From 6a751616f699f871e134a4e6b83d5d08d1ed41aa Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 31 Jan 2011 17:31:57 +0100 Subject: [PATCH 001/817] Remove unused variable mailboxes "mailboxes" is defined global and set to an empty list, but never used from anywhere within offlineimap. So let us just delete it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index cd09646..b60822c 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -102,7 +102,6 @@ def AccountHashGenerator(customconfig): retval[item.getname()] = item return retval -mailboxes = [] class Account(CustomConfig.ConfigHelperMixin): def __init__(self, config, name): @@ -291,7 +290,6 @@ class SyncableAccount(Account, AccountSynchronizationMixin): def syncfolder(accountname, remoterepos, remotefolder, localrepos, statusrepos, quick): - global mailboxes ui = getglobalui() ui.registerthread(accountname) try: From 465af6c481e5c7e10bb9c167209dd5e0e436d66e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 31 Jan 2011 15:53:35 +0100 Subject: [PATCH 002/817] accounts.py: Merge AccountSynchronizationMixin with SyncableAccount class AccountSynchronizationMixin was never used on its own and it is a very confusing class until you understand what it is used for. (It complemented the Account() class with a few methods to make Account() syncable. But we use the SyncableAccount class anyway, so merge the former Mixin' methods directly in there. This does away with a class that is not directly used, and was a case of over-object-orientation which confuses more than it helps. Touched up code documentation while going through the file. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index cd09646..aa8b88f 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -166,7 +166,12 @@ class Account(CustomConfig.ConfigHelperMixin): item.stopkeepalive() return sleepresult -class AccountSynchronizationMixin: + +class SyncableAccount(Account): + """A syncable IMAP account. + + Derives from class:`Account`.""" + def syncrunner(self, siglistener): self.ui.registerthread(self.name) self.ui.acct(self.name) @@ -285,12 +290,12 @@ class AccountSynchronizationMixin: self.ui.callhook("Hook return code: %d" % p.returncode) except: self.ui.warn("Exception occured while calling hook") - -class SyncableAccount(Account, AccountSynchronizationMixin): - pass + def syncfolder(accountname, remoterepos, remotefolder, localrepos, statusrepos, quick): + """This function is called as target for the + InstanceLimitedThread invokation in SyncableAccount.""" global mailboxes ui = getglobalui() ui.registerthread(accountname) From b768d2d28bd882a2e4e5971ed5fb848a0c24a8fb Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 27 Jan 2011 21:17:35 +0100 Subject: [PATCH 003/817] Makefile: get version number dynamically Avoid static version number in the Makefile. It's a possible source of errors, especially for me. Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 8b599f9..bb960e4 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,8 @@ New Features Changes ------- +* Makefile use magic to find the version number. + Bug Fixes --------- diff --git a/Makefile b/Makefile index 7d45f38..597c49b 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -VERSION=6.3.2 +VERSION=`./offlineimap.py --version` TARGZ=offlineimap_$(VERSION).tar.gz SHELL=/bin/bash RST2HTML=`type rst2html 2>/dev/null 2>&1 && echo rst2html || echo rst2html.py` From 6398bf7795a3f21678f364cc14ae6145f1f4e0b6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 31 Jan 2011 15:50:18 +0100 Subject: [PATCH 004/817] Don't require the string module There is no need for using the string module if all we want is to split a string at the white space. All pythons since at least 2.4 can do that. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 77d6f36..b1c343b 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -18,7 +18,6 @@ import imaplib import rfc822 -import string import random import binascii import re @@ -111,7 +110,7 @@ class IMAPFolder(BaseFolder): self.imapserver.releaseconnection(imapobj) # Discard the message number. - messagestr = string.split(response[0], maxsplit = 1)[1] + messagestr = response[0].split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not options.has_key('UID'): return True @@ -194,7 +193,7 @@ class IMAPFolder(BaseFolder): self.imapserver.releaseconnection(imapobj) for messagestr in response: # Discard the message number. - messagestr = string.split(messagestr, maxsplit = 1)[1] + messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not options.has_key('UID'): self.ui.warn('No UID in message with options %s' %\ From 838a67bc40fd82909789f497bf437333983d3d2d Mon Sep 17 00:00:00 2001 From: Thomas Jost Date: Wed, 23 Feb 2011 10:00:14 +0100 Subject: [PATCH 005/817] Support subjectAltName in SSL certificates Signed-off-by: Thomas Jost Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ offlineimap/imaplibutil.py | 30 +++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index bb960e4..44b4a87 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,8 @@ others. New Features ------------ +* SSL: support subjectAltName. + Changes ------- diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index cf82996..69d3983 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -17,10 +17,9 @@ import re, socket, time, subprocess from offlineimap.ui import getglobalui -from imaplib import * # Import the symbols we need that aren't exported by default -from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num +from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num, IMAP4, IMAP4_SSL try: import ssl @@ -149,22 +148,35 @@ class WrappedIMAP4_SSL(IMAP4_SSL): def _verifycert(self, cert, hostname): '''Verify that cert (in socket.getpeercert() format) matches hostname. - CRLs and subjectAltName are not handled. + CRLs are not handled. Returns error message if any problems are found and None on success. ''' if not cert: return ('no certificate received') dnsname = hostname.lower() + certnames = [] + + # First read commonName for s in cert.get('subject', []): key, value = s[0] if key == 'commonName': - certname = value.lower() - if (certname == dnsname or - '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): - return None - return ('certificate is for %s') % certname - return ('no commonName found in certificate') + certnames.append(value.lower()) + if len(certnames) == 0: + return ('no commonName found in certificate') + + # Then read subjectAltName + for key, value in cert.get('subjectAltName', []): + if key == 'DNS': + certnames.append(value.lower()) + + # And finally try to match hostname with one of these names + for certname in certnames: + if (certname == dnsname or + '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): + return None + + return ('no matching domain name found in certificate') def _read_upto (self, n): """Read up to n bytes, emptying existing _readbuffer first""" From 966841c8d6338cc9b756bfa5f0a3d2d7c3247781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Melis?= Date: Tue, 1 Mar 2011 14:49:02 +0100 Subject: [PATCH 006/817] Allow SSL connections to send keep-alive messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This seems to prevent mysterious hangs with SSL imap servers (especially gmail?) and does not harm in any case. So let us enable keep-alive messages for ssl connections. Our thread pool should be made more robust against closed SSL connections (which do not always seem to raise Exceptions), and not deadlock while waiting for resources or data that will never arrive. Reviewed-by: Sebastian Spaeth Signed-off-by: Gábor Melis Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ offlineimap/imaplibutil.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 44b4a87..8addd4f 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -23,6 +23,8 @@ Changes Bug Fixes --------- +* Allow SSL connections to send keep-alive messages. + Pending for the next major release ================================== diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 69d3983..6752328 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -110,6 +110,10 @@ class WrappedIMAP4_SSL(IMAP4_SSL): # FIXME raise socket.error(last_error) + # Allow sending of keep-alive message seems to prevent some servers + # from closing SSL on us leading to deadlocks + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + #connected to socket, now wrap it in SSL try: if self._cacertfile: From d5e7620ce939a5d81bb342fee3af2c670c397b46 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Mar 2011 11:05:15 +0100 Subject: [PATCH 007/817] Create an abstract Repository class A Repository() returns the correctly instanciated dervivate of a BaseRepository, depending on the parameters passed to it. The returned instance is eg an ImapRepository(). This makes the code look nicer, and we have less functions lying around outside of classes (no more global LoadRepository() function). This will also enable us to conveniently hand back a LocalStatusRepository based on SQLITE rather than plain text, if the user configures this to be the experimental and optional backend (once it exists). Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 + offlineimap/accounts.py | 13 +++--- offlineimap/repository/Base.py | 26 +---------- offlineimap/repository/Gmail.py | 2 +- offlineimap/repository/IMAP.py | 2 +- offlineimap/repository/__init__.py | 72 +++++++++++++++++++++++++++++- 6 files changed, 82 insertions(+), 35 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index d9977a2..c0e1d40 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,8 @@ New Features Changes ------- +* Rework the repository module + Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 8c03e69..7a0f5a9 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -16,7 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap import threadutil, mbnames, CustomConfig -import offlineimap.repository.Base, offlineimap.repository.LocalStatus +from offlineimap.repository import Repository from offlineimap.ui import getglobalui from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread from subprocess import Popen, PIPE @@ -175,13 +175,10 @@ class AccountSynchronizationMixin: if not os.path.exists(accountmetadata): os.mkdir(accountmetadata, 0700) - self.remoterepos = offlineimap.repository.Base.LoadRepository(self.getconf('remoterepository'), self, 'remote') - - # Connect to the local repository. - self.localrepos = offlineimap.repository.Base.LoadRepository(self.getconf('localrepository'), self, 'local') - - # Connect to the local cache. - self.statusrepos = offlineimap.repository.LocalStatus.LocalStatusRepository(self.getconf('localrepository'), self) + # get all three repositories + self.remoterepos = Repository(self, 'remote') + self.localrepos = Repository(self, 'local') + self.statusrepos = Repository(self, 'status') #might need changes here to ensure that one account sync does not crash others... if not self.refreshperiod: diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 7e1dd4d..eae41b2 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -16,32 +16,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import CustomConfig -from offlineimap.ui import getglobalui import os.path import traceback - -def LoadRepository(name, account, reqtype): - from offlineimap.repository.Gmail import GmailRepository - from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository - from offlineimap.repository.Maildir import MaildirRepository - if reqtype == 'remote': - # For now, we don't support Maildirs on the remote side. - typemap = {'IMAP': IMAPRepository, - 'Gmail': GmailRepository} - elif reqtype == 'local': - typemap = {'IMAP': MappedIMAPRepository, - 'Maildir': MaildirRepository} - else: - raise ValueError, "Request type %s not supported" % reqtype - config = account.getconfig() - repostype = config.get('Repository ' + name, 'type').strip() - try: - repo = typemap[repostype] - except KeyError: - raise Exception, "'%s' repository not supported for %s repositories."%\ - (repostype, reqtype) - return repo(name, account) +from offlineimap import CustomConfig +from offlineimap.ui import getglobalui class BaseRepository(CustomConfig.ConfigHelperMixin): def __init__(self, reposname, account): diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 4793db7..a586460 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from IMAP import IMAPRepository +from offlineimap.repository.IMAP import IMAPRepository from offlineimap import folder, imaputil from offlineimap.imapserver import IMAPServer diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 3bfa5db..2bc9718 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from Base import BaseRepository +from offlineimap.repository.Base import BaseRepository from offlineimap import folder, imaputil, imapserver from offlineimap.folder.UIDMaps import MappedIMAPFolder from offlineimap.threadutil import ExitNotifyThread diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py index be5c29e..c63f6db 100644 --- a/offlineimap/repository/__init__.py +++ b/offlineimap/repository/__init__.py @@ -1 +1,71 @@ -__all__ = ['Gmail', 'IMAP', 'Base', 'Maildir', 'LocalStatus'] +# Copyright (C) 2002-2007 John Goerzen +# 2010 Sebastian Spaeth and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository +from offlineimap.repository.Gmail import GmailRepository +from offlineimap.repository.Maildir import MaildirRepository +from offlineimap.repository.LocalStatus import LocalStatusRepository + +class Repository(object): + """Abstract class that returns the correct Repository type + instance based on 'account' and 'reqtype', e.g. a + class:`ImapRepository` instance.""" + + def __new__(cls, account, reqtype): + """ + :param account: :class:`Account` + :param regtype: 'remote', 'local', or 'status'""" + + if reqtype == 'remote': + name = account.getconf('remoterepository') + # We don't support Maildirs on the remote side. + typemap = {'IMAP': IMAPRepository, + 'Gmail': GmailRepository} + + elif reqtype == 'local': + name = account.getconf('localrepository') + typemap = {'IMAP': MappedIMAPRepository, + 'Maildir': MaildirRepository} + + elif reqtype == 'status': + # create and return a LocalStatusRepository + name = account.getconf('localrepository') + return LocalStatusRepository(name, account) + + else: + raise ValueError, "Request type %s not supported" % reqtype + + config = account.getconfig() + repostype = config.get('Repository ' + name, 'type').strip() + try: + repo = typemap[repostype] + except KeyError: + raise Exception, "'%s' repository not supported for %s repositories."%\ + (repostype, reqtype) + return repo(name, account) + + + def __init__(self, account, reqtype): + """Load the correct Repository type and return that. The + __init__ of the corresponding Repository class will be + executed instead of this stub + + :param account: :class:`Account` + :param regtype: 'remote', 'local', or 'status' + """ + pass + From 472476db6ff9028b3fa9cdda3d636ffe0521d2da Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Mar 2011 12:35:09 +0100 Subject: [PATCH 008/817] Remove over-verbose debug options The debug output for dequote, optionsplit is very verbose, outputing what the functions are called with and what they return. Those functions are now very old mature and rather simple, so it suffices to output their return value rather than cluttering out log with too much uninteresting garbarge. This makes log files much more readable. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaputil.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index fe74854..c59f1f5 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -31,7 +31,6 @@ def dequote(string): This function does NOT consider parenthised lists to be quoted. """ - debug("dequote() called with input:", string) if not (string[0] == '"' and string[-1] == '"'): return string string = string[1:-1] # Strip off quotes. @@ -46,7 +45,6 @@ def flagsplit(string): return imapsplit(string[1:-1]) def options2hash(list): - debug("options2hash called with input:", list) retval = {} counter = 0 while (counter < len(list)): @@ -68,7 +66,6 @@ def imapsplit(imapstring): ['(\\HasNoChildren)', '"."', '"INBOX.Sent"']""" - debug("imapsplit() called with input:", imapstring) if type(imapstring) != types.StringType: debug("imapsplit() got a non-string input; working around.") # Sometimes, imaplib will throw us a tuple if the input From 0811beb03d11a1e85d1833995f21b07eed5511d0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Mar 2011 10:43:03 +0100 Subject: [PATCH 009/817] Provide better error message for invalid 'reference' setting When e.g. specifying an invalid 'reference' value for an IMAP server to a root folder that does not exist, we would previously have crashed with a nonsensical and non-intuitive error message (trying to address an element of a NoneType). This will also raise an Exception (which should be ok, given that this is really a misconfiguration on the user side), but it will explain to the user WHY it exited. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 2a9f247..0d7c621 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -302,6 +302,14 @@ class IMAPServer: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] + if listres == [None] or listres == None: + # No Folders were returned. This occurs, e.g. if the + # 'reference' prefix does not exist on the mail + # server. Raise exception. + err = "Server '%s' returned no folders in '%s'" % \ + (self.repos.getname(), self.reference) + self.ui.warn(err) + raise Exception(err) self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) From b293e91a0e049b6c4f157423790d918a64222c0d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Mar 2011 10:39:54 +0100 Subject: [PATCH 010/817] Improve documentation on "reference" option It was not clear how the reference value is used. Improve this by exposing the internal path build system for the 'reference' option so that users know what value is expected. Also, explain how it impacts both nametrans and folderfilter options. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index d3be8be..72a3fa8 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -327,10 +327,14 @@ remoteuser = username ########## Advanced settings -# Some IMAP servers need a "reference" which often refers to the -# "folder root". This is most commonly needed with UW IMAP, where -# you might need to specify the directory in which your mail is -# stored. Most users will not need this. +# Some IMAP servers need a "reference" which often refers to the "folder +# root". This is most commonly needed with UW IMAP, where you might +# need to specify the directory in which your mail is stored. The +# 'reference' value will be prefixed to all folder paths refering to +# that repository. E.g. accessing folder 'INBOX' with reference = Mail +# will try to access Mail/INBOX. Note that the nametrans and +# folderfilter functions will still apply the full path including the +# reference prefix. Most users will not need this. # # reference = Mail From cc64a0952c6c940dce1989aae977413b6effeb5c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Mar 2011 11:43:22 +0100 Subject: [PATCH 011/817] Make self.ui available in all Repository() derivatives This enables us to make use of self.ui in all repositories without having to import and use getglobalui() in all types of repositories and all places. Note that this patch only makes this available, it does not yet make use of it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 7e1dd4d..10adca4 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -45,6 +45,7 @@ def LoadRepository(name, account, reqtype): class BaseRepository(CustomConfig.ConfigHelperMixin): def __init__(self, reposname, account): + self.ui = getglobalui() self.account = account self.config = account.getconfig() self.name = reposname @@ -166,7 +167,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin): except (KeyboardInterrupt): raise except: - getglobalui().warn("ERROR Attempting to create folder " \ + self.ui.warn("ERROR Attempting to create folder " \ + key + ":" +traceback.format_exc()) # From 93f7d0bd1f31f5ce6b4dd6de8145a3a06996ade1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Mar 2011 11:43:23 +0100 Subject: [PATCH 012/817] Enable debug output to see what folderfilter actually filters out It is currently very hard to find out what folderfilter actually does and makes it hard to debug for a user. With this patch if the user has enabled "-d imap" (even better would perhaps be a different debug type for this kind of thing?), we see a message "Filtering out folder 'foo' due to folderfilter" in the logs. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/IMAP.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 3bfa5db..76ff2c9 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -27,6 +27,7 @@ class IMAPRepository(BaseRepository): def __init__(self, reposname, account): """Initialize an IMAPRepository object.""" BaseRepository.__init__(self, reposname, account) + # self.ui is being set by the BaseRepository self.imapserver = imapserver.ConfigedIMAPServer(self) self.folders = None self.nametrans = lambda foldername: foldername @@ -250,6 +251,8 @@ class IMAPRepository(BaseRepository): continue foldername = imaputil.dequote(name) if not self.folderfilter(foldername): + self.ui.debug('imap',"Filtering out '%s' due to folderfilter" %\ + foldername) continue retval.append(self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), From ab3900e479199c2c74d26f3393f5485bd93d21bb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 1 Mar 2011 15:31:43 +0100 Subject: [PATCH 013/817] Remove unneeded workaround for Darwin There is a clumsy workaround for Darwin that chunks reads into 8kb blocks to avoid huge memory allocations. First, this fix should not only be required but on FreeBSD2.6 too (see http://bugs.python.org/issue3531). Second, decent python versions (I checked 2.6) already chunk in the SSL case anyway, so there is no need to do that again. Remove that level of indirection. http://evanjones.ca/python-memory.html claims that this problem has been fixed since python 2.5, so we might consider removing the workaround completely even for the non-SSL case. Increase the chunk size on Mac from 8kb to 64kb. Even Macs should be able to take that amount of memory usage nowadays. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 2a9f247..fa56af7 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -60,37 +60,23 @@ class UsefulIMAPMixIn: imaplibutil.new_mesg(self, s, secs) class UsefulIMAP4(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): - # This is a hack around Darwin's implementation of realloc() (which # Python uses inside the socket code). On Darwin, we split the - # message into 100k chunks, which should be small enough - smaller - # might start seriously hurting performance ... + # message into small chunks. + # see http://bugs.python.org/issue3531 def read(self, size): if (system() == 'Darwin') and (size>0) : read = 0 io = StringIO() while read < size: - data = imaplib.IMAP4.read (self, min(size-read,8192)) + data = imaplib.IMAP4.read (self, min(size-read, 65536)) read += len(data) io.write(data) return io.getvalue() else: return imaplib.IMAP4.read (self, size) -class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): - # This is the same hack as above, to be used in the case of an SSL - # connexion. - def read(self, size): - if (system() == 'Darwin') and (size>0) : - read = 0 - io = StringIO() - while read < size: - data = imaplibutil.WrappedIMAP4_SSL.read (self, min(size-read,8192)) - read += len(data) - io.write(data) - return io.getvalue() - else: - return imaplibutil.WrappedIMAP4_SSL.read (self,size) +class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): pass class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass From 419f27418ec4a8b27ac2c98bb6f3bb2169bafa28 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 4 Mar 2011 17:34:20 +0100 Subject: [PATCH 014/817] Simplify & document savemessage_getnewheader savemessage_getnewheader was an undocmented, cryptic and overengineered function. It generates a new unique value that can be used as a mail header to be inserted. For this it used LOTS of randomness sources: hash of the mail content, hash of the folder name, hash of the repository name, the current time, a random() value, and the offlineimap version string. All we need is something random. So reduce this to hash of content appended by a random integer. Sufficient and somewhat faster to calculate. Rename the function to actually describe accurately what it does or would you have guessed that savemessage_getnewheader() did nothing more than returning ('X-OfflineIMAP', )? Rename to generate_randomheader() to make it clearer what this is all about. Also document the function, describing what it does, and what it returns. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index b1c343b..8dd9ee6 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -25,7 +25,7 @@ import time from StringIO import StringIO from copy import copy from Base import BaseFolder -from offlineimap import imaputil, imaplibutil, __version__ +from offlineimap import imaputil, imaplibutil class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, visiblename, accountname, repository): @@ -226,14 +226,30 @@ class IMAPFolder(BaseFolder): def getmessageflags(self, uid): return self.messagelist[uid]['flags'] - def savemessage_getnewheader(self, content): + def generate_randomheader(self, content): + """Returns a unique X-OfflineIMAP header + + Generate an 'X-OfflineIMAP' mail header which contains a random + unique value (which is based on the mail content, and a random + number). This header allows us to fetch a mail after APPENDing + it to an IMAP server and thus find out the UID that the server + assigned it. + + :returns: (headername, headervalue) tuple, consisting of strings + headername == 'X-OfflineIMAP' and headervalue will be a + random string + """ headername = 'X-OfflineIMAP' - headervalue = '%s-' % str(binascii.crc32(content)).replace('-', 'x') - headervalue += binascii.hexlify(self.repository.getname()) + '-' - headervalue += binascii.hexlify(self.getname()) - headervalue += '-%d-' % long(time.time()) - headervalue += str(self.randomgenerator.random()).replace('.', '') - headervalue += '-v' + __version__ + # We need a random component too. If we ever upload the same + # mail twice (e.g. in different folders), we would still need to + # get the UID for the correct one. As we won't have too many + # mails with identical content, the randomness requirements are + # not extremly critial though. + + # compute unsigned crc32 of 'content' as unique hash + # NB: crc32 returns unsigned only starting with python 3.0 + headervalue = str( binascii.crc32(content) & 0xffffffff ) + '-' + headervalue += str(self.randomgenerator.randint(0,9999999999)) return (headername, headervalue) def savemessage_addheader(self, content, headername, headervalue): @@ -332,8 +348,8 @@ class IMAPFolder(BaseFolder): content = re.sub("(? Date: Fri, 4 Mar 2011 17:34:21 +0100 Subject: [PATCH 015/817] Factor out the date guessing/retrieving savemessage was too long and complex. Factor out the date guessing part of the function and put it into a function of its own. The logic of the date guessing is the same, however, we do not use the imaplib.Time2InternalDate() function as it is buggy (http://bugs.python.org/issue11024) and returns localized patches. So we create INTERNALDATE ourselves and pass it to append() as a string. This commit fixes a bug that international users used to pass an invalid date to the IMAP server, which the server will either ignore or complain about. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +- offlineimap/folder/IMAP.py | 128 +++++++++++++++++++++++++++---------- 2 files changed, 96 insertions(+), 35 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index fe896f0..e099689 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -26,8 +26,9 @@ Bug Fixes * Allow SSL connections to send keep-alive messages. * Fix regression (UIBase is no more). - * Make profiling mode really enforce single-threading +* Do not send localized date strings to the IMAP server as it will + either ignore or refuse them. Pending for the next major release ================================== diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8dd9ee6..eab7ffb 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -299,52 +299,112 @@ class IMAPFolder(BaseFolder): matchinguids.sort() return long(matchinguids[0]) + + def getmessageinternaldate(self, content, rtime=None): + """Parses mail and returns an INTERNALDATE string + + It will use information in the following order, falling back as an attempt fails: + - rtime parameter + - Date header of email + + We return None, if we couldn't find a valid date. In this case + the IMAP server will use the server local time when appening + (per RFC). + + Note, that imaplib's Time2Internaldate is inherently broken as + it returns localized date strings which are invalid for IMAP + servers. However, that function is called for *every* append() + internally. So we need to either pass in `None` or the correct + string (in which case Time2Internaldate() will do nothing) to + append(). The output of this function is designed to work as + input to the imapobj.append() function. + + TODO: We should probably be returning a bytearray rather than a + string here, because the IMAP server will expect plain + ASCII. However, imaplib.Time2INternaldate currently returns a + string so we go with the same for now. + + :param rtime: epoch timestamp to be used rather than analyzing + the email. + :returns: string in the form of "DD-Mmm-YYYY HH:MM:SS +HHMM" + (including double quotes) or `None` in case of failure + (which is fine as value for append).""" + if rtime is None: + message = rfc822.Message(StringIO(content)) + # parsedate returns a 9-tuple that can be passed directly to + # time.mktime(); Will be None if missing or not in a valid + # format. Note that indexes 6, 7, and 8 of the result tuple are + # not usable. + datetuple = rfc822.parsedate(message.getheader('Date')) + + if datetuple is None: + #could not determine the date, use the local time. + return None + else: + #rtime is set, use that instead + datetuple = time.localtime(rtime) + + try: + # Check for invalid dates + if datetuple[0] < 1981: + raise ValueError + + # Check for invalid dates + datetuple_check = time.localtime(time.mktime(datetuple)) + if datetuple[:2] != datetuple_check[:2]: + raise ValueError + + except (ValueError, OverflowError): + # Argh, sometimes it's a valid format but year is 0102 + # or something. Argh. It seems that Time2Internaldate + # will rause a ValueError if the year is 0102 but not 1902, + # but some IMAP servers nonetheless choke on 1902. + self.ui.debug("Message with invalid date %s. Server will use local time." % datetuple) + return None + + #produce a string representation of datetuple that works as + #INTERNALDATE + num2mon = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', + 7:'Jul', 8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'} + + if datetuple.tm_isdst == '1': + zone = -time.altzone + else: + zone = -time.timezone + offset_h, offset_m = divmod(zone//60, 60) + + internaldate = '"%02d-%s-%04d %02d:%02d:%02d %+03d%02d"' \ + % (datetuple.tm_mday, num2mon[datetuple.tm_mon], datetuple.tm_year, \ + datetuple.tm_hour, datetuple.tm_min, datetuple.tm_sec, offset_h, offset_m) + + return internaldate + def savemessage(self, uid, content, flags, rtime): + """Save the message on the Server + + This backend always assigns a new uid, so the uid arg is ignored. + + This function will update the self.messagelist dict to contain + the new message after sucessfully saving it. + + :param rtime: A timestamp to be + :returns: the UID of the new message as assigned by the + server. If the folder is read-only it will return 0.""" imapobj = self.imapserver.acquireconnection() self.ui.debug('imap', 'savemessage: called') + try: try: imapobj.select(self.getfullname()) # Needed for search except imapobj.readonly: self.ui.msgtoreadonly(self, uid, content, flags) # Return indicating message taken, but no UID assigned. - # Fudge it. return 0 - # This backend always assigns a new uid, so the uid arg is ignored. - # In order to get the new uid, we need to save off the message ID. + # get the date of the message file, so we can pass it to the server. + date = self.getmessageinternaldate(content, rtime) - message = rfc822.Message(StringIO(content)) - datetuple_msg = rfc822.parsedate(message.getheader('Date')) - # Will be None if missing or not in a valid format. - - # If time isn't known - if rtime == None and datetuple_msg == None: - datetuple = time.localtime() - elif rtime == None: - datetuple = datetuple_msg - else: - datetuple = time.localtime(rtime) - - try: - if datetuple[0] < 1981: - raise ValueError - - # Check for invalid date - datetuple_check = time.localtime(time.mktime(datetuple)) - if datetuple[:2] != datetuple_check[:2]: - raise ValueError - - # This could raise a value error if it's not a valid format. - date = imaplib.Time2Internaldate(datetuple) - except (ValueError, OverflowError): - # Argh, sometimes it's a valid format but year is 0102 - # or something. Argh. It seems that Time2Internaldate - # will rause a ValueError if the year is 0102 but not 1902, - # but some IMAP servers nonetheless choke on 1902. - date = imaplib.Time2Internaldate(time.localtime()) - - self.ui.debug('imap', 'savemessage: using date ' + str(date)) + self.ui.debug('imap', 'savemessage: using date %s' % date) content = re.sub("(? Date: Fri, 4 Mar 2011 17:34:22 +0100 Subject: [PATCH 016/817] Remove convoluted assert statements The working horse of the savemessage() function, imaplib.append() was hidden away in an assert statement. Pull the real functions out of the asserts and simply assert on the return values. This looks less convoluted and makes this easier to understand in my opinion. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index eab7ffb..17b849b 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -252,6 +252,7 @@ class IMAPFolder(BaseFolder): headervalue += str(self.randomgenerator.randint(0,9999999999)) return (headername, headervalue) + def savemessage_addheader(self, content, headername, headervalue): self.ui.debug('imap', 'savemessage_addheader: called to add %s: %s' % (headername, @@ -271,6 +272,7 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer)) return leader + newline + trailer + def savemessage_searchforheader(self, imapobj, headername, headervalue): if imapobj.untagged_responses.has_key('APPENDUID'): return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1]) @@ -417,12 +419,16 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage: new content length is ' + \ str(len(content))) - assert(imapobj.append(self.getfullname(), + # TODO: append could raise a ValueError if the date is not in + # valid format...? + (typ,dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), - date, content)[0] == 'OK') + date, content) + assert(typ == 'OK') # Checkpoint. Let it write out the messages, etc. - assert(imapobj.check()[0] == 'OK') + (typ,dat) = imapobj.check() + assert(typ == 'OK') # Keep trying until we get the UID. self.ui.debug('imap', 'savemessage: first attempt to get new UID') From 3b8e1f91cd6bb70799714cce3437b7d58c52b2c6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 4 Mar 2011 17:34:23 +0100 Subject: [PATCH 017/817] Implement APPENDUID support Rather than inserting our own home-grown header, everytime we save a message to an IMAP server, we check if we suport the UIDPLUS extension which provides us with an APPENDUID reply. Use that to find the new UID if possible, but keep the old way if we don't have that extension. If a folder is read-only, return the uid that we have passed in per API description in folder.Base.py Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 ++ offlineimap/folder/IMAP.py | 75 ++++++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 31 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index e099689..f68d747 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -12,6 +12,9 @@ others. New Features ------------ +* Implement UIDPLUS extension support. OfflineIMAP will now not insert + an X-OfflineIMAP header if the mail server supports the UIDPLUS + extension. * SSL: support subjectAltName. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 17b849b..a419664 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -274,9 +274,6 @@ class IMAPFolder(BaseFolder): def savemessage_searchforheader(self, imapobj, headername, headervalue): - if imapobj.untagged_responses.has_key('APPENDUID'): - return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1]) - self.ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \ (headername, headervalue)) # Now find the UID it got. @@ -389,38 +386,42 @@ class IMAPFolder(BaseFolder): This function will update the self.messagelist dict to contain the new message after sucessfully saving it. - :param rtime: A timestamp to be + :param rtime: A timestamp to be used as the mail date :returns: the UID of the new message as assigned by the server. If the folder is read-only it will return 0.""" - imapobj = self.imapserver.acquireconnection() self.ui.debug('imap', 'savemessage: called') try: + imapobj = self.imapserver.acquireconnection() + try: - imapobj.select(self.getfullname()) # Needed for search + imapobj.select(self.getfullname()) # Needed for search and making the box READ-WRITE except imapobj.readonly: + # readonly exception. Return original uid to notify that + # we did not save the message. (see savemessage in Base.py) self.ui.msgtoreadonly(self, uid, content, flags) - # Return indicating message taken, but no UID assigned. - return 0 - + return uid + + # UIDPLUS extension provides us with an APPENDUID response to our append() + use_uidplus = 'UIDPLUS' in imapobj.capabilities + # get the date of the message file, so we can pass it to the server. date = self.getmessageinternaldate(content, rtime) - self.ui.debug('imap', 'savemessage: using date %s' % date) + content = re.sub("(? Date: Sun, 6 Mar 2011 10:20:14 +0100 Subject: [PATCH 018/817] Set maxconnections default to 2 Multithreading speeds up account syncing a lot and the offlineimap defaults are very conservative. Let's make it use 2 IMAP connections by default to gain some of the benefits that offlineimap offers. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 2 +- offlineimap/init.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index d3be8be..cf731a4 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -341,7 +341,7 @@ remoteuser = username # cases, it may slow things down. The safe answer is 1. You should # probably never set it to a value more than 5. -maxconnections = 1 +maxconnections = 2 # OfflineIMAP normally closes IMAP server connections between refreshes if # the global option autorefresh is specified. If you wish it to keep the diff --git a/offlineimap/init.py b/offlineimap/init.py index 8aebb23..8029470 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -295,8 +295,9 @@ class OfflineImap: remoterepos = None localrepos = None - threadutil.initInstanceLimit("ACCOUNTLIMIT", - config.getdefaultint("general", "maxsyncaccounts", 1)) + threadutil.initInstanceLimit('ACCOUNTLIMIT', + config.getdefaultint('general', + 'maxsyncaccounts', 1)) for reposname in config.getsectionlist('Repository'): for instancename in ["FOLDER_" + reposname, @@ -305,7 +306,8 @@ class OfflineImap: threadutil.initInstanceLimit(instancename, 1) else: threadutil.initInstanceLimit(instancename, - config.getdefaultint('Repository ' + reposname, "maxconnections", 1)) + config.getdefaultint('Repository ' + reposname, + 'maxconnections', 2)) siglisteners = [] def sig_handler(signum, frame): if signum == signal.SIGUSR1: From 4e28c7c93fdb6dd105ce482ac9b8d4d78d2da35b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 6 Mar 2011 11:04:46 +0100 Subject: [PATCH 019/817] Allow to use nicer UI names The previous ui names were pretty unwieldy. Is it TTYUI.TTY or TTY.TTYUI? Do I have to use capitals and where? Simplify the names by making them case insensitive and by dropping everything before the dot. So "Curses.Blinkenlights" can now be invoked as "blinkenlights" or "BLINKENLIGHTS". The old names will still work just fine so the transition should be smooth. We issue a warning that the long names are deprecated. Document in offlineimap.conf that we don't accept lists of fallback UIs, but only one UI option (this was already the case before this commit but still wrongly documented). The list of accepted ui names is: ttyui (default), basic, quiet, machineui, blinkenlights Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +++ docs/MANUAL.rst | 52 ++++++++++++++++++-------------------- offlineimap.conf | 18 ++++++------- offlineimap/init.py | 12 ++++++--- offlineimap/ui/__init__.py | 10 ++++---- 5 files changed, 50 insertions(+), 45 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index f68d747..a3c70ec 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -23,6 +23,9 @@ Changes * Makefile use magic to find the version number. * Rework the repository module +* Change UI names to Blinkenlights,TTYUI,Basic,Quiet,MachineUI. + Old names will still work, but are deprecated. + Document that we don't accept a list of UIs anymore. Bug Fixes --------- diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 67fa383..9ef4eea 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -134,30 +134,29 @@ OPTIONS Specifies an alternative user interface module to use. This overrides the default specified in the configuration file. The pre-defined options are - listed in the User Interfaces section. + listed in the User Interfaces section. The interface name is case insensitive. User Interfaces =============== -OfflineIMAP has a pluggable user interface system that lets you choose how the -program communicates information to you. There are two graphical interfaces, -two terminal interfaces, and two noninteractive interfaces suitable for -scripting or logging purposes. The ui option in the configuration file -specifies user interface preferences. The -u command-line option can override -the configuration file setting. The available values for the configuration file -or command-line are described in this section. +OfflineIMAP has various user interfaces that let you choose how the +program communicates information to you. The 'ui' option in the +configuration file specifies the user interface. The -u command-line +option overrides the configuration file setting. The available values +for the configuration file or command-line are described in this +section. -Curses.Blinkenlights --------------------- +Blinkenlights +--------------- -Curses.Blinkenlights is an interface designed to be sleek, fun to watch, and +Blinkenlights is an interface designed to be sleek, fun to watch, and informative of the overall picture of what OfflineIMAP is doing. I consider it to be the best general-purpose interface in OfflineIMAP. -Curses.Blinkenlights contains a row of "LEDs" with command buttons and a log. +Blinkenlights contains a row of "LEDs" with command buttons and a log. The log shows more detail about what is happening and is color-coded to match the color of the lights. @@ -228,18 +227,18 @@ English-speaking world. One version ran in its entirety as follows: | pockets muss; relaxen und watchen das blinkenlichten. -TTY.TTYUI +TTYUI --------- -TTY.TTYUI interface is for people running in basic, non-color terminals. It +TTYUI interface is for people running in basic, non-color terminals. It prints out basic status messages and is generally friendly to use on a console or xterm. -Noninteractive.Basic +Basic -------------------- -Noninteractive.Basic is designed for situations in which OfflineIMAP will be run +Basic is designed for situations in which OfflineIMAP will be run non-attended and the status of its execution will be logged. You might use it, for instance, to have the system run automatically and e-mail you the results of the synchronization. This user interface is not capable of reading a password @@ -247,20 +246,19 @@ from the keyboard; account passwords must be specified using one of the configuration file options. -Noninteractive.Quiet --------------------- +Quiet +----- -Noninteractive.Quiet is designed for non-attended running in situations where -normal status messages are not desired. It will output nothing except errors -and serious warnings. Like Noninteractive.Basic, this user interface is not -capable of reading a password from the keyboard; account passwords must be -specified using one of the configuration file options. +Quiet is designed for non-attended running in situations where normal +status messages are not desired. It will output nothing except errors +and serious warnings. Like Noninteractive.Basic, this user interface is +not capable of reading a password from the keyboard; account passwords +must be specified using one of the configuration file options. +MachineUI +--------- -Machine.MachineUI ------------------ - -Machine.MachineUI generates output in a machine-parsable format. It is designed +MachineUI generates output in a machine-parsable format. It is designed for other programs that will interface to OfflineIMAP. diff --git a/offlineimap.conf b/offlineimap.conf index 72a3fa8..6032fd8 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -55,19 +55,17 @@ maxsyncaccounts = 1 # fails, the second, and so forth. # # The pre-defined options are: -# Curses.Blinkenlights -- A text-based (terminal) interface similar to -# Tk.Blinkenlights -# TTY.TTYUI -- a text-based (terminal) interface -# Noninteractive.Basic -- Noninteractive interface suitable for cronning -# Noninteractive.Quiet -- Noninteractive interface, generates no output -# except for errors. -# Machine.MachineUI -- Interactive interface suitable for machine -# parsing. +# Blinkenlights -- A fancy (terminal) interface +# TTYUI -- a text-based (terminal) interface +# Basic -- Noninteractive interface suitable for cron'ing +# Quiet -- Noninteractive interface, generates no output +# except for errors. +# MachineUI -- Interactive interface suitable for machine +# parsing. # # You can override this with a command-line option -u. -ui = Curses.Blinkenlights, TTY.TTYUI, - Noninteractive.Basic, Noninteractive.Quiet +ui = Blinkenlights # If you try to synchronize messages to a read-only folder, # OfflineIMAP will generate a warning. If you want to suppress these diff --git a/offlineimap/init.py b/offlineimap/init.py index 8aebb23..38c68b4 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -187,12 +187,18 @@ class OfflineImap: section = "general" config.set(section, key, value) - #init the ui, cmd line option overrides config file - ui_type = config.getdefault('general','ui', 'TTY.TTYUI') + #which ui to use? cmd line option overrides config file + ui_type = config.getdefault('general','ui', 'ttyui') if options.interface != None: ui_type = options.interface + if '.' in ui_type: + #transform Curses.Blinkenlights -> Blinkenlights + ui_type = ui_type.split('.')[-1] + logging.warning('Using old interface name, consider using one ' + 'of %s' % ', '.join(UI_LIST.keys())) try: - ui = UI_LIST[ui_type](config) + # create the ui class + ui = UI_LIST[ui_type.lower()](config) except KeyError: logging.error("UI '%s' does not exist, choose one of: %s" % \ (ui_type,', '.join(UI_LIST.keys()))) diff --git a/offlineimap/ui/__init__.py b/offlineimap/ui/__init__.py index 83d81c6..102cbfd 100644 --- a/offlineimap/ui/__init__.py +++ b/offlineimap/ui/__init__.py @@ -18,14 +18,14 @@ from offlineimap.ui.UIBase import getglobalui, setglobalui from offlineimap.ui import TTY, Noninteractive, Machine -UI_LIST = {'TTY.TTYUI': TTY.TTYUI, - 'Noninteractive.Basic': Noninteractive.Basic, - 'Noninteractive.Quiet': Noninteractive.Quiet, - 'Machine.MachineUI': Machine.MachineUI} +UI_LIST = {'ttyui': TTY.TTYUI, + 'basic': Noninteractive.Basic, + 'quiet': Noninteractive.Quiet, + 'machineui': Machine.MachineUI} #add Blinkenlights UI if it imports correctly (curses installed) try: from offlineimap.ui import Curses - UI_LIST['Curses.Blinkenlights'] = Curses.Blinkenlights + UI_LIST['blinkenlights'] = Curses.Blinkenlights except ImportError: pass From fc03475b9ec32352b36127d8c3d5fa02d8c635ea Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 6 Mar 2011 11:17:12 +0100 Subject: [PATCH 020/817] Remove thread Lock() when saving UIDvalidity Removing this lock makes the function not threadsafe, but then it is only ever called from one thread, the main account syncer. Also, it doesn't make it worse than most of the other functions in that class which are also not threadsafe. Removing this makes the code simpler, and removes the need to import the threading module. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 8e6a6b3..5238f1e 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -16,7 +16,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import * from offlineimap import threadutil from offlineimap.ui import getglobalui import os.path @@ -26,7 +25,6 @@ import traceback class BaseFolder: def __init__(self): - self.uidlock = Lock() self.ui = getglobalui() def getname(self): @@ -83,6 +81,11 @@ class BaseFolder: return foldername def isuidvalidityok(self): + """Does the cached UID match the real UID + + If required it caches the UID. In this case the function is not + threadsafe. So don't attempt to call it from concurrent threads.""" + if self.getsaveduidvalidity() != None: return self.getsaveduidvalidity() == self.getuidvalidity() else: @@ -106,17 +109,18 @@ class BaseFolder: return self._base_saved_uidvalidity def saveuidvalidity(self): + """Save the UID value of the folder to the status + + This function is not threadsafe, so don't attempt to call it + from concurrent threads.""" newval = self.getuidvalidity() uidfilename = self._getuidfilename() - self.uidlock.acquire() - try: - file = open(uidfilename + ".tmp", "wt") - file.write("%d\n" % newval) - file.close() - os.rename(uidfilename + ".tmp", uidfilename) - self._base_saved_uidvalidity = newval - finally: - self.uidlock.release() + + file = open(uidfilename + ".tmp", "wt") + file.write("%d\n" % newval) + file.close() + os.rename(uidfilename + ".tmp", uidfilename) + self._base_saved_uidvalidity = newval def getuidvalidity(self): raise NotImplementedException @@ -425,5 +429,3 @@ class BaseFolder: except: self.ui.warn("ERROR attempting to sync flags " \ + "for account " + self.getaccountname() + ":" + traceback.format_exc()) - - From 3eee8213824d369a05ce927d6bb78145dc33b7ac Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 6 Mar 2011 20:03:04 +0100 Subject: [PATCH 021/817] Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 5 +- offlineimap/accounts.py | 25 +-- offlineimap/folder/Base.py | 353 +++++++++++++++++++++---------------- 3 files changed, 209 insertions(+), 174 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index a3c70ec..edc8254 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -15,7 +15,6 @@ New Features * Implement UIDPLUS extension support. OfflineIMAP will now not insert an X-OfflineIMAP header if the mail server supports the UIDPLUS extension. - * SSL: support subjectAltName. Changes @@ -26,6 +25,10 @@ Changes * Change UI names to Blinkenlights,TTYUI,Basic,Quiet,MachineUI. Old names will still work, but are deprecated. Document that we don't accept a list of UIs anymore. +* Reworked the syncing strategy. The only user-visible change is that + blowing away LocalStatus will not require you to redownload ALL of + your mails if you still have the local Maildir. It will simply + recreate LocalStatus. Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 32e48dd..0f8c7be 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -350,29 +350,14 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, ui.messagelistloaded(remoterepos, remotefolder, len(remotefolder.getmessagelist().keys())) - - # - - if not statusfolder.isnewfolder(): - # Delete local copies of remote messages. This way, - # if a message's flag is modified locally but it has been - # deleted remotely, we'll delete it locally. Otherwise, we - # try to modify a deleted message's flags! This step - # need only be taken if a statusfolder is present; otherwise, - # there is no action taken *to* the remote repository. - - remotefolder.syncmessagesto_delete(localfolder, [localfolder, - statusfolder]) - ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) - localfolder.syncmessagesto(statusfolder, [remotefolder, statusfolder]) - # Synchronize remote changes. ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) - remotefolder.syncmessagesto(localfolder, [localfolder, statusfolder]) + remotefolder.syncmessagesto(localfolder, statusfolder) + # Synchronize local changes + ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) + localfolder.syncmessagesto(remotefolder, statusfolder) + - # Make sure the status folder is up-to-date. - ui.syncingmessages(localrepos, localfolder, statusrepos, statusfolder) - localfolder.syncmessagesto(statusfolder) statusfolder.save() localrepos.restore_atime() except (KeyboardInterrupt, SystemExit): diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 5238f1e..fb8519b 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -206,68 +206,89 @@ class BaseFolder: for uid in uidlist: self.deletemessage(uid) - def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1): - if register: - self.ui.registerthread(self.getaccountname()) - self.ui.copyingmessage(uid, self, applyto) + def syncmessagesto_neguid_msg(self, uid, dstfolder, statusfolder, + register = 1): + """Copy a single message from self to dests. + + This is called by meth:`syncmessagesto_neguid`, possibly in a + new thread. It does not return anything. + + :param dstfolder: A BaseFolder-derived instance + :param statusfolder: A LocalStatusFolder instance + :param register: If True, output that a new thread was created + (and register it with the ui).""" successobject = None successuid = None + + if register: + self.ui.registerthread(self.getaccountname()) + self.ui.copyingmessage(uid, self, [dstfolder]) + message = self.getmessage(uid) flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - for tryappend in applyto: - successuid = tryappend.savemessage(uid, message, flags, rtime) - if successuid >= 0: - successobject = tryappend - break - # Did we succeed? - if successobject != None: - if successuid: # Only if IMAP actually assigned a UID - # Copy the message to the other remote servers. - for appendserver in \ - [x for x in applyto if x != successobject]: - appendserver.savemessage(successuid, message, flags, rtime) - # Copy to its new name on the local server and delete - # the one without a UID. - self.savemessage(successuid, message, flags, rtime) - self.deletemessage(uid) # It'll be re-downloaded. - else: - # Did not find any server to take this message. Ignore. - pass + + #Save messages to dstfolder and see if a valid UID was returned + successuid = dstfolder.savemessage(uid, message, flags, rtime) + + #Succeeded? -> IMAP actually assigned a UID + #If successuid remained negative, no server was willing to assign us + #an UID. Ignore message. + if successuid >= 0: + # Copy the message to the statusfolder + statusfolder.savemessage(successuid, message, flags, rtime) + # Copy to its new name son the local server and delete + # the one without a UID. + self.savemessage(successuid, message, flags, rtime) + #TODO: above means, we read in the message and write it out + #to the very same dir with a different UID. Investigate if we + #cannot simply rename it. + + # delete the negative uid message. We have it with a good UID now. + self.deletemessage(uid) - def syncmessagesto_neguid(self, dest, applyto): + def syncmessagesto_neguid(self, dstfolder, statusfolder): """Pass 1 of folder synchronization. - Look for messages in self with a negative uid. These are messages in - Maildirs that were not added by us. Try to add them to the dests, - and once that succeeds, get the UID, add it to the others for real, - add it to local for real, and delete the fake one.""" + Look for messages in self with a negative uid. These are + messages in Maildirs that were not added by us. Try to add them + to the dstfolder. If that succeeds, get the new UID, add + it to the statusfolder, add it to local for real, and delete the + old fake (negative) one. + + :param dstfolder: A BaseFolder-derived instance + :param statusfolder: A LocalStatusFolder instance""" uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0] threads = [] - usethread = None - if applyto != None: - usethread = applyto[0] - for uid in uidlist: - if usethread and usethread.suggeststhreads(): - usethread.waitforthread() + if dstfolder.suggeststhreads(): + dstfolder.waitforthread() thread = threadutil.InstanceLimitedThread(\ - usethread.getcopyinstancelimit(), + dstfolder.getcopyinstancelimit(), target = self.syncmessagesto_neguid_msg, name = "New msg sync from %s" % self.getvisiblename(), - args = (uid, dest, applyto)) + args = (uid, dstfolder, statusfolder)) thread.setDaemon(1) thread.start() threads.append(thread) else: - self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0) + self.syncmessagesto_neguid_msg(uid, dstfolder, statusfolder, + register = 0) + #wait for all uploads to finish for thread in threads: thread.join() - def copymessageto(self, uid, applyto, register = 1): + def copymessageto(self, uid, dstfolder, statusfolder, register = 1): + """Copies a message from self to dst if needed, updating the status + + :param uid: uid of the message to be copied. + :param dstfolder: A BaseFolder-derived instance + :param statusfolder: A LocalStatusFolder instance + :param register: whether we should register a new thread." + :returns: Nothing on success, or raises an Exception.""" # Sometimes, it could be the case that if a sync takes awhile, # a message might be deleted from the maildir before it can be # synced to the status cache. This is only a problem with @@ -276,156 +297,182 @@ class BaseFolder: try: if register: self.ui.registerthread(self.getaccountname()) - self.ui.copyingmessage(uid, self, applyto) - message = '' - # If any of the destinations actually stores the message body, - # load it up. - - for object in applyto: - if object.storesmessages(): - message = self.getmessage(uid) - break + + message = None flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - for object in applyto: - newuid = object.savemessage(uid, message, flags, rtime) - if newuid > 0 and newuid != uid: - # Change the local uid. - self.savemessage(newuid, message, flags, rtime) - self.deletemessage(uid) - uid = newuid + + if uid in dstfolder.getmessagelist(): + # dst has message with that UID already, only update status + statusfolder.savemessage(uid, None, flags, rtime) + return + + # really need to copy to dst... + self.ui.copyingmessage(uid, self, [dstfolder]) + + # If any of the destinations actually stores the message body, + # load it up. + if dstfolder.storesmessages(): + message = self.getmessage(uid) + + newuid = dstfolder.savemessage(uid, message, flags, rtime) + if newuid > 0 and newuid != uid: + # Change the local uid. + self.savemessage(newuid, message, flags, rtime) + self.deletemessage(uid) + uid = newuid + statusfolder.savemessage(uid, message, flags, rtime) except (KeyboardInterrupt): raise except: self.ui.warn("ERROR attempting to copy message " + str(uid) \ - + " for account " + self.getaccountname() + ":" + traceback.format_exc()) - + + " for account " + self.getaccountname() + ":" \ + + traceback.format_exc()) + raise - def syncmessagesto_copy(self, dest, applyto): - """Pass 2 of folder synchronization. + def syncmessagesto_copy(self, dstfolder, statusfolder): + """Pass2: Copy locally existing messages - Look for messages present in self but not in dest. If any, add - them to dest.""" + This will copy messages with a valid UID but are not on the + other side yet. The strategy is: + + 1) Look for messages present in self but not in statusfolder. + 2) invoke copymessageto() on those which: + - If dstfolder doesn't have it yet, add them to dstfolder. + - Update statusfolder + """ threads = [] - - dest_messagelist = dest.getmessagelist() - for uid in self.getmessagelist().keys(): - if uid < 0: # Ignore messages that pass 1 missed. - continue - if not uid in dest_messagelist: - if self.suggeststhreads(): - self.waitforthread() - thread = threadutil.InstanceLimitedThread(\ - self.getcopyinstancelimit(), - target = self.copymessageto, - name = "Copy message %d from %s" % (uid, - self.getvisiblename()), - args = (uid, applyto)) - thread.setDaemon(1) - thread.start() - threads.append(thread) - else: - self.copymessageto(uid, applyto, register = 0) + + copylist = filter(lambda uid: uid>=0 and not \ + uid in statusfolder.getmessagelist(), + self.getmessagelist().keys()) + for uid in copylist: + if self.suggeststhreads(): + self.waitforthread() + thread = threadutil.InstanceLimitedThread(\ + self.getcopyinstancelimit(), + target = self.copymessageto, + name = "Copy message %d from %s" % (uid, + self.getvisiblename()), + args = (uid, dstfolder, statusfolder)) + thread.setDaemon(1) + thread.start() + threads.append(thread) + else: + self.copymessageto(uid, dstfolder, statusfolder, register = 0) + for thread in threads: thread.join() - def syncmessagesto_delete(self, dest, applyto): - """Pass 3 of folder synchronization. + def syncmessagesto_delete(self, dstfolder, statusfolder): + """Pass 3: Remove locally deleted messages on dst - Look for message present in dest but not in self. - If any, delete them.""" - deletelist = [] - self_messagelist = self.getmessagelist() - for uid in dest.getmessagelist().keys(): - if uid < 0: - continue - if not uid in self_messagelist: - deletelist.append(uid) + Get all UIDS in statusfolder but not self. These are messages + that were deleted in 'self'. Delete those from dstfolder and + statusfolder.""" + deletelist = filter(lambda uid: uid>=0 \ + and not uid in self.getmessagelist(), + statusfolder.getmessagelist().keys()) if len(deletelist): - self.ui.deletingmessages(deletelist, applyto) - for object in applyto: - object.deletemessages(deletelist) + self.ui.deletingmessages(deletelist, [dstfolder]) + # delete in statusfolder first to play safe. In case of abort, we + # won't lose message, we will just retransmit some unneccessary. + for folder in [statusfolder, dstfolder]: + folder.deletemessages(deletelist) - def syncmessagesto_flags(self, dest, applyto): - """Pass 4 of folder synchronization. - - Look for any flag matching issues -- set dest message to have the - same flags that we have.""" - - # As an optimization over previous versions, we store up which flags - # are being used for an add or a delete. For each flag, we store - # a list of uids to which it should be added. Then, we can call - # addmessagesflags() to apply them in bulk, rather than one - # call per message as before. This should result in some significant - # performance improvements. + def syncmessagesto_flags(self, dstfolder, statusfolder): + """Pass 4: Flag synchronization + Compare flag mismatches in self with those in statusfolder. If + msg has a valid UID and exists on dstfolder (has not e.g. been + deleted there), sync the flag change to both dstfolder and + statusfolder. + """ + # For each flag, we store a list of uids to which it should be + # added. Then, we can call addmessagesflags() to apply them in + # bulk, rather than one call per message. addflaglist = {} delflaglist = {} - for uid in self.getmessagelist().keys(): - if uid < 0: # Ignore messages missed by pass 1 + # Ignore messages with negative UIDs missed by pass 1 + # also don't do anything if the message has been deleted remotely + if uid < 0 or not uid in dstfolder.getmessagelist(): continue - selfflags = self.getmessageflags(uid) - destflags = dest.getmessageflags(uid) - addflags = [x for x in selfflags if x not in destflags] + selfflags = self.getmessageflags(uid) + statusflags = statusfolder.getmessageflags(uid) + #if we could not get message flags from LocalStatus, assume empty. + if statusflags is None: + statusflags = [] + addflags = [x for x in selfflags if x not in statusflags] for flag in addflags: if not flag in addflaglist: addflaglist[flag] = [] addflaglist[flag].append(uid) - delflags = [x for x in destflags if x not in selfflags] + delflags = [x for x in statusflags if x not in selfflags] for flag in delflags: if not flag in delflaglist: delflaglist[flag] = [] delflaglist[flag].append(uid) - for object in applyto: - for flag in addflaglist.keys(): - self.ui.addingflags(addflaglist[flag], flag, [object]) - object.addmessagesflags(addflaglist[flag], [flag]) - for flag in delflaglist.keys(): - self.ui.deletingflags(delflaglist[flag], flag, [object]) - object.deletemessagesflags(delflaglist[flag], [flag]) + for flag in addflaglist.keys(): + self.ui.addingflags(addflaglist[flag], flag, [dstfolder]) + dstfolder.addmessagesflags(addflaglist[flag], [flag]) + statusfolder.addmessagesflags(addflaglist[flag], [flag]) + + for flag in delflaglist.keys(): + self.ui.deletingflags(delflaglist[flag], flag, [dstfolder]) + dstfolder.deletemessagesflags(delflaglist[flag], [flag]) + statusfolder.deletemessagesflags(delflaglist[flag], [flag]) - def syncmessagesto(self, dest, applyto = None): - """Syncs messages in this folder to the destination. - If applyto is specified, it should be a list of folders (don't forget - to include dest!) to which all write actions should be applied. - It defaults to [dest] if not specified. It is important that - the UID generator be listed first in applyto; that is, the other - applyto ones should be the ones that "copy" the main action.""" - if applyto == None: - applyto = [dest] + def syncmessagesto(self, dstfolder, statusfolder): + """Syncs messages in this folder to the destination dstfolder. - try: - self.syncmessagesto_neguid(dest, applyto) - except (KeyboardInterrupt): - raise - except: - self.ui.warn("ERROR attempting to handle negative uids " \ - + "for account " + self.getaccountname() + ":" + traceback.format_exc()) + This is the high level entry for syncing messages in one direction. + Syncsteps are: - #all threads launched here are in try / except clauses when they copy anyway... - self.syncmessagesto_copy(dest, applyto) + Pass1: Transfer new local messages + Upload msg with negative/no UIDs to dstfolder. dstfolder + might assign that message a new UID. Update statusfolder. - try: - self.syncmessagesto_delete(dest, applyto) - except (KeyboardInterrupt): - raise - except: - self.ui.warn("ERROR attempting to delete messages " \ - + "for account " + self.getaccountname() + ":" + traceback.format_exc()) + Pass2: Copy locally existing messages + Copy messages in self, but not statusfolder to dstfolder if not + already in dstfolder. Update statusfolder. - # Now, the message lists should be identical wrt the uids present. - # (except for potential negative uids that couldn't be placed - # anywhere) + Pass3: Remove locally deleted messages + Get all UIDS in statusfolder but not self. These are messages + that were deleted in 'self'. Delete those from dstfolder and + statusfolder. - try: - self.syncmessagesto_flags(dest, applyto) - except (KeyboardInterrupt): - raise - except: - self.ui.warn("ERROR attempting to sync flags " \ - + "for account " + self.getaccountname() + ":" + traceback.format_exc()) + After this pass, the message lists should be identical wrt the + uids present (except for potential negative uids that couldn't + be placed anywhere). + + Pass4: Synchronize flag changes + Compare flag mismatches in self with those in statusfolder. If + msg has a valid UID and exists on dstfolder (has not e.g. been + deleted there), sync the flag change to both dstfolder and + statusfolder. + + + :param dstfolder: Folderinstance to sync the msgs to. + :param statusfolder: LocalStatus instance to sync against. + """ + passes = [('uploading negative UIDs', self.syncmessagesto_neguid), + ('copying messages' , self.syncmessagesto_copy), + ('deleting messages' , self.syncmessagesto_delete), + ('syncing flags' , self.syncmessagesto_flags)] + + for (passdesc, action) in passes: + try: + action(dstfolder, statusfolder) + except (KeyboardInterrupt): + raise + except: + self.ui.warn("ERROR attempting to sync flags " \ + + "for account " + self.getaccountname() \ + + ":" + traceback.format_exc()) + + raise From ab1df868c29aee10dd71055b6bc4175e7b3287ec Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 6 Mar 2011 20:03:05 +0100 Subject: [PATCH 022/817] String representation of a Folder is its name This enables us to just use the folder instance in the ui output and get a name rather than having to call getname() all the time. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index fb8519b..2e0b177 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -31,6 +31,9 @@ class BaseFolder: """Returns name""" return self.name + def __str__(self): + return self.name + def suggeststhreads(self): """Returns true if this folder suggests using threads for actions; false otherwise. Probably only IMAP will return true.""" From 387fbf3aaaec21b0119385103f18dce398ad7a01 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 6 Mar 2011 20:03:06 +0100 Subject: [PATCH 023/817] ui: clean up importment statements They were not PEP-8 formatted, and some imports were simply unnecessary. Removed those. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Machine.py | 6 ++++-- offlineimap/ui/UIBase.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index e5988e5..e8864ab 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -15,9 +15,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import urllib, sys, re, time, traceback, threading, thread +import urllib +import sys +import time from UIBase import UIBase -from threading import * +from threading import currentThread, Lock import offlineimap protocol = '6.0.0' diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 6621a62..03b76e0 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -16,7 +16,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import re, time, sys, traceback, threading, thread +import re +import time +import sys +import traceback +import threading from StringIO import StringIO from Queue import Empty import offlineimap From 7a2a02254e238898da8e053e9181d5147ca22a41 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 6 Mar 2011 20:03:07 +0100 Subject: [PATCH 024/817] Don't pass list to ui.adding/deletingflags We only have one "dstfolder" at a time when deleting/adding flags, so no need to pass in a list of those to the ui functions that output the log info. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 4 ++-- offlineimap/ui/Blinkenlights.py | 8 ++++---- offlineimap/ui/Machine.py | 10 ++++------ offlineimap/ui/UIBase.py | 13 ++++++------- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 2e0b177..5c06e6e 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -421,12 +421,12 @@ class BaseFolder: delflaglist[flag].append(uid) for flag in addflaglist.keys(): - self.ui.addingflags(addflaglist[flag], flag, [dstfolder]) + self.ui.addingflags(addflaglist[flag], flag, dstfolder) dstfolder.addmessagesflags(addflaglist[flag], [flag]) statusfolder.addmessagesflags(addflaglist[flag], [flag]) for flag in delflaglist.keys(): - self.ui.deletingflags(delflaglist[flag], flag, [dstfolder]) + self.ui.deletingflags(delflaglist[flag], flag, dstfolder) dstfolder.deletemessagesflags(delflaglist[flag], [flag]) statusfolder.deletemessagesflags(delflaglist[flag], [flag]) diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py index 08303a4..2f11aa4 100644 --- a/offlineimap/ui/Blinkenlights.py +++ b/offlineimap/ui/Blinkenlights.py @@ -66,13 +66,13 @@ class BlinkenBase: s.gettf().setcolor('red') s.__class__.__bases__[-1].deletingmessage(s, uid, destlist) - def addingflags(s, uidlist, flags, destlist): + def addingflags(s, uidlist, flags, dest): s.gettf().setcolor('yellow') - s.__class__.__bases__[-1].addingflags(s, uidlist, flags, destlist) + s.__class__.__bases__[-1].addingflags(s, uidlist, flags, dest) - def deletingflags(s, uidlist, flags, destlist): + def deletingflags(s, uidlist, flags, dest): s.gettf().setcolor('pink') - s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, destlist) + s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, dest) def warn(s, msg, minor = 0): if minor: diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index e8864ab..37b5e9b 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -126,17 +126,15 @@ class MachineUI(UIBase): ds = s.folderlist(destlist) s._printData('deletingmessages', "%s\n%s" % (s.uidlist(uidlist), ds)) - def addingflags(s, uidlist, flags, destlist): - ds = s.folderlist(destlist) + def addingflags(s, uidlist, flags, dest): s._printData("addingflags", "%s\n%s\n%s" % (s.uidlist(uidlist), "\f".join(flags), - ds)) + dest)) - def deletingflags(s, uidlist, flags, destlist): - ds = s.folderlist(destlist) + def deletingflags(s, uidlist, flags, dest): s._printData('deletingflags', "%s\n%s\n%s" % (s.uidlist(uidlist), "\f".join(flags), - ds)) + dest)) def threadException(s, thread): print s.getThreadExceptionString(thread) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 03b76e0..fedcdac 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -261,17 +261,16 @@ class UIBase: ", ".join([str(u) for u in uidlist]), ds)) - def addingflags(s, uidlist, flags, destlist): + def addingflags(s, uidlist, flags, dest): if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Adding flags %s to %d messages on %s" % \ - (", ".join(flags), len(uidlist), ds)) + s._msg("Adding flag %s to %d messages on %s" % \ + (", ".join(flags), len(uidlist), dest)) - def deletingflags(s, uidlist, flags, destlist): + def deletingflags(s, uidlist, flags, dest): if s.verbose >= 0: ds = s.folderlist(destlist) - s._msg("Deleting flags %s to %d messages on %s" % \ - (", ".join(flags), len(uidlist), ds)) + s._msg("Deleting flag %s from %d messages on %s" % \ + (", ".join(flags), len(uidlist), dest)) ################################################## Threads From b94bf792585a9297851bdb965c4bbd6bdadeeb42 Mon Sep 17 00:00:00 2001 From: Haojun Bao Date: Thu, 3 Mar 2011 17:48:26 +0800 Subject: [PATCH 025/817] fix hang because of infinite loop reading EOF Read() should return empty string when EOF happen, instead of looping forever. This is the right semantics of read(), and a wrapped version should not change it. If you read the read(2) system call manpage, it tells you that when EOF is seen, return value is 0; it does not say ``loop forever when EOF happen''. After the EOF detection is patched you can see the following exception: WARNING: ERROR attempting to copy message 344 for account Gmail:Traceback (most recent call last): File "/usr/lib/pymodules/python2.6/offlineimap/folder/Base.py", line 282, in copymessageto message = self.getmessage(uid) File "/usr/lib/pymodules/python2.6/offlineimap/folder/IMAP.py", line 216, in getmessage initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])') File "/usr/lib/python2.6/imaplib.py", line 753, in uid typ, dat = self._simple_command(name, command, *args) File "/usr/lib/python2.6/imaplib.py", line 1060, in _simple_command return self._command_complete(name, self._command(name, *args)) File "/usr/lib/python2.6/imaplib.py", line 890, in _command_complete raise self.abort('command: %s => %s' % (name, val)) abort: command: UID => socket error: EOF Signed-off-by: Bao Haojun Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 1 + offlineimap/imaplibutil.py | 9 ++++++++- offlineimap/imapserver.py | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index e754f50..b8c82f8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,6 +19,7 @@ Changes Bug Fixes --------- +* Fix hang because of infinite loop reading EOF. * Fix regression (UIBase is no more). * Make profiling mode really enforce single-threading diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index cf82996..4f69d41 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -48,7 +48,10 @@ class IMAP4_Tunnel(IMAP4): def read(self, size): retval = '' while len(retval) < size: - retval += self.infd.read(size - len(retval)) + buf = self.infd.read(size - len(retval)) + if not buf: + break + retval += buf return retval def readline(self): @@ -188,6 +191,8 @@ class WrappedIMAP4_SSL(IMAP4_SSL): read = 0 while read < n: data = self._read_upto (n-read) + if not data: + break read += len(data) chunks.append(data) @@ -200,6 +205,8 @@ class WrappedIMAP4_SSL(IMAP4_SSL): retval = '' while 1: linebuf = self._read_upto(1024) + if not linebuf: + return retval nlindex = linebuf.find("\n") if nlindex != -1: retval += linebuf[:nlindex + 1] diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 2a9f247..c298aa3 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -71,6 +71,8 @@ class UsefulIMAP4(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): io = StringIO() while read < size: data = imaplib.IMAP4.read (self, min(size-read,8192)) + if not data: + break read += len(data) io.write(data) return io.getvalue() @@ -86,6 +88,8 @@ class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): io = StringIO() while read < size: data = imaplibutil.WrappedIMAP4_SSL.read (self, min(size-read,8192)) + if not data: + break read += len(data) io.write(data) return io.getvalue() From b18bde70c9aa805141381a48aceb03b89425b0a2 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 7 Mar 2011 22:23:21 +0100 Subject: [PATCH 026/817] doc: rst2xxx: fix ignoring output while determining the command name Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ Makefile | 2 +- docs/Makefile | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index b8c82f8..6d28f42 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,6 +19,8 @@ Changes Bug Fixes --------- +* Fix ignoring output while determining the rst2xxx command name to build + documentation. * Fix hang because of infinite loop reading EOF. * Fix regression (UIBase is no more). diff --git a/Makefile b/Makefile index 7d45f38..3e5cd93 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ VERSION=6.3.2 TARGZ=offlineimap_$(VERSION).tar.gz SHELL=/bin/bash -RST2HTML=`type rst2html 2>/dev/null 2>&1 && echo rst2html || echo rst2html.py` +RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` all: build diff --git a/docs/Makefile b/docs/Makefile index fff639d..70a99a2 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,8 +5,8 @@ SOURCES = $(wildcard *.rst) HTML_TARGETS = $(patsubst %.rst,%.html,$(SOURCES)) RM = rm -RST2HTML=`type rst2html 2>/dev/null 2>&1 && echo rst2html || echo rst2html.py` -RST2MAN=`type rst2man 2>/dev/null 2>&1 && echo rst2man || echo rst2man.py` +RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` +RST2MAN=`type rst2man >/dev/null 2>&1 && echo rst2man || echo rst2man.py` all: html From efcce01d640ade3b03596fe991d466e4f5bbf4fe Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 9 Mar 2011 08:44:41 +0100 Subject: [PATCH 027/817] Declutter TTY output Previously we would output: Folder sync sspaeth.de[INBOX.INBOX201004]: Syncing INBOX.INBOX201004: IMAP -> Maildir Folder sync sspaeth.de[INBOX.INBOX201006]: Syncing INBOX.INBOX201006: IMAP -> Maildir Folder sync sspaeth.de[INBOX.INBOX201009]: Syncing INBOX.INBOX201009: IMAP -> Maildir which is very repetitive and cluttered. By naming the folder sync threads just according to the account and not the folder, the output looks much nicer: Folder sync [sspaeth.de]: Syncing INBOX.INBOX201004: IMAP -> Maildir Syncing INBOX.INBOX201006: IMAP -> Maildir Syncing INBOX.INBOX201009: IMAP -> Maildir If syncing multiple accounts in parallel, we will still get headers indicating the account: Folder sync [sspaeth.de]: Syncing INBOX: IMAP -> Maildir Syncing INBOX.INBOX201006: IMAP -> Maildir Folder sync [gmail]: Syncing INBOX: IMAP -> Maildir This is a small fix that makes the output much nicer in my opinion. Also don't output the thread name if we are in the MainThread, e.g. when we output the initial offlineimap banner. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ offlineimap/accounts.py | 3 +-- offlineimap/ui/TTY.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index b8c82f8..888f328 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,8 @@ New Features Changes ------- +* TTYUI ouput improved. + Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 8c03e69..7e87909 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -252,8 +252,7 @@ class AccountSynchronizationMixin: thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, - name = "Folder sync %s[%s]" % \ - (self.name, remotefolder.getvisiblename()), + name = "Folder sync [%s]" % self.name, args = (self.name, remoterepos, remotefolder, localrepos, statusrepos, quick)) thread.setDaemon(1) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index ee18dfa..d88de1b 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -41,7 +41,8 @@ class TTYUI(UIBase): threadname = currentThread().name except AttributeError: threadname = currentThread().getName() - if (threadname == s._lastThreaddisplay): + if (threadname == s._lastThreaddisplay \ + or threadname == 'MainThread'): print " %s" % msg else: print "%s:\n %s" % (threadname, msg) From 4d352a528a3a74a457a6e3cd9dfdd565fee661df Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 9 Mar 2011 08:53:20 +0100 Subject: [PATCH 028/817] folder: Implement helper function getmessagecount() Rather than always having to call len(getmessagelist.keys()) as was done before. No functional change, just nicer looking code. Also the SQLite backend or other backends could implement more efficient implementations. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 6 +++--- offlineimap/folder/Base.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 35550c1..7ffdfc1 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -324,13 +324,13 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) ui.loadmessagelist(localrepos, localfolder) localfolder.cachemessagelist() - ui.messagelistloaded(localrepos, localfolder, len(localfolder.getmessagelist().keys())) + ui.messagelistloaded(localrepos, localfolder, localfolder.getmessagecount()) # If either the local or the status folder has messages and there is a UID # validity problem, warn and abort. If there are no messages, UW IMAPd # loses UIDVALIDITY. But we don't really need it if both local folders are # empty. So, in that case, just save it off. - if len(localfolder.getmessagelist()) or len(statusfolder.getmessagelist()): + if localfolder.getmessagecount() or statusfolder.getmessagecount(): if not localfolder.isuidvalidityok(): ui.validityproblem(localfolder) localrepos.restore_atime() @@ -347,7 +347,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, ui.loadmessagelist(remoterepos, remotefolder) remotefolder.cachemessagelist() ui.messagelistloaded(remoterepos, remotefolder, - len(remotefolder.getmessagelist().keys())) + remotefolder.getmessagecount()) # Synchronize remote changes. ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 5c06e6e..ffbbbbb 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -139,6 +139,10 @@ class BaseFolder: You must call cachemessagelist() before calling this function!""" raise NotImplementedException + def getmessagecount(self): + """Gets the number of messages.""" + return len(self.getmessagelist()) + def getmessage(self, uid): """Returns the content of the specified message.""" raise NotImplementedException From 6c6fdfc76946c9218e55a1bbb2e9e628bf93e243 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 11 Mar 2011 10:50:18 +0100 Subject: [PATCH 029/817] folder: Implement helper functions uidexists() and getmessageuidlist() More convenient way to test if a certain uid exists and getting a list of all uids. Also, the SQL backend will have efficient overrides for these methods. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index ffbbbbb..4bc5aab 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -139,6 +139,15 @@ class BaseFolder: You must call cachemessagelist() before calling this function!""" raise NotImplementedException + def uidexists(self, uid): + """Returns True if uid exists""" + return uid in self.getmessagelist() + + def getmessageuidlist(self): + """Gets a list of UIDs. + You may have to call cachemessagelist() before calling this function!""" + return self.getmessagelist().keys() + def getmessagecount(self): """Gets the number of messages.""" return len(self.getmessagelist()) @@ -267,7 +276,7 @@ class BaseFolder: :param dstfolder: A BaseFolder-derived instance :param statusfolder: A LocalStatusFolder instance""" - uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0] + uidlist = [uid for uid in self.getmessageuidlist() if uid < 0] threads = [] for uid in uidlist: @@ -351,8 +360,8 @@ class BaseFolder: threads = [] copylist = filter(lambda uid: uid>=0 and not \ - uid in statusfolder.getmessagelist(), - self.getmessagelist().keys()) + statusfolder.uidexists(uid), + self.getmessageuidlist()) for uid in copylist: if self.suggeststhreads(): self.waitforthread() @@ -378,8 +387,8 @@ class BaseFolder: that were deleted in 'self'. Delete those from dstfolder and statusfolder.""" deletelist = filter(lambda uid: uid>=0 \ - and not uid in self.getmessagelist(), - statusfolder.getmessagelist().keys()) + and not self.uidexists(uid), + statusfolder.getmessageuidlist()) if len(deletelist): self.ui.deletingmessages(deletelist, [dstfolder]) # delete in statusfolder first to play safe. In case of abort, we @@ -400,10 +409,10 @@ class BaseFolder: # bulk, rather than one call per message. addflaglist = {} delflaglist = {} - for uid in self.getmessagelist().keys(): + for uid in self.getmessageuidlist(): # Ignore messages with negative UIDs missed by pass 1 # also don't do anything if the message has been deleted remotely - if uid < 0 or not uid in dstfolder.getmessagelist(): + if uid < 0 or not dstfolder.uidexists(uid): continue selfflags = self.getmessageflags(uid) From 90949a4bfc20c074696b89f1966299f879838789 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 11 Mar 2011 20:20:12 +0100 Subject: [PATCH 030/817] UIBase: fix regression while deletingflags In commit 7a2a0225 [Don't pass list to ui.adding/deletingflags] we changed the list logic for a per folder logic but forgot to remove one instance of "destlist" which isn't valid anymore. Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index fedcdac..02593e6 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -268,7 +268,6 @@ class UIBase: def deletingflags(s, uidlist, flags, dest): if s.verbose >= 0: - ds = s.folderlist(destlist) s._msg("Deleting flag %s from %d messages on %s" % \ (", ".join(flags), len(uidlist), dest)) From 10024731e68b95cee7f528a9241ebda822b8cffb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 11 Mar 2011 19:19:26 +0100 Subject: [PATCH 031/817] threadutil: Don't require the StringIO module The only reason we used it here was to do a traceback.print_exc(StringIO()) to get a string of our traceback. But we can simply use traceback.format_exc() which exists since python 2.4. One less module (and it is in the way to python 3 compatability too) Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/threadutil.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 09c742e..f3451b8 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -17,7 +17,6 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import * -from StringIO import StringIO from Queue import Queue, Empty import sys, traceback, thread, time from offlineimap.ui import getglobalui @@ -173,9 +172,8 @@ class ExitNotifyThread(Thread): self.setExitCause('EXCEPTION') if sys: self.setExitException(sys.exc_info()[1]) - sbuf = StringIO() - traceback.print_exc(file = sbuf) - self.setExitStackTrace(sbuf.getvalue()) + tb = traceback.format_exc() + self.setExitStackTrace(tb) else: self.setExitCause('NORMAL') if not hasattr(self, 'exitmessage'): From 2946a1ea5de4da5f0ef830ead4aa0f1b0fa5ae9f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 12 Mar 2011 14:05:28 +0100 Subject: [PATCH 032/817] Convert rfc822 module to email module The rfc822 module has been deprecated since python 2.3, and conversion to the email module is straightforward, so let us do that. rfc822 is completely gone in python3. This also fixes a bug that led to offlineimap abortion (but that code path is apparently usually not exercised so I did not notice: rfc822|email.utils.parsedate return a tuple which has no named attributes, but we were using them later in that function. So pass the tuple into a struct_time() to get named attributes. While reading the docs, I noticed that email.parsedate returns invalid daylight savings information (is_dst attribute), and we are using it anyway. Oh well, the imap server might think the mails are off by an hour at worst. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index a419664..8cf57d4 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -17,12 +17,11 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import imaplib -import rfc822 +import email import random import binascii import re import time -from StringIO import StringIO from copy import copy from Base import BaseFolder from offlineimap import imaputil, imaplibutil @@ -329,16 +328,17 @@ class IMAPFolder(BaseFolder): (including double quotes) or `None` in case of failure (which is fine as value for append).""" if rtime is None: - message = rfc822.Message(StringIO(content)) + message = email.message_from_string(content) # parsedate returns a 9-tuple that can be passed directly to # time.mktime(); Will be None if missing or not in a valid # format. Note that indexes 6, 7, and 8 of the result tuple are # not usable. - datetuple = rfc822.parsedate(message.getheader('Date')) - + datetuple = email.utils.parsedate(message.get('Date')) if datetuple is None: #could not determine the date, use the local time. return None + #make it a real struct_time, so we have named attributes + datetuple = time.struct_time(datetuple) else: #rtime is set, use that instead datetuple = time.localtime(rtime) @@ -358,7 +358,8 @@ class IMAPFolder(BaseFolder): # or something. Argh. It seems that Time2Internaldate # will rause a ValueError if the year is 0102 but not 1902, # but some IMAP servers nonetheless choke on 1902. - self.ui.debug("Message with invalid date %s. Server will use local time." % datetuple) + self.ui.debug("Message with invalid date %s. Server will use local time." \ + % datetuple) return None #produce a string representation of datetuple that works as @@ -366,6 +367,7 @@ class IMAPFolder(BaseFolder): num2mon = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'} + #tm_isdst coming from email.parsedate is not usable, we still use it here, mhh if datetuple.tm_isdst == '1': zone = -time.altzone else: From e2354fd37c85f2582f858b2a68c15872dc73c4c4 Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Tue, 8 Mar 2011 10:05:14 -0500 Subject: [PATCH 033/817] Remove some unneeded imports Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Gmail.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 59d3253..2a82833 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -20,8 +20,7 @@ """ from IMAP import IMAPFolder -import imaplib -from offlineimap import imaputil, imaplibutil +from offlineimap import imaputil from copy import copy From 0a76f0a23dd729c074c03bb73ac70ded2003490e Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Tue, 8 Mar 2011 10:05:15 -0500 Subject: [PATCH 034/817] Import newest version of imaplib2 This change does not do anything yet with imaplib2, merely makes it available for future commits. This file is identical to the one at http://sydney.edu.au/engineering/it/~piers/python/imaplib2 . imaplib2, written by the same guy who wrote imaplib, is very different from imaplib itself. Calling it a modified version from the standard distribution is misleading. It's more like a complete rewrite. As such, it's not really possible to summarize what was changed. The largest thing is that imaplib2 is "threaded". Instead of doing blocking writes/reads on the socket during/after every command, imaplib2 forks off threads to read and write to the socket based on input and output buffers. This opens the door to asynchronous commands (every command is potentially asynchronous, according to the docs), and in particular IDLE, which is by definition an asynchronous command. The author writes: "imaplib2 can be substituted for imaplib in existing clients with no changes in the code", but that's pretty misleading. It might be true for certain simple users of imaplib, but for us it's completely false. Among other things, how untagged responses are stored in-memory is different -- instead of a hash table, it's a list. I'm guessing this is to preserve order of responses. I think there are other miscellaneous improvements, like I think imaplib2 is IPv6 safe out-of-the-box, but I haven't conducted an extremely thorough examination of the differences :) Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 2323 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 2323 insertions(+) create mode 100755 offlineimap/imaplib2.py diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py new file mode 100755 index 0000000..20b90ae --- /dev/null +++ b/offlineimap/imaplib2.py @@ -0,0 +1,2323 @@ +#!/usr/bin/env python + +"""Threaded IMAP4 client. + +Based on RFC 2060 and original imaplib module. + +Public classes: IMAP4 + IMAP4_SSL + IMAP4_stream + +Public functions: Internaldate2Time + ParseFlags + Time2Internaldate +""" + + +__all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", + "Internaldate2Time", "ParseFlags", "Time2Internaldate") + +__version__ = "2.20" +__release__ = "2" +__revision__ = "20" +__credits__ = """ +Authentication code contributed by Donn Cave June 1998. +String method conversion by ESR, February 2001. +GET/SETACL contributed by Anthony Baxter April 2001. +IMAP4_SSL contributed by Tino Lange March 2002. +GET/SETQUOTA contributed by Andreas Zeidler June 2002. +PROXYAUTH contributed by Rick Holbert November 2002. +IDLE via threads suggested by Philippe Normand January 2005. +GET/SETANNOTATION contributed by Tomas Lindroos June 2005. +COMPRESS/DEFLATE contributed by Bron Gondwana May 2009. +STARTTLS from Jython's imaplib by Alan Kennedy. +ID contributed by Dave Baggett November 2009. +Improved untagged responses handling suggested by Dave Baggett November 2009. +Improved thread naming, and 0 read detection contributed by Grant Edwards June 2010. +Improved timeout handling contributed by Ivan Vovnenko October 2010. +Timeout handling further improved by Ethan Glasser-Camp December 2010.""" +__author__ = "Piers Lauder " +__URL__ = "http://janeelix.com/piers/python/imaplib2" +__license__ = "Python License" + +import binascii, os, Queue, random, re, select, socket, sys, time, threading, zlib + +select_module = select + +# Globals + +CRLF = '\r\n' +Debug = None # Backward compatibility +IMAP4_PORT = 143 +IMAP4_SSL_PORT = 993 + +IDLE_TIMEOUT_RESPONSE = '* IDLE TIMEOUT\r\n' +IDLE_TIMEOUT = 60*29 # Don't stay in IDLE state longer +READ_POLL_TIMEOUT = 30 # Without this timeout interrupted network connections can hang reader + +AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first + +# Commands + +CMD_VAL_STATES = 0 +CMD_VAL_ASYNC = 1 +NONAUTH, AUTH, SELECTED, LOGOUT = 'NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT' + +Commands = { + # name valid states asynchronous + 'APPEND': ((AUTH, SELECTED), False), + 'AUTHENTICATE': ((NONAUTH,), False), + 'CAPABILITY': ((NONAUTH, AUTH, SELECTED), True), + 'CHECK': ((SELECTED,), True), + 'CLOSE': ((SELECTED,), False), + 'COMPRESS': ((AUTH,), False), + 'COPY': ((SELECTED,), True), + 'CREATE': ((AUTH, SELECTED), True), + 'DELETE': ((AUTH, SELECTED), True), + 'DELETEACL': ((AUTH, SELECTED), True), + 'EXAMINE': ((AUTH, SELECTED), False), + 'EXPUNGE': ((SELECTED,), True), + 'FETCH': ((SELECTED,), True), + 'GETACL': ((AUTH, SELECTED), True), + 'GETANNOTATION':((AUTH, SELECTED), True), + 'GETQUOTA': ((AUTH, SELECTED), True), + 'GETQUOTAROOT': ((AUTH, SELECTED), True), + 'ID': ((NONAUTH, AUTH, SELECTED), True), + 'IDLE': ((SELECTED,), False), + 'LIST': ((AUTH, SELECTED), True), + 'LOGIN': ((NONAUTH,), False), + 'LOGOUT': ((NONAUTH, AUTH, LOGOUT, SELECTED), False), + 'LSUB': ((AUTH, SELECTED), True), + 'MYRIGHTS': ((AUTH, SELECTED), True), + 'NAMESPACE': ((AUTH, SELECTED), True), + 'NOOP': ((NONAUTH, AUTH, SELECTED), True), + 'PARTIAL': ((SELECTED,), True), + 'PROXYAUTH': ((AUTH,), False), + 'RENAME': ((AUTH, SELECTED), True), + 'SEARCH': ((SELECTED,), True), + 'SELECT': ((AUTH, SELECTED), False), + 'SETACL': ((AUTH, SELECTED), False), + 'SETANNOTATION':((AUTH, SELECTED), True), + 'SETQUOTA': ((AUTH, SELECTED), False), + 'SORT': ((SELECTED,), True), + 'STARTTLS': ((NONAUTH,), False), + 'STATUS': ((AUTH, SELECTED), True), + 'STORE': ((SELECTED,), True), + 'SUBSCRIBE': ((AUTH, SELECTED), False), + 'THREAD': ((SELECTED,), True), + 'UID': ((SELECTED,), True), + 'UNSUBSCRIBE': ((AUTH, SELECTED), False), + } + +UID_direct = ('SEARCH', 'SORT', 'THREAD') + + +def Int2AP(num): + + """string = Int2AP(num) + Return 'num' converted to a string using characters from the set 'A'..'P' + """ + + val, a2p = [], 'ABCDEFGHIJKLMNOP' + num = int(abs(num)) + while num: + num, mod = divmod(num, 16) + val.insert(0, a2p[mod]) + return ''.join(val) + + + +class Request(object): + + """Private class to represent a request awaiting response.""" + + def __init__(self, parent, name=None, callback=None, cb_arg=None): + self.name = name + self.callback = callback # Function called to process result + self.callback_arg = cb_arg # Optional arg passed to "callback" + + self.tag = '%s%s' % (parent.tagpre, parent.tagnum) + parent.tagnum += 1 + + self.ready = threading.Event() + self.response = None + self.aborted = None + self.data = None + + + def abort(self, typ, val): + self.aborted = (typ, val) + self.deliver(None) + + + def get_response(self, exc_fmt=None): + self.callback = None + self.ready.wait() + + if self.aborted is not None: + typ, val = self.aborted + if exc_fmt is None: + exc_fmt = '%s - %%s' % typ + raise typ(exc_fmt % str(val)) + + return self.response + + + def deliver(self, response): + if self.callback is not None: + self.callback((response, self.callback_arg, self.aborted)) + return + + self.response = response + self.ready.set() + + + + +class IMAP4(object): + + """Threaded IMAP4 client class. + + Instantiate with: + IMAP4(host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 port); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response (default: no timeout). + + All IMAP4rev1 commands are supported by methods of the same name. + + Each command returns a tuple: (type, [data, ...]) where 'type' + is usually 'OK' or 'NO', and 'data' is either the text from the + tagged response, or untagged results from command. Each 'data' is + either a string, or a tuple. If a tuple, then the first part is the + header of the response, and the second part contains the data (ie: + 'literal' value). + + Errors raise the exception class .error(""). + IMAP4 server errors raise .abort(""), which is + a sub-class of 'error'. Mailbox status changes from READ-WRITE to + READ-ONLY raise the exception class .readonly(""), + which is a sub-class of 'abort'. + + "error" exceptions imply a program error. + "abort" exceptions imply the connection should be reset, and + the command re-tried. + "readonly" exceptions imply the command should be re-tried. + + All commands take two optional named arguments: + 'callback' and 'cb_arg' + If 'callback' is provided then the command is asynchronous, so after + the command is queued for transmission, the call returns immediately + with the tuple (None, None). + The result will be posted by invoking "callback" with one arg, a tuple: + callback((result, cb_arg, None)) + or, if there was a problem: + callback((None, cb_arg, (exception class, reason))) + + Otherwise the command is synchronous (waits for result). But note + that state-changing commands will both block until previous commands + have completed, and block subsequent commands until they have finished. + + All (non-callback) arguments to commands are converted to strings, + except for AUTHENTICATE, and the last argument to APPEND which is + passed as an IMAP4 literal. If necessary (the string contains any + non-printing characters or white-space and isn't enclosed with either + parentheses or double quotes) each string is quoted. However, the + 'password' argument to the LOGIN command is always quoted. If you + want to avoid having an argument string quoted (eg: the 'flags' + argument to STORE) then enclose the string in parentheses (eg: + "(\Deleted)"). + + There is one instance variable, 'state', that is useful for tracking + whether the client needs to login to the server. If it has the + value "AUTH" after instantiating the class, then the connection + is pre-authenticated (otherwise it will be "NONAUTH"). Selecting a + mailbox changes the state to be "SELECTED", closing a mailbox changes + back to "AUTH", and once the client has logged out, the state changes + to "LOGOUT" and no further commands may be issued. + + Note: to use this module, you must read the RFCs pertaining to the + IMAP4 protocol, as the semantics of the arguments to each IMAP4 + command are left to the invoker, not to mention the results. Also, + most IMAP servers implement a sub-set of the commands available here. + + Note also that you must call logout() to shut down threads before + discarding an instance. + """ + + class error(Exception): pass # Logical errors - debug required + class abort(error): pass # Service errors - close and retry + class readonly(abort): pass # Mailbox status changed to READ-ONLY + + + continuation_cre = re.compile(r'\+( (?P.*))?') + literal_cre = re.compile(r'.*{(?P\d+)}$') + mapCRLF_cre = re.compile(r'\r\n|\r|\n') + # Need to quote "atom-specials" :- + # "(" / ")" / "{" / SP / 0x00 - 0x1f / 0x7f / "%" / "*" / DQUOTE / "\" / "]" + # so match not the inverse set + mustquote_cre = re.compile(r"[^!#$&'+,./0-9:;<=>?@A-Z\[^_`a-z|}~-]") + response_code_cre = re.compile(r'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') + untagged_response_cre = re.compile(r'\* (?P[A-Z-]+)( (?P.*))?') + untagged_status_cre = re.compile(r'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?') + + + def __init__(self, host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None): + + self.state = NONAUTH # IMAP4 protocol state + self.literal = None # A literal argument to a command + self.tagged_commands = {} # Tagged commands awaiting response + self.untagged_responses = [] # [[typ: [data, ...]], ...] + self.mailbox = None # Current mailbox selected + self.mailboxes = {} # Untagged responses state per mailbox + self.is_readonly = False # READ-ONLY desired state + self.idle_rqb = None # Server IDLE Request - see _IdleCont + self.idle_timeout = None # Must prod server occasionally + + self._expecting_data = 0 # Expecting message data + self._accumulated_data = [] # Message data accumulated so far + self._literal_expected = None # Message data descriptor + + self.compressor = None # COMPRESS/DEFLATE if not None + self.decompressor = None + + # Create unique tag for this session, + # and compile tagged response matcher. + + self.tagnum = 0 + self.tagpre = Int2AP(random.randint(4096, 65535)) + self.tagre = re.compile(r'(?P' + + self.tagpre + + r'\d+) (?P[A-Z]+) (?P.*)') + + if __debug__: self._init_debug(debug, debug_file) + + self.resp_timeout = timeout # Timeout waiting for command response + + if timeout is not None and timeout < READ_POLL_TIMEOUT: + self.read_poll_timeout = timeout + else: + self.read_poll_timeout = READ_POLL_TIMEOUT + + # Open socket to server. + + self.open(host, port) + + if __debug__: + if debug: + self._mesg('connected to %s on port %s' % (self.host, self.port)) + + # Threading + + if identifier is not None: + self.identifier = identifier + else: + self.identifier = self.host + if self.identifier: + self.identifier += ' ' + + self.Terminate = False + + self.state_change_free = threading.Event() + self.state_change_pending = threading.Lock() + self.commands_lock = threading.Lock() + + self.ouq = Queue.Queue(10) + self.inq = Queue.Queue() + + self.wrth = threading.Thread(target=self._writer) + self.wrth.start() + self.rdth = threading.Thread(target=self._reader) + self.rdth.start() + self.inth = threading.Thread(target=self._handler) + self.inth.start() + + # Get server welcome message, + # request and store CAPABILITY response. + + try: + self.welcome = self._request_push(tag='continuation').get_response('IMAP4 protocol error: %s')[1] + + if self._get_untagged_response('PREAUTH'): + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + elif self._get_untagged_response('OK'): + if __debug__: self._log(1, 'state => NONAUTH') + else: + raise self.error(self.welcome) + + typ, dat = self.capability() + if dat == [None]: + raise self.error('no CAPABILITY response from server') + self.capabilities = tuple(dat[-1].upper().split()) + if __debug__: self._log(1, 'CAPABILITY: %r' % (self.capabilities,)) + + for version in AllowedVersions: + if not version in self.capabilities: + continue + self.PROTOCOL_VERSION = version + break + else: + raise self.error('server not IMAP4 compliant') + except: + self._close_threads() + raise + + + def __getattr__(self, attr): + # Allow UPPERCASE variants of IMAP4 command methods. + if attr in Commands: + return getattr(self, attr.lower()) + raise AttributeError("Unknown IMAP4 command: '%s'" % attr) + + + + # Overridable methods + + + def open(self, host=None, port=None): + """open(host=None, port=None) + Setup connection to remote server on "host:port" + (default: localhost:standard IMAP4 port). + This connection will be used by the routines: + read, send, shutdown, socket.""" + + self.host = host is not None and host or '' + self.port = port is not None and port or IMAP4_PORT + self.sock = self.open_socket() + self.read_fd = self.sock.fileno() + + + def open_socket(self): + """open_socket() + Open socket choosing first address family available.""" + + msg = (-1, 'could not open socket') + for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + s = socket.socket(af, socktype, proto) + except socket.error, msg: + continue + try: + s.connect(sa) + except socket.error, msg: + s.close() + continue + break + else: + raise socket.error(msg) + + return s + + + def start_compressing(self): + """start_compressing() + Enable deflate compression on the socket (RFC 4978).""" + + # rfc 1951 - pure DEFLATE, so use -15 for both windows + self.decompressor = zlib.decompressobj(-15) + self.compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) + + + def read(self, size): + """data = read(size) + Read at most 'size' bytes from remote.""" + + if self.decompressor is None: + return self.sock.recv(size) + + if self.decompressor.unconsumed_tail: + data = self.decompressor.unconsumed_tail + else: + data = self.sock.recv(8192) + + return self.decompressor.decompress(data, size) + + + def send(self, data): + """send(data) + Send 'data' to remote.""" + + if self.compressor is not None: + data = self.compressor.compress(data) + data += self.compressor.flush(zlib.Z_SYNC_FLUSH) + + self.sock.sendall(data) + + + def shutdown(self): + """shutdown() + Close I/O established in "open".""" + + self.sock.close() + + + def socket(self): + """socket = socket() + Return socket instance used to connect to IMAP4 server.""" + + return self.sock + + + + # Utility methods + + + def enable_compression(self): + """enable_compression() + Ask the server to start compressing the connection. + Should be called from user of this class after instantiation, as in: + if 'COMPRESS=DEFLATE' in imapobj.capabilities: + imapobj.enable_compression()""" + + try: + typ, dat = self._simple_command('COMPRESS', 'DEFLATE') + if typ == 'OK': + self.start_compressing() + if __debug__: self._log(1, 'Enabled COMPRESS=DEFLATE') + finally: + self.state_change_pending.release() + + + def pop_untagged_responses(self): + """ for typ,data in pop_untagged_responses(): pass + Generator for any remaining untagged responses. + Returns and removes untagged responses in order of reception. + Use at your own risk!""" + + while self.untagged_responses: + self.commands_lock.acquire() + try: + yield self.untagged_responses.pop(0) + finally: + self.commands_lock.release() + + + def recent(self, **kw): + """(typ, [data]) = recent() + Return 'RECENT' responses if any exist, + else prompt server for an update using the 'NOOP' command. + 'data' is None if no new messages, + else list of RECENT responses, most recent last.""" + + name = 'RECENT' + + data = [] + while True: + dat = self._get_untagged_response(name) + if not dat: + break + data += dat + + if data: + return self._deliver_dat(name, data, kw) + kw['untagged_response'] = name + return self.noop(**kw) # Prod server for response + + + def response(self, code, **kw): + """(code, [data]) = response(code) + Return data for response 'code' if received, or None. + Old value for response 'code' is cleared.""" + + typ, dat = self._untagged_response(code, [None], code.upper()) + return self._deliver_dat(typ, dat, kw) + + + + + # IMAP4 commands + + + def append(self, mailbox, flags, date_time, message, **kw): + """(typ, [data]) = append(mailbox, flags, date_time, message) + Append message to named mailbox. + All args except `message' can be None.""" + + name = 'APPEND' + if not mailbox: + mailbox = 'INBOX' + if flags: + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags + else: + flags = None + if date_time: + date_time = Time2Internaldate(date_time) + else: + date_time = None + self.literal = self.mapCRLF_cre.sub(CRLF, message) + try: + return self._simple_command(name, mailbox, flags, date_time, **kw) + finally: + self.state_change_pending.release() + + + def authenticate(self, mechanism, authobject, **kw): + """(typ, [data]) = authenticate(mechanism, authobject) + Authenticate command - requires response processing. + + 'mechanism' specifies which authentication mechanism is to + be used - it must appear in .capabilities in the + form AUTH=. + + 'authobject' must be a callable object: + + data = authobject(response) + + It will be called to process server continuation responses. + It should return data that will be encoded and sent to server. + It should return None if the client abort response '*' should + be sent instead.""" + + self.literal = _Authenticator(authobject).process + try: + typ, dat = self._simple_command('AUTHENTICATE', mechanism.upper()) + if typ != 'OK': + self._deliver_exc(self.error, dat[-1], kw) + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + finally: + self.state_change_pending.release() + return self._deliver_dat(typ, dat, kw) + + + def capability(self, **kw): + """(typ, [data]) = capability() + Fetch capabilities list from server.""" + + name = 'CAPABILITY' + kw['untagged_response'] = name + return self._simple_command(name, **kw) + + + def check(self, **kw): + """(typ, [data]) = check() + Checkpoint mailbox on server.""" + + return self._simple_command('CHECK', **kw) + + + def close(self, **kw): + """(typ, [data]) = close() + Close currently selected mailbox. + + Deleted messages are removed from writable mailbox. + This is the recommended command before 'LOGOUT'.""" + + if self.state != 'SELECTED': + raise self.error('No mailbox selected.') + try: + typ, dat = self._simple_command('CLOSE') + finally: + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + self.state_change_pending.release() + return self._deliver_dat(typ, dat, kw) + + + def copy(self, message_set, new_mailbox, **kw): + """(typ, [data]) = copy(message_set, new_mailbox) + Copy 'message_set' messages onto end of 'new_mailbox'.""" + + return self._simple_command('COPY', message_set, new_mailbox, **kw) + + + def create(self, mailbox, **kw): + """(typ, [data]) = create(mailbox) + Create new mailbox.""" + + return self._simple_command('CREATE', mailbox, **kw) + + + def delete(self, mailbox, **kw): + """(typ, [data]) = delete(mailbox) + Delete old mailbox.""" + + return self._simple_command('DELETE', mailbox, **kw) + + + def deleteacl(self, mailbox, who, **kw): + """(typ, [data]) = deleteacl(mailbox, who) + Delete the ACLs (remove any rights) set for who on mailbox.""" + + return self._simple_command('DELETEACL', mailbox, who, **kw) + + + def examine(self, mailbox='INBOX', **kw): + """(typ, [data]) = examine(mailbox='INBOX', readonly=False) + Select a mailbox for READ-ONLY access. (Flushes all untagged responses.) + 'data' is count of messages in mailbox ('EXISTS' response). + Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so + other responses should be obtained via "response('FLAGS')" etc.""" + + return self.select(mailbox=mailbox, readonly=True, **kw) + + + def expunge(self, **kw): + """(typ, [data]) = expunge() + Permanently remove deleted items from selected mailbox. + Generates 'EXPUNGE' response for each deleted message. + 'data' is list of 'EXPUNGE'd message numbers in order received.""" + + name = 'EXPUNGE' + kw['untagged_response'] = name + return self._simple_command(name, **kw) + + + def fetch(self, message_set, message_parts, **kw): + """(typ, [data, ...]) = fetch(message_set, message_parts) + Fetch (parts of) messages. + 'message_parts' should be a string of selected parts + enclosed in parentheses, eg: "(UID BODY[TEXT])". + 'data' are tuples of message part envelope and data, + followed by a string containing the trailer.""" + + name = 'FETCH' + kw['untagged_response'] = name + return self._simple_command(name, message_set, message_parts, **kw) + + + def getacl(self, mailbox, **kw): + """(typ, [data]) = getacl(mailbox) + Get the ACLs for a mailbox.""" + + kw['untagged_response'] = 'ACL' + return self._simple_command('GETACL', mailbox, **kw) + + + def getannotation(self, mailbox, entry, attribute, **kw): + """(typ, [data]) = getannotation(mailbox, entry, attribute) + Retrieve ANNOTATIONs.""" + + kw['untagged_response'] = 'ANNOTATION' + return self._simple_command('GETANNOTATION', mailbox, entry, attribute, **kw) + + + def getquota(self, root, **kw): + """(typ, [data]) = getquota(root) + Get the quota root's resource usage and limits. + (Part of the IMAP4 QUOTA extension defined in rfc2087.)""" + + kw['untagged_response'] = 'QUOTA' + return self._simple_command('GETQUOTA', root, **kw) + + + def getquotaroot(self, mailbox, **kw): + # Hmmm, this is non-std! Left for backwards-compatibility, sigh. + # NB: usage should have been defined as: + # (typ, [QUOTAROOT responses...]) = getquotaroot(mailbox) + # (typ, [QUOTA responses...]) = response('QUOTA') + """(typ, [[QUOTAROOT responses...], [QUOTA responses...]]) = getquotaroot(mailbox) + Get the list of quota roots for the named mailbox.""" + + typ, dat = self._simple_command('GETQUOTAROOT', mailbox) + typ, quota = self._untagged_response(typ, dat, 'QUOTA') + typ, quotaroot = self._untagged_response(typ, dat, 'QUOTAROOT') + return self._deliver_dat(typ, [quotaroot, quota], kw) + + + def id(self, *kv_pairs, **kw): + """(typ, [data]) = .id(kv_pairs) + 'data' is list of ID key value pairs. + Request information for problem analysis and determination. + The ID extension is defined in RFC 2971. """ + + name = 'ID' + kw['untagged_response'] = name + return self._simple_command(name, *kv_pairs, **kw) + + + def idle(self, timeout=None, **kw): + """"(typ, [data]) = idle(timeout=None) + Put server into IDLE mode until server notifies some change, + or 'timeout' (secs) occurs (default: 29 minutes), + or another IMAP4 command is scheduled.""" + + name = 'IDLE' + self.literal = _IdleCont(self, timeout).process + try: + return self._simple_command(name, **kw) + finally: + self.state_change_pending.release() + + + def list(self, directory='""', pattern='*', **kw): + """(typ, [data]) = list(directory='""', pattern='*') + List mailbox names in directory matching pattern. + 'data' is list of LIST responses. + + NB: for 'pattern': + % matches all except separator ( so LIST "" "%" returns names at root) + * matches all (so LIST "" "*" returns whole directory tree from root)""" + + name = 'LIST' + kw['untagged_response'] = name + return self._simple_command(name, directory, pattern, **kw) + + + def login(self, user, password, **kw): + """(typ, [data]) = login(user, password) + Identify client using plaintext password. + NB: 'password' will be quoted.""" + + try: + typ, dat = self._simple_command('LOGIN', user, self._quote(password)) + if typ != 'OK': + self._deliver_exc(self.error, dat[-1], kw) + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + finally: + self.state_change_pending.release() + return self._deliver_dat(typ, dat, kw) + + + def login_cram_md5(self, user, password, **kw): + """(typ, [data]) = login_cram_md5(user, password) + Force use of CRAM-MD5 authentication.""" + + self.user, self.password = user, password + return self.authenticate('CRAM-MD5', self._CRAM_MD5_AUTH, **kw) + + + def _CRAM_MD5_AUTH(self, challenge): + """Authobject to use with CRAM-MD5 authentication.""" + import hmac + return self.user + " " + hmac.HMAC(self.password, challenge).hexdigest() + + + def logout(self, **kw): + """(typ, [data]) = logout() + Shutdown connection to server. + Returns server 'BYE' response. + NB: You must call this to shut down threads before discarding an instance.""" + + self.state = LOGOUT + if __debug__: self._log(1, 'state => LOGOUT') + + try: + typ, dat = self._simple_command('LOGOUT') + except: + typ, dat = 'NO', ['%s: %s' % sys.exc_info()[:2]] + if __debug__: self._log(1, dat) + + self._close_threads() + + self.state_change_pending.release() + + if __debug__: self._log(1, 'connection closed') + + bye = self._get_untagged_response('BYE', leave=True) + if bye: + typ, dat = 'BYE', bye + return self._deliver_dat(typ, dat, kw) + + + def lsub(self, directory='""', pattern='*', **kw): + """(typ, [data, ...]) = lsub(directory='""', pattern='*') + List 'subscribed' mailbox names in directory matching pattern. + 'data' are tuples of message part envelope and data.""" + + name = 'LSUB' + kw['untagged_response'] = name + return self._simple_command(name, directory, pattern, **kw) + + + def myrights(self, mailbox, **kw): + """(typ, [data]) = myrights(mailbox) + Show my ACLs for a mailbox (i.e. the rights that I have on mailbox).""" + + name = 'MYRIGHTS' + kw['untagged_response'] = name + return self._simple_command(name, mailbox, **kw) + + + def namespace(self, **kw): + """(typ, [data, ...]) = namespace() + Returns IMAP namespaces ala rfc2342.""" + + name = 'NAMESPACE' + kw['untagged_response'] = name + return self._simple_command(name, **kw) + + + def noop(self, **kw): + """(typ, [data]) = noop() + Send NOOP command.""" + + if __debug__: self._dump_ur(3) + return self._simple_command('NOOP', **kw) + + + def partial(self, message_num, message_part, start, length, **kw): + """(typ, [data, ...]) = partial(message_num, message_part, start, length) + Fetch truncated part of a message. + 'data' is tuple of message part envelope and data. + NB: obsolete.""" + + name = 'PARTIAL' + kw['untagged_response'] = 'FETCH' + return self._simple_command(name, message_num, message_part, start, length, **kw) + + + def proxyauth(self, user, **kw): + """(typ, [data]) = proxyauth(user) + Assume authentication as 'user'. + (Allows an authorised administrator to proxy into any user's mailbox.)""" + + try: + return self._simple_command('PROXYAUTH', user, **kw) + finally: + self.state_change_pending.release() + + + def rename(self, oldmailbox, newmailbox, **kw): + """(typ, [data]) = rename(oldmailbox, newmailbox) + Rename old mailbox name to new.""" + + return self._simple_command('RENAME', oldmailbox, newmailbox, **kw) + + + def search(self, charset, *criteria, **kw): + """(typ, [data]) = search(charset, criterion, ...) + Search mailbox for matching messages. + 'data' is space separated list of matching message numbers.""" + + name = 'SEARCH' + kw['untagged_response'] = name + if charset: + return self._simple_command(name, 'CHARSET', charset, *criteria, **kw) + return self._simple_command(name, *criteria, **kw) + + + def select(self, mailbox='INBOX', readonly=False, **kw): + """(typ, [data]) = select(mailbox='INBOX', readonly=False) + Select a mailbox. (Restores any previous untagged responses.) + 'data' is count of messages in mailbox ('EXISTS' response). + Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so + other responses should be obtained via "response('FLAGS')" etc.""" + + self.commands_lock.acquire() + # Save state of old mailbox, restore state for new... + self.mailboxes[self.mailbox] = self.untagged_responses + self.untagged_responses = self.mailboxes.setdefault(mailbox, []) + self.commands_lock.release() + + self.mailbox = mailbox + + self.is_readonly = readonly and True or False + if readonly: + name = 'EXAMINE' + else: + name = 'SELECT' + try: + rqb = self._command(name, mailbox) + typ, dat = rqb.get_response('command: %s => %%s' % rqb.name) + if typ != 'OK': + if self.state == SELECTED: + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + if typ == 'BAD': + self._deliver_exc(self.error, '%s command error: %s %s. Data: %.100s' % (name, typ, dat, mailbox), kw) + return self._deliver_dat(typ, dat, kw) + self.state = SELECTED + if __debug__: self._log(1, 'state => SELECTED') + finally: + self.state_change_pending.release() + + if self._get_untagged_response('READ-ONLY', leave=True) and not readonly: + if __debug__: self._dump_ur(1) + self._deliver_exc(self.readonly, '%s is not writable' % mailbox, kw) + typ, dat = self._untagged_response(typ, [None], 'EXISTS') + return self._deliver_dat(typ, dat, kw) + + + def setacl(self, mailbox, who, what, **kw): + """(typ, [data]) = setacl(mailbox, who, what) + Set a mailbox acl.""" + + try: + return self._simple_command('SETACL', mailbox, who, what, **kw) + finally: + self.state_change_pending.release() + + + def setannotation(self, *args, **kw): + """(typ, [data]) = setannotation(mailbox[, entry, attribute]+) + Set ANNOTATIONs.""" + + kw['untagged_response'] = 'ANNOTATION' + return self._simple_command('SETANNOTATION', *args, **kw) + + + def setquota(self, root, limits, **kw): + """(typ, [data]) = setquota(root, limits) + Set the quota root's resource limits.""" + + kw['untagged_response'] = 'QUOTA' + try: + return self._simple_command('SETQUOTA', root, limits, **kw) + finally: + self.state_change_pending.release() + + + def sort(self, sort_criteria, charset, *search_criteria, **kw): + """(typ, [data]) = sort(sort_criteria, charset, search_criteria, ...) + IMAP4rev1 extension SORT command.""" + + name = 'SORT' + if (sort_criteria[0],sort_criteria[-1]) != ('(',')'): + sort_criteria = '(%s)' % sort_criteria + kw['untagged_response'] = name + return self._simple_command(name, sort_criteria, charset, *search_criteria, **kw) + + + def starttls(self, keyfile=None, certfile=None, **kw): + """(typ, [data]) = starttls(keyfile=None, certfile=None) + Start TLS negotiation as per RFC 2595.""" + + name = 'STARTTLS' + + if name not in self.capabilities: + raise self.abort('TLS not supported by server') + + if hasattr(self, '_tls_established') and self._tls_established: + raise self.abort('TLS session already established') + + try: + typ, dat = self._simple_command(name) + finally: + self.state_change_pending.release() + + if typ == 'OK': + import ssl + self.sock = ssl.wrap_socket(self.sock, keyfile, certfile) + self.read_fd = self.sock.fileno() + + typ, dat = self.capability() + if dat == [None]: + raise self.error('no CAPABILITY response from server') + self.capabilities = tuple(dat[-1].upper().split()) + self._tls_established = True + else: + raise self.error("Couldn't establish TLS session: %s" % dat) + + typ, dat = self._untagged_response(typ, dat, name) + return self._deliver_dat(typ, dat, kw) + + + def status(self, mailbox, names, **kw): + """(typ, [data]) = status(mailbox, names) + Request named status conditions for mailbox.""" + + name = 'STATUS' + kw['untagged_response'] = name + return self._simple_command(name, mailbox, names, **kw) + + + def store(self, message_set, command, flags, **kw): + """(typ, [data]) = store(message_set, command, flags) + Alters flag dispositions for messages in mailbox.""" + + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags # Avoid quoting the flags + kw['untagged_response'] = 'FETCH' + return self._simple_command('STORE', message_set, command, flags, **kw) + + + def subscribe(self, mailbox, **kw): + """(typ, [data]) = subscribe(mailbox) + Subscribe to new mailbox.""" + + try: + return self._simple_command('SUBSCRIBE', mailbox, **kw) + finally: + self.state_change_pending.release() + + + def thread(self, threading_algorithm, charset, *search_criteria, **kw): + """(type, [data]) = thread(threading_alogrithm, charset, search_criteria, ...) + IMAPrev1 extension THREAD command.""" + + name = 'THREAD' + kw['untagged_response'] = name + return self._simple_command(name, threading_algorithm, charset, *search_criteria, **kw) + + + def uid(self, command, *args, **kw): + """(typ, [data]) = uid(command, arg, ...) + Execute "command arg ..." with messages identified by UID, + rather than message number. + Assumes 'command' is legal in current state. + Returns response appropriate to 'command'.""" + + command = command.upper() + if command in UID_direct: + resp = command + else: + resp = 'FETCH' + kw['untagged_response'] = resp + return self._simple_command('UID', command, *args, **kw) + + + def unsubscribe(self, mailbox, **kw): + """(typ, [data]) = unsubscribe(mailbox) + Unsubscribe from old mailbox.""" + + try: + return self._simple_command('UNSUBSCRIBE', mailbox, **kw) + finally: + self.state_change_pending.release() + + + def xatom(self, name, *args, **kw): + """(typ, [data]) = xatom(name, arg, ...) + Allow simple extension commands notified by server in CAPABILITY response. + Assumes extension command 'name' is legal in current state. + Returns response appropriate to extension command 'name'.""" + + name = name.upper() + if not name in Commands: + Commands[name] = ((self.state,), False) + try: + return self._simple_command(name, *args, **kw) + finally: + if self.state_change_pending.locked(): + self.state_change_pending.release() + + + + # Internal methods + + + def _append_untagged(self, typ, dat): + + # Append new 'dat' to end of last untagged response if same 'typ', + # else append new response. + + if dat is None: dat = '' + + self.commands_lock.acquire() + + if self.untagged_responses: + urn, urd = self.untagged_responses[-1] + if urn != typ: + urd = None + else: + urd = None + + if urd is None: + urd = [] + self.untagged_responses.append([typ, urd]) + + urd.append(dat) + + self.commands_lock.release() + + if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%s"]' % (typ, len(urd)-1, dat)) + + + def _check_bye(self): + + bye = self._get_untagged_response('BYE', leave=True) + if bye: + raise self.abort(bye[-1]) + + + def _checkquote(self, arg): + + # Must quote command args if "atom-specials" present, + # and not already quoted. + + if not isinstance(arg, basestring): + return arg + if len(arg) >= 2 and (arg[0],arg[-1]) in (('(',')'),('"','"')): + return arg + if arg and self.mustquote_cre.search(arg) is None: + return arg + return self._quote(arg) + + + def _command(self, name, *args, **kw): + + if Commands[name][CMD_VAL_ASYNC]: + cmdtyp = 'async' + else: + cmdtyp = 'sync' + + if __debug__: self._log(1, '[%s] %s %s' % (cmdtyp, name, args)) + + self.state_change_pending.acquire() + + self._end_idle() + + if cmdtyp == 'async': + self.state_change_pending.release() + else: + # Need to wait for all async commands to complete + self._check_bye() + self.commands_lock.acquire() + if self.tagged_commands: + self.state_change_free.clear() + need_event = True + else: + need_event = False + self.commands_lock.release() + if need_event: + if __debug__: self._log(4, 'sync command %s waiting for empty commands Q' % name) + self.state_change_free.wait() + if __debug__: self._log(4, 'sync command %s proceeding' % name) + + if self.state not in Commands[name][CMD_VAL_STATES]: + self.literal = None + raise self.error('command %s illegal in state %s' + % (name, self.state)) + + self._check_bye() + + for typ in ('OK', 'NO', 'BAD'): + self._get_untagged_response(typ) + + if self._get_untagged_response('READ-ONLY', leave=True) and not self.is_readonly: + self.literal = None + raise self.readonly('mailbox status changed to READ-ONLY') + + if self.Terminate: + raise self.abort('connection closed') + + rqb = self._request_push(name=name, **kw) + + data = '%s %s' % (rqb.tag, name) + for arg in args: + if arg is None: continue + data = '%s %s' % (data, self._checkquote(arg)) + + literal = self.literal + if literal is not None: + self.literal = None + if isinstance(literal, basestring): + literator = None + data = '%s {%s}' % (data, len(literal)) + else: + literator = literal + + if __debug__: self._log(4, 'data=%s' % data) + + rqb.data = '%s%s' % (data, CRLF) + + if literal is None: + self.ouq.put(rqb) + return rqb + + # Must setup continuation expectancy *before* ouq.put + crqb = self._request_push(tag='continuation') + + self.ouq.put(rqb) + + while True: + # Wait for continuation response + + ok, data = crqb.get_response('command: %s => %%s' % name) + if __debug__: self._log(3, 'continuation => %s, %s' % (ok, data)) + + # NO/BAD response? + + if not ok: + break + + # Send literal + + if literator is not None: + literal = literator(data, rqb) + + if literal is None: + break + + if literator is not None: + # Need new request for next continuation response + crqb = self._request_push(tag='continuation') + + if __debug__: self._log(4, 'write literal size %s' % len(literal)) + crqb.data = '%s%s' % (literal, CRLF) + self.ouq.put(crqb) + + if literator is None: + break + + return rqb + + + def _command_complete(self, rqb, kw): + + # Called for non-callback commands + + typ, dat = rqb.get_response('command: %s => %%s' % rqb.name) + self._check_bye() + if typ == 'BAD': + if __debug__: self._print_log() + raise self.error('%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) + if 'untagged_response' in kw: + return self._untagged_response(typ, dat, kw['untagged_response']) + return typ, dat + + + def _command_completer(self, (response, cb_arg, error)): + + # Called for callback commands + rqb, kw = cb_arg + rqb.callback = kw['callback'] + rqb.callback_arg = kw.get('cb_arg') + if error is not None: + if __debug__: self._print_log() + typ, val = error + rqb.abort(typ, val) + return + bye = self._get_untagged_response('BYE', leave=True) + if bye: + rqb.abort(self.abort, bye[-1]) + return + typ, dat = response + if typ == 'BAD': + if __debug__: self._print_log() + rqb.abort(self.error, '%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) + return + if 'untagged_response' in kw: + response = self._untagged_response(typ, dat, kw['untagged_response']) + rqb.deliver(response) + + + def _deliver_dat(self, typ, dat, kw): + + if 'callback' in kw: + kw['callback'](((typ, dat), kw.get('cb_arg'), None)) + return typ, dat + + + def _deliver_exc(self, exc, dat, kw): + + if 'callback' in kw: + kw['callback']((None, kw.get('cb_arg'), (exc, dat))) + raise exc(dat) + + + def _end_idle(self): + + irqb = self.idle_rqb + if irqb is None: + return + self.idle_rqb = None + self.idle_timeout = None + irqb.data = 'DONE%s' % CRLF + self.ouq.put(irqb) + if __debug__: self._log(2, 'server IDLE finished') + + + def _get_untagged_response(self, name, leave=False): + + self.commands_lock.acquire() + + for i, (typ, dat) in enumerate(self.untagged_responses): + if typ == name: + if not leave: + del self.untagged_responses[i] + self.commands_lock.release() + if __debug__: self._log(5, '_get_untagged_response(%s) => %s' % (name, dat)) + return dat + + self.commands_lock.release() + return None + + + def _match(self, cre, s): + + # Run compiled regular expression 'cre' match method on 's'. + # Save result, return success. + + self.mo = cre.match(s) + return self.mo is not None + + + def _put_response(self, resp): + + if self._expecting_data > 0: + rlen = len(resp) + dlen = min(self._expecting_data, rlen) + self._expecting_data -= dlen + if rlen <= dlen: + self._accumulated_data.append(resp) + return + self._accumulated_data.append(resp[:dlen]) + resp = resp[dlen:] + + if self._accumulated_data: + typ, dat = self._literal_expected + self._append_untagged(typ, (dat, ''.join(self._accumulated_data))) + self._accumulated_data = [] + + # Protocol mandates all lines terminated by CRLF + resp = resp[:-2] + + if 'continuation' in self.tagged_commands: + continuation_expected = True + else: + continuation_expected = False + + if self._literal_expected is not None: + dat = resp + if self._match(self.literal_cre, dat): + self._literal_expected[1] = dat + self._expecting_data = int(self.mo.group('size')) + if __debug__: self._log(4, 'expecting literal size %s' % self._expecting_data) + return + typ = self._literal_expected[0] + self._literal_expected = None + self._append_untagged(typ, dat) # Tail + if __debug__: self._log(4, 'literal completed') + else: + # Command completion response? + if self._match(self.tagre, resp): + tag = self.mo.group('tag') + typ = self.mo.group('type') + dat = self.mo.group('data') + if not tag in self.tagged_commands: + if __debug__: self._log(1, 'unexpected tagged response: %s' % resp) + else: + self._request_pop(tag, (typ, [dat])) + else: + dat2 = None + + # '*' (untagged) responses? + + if not self._match(self.untagged_response_cre, resp): + if self._match(self.untagged_status_cre, resp): + dat2 = self.mo.group('data2') + + if self.mo is None: + # Only other possibility is '+' (continuation) response... + + if self._match(self.continuation_cre, resp): + if not continuation_expected: + if __debug__: self._log(1, "unexpected continuation response: '%s'" % resp) + return + self._request_pop('continuation', (True, self.mo.group('data'))) + return + + if __debug__: self._log(1, "unexpected response: '%s'" % resp) + return + + typ = self.mo.group('type') + dat = self.mo.group('data') + if dat is None: dat = '' # Null untagged response + if dat2: dat = dat + ' ' + dat2 + + # Is there a literal to come? + + if self._match(self.literal_cre, dat): + self._expecting_data = int(self.mo.group('size')) + if __debug__: self._log(4, 'read literal size %s' % self._expecting_data) + self._literal_expected = [typ, dat] + return + + self._append_untagged(typ, dat) + + if typ != 'OK': + self._end_idle() + + # Bracketed response information? + + if typ in ('OK', 'NO', 'BAD') and self._match(self.response_code_cre, dat): + self._append_untagged(self.mo.group('type'), self.mo.group('data')) + + # Command waiting for aborted continuation response? + + if continuation_expected: + self._request_pop('continuation', (False, resp)) + + # Bad news? + + if typ in ('NO', 'BAD', 'BYE'): + if typ == 'BYE': + self.Terminate = True + if __debug__: self._log(1, '%s response: %s' % (typ, dat)) + + + def _quote(self, arg): + + return '"%s"' % arg.replace('\\', '\\\\').replace('"', '\\"') + + + def _request_pop(self, name, data): + + if __debug__: self._log(4, '_request_pop(%s, %s)' % (name, data)) + self.commands_lock.acquire() + rqb = self.tagged_commands.pop(name) + if not self.tagged_commands: + self.state_change_free.set() + self.commands_lock.release() + rqb.deliver(data) + + + def _request_push(self, tag=None, name=None, **kw): + + self.commands_lock.acquire() + rqb = Request(self, name=name, **kw) + if tag is None: + tag = rqb.tag + self.tagged_commands[tag] = rqb + self.commands_lock.release() + if __debug__: self._log(4, '_request_push(%s, %s, %s)' % (tag, name, `kw`)) + return rqb + + + def _simple_command(self, name, *args, **kw): + + if 'callback' in kw: + rqb = self._command(name, callback=self._command_completer, *args) + rqb.callback_arg = (rqb, kw) + return (None, None) + return self._command_complete(self._command(name, *args), kw) + + + def _untagged_response(self, typ, dat, name): + + if typ == 'NO': + return typ, dat + data = self._get_untagged_response(name) + if not data: + return typ, [None] + return typ, data + + + + # Threads + + + def _close_threads(self): + + if __debug__: self._log(1, '_close_threads') + + self.ouq.put(None) + self.wrth.join() + + if __debug__: self._log(1, 'call shutdown') + + self.shutdown() + + self.rdth.join() + self.inth.join() + + + def _handler(self): + + resp_timeout = self.resp_timeout + + threading.currentThread().setName(self.identifier + 'handler') + + time.sleep(0.1) # Don't start handling before main thread ready + + if __debug__: self._log(1, 'starting') + + typ, val = self.abort, 'connection terminated' + + while not self.Terminate: + try: + if self.idle_timeout is not None: + timeout = self.idle_timeout - time.time() + if timeout <= 0: + timeout = 1 + if __debug__: + if self.idle_rqb is not None: + self._log(5, 'server IDLING, timeout=%.2f' % timeout) + else: + timeout = resp_timeout + line = self.inq.get(True, timeout) + except Queue.Empty: + if self.idle_rqb is None: + if resp_timeout is not None and self.tagged_commands: + if __debug__: self._log(1, 'response timeout') + typ, val = self.abort, 'no response after %s secs' % resp_timeout + break + continue + if self.idle_timeout > time.time(): + continue + if __debug__: self._log(2, 'server IDLE timedout') + line = IDLE_TIMEOUT_RESPONSE + + if line is None: + if __debug__: self._log(1, 'inq None - terminating') + break + + if not isinstance(line, basestring): + typ, val = line + break + + try: + self._put_response(line) + except: + typ, val = self.error, 'program error: %s - %s' % sys.exc_info()[:2] + break + + self.Terminate = True + + if __debug__: self._log(1, 'terminating: %s' % `val`) + + while not self.ouq.empty(): + try: + self.ouq.get_nowait().abort(typ, val) + except Queue.Empty: + break + self.ouq.put(None) + + self.commands_lock.acquire() + for name in self.tagged_commands.keys(): + rqb = self.tagged_commands.pop(name) + rqb.abort(typ, val) + self.state_change_free.set() + self.commands_lock.release() + + if __debug__: self._log(1, 'finished') + + + if hasattr(select_module, "poll"): + + def _reader(self): + + threading.currentThread().setName(self.identifier + 'reader') + + if __debug__: self._log(1, 'starting using poll') + + def poll_error(state): + PollErrors = { + select.POLLERR: 'Error', + select.POLLHUP: 'Hang up', + select.POLLNVAL: 'Invalid request: descriptor not open', + } + return ' '.join([PollErrors[s] for s in PollErrors.keys() if (s & state)]) + + line_part = '' + + poll = select.poll() + + poll.register(self.read_fd, select.POLLIN) + + rxzero = 0 + read_poll_timeout = self.read_poll_timeout * 1000 # poll() timeout is in millisecs + + while not self.Terminate: + if self.state == LOGOUT: + timeout = 1 + else: + timeout = read_poll_timeout + try: + r = poll.poll(timeout) + if __debug__: self._log(5, 'poll => %s' % `r`) + if not r: + continue # Timeout + + fd,state = r[0] + + if state & select.POLLIN: + data = self.read(32768) # Drain ssl buffer if present + start = 0 + dlen = len(data) + if __debug__: self._log(5, 'rcvd %s' % dlen) + if dlen == 0: + rxzero += 1 + if rxzero > 5: + raise IOError("Too many read 0") + time.sleep(0.1) + else: + rxzero = 0 + while True: + stop = data.find('\n', start) + if stop < 0: + line_part += data[start:] + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop] + if __debug__: self._log(4, '< %s' % line) + self.inq.put(line) + + if state & ~(select.POLLIN): + raise IOError(poll_error(state)) + except: + reason = 'socket error: %s - %s' % sys.exc_info()[:2] + if __debug__: + if not self.Terminate: + self._print_log() + if self.debug: self.debug += 4 # Output all + self._log(1, reason) + self.inq.put((self.abort, reason)) + break + + poll.unregister(self.read_fd) + + if __debug__: self._log(1, 'finished') + + else: + + # No "poll" - use select() + + def _reader(self): + + threading.currentThread().setName(self.identifier + 'reader') + + if __debug__: self._log(1, 'starting using select') + + line_part = '' + + rxzero = 0 + read_poll_timeout = self.read_poll_timeout + + while not self.Terminate: + if self.state == LOGOUT: + timeout = 1 + else: + timeout = read_poll_timeout + try: + r,w,e = select.select([self.read_fd], [], [], timeout) + if __debug__: self._log(5, 'select => %s, %s, %s' % (r,w,e)) + if not r: # Timeout + continue + + data = self.read(32768) # Drain ssl buffer if present + start = 0 + dlen = len(data) + if __debug__: self._log(5, 'rcvd %s' % dlen) + if dlen == 0: + rxzero += 1 + if rxzero > 5: + raise IOError("Too many read 0") + time.sleep(0.1) + else: + rxzero = 0 + while True: + stop = data.find('\n', start) + if stop < 0: + line_part += data[start:] + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop] + if __debug__: self._log(4, '< %s' % line) + self.inq.put(line) + except: + reason = 'socket error: %s - %s' % sys.exc_info()[:2] + if __debug__: + if not self.Terminate: + self._print_log() + if self.debug: self.debug += 4 # Output all + self._log(1, reason) + self.inq.put((self.abort, reason)) + break + + if __debug__: self._log(1, 'finished') + + + def _writer(self): + + threading.currentThread().setName(self.identifier + 'writer') + + if __debug__: self._log(1, 'starting') + + reason = 'Terminated' + + while not self.Terminate: + rqb = self.ouq.get() + if rqb is None: + break # Outq flushed + + try: + self.send(rqb.data) + if __debug__: self._log(4, '> %s' % rqb.data) + except: + reason = 'socket error: %s - %s' % sys.exc_info()[:2] + if __debug__: + if not self.Terminate: + self._print_log() + if self.debug: self.debug += 4 # Output all + self._log(1, reason) + rqb.abort(self.abort, reason) + break + + self.inq.put((self.abort, reason)) + + if __debug__: self._log(1, 'finished') + + + + # Debugging + + + if __debug__: + + def _init_debug(self, debug=None, debug_file=None): + self.debug = debug is not None and debug or Debug is not None and Debug or 0 + self.debug_file = debug_file is not None and debug_file or sys.stderr + + self.debug_lock = threading.Lock() + self._cmd_log_len = 20 + self._cmd_log_idx = 0 + self._cmd_log = {} # Last `_cmd_log_len' interactions + if self.debug: + self._mesg('imaplib2 version %s' % __version__) + self._mesg('imaplib2 debug level %s' % self.debug) + + + def _dump_ur(self, lvl): + if lvl > self.debug: + return + + l = self.untagged_responses + if not l: + return + + t = '\n\t\t' + l = map(lambda x:'%s: "%s"' % (x[0], x[1][0] and '" "'.join(x[1]) or ''), l) + self.debug_lock.acquire() + self._mesg('untagged responses dump:%s%s' % (t, t.join(l))) + self.debug_lock.release() + + + def _log(self, lvl, line): + if lvl > self.debug: + return + + if line[-2:] == CRLF: + line = line[:-2] + '\\r\\n' + + tn = threading.currentThread().getName() + + if lvl == 1 or self.debug >= 4: + self.debug_lock.acquire() + self._mesg(line, tn) + self.debug_lock.release() + return + + # Keep log of last `_cmd_log_len' interactions for debugging. + self.debug_lock.acquire() + self._cmd_log[self._cmd_log_idx] = (line, tn, time.time()) + self._cmd_log_idx += 1 + if self._cmd_log_idx >= self._cmd_log_len: + self._cmd_log_idx = 0 + self.debug_lock.release() + + + def _mesg(self, s, tn=None, secs=None): + if secs is None: + secs = time.time() + if tn is None: + tn = threading.currentThread().getName() + tm = time.strftime('%M:%S', time.localtime(secs)) + self.debug_file.write(' %s.%02d %s %s\n' % (tm, (secs*100)%100, tn, s)) + self.debug_file.flush() + + + def _print_log(self): + self.debug_lock.acquire() + i, n = self._cmd_log_idx, self._cmd_log_len + if n: self._mesg('last %d log messages:' % n) + while n: + try: + self._mesg(*self._cmd_log[i]) + except: + pass + i += 1 + if i >= self._cmd_log_len: + i = 0 + n -= 1 + self.debug_lock.release() + + + +class IMAP4_SSL(IMAP4): + + """IMAP4 client class over SSL connection + + Instantiate with: + IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, debug=None, debug_file=None, identifier=None, timeout=None) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 SSL port); + keyfile - PEM formatted file that contains your private key (default: None); + certfile - PEM formatted certificate chain file (default: None); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response. + + For more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, host=None, port=None, keyfile=None, certfile=None, debug=None, debug_file=None, identifier=None, timeout=None): + self.keyfile = keyfile + self.certfile = certfile + IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout) + + + def open(self, host=None, port=None): + """open(host=None, port=None) + Setup secure connection to remote server on "host:port" + (default: localhost:standard IMAP4 SSL port). + This connection will be used by the routines: + read, send, shutdown, socket, ssl.""" + + self.host = host is not None and host or '' + self.port = port is not None and port or IMAP4_SSL_PORT + self.sock = self.open_socket() + + try: + import ssl + self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile) + except ImportError: + self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile) + + self.read_fd = self.sock.fileno() + + + def read(self, size): + """data = read(size) + Read at most 'size' bytes from remote.""" + + if self.decompressor is None: + return self.sslobj.read(size) + + if self.decompressor.unconsumed_tail: + data = self.decompressor.unconsumed_tail + else: + data = self.sslobj.read(8192) + + return self.decompressor.decompress(data, size) + + + def send(self, data): + """send(data) + Send 'data' to remote.""" + + if self.compressor is not None: + data = self.compressor.compress(data) + data += self.compressor.flush(zlib.Z_SYNC_FLUSH) + + # NB: socket.ssl needs a "sendall" method to match socket objects. + bytes = len(data) + while bytes > 0: + sent = self.sslobj.write(data) + if sent == bytes: + break # avoid copy + data = data[sent:] + bytes = bytes - sent + + + def ssl(self): + """ssl = ssl() + Return socket.ssl instance used to communicate with the IMAP4 server.""" + + return self.sslobj + + + +class IMAP4_stream(IMAP4): + + """IMAP4 client class over a stream + + Instantiate with: + IMAP4_stream(command, debug=None, debug_file=None, identifier=None, timeout=None) + + command - string that can be passed to subprocess.Popen(); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response. + + For more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, command, debug=None, debug_file=None, identifier=None, timeout=None): + self.command = command + self.host = command + self.port = None + self.sock = None + self.writefile, self.readfile = None, None + self.read_fd = None + IMAP4.__init__(self, None, None, debug, debug_file, identifier, timeout) + + + def open(self, host=None, port=None): + """open(host=None, port=None) + Setup a stream connection via 'self.command'. + This connection will be used by the routines: + read, send, shutdown, socket.""" + + from subprocess import Popen, PIPE + + self._P = Popen(self.command, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True) + self.writefile, self.readfile = self._P.stdin, self._P.stdout + self.read_fd = self.readfile.fileno() + + + def read(self, size): + """Read 'size' bytes from remote.""" + + if self.decompressor is None: + return os.read(self.read_fd, size) + + if self.decompressor.unconsumed_tail: + data = self.decompressor.unconsumed_tail + else: + data = os.read(self.read_fd, 8192) + + return self.decompressor.decompress(data, size) + + + def send(self, data): + """Send data to remote.""" + + if self.compressor is not None: + data = self.compressor.compress(data) + data += self.compressor.flush(zlib.Z_SYNC_FLUSH) + + self.writefile.write(data) + self.writefile.flush() + + + def shutdown(self): + """Close I/O established in "open".""" + + self.readfile.close() + self.writefile.close() + + +class _Authenticator(object): + + """Private class to provide en/de-coding + for base64 authentication conversation.""" + + def __init__(self, mechinst): + self.mech = mechinst # Callable object to provide/process data + + def process(self, data, rqb): + ret = self.mech(self.decode(data)) + if ret is None: + return '*' # Abort conversation + return self.encode(ret) + + def encode(self, inp): + # + # Invoke binascii.b2a_base64 iteratively with + # short even length buffers, strip the trailing + # line feed from the result and append. "Even" + # means a number that factors to both 6 and 8, + # so when it gets to the end of the 8-bit input + # there's no partial 6-bit output. + # + oup = '' + while inp: + if len(inp) > 48: + t = inp[:48] + inp = inp[48:] + else: + t = inp + inp = '' + e = binascii.b2a_base64(t) + if e: + oup = oup + e[:-1] + return oup + + def decode(self, inp): + if not inp: + return '' + return binascii.a2b_base64(inp) + + + + +class _IdleCont(object): + + """When process is called, server is in IDLE state + and will send asynchronous changes.""" + + def __init__(self, parent, timeout): + self.parent = parent + self.timeout = timeout is not None and timeout or IDLE_TIMEOUT + self.parent.idle_timeout = self.timeout + time.time() + + def process(self, data, rqb): + self.parent.idle_rqb = rqb + self.parent.idle_timeout = self.timeout + time.time() + if __debug__: self.parent._log(2, 'server IDLE started, timeout in %.2f secs' % self.timeout) + return None + + + +Mon2num = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, + 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} + +InternalDate = re.compile(r'.*INTERNALDATE "' + r'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' + r' (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])' + r' (?P[-+])(?P[0-9][0-9])(?P[0-9][0-9])' + r'"') + + +def Internaldate2Time(resp): + + """time_tuple = Internaldate2Time(resp) + Convert IMAP4 INTERNALDATE to UT.""" + + mo = InternalDate.match(resp) + if not mo: + return None + + mon = Mon2num[mo.group('mon')] + zonen = mo.group('zonen') + + day = int(mo.group('day')) + year = int(mo.group('year')) + hour = int(mo.group('hour')) + min = int(mo.group('min')) + sec = int(mo.group('sec')) + zoneh = int(mo.group('zoneh')) + zonem = int(mo.group('zonem')) + + # INTERNALDATE timezone must be subtracted to get UT + + zone = (zoneh*60 + zonem)*60 + if zonen == '-': + zone = -zone + + tt = (year, mon, day, hour, min, sec, -1, -1, -1) + + utc = time.mktime(tt) + + # Following is necessary because the time module has no 'mkgmtime'. + # 'mktime' assumes arg in local timezone, so adds timezone/altzone. + + lt = time.localtime(utc) + if time.daylight and lt[-1]: + zone = zone + time.altzone + else: + zone = zone + time.timezone + + return time.localtime(utc - zone) + +Internaldate2tuple = Internaldate2Time # (Backward compatible) + + + +def Time2Internaldate(date_time): + + """'"DD-Mmm-YYYY HH:MM:SS +HHMM"' = Time2Internaldate(date_time) + Convert 'date_time' to IMAP4 INTERNALDATE representation.""" + + if isinstance(date_time, (int, float)): + tt = time.localtime(date_time) + elif isinstance(date_time, (tuple, time.struct_time)): + tt = date_time + elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'): + return date_time # Assume in correct format + else: + raise ValueError("date_time not of a known type") + + dt = time.strftime("%d-%b-%Y %H:%M:%S", tt) + if dt[0] == '0': + dt = ' ' + dt[1:] + if time.daylight and tt[-1]: + zone = -time.altzone + else: + zone = -time.timezone + return '"' + dt + " %+03d%02d" % divmod(zone//60, 60) + '"' + + + +FLAGS_cre = re.compile(r'.*FLAGS \((?P[^\)]*)\)') + +def ParseFlags(resp): + + """('flag', ...) = ParseFlags(line) + Convert IMAP4 flags response to python tuple.""" + + mo = FLAGS_cre.match(resp) + if not mo: + return () + + return tuple(mo.group('flags').split()) + + + +if __name__ == '__main__': + + # To test: invoke either as 'python imaplib2.py [IMAP4_server_hostname]', + # or as 'python imaplib2.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' + # or as 'python imaplib2.py -l "keyfile[:certfile]" [IMAP4_SSL_server_hostname]' + + import getopt, getpass + + try: + optlist, args = getopt.getopt(sys.argv[1:], 'd:l:s:p:') + except getopt.error, val: + optlist, args = (), () + + debug, port, stream_command, keyfile, certfile = (None,)*5 + for opt,val in optlist: + if opt == '-d': + debug = int(val) + elif opt == '-l': + try: + keyfile,certfile = val.split(':') + except ValueError: + keyfile,certfile = val,val + elif opt == '-p': + port = int(val) + elif opt == '-s': + stream_command = val + if not args: args = (stream_command,) + + if not args: args = ('',) + if not port: port = (keyfile is not None) and IMAP4_SSL_PORT or IMAP4_PORT + + host = args[0] + + USER = getpass.getuser() + + data = open(os.path.exists("test.data") and "test.data" or __file__).read(1000) + test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)s%(data)s' \ + % {'user':USER, 'lf':'\n', 'data':data} + test_seq1 = [ + ('list', ('""', '%')), + ('create', ('/tmp/imaplib2_test.0',)), + ('rename', ('/tmp/imaplib2_test.0', '/tmp/imaplib2_test.1')), + ('CREATE', ('/tmp/imaplib2_test.2',)), + ('append', ('/tmp/imaplib2_test.2', None, None, test_mesg)), + ('list', ('/tmp', 'imaplib2_test*')), + ('select', ('/tmp/imaplib2_test.2',)), + ('search', (None, 'SUBJECT', 'IMAP4 test')), + ('fetch', ('1', '(FLAGS INTERNALDATE RFC822)')), + ('store', ('1', 'FLAGS', '(\Deleted)')), + ('namespace', ()), + ('expunge', ()), + ('recent', ()), + ('close', ()), + ] + + test_seq2 = ( + ('select', ()), + ('response',('UIDVALIDITY',)), + ('response', ('EXISTS',)), + ('append', (None, None, None, test_mesg)), + ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')), + ('uid', ('SEARCH', 'ALL')), + ('uid', ('THREAD', 'references', 'UTF-8', '(SEEN)')), + ('recent', ()), + ) + + AsyncError = None + + def responder((response, cb_arg, error)): + global AsyncError + cmd, args = cb_arg + if error is not None: + AsyncError = error + M._mesg('[cb] ERROR %s %.100s => %s' % (cmd, args, error)) + return + typ, dat = response + M._mesg('[cb] %s %.100s => %s %.100s' % (cmd, args, typ, dat)) + if typ == 'NO': + AsyncError = (Exception, dat[0]) + + def run(cmd, args, cb=True): + if AsyncError: + M.logout() + typ, val = AsyncError + raise typ(val) + M._mesg('%s %.100s' % (cmd, args)) + try: + if cb: + typ, dat = getattr(M, cmd)(callback=responder, cb_arg=(cmd, args), *args) + if M.debug: + M._mesg('%s %.100s => %s %.100s' % (cmd, args, typ, dat)) + else: + typ, dat = getattr(M, cmd)(*args) + M._mesg('%s %.100s => %s %.100s' % (cmd, args, typ, dat)) + except: + M.logout() + raise + if typ == 'NO': + M.logout() + raise Exception(dat[0]) + return dat + + try: + threading.currentThread().setName('main') + + if keyfile is not None: + if not keyfile: keyfile = None + if not certfile: certfile = None + M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, debug=debug, identifier='', timeout=10) + elif stream_command: + M = IMAP4_stream(stream_command, debug=debug, identifier='', timeout=10) + else: + M = IMAP4(host=host, port=port, debug=debug, identifier='', timeout=10) + if M.state != 'AUTH': # Login needed + PASSWD = getpass.getpass("IMAP password for %s on %s: " % (USER, host or "localhost")) + test_seq1.insert(0, ('login', (USER, PASSWD))) + M._mesg('PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) + if 'COMPRESS=DEFLATE' in M.capabilities: + M.enable_compression() + + for cmd,args in test_seq1: + run(cmd, args) + + for ml in run('list', ('/tmp/', 'imaplib2_test%'), cb=False): + mo = re.match(r'.*"([^"]+)"$', ml) + if mo: path = mo.group(1) + else: path = ml.split()[-1] + run('delete', (path,)) + + for cmd,args in test_seq2: + if (cmd,args) != ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')): + run(cmd, args) + continue + + dat = run(cmd, args, cb=False) + uid = dat[-1].split() + if not uid: continue + run('uid', ('FETCH', uid[-1], + '(FLAGS INTERNALDATE RFC822.SIZE RFC822.HEADER RFC822.TEXT)')) + run('uid', ('STORE', uid[-1], 'FLAGS', '(\Deleted)')) + run('expunge', ()) + + if 'IDLE' in M.capabilities: + run('idle', (2,), cb=False) + run('idle', (99,), cb=True) # Asynchronous, to test interruption of 'idle' by 'noop' + time.sleep(1) + run('noop', (), cb=False) + + run('logout', (), cb=False) + + if debug: + M._mesg('') + M._print_log() + M._mesg('') + M._mesg('unused untagged responses in order, most recent last:') + for typ,dat in M.pop_untagged_responses(): M._mesg('\t%s %s' % (typ, dat)) + + print 'All tests OK.' + + except: + print 'Tests failed.' + + if not debug: + print ''' +If you would like to see debugging output, +try: %s -d5 +''' % sys.argv[0] + + raise From f9413226b8aa6413bbc6a4d586be8d31b575089d Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Tue, 8 Mar 2011 10:05:16 -0500 Subject: [PATCH 035/817] Import imaplib2 instead of imaplib imaplib2 has slightly different semantics than standard imaplib, so this patch will break the build, but I thought it was helpful to have it as a separate commit. Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 1 - offlineimap/imaplibutil.py | 3 ++- offlineimap/imapserver.py | 2 +- offlineimap/init.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8cf57d4..8fd1b88 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -16,7 +16,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import imaplib import email import random import binascii diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index b03e80b..46c207d 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -17,9 +17,10 @@ import re, socket, time, subprocess from offlineimap.ui import getglobalui +from offlineimap.imaplib2 import * # Import the symbols we need that aren't exported by default -from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num, IMAP4, IMAP4_SSL +from offlineimap.imaplib2 import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num try: import ssl diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 7ead9a4..4e01a79 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import imaplib +from offlineimap import imaplib2 as imaplib from offlineimap import imaplibutil, imaputil, threadutil from offlineimap.ui import getglobalui from threading import * diff --git a/offlineimap/init.py b/offlineimap/init.py index 7391727..96d4e50 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -19,7 +19,7 @@ import os import sys import threading -import imaplib +import offlineimap.imaplib2 as imaplib import signal import socket import logging From 1bf4bee5e630521c50e79081cea82195826a36bb Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Thu, 10 Mar 2011 15:36:20 -0500 Subject: [PATCH 036/817] Update to match semantics of new imaplib2 The biggest change here is that imapobj.untagged_responses is no longer a dictionary, but a list. To access it, I use the semi-private _get_untagged_response method. * offlineimap/folder/IMAP.py (IMAPFolder.quickchanged, IMAPFolder.cachemessagelist): imaplib2 now explicitly removes its EXISTS response on select(), so instead we use the return values from select() to get the number of messages. * offlineimap/imapserver.py (UsefulIMAPMixIn.select): imaplib2 now stores untagged_responses for different mailboxes, which confuses us because it seems like our mailboxes are "still" in read-only mode when we just re-opened them. Additionally, we have to return the value from imaplib2's select() so that the above thing works. * offlineimap/imapserver.py (UsefulIMAPMixIn._mesg): imaplib2 now calls _mesg with the name of a thread, so we display this information in debug output. This requires a corresponding change to imaplibutil.new_mesg. * offlineimap/imaplibutil.py: We override IMAP4_SSL.open, whose default arguments have changed, so update the default arguments. We also subclass imaplib.IMAP4 in a few different places, which now relies on having a read_fd file descriptor to poll on. Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 53 ++++++++++++++++++++------------------ offlineimap/imaplibutil.py | 17 +++++++++--- offlineimap/imapserver.py | 7 +++-- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8fd1b88..89848b2 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -71,7 +71,7 @@ class IMAPFolder(BaseFolder): try: # Primes untagged_responses self.selectro(imapobj) - return long(imapobj.untagged_responses['UIDVALIDITY'][0]) + return long(imapobj._get_untagged_response('UIDVALIDITY', True)[0]) finally: self.imapserver.releaseconnection(imapobj) @@ -82,17 +82,16 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: # Primes untagged_responses - imapobj.select(self.getfullname(), readonly = 1, force = 1) - try: - # 1. Some mail servers do not return an EXISTS response - # if the folder is empty. 2. ZIMBRA servers can return - # multiple EXISTS replies in the form 500, 1000, 1500, - # 1623 so check for potentially multiple replies. - maxmsgid = 0 - for msgid in imapobj.untagged_responses['EXISTS']: - maxmsgid = max(long(msgid), maxmsgid) - except KeyError: + imaptype, imapdata = imapobj.select(self.getfullname(), readonly = 1, force = 1) + # 1. Some mail servers do not return an EXISTS response + # if the folder is empty. 2. ZIMBRA servers can return + # multiple EXISTS replies in the form 500, 1000, 1500, + # 1623 so check for potentially multiple replies. + if imapdata == [None]: return True + maxmsgid = 0 + for msgid in imapdata: + maxmsgid = max(long(msgid), maxmsgid) # Different number of messages than last time? if maxmsgid != len(statusfolder.getmessagelist()): @@ -127,7 +126,7 @@ class IMAPFolder(BaseFolder): try: # Primes untagged_responses - imapobj.select(self.getfullname(), readonly = 1, force = 1) + imaptype, imapdata = imapobj.select(self.getfullname(), readonly = 1, force = 1) maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1) maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1) @@ -169,17 +168,20 @@ class IMAPFolder(BaseFolder): # No messages; return return else: - try: - # 1. Some mail servers do not return an EXISTS response - # if the folder is empty. 2. ZIMBRA servers can return - # multiple EXISTS replies in the form 500, 1000, 1500, - # 1623 so check for potentially multiple replies. - maxmsgid = 0 - for msgid in imapobj.untagged_responses['EXISTS']: - maxmsgid = max(long(msgid), maxmsgid) - messagesToFetch = '1:%d' % maxmsgid; - except KeyError: + # 1. Some mail servers do not return an EXISTS response + # if the folder is empty. 2. ZIMBRA servers can return + # multiple EXISTS replies in the form 500, 1000, 1500, + # 1623 so check for potentially multiple replies. + if imapdata == [None]: return + + maxmsgid = 0 + for msgid in imapdata: + maxmsgid = max(long(msgid), maxmsgid) + + maxmsgid = long(imapdata[0]) + messagesToFetch = '1:%d' % maxmsgid; + if maxmsgid < 1: #no messages; return return @@ -437,10 +439,11 @@ class IMAPFolder(BaseFolder): # get the new UID from the APPENDUID response, it could look like # OK [APPENDUID 38505 3955] APPEND completed # with 38505 bein folder UIDvalidity and 3955 the new UID - if not imapobj.untagged_responses.has_key('APPENDUID'): - self.ui.warn("Server supports UIDPLUS but got no APPENDUID appending a message.") + if not imapobj._get_untagged_response('APPENDUID', True): + self.ui.warn("Server supports UIDPLUS but got no APPENDUID " + "appending a message.") return 0 - uid = long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1]) + uid = long(imapobj._get_untagged_response('APPENDUID', True)[-1].split(' ')[1]) else: # we don't support UIDPLUS diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 46c207d..f902fce 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -17,6 +17,7 @@ import re, socket, time, subprocess from offlineimap.ui import getglobalui +import threading from offlineimap.imaplib2 import * # Import the symbols we need that aren't exported by default @@ -44,6 +45,8 @@ class IMAP4_Tunnel(IMAP4): self.process = subprocess.Popen(host, shell=True, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) (self.outfd, self.infd) = (self.process.stdin, self.process.stdout) + # imaplib2 polls on this fd + self.read_fd = self.infd.fileno() def read(self, size): retval = '' @@ -66,11 +69,13 @@ class IMAP4_Tunnel(IMAP4): self.process.wait() -def new_mesg(self, s, secs=None): +def new_mesg(self, s, tn=None, secs=None): if secs is None: secs = time.time() + if tn is None: + tn = threading.currentThread().getName() tm = time.strftime('%M:%S', time.localtime(secs)) - getglobalui().debug('imap', ' %s.%02d %s' % (tm, (secs*100)%100, s)) + getglobalui().debug('imap', ' %s.%02d %s %s' % (tm, (secs*100)%100, tn, s)) class WrappedIMAP4_SSL(IMAP4_SSL): """Provides an improved version of the standard IMAP4_SSL @@ -85,7 +90,7 @@ class WrappedIMAP4_SSL(IMAP4_SSL): del kwargs['cacertfile'] IMAP4_SSL.__init__(self, *args, **kwargs) - def open(self, host = '', port = IMAP4_SSL_PORT): + def open(self, host=None, port=None): """Do whatever IMAP4_SSL would do in open, but call sslwrap with cert verification""" #IMAP4_SSL.open(self, host, port) uses the below 2 lines: @@ -148,6 +153,9 @@ class WrappedIMAP4_SSL(IMAP4_SSL): if error: raise ssl.SSLError("SSL Certificate host name mismatch: %s" % error) + # imaplib2 uses this to poll() + self.read_fd = self.sock.fileno() + #TODO: Done for now. We should implement a mutt-like behavior #that offers the users to accept a certificate (presenting a #fingerprint of it) (get via self.sslobj.getpeercert()), and @@ -263,6 +271,9 @@ class WrappedIMAP4(IMAP4): raise socket.error(last_error) self.file = self.sock.makefile('rb') + # imaplib2 uses this to poll() + self.read_fd = self.sock.fileno() + mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]") def Internaldate2epoch(resp): diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 4e01a79..93fc4db 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -48,6 +48,8 @@ class UsefulIMAPMixIn: and self.is_readonly == readonly: # No change; return. return + # Wipe out all old responses, to maintain semantics with old imaplib2 + del self.untagged_responses[:] result = self.__class__.__bases__[1].select(self, mailbox, readonly) if result[0] != 'OK': raise ValueError, "Error from select: %s" % str(result) @@ -55,9 +57,10 @@ class UsefulIMAPMixIn: self.selectedfolder = mailbox else: self.selectedfolder = None + return result - def _mesg(self, s, secs=None): - imaplibutil.new_mesg(self, s, secs) + def _mesg(self, s, tn=None, secs=None): + imaplibutil.new_mesg(self, s, tn, secs) class UsefulIMAP4(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): # This is a hack around Darwin's implementation of realloc() (which From 197030da1aad3a4d5b0ae343fcf0feeced077e5d Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Tue, 8 Mar 2011 10:05:18 -0500 Subject: [PATCH 037/817] Remove obsolete read(), readline(), _read_upto() methods For read(), the imaplib2 version seems to work perfectly well. The others aren't used any more, either by imaplib2, nor by us, so we may as well get rid of them. Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 47 -------------------------------------- 1 file changed, 47 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index f902fce..3bd33ea 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -194,53 +194,6 @@ class WrappedIMAP4_SSL(IMAP4_SSL): return ('no matching domain name found in certificate') - def _read_upto (self, n): - """Read up to n bytes, emptying existing _readbuffer first""" - bytesfrombuf = min(n, len(self._readbuf)) - if bytesfrombuf: - # Return the stuff in readbuf, even if less than n. - # It might contain the rest of the line, and if we try to - # read more, might block waiting for data that is not - # coming to arrive. - retval = self._readbuf[:bytesfrombuf] - self._readbuf = self._readbuf[bytesfrombuf:] - return retval - return self.sslobj.read(min(n, 16384)) - - def read(self, n): - """Read exactly n bytes - - As done in IMAP4_SSL.read() API. If read returns less than n - bytes, things break left and right.""" - chunks = [] - read = 0 - while read < n: - data = self._read_upto (n-read) - if not data: - break - read += len(data) - chunks.append(data) - - return ''.join(chunks) - - def readline(self): - """Get the next line. This implementation is more efficient - than IMAP4_SSL.readline() which reads one char at a time and - reassembles the string by appending those chars. Uggh.""" - retval = '' - while 1: - linebuf = self._read_upto(1024) - if not linebuf: - return retval - nlindex = linebuf.find("\n") - if nlindex != -1: - retval += linebuf[:nlindex + 1] - self._readbuf = linebuf[nlindex + 1:] + self._readbuf - return retval - else: - retval += linebuf - - class WrappedIMAP4(IMAP4): """Improved version of imaplib.IMAP4 that can also connect to IPv6""" From a139d9deedad16eb621b7d552260bc3066cb4bd0 Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Tue, 8 Mar 2011 10:05:24 -0500 Subject: [PATCH 038/817] Throw away broken connections Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 93fc4db..610b6a5 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -154,7 +154,11 @@ class IMAPServer: """Releases a connection, returning it to the pool.""" self.connectionlock.acquire() self.assignedconnections.remove(connection) - self.availableconnections.append(connection) + # Don't reuse broken connections + if connection.Terminate: + connection.logout() + else: + self.availableconnections.append(connection) self.connectionlock.release() self.semaphore.release() From 5ea95002f50492db9fda00ec5ea7e25994828baf Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Tue, 8 Mar 2011 10:05:25 -0500 Subject: [PATCH 039/817] Create new connections with a timeout imaplib2 does not use socket, so does not know about the defaulttimeout we set based on the config. Instead, we explicitly pass the default timeout. Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 610b6a5..c2e2d50 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -20,7 +20,7 @@ from offlineimap import imaplib2 as imaplib from offlineimap import imaplibutil, imaputil, threadutil from offlineimap.ui import getglobalui from threading import * -import thread, hmac, os, time +import thread, hmac, os, time, socket import base64 from StringIO import StringIO @@ -243,16 +243,18 @@ class IMAPServer: # Generate a new connection. if self.tunnel: self.ui.connecting('tunnel', self.tunnel) - imapobj = UsefulIMAP4_Tunnel(self.tunnel) + imapobj = UsefulIMAP4_Tunnel(self.tunnel, timeout=socket.getdefaulttimeout()) success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) imapobj = UsefulIMAP4_SSL(self.hostname, self.port, - self.sslclientkey, self.sslclientcert, + self.sslclientkey, self.sslclientcert, + timeout=socket.getdefaulttimeout(), cacertfile = self.sslcacertfile) else: self.ui.connecting(self.hostname, self.port) - imapobj = UsefulIMAP4(self.hostname, self.port) + imapobj = UsefulIMAP4(self.hostname, self.port, + timeout=socket.getdefaulttimeout()) imapobj.mustquote = imaplibutil.mustquote From 44eefae0433e5685090cd05486989d2a1bee63d2 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 11 Mar 2011 22:13:21 +0100 Subject: [PATCH 040/817] cleanup import statements and conform to PEP-8 Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 6 +++--- offlineimap/folder/Base.py | 1 - offlineimap/folder/LocalStatus.py | 3 ++- offlineimap/folder/Maildir.py | 6 ++++-- offlineimap/folder/UIDMaps.py | 4 +--- offlineimap/imaplibutil.py | 5 ++++- offlineimap/imapserver.py | 5 ++++- offlineimap/imaputil.py | 4 +++- offlineimap/repository/Gmail.py | 3 +-- offlineimap/repository/LocalStatus.py | 3 ++- offlineimap/repository/Maildir.py | 3 +-- offlineimap/syncmaster.py | 2 +- offlineimap/threadutil.py | 4 +++- offlineimap/ui/Noninteractive.py | 3 ++- offlineimap/ui/TTY.py | 2 +- 15 files changed, 32 insertions(+), 22 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 7ffdfc1..e806151 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -18,11 +18,11 @@ from offlineimap import threadutil, mbnames, CustomConfig from offlineimap.repository import Repository from offlineimap.ui import getglobalui -from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread +from offlineimap.threadutil import InstanceLimitedThread from subprocess import Popen, PIPE -from threading import Event, Lock +from threading import Lock import os -from Queue import Queue, Empty +from Queue import Queue import sys import traceback diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 4bc5aab..320f5b2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -20,7 +20,6 @@ from offlineimap import threadutil from offlineimap.ui import getglobalui import os.path import re -import sys import traceback class BaseFolder: diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 50a0f55..623829a 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -17,7 +17,8 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from Base import BaseFolder -import os, threading +import os +import threading magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index a01d071..083325a 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -16,9 +16,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import os.path, os, re, time, socket +import socket +import time +import re +import os from Base import BaseFolder -from offlineimap import imaputil from threading import Lock try: diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index e7394b5..db4ab11 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -17,10 +17,8 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import * -from offlineimap import threadutil -from offlineimap.threadutil import InstanceLimitedThread from IMAP import IMAPFolder -import os.path, re +import os.path class MappingFolderMixIn: def _initmapping(self): diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 3bd33ea..4273d62 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -15,7 +15,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import re, socket, time, subprocess +import re +import socket +import time +import subprocess from offlineimap.ui import getglobalui import threading from offlineimap.imaplib2 import * diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index c2e2d50..f05c743 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -20,7 +20,10 @@ from offlineimap import imaplib2 as imaplib from offlineimap import imaplibutil, imaputil, threadutil from offlineimap.ui import getglobalui from threading import * -import thread, hmac, os, time, socket +import thread +import time +import hmac +import socket import base64 from StringIO import StringIO diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index c59f1f5..101f9a6 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -16,7 +16,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import re, string, types +import re +import string +import types from offlineimap.ui import getglobalui quotere = re.compile('^("(?:[^"]|\\\\")*")') diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index a586460..97637b8 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -16,8 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap.repository.IMAP import IMAPRepository -from offlineimap import folder, imaputil -from offlineimap.imapserver import IMAPServer +from offlineimap import folder class GmailRepository(IMAPRepository): """Gmail IMAP repository. diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 92e392d..a4e036b 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -19,7 +19,8 @@ from Base import BaseRepository from offlineimap import folder import offlineimap.folder.LocalStatus -import os, re +import os +import re class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 648440a..0af4d1f 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -17,9 +17,8 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from Base import BaseRepository -from offlineimap import folder, imaputil +from offlineimap import folder from offlineimap.ui import getglobalui -from mailbox import Maildir import os from stat import * diff --git a/offlineimap/syncmaster.py b/offlineimap/syncmaster.py index 5a0a2fb..d21cfac 100644 --- a/offlineimap/syncmaster.py +++ b/offlineimap/syncmaster.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap.threadutil import threadlist, InstanceLimitedThread, ExitNotifyThread +from offlineimap.threadutil import threadlist, InstanceLimitedThread from offlineimap.accounts import SyncableAccount, SigListener from threading import currentThread diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index f3451b8..8644322 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -18,7 +18,9 @@ from threading import * from Queue import Queue, Empty -import sys, traceback, thread, time +import traceback +import thread +import sys from offlineimap.ui import getglobalui profiledir = None diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index 9cd5eca..33248dd 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -16,7 +16,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import sys, time +import sys +import time from UIBase import UIBase class Basic(UIBase): diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index d88de1b..61019c1 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -18,7 +18,7 @@ from UIBase import UIBase from getpass import getpass -import select, sys +import sys from threading import * class TTYUI(UIBase): From fa2e9f5a6a5134c23019de78f2ad5f6cff17dc78 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 15 Mar 2011 18:49:43 +0100 Subject: [PATCH 041/817] update Changelog Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 25946c7..6f50c06 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -20,6 +20,7 @@ New Features Changes ------- +* Use imaplib2 instead of imaplib. * Makefile use magic to find the version number. * Rework the repository module * Change UI names to Blinkenlights,TTYUI,Basic,Quiet,MachineUI. @@ -47,6 +48,7 @@ Pending for the next major release ================================== * UIs get shorter and nicer names. (API changing) +* Implement IDLE feature. (delayed until next major release) Stalled From dc3ad723c9b60eb766bf6c9af93f5818d72be9f2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 16 Mar 2011 16:24:07 +0100 Subject: [PATCH 042/817] Give some love to UIDMaps - Some documentation improvements, this is a severely underdocumented class. This still needs some further improvements though. - Don't use apply(Baseclass) (which is going away in Python 3), use IMAPFolder.__init__(self, *args, **kwargs). - Don't call ValueError, string. It is ValueError(string) Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 19 ++++++++++--------- offlineimap/folder/UIDMaps.py | 34 ++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 4bc5aab..a545783 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -158,20 +158,21 @@ class BaseFolder: def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. - If the uid is < 0, the backend should assign a new uid and return it. - If the backend cannot assign a new uid, it returns the uid passed in - WITHOUT saving the message. + If the uid is < 0: The backend should assign a new uid and + return it. In case it cannot assign a new uid, it returns + the negative uid passed in WITHOUT saving the message. - If the backend CAN assign a new uid, but cannot find out what this UID - is (as is the case with many IMAP servers), it returns 0 but DOES save - the message. + If the backend CAN assign a new uid, but cannot find out what + this UID is (as is the case with some IMAP servers), it + returns 0 but DOES save the message. - IMAP backend should be the only one that can assign a new uid. + IMAP backend should be the only one that can assign a new + uid. If the uid is > 0, the backend should set the uid to this, if it can. - If it cannot set the uid to that, it will save it anyway. - It will return the uid assigned in any case. + If it cannot set the uid to that, it will save it anyway. + It will return the uid assigned in any case. """ raise NotImplementedException diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index e7394b5..78e4357 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -23,6 +23,14 @@ from IMAP import IMAPFolder import os.path, re class MappingFolderMixIn: + """Helper class to map between Folder() instances where both side assign a uid + + Instance variables (self.): + r2l: dict mapping message uids: self.r2l[remoteuid]=localuid + l2r: dict mapping message uids: self.r2l[localuid]=remoteuid + #TODO: what is the difference, how are they used? + diskr2l: dict mapping message uids: self.r2l[remoteuid]=localuid + diskl2r: dict mapping message uids: self.r2l[localuid]=remoteuid""" def _initmapping(self): self.maplock = Lock() (self.diskr2l, self.diskl2r) = self._loadmaps() @@ -131,30 +139,32 @@ class MappingFolderMixIn: def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. - If the uid is < 0, the backend should assign a new uid and return it. - If the backend cannot assign a new uid, it returns the uid passed in - WITHOUT saving the message. - - If the backend CAN assign a new uid, but cannot find out what this UID - is (as is the case with many IMAP servers), it returns 0 but DOES save - the message. - - IMAP backend should be the only one that can assign a new uid. + The UIDMaps class will not return a newly assigned uid, as it + internally maps different uids between IMAP servers. So a + successful savemessage() invocation will return the same uid it + has been invoked with. As it maps between 2 IMAP servers which + means the source message must already have an uid, it requires a + positive uid to be passed in. Passing in a message with a + negative uid will do nothing and return the negative uid. If the uid is > 0, the backend should set the uid to this, if it can. If it cannot set the uid to that, it will save it anyway. It will return the uid assigned in any case. """ + # Mapped UID instances require the source to already have a + # positive UID, so simply return here. if uid < 0: - # We cannot assign a new uid. return uid + + #if msg uid already exists, just modify the flags if uid in self.r2l: self.savemessageflags(uid, flags) return uid + newluid = self._mb.savemessage(self, -1, content, flags, rtime) if newluid < 1: - raise ValueError, "Backend could not find uid for message" + raise ValueError("Backend could not find uid for message") self.maplock.acquire() try: self.diskl2r[newluid] = uid @@ -221,5 +231,5 @@ class MappingFolderMixIn: # Define a class for local part of IMAP. class MappedIMAPFolder(MappingFolderMixIn, IMAPFolder): def __init__(self, *args, **kwargs): - apply(IMAPFolder.__init__, (self,) + args, kwargs) + IMAPFolder.__init__(self, *args, **kwargs) self._initmapping() From fd28c5a2d345327fc3784a820c59f408f376adc6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 16 Mar 2011 16:03:35 +0100 Subject: [PATCH 043/817] folder/IMAP: savemessage() should just save flags if uid already exists As the LocalStatus and UIDMap backend already did: If the uid already exists for savemessage(), only modify the flags and don't append a new message. We don't invoke savemessage() on messages that already exist in our sync logic, so this has no change on our current behavior. But it makes backends befave more consistent with each other. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 89848b2..74a9b87 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -394,6 +394,11 @@ class IMAPFolder(BaseFolder): server. If the folder is read-only it will return 0.""" self.ui.debug('imap', 'savemessage: called') + # already have it, just save modified flags + if uid > 0 and uid in self.messagelist: + self.savemessageflags(uid, flags) + return uid + try: imapobj = self.imapserver.acquireconnection() From fe0e17e45f45c0efb1ed933d8313575ea5f9ddea Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 16 Mar 2011 18:04:25 +0100 Subject: [PATCH 044/817] v6.3.3-rc1 Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 25 --------------------- Changelog.rst | 55 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 6f50c06..bb63372 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -12,38 +12,13 @@ others. New Features ------------ -* Implement UIDPLUS extension support. OfflineIMAP will now not insert - an X-OfflineIMAP header if the mail server supports the UIDPLUS - extension. -* SSL: support subjectAltName. Changes ------- -* Use imaplib2 instead of imaplib. -* Makefile use magic to find the version number. -* Rework the repository module -* Change UI names to Blinkenlights,TTYUI,Basic,Quiet,MachineUI. - Old names will still work, but are deprecated. - Document that we don't accept a list of UIs anymore. -* Reworked the syncing strategy. The only user-visible change is that - blowing away LocalStatus will not require you to redownload ALL of - your mails if you still have the local Maildir. It will simply - recreate LocalStatus. -* TTYUI ouput improved. - Bug Fixes --------- -* Fix ignoring output while determining the rst2xxx command name to build - documentation. -* Fix hang because of infinite loop reading EOF. -* Allow SSL connections to send keep-alive messages. -* Fix regression (UIBase is no more). -* Make profiling mode really enforce single-threading -* Do not send localized date strings to the IMAP server as it will - either ignore or refuse them. - Pending for the next major release ================================== diff --git a/Changelog.rst b/Changelog.rst index 78554cd..312742f 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,61 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.3.3-rc1 (2011-03-16) +=================================== + +Notes +----- + +Here is time to begin the tests cycle. If feature topics are sent, I may merge +or delay them until the next stable release. + +Main change comes from the migration from imaplib to imaplib2. It's internal +code changes and doesn't impact users. UIDPLUS and subjectAltName for SSL are +also great improvements. + +This release includes a hang fix due to infinite loop. Users seeing OfflineIMAP +hang and consuming a lot of CPU are asked to update. + +That beeing said, this is still an early release canditate you should use for +non-critical data only! + +New Features +------------ + +* Implement UIDPLUS extension support. OfflineIMAP will now not insert + an X-OfflineIMAP header if the mail server supports the UIDPLUS + extension. +* SSL: support subjectAltName. + +Changes +------- + +* Use imaplib2 instead of imaplib. +* Makefile use magic to find the version number. +* Rework the repository module +* Change UI names to Blinkenlights,TTYUI,Basic,Quiet,MachineUI. + Old names will still work, but are deprecated. + Document that we don't accept a list of UIs anymore. +* Reworked the syncing strategy. The only user-visible change is that + blowing away LocalStatus will not require you to redownload ALL of + your mails if you still have the local Maildir. It will simply + recreate LocalStatus. +* TTYUI ouput improved. +* Code cleanups. + +Bug Fixes +--------- + +* Fix ignoring output while determining the rst2xxx command name to build + documentation. +* Fix hang because of infinite loop reading EOF. +* Allow SSL connections to send keep-alive messages. +* Fix regression (UIBase is no more). +* Make profiling mode really enforce single-threading +* Do not send localized date strings to the IMAP server as it will + either ignore or refuse them. + OfflineIMAP v6.3.2 (2010-02-21) =============================== From 3f77afeb8a9f0967eb7334da62d0527fb938981c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 22 Mar 2011 11:01:52 +0100 Subject: [PATCH 045/817] offlineimap.conf: Clarify password options via netrc Document that only one user name per host name can be given via netrc file. Reformat the enumeration text. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index c0ff8d7..987b6b4 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -289,39 +289,37 @@ ssl = yes # Specify the remote user name. remoteuser = username -# There are five ways to give the password for the remote IMAP -# server: +# There are five ways to specify the password for the IMAP server: # -# 1. No password at all specified in the config file. If a matching entry is -# found in ~/.netrc (see netrc (5) for information) the password from the -# matching entry will be used. If there is no ~/.netrc file but there is an -# /etc/netrc file, the password will instead be taken from there. Otherwise -# you will be prompted for the password when OfflineIMAP starts. +# 1. No password at all specified in the config file. +# If a matching entry is found in ~/.netrc (see netrc (5) for +# information) this password will be used. Do note that netrc only +# allows one entry per hostname. If there is no ~/.netrc file but +# there is an /etc/netrc file, the password will instead be taken +# from there. Otherwise you will be prompted for the password when +# OfflineIMAP starts when using a UI that supports this. # # 2. The remote password stored in this file with the remotepass # option. Example: -# -# remotepass = mypassword +# remotepass = mypassword # # 3. The remote password stored as a single line in an external # file, which is referenced by the remotefile option. Example: -# -# remotepassfile = ~/Password.IMAP.Account1 +# remotepassfile = ~/Password.IMAP.Account1 # # 4. With a preauth tunnel. With this method, you invoke an external -# program that is guaranteed *NOT* to ask for a password, but rather -# to read from stdin and write to stdout an IMAP procotol stream -# that begins life in the PREAUTH state. When you use a tunnel, -# you do NOT specify a user or password (if you do, they'll be -# ignored.) Instead, you specify a preauthtunnel, as this -# example illustrates for Courier IMAP on Debian: +# program that is guaranteed *NOT* to ask for a password, but rather +# to read from stdin and write to stdout an IMAP procotol stream that +# begins life in the PREAUTH state. When you use a tunnel, you do +# NOT specify a user or password (if you do, they'll be ignored.) +# Instead, you specify a preauthtunnel, as this example illustrates +# for Courier IMAP on Debian: +# preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' # -# preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' -# -# 5. If you are using Kerberos and have the Python Kerberos package installed, -# you should not specify a remotepass. If the user has a valid -# Kerberos TGT, OfflineIMAP will figure out the rest all by itself, and -# fall back to password authentication if needed. +# 5. If you are using Kerberos and have the Python Kerberos package +# installed, you should not specify a remotepass. If the user has a +# valid Kerberos TGT, OfflineIMAP will figure out the rest all by +# itself, and fall back to password authentication if needed. ########## Advanced settings From 9a277cfd022ca0aaf0513ca407c05225f5fdc049 Mon Sep 17 00:00:00 2001 From: Vincent Beffara Date: Tue, 22 Mar 2011 15:51:42 +0100 Subject: [PATCH 046/817] Remove a darwin-specific workaround for read() Because of a buggy realloc() implementation in earlier versions of Python on Mac OS X, we had to cut reads into manageable chunks by hand; this is no more needed with Python 2.6, and besides it causes problems with imaplib2, which we now use. Revert the special case to use the system's read() instead, which is now safe. Signed-off-by: Vincent Beffara Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index f05c743..fc7a503 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -65,24 +65,7 @@ class UsefulIMAPMixIn: def _mesg(self, s, tn=None, secs=None): imaplibutil.new_mesg(self, s, tn, secs) -class UsefulIMAP4(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): - # This is a hack around Darwin's implementation of realloc() (which - # Python uses inside the socket code). On Darwin, we split the - # message into small chunks. - # see http://bugs.python.org/issue3531 - def read(self, size): - if (system() == 'Darwin') and (size>0) : - read = 0 - io = StringIO() - while read < size: - data = imaplib.IMAP4.read (self, min(size-read, 65536)) - if not data: - break - read += len(data) - io.write(data) - return io.getvalue() - else: - return imaplib.IMAP4.read (self, size) +class UsefulIMAP4(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): pass class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): pass From 9f03f41b7015a1b594567d634ab11a9476979c69 Mon Sep 17 00:00:00 2001 From: Vincent Beffara Date: Tue, 22 Mar 2011 15:51:43 +0100 Subject: [PATCH 047/817] Move UsefulIMAPMixIn to imaplibutil.py Now that we do not need any system-specific hack, the three UsefulIMAP4 classes have become empty and should be removed. This implies importing UsefulIMAPMixIn directly from the classes defined in imaplibutil. Prepare this change by moving the code into imaplibutil.py. Functionally this is a no-op. Signed-off-by: Vincent Beffara Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 27 +++++++++++++++++++++++++++ offlineimap/imapserver.py | 33 +++------------------------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 4273d62..651eb71 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -32,6 +32,33 @@ except ImportError: #fails on python <2.6 pass +class UsefulIMAPMixIn: + def getstate(self): + return self.state + def getselectedfolder(self): + if self.getstate() == 'SELECTED': + return self.selectedfolder + return None + + def select(self, mailbox='INBOX', readonly=None, force = 0): + if (not force) and self.getselectedfolder() == mailbox \ + and self.is_readonly == readonly: + # No change; return. + return + # Wipe out all old responses, to maintain semantics with old imaplib2 + del self.untagged_responses[:] + result = self.__class__.__bases__[1].select(self, mailbox, readonly) + if result[0] != 'OK': + raise ValueError, "Error from select: %s" % str(result) + if self.getstate() == 'SELECTED': + self.selectedfolder = mailbox + else: + self.selectedfolder = None + return result + + def _mesg(self, s, tn=None, secs=None): + new_mesg(self, s, tn, secs) + class IMAP4_Tunnel(IMAP4): """IMAP4 client class over a tunnel diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index fc7a503..96f0d1f 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -38,38 +38,11 @@ try: except ImportError: pass -class UsefulIMAPMixIn: - def getstate(self): - return self.state - def getselectedfolder(self): - if self.getstate() == 'SELECTED': - return self.selectedfolder - return None +class UsefulIMAP4(imaplibutil.UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): pass - def select(self, mailbox='INBOX', readonly=None, force = 0): - if (not force) and self.getselectedfolder() == mailbox \ - and self.is_readonly == readonly: - # No change; return. - return - # Wipe out all old responses, to maintain semantics with old imaplib2 - del self.untagged_responses[:] - result = self.__class__.__bases__[1].select(self, mailbox, readonly) - if result[0] != 'OK': - raise ValueError, "Error from select: %s" % str(result) - if self.getstate() == 'SELECTED': - self.selectedfolder = mailbox - else: - self.selectedfolder = None - return result +class UsefulIMAP4_SSL(imaplibutil.UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): pass - def _mesg(self, s, tn=None, secs=None): - imaplibutil.new_mesg(self, s, tn, secs) - -class UsefulIMAP4(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): pass - -class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): pass - -class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass +class UsefulIMAP4_Tunnel(imaplibutil.UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass class IMAPServer: GSS_STATE_STEP = 0 From bb48b6deafc0b632bf718e4bbc557e69b3275fe9 Mon Sep 17 00:00:00 2001 From: Vincent Beffara Date: Tue, 22 Mar 2011 15:51:44 +0100 Subject: [PATCH 048/817] Get rid of the UsefulIMAP4 classes The three classes with names starting with UsefulIMAP4 were used for two purposes, to include the UsefulIMAPMixIn class and to implement various system-specific kludges. None of these kludges remain, so it is cleaner to include UsefulIMAPMixIn directly in imaplibutil and forget about them for good. Signed-off-by: Vincent Beffara Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 6 +++--- offlineimap/imapserver.py | 21 ++++++++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 651eb71..14b11c2 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -59,7 +59,7 @@ class UsefulIMAPMixIn: def _mesg(self, s, tn=None, secs=None): new_mesg(self, s, tn, secs) -class IMAP4_Tunnel(IMAP4): +class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): """IMAP4 client class over a tunnel Instantiate with: IMAP4_Tunnel(tunnelcmd) @@ -107,7 +107,7 @@ def new_mesg(self, s, tn=None, secs=None): tm = time.strftime('%M:%S', time.localtime(secs)) getglobalui().debug('imap', ' %s.%02d %s %s' % (tm, (secs*100)%100, tn, s)) -class WrappedIMAP4_SSL(IMAP4_SSL): +class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): """Provides an improved version of the standard IMAP4_SSL It provides a better readline() implementation as impaplib's @@ -224,7 +224,7 @@ class WrappedIMAP4_SSL(IMAP4_SSL): return ('no matching domain name found in certificate') -class WrappedIMAP4(IMAP4): +class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): """Improved version of imaplib.IMAP4 that can also connect to IPv6""" def open(self, host = '', port = IMAP4_PORT): diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 96f0d1f..c474a01 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -38,12 +38,6 @@ try: except ImportError: pass -class UsefulIMAP4(imaplibutil.UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): pass - -class UsefulIMAP4_SSL(imaplibutil.UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): pass - -class UsefulIMAP4_Tunnel(imaplibutil.UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass - class IMAPServer: GSS_STATE_STEP = 0 GSS_STATE_WRAP = 1 @@ -202,18 +196,19 @@ class IMAPServer: # Generate a new connection. if self.tunnel: self.ui.connecting('tunnel', self.tunnel) - imapobj = UsefulIMAP4_Tunnel(self.tunnel, timeout=socket.getdefaulttimeout()) + imapobj = imaplibutil.IMAP4_Tunnel(self.tunnel, + timeout=socket.getdefaulttimeout()) success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) - imapobj = UsefulIMAP4_SSL(self.hostname, self.port, - self.sslclientkey, self.sslclientcert, - timeout=socket.getdefaulttimeout(), - cacertfile = self.sslcacertfile) + imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, self.port, + self.sslclientkey, self.sslclientcert, + timeout=socket.getdefaulttimeout(), + cacertfile = self.sslcacertfile) else: self.ui.connecting(self.hostname, self.port) - imapobj = UsefulIMAP4(self.hostname, self.port, - timeout=socket.getdefaulttimeout()) + imapobj = imaplibutil.WrappedIMAP4(self.hostname, self.port, + timeout=socket.getdefaulttimeout()) imapobj.mustquote = imaplibutil.mustquote From 105da1b0c35d6762865a56d10eaf2246716a11cb Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Thu, 24 Mar 2011 14:51:10 -0400 Subject: [PATCH 049/817] Always logout() on imaplib2 objects, even during exceptions Without this, trying to Ctrl-C out of offlineimap will go into a hang. Signed-off-by: Ethan Glasser-Camp Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ offlineimap/accounts.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 0a85888..39490a1 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,6 +19,8 @@ Changes Bug Fixes --------- +* Fix hang when using Ctrl+C in some cases. + Pending for the next major release ================================== diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 4903e39..7edfa37 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -265,10 +265,9 @@ class SyncableAccount(Account): mbnames.write() localrepos.forgetfolders() remoterepos.forgetfolders() + finally: localrepos.holdordropconnections() remoterepos.holdordropconnections() - finally: - pass hook = self.getconf('postsynchook', '') self.callhook(hook) From 09515f8f90cec06ffd880456b85d5c9c8a524d14 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 25 Mar 2011 10:19:30 +0100 Subject: [PATCH 050/817] Increase compatability with Gmail When uploading a new message to Gmail we need to find out the UID it assigned it, but Gmail does not advertize the UIDPLUS extension (in all cases) and it fails to find the email that we just uploaded when searching for it. This prevented us effectively from uploading to gmail. See analysis in http://lists.alioth.debian.org/pipermail/offlineimap-project/2011-March/001449.html for details on what is going wrong. This patch increases compatability with Gmail by checking for APPENDUID responses to an APPEND action even if the server did not claim to support it. This restores the capability to upload messages to the *broken* Gmail IMAP implementation. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/folder/IMAP.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 39490a1..79006fd 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,9 @@ New Features Changes ------- +* Increase compatability with Gmail servers which claim to not support + the UIDPLUS extension but in reality do. + Bug Fixes --------- diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 74a9b87..828070c 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -439,8 +439,9 @@ class IMAPFolder(BaseFolder): (typ,dat) = imapobj.check() assert(typ == 'OK') - # get the UID. - if use_uidplus: + # get the new UID. Test for APPENDUID response even if the + # server claims to not support it, as e.g. Gmail does :-( + if use_uidplus or imapobj._get_untagged_response('APPENDUID', True): # get the new UID from the APPENDUID response, it could look like # OK [APPENDUID 38505 3955] APPEND completed # with 38505 bein folder UIDvalidity and 3955 the new UID From 954655b7eccfc2e0a26e392ff897670fe2bcad17 Mon Sep 17 00:00:00 2001 From: David Favro Date: Sat, 26 Mar 2011 19:43:37 -0400 Subject: [PATCH 051/817] Fixed bug: wrong number of arguments to debug() [IMAP.py]. Signed-off-by: David Favro Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 828070c..8c7b68d 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -359,7 +359,7 @@ class IMAPFolder(BaseFolder): # or something. Argh. It seems that Time2Internaldate # will rause a ValueError if the year is 0102 but not 1902, # but some IMAP servers nonetheless choke on 1902. - self.ui.debug("Message with invalid date %s. Server will use local time." \ + self.ui.debug('imap', "Message with invalid date %s. Server will use local time." \ % datetuple) return None From ca012d3a81428b6bc74274ee036391c591477b0f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 28 Mar 2011 10:19:19 -0400 Subject: [PATCH 052/817] Implement more efficient functions for the MappedUID case We are calling getmessagelist() internally a lot, e.g. just to check if a UID exists (from uidexist()). This is a very expensive operation in the UIDMapped case, as we reconstruct the whole messagelist dict every single time, involving lots of copying etc. So we provide more efficient implementations for the uidexists() getmessageuidlist() and getmessagecount() functions that are fast in the UIDMapped case. This should solve the performance regression that was recently observed in the Mapped UID case. Signed-off-by: Sebastian Spaeth Reviewed-and-tested-by: Vincent Beffara Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/UIDMaps.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 039a41d..43b28e4 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -107,9 +107,30 @@ class MappingFolderMixIn: finally: self.maplock.release() + def uidexists(self, ruid): + """Checks if the (remote) UID exists in this Folder""" + # This implementation overrides the one in BaseFolder, as it is + # much more efficient for the mapped case. + return ruid in self.r2l + + def getmessageuidlist(self): + """Gets a list of (remote) UIDs. + You may have to call cachemessagelist() before calling this function!""" + # This implementation overrides the one in BaseFolder, as it is + # much more efficient for the mapped case. + return self.r2l.keys() + + def getmessagecount(self): + """Gets the number of messages in this folder. + You may have to call cachemessagelist() before calling this function!""" + # This implementation overrides the one in BaseFolder, as it is + # much more efficient for the mapped case. + return len(self.r2l) + def getmessagelist(self): - """Gets the current message list. - You must call cachemessagelist() before calling this function!""" + """Gets the current message list. This function's implementation + is quite expensive for the mapped UID case. You must call + cachemessagelist() before calling this function!""" retval = {} localhash = self._mb.getmessagelist(self) From 58220fd8e71cb6312edc0fe4c17a8a3f877e36e5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 28 Mar 2011 10:19:20 -0400 Subject: [PATCH 053/817] Replace calls to getmessagelist() to alternatives getmessagelist() is slow for the mapped UID case, so replace some of its occurences with calls that are optimized for this case, ie getmessagecount() and uidexists(). Signed-off-by: Sebastian Spaeth Reviewed-and-tested-by: Vincent Beffara Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 2 +- offlineimap/folder/IMAP.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index fd8496e..ad11158 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -318,7 +318,7 @@ class BaseFolder: flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - if uid in dstfolder.getmessagelist(): + if dstfolder.uidexists(uid): # dst has message with that UID already, only update status statusfolder.savemessage(uid, None, flags, rtime) return diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8c7b68d..f2d0dda 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -94,7 +94,7 @@ class IMAPFolder(BaseFolder): maxmsgid = max(long(msgid), maxmsgid) # Different number of messages than last time? - if maxmsgid != len(statusfolder.getmessagelist()): + if maxmsgid != statusfolder.getmessagecount(): return True if maxmsgid < 1: @@ -112,7 +112,7 @@ class IMAPFolder(BaseFolder): if not options.has_key('UID'): return True uid = long(options['UID']) - saveduids = statusfolder.getmessagelist().keys() + saveduids = statusfolder.getmessageuidlist() saveduids.sort() if uid != saveduids[-1]: return True @@ -395,7 +395,7 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage: called') # already have it, just save modified flags - if uid > 0 and uid in self.messagelist: + if uid > 0 and self.uidexists(uid): self.savemessageflags(uid, flags) return uid From f584d6fee6fab321a4255246f86dead8da4d205c Mon Sep 17 00:00:00 2001 From: Daniel Shahaf Date: Sun, 3 Apr 2011 05:49:20 +0300 Subject: [PATCH 054/817] Document a feature in the FAQ Add a FAQ entry for the Blinkenlight UI's 'Force an immediate resync' feature. Signed-off-by: Daniel Shahaf Signed-off-by: Nicolas Sebrecht --- docs/FAQ.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 9b66e29..fcc55b3 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -205,6 +205,20 @@ How is OfflineIMAP conformance? * Maildir as specified in the Maildir manpage and the qmail website * Standard Python 2.6 as implemented on POSIX-compliant systems +Can I force OfflineIMAP to sync a folder right now? +--------------------------------------------------- + +Yes, if you use the `Blinkenlights` UI. That UI shows the active accounts +as follows:: + + 4: [active] *Control: . + 3: [ 4:36] personal: + 2: [ 3:37] work: . + 1: [ 6:28] uni: + +Simply press the appropriate digit (`3` for `personal`, etc.) to resync that +account immediately. This will be ignored if a resync is already in progress +for that account. Configuration Questions ======================= From 69d2185c2375d011f89e26f146126e0d7e3c1050 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 4 Apr 2011 20:03:36 +0200 Subject: [PATCH 055/817] v6.3.3-rc2 Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 5 ----- Changelog.rst | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 79006fd..0a85888 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,14 +16,9 @@ New Features Changes ------- -* Increase compatability with Gmail servers which claim to not support - the UIDPLUS extension but in reality do. - Bug Fixes --------- -* Fix hang when using Ctrl+C in some cases. - Pending for the next major release ================================== diff --git a/Changelog.rst b/Changelog.rst index 312742f..db8ca9a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,46 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.3.3-rc2 (2011-04-07) +=================================== + +Notes +----- + +We are now at the third week of the -rc1 cycle. I think it's welcome to begin +the -rc2 cycle. Things are highly calming down in the code even if we had +much more feedbacks than usual. Keep going your effort! + +I'd like to thank reporters who involved in this cycle: + - Баталов Григорий + - Alexander Skwar + - Christoph Höger + - dtk + - Greg Grossmeier + - h2oz7v + - Iain Dalton + - Pan Tsu + - Vincent Beffara + - Will Styler + (my apologies if I forget somebody) + +...and all active developers, of course! + +The imaplib2 migration looks to go the right way to be definetly released but +still needs more tests. So, here we go... + +Changes +------- + +* Increase compatability with Gmail servers which claim to not support + the UIDPLUS extension but in reality do. + +Bug Fixes +--------- + +* Fix hang when using Ctrl+C in some cases. + + OfflineIMAP v6.3.3-rc1 (2011-03-16) =================================== From d762175af410eb0f9bd23bf403bade6847879521 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 5 Apr 2011 12:16:43 +0200 Subject: [PATCH 056/817] Make -f option with with folder names with spaces. Previously ALL spaces had been stripped off. Now, only strip spaces around the comma, so -f "INBOX, Deleted Mails" will work. You will still need to quote or escape spaces so the shell hand the list as one command line argument to offlineimap. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +++ offlineimap/init.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 0a85888..969655b 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,9 @@ New Features Changes ------- +* the -f option did not work with Folder names with spaces. It works + now, use with quoting e.g. -f "INBOX, Deleted Mails". + Bug Fixes --------- diff --git a/offlineimap/init.py b/offlineimap/init.py index 96d4e50..296b84b 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -238,8 +238,10 @@ class OfflineImap: for section in accounts.getaccountlist(config): config.set('Account ' + section, "quick", '-1') + #custom folder list specified? if options.folders: - foldernames = options.folders.replace(" ", "").split(",") + foldernames = map(lambda s: s.strip(), + options.folders.split(",")) folderfilter = "lambda f: f in %s" % foldernames folderincludes = "[]" for accountname in accounts.getaccountlist(config): From e37441cd19ae9f216c993650b0d8fded6c2f9a14 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 11 Apr 2011 18:09:07 +0200 Subject: [PATCH 057/817] folder/Maildir: Make use of helper functions quickchanged() was iterating a lot, make use of some of the helper functions that had been introduced recently and document the function a bit better. No functional change. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 083325a..02e64d2 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -162,16 +162,17 @@ class MaildirFolder(BaseFolder): return retval def quickchanged(self, statusfolder): + """Returns True if the Maildir has changed""" self.cachemessagelist() - savedmessages = statusfolder.getmessagelist() - if len(self.messagelist) != len(savedmessages): + # Folder has different uids than statusfolder => TRUE + if sorted(self.getmessageuidlist()) != \ + sorted(statusfolder.getmessageuidlist()): return True - for uid in self.messagelist.keys(): - if uid not in savedmessages: + # Also check for flag changes, it's quick on a Maildir + for (uid, message) in self.getmessagelist().iteritems(): + if message['flags'] != statusfolder.getmessageflags(uid): return True - if self.messagelist[uid]['flags'] != savedmessages[uid]['flags']: - return True - return False + return False #Nope, nothing changed def cachemessagelist(self): if self.messagelist is None: From fdf22400b1c4108dd97e2b2c0d93c1e9ec94419c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 11 Apr 2011 17:24:44 +0200 Subject: [PATCH 058/817] imaplib2: Bump from 2.20 to 2.22 This contains a fixed Time2InternalDate function and a more robust socket connection, trying twice and raising an error only when that fails (I believe). The actual code changes are rather minor. Signed-off-by: Sebastian Spaeth Reviewed-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) mode change 100755 => 100644 offlineimap/imaplib2.py diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py old mode 100755 new mode 100644 index 20b90ae..be34025 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.20" +__version__ = "2.22" __release__ = "2" -__revision__ = "20" +__revision__ = "22" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -35,12 +35,13 @@ ID contributed by Dave Baggett November 2009. Improved untagged responses handling suggested by Dave Baggett November 2009. Improved thread naming, and 0 read detection contributed by Grant Edwards June 2010. Improved timeout handling contributed by Ivan Vovnenko October 2010. -Timeout handling further improved by Ethan Glasser-Camp December 2010.""" +Timeout handling further improved by Ethan Glasser-Camp December 2010. +Time2Internaldate() patch to match RFC2060 specification of English month names from http://bugs.python.org/issue11024 March 2011.""" __author__ = "Piers Lauder " __URL__ = "http://janeelix.com/piers/python/imaplib2" __license__ = "Python License" -import binascii, os, Queue, random, re, select, socket, sys, time, threading, zlib +import binascii, errno, os, Queue, random, re, select, socket, sys, time, threading, zlib select_module = select @@ -404,7 +405,15 @@ class IMAP4(object): except socket.error, msg: continue try: - s.connect(sa) + for i in (0, 1): + try: + s.connect(sa) + break + except socket.error, msg: + if len(msg.args) < 2 or msg.args[0] != errno.EINTR: + raise + else: + raise socket.error(msg) except socket.error, msg: s.close() continue @@ -2057,8 +2066,10 @@ class _IdleCont(object): -Mon2num = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, - 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} +MonthNames = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + +Mon2num = dict(zip((x.encode() for x in MonthNames[1:]), range(1, 13))) InternalDate = re.compile(r'.*INTERNALDATE "' r'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' @@ -2126,14 +2137,13 @@ def Time2Internaldate(date_time): else: raise ValueError("date_time not of a known type") - dt = time.strftime("%d-%b-%Y %H:%M:%S", tt) - if dt[0] == '0': - dt = ' ' + dt[1:] if time.daylight and tt[-1]: zone = -time.altzone else: zone = -time.timezone - return '"' + dt + " %+03d%02d" % divmod(zone//60, 60) + '"' + return ('"%2d-%s-%04d %02d:%02d:%02d %+03d%02d"' % + ((tt[2], MonthNames[tt[1]], tt[0]) + tt[3:6] + + divmod(zone//60, 60))) From 9e734006f6dbc2b02626b77e8018f9b448a0640e Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Wed, 13 Apr 2011 04:52:18 -0400 Subject: [PATCH 059/817] Fix IMAP4_Tunnel to work with imaplib2 * IMAP4_Tunnel constructor should support base-class arguments, in order to support the timeout argument. * IMAP4_Tunnel needs to store the member IMAP4.host, which is normally done in IMAP4.open(). * Update IMAP4_Tunnel.read() and IMAP4_Tunnel.send(). We turn on nonblocking mode for these sockets, so we can return immediately with whatever data is available. Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ offlineimap/imaplibutil.py | 44 +++++++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 969655b..d51a4f3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -22,6 +22,8 @@ Changes Bug Fixes --------- +* Fix IMAP4 tunnel with imaplib2. + Pending for the next major release ================================== diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 14b11c2..7c98fef 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -15,6 +15,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os +import fcntl import re import socket import time @@ -67,32 +69,50 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): tunnelcmd -- shell command to generate the tunnel. The result will be in PREAUTH stage.""" - def __init__(self, tunnelcmd): - IMAP4.__init__(self, tunnelcmd) + def __init__(self, tunnelcmd, **kwargs): + IMAP4.__init__(self, tunnelcmd, **kwargs) def open(self, host, port): """The tunnelcmd comes in on host!""" + self.host = host self.process = subprocess.Popen(host, shell=True, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) (self.outfd, self.infd) = (self.process.stdin, self.process.stdout) # imaplib2 polls on this fd self.read_fd = self.infd.fileno() - def read(self, size): - retval = '' - while len(retval) < size: - buf = self.infd.read(size - len(retval)) - if not buf: - break - retval += buf - return retval + self.set_nonblocking(self.read_fd) - def readline(self): - return self.infd.readline() + def set_nonblocking(self, fd): + "Mark fd as nonblocking" + # get the file's current flag settings + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + # clear non-blocking mode from flags + fl = fl & ~os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, fl) + + def read(self, size): + """data = read(size) + Read at most 'size' bytes from remote.""" + + if self.decompressor is None: + return os.read(self.read_fd, size) + + if self.decompressor.unconsumed_tail: + data = self.decompressor.unconsumed_tail + else: + data = os.read(self.read_fd, 8192) + + return self.decompressor.decompress(data, size) def send(self, data): + if self.compressor is not None: + data = self.compressor.compress(data) + data += self.compressor.flush(zlib.Z_SYNC_FLUSH) + self.outfd.write(data) + def shutdown(self): self.infd.close() self.outfd.close() From 38dfa1d684d2f4d27274eb0cadcf54707ad2110f Mon Sep 17 00:00:00 2001 From: dtk Date: Wed, 13 Apr 2011 10:48:51 +0200 Subject: [PATCH 060/817] Improve documentation on quick syncs Make clear that quick syncs do not happen inbetween full syncs (ie they are part of the regular autorefresh intervals and don't happen within them). This part of the documentation had confused many. Signed-off-by: dtk Modified-by: Sebastian Spaeth Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 987b6b4..869ac96 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -171,13 +171,17 @@ remoterepository = RemoteExample # autorefresh = 5 -# You can tell offlineimap to do a number of quicker synchronizations -# between full updates. A quick synchronization only synchronizes -# if a Maildir folder has changed, or if an IMAP folder has received -# new messages or had messages deleted. It does not update if the -# only changes were to IMAP flags. Specify 0 to never do quick updates, -# -1 to always do quick updates, or a positive integer to do that many -# quick updates between each full synchronization (requires autorefresh). +# OfflineImap can replace a number of full updates by quick +# synchronizations. It only synchronizes a folder if 1) a Maildir +# folder has changed, or 2) if an IMAP folder has received new messages +# or had messages deleted, ie it does not update if only IMAP flags have +# changed. Full updates need to fetch ALL flags for all messages, so +# this makes quite a performance difference (especially if syncing +# between two IMAP servers). +# Specify 0 for never, -1 for always (works even in non-autorefresh +# mode), or a positive integer to do quick updates before doing +# another full synchronization (requires autorefresh). Updates are +# always performed after minutes, be they quick or full. # quick = 10 From 84db2c50ac8e2145160e9bf5aa468c8001ae7025 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 14 Apr 2011 20:18:17 +0200 Subject: [PATCH 061/817] v6.3.3-rc3 Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 5 ----- Changelog.rst | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index d51a4f3..0a85888 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,14 +16,9 @@ New Features Changes ------- -* the -f option did not work with Folder names with spaces. It works - now, use with quoting e.g. -f "INBOX, Deleted Mails". - Bug Fixes --------- -* Fix IMAP4 tunnel with imaplib2. - Pending for the next major release ================================== diff --git a/Changelog.rst b/Changelog.rst index db8ca9a..f209627e 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,32 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. + +OfflineIMAP v6.3.3-rc3 (2011-04-19) +=================================== + +Notes +----- + +It's more than a week since the previous release. Most of the issues raised were +discussed and fixed since last release. I think we can be glad and confident for +the future while the project live his merry life. + +Changes +------- + +* The -f option did not work with Folder names with spaces. It works + now, use with quoting e.g. -f "INBOX, Deleted Mails". +* Improved documentation. +* Bump from imaplib2 v2.20 to v2.22. +* Code refactoring. + +Bug Fixes +--------- + +* Fix IMAP4 tunnel with imaplib2. + + OfflineIMAP v6.3.3-rc2 (2011-04-07) =================================== From 9136bdb0ba78700db27422a574645038fd0543d1 Mon Sep 17 00:00:00 2001 From: Thomas Kahle Date: Wed, 20 Apr 2011 00:27:34 +0200 Subject: [PATCH 062/817] Adding an entry to offlineimap.conf that explain how to use python code to query for a password. Signed-off-by: Thomas Kahle Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/offlineimap.conf b/offlineimap.conf index 869ac96..7018090 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -293,7 +293,7 @@ ssl = yes # Specify the remote user name. remoteuser = username -# There are five ways to specify the password for the IMAP server: +# There are six ways to specify the password for the IMAP server: # # 1. No password at all specified in the config file. # If a matching entry is found in ~/.netrc (see netrc (5) for @@ -324,6 +324,15 @@ remoteuser = username # installed, you should not specify a remotepass. If the user has a # valid Kerberos TGT, OfflineIMAP will figure out the rest all by # itself, and fall back to password authentication if needed. +# +# 6. Using arbitrary python code. With this method, you invoke a +# function from your pythonfile. To use this method assign the name +# of the function to the variable 'remotepasseval'. Example: +# remotepasseval = get_password("imap.example.net") +# You can also query for the username: +# remoteusereval = get_username("imap.example.net") +# This method can be used to design more elaborate setups, e.g. by +# querying the gnome-keyring via its python bindings. ########## Advanced settings From 3b096952364b572fa117261c2369472b0d41e44d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 21 Apr 2011 18:24:50 +0200 Subject: [PATCH 063/817] v6.3.3 Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 21 ++++++++++++++++++++- offlineimap/__init__.py | 2 +- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index f209627e..d9d6516 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,25 @@ ChangeLog releases announces. +OfflineIMAP v6.3.3 (2011- - ) +=============================== + +Notes +----- + +Make this last candidate cycle short. It looks like we don't need more tests as +most issues were raised and solved in the second round. Also, we have huge work +to merge big and expected features into OfflineIMAP. + +Thanks to all contributors, again. With such a contribution rate, we can release +stable faster. I hope it will be confirmed in the longer run! + +Changes +------- + +* Improved documentation for querying password. + + OfflineIMAP v6.3.3-rc3 (2011-04-19) =================================== @@ -93,7 +112,7 @@ also great improvements. This release includes a hang fix due to infinite loop. Users seeing OfflineIMAP hang and consuming a lot of CPU are asked to update. -That beeing said, this is still an early release canditate you should use for +That beeing said, this is still an early release candidate you should use for non-critical data only! New Features diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 4a760df..273faef 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.2" +__version__ = "6.3.3" __copyright__ = "Copyright (C) 2002 - 2010 John Goerzen" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 12e11429b51470c2ab0983f104bea400666c64ac Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 24 Apr 2011 18:43:35 +0200 Subject: [PATCH 064/817] Changelog: add missing date for v6.3.3 Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index d9d6516..b2912b3 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,7 +12,7 @@ ChangeLog releases announces. -OfflineIMAP v6.3.3 (2011- - ) +OfflineIMAP v6.3.3 (2011-04-24) =============================== Notes From 36eb37b47df3eccedd8dd0d7dd78cf0fb9305b66 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 11 Apr 2011 18:33:11 +0200 Subject: [PATCH 065/817] IMAP: reduce quickchanged() checks For each folder we were making a second IMAP request asking for the latest UID and compared that with the highest UID in our statusfolder. This catched the case that 1 mail has been deleted by someone else and another one has arrived since we checked, so that the total number of mails appears to not have changed. We don't capture anymore this case in the quickchanged() case. It improves my performance from 8 to about 7.5 seconds per check (with lots of variation) and we would benefit even more in the IMAP<->IMAP case as we do one additional IMAP lookup per folder on each side then. Do cleanups on whitespaces while in this file. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index f2d0dda..046c792 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -74,7 +74,7 @@ class IMAPFolder(BaseFolder): return long(imapobj._get_untagged_response('UIDVALIDITY', True)[0]) finally: self.imapserver.releaseconnection(imapobj) - + def quickchanged(self, statusfolder): # An IMAP folder has definitely changed if the number of # messages or the UID of the last message have changed. Otherwise @@ -97,26 +97,8 @@ class IMAPFolder(BaseFolder): if maxmsgid != statusfolder.getmessagecount(): return True - if maxmsgid < 1: - # No messages; return - return False - - # Now, get the UID for the last message. - response = imapobj.fetch('%d' % maxmsgid, '(UID)')[1] finally: self.imapserver.releaseconnection(imapobj) - - # Discard the message number. - messagestr = response[0].split(' ', 1)[1] - options = imaputil.flags2hash(messagestr) - if not options.has_key('UID'): - return True - uid = long(options['UID']) - saveduids = statusfolder.getmessageuidlist() - saveduids.sort() - if uid != saveduids[-1]: - return True - return False # TODO: Make this so that it can define a date that would be the oldest messages etc. @@ -216,13 +198,13 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'Returned object from fetching %d: %s' % \ (uid, str(initialresult))) return initialresult[1][0][1].replace("\r\n", "\n") - + finally: self.imapserver.releaseconnection(imapobj) def getmessagetime(self, uid): return self.messagelist[uid]['time'] - + def getmessageflags(self, uid): return self.messagelist[uid]['flags'] @@ -416,16 +398,16 @@ class IMAPFolder(BaseFolder): # get the date of the message file, so we can pass it to the server. date = self.getmessageinternaldate(content, rtime) self.ui.debug('imap', 'savemessage: using date %s' % date) - + content = re.sub("(? Date: Thu, 24 Mar 2011 17:45:21 +0100 Subject: [PATCH 066/817] Remove upload neguid pass from sync logic In order to optimize performance, we fold the 1st and 2nd pass of our sync strategy into one. They were essentially doing the same thing: uploading a message to the other side. The only difference was that in one case we have a negative UID locally, and in the other case, we have a positive one already. This saves some time, as we don't have to run through that function on IMAP servers anyway (they always have positive UIDs), and 2nd were we stalling further copying until phase 1 was finished. So uploading a single new message would prevent us from starting to copy existing regular messages. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 4 + offlineimap/folder/Base.py | 140 +++++++++------------------------- offlineimap/folder/UIDMaps.py | 5 -- 3 files changed, 40 insertions(+), 109 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 0a85888..69169c1 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,10 @@ New Features Changes ------- +* Reduced our sync logic from 4 passes to 3 passes (integrating upload of + "new" and "existing" messages into one function). This should result in a + slight speedup. + Bug Fixes --------- diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index ad11158..991a39b 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -222,81 +222,6 @@ class BaseFolder: for uid in uidlist: self.deletemessage(uid) - def syncmessagesto_neguid_msg(self, uid, dstfolder, statusfolder, - register = 1): - """Copy a single message from self to dests. - - This is called by meth:`syncmessagesto_neguid`, possibly in a - new thread. It does not return anything. - - :param dstfolder: A BaseFolder-derived instance - :param statusfolder: A LocalStatusFolder instance - :param register: If True, output that a new thread was created - (and register it with the ui).""" - successobject = None - successuid = None - - if register: - self.ui.registerthread(self.getaccountname()) - self.ui.copyingmessage(uid, self, [dstfolder]) - - message = self.getmessage(uid) - flags = self.getmessageflags(uid) - rtime = self.getmessagetime(uid) - - #Save messages to dstfolder and see if a valid UID was returned - successuid = dstfolder.savemessage(uid, message, flags, rtime) - - #Succeeded? -> IMAP actually assigned a UID - #If successuid remained negative, no server was willing to assign us - #an UID. Ignore message. - if successuid >= 0: - # Copy the message to the statusfolder - statusfolder.savemessage(successuid, message, flags, rtime) - # Copy to its new name son the local server and delete - # the one without a UID. - self.savemessage(successuid, message, flags, rtime) - #TODO: above means, we read in the message and write it out - #to the very same dir with a different UID. Investigate if we - #cannot simply rename it. - - # delete the negative uid message. We have it with a good UID now. - self.deletemessage(uid) - - - def syncmessagesto_neguid(self, dstfolder, statusfolder): - """Pass 1 of folder synchronization. - - Look for messages in self with a negative uid. These are - messages in Maildirs that were not added by us. Try to add them - to the dstfolder. If that succeeds, get the new UID, add - it to the statusfolder, add it to local for real, and delete the - old fake (negative) one. - - :param dstfolder: A BaseFolder-derived instance - :param statusfolder: A LocalStatusFolder instance""" - - uidlist = [uid for uid in self.getmessageuidlist() if uid < 0] - threads = [] - - for uid in uidlist: - if dstfolder.suggeststhreads(): - dstfolder.waitforthread() - thread = threadutil.InstanceLimitedThread(\ - dstfolder.getcopyinstancelimit(), - target = self.syncmessagesto_neguid_msg, - name = "New msg sync from %s" % self.getvisiblename(), - args = (uid, dstfolder, statusfolder)) - thread.setDaemon(1) - thread.start() - threads.append(thread) - else: - self.syncmessagesto_neguid_msg(uid, dstfolder, statusfolder, - register = 0) - #wait for all uploads to finish - for thread in threads: - thread.join() - def copymessageto(self, uid, dstfolder, statusfolder, register = 1): """Copies a message from self to dst if needed, updating the status @@ -311,33 +236,46 @@ class BaseFolder: # self.getmessage(). So, don't call self.getmessage unless # really needed. try: - if register: + if register: # output that we start a new thread self.ui.registerthread(self.getaccountname()) message = None flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - if dstfolder.uidexists(uid): + if uid > 0 and dstfolder.uidexists(uid): # dst has message with that UID already, only update status statusfolder.savemessage(uid, None, flags, rtime) return - # really need to copy to dst... self.ui.copyingmessage(uid, self, [dstfolder]) - # If any of the destinations actually stores the message body, # load it up. if dstfolder.storesmessages(): - message = self.getmessage(uid) + message = self.getmessage(uid) + #Succeeded? -> IMAP actually assigned a UID. If newid + #remained negative, no server was willing to assign us an + #UID. If newid is 0, saving succeeded, but we could not + #retrieve the new UID. Ignore message in this case. newuid = dstfolder.savemessage(uid, message, flags, rtime) - if newuid > 0 and newuid != uid: - # Change the local uid. - self.savemessage(newuid, message, flags, rtime) - self.deletemessage(uid) - uid = newuid - statusfolder.savemessage(uid, message, flags, rtime) + if newuid > 0: + if newuid != uid: + # Got new UID, change the local uid. + #TODO: Maildir could do this with a rename rather than + #load/save/del operation, IMPLEMENT a changeuid() + #function or so. + self.savemessage(newuid, message, flags, rtime) + self.deletemessage(uid) + uid = newuid + # Save uploaded status in the statusfolder + statusfolder.savemessage(uid, message, flags, rtime) + else: + raise UserWarning("Trying to save msg (uid %d) on folder " + "%s returned invalid uid %d" % \ + (uid, + dstfolder.getvisiblename(), + newuid)) except (KeyboardInterrupt): raise except: @@ -347,10 +285,10 @@ class BaseFolder: raise def syncmessagesto_copy(self, dstfolder, statusfolder): - """Pass2: Copy locally existing messages + """Pass1: Copy locally existing messages not on the other side - This will copy messages with a valid UID but are not on the - other side yet. The strategy is: + This will copy messages to dstfolder that exist locally but are + not in the statusfolder yet. The strategy is: 1) Look for messages present in self but not in statusfolder. 2) invoke copymessageto() on those which: @@ -359,7 +297,7 @@ class BaseFolder: """ threads = [] - copylist = filter(lambda uid: uid>=0 and not \ + copylist = filter(lambda uid: not \ statusfolder.uidexists(uid), self.getmessageuidlist()) for uid in copylist: @@ -381,7 +319,7 @@ class BaseFolder: thread.join() def syncmessagesto_delete(self, dstfolder, statusfolder): - """Pass 3: Remove locally deleted messages on dst + """Pass 2: Remove locally deleted messages on dst Get all UIDS in statusfolder but not self. These are messages that were deleted in 'self'. Delete those from dstfolder and @@ -397,7 +335,7 @@ class BaseFolder: folder.deletemessages(deletelist) def syncmessagesto_flags(self, dstfolder, statusfolder): - """Pass 4: Flag synchronization + """Pass 3: Flag synchronization Compare flag mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been @@ -449,15 +387,12 @@ class BaseFolder: This is the high level entry for syncing messages in one direction. Syncsteps are: - Pass1: Transfer new local messages - Upload msg with negative/no UIDs to dstfolder. dstfolder - might assign that message a new UID. Update statusfolder. - - Pass2: Copy locally existing messages + Pass1: Copy locally existing messages Copy messages in self, but not statusfolder to dstfolder if not - already in dstfolder. Update statusfolder. + already in dstfolder. dstfolder might assign a new UID (e.g. if + uploading to IMAP). Update statusfolder. - Pass3: Remove locally deleted messages + Pass2: Remove locally deleted messages Get all UIDS in statusfolder but not self. These are messages that were deleted in 'self'. Delete those from dstfolder and statusfolder. @@ -466,18 +401,16 @@ class BaseFolder: uids present (except for potential negative uids that couldn't be placed anywhere). - Pass4: Synchronize flag changes + Pass3: Synchronize flag changes Compare flag mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to both dstfolder and statusfolder. - :param dstfolder: Folderinstance to sync the msgs to. :param statusfolder: LocalStatus instance to sync against. """ - passes = [('uploading negative UIDs', self.syncmessagesto_neguid), - ('copying messages' , self.syncmessagesto_copy), + passes = [('copying messages' , self.syncmessagesto_copy), ('deleting messages' , self.syncmessagesto_delete), ('syncing flags' , self.syncmessagesto_flags)] @@ -490,5 +423,4 @@ class BaseFolder: self.ui.warn("ERROR attempting to sync flags " \ + "for account " + self.getaccountname() \ + ":" + traceback.format_exc()) - raise diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 43b28e4..bd0f43a 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -242,11 +242,6 @@ class MappingFolderMixIn: self._mb.deletemessages(self, self._uidlist(self.r2l, uidlist)) self._mapped_delete(uidlist) - #def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1): - # does not need changes because it calls functions that make the changes - # same goes for all other sync messages types. - - # Define a class for local part of IMAP. class MappedIMAPFolder(MappingFolderMixIn, IMAPFolder): def __init__(self, *args, **kwargs): From fa0b1ef8eefe583e577bdfba260001352ccc6032 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 27 Apr 2011 10:37:15 +0200 Subject: [PATCH 067/817] Add beginnings of sphinx-based code documentation The docs still need some meat, but the infrastructure is in place. THis allows us to generate the nice looking API documentation that many python projects already have. We should document our API better, providing an overview of the functionality available. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- .gitignore | 1 + docs/Makefile | 9 +- docs/dev-doc-src/conf.py | 200 +++++++++++++++++++++++++++++++ docs/dev-doc-src/index.rst | 82 +++++++++++++ docs/dev-doc-src/offlineimap.rst | 74 ++++++++++++ 5 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 docs/dev-doc-src/conf.py create mode 100644 docs/dev-doc-src/index.rst create mode 100644 docs/dev-doc-src/offlineimap.rst diff --git a/.gitignore b/.gitignore index 553b655..9307862 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Generated files +/docs/dev-doc/ /build/ *.pyc offlineimap.1 diff --git a/docs/Makefile b/docs/Makefile index 70a99a2..8fd880d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -7,8 +7,9 @@ HTML_TARGETS = $(patsubst %.rst,%.html,$(SOURCES)) RM = rm RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` RST2MAN=`type rst2man >/dev/null 2>&1 && echo rst2man || echo rst2man.py` +SPHINXBUILD = sphinx-build -all: html +all: html dev-doc html: $(HTML_TARGETS) @@ -21,6 +22,12 @@ offlineimap.1: MANUAL.rst $(RST2MAN) MANUAL.rst offlineimap.1 cp -f offlineimap.1 .. +dev-doc: + $(SPHINXBUILD) -b html -d dev-doc/doctrees dev-doc-src dev-doc/html + clean: $(RM) -f $(HTML_TARGETS) $(RM) -f offlineimap.1 ../offlineimap.1 + $(RM) -rf dev-doc/* + +.PHONY: dev-doc diff --git a/docs/dev-doc-src/conf.py b/docs/dev-doc-src/conf.py new file mode 100644 index 0000000..9269a51 --- /dev/null +++ b/docs/dev-doc-src/conf.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# pyDNS documentation build configuration file, created by +# sphinx-quickstart on Tue Feb 2 10:00:47 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0,os.path.abspath('../..')) + +from offlineimap import __version__,__author__ +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo'] +autoclass_content = "both" + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'OfflineImap' +copyright = u'2002-2010, ' + __author__ + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = __version__ +# The full version, including alpha/beta/rc tags. +release = __version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = False + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['../html'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +html_use_modindex = False + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'offlineimapdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'offlineimap.tex', u'OfflineImap Documentation', + u'OfflineImap contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/dev-doc-src/index.rst b/docs/dev-doc-src/index.rst new file mode 100644 index 0000000..dff8620 --- /dev/null +++ b/docs/dev-doc-src/index.rst @@ -0,0 +1,82 @@ +.. OfflineImap documentation master file + +.. currentmodule:: offlineimap + +Welcome to :mod:`offlineimaps`'s documentation +=========================================== + +The :mod:`offlineimap` module provides the user interface for synchronization between IMAP servers and MailDirs or between IMAP servers. The homepage containing the source code repository can be found at the `offlineimap wiki `_. + +Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. A folder is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative :mod:`offlineimap.folder`. + +.. moduleauthor:: John Goerzen, Sebastian Spaeth + +:License: This module is covered under the GNU GPL v2 (or later). + +This page contains the main API overview of OfflineImap |release|. + +Notmuch can be imported as:: + + from offlineimap import OfflineImap + +More information on specific topics can be found on the following pages: + +.. toctree:: + :maxdepth: 1 + + offlineimap + +:mod:`offlineimap` -- The OfflineImap module +================================================= + +.. automodule:: offlineimap + +.. autoclass:: offlineimap.OfflineImap(cmdline_opts = None) + + .. automethod:: parse_commandline + + .. automethod:: write_pidfile + + .. automethod:: delete_pidfile + + .. automethod:: lock + + .. automethod:: unlock + + .. autoattribute:: ui + + :todo: Document + +:mod:`offlineimap.folder` -- Basic representation of a local or remote Mail folder +--------------------------------------------------------------------------------------------------------- + +.. autoclass:: offlineimap.folder.Base.BaseFolder + :members: + :inherited-members: + +.. .. note:: :meth:`foo` +.. .. attribute:: Database.MODE + + Defines constants that are used as the mode in which to open a database. + + MODE.READ_ONLY + Open the database in read-only mode + + MODE.READ_WRITE + Open the database in read-write mode + + +:exc:`OfflineImapException` -- A Notmuch execution error +------------------------------------------------ +.. autoexception:: offlineimap.OfflineImapException + :members: + + This execption inherits directly from :exc:`Exception` and is raised on errors during the notmuch execution. + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` + diff --git a/docs/dev-doc-src/offlineimap.rst b/docs/dev-doc-src/offlineimap.rst new file mode 100644 index 0000000..f206c11 --- /dev/null +++ b/docs/dev-doc-src/offlineimap.rst @@ -0,0 +1,74 @@ +The offlineimap 'binary' +======================== + +Offlineimap is invoked with the following pattern: `offlineimap [args...]`. + +Where [args...] are as follows: + +Options: + --version show program's version number and exit + -h, --help show this help message and exit + -1 Disable all multithreading operations and use solely a + single-thread sync. This effectively sets the + maxsyncaccounts and all maxconnections configuration + file variables to 1. + -P DIR Sets OfflineIMAP into profile mode. The program will + create DIR (it must not already exist). As it runs, + Python profiling information about each thread is + logged into profiledir. Please note: This option is + present for debugging and optimization only, and + should NOT be used unless you have a specific reason + to do so. It will significantly slow program + performance, may reduce reliability, and can generate + huge amounts of data. This option implies the + singlethreading option (-1). + -a ACCOUNTS Overrides the accounts section in the config file. + Lets you specify a particular account or set of + accounts to sync without having to edit the config + file. You might use this to exclude certain accounts, + or to sync some accounts that you normally prefer not + to. + -c FILE Specifies a configuration file to use in lieu of + ~/.offlineimaprc. + + -d type1,[type2...] Enables debugging for OfflineIMAP. This is useful if + you are trying to track down a malfunction or figure + out what is going on under the hood. I suggest that + you use this with -1 in order to make the results more + sensible. This option requires one or more debugtypes, + separated by commas. These define what exactly will + be debugged, and so far include the options: imap, + thread,maildir or ALL. The imap option will enable + IMAP protocol stream and parsing debugging. Note that + the output may contain passwords, so take care to + remove that from the debugging output before sending + it to anyone else. The maildir option will enable + debugging for certain Maildir operations. + + -l FILE Log to FILE + + -f folder1,[folder2...] + Only sync the specified folders. The 'folder's are the + *untranslated* foldernames. This command-line option + overrides any 'folderfilter' and 'folderincludes' + options in the configuration file. + + -k `[section:]option=value` + Override configuration file option. If"section" is + omitted, it defaults to "general". Any underscores + "_" in the section name are replaced with spaces: + for instance, to override option "autorefresh" in + the "[Account Personal]" section in the config file + one would use "-k Account_Personal:autorefresh=30". + + -o Run only once, ignoring any autorefresh setting in the + configuration file. + -q Run only quick synchronizations. Ignore any flag + updates on IMAP servers. + -u INTERFACE Specifies an alternative user interface to use. This + overrides the default specified in the configuration + file. The UI specified with -u will be forced to be + used, even if checks determine that it is not usable. + Possible interface choices are: Curses.Blinkenlights, + TTY.TTYUI, Noninteractive.Basic, Noninteractive.Quiet, + Machine.MachineUI From bb8546e846a23bc2b26af587c95073f5cb6192a4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 27 Apr 2011 12:15:49 +0200 Subject: [PATCH 068/817] Whitespace fixes Just clean up lines which end in whitespaces. Also adapt the copyright to the current year while touching the file. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 991a39b..b3fd747 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -1,6 +1,5 @@ # Base folder support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,7 +24,7 @@ import traceback class BaseFolder: def __init__(self): self.ui = getglobalui() - + def getname(self): """Returns name""" return self.name @@ -74,7 +73,7 @@ class BaseFolder: return self.getroot() + self.getsep() + self.getname() else: return self.getname() - + def getfolderbasename(self): foldername = self.getname() foldername = foldername.replace(self.repository.getsep(), '.') @@ -97,7 +96,7 @@ class BaseFolder: def _getuidfilename(self): return os.path.join(self.repository.getuiddir(), self.getfolderbasename()) - + def getsaveduidvalidity(self): if hasattr(self, '_base_saved_uidvalidity'): return self._base_saved_uidvalidity @@ -242,7 +241,7 @@ class BaseFolder: message = None flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - + if uid > 0 and dstfolder.uidexists(uid): # dst has message with that UID already, only update status statusfolder.savemessage(uid, None, flags, rtime) @@ -250,7 +249,7 @@ class BaseFolder: self.ui.copyingmessage(uid, self, [dstfolder]) # If any of the destinations actually stores the message body, - # load it up. + # load it up. if dstfolder.storesmessages(): message = self.getmessage(uid) @@ -380,7 +379,7 @@ class BaseFolder: self.ui.deletingflags(delflaglist[flag], flag, dstfolder) dstfolder.deletemessagesflags(delflaglist[flag], [flag]) statusfolder.deletemessagesflags(delflaglist[flag], [flag]) - + def syncmessagesto(self, dstfolder, statusfolder): """Syncs messages in this folder to the destination dstfolder. From 074ea14cf1141ff3f20c6562fac0c60b8d257024 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 27 Apr 2011 12:15:50 +0200 Subject: [PATCH 069/817] Make str() of Account() be its name So we can simply hand an account instance to error messages rather than having to call Account().getname() all the time. This is not used yet. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 7edfa37..a22bf7f 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -125,6 +125,9 @@ class Account(CustomConfig.ConfigHelperMixin): def getname(self): return self.name + def __str__(self): + return self.name + def getsection(self): return 'Account ' + self.getname() From 3e5d6f54501ac7335ad81836914454c10211b4f7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 27 Apr 2011 12:15:51 +0200 Subject: [PATCH 070/817] Make str() of Repository() be its name So we can simply hand a repository instance to error messages rather than having to call Repository().getname() all the time. This is not used yet. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 482cc2d..594f864 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -72,6 +72,9 @@ class BaseRepository(CustomConfig.ConfigHelperMixin): def getname(self): return self.name + def __str__(self): + return self.name + def getuiddir(self): return self.uiddir From e01968973decba461678836856d3ea4862885202 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 27 Apr 2011 12:15:52 +0200 Subject: [PATCH 071/817] Use "%s" % account rather than account.name uncritical patch, but we can make the code a bit shorter so why not. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index a22bf7f..f94a55c 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -256,7 +256,7 @@ class SyncableAccount(Account): thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, - name = "Folder sync [%s]" % self.name, + name = "Folder sync [%s]" % self, args = (self.name, remoterepos, remotefolder, localrepos, statusrepos, quick)) thread.setDaemon(1) From c8f80bc6d2dcb6d28bdff48dca34db025b76972f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 26 Apr 2011 12:31:32 +0200 Subject: [PATCH 072/817] We had been setting this variable twice This was in the code for a very long time, it seems. Remove one instance, no functional changes. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatus.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 623829a..7390dc1 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -29,7 +29,6 @@ class LocalStatusFolder(BaseFolder): self.sep = '.' self.config = config self.dofsync = config.getdefaultboolean("general", "fsync", True) - self.filename = os.path.join(root, name) self.filename = repository.getfolderfilename(name) self.messagelist = None self.repository = repository From 1ff628bd5cf6a36ff1925c246f6975d21432641a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 26 Apr 2011 15:18:54 +0200 Subject: [PATCH 073/817] folder/IMAP: cleanup getmessage() Add some comments how the data structures actually look like. Describe the function properly, and make sure we only hold on to the data connection as quickly as possible. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index f2d0dda..a5c71b5 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -209,16 +209,25 @@ class IMAPFolder(BaseFolder): return self.messagelist def getmessage(self, uid): + """Retrieve message with UID from the IMAP server (incl body) + + :returns: the message body + """ imapobj = self.imapserver.acquireconnection() try: imapobj.select(self.getfullname(), readonly = 1) - initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])') - self.ui.debug('imap', 'Returned object from fetching %d: %s' % \ - (uid, str(initialresult))) - return initialresult[1][0][1].replace("\r\n", "\n") - + res_type, data = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])') finally: self.imapserver.releaseconnection(imapobj) + assert res_type == 'OK', "Fetching message with UID '%d' failed" % uid + # data looks now e.g. [('320 (UID 17061 BODY[] + # {2565}','msgbody....')] we only asked for one message, + # and that msg is in data[0]. msbody is in [0][1] + data = data[0][1] + self.ui.debug('imap', '%s bytes returned from fetching %d: %s...%s' % \ + (len(data), uid, data[0:100], data[-100:])) + return data.replace("\r\n", "\n") + def getmessagetime(self, uid): return self.messagelist[uid]['time'] From 163f1eefc482e038195bf24d8b717df9f305abdd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 27 Apr 2011 11:44:24 +0200 Subject: [PATCH 074/817] Drop all connections when a sync failed in accounts.sync() we would holdordropconnections() after each sync. But depending on the repository configuration that might imply that offlineimap tries to keep the same connections. But when a sync failed, e.g. after a user had his computer suspended, it might be that our connections that we have are worthless. So definitely drop them after a failed sync. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +++ offlineimap/accounts.py | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 69169c1..19788af 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -23,6 +23,9 @@ Changes Bug Fixes --------- +* Drop connection if synchronisation failed. This is needed if resuming the + system from suspend mode gives a wrong connection. + Pending for the next major release ================================== diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index f94a55c..6b706de 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -268,7 +268,14 @@ class SyncableAccount(Account): mbnames.write() localrepos.forgetfolders() remoterepos.forgetfolders() - finally: + except: + #error while syncing. Drop all connections that we have, they + #might be bogus by now (e.g. after suspend) + localrepos.dropconnections() + remoterepos.dropconnections() + raise + else: + # sync went fine. Hold or drop depending on config localrepos.holdordropconnections() remoterepos.holdordropconnections() From 9bd7a21f16fb3a57cbf141b27f5d3f886dd82854 Mon Sep 17 00:00:00 2001 From: Dan Christensen Date: Thu, 28 Apr 2011 15:26:07 +0200 Subject: [PATCH 075/817] Don't strip whitespace in the -f option Allow leading and trailing spaces in folder names specified on the command line. Reviewed-by: Sebastian Spaeth Signed-off-by: Dan Christensen Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +++ offlineimap/init.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 0a85888..d0125b7 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,9 @@ New Features Changes ------- +- no whitespace is stripped from comma-separated arguments passed via + the -f option. + Bug Fixes --------- diff --git a/offlineimap/init.py b/offlineimap/init.py index 296b84b..84983a4 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -240,8 +240,7 @@ class OfflineImap: #custom folder list specified? if options.folders: - foldernames = map(lambda s: s.strip(), - options.folders.split(",")) + foldernames = options.folders.split(",") folderfilter = "lambda f: f in %s" % foldernames folderincludes = "[]" for accountname in accounts.getaccountlist(config): From 2ed1c357a06bb7de2e1eacc3feaebe627c54be3b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 2 May 2011 11:44:19 +0200 Subject: [PATCH 076/817] More detailed error output on corrupt UID mapping files This function will need much more "robustifying", but the very least we can do is to print the file name and line that are giving trouble. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ offlineimap/folder/UIDMaps.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 0a85888..2b109a3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,8 @@ New Features Changes ------- +* Give more detailed error when encountering a corrupt UID mapping file. + Bug Fixes --------- diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 43b28e4..e87e21e 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -51,7 +51,11 @@ class MappingFolderMixIn: line = file.readline() if not len(line): break - line = line.strip() + try: + line = line.strip() + except ValueError: + raise Exception("Corrupt line '%s' in UID mapping file '%s'" \ + %(line, mapfilename)) (str1, str2) = line.split(':') loc = long(str1) rem = long(str2) From 6add201436ff468d24ac2e75dac3b8aef43bdf1d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 2 May 2011 17:11:40 +0200 Subject: [PATCH 077/817] Improve the developer API documentation Improve the code documentation (still much more to do) and also add some more meat to the structure of the developer documentation. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- docs/dev-doc-src/conf.py | 4 +- docs/dev-doc-src/index.rst | 58 +++++++++++---------------- docs/dev-doc-src/offlineimap.rst | 13 +++++- docs/dev-doc-src/repository.rst | 68 ++++++++++++++++++++++++++++++++ docs/dev-doc-src/ui.rst | 27 +++++++++++++ offlineimap/accounts.py | 19 ++++++++- offlineimap/init.py | 7 ++-- offlineimap/ui/UIBase.py | 2 + 8 files changed, 153 insertions(+), 45 deletions(-) create mode 100644 docs/dev-doc-src/repository.rst create mode 100644 docs/dev-doc-src/ui.rst diff --git a/docs/dev-doc-src/conf.py b/docs/dev-doc-src/conf.py index 9269a51..f583a9c 100644 --- a/docs/dev-doc-src/conf.py +++ b/docs/dev-doc-src/conf.py @@ -122,7 +122,7 @@ html_theme = 'default' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['../html'] +#html_static_path = ['html'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -160,7 +160,7 @@ html_use_modindex = False #html_file_suffix = '' # Output file base name for HTML help builder. -htmlhelp_basename = 'offlineimapdoc' +htmlhelp_basename = 'dev-doc' # -- Options for LaTeX output -------------------------------------------------- diff --git a/docs/dev-doc-src/index.rst b/docs/dev-doc-src/index.rst index dff8620..f4d7067 100644 --- a/docs/dev-doc-src/index.rst +++ b/docs/dev-doc-src/index.rst @@ -3,13 +3,13 @@ .. currentmodule:: offlineimap Welcome to :mod:`offlineimaps`'s documentation -=========================================== +============================================== -The :mod:`offlineimap` module provides the user interface for synchronization between IMAP servers and MailDirs or between IMAP servers. The homepage containing the source code repository can be found at the `offlineimap wiki `_. +The :mod:`offlineimap` module provides the user interface for synchronization between IMAP servers and MailDirs or between IMAP servers. The homepage containing the source code repository can be found at the `offlineimap homepage `_. Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. A folder is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative :mod:`offlineimap.folder`. -.. moduleauthor:: John Goerzen, Sebastian Spaeth +.. moduleauthor:: John Goerzen, and many others. See AUTHORS and the git history for a full list. :License: This module is covered under the GNU GPL v2 (or later). @@ -24,59 +24,45 @@ More information on specific topics can be found on the following pages: .. toctree:: :maxdepth: 1 - offlineimap + repository + ui + offlineimap :mod:`offlineimap` -- The OfflineImap module -================================================= +============================================= -.. automodule:: offlineimap +.. module:: offlineimap .. autoclass:: offlineimap.OfflineImap(cmdline_opts = None) - .. automethod:: parse_commandline - - .. automethod:: write_pidfile - - .. automethod:: delete_pidfile - .. automethod:: lock - .. automethod:: unlock + .. automethod:: run - .. autoattribute:: ui +.. .. autoattribute:: ui :todo: Document -:mod:`offlineimap.folder` -- Basic representation of a local or remote Mail folder ---------------------------------------------------------------------------------------------------------- +:class:`offlineimap.account` +============================ -.. autoclass:: offlineimap.folder.Base.BaseFolder +An :class:`accounts.Account` connects two email repositories that are to be synced. It comes in two flavors, normal and syncable. + +.. autoclass:: offlineimap.accounts.Account + +.. autoclass:: offlineimap.accounts.SyncableAccount :members: :inherited-members: -.. .. note:: :meth:`foo` -.. .. attribute:: Database.MODE + .. autodata:: ui - Defines constants that are used as the mode in which to open a database. - - MODE.READ_ONLY - Open the database in read-only mode - - MODE.READ_WRITE - Open the database in read-write mode + Contains the current :mod:`offlineimap.ui`, and can be used for logging etc. :exc:`OfflineImapException` -- A Notmuch execution error ------------------------------------------------- +-------------------------------------------------------- + .. autoexception:: offlineimap.OfflineImapException :members: - This execption inherits directly from :exc:`Exception` and is raised on errors during the notmuch execution. - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`search` - + This execption inherits directly from :exc:`Exception` and is raised on errors during the offlineimap execution. diff --git a/docs/dev-doc-src/offlineimap.rst b/docs/dev-doc-src/offlineimap.rst index f206c11..bc98c16 100644 --- a/docs/dev-doc-src/offlineimap.rst +++ b/docs/dev-doc-src/offlineimap.rst @@ -1,5 +1,5 @@ -The offlineimap 'binary' -======================== +The offlineimap 'binary' command line options +============================================= Offlineimap is invoked with the following pattern: `offlineimap [args...]`. @@ -72,3 +72,12 @@ Options: Possible interface choices are: Curses.Blinkenlights, TTY.TTYUI, Noninteractive.Basic, Noninteractive.Quiet, Machine.MachineUI + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` + diff --git a/docs/dev-doc-src/repository.rst b/docs/dev-doc-src/repository.rst new file mode 100644 index 0000000..04025cd --- /dev/null +++ b/docs/dev-doc-src/repository.rst @@ -0,0 +1,68 @@ +.. currentmodule:: offlineimap.repository + +:mod:`offlineimap.repository` -- Email repositories +------------------------------------------------------------ + +A derivative of class +:class:`Base.BaseRepository` represents an email +repository depending on the type of storage, possible options are: + + * :class:`IMAPRepository`, + * :class:`MappedIMAPRepository` + * :class:`GmailRepository`, + * :class:`MaildirRepository`, or + * :class:`LocalStatusRepository`. + +Which class you need depends on your account +configuration. The helper class :class:`offlineimap.repository.Repository` is +an *autoloader*, that returns the correct class depending +on your configuration. So when you want to instanciate a new +:mod:`offlineimap.repository`, you will mostly do it through this class. + +.. autoclass:: offlineimap.repository.Repository + :members: + :inherited-members: + + + +:mod:`offlineimap.repository` -- Basic representation of a mail repository +-------------------------------------------------------------------------- +.. autoclass:: offlineimap.repository.Base.BaseRepository + :members: + :inherited-members: + :undoc-members: + +.. .. note:: :meth:`foo` +.. .. attribute:: Database.MODE + + Defines constants that are used as the mode in which to open a database. + + MODE.READ_ONLY + Open the database in read-only mode + + MODE.READ_WRITE + Open the database in read-write mode + +.. autoclass:: offlineimap.repository.IMAPRepository +.. autoclass:: offlineimap.repository.MappedIMAPRepository +.. autoclass:: offlineimap.repository.GmailRepository +.. autoclass:: offlineimap.repository.MaildirRepository +.. autoclass:: offlineimap.repository.LocalStatusRepository + +:mod:`offlineimap.folder` -- Basic representation of a local or remote Mail folder +--------------------------------------------------------------------------------------------------------- + +.. autoclass:: offlineimap.folder.Base.BaseFolder + :members: + :inherited-members: + :undoc-members: + +.. .. attribute:: Database.MODE + + Defines constants that are used as the mode in which to open a database. + + MODE.READ_ONLY + Open the database in read-only mode + + MODE.READ_WRITE + Open the database in read-write mode diff --git a/docs/dev-doc-src/ui.rst b/docs/dev-doc-src/ui.rst new file mode 100644 index 0000000..03da19b --- /dev/null +++ b/docs/dev-doc-src/ui.rst @@ -0,0 +1,27 @@ +:mod:`offlineimap.ui` -- A pluggable logging system +-------------------------------------------------------- + +.. currentmodule:: offlineimap.ui + +OfflineImap has various ui systems, that can be selected. They offer various functionalities. They must implement all functions that the :class:`offlineimap.ui.UIBase` offers. Early on, the ui must be set using :meth:`getglobalui` + +.. automethod:: offlineimap.ui.setglobalui +.. automethod:: offlineimap.ui.getglobalui + +Base UI plugin +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: offlineimap.ui.UIBase.UIBase + :members: + :inherited-members: + +.. .. note:: :meth:`foo` +.. .. attribute:: Database.MODE + + Defines constants that are used as the mode in which to open a database. + + MODE.READ_ONLY + Open the database in read-only mode + + MODE.READ_WRITE + Open the database in read-write mode diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 6b706de..c366810 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -105,11 +105,24 @@ def AccountHashGenerator(customconfig): class Account(CustomConfig.ConfigHelperMixin): + """Represents an account (ie. 2 repositories) to sync + + Most of the time you will actually want to use the derived + :class:`accounts.SyncableAccount` which contains all functions used + for syncing an account.""" + def __init__(self, config, name): + """ + :param config: Representing the offlineimap configuration file. + :type config: :class:`offlineimap.CustomConfig.CustomConfigParser` + + :param name: A string denoting the name of the Account + as configured""" self.config = config self.name = name self.metadatadir = config.getmetadatadir() self.localeval = config.getlocaleval() + #Contains the current :mod:`offlineimap.ui`, and can be used for logging etc. self.ui = getglobalui() self.refreshperiod = self.getconffloat('autorefresh', 0.0) self.quicknum = 0 @@ -171,9 +184,11 @@ class Account(CustomConfig.ConfigHelperMixin): class SyncableAccount(Account): - """A syncable IMAP account. + """A syncable email account connecting 2 repositories - Derives from class:`Account`.""" + Derives from :class:`accounts.Account` but contains the additional + functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`, + used for syncing.""" def syncrunner(self, siglistener): self.ui.registerthread(self.name) diff --git a/offlineimap/init.py b/offlineimap/init.py index 84983a4..54e7a27 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -41,9 +41,10 @@ lockfd = None class OfflineImap: """The main class that encapsulates the high level use of OfflineImap. - To invoke OfflineImap you would call it with: - oi = OfflineImap() - oi.run() + To invoke OfflineImap you would call it with:: + + oi = OfflineImap() + oi.run() """ def lock(self, config, ui): global lockfd, hasfcntl diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 02593e6..150818d 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -31,9 +31,11 @@ debugtypes = {'imap': 'IMAP protocol debugging', globalui = None def setglobalui(newui): + """Set the global ui object to be used for logging""" global globalui globalui = newui def getglobalui(): + """Return the current ui object""" global globalui return globalui From 3d4dc11a8bd9482aabefe685d797955e2d4332b1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 1 May 2011 20:18:28 +0200 Subject: [PATCH 078/817] Introduce an *empty* debug type This debug type will always be enabled whenever any debugging is enables and it outputs debug messages that cannot be categorized among any of imap, maildir (e.g. things that concern the sync logic). Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 3 ++- offlineimap/ui/UIBase.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 54e7a27..d160bbf 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -222,7 +222,8 @@ class OfflineImap: ui._msg("Debug mode: Forcing to singlethreaded.") options.singlethreaded = True - for type in options.debugtype.split(','): + debugtypes = options.debugtype.split(',') + [''] + for type in debugtypes: type = type.strip() ui.add_debug(type) if type.lower() == 'imap': diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 150818d..319000c 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -25,7 +25,8 @@ from StringIO import StringIO from Queue import Empty import offlineimap -debugtypes = {'imap': 'IMAP protocol debugging', +debugtypes = {'':'Other offlineimap related sync messages', + 'imap': 'IMAP protocol debugging', 'maildir': 'Maildir repository debugging', 'thread': 'Threading debugging'} From f3249e0856c414c3041b5ade03762e19e738ea64 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 1 May 2011 20:18:29 +0200 Subject: [PATCH 079/817] True 1-way sync (backup) This commit enables true 1-way syncing between repositories. This has often been demanded for backup purposes when you do not want to cause accidental modifications of your backup that would be propagated to the other side. This has been implemented by allowing to configure a Repository as 'readonly' to forbid any modification on it. 'readonly' applies to all the type of repositories. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 5 +++++ offlineimap.conf | 25 ++++++++++++++++--------- offlineimap/accounts.py | 24 +++++++++++++++++------- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index ed3a14e..ff559a0 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,11 @@ others. New Features ------------ +* Enable 1-way synchronization by settting a [Repository ...] to + readonly = True. When e.g. using offlineimap for backup purposes you + can thus make sure that no changes in your backup trickle back into + the main IMAP server. + Changes ------- diff --git a/offlineimap.conf b/offlineimap.conf index 7018090..f8506b2 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -1,6 +1,5 @@ # Sample configuration file -# Copyright (C) 2002-2005 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -67,13 +66,14 @@ maxsyncaccounts = 1 ui = Blinkenlights -# If you try to synchronize messages to a read-only folder, -# OfflineIMAP will generate a warning. If you want to suppress these -# warnings, set ignore-readonly to yes. Read-only IMAP folders allow -# reading but not modification, so if you try to change messages in -# the local copy of such a folder, the IMAP server will prevent -# OfflineIMAP from propagating those changes to the IMAP server. - +# If you try to synchronize messages to a folder which the IMAP server +# considers read-only, OfflineIMAP will generate a warning. If you want +# to suppress these warnings, set ignore-readonly to yes. Read-only +# IMAP folders allow reading but not modification, so if you try to +# change messages in the local copy of such a folder, the IMAP server +# will prevent OfflineIMAP from propagating those changes to the IMAP +# server. Note that ignore-readonly is unrelated to the "readonly" +# setting which prevents a repository from being modified at all. ignore-readonly = no ########## Advanced settings @@ -469,6 +469,11 @@ subscribedonly = no # # foldersort = lambda x, y: -cmp(x, y) +# Enable 1-way synchronization. When setting 'readonly' to True, this +# repository will not be modified during synchronization. Use to +# e.g. backup an IMAP server. The readonly setting can be applied to any +# type of Repository (Maildir, Imap, etc). +readonly = False [Repository GmailExample] @@ -509,3 +514,5 @@ realdelete = no # # spamfolder = [Google Mail]/Spam +# Enable 1-way synchronization. See above for explanation. +readonly = False diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index c366810..0b71377 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -260,8 +260,10 @@ class SyncableAccount(Account): remoterepos = self.remoterepos localrepos = self.localrepos statusrepos = self.statusrepos - self.ui.syncfolders(remoterepos, localrepos) - remoterepos.syncfoldersto(localrepos, [statusrepos]) + # replicate the folderstructure from REMOTE to LOCAL + if not localrepos.getconf('readonly', False): + self.ui.syncfolders(remoterepos, localrepos) + remoterepos.syncfoldersto(localrepos, [statusrepos]) siglistener.addfolders(remoterepos.getfolders(), bool(self.refreshperiod), quick) @@ -374,12 +376,20 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, remotefolder.getmessagecount()) # Synchronize remote changes. - ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) - remotefolder.syncmessagesto(localfolder, statusfolder) + if not localrepos.getconf('readonly', False): + ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) + remotefolder.syncmessagesto(localfolder, statusfolder) + else: + ui.debug('imap', "Not syncing to read-only repository '%s'" \ + % localrepos.getname()) + # Synchronize local changes - ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) - localfolder.syncmessagesto(remotefolder, statusfolder) - + if not remoterepos.getconf('readonly', False): + ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) + localfolder.syncmessagesto(remotefolder, statusfolder) + else: + ui.debug('', "Not syncing to read-only repository '%s'" \ + % remoterepos.getname()) statusfolder.save() localrepos.restore_atime() From 01b390bb71857f22efebcd56f2416a8df7b624f4 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 4 May 2011 19:03:35 +0200 Subject: [PATCH 080/817] INSTALL: explain how to install offlineimap using git Signed-off-by: Nicolas Sebrecht --- docs/INSTALL.rst | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index bf0fa80..684ca6f 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -58,27 +58,42 @@ program. System-Wide Installation, Other =============================== -Download the tar.gz version of the package from the website. Then run these -commands, making sure that you are the "root" user first:: +Check your distribution packaging tool, OfflineIMAP may already be packaged for +you. - tar -zxvf offlineimap_x.y.z.tar.gz - cd offlineimap-x.y.z - python2.2 setup.py install +System-Wide Installation, From source +===================================== -On some systems, you will need to use python instead of python2.6. Next, -proceed to below. You will type offlineimap to invoke the program. +Get your own copy of the official git repository at `OfflineIMAP`_:: + + git clone git://github.com/nicolas33/offlineimap.git + +This will download all the sources with history. By default, git set up the +local master branch up which is most likely what you want. If not, you can +checkout a particular release:: + + cd offlineimap + git checkout -b local_version v6.3.3 + +The latter creates a local branch called "local_version" of the v6.3.3 release. + +Then run these commands, to build the python package:: + + make clean + make + +Finally, install the program (as root):: + + python setup.py install + +Next, proceed to below. You will type offlineimap to invoke the program. Single-Account Installation =========================== -Download the tar.gz version of the package from the website. Then run these -commands:: - - tar -zxvf offlineimap_x.y.z.tar.gz - cd offlineimap-x.y.z - -When you want to run `OfflineIMAP`_, you will issue the cd command as above and -then type `./offlineimap.py`; there is no installation step necessary. +Download the git repository as described above. Instead of installing the +program as root, you type `./offlineimap.py`; there is no installation step +necessary. ============= Configuration From 3446b592f3886100a41f283ca6103a32a5111caf Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 May 2011 16:45:26 +0200 Subject: [PATCH 081/817] accounts: remove duplicated code paths Currently, account.syncrunner() has 2 separate duplicated code paths depending on whether we want to autorefresh after some waiting perios or not. Unify those code paths by setting "looping = False" in case self.refeshperiod == 0 after the first run. Behavior is identical to before. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 0b71377..f55a18d 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -202,22 +202,9 @@ class SyncableAccount(Account): self.localrepos = Repository(self, 'local') self.statusrepos = Repository(self, 'status') - #might need changes here to ensure that one account sync does not crash others... - if not self.refreshperiod: - try: - try: - self.sync(siglistener) - except (KeyboardInterrupt, SystemExit): - raise - except: - self.ui.warn("Error occured attempting to sync account " + self.name \ - + ": " + traceback.format_exc()) - finally: - self.ui.acctdone(self.name) - - return - - + # Might need changes here to ensure that one account sync does + # not crash others... + # Loop account synchronization if needed looping = 1 while looping: try: @@ -229,7 +216,7 @@ class SyncableAccount(Account): self.ui.warn("Error occured attempting to sync account " + self.name \ + ": " + traceback.format_exc()) finally: - looping = self.sleeper(siglistener) != 2 + looping = self.refreshperiod and self.sleeper(siglistener) != 2 self.ui.acctdone(self.name) From bc004a911a3e1058cc69e937b1ff150718710828 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 May 2011 16:45:24 +0200 Subject: [PATCH 082/817] Add OfflineImapError class This Exception can be thrown whenever a sync error occurs. It is used for outputting sensible error messages to the user and it denotes a "severity", it can tell offlineimap if we need to abort a message, a folder, a repo sync, or whether we should indeed abort immediately. The exception is not yet used in the code. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/__init__.py | 1 + offlineimap/error.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 offlineimap/error.py diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 273faef..0a55a97 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -18,6 +18,7 @@ to distribute it under the conditions laid out in COPYING.""" __homepage__ = "http://github.com/nicolas33/offlineimap" __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)." +from offlineimap.error import OfflineImapError # put this last, so we don't run into circular dependencies using # e.g. offlineimap.__version__. from offlineimap.init import OfflineImap diff --git a/offlineimap/error.py b/offlineimap/error.py new file mode 100644 index 0000000..d4d16be --- /dev/null +++ b/offlineimap/error.py @@ -0,0 +1,33 @@ +class OfflineImapError(Exception): + """An Error during offlineimap synchronization""" + + class ERROR: + """Severity levels""" + MESSAGE = 0 + FOLDER = 10 + REPO = 20 + CRITICAL = 30 + + def __init__(self, reason, severity, errcode=None): + """ + :param reason: Human readable string suitable for logging + + :param severity: denoting which operations should be + aborted. E.g. a ERROR.MESSAGE can occur on a faulty + message, but a ERROR.REPO occurs when the server is + offline. + + :param errcode: optional number denoting a predefined error + situation (which let's us exit with a predefined exit + value). So far, no errcodes have been defined yet. + + :type severity: OfflineImapError.ERROR value""" + self.errcode = errcode + self.severity = severity + + # 'reason' is stored in the Exception().args tuple. + super(OfflineImapError, self).__init__(reason) + + @property + def reason(self): + return self.args[0] From 0f45d89e34f85d02d46b06235acdb007b47de579 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 May 2011 16:45:25 +0200 Subject: [PATCH 083/817] Throw OfflineImapError on DNS error In case we misconfigured a server name or are otherwise offline, a socket.gaierror will be raised when attempting to connect. We catch that case and raise an OfflineImapError with severity ERROR.REPO, meaning we should stop syncing this account and continue with the next one. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index c474a01..6e58949 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -17,7 +17,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap import imaplib2 as imaplib -from offlineimap import imaplibutil, imaputil, threadutil +from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui from threading import * import thread @@ -28,6 +28,7 @@ import base64 from StringIO import StringIO from platform import system +from socket import gaierror try: # do we have a recent pykerberos? @@ -271,16 +272,29 @@ class IMAPServer: self.lastowner[imapobj] = thread.get_ident() self.connectionlock.release() return imapobj - except: - """If we are here then we did not succeed in getting a connection - - we should clean up and then re-raise the error...""" + except Exception, e: + """If we are here then we did not succeed in getting a + connection - we should clean up and then re-raise the + error...""" self.semaphore.release() #Make sure that this can be retried the next time... self.passworderror = None if(self.connectionlock.locked()): self.connectionlock.release() - raise + + if type(e) == gaierror: + #DNS related errors. Abort Repo sync + severity = OfflineImapError.ERROR.REPO + #TODO: special error msg for e.errno == 2 "Name or service not known"? + reason = "Could not resolve name '%s' for repository "\ + "'%s'. Make sure you have configured the ser"\ + "ver name correctly and that you are online."\ + % (self.hostname, self.reposname) + raise OfflineImapError(reason, severity) + else: + # re-raise all other errors + raise def connectionwait(self): """Waits until there is a connection available. Note that between From 4608d14836b8e72d65ae60c5861bbdd8a555a189 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 May 2011 16:45:27 +0200 Subject: [PATCH 084/817] Handle OfflineImapError of severity REPO and CRIT By aborting the account syncing, the looping and logging an error message. We will introduce a ui.error() rather than a ui.warn() function which saves all Exceptions in a Queue and outputs them at the end of the program. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index f55a18d..b76b48c 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import threadutil, mbnames, CustomConfig +from offlineimap import threadutil, mbnames, CustomConfig, OfflineImapError from offlineimap.repository import Repository from offlineimap.ui import getglobalui from offlineimap.threadutil import InstanceLimitedThread @@ -202,21 +202,28 @@ class SyncableAccount(Account): self.localrepos = Repository(self, 'local') self.statusrepos = Repository(self, 'status') - # Might need changes here to ensure that one account sync does - # not crash others... # Loop account synchronization if needed - looping = 1 + looping = True while looping: try: try: self.sync(siglistener) except (KeyboardInterrupt, SystemExit): raise + except OfflineImapError, e: + self.ui.warn(e.reason) + #stop looping and bubble up Exception if needed + if e.severity >= OfflineImapError.ERROR.REPO: + looping = 0 + if e.severity > OfflineImapError.ERROR.REPO: + raise except: - self.ui.warn("Error occured attempting to sync account " + self.name \ - + ": " + traceback.format_exc()) + self.ui.warn("Error occured attempting to sync "\ + "account '%s':\n"% (self, traceback.format_exc())) finally: - looping = self.refreshperiod and self.sleeper(siglistener) != 2 + looping = looping and \ + self.refreshperiod and \ + self.sleeper(siglistener) != 2 self.ui.acctdone(self.name) From f3774343db3a20bd1ee2db70361fdeab744e5017 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 May 2011 16:45:28 +0200 Subject: [PATCH 085/817] If we loop, exit the account synchronization after 3 failed attempts This should get rid of intermittent network failures, but lets us bail out on permanent errors. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index b76b48c..0eb49dc 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -202,8 +202,8 @@ class SyncableAccount(Account): self.localrepos = Repository(self, 'local') self.statusrepos = Repository(self, 'status') - # Loop account synchronization if needed - looping = True + # Loop account sync if needed (bail out after 3 failures) + looping = 3 while looping: try: try: @@ -214,16 +214,20 @@ class SyncableAccount(Account): self.ui.warn(e.reason) #stop looping and bubble up Exception if needed if e.severity >= OfflineImapError.ERROR.REPO: - looping = 0 - if e.severity > OfflineImapError.ERROR.REPO: + if looping: + looping -= 1 + if e.severity >= OfflineImapError.ERROR.CRITICAL: raise except: self.ui.warn("Error occured attempting to sync "\ "account '%s':\n"% (self, traceback.format_exc())) + else: + # after success sync, reset the looping counter to 3 + if self.refreshperiod: + looping = 3 finally: - looping = looping and \ - self.refreshperiod and \ - self.sleeper(siglistener) != 2 + if self.sleeper(siglistener) >= 2: + looping = 0 self.ui.acctdone(self.name) From 5c7d7ee44565c1e4ad8febc54b6d797df71ad183 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 May 2011 19:44:01 +0200 Subject: [PATCH 086/817] Output more detailed error on corrupt LocalStatus When our LocalStatus cache is corrupt, ie e.g. it contains lines not in the form number:number, we would previously just raise a ValueError stating things like "too many values". In case we encounter clearly corrupt LocalStatus cache entries, clearly raise an exception stating the filename and the line, so that people can attempt to repair it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatus.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 623829a..21cc571 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -78,8 +78,13 @@ class LocalStatusFolder(BaseFolder): assert(line == magicline) for line in file.xreadlines(): line = line.strip() - uid, flags = line.split(':') - uid = long(uid) + try: + uid, flags = line.split(':') + uid = long(uid) + except ValueError, e: + errstr = "Corrupt line '%s' in cache file '%s'" % (line, self.filename) + self.ui.warn(errstr) + raise ValueError(errstr) flags = [x for x in flags] self.messagelist[uid] = {'uid': uid, 'flags': flags} file.close() From deab62fbd84c003c5dacdf829faa1a89b9e74e9e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 May 2011 10:19:44 +0200 Subject: [PATCH 087/817] Fix the broken thread debugging Using threading._VERBOSE=1 is broken since python 2.6 till at least python 3.2, (http://bugs.python.org/issue4188) so we can't use it for our thread debugging. Remove the usage of threading._VERBOSE, and implement a "light thread debug log" that for now outputs information when a new thread is being registered and when it is being unregistered. I am sure we will be able to add more thread debugging information over the time. Besides '-d thread' this will re-enable the usage of -d 'all' for the most verbose debugging of all categories. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 1 + offlineimap/init.py | 2 -- offlineimap/ui/UIBase.py | 3 +++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 2b109a3..eacfea1 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -21,6 +21,7 @@ Changes Bug Fixes --------- +* Fix the offlineimap crash when invoking debug option 'thread' Pending for the next major release ================================== diff --git a/offlineimap/init.py b/offlineimap/init.py index 296b84b..287bbbf 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -226,8 +226,6 @@ class OfflineImap: ui.add_debug(type) if type.lower() == 'imap': imaplib.Debug = 5 - if type.lower() == 'thread': - threading._VERBOSE = 1 if options.runonce: # FIXME: maybe need a better diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 02593e6..a1cc864 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -89,11 +89,14 @@ class UIBase: (threading.currentThread().getName(), s.getthreadaccount(s), account) s.threadaccounts[threading.currentThread()] = account + s.debug('thread', "Register new thread '%s' (account '%s')" %\ + (threading.currentThread().getName(), account)) def unregisterthread(s, thr): """Recognizes a thread has exited.""" if s.threadaccounts.has_key(thr): del s.threadaccounts[thr] + s.debug('thread', "Unregister thread '%s'" % thr.getName()) def getthreadaccount(s, thr = None): if not thr: From f2ad4bc230ef70dfea0cd1c381d316dd83f0c038 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 May 2011 11:15:51 +0200 Subject: [PATCH 088/817] Fix typo to force singlethreading in debug mode A typo prevented us from enforcing singlethreading mode when selecting debugging. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 287bbbf..0aaa06e 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -219,7 +219,7 @@ class OfflineImap: if not ('thread' in options.debugtype.split(',') \ and options.singlethreading): ui._msg("Debug mode: Forcing to singlethreaded.") - options.singlethreaded = True + options.singlethreading = True for type in options.debugtype.split(','): type = type.strip() From 535f4592fcac1a4637350bc82bbae1d2da36ed6f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 May 2011 15:59:23 +0200 Subject: [PATCH 089/817] Replace 2 regexes with a single one No functional changes Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/LocalStatus.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index a4e036b..5cbf77b 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -34,8 +34,9 @@ class LocalStatusRepository(BaseRepository): return '.' def getfolderfilename(self, foldername): - foldername = re.sub('/\.$', '/dot', foldername) - foldername = re.sub('^\.$', 'dot', foldername) + """Return the full path of the status file""" + # replace with 'dot' if final path name is '.' + foldername = re.sub('(^|\/)\.$','\\1dot', foldername) return os.path.join(self.directory, foldername) def makefolder(self, foldername): From 2fc12e875a4c7706325e1ee100b0fe47f7af1ecb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 May 2011 15:59:24 +0200 Subject: [PATCH 090/817] Experimental LocalStatus stored in SQLite database Based on patches by Stewart Smith, updated by Rob Browning. plus: - Inherit LocalStatusSQLFolder from LocalStatusFolder This lets us remove all functions that are available via our ancestors classes and are not needed. - Don't fail if pysql import fails. Fail rather at runtime when needed. - When creating the db file, create a metadata table which contains the format version info, so we can upgrade nicely to other formats. - Create an upgrade_db() function which allows us to upgrade from any previous file format to the current one (including plain text) Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 2 +- offlineimap/folder/LocalStatus.py | 4 +- offlineimap/folder/LocalStatusSQLite.py | 192 ++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 offlineimap/folder/LocalStatusSQLite.py diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index b3fd747..a5fd242 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -21,7 +21,7 @@ import os.path import re import traceback -class BaseFolder: +class BaseFolder(object): def __init__(self): self.ui = getglobalui() diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 0310a52..467738d 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -30,12 +30,12 @@ class LocalStatusFolder(BaseFolder): self.config = config self.dofsync = config.getdefaultboolean("general", "fsync", True) self.filename = repository.getfolderfilename(name) - self.messagelist = None + self.messagelist = {} self.repository = repository self.savelock = threading.Lock() self.doautosave = 1 self.accountname = accountname - BaseFolder.__init__(self) + super(LocalStatusFolder, self).__init__() def getaccountname(self): return self.accountname diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py new file mode 100644 index 0000000..701592c --- /dev/null +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -0,0 +1,192 @@ +# Local status cache virtual folder: SQLite backend +# Copyright (C) 2009-2011 Stewart Smith and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os.path +import re +from LocalStatus import LocalStatusFolder, magicline +try: + from pysqlite2 import dbapi2 as sqlite +except: + pass #fail only if needed later on, not on import + +class LocalStatusSQLiteFolder(LocalStatusFolder): + """LocalStatus backend implemented with an SQLite database""" + #current version of the db format + cur_version = 1 + + def __init__(self, root, name, repository, accountname, config): + super(LocalStatusSQLiteFolder, self).__init__(root, name, + repository, + accountname, + config) + #Try to establish connection + try: + self.connection = sqlite.connect(self.filename) + except NameError: + # sqlite import had failed + raise UserWarning('SQLite backend chosen, but no sqlite python ' + 'bindings available. Please install.') + + #Test if the db version is current enough and if the db is + #readable. + try: + self.cursor = self.connection.cursor() + self.cursor.execute("SELECT value from metadata WHERE key='db_version'") + except sqlite.DatabaseError: + #db file missing or corrupt, recreate it. + self.connection.close() + self.upgrade_db(0) + else: + # fetch db version and upgrade if needed + version = int(self.cursor.fetchone()[0]) + self.cursor.close() + if version < LocalStatusSQLiteFolder.cur_version: + self.upgrade_db(version) + self.connection.close() + + def upgrade_db(self, from_ver): + """Upgrade the sqlite format from version 'from_ver' to current""" + if from_ver == 0: + # from_ver==0: no db existent: plain text migration? + self.create_db() + # below was derived from repository.getfolderfilename() logic + plaintextfilename = os.path.join( + self.repository.account.getaccountmeta(), + 'LocalStatus', + re.sub('(^|\/)\.$','\\1dot', self.name)) + # MIGRATE from plaintext if needed + if os.path.exists(plaintextfilename): + self.ui._msg('Migrating LocalStatus cache from plain text ' + 'to sqlite database for %s:%s' %\ + (self.repository, self)) + file = open(plaintextfilename, "rt") + line = file.readline().strip() + assert(line == magicline) + connection = sqlite.connect(self.filename) + cursor = connection.cursor() + for line in file.xreadlines(): + line = line.strip() + uid, flags = line.split(':') + uid = long(uid) + flags = [x for x in flags] + flags.sort() + flags = ''.join(flags) + self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)', + (uid,flags)) + file.close() + self.connection.commit() + os.rename(plaintextfilename, plaintextfilename + ".old") + self.connection.close() + # Future version upgrades come here... + # if from_ver <= 1: ... #upgrade from 1 to 2 + # if from_ver <= 2: ... #upgrade from 2 to 3 + + def create_db(self): + """Create a new db file""" + self.ui._msg('Creating new Local Status db for %s:%s' \ + % (self.repository, self)) + connection = sqlite.connect(self.filename) + cursor = connection.cursor() + cursor.execute('CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128))') + cursor.execute("INSERT INTO metadata VALUES('db_version', '1')") + cursor.execute('CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50))') + self.autosave() #commit if needed + + def isnewfolder(self): + # testing the existence of the db file won't work. It is created + # as soon as this class instance was intitiated. So say it is a + # new folder when there are no messages at all recorded in it. + return self.getmessagecount() > 0 + + def deletemessagelist(self): + """delete all messages in the db""" + self.cursor.execute('DELETE FROM status') + + def cachemessagelist(self): + self.messagelist = {} + self.cursor.execute('SELECT id,flags from status') + for row in self.cursor: + flags = [x for x in row[1]] + self.messagelist[row[0]] = {'uid': row[0], 'flags': flags} + + def save(self): + #Noop in this backend + pass + + def uidexists(self,uid): + self.cursor.execute('SELECT id FROM status WHERE id=:id',{'id': uid}) + for row in self.cursor: + if(row[0]==uid): + return 1 + return 0 + + def getmessageuidlist(self): + self.cursor.execute('SELECT id from status') + r = [] + for row in self.cursor: + r.append(row[0]) + return r + + def getmessagecount(self): + self.cursor.execute('SELECT count(id) from status'); + row = self.cursor.fetchone() + return row[0] + + def savemessage(self, uid, content, flags, rtime): + if uid < 0: + # We cannot assign a uid. + return uid + + if self.uidexists(uid): # already have it + self.savemessageflags(uid, flags) + return uid + + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + flags.sort() + flags = ''.join(flags) + self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)', + (uid,flags)) + self.autosave() + return uid + + def getmessageflags(self, uid): + self.cursor.execute('SELECT flags FROM status WHERE id=:id', + {'id': uid}) + for row in self.cursor: + flags = [x for x in row[0]] + return flags + assert False,"getmessageflags() called on non-existing message" + + def getmessagetime(self, uid): + return self.messagelist[uid]['time'] + + def savemessageflags(self, uid, flags): + self.messagelist[uid] = {'uid': uid, 'flags': flags} + flags.sort() + flags = ''.join(flags) + self.cursor.execute('UPDATE status SET flags=? WHERE id=?',(flags,uid)) + self.autosave() + + def deletemessages(self, uidlist): + # Weed out ones not in self.messagelist + uidlist = [uid for uid in uidlist if uid in self.messagelist] + if not len(uidlist): + return + + for uid in uidlist: + del(self.messagelist[uid]) + #if self.uidexists(uid): + self.cursor.execute('DELETE FROM status WHERE id=:id', {'id': uid}) From bd3766eca54c19c08d426a5e9af9cfa47d196e6a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 May 2011 15:59:25 +0200 Subject: [PATCH 091/817] Make getnicename work again for classes that derive from object() Python's new style classes derive from object and str(class().__class__) will return a slightly different format. class().__class.__name__ will still work for both old and new style classes, so use that instead. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 677d19b..4b0ca32 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -142,7 +142,10 @@ class UIBase: raise Exception, "Another OfflineIMAP is running with the same metadatadir; exiting." def getnicename(s, object): - prelimname = str(object.__class__).split('.')[-1] + """Return the type of a repository or Folder as string + + (IMAP, Gmail, Maildir, etc...)""" + prelimname = object.__class__.__name__.split('.')[-1] # Strip off extra stuff. return re.sub('(Folder|Repository)', '', prelimname) From e1e9c8e831b9ee46a572340033b9c861962a19d0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 May 2011 15:59:26 +0200 Subject: [PATCH 092/817] Use self.doautosave rather than self.dofsync doautosave was a useless variable before (it was *always* 1). So we remove the self.dofsync variable and store in doautosave whether we should fsync as often as possible (which really hurts performance). The sqlite backend could (at one point) use the doautosave variable to determine if it should autocommit after each modification. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatus.py | 18 +++++++----------- offlineimap/folder/LocalStatusSQLite.py | 6 +++--- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 467738d..5113345 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -28,12 +28,12 @@ class LocalStatusFolder(BaseFolder): self.root = root self.sep = '.' self.config = config - self.dofsync = config.getdefaultboolean("general", "fsync", True) self.filename = repository.getfolderfilename(name) self.messagelist = {} self.repository = repository self.savelock = threading.Lock() - self.doautosave = 1 + self.doautosave = config.getdefaultboolean("general", "fsync", False) + """Should we perform fsyncs as often as possible?""" self.accountname = accountname super(LocalStatusFolder, self).__init__() @@ -88,10 +88,6 @@ class LocalStatusFolder(BaseFolder): self.messagelist[uid] = {'uid': uid, 'flags': flags} file.close() - def autosave(self): - if self.doautosave: - self.save() - def save(self): self.savelock.acquire() try: @@ -103,12 +99,12 @@ class LocalStatusFolder(BaseFolder): flags = ''.join(flags) file.write("%s:%s\n" % (msg['uid'], flags)) file.flush() - if self.dofsync: + if self.doautosave: os.fsync(file.fileno()) file.close() os.rename(self.filename + ".tmp", self.filename) - if self.dofsync: + if self.doautosave: fd = os.open(os.path.dirname(self.filename), os.O_RDONLY) os.fsync(fd) os.close(fd) @@ -129,7 +125,7 @@ class LocalStatusFolder(BaseFolder): return uid self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} - self.autosave() + self.save() return uid def getmessageflags(self, uid): @@ -140,7 +136,7 @@ class LocalStatusFolder(BaseFolder): def savemessageflags(self, uid, flags): self.messagelist[uid]['flags'] = flags - self.autosave() + self.save() def deletemessage(self, uid): self.deletemessages([uid]) @@ -153,4 +149,4 @@ class LocalStatusFolder(BaseFolder): for uid in uidlist: del(self.messagelist[uid]) - self.autosave() + self.save() diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 701592c..c8c179f 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -103,7 +103,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): cursor.execute('CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128))') cursor.execute("INSERT INTO metadata VALUES('db_version', '1')") cursor.execute('CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50))') - self.autosave() #commit if needed + self.save() #commit if needed def isnewfolder(self): # testing the existence of the db file won't work. It is created @@ -159,7 +159,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): flags = ''.join(flags) self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)', (uid,flags)) - self.autosave() + self.save() return uid def getmessageflags(self, uid): @@ -178,7 +178,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): flags.sort() flags = ''.join(flags) self.cursor.execute('UPDATE status SET flags=? WHERE id=?',(flags,uid)) - self.autosave() + self.save() def deletemessages(self, uidlist): # Weed out ones not in self.messagelist From af25c2779f6c8a50aa6e5702e6359b0aaa8220df Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 May 2011 15:59:28 +0200 Subject: [PATCH 093/817] repository.LocalStatus: Remove code duplication Make getfolders() invoke getfolder() for each folder rather than duplicating code. Also add a forgetfolders() implementation. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/LocalStatus.py | 32 +++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 5cbf77b..722f6c3 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -48,24 +48,28 @@ class LocalStatusRepository(BaseRepository): os.fsync(file.fileno()) file.close() os.rename(filename + ".tmp", filename) - # Invalidate the cache. self.folders = None - def getfolders(self): - retval = [] - for folder in os.listdir(self.directory): - retval.append(folder.LocalStatus.LocalStatusFolder(self.directory, - folder, self, self.accountname, - self.config)) - return retval - def getfolder(self, foldername): - return folder.LocalStatus.LocalStatusFolder(self.directory, foldername, - self, self.accountname, - self.config) + """Return the Folder() object for a foldername""" + return self.LocalStatusFolderClass(self.directory, foldername, + self, self.accountname, + self.config) + def getfolders(self): + """Returns a list of ALL folders on this server. - + This is currently nowhere used in the code.""" + if self._folders != None: + return self._folders + + for folder in os.listdir(self.directory): + self._folders = retval.append(self.getfolder(folder)) + return self._folders + + def forgetfolders(self): + """Forgets the cached list of folders, if any. Useful to run + after a sync run.""" + self._folders = None - From 0af9ef70a764c59503519a7e37d8e721ef557feb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 May 2011 11:00:56 +0200 Subject: [PATCH 094/817] Factor out SQL retries Test if sqlite is multithreading-safe and bail out if not. sqlite versions since at least 2008 are. But, as it still causes errors when 2 threads try to write to the same connection simultanously (We get a "cannot start transaction within a transaction" error), we protect writes with a per class, ie per-connection lock. Factor out the retrying to write when the database is locked. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 200 +++++++++++++++--------- 1 file changed, 125 insertions(+), 75 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index c8c179f..63452f8 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -16,15 +16,29 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os.path import re +from threading import Lock from LocalStatus import LocalStatusFolder, magicline try: - from pysqlite2 import dbapi2 as sqlite + import sqlite3 as sqlite except: pass #fail only if needed later on, not on import class LocalStatusSQLiteFolder(LocalStatusFolder): - """LocalStatus backend implemented with an SQLite database""" - #current version of the db format + """LocalStatus backend implemented with an SQLite database + + As python-sqlite currently does not allow to access the same sqlite + objects from various threads, we need to open get and close a db + connection and cursor for all operations. This is a big disadvantage + and we might want to investigate if we cannot hold an object open + for a thread somehow.""" + #though. According to sqlite docs, you need to commit() before + #the connection is closed or your changes will be lost!""" + #get db connection which autocommits + #connection = sqlite.connect(self.filename, isolation_level=None) + #cursor = connection.cursor() + #return connection, cursor + + #current version of our db format cur_version = 1 def __init__(self, root, name, repository, accountname, config): @@ -32,33 +46,68 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): repository, accountname, config) - #Try to establish connection + + # dblock protects against concurrent writes in same connection + self._dblock = Lock() + #Try to establish connection, no need for threadsafety in __init__ try: - self.connection = sqlite.connect(self.filename) + self.connection = sqlite.connect(self.filename, check_same_thread = False) except NameError: # sqlite import had failed raise UserWarning('SQLite backend chosen, but no sqlite python ' 'bindings available. Please install.') - #Test if the db version is current enough and if the db is - #readable. + #Make sure sqlite is in multithreading SERIALIZE mode + assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' + + #Test if db version is current enough and if db is readable. try: - self.cursor = self.connection.cursor() - self.cursor.execute("SELECT value from metadata WHERE key='db_version'") + cursor = self.connection.execute("SELECT value from metadata WHERE key='db_version'") except sqlite.DatabaseError: #db file missing or corrupt, recreate it. - self.connection.close() self.upgrade_db(0) else: # fetch db version and upgrade if needed - version = int(self.cursor.fetchone()[0]) - self.cursor.close() + version = int(cursor.fetchone()[0]) if version < LocalStatusSQLiteFolder.cur_version: self.upgrade_db(version) - self.connection.close() + + def sql_write(self, sql, vars=None): + """execute some SQL retrying if the db was locked. + + :param sql: the SQL string passed to execute() :param args: the + variable values to `sql`. E.g. (1,2) or {uid:1, flags:'T'}. See + sqlite docs for possibilities. + :returns: the Cursor() or raises an Exception""" + success = False + while not success: + self._dblock.acquire() + try: + if vars is None: + cursor = self.connection.execute(sql) + else: + cursor = self.connection.execute(sql, vars) + success = True + self.connection.commit() + except sqlite.OperationalError, e: + if e.args[0] == 'cannot commit - no transaction is active': + pass + elif e.args[0] == 'database is locked': + self.ui.debug('', "Locked sqlite database, retrying.") + success = False + else: + raise + finally: + self._dblock.release() + return cursor def upgrade_db(self, from_ver): """Upgrade the sqlite format from version 'from_ver' to current""" + + if hasattr(self, 'connection'): + self.connection.close() #close old connections first + self.connection = sqlite.connect(self.filename, check_same_thread = False) + if from_ver == 0: # from_ver==0: no db existent: plain text migration? self.create_db() @@ -74,22 +123,18 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): (self.repository, self)) file = open(plaintextfilename, "rt") line = file.readline().strip() - assert(line == magicline) - connection = sqlite.connect(self.filename) - cursor = connection.cursor() + data = [] for line in file.xreadlines(): - line = line.strip() - uid, flags = line.split(':') + uid, flags = line.strip().split(':') uid = long(uid) - flags = [x for x in flags] - flags.sort() - flags = ''.join(flags) - self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)', - (uid,flags)) - file.close() + flags = list(flags) + flags = ''.join(sorted(flags)) + data.append((uid,flags)) + self.connection.executemany('INSERT INTO status (id,flags) VALUES (?,?)', + data) self.connection.commit() + file.close() os.rename(plaintextfilename, plaintextfilename + ".old") - self.connection.close() # Future version upgrades come here... # if from_ver <= 1: ... #upgrade from 1 to 2 # if from_ver <= 2: ... #upgrade from 2 to 3 @@ -98,12 +143,15 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): """Create a new db file""" self.ui._msg('Creating new Local Status db for %s:%s' \ % (self.repository, self)) - connection = sqlite.connect(self.filename) - cursor = connection.cursor() - cursor.execute('CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128))') - cursor.execute("INSERT INTO metadata VALUES('db_version', '1')") - cursor.execute('CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50))') - self.save() #commit if needed + if hasattr(self, 'connection'): + self.connection.close() #close old connections first + self.connection = sqlite.connect(self.filename, check_same_thread = False) + self.connection.executescript(""" + CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128)); + INSERT INTO metadata VALUES('db_version', '1'); + CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50)); + """) + self.connection.commit() def isnewfolder(self): # testing the existence of the db file won't work. It is created @@ -113,37 +161,54 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): def deletemessagelist(self): """delete all messages in the db""" - self.cursor.execute('DELETE FROM status') + self.sql_write('DELETE FROM status') def cachemessagelist(self): self.messagelist = {} - self.cursor.execute('SELECT id,flags from status') - for row in self.cursor: - flags = [x for x in row[1]] - self.messagelist[row[0]] = {'uid': row[0], 'flags': flags} + cursor = self.connection.execute('SELECT id,flags from status') + for row in cursor: + flags = [x for x in row[1]] + self.messagelist[row[0]] = {'uid': row[0], 'flags': flags} def save(self): #Noop in this backend pass - def uidexists(self,uid): - self.cursor.execute('SELECT id FROM status WHERE id=:id',{'id': uid}) - for row in self.cursor: - if(row[0]==uid): - return 1 - return 0 - - def getmessageuidlist(self): - self.cursor.execute('SELECT id from status') - r = [] - for row in self.cursor: - r.append(row[0]) - return r - - def getmessagecount(self): - self.cursor.execute('SELECT count(id) from status'); - row = self.cursor.fetchone() - return row[0] + # Following some pure SQLite functions, where we chose to use + # BaseFolder() methods instead. Doing those on the in-memory list is + # quicker anyway. If our db becomes so big that we don't want to + # maintain the in-memory list anymore, these might come in handy + # in the future though. + # + #def uidexists(self,uid): + # conn, cursor = self.get_cursor() + # with conn: + # cursor.execute('SELECT id FROM status WHERE id=:id',{'id': uid}) + # return cursor.fetchone() + # This would be the pure SQLite solution, use BaseFolder() method, + # to avoid threading with sqlite... + #def getmessageuidlist(self): + # conn, cursor = self.get_cursor() + # with conn: + # cursor.execute('SELECT id from status') + # r = [] + # for row in cursor: + # r.append(row[0]) + # return r + #def getmessagecount(self): + # conn, cursor = self.get_cursor() + # with conn: + # cursor.execute('SELECT count(id) from status'); + # return cursor.fetchone()[0] + #def getmessageflags(self, uid): + # conn, cursor = self.get_cursor() + # with conn: + # cursor.execute('SELECT flags FROM status WHERE id=:id', + # {'id': uid}) + # for row in cursor: + # flags = [x for x in row[0]] + # return flags + # assert False,"getmessageflags() called on non-existing message" def savemessage(self, uid, content, flags, rtime): if uid < 0: @@ -155,38 +220,23 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): return uid self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} - flags.sort() - flags = ''.join(flags) - self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)', - (uid,flags)) - self.save() + flags = ''.join(sorted(flags)) + self.sql_write('INSERT INTO status (id,flags) VALUES (?,?)', + (uid,flags)) return uid - def getmessageflags(self, uid): - self.cursor.execute('SELECT flags FROM status WHERE id=:id', - {'id': uid}) - for row in self.cursor: - flags = [x for x in row[0]] - return flags - assert False,"getmessageflags() called on non-existing message" - - def getmessagetime(self, uid): - return self.messagelist[uid]['time'] - def savemessageflags(self, uid, flags): self.messagelist[uid] = {'uid': uid, 'flags': flags} flags.sort() flags = ''.join(flags) - self.cursor.execute('UPDATE status SET flags=? WHERE id=?',(flags,uid)) - self.save() + self.sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) def deletemessages(self, uidlist): # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if uid in self.messagelist] if not len(uidlist): return - for uid in uidlist: del(self.messagelist[uid]) - #if self.uidexists(uid): - self.cursor.execute('DELETE FROM status WHERE id=:id', {'id': uid}) + self.sql_write('DELETE FROM status WHERE id=?', + uidlist) From 0318c6ad340b099b2b683bb16ed34ab89bf3941c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 May 2011 11:00:57 +0200 Subject: [PATCH 095/817] Create LocalStatus or LocalStatusSQLite folders Depending on the configuration we use the plain text or the new experimental sqlite backend for the LocalStatus cache. Make plain text the default status backend but allow people to configure status_backend=sqlite in their [Account ...] section. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 11 +++----- offlineimap.conf | 17 +++++++++++++ offlineimap/repository/LocalStatus.py | 36 +++++++++++++++++++++------ 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index fe70cfe..66501e2 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -17,6 +17,9 @@ New Features readonly = True. When e.g. using offlineimap for backup purposes you can thus make sure that no changes in your backup trickle back into the main IMAP server. +* optional: experimental SQLite-based backend for the LocalStatus + cache. Plain text remains the default. Enable by setting + status_backend=sqlite in the [Account ...] section Changes ------- @@ -41,11 +44,3 @@ Pending for the next major release * UIs get shorter and nicer names. (API changing) * Implement IDLE feature. (delayed until next major release) - - -Stalled -======= - -* Learn Sqlite support. - Stalled: it would need to learn the ability to choose between the current - format and SQL to help testing the long term. diff --git a/offlineimap.conf b/offlineimap.conf index f8506b2..33a0810 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -197,6 +197,23 @@ remoterepository = RemoteExample # You can also specify parameters to the commands # presynchook = imapfilter -c someotherconfig.lua +# OfflineImap caches the state of the synchronisation to e.g. be able to +# determine if a mail has been deleted on one side or added on the +# other. +# +# The default and historical backend is 'plain' which writes out the +# state in plain text files. On Repositories with large numbers of +# mails, the performance might not be optimal, as we write out the +# complete file for each change. Another new backend 'sqlite' is +# available which stores the status in sqlite databases. BE AWARE THIS +# IS EXPERIMENTAL STUFF. +# +# If you switch the backend, you may want to delete the old cache +# directory in ~/.offlineimap/Account-/LocalStatus manually +# once you are sure that things work. +# +#status_backend = plain + # If you have a limited amount of bandwidth available you can exclude larger # messages (e.g. those with large attachments etc). If you do this it # will appear to offlineimap that these messages do not exist at all. They diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 722f6c3..7d38596 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -18,7 +18,8 @@ from Base import BaseRepository from offlineimap import folder -import offlineimap.folder.LocalStatus +from offlineimap.folder.LocalStatus import LocalStatusFolder +from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder import os import re @@ -26,9 +27,25 @@ class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): BaseRepository.__init__(self, reposname, account) self.directory = os.path.join(account.getaccountmeta(), 'LocalStatus') + + #statusbackend can be 'plain' or 'sqlite' + backend = self.account.getconf('status_backend', 'plain') + if backend == 'sqlite': + self._backend = 'sqlite' + self.LocalStatusFolderClass = LocalStatusSQLiteFolder + self.directory += '-sqlite' + elif backend == 'plain': + self._backend = 'plain' + self.LocalStatusFolderClass = LocalStatusFolder + else: + raise SyntaxWarning("Unknown status_backend '%s' for account '%s'" \ + % (backend, account.name)) + if not os.path.exists(self.directory): os.mkdir(self.directory, 0700) - self.folders = None + + # self._folders is a list of LocalStatusFolders() + self._folders = None def getsep(self): return '.' @@ -40,16 +57,22 @@ class LocalStatusRepository(BaseRepository): return os.path.join(self.directory, foldername) def makefolder(self, foldername): - # "touch" the file, truncating it. + """Create a LocalStatus Folder + + Empty Folder for plain backend. NoOp for sqlite backend as those + are created on demand.""" + # Invalidate the cache. + self._folders = None + if self._backend == 'sqlite': + return + filename = self.getfolderfilename(foldername) file = open(filename + ".tmp", "wt") file.write(offlineimap.folder.LocalStatus.magicline + '\n') - file.flush() - os.fsync(file.fileno()) file.close() os.rename(filename + ".tmp", filename) # Invalidate the cache. - self.folders = None + self._folders = None def getfolder(self, foldername): """Return the Folder() object for a foldername""" @@ -72,4 +95,3 @@ class LocalStatusRepository(BaseRepository): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" self._folders = None - From f4081985dc12212fa83f8a676f7c49a6cfae25e3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 May 2011 11:11:20 +0200 Subject: [PATCH 096/817] Prettify and use new uidexists() helper function Make the folder classes use uidexists() more. Add some code documentation while going through. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- offlineimap/folder/Maildir.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 6ff73ec..9f60ded 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -562,7 +562,7 @@ class IMAPFolder(BaseFolder): def deletemessages_noconvert(self, uidlist): # Weed out ones not in self.messagelist - uidlist = [uid for uid in uidlist if uid in self.messagelist] + uidlist = [uid for uid in uidlist if self.uidexists(uid)] if not len(uidlist): return diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 02e64d2..e360619 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -295,8 +295,16 @@ class MaildirFolder(BaseFolder): assert final_dir != tmpdir def deletemessage(self, uid): - if not uid in self.messagelist: + """Unlinks a message file from the Maildir. + + :param uid: UID of a mail message + :type uid: String + :return: Nothing, or an Exception if UID but no corresponding file + found. + """ + if not self.uidexists(uid): return + filename = self.messagelist[uid]['filename'] try: os.unlink(filename) From dd82f213f006af54dbd83165a423e28cbefea2d5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 May 2011 18:27:28 +0200 Subject: [PATCH 097/817] accounts: fix broken warn statement Use two %s in the message for both string parameters. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 0eb49dc..e4499fa 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -219,8 +219,8 @@ class SyncableAccount(Account): if e.severity >= OfflineImapError.ERROR.CRITICAL: raise except: - self.ui.warn("Error occured attempting to sync "\ - "account '%s':\n"% (self, traceback.format_exc())) + self.ui.warn("Error occured attempting to sync account "\ + "'%s':\n%s"% (self, traceback.format_exc())) else: # after success sync, reset the looping counter to 3 if self.refreshperiod: From b3a383d1519a51d3002ca54c262870034f15dc48 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 May 2011 18:39:13 +0200 Subject: [PATCH 098/817] accounts: handle OfflineImapError severity FOLDER Throw an OfflineImapError when SELECTing a folder is unsuccessful and bail out with a FOLDER serverity. In accounts.py catch all OfflineImapErrors and either just log the error and skip the folder or bubble it up if it's severe. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 7 +++++++ offlineimap/folder/IMAP.py | 6 +++++- offlineimap/imaplibutil.py | 12 ++++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index e4499fa..3e8fe18 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -393,6 +393,13 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, localrepos.restore_atime() except (KeyboardInterrupt, SystemExit): raise + except OfflineImapError, e: + # bubble up severe Errors, skip folder otherwise + if e.severity > OfflineImapError.ERROR.FOLDER: + raise + else: + ui.warn("Aborting folder sync '%s' [acc: '%s']\nReason was: %s" %\ + (localfolder.name, accountname, e.reason)) except: ui.warn("ERROR in syncfolder for %s folder %s: %s" % \ (accountname,remotefolder.getvisiblename(), diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 9f60ded..9c91ddd 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -43,9 +43,13 @@ class IMAPFolder(BaseFolder): def selectro(self, imapobj): """Select this folder when we do not need write access. + Prefer SELECT to EXAMINE if we can, since some servers (Courier) do not stabilize UID validity until the folder is - selected.""" + selected. + .. todo: Still valid? Needs verification + + :returns: raises :exc:`OfflineImapError` severity FOLDER on error""" try: imapobj.select(self.getfullname()) except imapobj.readonly: diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 7c98fef..5faf036 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -23,8 +23,8 @@ import time import subprocess from offlineimap.ui import getglobalui import threading +from offlineimap import OfflineImapError from offlineimap.imaplib2 import * - # Import the symbols we need that aren't exported by default from offlineimap.imaplib2 import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num @@ -43,6 +43,10 @@ class UsefulIMAPMixIn: return None def select(self, mailbox='INBOX', readonly=None, force = 0): + """Selects a mailbox on the IMAP server + + :returns: 'OK' on success, nothing if the folder was already + selected or raises an :exc:`OfflineImapError`""" if (not force) and self.getselectedfolder() == mailbox \ and self.is_readonly == readonly: # No change; return. @@ -51,7 +55,11 @@ class UsefulIMAPMixIn: del self.untagged_responses[:] result = self.__class__.__bases__[1].select(self, mailbox, readonly) if result[0] != 'OK': - raise ValueError, "Error from select: %s" % str(result) + #in case of error, bail out with OfflineImapError + errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\ + (mailbox, result) + severity = OfflineImapError.ERROR.FOLDER + raise OfflineImapError(errstr, severity) if self.getstate() == 'SELECTED': self.selectedfolder = mailbox else: From 89619838b0d393d138dcaecef57e488f06322519 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 May 2011 17:40:32 +0200 Subject: [PATCH 099/817] Remove weird SigListener class The SigListener class was used to queue folders that we need to sync and to receive "resync" and "abort" signals. It was undocumented and weird and we had to pass "siglisteners" through the whole program. Simply do away with it, and make 2 functions in the Account() class: set_abort_event and get_abort_event which can be used to set and check for such signals. This way we do not need to pass siglisteners all over the place. Tested Blinkenlights and TTYUI uis to make sure that SIGUSR1 and SIGUSR2 actually still work. Document those signals in MANUAL.rst. They were completly undocumented. This simplifies the code and interdependencies by passing less stuff around. Removes an undocumented and weirdly named class. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 15 +++ offlineimap/accounts.py | 195 ++++++++++++++------------------ offlineimap/init.py | 37 +++--- offlineimap/syncmaster.py | 14 +-- offlineimap/threadutil.py | 4 - offlineimap/ui/Blinkenlights.py | 4 +- offlineimap/ui/Curses.py | 4 +- offlineimap/ui/UIBase.py | 22 ++-- 8 files changed, 129 insertions(+), 166 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 9ef4eea..62b0d3b 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -262,6 +262,21 @@ MachineUI generates output in a machine-parsable format. It is designed for other programs that will interface to OfflineIMAP. +Signals +======= + +OfflineImap listens to the unix signals SIGUSR1 and SIGUSR2. + +If sent a SIGUSR1 it will abort any current (or next future) sleep of all +accounts that are configured to "autorefresh". In effect, this will trigger a +full sync of all accounts to be performed as soon as possible. + +If sent a SIGUSR2, it will stop "autorefresh mode" for all accounts. That is, +accounts will abort any current sleep and will exit after a currently running +synchronization has finished. This signal can be used to gracefully exit out of +a running offlineimap "daemon". + + KNOWN BUGS ========== diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 0eb49dc..d7e44c2 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -20,76 +20,11 @@ from offlineimap.repository import Repository from offlineimap.ui import getglobalui from offlineimap.threadutil import InstanceLimitedThread from subprocess import Popen, PIPE -from threading import Lock +from threading import Event import os -from Queue import Queue import sys import traceback -class SigListener(Queue): - def __init__(self): - self.folderlock = Lock() - self.folders = None - Queue.__init__(self, 20) - def put_nowait(self, sig): - self.folderlock.acquire() - try: - if sig == 1: - if self.folders is None or not self.autorefreshes: - # folders haven't yet been added, or this account is once-only; drop signal - return - elif self.folders: - for foldernr in range(len(self.folders)): - # requeue folder - self.folders[foldernr][1] = True - self.quick = False - return - # else folders have already been cleared, put signal... - finally: - self.folderlock.release() - Queue.put_nowait(self, sig) - def addfolders(self, remotefolders, autorefreshes, quick): - self.folderlock.acquire() - try: - self.folders = [] - self.quick = quick - self.autorefreshes = autorefreshes - for folder in remotefolders: - # new folders are queued - self.folders.append([folder, True]) - finally: - self.folderlock.release() - def clearfolders(self): - self.folderlock.acquire() - try: - for folder, queued in self.folders: - if queued: - # some folders still in queue - return False - self.folders[:] = [] - return True - finally: - self.folderlock.release() - def queuedfolders(self): - self.folderlock.acquire() - try: - dirty = True - while dirty: - dirty = False - for foldernr, (folder, queued) in enumerate(self.folders): - if queued: - # mark folder as no longer queued - self.folders[foldernr][1] = False - dirty = True - quick = self.quick - self.folderlock.release() - yield (folder, quick) - self.folderlock.acquire() - except: - self.folderlock.release() - raise - self.folderlock.release() - def getaccountlist(customconfig): return customconfig.getsectionlist('Account') @@ -110,6 +45,8 @@ class Account(CustomConfig.ConfigHelperMixin): Most of the time you will actually want to use the derived :class:`accounts.SyncableAccount` which contains all functions used for syncing an account.""" + #signal gets set when we should stop looping + abort_signal = Event() def __init__(self, config, name): """ @@ -144,13 +81,49 @@ class Account(CustomConfig.ConfigHelperMixin): def getsection(self): return 'Account ' + self.getname() - def sleeper(self, siglistener): - """Sleep handler. Returns same value as UIBase.sleep: - 0 if timeout expired, 1 if there was a request to cancel the timer, - and 2 if there is a request to abort the program. + @classmethod + def set_abort_event(cls, config, signum): + """Set skip sleep/abort event for all accounts - Also, returns 100 if configured to not sleep at all.""" - + If we want to skip a current (or the next) sleep, or if we want + to abort an autorefresh loop, the main thread can use + set_abort_event() to send the corresponding signal. Signum = 1 + implies that we want all accounts to abort or skip the current + or next sleep phase. Signum = 2 will end the autorefresh loop, + ie all accounts will return after they finished a sync. + + This is a class method, it will send the signal to all accounts. + """ + if signum == 1: + # resync signal, set config option for all accounts + for acctsection in getaccountlist(config): + config.set('Account ' + acctsection, "skipsleep", '1') + elif signum == 2: + # don't autorefresh anymore + cls.abort_signal.set() + + def get_abort_event(self): + """Checks if an abort signal had been sent + + If the 'skipsleep' config option for this account had been set, + with `set_abort_event(config, 1)` it will get cleared in this + function. Ie, we will only skip one sleep and not all. + + :returns: True, if the main thread had called + :meth:`set_abort_event` earlier, otherwise 'False'. + """ + skipsleep = self.getconfboolean("skipsleep", 0) + if skipsleep: + self.config.set(self.getsection(), "skipsleep", '0') + return skipsleep or Account.abort_signal.is_set() + + def sleeper(self): + """Sleep if the account is set to autorefresh + + :returns: 0:timeout expired, 1: canceled the timer, + 2:request to abort the program, + 100: if configured to not sleep at all. + """ if not self.refreshperiod: return 100 @@ -165,22 +138,18 @@ class Account(CustomConfig.ConfigHelperMixin): item.startkeepalive() refreshperiod = int(self.refreshperiod * 60) -# try: -# sleepresult = siglistener.get_nowait() -# # retrieved signal before sleep started -# if sleepresult == 1: -# # catching signal 1 here means folders were cleared before signal was posted -# pass -# except Empty: -# sleepresult = self.ui.sleep(refreshperiod, siglistener) - sleepresult = self.ui.sleep(refreshperiod, siglistener) - if sleepresult == 1: - self.quicknum = 0 + sleepresult = self.ui.sleep(refreshperiod, self) # Cancel keepalive for item in kaobjs: item.stopkeepalive() - return sleepresult + + if sleepresult: + if Account.abort_signal.is_set(): + return 2 + self.quicknum = 0 + return 1 + return 0 class SyncableAccount(Account): @@ -190,14 +159,13 @@ class SyncableAccount(Account): functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`, used for syncing.""" - def syncrunner(self, siglistener): + def syncrunner(self): self.ui.registerthread(self.name) self.ui.acct(self.name) accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): os.mkdir(accountmetadata, 0700) - # get all three repositories self.remoterepos = Repository(self, 'remote') self.localrepos = Repository(self, 'local') self.statusrepos = Repository(self, 'status') @@ -207,36 +175,40 @@ class SyncableAccount(Account): while looping: try: try: - self.sync(siglistener) + self.sync() except (KeyboardInterrupt, SystemExit): raise except OfflineImapError, e: self.ui.warn(e.reason) - #stop looping and bubble up Exception if needed + # Stop looping and bubble up Exception if needed. if e.severity >= OfflineImapError.ERROR.REPO: - if looping: + if looping: looping -= 1 if e.severity >= OfflineImapError.ERROR.CRITICAL: raise except: - self.ui.warn("Error occured attempting to sync "\ - "account '%s':\n"% (self, traceback.format_exc())) + self.ui.warn("Error occured attempting to sync account " + "'%s':\n" % (self, traceback.format_exc())) else: # after success sync, reset the looping counter to 3 if self.refreshperiod: looping = 3 finally: - if self.sleeper(siglistener) >= 2: + if looping and self.sleeper() >= 2: looping = 0 self.ui.acctdone(self.name) - def getaccountmeta(self): return os.path.join(self.metadatadir, 'Account-' + self.name) - def sync(self, siglistener): - # We don't need an account lock because syncitall() goes through - # each account once, then waits for all to finish. + def sync(self): + """Synchronize the account once, then return + + Assumes that `self.remoterepos`, `self.localrepos`, and + `self.statusrepos` has already been populated, so it should only + be called from the :meth:`syncrunner` function. + """ + folderthreads = [] hook = self.getconf('presynchook', '') self.callhook(hook) @@ -263,23 +235,20 @@ class SyncableAccount(Account): self.ui.syncfolders(remoterepos, localrepos) remoterepos.syncfoldersto(localrepos, [statusrepos]) - siglistener.addfolders(remoterepos.getfolders(), bool(self.refreshperiod), quick) - - while True: - folderthreads = [] - for remotefolder, quick in siglistener.queuedfolders(): - thread = InstanceLimitedThread(\ - instancename = 'FOLDER_' + self.remoterepos.getname(), - target = syncfolder, - name = "Folder sync [%s]" % self, - args = (self.name, remoterepos, remotefolder, localrepos, - statusrepos, quick)) - thread.setDaemon(1) - thread.start() - folderthreads.append(thread) - threadutil.threadsreset(folderthreads) - if siglistener.clearfolders(): - break + # iterate through all folders on the remote repo and sync + for remotefolder in remoterepos.getfolders(): + thread = InstanceLimitedThread(\ + instancename = 'FOLDER_' + self.remoterepos.getname(), + target = syncfolder, + name = "Folder sync [%s]" % self, + args = (self.name, remoterepos, remotefolder, localrepos, + statusrepos, quick)) + thread.setDaemon(1) + thread.start() + folderthreads.append(thread) + # wait for all threads to finish + for thr in folderthreads: + thr.join() mbnames.write() localrepos.forgetfolders() remoterepos.forgetfolders() diff --git a/offlineimap/init.py b/offlineimap/init.py index ba353c5..7aa8f0b 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -254,7 +254,7 @@ class OfflineImap: config.set(section, "folderincludes", folderincludes) self.lock(config, ui) - + self.config = config def sigterm_handler(signum, frame): # die immediately @@ -315,21 +315,14 @@ class OfflineImap: threadutil.initInstanceLimit(instancename, config.getdefaultint('Repository ' + reposname, 'maxconnections', 2)) - siglisteners = [] - def sig_handler(signum, frame): - if signum == signal.SIGUSR1: - # tell each account to do a full sync asap - signum = (1,) - elif signum == signal.SIGHUP: - # tell each account to die asap - signum = (2,) - elif signum == signal.SIGUSR2: - # tell each account to do a full sync asap, then die - signum = (1, 2) - # one listener per account thread (up to maxsyncaccounts) - for listener in siglisteners: - for sig in signum: - listener.put_nowait(sig) + def sig_handler(sig, frame): + if sig == signal.SIGUSR1 or sig == signal.SIGHUP: + # tell each account to stop sleeping + accounts.Account.set_abort_event(self.config, 1) + elif sig == signal.SIGUSR2: + # tell each account to stop looping + accounts.Account.set_abort_event(self.config, 2) + signal.signal(signal.SIGHUP,sig_handler) signal.signal(signal.SIGUSR1,sig_handler) signal.signal(signal.SIGUSR2,sig_handler) @@ -340,14 +333,13 @@ class OfflineImap: if options.singlethreading: #singlethreaded - self.sync_singlethreaded(syncaccounts, config, siglisteners) + self.sync_singlethreaded(syncaccounts, config) else: # multithreaded t = threadutil.ExitNotifyThread(target=syncmaster.syncitall, name='Sync Runner', kwargs = {'accounts': syncaccounts, - 'config': config, - 'siglisteners': siglisteners}) + 'config': config}) t.setDaemon(1) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) @@ -360,16 +352,13 @@ class OfflineImap: except: ui.mainException() - def sync_singlethreaded(self, accs, config, siglisteners): + def sync_singlethreaded(self, accs, config): """Executed if we do not want a separate syncmaster thread :param accs: A list of accounts that should be synced :param config: The CustomConfig object - :param siglisteners: The signal listeners list, defined in run() """ for accountname in accs: account = offlineimap.accounts.SyncableAccount(config, accountname) - siglistener = offlineimap.accounts.SigListener() - siglisteners.append(siglistener) threading.currentThread().name = "Account sync %s" % accountname - account.syncrunner(siglistener=siglistener) + account.syncrunner() diff --git a/offlineimap/syncmaster.py b/offlineimap/syncmaster.py index d21cfac..3aea6d2 100644 --- a/offlineimap/syncmaster.py +++ b/offlineimap/syncmaster.py @@ -17,26 +17,22 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap.threadutil import threadlist, InstanceLimitedThread -from offlineimap.accounts import SyncableAccount, SigListener +from offlineimap.accounts import SyncableAccount from threading import currentThread -def syncaccount(threads, config, accountname, siglisteners): +def syncaccount(threads, config, accountname): account = SyncableAccount(config, accountname) - siglistener = SigListener() thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT', target = account.syncrunner, - name = "Account sync %s" % accountname, - kwargs = {'siglistener': siglistener} ) - # the Sync Runner thread is the only one that will mutate siglisteners - siglisteners.append(siglistener) + name = "Account sync %s" % accountname) thread.setDaemon(1) thread.start() threads.add(thread) -def syncitall(accounts, config, siglisteners): +def syncitall(accounts, config): currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE') threads = threadlist() for accountname in accounts: - syncaccount(threads, config, accountname, siglisteners) + syncaccount(threads, config, accountname) # Wait for the threads to finish. threads.reset() diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 8644322..7ffcbe6 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -45,10 +45,6 @@ def semaphorereset(semaphore, originalstate): def semaphorewait(semaphore): semaphore.acquire() semaphore.release() - -def threadsreset(threadlist): - for thr in threadlist: - thr.join() class threadlist: def __init__(self): diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py index 2f11aa4..257dca2 100644 --- a/offlineimap/ui/Blinkenlights.py +++ b/offlineimap/ui/Blinkenlights.py @@ -132,10 +132,10 @@ class BlinkenBase: s.gettf().setcolor('white') s.__class__.__bases__[-1].callhook(s, msg) - def sleep(s, sleepsecs, siglistener): + def sleep(s, sleepsecs, account): s.gettf().setcolor('red') s.getaccountframe().startsleep(sleepsecs) - return UIBase.sleep(s, sleepsecs, siglistener) + return UIBase.sleep(s, sleepsecs, account) def sleeping(s, sleepsecs, remainingsecs): if remainingsecs and s.gettf().getcolor() == 'black': diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index b5a10da..cceeaaa 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -557,10 +557,10 @@ class Blinkenlights(BlinkenBase, UIBase): s.c.stop() UIBase.mainException(s) - def sleep(s, sleepsecs, siglistener): + def sleep(s, sleepsecs, account): s.gettf().setcolor('red') s._msg("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return BlinkenBase.sleep(s, sleepsecs, siglistener) + return BlinkenBase.sleep(s, sleepsecs, account) if __name__ == '__main__': x = Blinkenlights(None) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 4b0ca32..8047c15 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -342,24 +342,22 @@ class UIBase: ################################################## Other - def sleep(s, sleepsecs, siglistener): + def sleep(s, sleepsecs, account): """This function does not actually output anything, but handles the overall sleep, dealing with updates as necessary. It will, however, call sleeping() which DOES output something. - Returns 0 if timeout expired, 1 if there is a request to cancel - the timer, and 2 if there is a request to abort the program.""" - - abortsleep = 0 + :returns: 0/False if timeout expired, 1/2/True if there is a + request to cancel the timer. + """ + abortsleep = False while sleepsecs > 0 and not abortsleep: - try: - abortsleep = siglistener.get_nowait() - # retrieved signal while sleeping: 1 means immediately resynch, 2 means immediately die - except Empty: - # no signal + if account.get_abort_event(): + abortsleep = True + else: abortsleep = s.sleeping(10, sleepsecs) - sleepsecs -= 10 - s.sleeping(0, 0) # Done sleeping. + sleepsecs -= 10 + s.sleeping(0, 0) # Done sleeping. return abortsleep def sleeping(s, sleepsecs, remainingsecs): From cff792d38120a0212e58f307cb61f215f86172ff Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 May 2011 20:28:17 +0200 Subject: [PATCH 100/817] LocalStatusSQLite: Fix bug when deleting messages The syntax was not right, and deleting messages from the LocalStatus failed. (We passed in the full list of uids and we need to pass in one uid at a time (as a tuple). Deleting messages works now. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 63452f8..f470599 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -238,5 +238,5 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): return for uid in uidlist: del(self.messagelist[uid]) - self.sql_write('DELETE FROM status WHERE id=?', - uidlist) + #TODO: we want a way to do executemany(.., uidlist) to delete all + self.sql_write('DELETE FROM status WHERE id=?', (uid, )) From e7439b5d774e62a9477f67ad3d3bc2bb3c3a5657 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 7 May 2011 14:37:23 +0200 Subject: [PATCH 101/817] doc: introduce a new HACKING file This aims to help newcomers to be very fast to involve into the project. Signed-off-by: Nicolas Sebrecht --- docs/HACKING.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/HACKING.rst diff --git a/docs/HACKING.rst b/docs/HACKING.rst new file mode 100644 index 0000000..623583f --- /dev/null +++ b/docs/HACKING.rst @@ -0,0 +1,23 @@ +.. -*- coding: utf-8 -*- + +.. _OfflineIMAP: https://github.com/nicolas33/offlineimap + +.. contents:: +.. sectnum:: + +=================== +HACKING OFFLINEIMAP +=================== + +Welcome to the `OfflineIMAP`_ project. You'll find here all the information you +need to start hacking OfflineIMAP. Be aware there are a lot of very usefull tips +in the mailing list. You may want to subscribe if you didn't, yet. This is +where you'll get help. + +=== +API +=== + +API is documented in the dev-doc-src directory using the sphinx tools (also used +for python itself). This is a WIP. Contributions in this area would be very +appreciated. From 9a9379f8f33dab376aafb6fdc9f3b0f8b04227f8 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 8 May 2011 17:03:58 +0200 Subject: [PATCH 102/817] add documentation about Git Signed-off-by: Nicolas Sebrecht --- docs/FAQ.rst | 55 +++++++++++++++ docs/HACKING.rst | 171 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 222 insertions(+), 4 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index fcc55b3..cd612ac 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -355,6 +355,22 @@ accounts. Miscellaneous Questions ======================= +I'm using git to install OfflineIMAP and found these branches called "master", "maint", "next", "pu" and "gh-pages". What are they? +----------------------------------------------------------------------------------------------------------------------------------- + +To be brief: + +* **gh-pages**: branch used to maintain the home page at github. +* **master**: classical mainline branch. +* **next**: this is the branch for recent merged patches. Used for testing OfflineIMAP. +* **pu** ("proposed updates"): patches not ready for inclusion. This should **never** be checkouted! +* **maint**: our long-living maintenance branch. We maintain this branch + (security and bugfixes) for users who don't want or can't upgrade to the + latest release. + +For more information about the branching model and workflow, see the HACKING page. + + Why are your Maildir message filenames so long? ----------------------------------------------- @@ -396,3 +412,42 @@ written in Korn, so you’ll need ksh, pdksh, or mksh to run it:: do ( exec /usr/bin/offlineimap -u Noninteractive.Quiet ) sleep 60 # prevents extended failure condition + + +Contributing +============ + +How to submit a patch? +---------------------- + +If you want to send regular patches, you should first subscribe to the `mailing +list`_. This is not a pre-requisite, though. + +Next, you'll find documentation in the docs/ directory, especially the HACKING +page. + +You'll need to get a clone from the official `OfflineIMAP`_ repository and +configure Git. Then, read the SubmittingPatches.rst page in your local +repository or at +https://github.com/nicolas33/offlineimap/blob/master/SubmittingPatches.rst#readme +. + +To send a patch, we recommend using 'git send-email'. + + +Where from should my patches be based on? +----------------------------------------- + +Depends. If you're not sure, it should start off of the master branch. master is +the branch where new patches should be based on by default. + +Obvious materials for next release (e.g. new features) start off of current +next. Also, next is the natural branch to write patches on top of commits not +already in master. + +A fix for a very old bug or security issue may start off of maint. This isn't +needed since such fix are backported by the maintainer, though. + +Finally, a work on very active or current development can start from a topic +next. This clearly means you **need** this topic as a base for what is intended. + diff --git a/docs/HACKING.rst b/docs/HACKING.rst index 623583f..e21afd7 100644 --- a/docs/HACKING.rst +++ b/docs/HACKING.rst @@ -2,11 +2,8 @@ .. _OfflineIMAP: https://github.com/nicolas33/offlineimap -.. contents:: -.. sectnum:: - =================== -HACKING OFFLINEIMAP +Hacking OfflineIMAP =================== Welcome to the `OfflineIMAP`_ project. You'll find here all the information you @@ -14,6 +11,172 @@ need to start hacking OfflineIMAP. Be aware there are a lot of very usefull tips in the mailing list. You may want to subscribe if you didn't, yet. This is where you'll get help. +.. contents:: +.. sectnum:: + + +================================= +Git: Branching Model And Workflow +================================= + +Introduction +============ + +In order to involve into OfflineIMAP you need some knowledges about Git and our +workflow. Don't be afraid if you don't know much, we would be pleased to help +you. + +Release cycles +============== + +We use a classical cycle based workflow: + +1. A stable release is out. + +2. Feature topics are sent, discussed and merged. + +3. When enough work was merged, we start the freeze cycle: the first release + candidate is out. + +4. During the freeze cycle, no more features are merged. It's time to test + OfflineIMAP. New candidates version are released. The more we are late in -rc + releases the less patches are merged but bug fixes. + +5. When we think a release is stable enough, we restart from step 1. + + +Branching model +=============== + +The branching model with use in OfflineIMAP is very near from the Git project. +We use a topic oriented workflow. A topic may be one or more patches. + +The branches you'll find in the official repository are: + +* gh-pages +* master +* next +* pu +* maint + +gh-pages +-------- + +This comes from a feature offered by Github. We maintain the online home github +page using this branch. + +master +------ + +If you're not sure what branch you should use, this one is for you. This is the +mainline. Simple users should use this branch to follow OfflineIMAP's evolution. + +Usually, patches submitted to the mailing list should start off of this branch. + +next +---- + +Patches recently merged are good candidates for this branch. The content of next +is merged into the mainline (master) at release time for both stable and -rc +releases. + +When patches are sent to the mailing list, contributors discuss about them. Once +done and when patches looks ready for mainline, patches are first merged into +next. Advanced users and testers use this branch to test last merged patches +before they hit the mainline. This helps not introducing strong breackages +directly in master. + +pu +-- + +pu stands for "proposed updates". If a topic is not ready for master nor next, +it may be merged into pu. This branch only help developers to work on someone +else topic or an earlier pending topic. + +This branch is **not intended to be checkouted**; never. Even developers don't +do that. Due to the way pu is built you can't expect content there to work in +any way... unless you clearly want to run into troubles. + +Developers can extract a topic from this branch to work on it. See the following +section "Extract a topic from pu" in this documentation. + +maint +----- + +This is the maintenance branch. It gets its own releases starting from an old +stable release. It helps both users having troubles with last stable releases +and users not wanting latest features or so to still benefit from strong bug +fixes and security fixes. + + +Working with Git +================ + +Extract a topic from pu +----------------------- + +pu is built this way:: + + git checkout pu + git reset --keep next + git merge --no-ff -X theirs topic1 + git merge --no-ff -X theirs topic2 + git merge --no-ff -X theirs blue + git merge --no-ff -X theirs orange + ... + +As a consequence: + +1. Each topic merged uses a merge commit. A merge commit is a commit having 2 + ancestors. Actually, Git allows more than 2 parents but we don't use this + feature. It's intended. + +2. Paths in pu may mix up multiple versions if all the topics don't use the same + base commit. This is very often the case as topics aren't rebased: it guarantees + each topic is strictly identical to the last version sent to the mailing list. + No surprise. + + +What you need to extract a particular topic is the sha1 of the tip of that +branch (the last commit of the topic). Assume you want the branch of the topic +called 'blue'. First, look at the log given by this command:: + + git log --reverse --merges --parents origin/next..origin/pu + +With this command you ask for the log: + +* from next to pu +* in reverse order (older first) +* merge commits only +* with the sha1 of the ancestors + +In this list, find the topic you're looking for, basing you search on the lines +like:: + + Merge branch 'topic/name' into pu + +By convention, it has the form /. When you're at +it, pick the topic ancestor sha1. It's always the last sha1 in the line starting +by 'commit'. For you to know: + +* the first is the sha1 of the commit you see: the merge commit +* the following sha1 is the ancestor of the branch checkouted at merge time + (always the previous merged topic or the ancien next in our case) +* last is the branch merged + +Giving:: + + commit sha1_of_merge_commit sha1_of_ancient_pu sha1_of_topic_blue + +Then, you only have to checkout the topic from there:: + + git checkout -b blue sha1_of_topic_blue + +and you're done! You've just created a new branch called "blue" with the blue +content. Be aware this topic is almostly not updated against current next +branch. ,-) + + === API === From 8465c832c397a39acead54b0a234047d38f2fdd5 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 8 May 2011 18:37:07 +0200 Subject: [PATCH 103/817] FAQ: insert entry to explain how to test OfflineIMAP Signed-off-by: Nicolas Sebrecht --- docs/FAQ.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index cd612ac..1ea837b 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -417,6 +417,29 @@ written in Korn, so you’ll need ksh, pdksh, or mksh to run it:: Contributing ============ +How to test OfflineIMAP? +------------------------ + +We don't have a testing tool, for now. As a IMAP client, we need an available +IMAP server for that purpose. But it doesn't mean you can do anything. + +Recent patches are merged in the next branch before beeing in the mainline. Once +you have your own copy of the official repository, track this next branch:: + + git checkout -t origin/next + +Update this branch in a regular basis with:: + + git checkout next + git pull + +Notice you're not supposed to install OfflineIMAP each time. You may simply +run it like this:: + + ./offlineimap.py + +The choice is up to you. :-) + How to submit a patch? ---------------------- From 632f1fe61f9e7700ac46a5a077d94fe652be2a09 Mon Sep 17 00:00:00 2001 From: Daniel Shahaf Date: Sun, 8 May 2011 22:55:55 +0300 Subject: [PATCH 104/817] FAQ: add two entries concerning 'sslcacertfile' Add a FAQ entry about non-verifying SSL certificates by default, and another about how to generate a certificates file to feed to the 'sslcacertfile' repository configuration item. Signed-off-by: Daniel Shahaf Signed-off-by: Nicolas Sebrecht --- docs/FAQ.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index fcc55b3..ae4fbe2 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -22,6 +22,7 @@ Please feel free to ask questions and/or provide answers; send email to the .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _OfflineIMAP: https://github.com/nicolas33/offlineimap +.. _ssl.wrap_socket: http://docs.python.org/library/ssl.html#ssl.wrap_socket OfflineIMAP @@ -252,6 +253,33 @@ What is the mailbox name recorder (mbnames) for? Some mail readers, such as mutt, are not capable of automatically determining the names of your mailboxes. OfflineIMAP can help these programs by writing the names of the folders in a format you specify. See the example offlineimap.conf for details. +Does OfflineIMAP verify SSL certificates? +----------------------------------------- + +By default, no. However, as of version 6.3.2, it is possible to enforce verification +of SSL certificate on a per-repository basis by setting the `sslcacertfile` option in the +config file. (See the example offlineimap.conf for details.) + +How do I generate an `sslcacertfile` file? +------------------------------------------ + +The `sslcacertfile` file must contain an SSL certificate (or a concatenated +certificates chain) in PEM format. (See the documentation of +`ssl.wrap_socket`_'s `certfile` parameter for the gory details.) The following +command should generate a file in the proper format:: + + openssl s_client -CApath /etc/ssl/certs -connect ${hostname}:imaps -showcerts \ + | perl -ne 'print if /BEGIN/../END/; print STDERR if /return/' > $sslcacertfile + ^D + +Before using the resulting file, ensure that openssl verified the certificate +successfully. + +The path `/etc/ssl/certs` is not standardized; your system may store +SSL certificates elsewhere. (On some systems it may be in +`/usr/local/share/certs/`.) + + IMAP Server Notes ================= From c70a6217095ffd37f7ff928cf5cf59adeb1c43db Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 May 2011 22:46:08 +0200 Subject: [PATCH 105/817] Limit msg body length in debug output We were outputting full message bodies to the debug log (often stderr), and then again (as they go over the imaplib2 wire, imaplib logs everything too). Not only is quite a privacy issue when sending in debug logs but it can also freeze a console for quite some time. Plus it bloats debug logs A LOT. Only output the first and last 100 bytes of each message body to the debug log (we still get the full body from imaplib2 logging). This limits privacy issues when handing the log to someone else, but usually still contains all the interesting bits that we want to see in a log. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 50 ++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 9c91ddd..b8d91c7 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -203,17 +203,32 @@ class IMAPFolder(BaseFolder): try: imapobj.select(self.getfullname(), readonly = 1) res_type, data = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])') + assert res_type == 'OK', "Fetching message with UID '%d' failed" % uid + # data looks now e.g. [('320 (UID 17061 BODY[] + # {2565}','msgbody....')] we only asked for one message, + # and that msg is in data[0]. msbody is in [0][1] + + #NB & TODO: When the message on the IMAP server has been + #deleted in the mean time, it will respond with an 'OK' + #res_type, but it will simply not send any data. This will + #lead to a crash in the below line. We need urgently to + #detect this, protect from this and need to think about what + #to return in this case. Probably returning `None` in this + #case would be good. But we need to make sure that all + #Backends behave the same, and that we actually check the + #return value and behave accordingly. + data = data[0][1].replace("\r\n", "\n") + + if len(data)>200: + dbg_output = "%s...%s" % (str(data)[:150], + str(data)[-50:]) + else: + dbg_output = data + self.ui.debug('imap', "Returned object from fetching %d: '%s'" % + (uid, dbg_output)) finally: self.imapserver.releaseconnection(imapobj) - assert res_type == 'OK', "Fetching message with UID '%d' failed" % uid - # data looks now e.g. [('320 (UID 17061 BODY[] - # {2565}','msgbody....')] we only asked for one message, - # and that msg is in data[0]. msbody is in [0][1] - data = data[0][1] - self.ui.debug('imap', '%s bytes returned from fetching %d: %s...%s' % \ - (len(data), uid, data[0:100], data[-100:])) - return data.replace("\r\n", "\n") - + return data def getmessagetime(self, uid): return self.messagelist[uid]['time'] @@ -410,21 +425,24 @@ class IMAPFolder(BaseFolder): # get the date of the message file, so we can pass it to the server. date = self.getmessageinternaldate(content, rtime) - self.ui.debug('imap', 'savemessage: using date %s' % date) - content = re.sub("(?200: + dbg_output = "%s...%s" % (content[:150], + content[-50:]) + else: + dbg_output = content + + self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % + (date, dbg_output)) - # TODO: - append could raise a ValueError if the date is not in - # valid format...? (typ,dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) From b25a72f8c8b216d788225d3bf1796f5f1f6c1e7d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 9 May 2011 22:42:15 +0200 Subject: [PATCH 106/817] cleanup: remove uneeded imports Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 3 +-- offlineimap/folder/LocalStatusSQLite.py | 2 +- offlineimap/imapserver.py | 3 --- offlineimap/repository/LocalStatus.py | 1 - 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 7f26628..a989441 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -15,14 +15,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import threadutil, mbnames, CustomConfig, OfflineImapError +from offlineimap import mbnames, CustomConfig, OfflineImapError from offlineimap.repository import Repository from offlineimap.ui import getglobalui from offlineimap.threadutil import InstanceLimitedThread from subprocess import Popen, PIPE from threading import Event import os -import sys import traceback def getaccountlist(customconfig): diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index f470599..a57992b 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -17,7 +17,7 @@ import os.path import re from threading import Lock -from LocalStatus import LocalStatusFolder, magicline +from LocalStatus import LocalStatusFolder try: import sqlite3 as sqlite except: diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 6e58949..038103c 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -16,7 +16,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import imaplib2 as imaplib from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui from threading import * @@ -26,8 +25,6 @@ import hmac import socket import base64 -from StringIO import StringIO -from platform import system from socket import gaierror try: diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 7d38596..cc22dd2 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -17,7 +17,6 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from Base import BaseRepository -from offlineimap import folder from offlineimap.folder.LocalStatus import LocalStatusFolder from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder import os From a39128516b803f31965bbfad00646bff2a53fc62 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 9 May 2011 22:43:03 +0200 Subject: [PATCH 107/817] explicitly define symbols to import instead of 'import *' Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 4 +--- offlineimap/imapserver.py | 2 +- offlineimap/mbnames.py | 2 +- offlineimap/threadutil.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 5faf036..6c6142c 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -24,9 +24,7 @@ import subprocess from offlineimap.ui import getglobalui import threading from offlineimap import OfflineImapError -from offlineimap.imaplib2 import * -# Import the symbols we need that aren't exported by default -from offlineimap.imaplib2 import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num +from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, IMAP4_PORT, InternalDate, Mon2num try: import ssl diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 038103c..bfe3796 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -18,7 +18,7 @@ from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui -from threading import * +from threading import Lock, BoundedSemaphore import thread import time import hmac diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 5ff0f29..fb8af74 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -18,7 +18,7 @@ import os.path import re # for folderfilter -from threading import * +from threading import Lock boxes = {} config = None diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 7ffcbe6..cef2195 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import * +from threading import Lock, Thread, BoundedSemaphore from Queue import Queue, Empty import traceback import thread From 8a34edc8ca7657a4f88c5a2c624154ec750610f1 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 11 May 2011 18:25:13 +0200 Subject: [PATCH 108/817] cleanup import satements - conform to PEP8 - explicitly define symbols instead of 'import *' - remove unused import Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/IMAP.py | 8 ++++++-- offlineimap/ui/Curses.py | 2 +- offlineimap/ui/TTY.py | 2 +- offlineimap/ui/UIBase.py | 1 - offlineimap/ui/debuglock.py | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 18e3b4e..0bc84eb 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -20,8 +20,12 @@ from offlineimap.repository.Base import BaseRepository from offlineimap import folder, imaputil, imapserver from offlineimap.folder.UIDMaps import MappedIMAPFolder from offlineimap.threadutil import ExitNotifyThread -import re, types, os, netrc, errno -from threading import * +from threading import Event +import re +import types +import os +import netrc +import errno class IMAPRepository(BaseRepository): def __init__(self, reposname, account): diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index cceeaaa..9e0b9d5 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -21,7 +21,7 @@ import time import sys import os import signal -import curses, curses.panel, curses.textpad, curses.wrapper +import curses from Blinkenlights import BlinkenBase from UIBase import UIBase import offlineimap diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 61019c1..cba573f 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -19,7 +19,7 @@ from UIBase import UIBase from getpass import getpass import sys -from threading import * +from threading import Lock, currentThread class TTYUI(UIBase): def __init__(s, config, verbose = 0): diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 8047c15..4c60b2a 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -22,7 +22,6 @@ import sys import traceback import threading from StringIO import StringIO -from Queue import Empty import offlineimap debugtypes = {'':'Other offlineimap related sync messages', diff --git a/offlineimap/ui/debuglock.py b/offlineimap/ui/debuglock.py index 7873a1f..4f2a4f9 100644 --- a/offlineimap/ui/debuglock.py +++ b/offlineimap/ui/debuglock.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import * +from threading import Lock, currentThread import traceback logfile = open("/tmp/logfile", "wt") loglock = Lock() From d5493fe8942069520d0337975fc7dae9efda3b60 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 11 May 2011 19:34:59 +0200 Subject: [PATCH 109/817] threadutil: explicitly import get_ident from thread The python module thread is the low-level module we should avoid to use in favor of threading. We still need it to support old python because Thread.ident doesn't exist before python 2.6: http://docs.python.org/library/threading.html#threading.Thread.ident Make it clear we should avoid it. Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 9 ++++----- offlineimap/threadutil.py | 5 ++--- offlineimap/ui/Blinkenlights.py | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index bfe3796..e5c9c18 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -19,7 +19,7 @@ from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui from threading import Lock, BoundedSemaphore -import thread +from thread import get_ident # python < 2.6 support import time import hmac import socket @@ -167,11 +167,10 @@ class IMAPServer: # Try to find one that previously belonged to this thread # as an optimization. Start from the back since that's where # they're popped on. - threadid = thread.get_ident() imapobj = None for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] - if self.lastowner[tryobj] == threadid: + if self.lastowner[tryobj] == get_ident(): imapobj = tryobj del(self.availableconnections[i]) break @@ -179,7 +178,7 @@ class IMAPServer: imapobj = self.availableconnections[0] del(self.availableconnections[0]) self.assignedconnections.append(imapobj) - self.lastowner[imapobj] = thread.get_ident() + self.lastowner[imapobj] = get_ident() self.connectionlock.release() return imapobj @@ -266,7 +265,7 @@ class IMAPServer: self.connectionlock.acquire() self.assignedconnections.append(imapobj) - self.lastowner[imapobj] = thread.get_ident() + self.lastowner[imapobj] = get_ident() self.connectionlock.release() return imapobj except Exception, e: diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index cef2195..fe4b5fd 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -19,7 +19,7 @@ from threading import Lock, Thread, BoundedSemaphore from Queue import Queue, Empty import traceback -import thread +from thread import get_ident # python < 2.6 support import sys from offlineimap.ui import getglobalui @@ -149,7 +149,6 @@ class ExitNotifyThread(Thread): exited and to provide for the ability for it to find out why.""" def run(self): global exitthreads, profiledir - self.threadid = thread.get_ident() try: if not profiledir: # normal case Thread.run(self) @@ -164,7 +163,7 @@ class ExitNotifyThread(Thread): except SystemExit: pass prof.dump_stats( \ - profiledir + "/" + str(self.threadid) + "_" + \ + profiledir + "/" + str(get_ident()) + "_" + \ self.getName() + ".prof") except: self.setExitCause('EXCEPTION') diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py index 257dca2..2160100 100644 --- a/offlineimap/ui/Blinkenlights.py +++ b/offlineimap/ui/Blinkenlights.py @@ -18,7 +18,7 @@ from threading import RLock, currentThread from offlineimap.ui.UIBase import UIBase -import thread +from thread import get_ident # python < 2.6 support class BlinkenBase: """This is a mix-in class that should be mixed in with either UIBase @@ -103,7 +103,7 @@ class BlinkenBase: UIBase.threadExited(s, thread) def gettf(s): - threadid = thread.get_ident() + threadid = get_ident() accountname = s.getthreadaccount() s.tflock.acquire() From 136237b7dc99d7470a714fdbe146bcce93d36a2a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 11 May 2011 20:55:54 +0200 Subject: [PATCH 110/817] refactoring: simplify the semaphorewait logic The semaphorewait()/waitforthread() logic is usefull for IMAP starting connections. We actually use it in imapserver only. This patch removes the over-engineered factorized methods. It tend to simplify the code by cleaning out a chain of two direct calls with no other processes. Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 5 ----- offlineimap/imapserver.py | 3 ++- offlineimap/threadutil.py | 4 ---- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index a5fd242..608d361 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -37,11 +37,6 @@ class BaseFolder(object): false otherwise. Probably only IMAP will return true.""" return 0 - def waitforthread(self): - """For threading folders, waits until there is a resource available - before firing off a thread. For all others, returns immediately.""" - pass - def getcopyinstancelimit(self): """For threading folders, returns the instancelimitname for InstanceLimitedThreads.""" diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index e5c9c18..7703879 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -300,7 +300,8 @@ class IMAPServer: to copy messages, then have them all wait for 3 available connections. It's OK if we have maxconnections + 1 or 2 threads, which is what this will help us do.""" - threadutil.semaphorewait(self.semaphore) + self.semaphore.acquire() + self.semaphore.release() def close(self): # Make sure I own all the semaphores. Let the threads finish diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index fe4b5fd..24486e2 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -42,10 +42,6 @@ def semaphorereset(semaphore, originalstate): for i in range(originalstate): semaphore.release() -def semaphorewait(semaphore): - semaphore.acquire() - semaphore.release() - class threadlist: def __init__(self): self.lock = Lock() From 21875852eb2f7978d2ce24d8a93db995060caf49 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 11 May 2011 18:56:56 +0200 Subject: [PATCH 111/817] cleanup: remove unused initextnotify() function and inited variable Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 1 - offlineimap/threadutil.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 7aa8f0b..3aaa679 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -328,7 +328,6 @@ class OfflineImap: signal.signal(signal.SIGUSR2,sig_handler) #various initializations that need to be performed: - threadutil.initexitnotify() #TODO: Why? offlineimap.mbnames.init(config, syncaccounts) if options.singlethreading: diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 24486e2..65028ee 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -83,15 +83,6 @@ class threadlist: ###################################################################### exitthreads = Queue(100) -inited = 0 - -def initexitnotify(): - """Initialize the exit notify system. This MUST be called from the - SAME THREAD that will call monitorloop BEFORE it calls monitorloop. - This SHOULD be called before the main thread starts any other - ExitNotifyThreads, or else it may miss the ability to catch the exit - status from them!""" - pass def exitnotifymonitorloop(callback): """An infinite "monitoring" loop watching for finished ExitNotifyThread's. From 694e5772da70a6530e23d45414cfae95fc625545 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 12 May 2011 20:00:08 +0200 Subject: [PATCH 112/817] threadutil: remove broken os.exit() We are missing the import of 'os' python module since commit d839be3c61888a837b (Sat Apr 16 20:33:35 2005 +0100) which was when John switched from SVN to Git. Happily, it help us today: we still had no feedback for this missing import, 6 years later. So, we can remove the os.exit() call safely. That beeing said, we still don't know if the above sys.exit() was ever touched. My guess is that it never was. Keep this (hopefully) commented statement to ensure the thread terminate and not play too much with the Murphy's law. :-) Signed-off-by: Nicolas Sebrecht --- offlineimap/threadutil.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 65028ee..155a563 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -122,12 +122,10 @@ def threadexited(thread): raise SystemExit ui.threadException(thread) # Expected to terminate sys.exit(100) # Just in case... - os._exit(100) elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE': ui.terminate() # Just in case... sys.exit(100) - os._exit(100) else: ui.threadExited(thread) From 41c21f5a9f99a916b5a5c01b964e23632676d22b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 12 May 2011 20:59:26 +0200 Subject: [PATCH 113/817] fix: allow debugtype 'thread' from command line commit f7e9d95 [Imply single-threaded mode with -d(ebug) command line option] was broken. It pretends to imply singlethreading each time _unless_ user explicitly asks for the debug 'thread' option. Change the "force singlethreading" check from (force single threading if last column is true) +------------------------------------------+----------------+ | | | +---------------------------------+ | | |((SINGLETHREADING THREAD_MODE) | "AND") | "NOT" | +------------------+--------------+--------+----------------+ | True | True | True | False (wrong) | +------------------+--------------+--------+----------------+ | True | False | False | True | +------------------+--------------+--------+----------------+ | False | True | False | True (wrong) | +------------------+--------------+--------+----------------+ | False | False | False | True | +------------------------------------------+----------------+ To the correct one +--------------------------------------------------+-------+ | | | +-----------------------------------------+ | | |(("NOT" SINGLETHREADING THREAD_MODE) | "AND") | "NOT" | +-------------------------+---------------+--------+-------+ | False | True | False | True | +-------------------------+---------------+--------+-------+ | False | False | False | True | +-------------------------+---------------+--------+-------+ | True | True | True | False | +-------------------------+---------------+--------+-------+ | True | False | False | True | +-------------------------+---------------+----------------+ Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 3aaa679..7258365 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -218,7 +218,7 @@ class OfflineImap: options.debugtype = 'imap,maildir,thread' #force single threading? if not ('thread' in options.debugtype.split(',') \ - and options.singlethreading): + and not options.singlethreading): ui._msg("Debug mode: Forcing to singlethreaded.") options.singlethreading = True From 2d31abde76935057054711d1a3563f711a4250ec Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 May 2011 18:45:07 +0200 Subject: [PATCH 114/817] v6.3.4-rc1 Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 19 -------------- Changelog.rst | 56 +++++++++++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 66501e2..fd445ac 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,31 +13,12 @@ others. New Features ------------ -* Enable 1-way synchronization by settting a [Repository ...] to - readonly = True. When e.g. using offlineimap for backup purposes you - can thus make sure that no changes in your backup trickle back into - the main IMAP server. -* optional: experimental SQLite-based backend for the LocalStatus - cache. Plain text remains the default. Enable by setting - status_backend=sqlite in the [Account ...] section - Changes ------- -* Reduced our sync logic from 4 passes to 3 passes (integrating upload of - "new" and "existing" messages into one function). This should result in a - slight speedup. -* No whitespace is stripped from comma-separated arguments passed via - the -f option. -* Give more detailed error when encountering a corrupt UID mapping file. - Bug Fixes --------- -* Drop connection if synchronisation failed. This is needed if resuming the - system from suspend mode gives a wrong connection. -* Fix the offlineimap crash when invoking debug option 'thread' - Pending for the next major release ================================== diff --git a/Changelog.rst b/Changelog.rst index b2912b3..0f5d6d1 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,62 @@ ChangeLog releases announces. +OfflineIMAP v6.3.4-rc1 (2011-05-16) +=================================== + +Notes +----- + +Welcome to the v6.3.4 pre-release cycle. Your favorite IMAP tool wins 2 new +features which were asked for a long time: +* an experimental SQL-based backend for the local cache; +* one-way synchronization cabability. + +Logic synchronization is reviewed and simplified (from 4 to 3 passes) giving +improved performance. + +Lot of work was done to give OfflineIMAP a better code base. Raised errors can +now rely on a new error system and should become the default in the coming +releases. + +As usual, we ask our users to test this release as much as possible, especially +the SQL backend. Have fun! + +New Features +------------ + +* Begin sphinx-based documentation for the code. +* Enable 1-way synchronization by settting a [Repository ...] to + readonly = True. When e.g. using offlineimap for backup purposes you + can thus make sure that no changes in your backup trickle back into + the main IMAP server. +* Optional: experimental SQLite-based backend for the LocalStatus + cache. Plain text remains the default. + +Changes +------- + +* Start a enhanced error handling background system. This is designed to not + stop a whole sync process on all errors (not much used, yet). +* Documentation improvements: the FAQ wins new entries and add a new HACKING + file for developers. +* Lot of code cleanups. +* Reduced our sync logic from 4 passes to 3 passes (integrating upload of + "new" and "existing" messages into one function). This should result in a + slight speedup. +* No whitespace is stripped from comma-separated arguments passed via + the -f option. +* Give more detailed error when encountering a corrupt UID mapping file. + +Bug Fixes +--------- + +* Drop connection if synchronization failed. This is needed if resuming the + system from suspend mode gives a wrong connection. +* Fix the offlineimap crash when invoking debug option 'thread'. +* Make 'thread' command line option work. + + OfflineIMAP v6.3.3 (2011-04-24) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 0a55a97..d7313cc 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.3" +__version__ = "6.3.4-rc1" __copyright__ = "Copyright (C) 2002 - 2010 John Goerzen" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 89a5d25263c17f9dffa21880f7f8a04b0a79a292 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 17 May 2011 18:19:31 +0200 Subject: [PATCH 115/817] fix broken ui Blinkenlights in multi-threaded mode This is a regression introduced by commit d5493fe89420 [threadutil: explicitly import get_ident from thread]. The threadid attribute was wrongly removed from the ExitNotifyThread class. Restore it. Tested-by: Mark Foxwell Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/threadutil.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 155a563..bb3b63c 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -134,6 +134,7 @@ class ExitNotifyThread(Thread): exited and to provide for the ability for it to find out why.""" def run(self): global exitthreads, profiledir + self.threadid = get_ident() try: if not profiledir: # normal case Thread.run(self) @@ -148,7 +149,7 @@ class ExitNotifyThread(Thread): except SystemExit: pass prof.dump_stats( \ - profiledir + "/" + str(get_ident()) + "_" + \ + profiledir + "/" + str(self.threadid) + "_" + \ self.getName() + ".prof") except: self.setExitCause('EXCEPTION') From 52cefb582cd96e6a54297b2aa35c40a3b3e2a303 Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Thu, 19 May 2011 15:02:26 -0400 Subject: [PATCH 116/817] Recognize configuration for idlefolders Mark this option as experimental and document its shortcomings in MANUAL.rst. This code was originally by James Bunton . Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 11 +++++++++++ offlineimap.conf | 17 +++++++++++++++++ offlineimap/imapserver.py | 6 +++++- offlineimap/repository/IMAP.py | 12 +++++++++++- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 62b0d3b..26f31a6 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -288,6 +288,17 @@ KNOWN BUGS last stable version and send us a report to the mailing list including the full log. +* IDLE support is incomplete and experimental. Bugs may be encountered. + + * No hook exists for "run after an IDLE response". Email will + show up, but may not be processed until the next refresh cycle. + + * nametrans may not be supported correctly. + + * IMAP IDLE <-> IMAP IDLE doesn't work yet. + + * IDLE may only work "once" per refresh. If you encounter this bug, + please send a report to the list! SEE ALSO ======== diff --git a/offlineimap.conf b/offlineimap.conf index 33a0810..9329c66 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -364,6 +364,23 @@ remoteuser = username # # reference = Mail +# In between synchronisations, OfflineIMAP can monitor mailboxes for new +# messages using the IDLE command. If you want to enable this, specify here +# the folders you wish to monitor. Note that the IMAP protocol requires a +# separate connection for each folder monitored in this way, so setting +# this option will force settings for: +# maxconnections - to be at least the number of folders you give +# holdconnectionopen - to be true +# keepalive - to be 29 minutes unless you specify otherwise +# +# This feature isn't complete and may well have problems. BE AWARE THIS +# IS EXPERIMENTAL STUFF. See the manual for more details. +# +# This option should return a Python list. For example +# +# idlefolders = ['INBOX', 'INBOX.Alerts'] +# + # OfflineIMAP can use multiple connections to the server in order # to perform multiple synchronization actions simultaneously. # This may place a higher burden on the server. In most cases, diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 7703879..58e7e57 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -43,7 +43,7 @@ class IMAPServer: username = None, password = None, hostname = None, port = None, ssl = 1, maxconnections = 1, tunnel = None, reference = '""', sslclientcert = None, sslclientkey = None, - sslcacertfile= None): + sslcacertfile = None, idlefolders = []): self.ui = getglobalui() self.reposname = reposname self.config = config @@ -72,6 +72,7 @@ class IMAPServer: self.semaphore = BoundedSemaphore(self.maxconnections) self.connectionlock = Lock() self.reference = reference + self.idlefolders = idlefolders self.gss_step = self.GSS_STATE_STEP self.gss_vc = None self.gssapi = False @@ -386,6 +387,7 @@ class ConfigedIMAPServer(IMAPServer): sslclientkey = self.repos.getsslclientkey() sslcacertfile = self.repos.getsslcacertfile() reference = self.repos.getreference() + idlefolders = self.repos.getidlefolders() server = None password = None @@ -397,6 +399,7 @@ class ConfigedIMAPServer(IMAPServer): IMAPServer.__init__(self, self.config, self.repos.getname(), tunnel = usetunnel, reference = reference, + idlefolders = idlefolders, maxconnections = self.repos.getmaxconnections()) else: if not password: @@ -405,6 +408,7 @@ class ConfigedIMAPServer(IMAPServer): user, password, host, port, ssl, self.repos.getmaxconnections(), reference = reference, + idlefolders = idlefolders, sslclientcert = sslclientcert, sslclientkey = sslclientkey, sslcacertfile = sslcacertfile) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 0bc84eb..1d40a8f 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -79,9 +79,13 @@ class IMAPRepository(BaseRepository): self.imapserver.close() def getholdconnectionopen(self): + if self.getidlefolders(): + return 1 return self.getconfboolean("holdconnectionopen", 0) def getkeepalive(self): + if self.getidlefolders(): + return 29*60 return self.getconfint("keepalive", 0) def getsep(self): @@ -163,8 +167,14 @@ class IMAPRepository(BaseRepository): def getreference(self): return self.getconf('reference', '""') + def getidlefolders(self): + localeval = self.localeval + return localeval.eval(self.getconf('idlefolders', '[]')) + def getmaxconnections(self): - return self.getconfint('maxconnections', 1) + num1 = len(self.getidlefolders()) + num2 = self.getconfint('maxconnections', 1) + return max(num1, num2) def getexpunge(self): return self.getconfboolean('expunge', 1) From 9fa6bcdf82e08f59c7c92452d0696005d94bcf50 Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Thu, 19 May 2011 15:02:27 -0400 Subject: [PATCH 117/817] Add IdleThread class This encapsulates the logic for sending a keepalive/IDLE call, including starting a sync if needed. This code was originally by James Bunton . I modified the idle() method to put the select() call after acquireconnection(). Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 60 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 58e7e57..ebd867e 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -18,8 +18,9 @@ from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui -from threading import Lock, BoundedSemaphore +from threading import Lock, BoundedSemaphore, Thread, Event, currentThread from thread import get_ident # python < 2.6 support +import offlineimap.accounts import time import hmac import socket @@ -367,6 +368,63 @@ class IMAPServer: self.ui.debug('imap', 'keepalive: bottom of loop') +class IdleThread(object): + def __init__(self, parent, folder=None): + self.parent = parent + self.folder = folder + self.event = Event() + if folder is None: + self.thread = Thread(target=self.noop) + else: + self.thread = Thread(target=self.idle) + self.thread.setDaemon(1) + + def start(self): + self.thread.start() + + def stop(self): + self.event.set() + + def join(self): + self.thread.join() + + def noop(self): + imapobj = self.parent.acquireconnection() + imapobj.noop() + self.event.wait() + self.parent.releaseconnection(imapobj) + + def dosync(self): + remoterepos = self.parent.repos + account = remoterepos.account + localrepos = account.localrepos + remoterepos = account.remoterepos + statusrepos = account.statusrepos + remotefolder = remoterepos.getfolder(self.folder) + offlineimap.accounts.syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False) + ui = getglobalui() + ui.unregisterthread(currentThread()) + + def idle(self): + while True: + if self.event.isSet(): + return + self.needsync = False + def callback(args): + if not self.event.isSet(): + self.needsync = True + self.event.set() + imapobj = self.parent.acquireconnection() + imapobj.select(self.folder) + imapobj.idle(callback=callback) + self.event.wait() + if self.event.isSet(): + imapobj.noop() + self.parent.releaseconnection(imapobj) + if self.needsync: + self.event.clear() + self.dosync() + class ConfigedIMAPServer(IMAPServer): """This class is designed for easier initialization given a ConfigParser object and an account name. The passwordhash is used if From d47bd1ff890bc9612797666a0f328bb20be4c22e Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Thu, 19 May 2011 15:02:28 -0400 Subject: [PATCH 118/817] Change keepalive() to spawn IdleThreads This is the commit that enables IDLE support. In order to do this, we hijack the keepalive method. Instead of just sending NOOPs, it now sends IDLE and responds accordingly, thanks to the IdleThread class. This code was originally by James Bunton . Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index ebd867e..e5613fb 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -21,7 +21,6 @@ from offlineimap.ui import getglobalui from threading import Lock, BoundedSemaphore, Thread, Event, currentThread from thread import get_ident # python < 2.6 support import offlineimap.accounts -import time import hmac import socket import base64 @@ -330,8 +329,6 @@ class IMAPServer: self.ui.debug('imap', 'keepalive thread started') while 1: self.ui.debug('imap', 'keepalive: top of loop') - time.sleep(timeout) - self.ui.debug('imap', 'keepalive: after wait') if event.isSet(): self.ui.debug('imap', 'keepalive: event is set; exiting') return @@ -342,29 +339,27 @@ class IMAPServer: self.connectionlock.release() self.ui.debug('imap', 'keepalive: connectionlock released') threads = [] - imapobjs = [] for i in range(numconnections): self.ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections)) - imapobj = self.acquireconnection() - self.ui.debug('imap', 'keepalive: connection %d acquired' % i) - imapobjs.append(imapobj) - thr = threadutil.ExitNotifyThread(target = imapobj.noop) - thr.setDaemon(1) - thr.start() - threads.append(thr) + if len(self.idlefolders) > i: + idler = IdleThread(self, self.idlefolders[i]) + else: + idler = IdleThread(self) + idler.start() + threads.append(idler) self.ui.debug('imap', 'keepalive: thread started') + self.ui.debug('imap', 'keepalive: waiting for timeout') + event.wait(timeout) + self.ui.debug('imap', 'keepalive: after wait') + self.ui.debug('imap', 'keepalive: joining threads') - for thr in threads: + for idler in threads: # Make sure all the commands have completed. - thr.join() - - self.ui.debug('imap', 'keepalive: releasing connections') - - for imapobj in imapobjs: - self.releaseconnection(imapobj) + idler.stop() + idler.join() self.ui.debug('imap', 'keepalive: bottom of loop') From 36a06801915af61be8503807a465a39025c2b8b7 Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Sun, 22 May 2011 03:19:29 -0400 Subject: [PATCH 119/817] Add check for IDLE in capabilities If the server doesn't support IDLE, we fall back to the standard noop() keepalive. This commit was originally by James Bunton . Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index e5613fb..96e8d23 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -411,7 +411,15 @@ class IdleThread(object): self.event.set() imapobj = self.parent.acquireconnection() imapobj.select(self.folder) - imapobj.idle(callback=callback) + if "IDLE" in imapobj.capabilities: + imapobj.idle(callback=callback) + else: + ui = getglobalui() + ui.warn("IMAP IDLE not supported on connection to %s." + "Falling back to old behavior: sleeping until next" + "refresh cycle." + %(imapobj.identifier,)) + imapobj.noop() self.event.wait() if self.event.isSet(): imapobj.noop() From 2cc2ead503ba9491d41c36d8f9d410312171f6ef Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Thu, 19 May 2011 15:02:30 -0400 Subject: [PATCH 120/817] Let the user configure how long to IDLE for This commit was originally by James Bunton . Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/IMAP.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 1d40a8f..3517d88 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -84,9 +84,11 @@ class IMAPRepository(BaseRepository): return self.getconfboolean("holdconnectionopen", 0) def getkeepalive(self): - if self.getidlefolders(): + num = self.getconfint("keepalive", 0) + if num == 0 and self.getidlefolders(): return 29*60 - return self.getconfint("keepalive", 0) + else: + return num def getsep(self): return self.imapserver.delim From da36c5c1e7d4df6d191da468bce1654e0cc820f6 Mon Sep 17 00:00:00 2001 From: Tom Lawton Date: Thu, 19 May 2011 15:02:31 -0400 Subject: [PATCH 121/817] Handle abort messages from GMail Without this patch, we try to NOOP on a bad connection and crash messily. Signed-off-by: Tom Lawton Signed-off-by: Ethan Glasser-Camp Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 96e8d23..147e84b 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -405,10 +405,18 @@ class IdleThread(object): if self.event.isSet(): return self.needsync = False + self.imapaborted = False def callback(args): - if not self.event.isSet(): - self.needsync = True - self.event.set() + result, cb_arg, exc_data = args + if exc_data is None: + if not self.event.isSet(): + self.needsync = True + self.event.set() + else: + # We got an "abort" signal. + self.imapaborted = True + self.stop() + imapobj = self.parent.acquireconnection() imapobj.select(self.folder) if "IDLE" in imapobj.capabilities: @@ -422,7 +430,11 @@ class IdleThread(object): imapobj.noop() self.event.wait() if self.event.isSet(): - imapobj.noop() + # Can't NOOP on a bad connection. + if not self.imapaborted: + imapobj.noop() + # We don't do event.clear() so that we'll fall out + # of the loop next time around. self.parent.releaseconnection(imapobj) if self.needsync: self.event.clear() From fc6c12d96cc31827f49e227a0092cf4142b88e25 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 24 May 2011 23:45:39 +0200 Subject: [PATCH 122/817] improve message "error 111" if connection failed Raise OfflineImapError with severity REPO explaining that the connection failed. Before, no valuable information was given to the user. Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 8 ++++---- offlineimap/imapserver.py | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 6c6142c..f501af6 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -172,8 +172,8 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): else: self.sock.close() if last_error != 0: - # FIXME - raise socket.error(last_error) + raise Exception("can't open socket; error: %s"\ + % socket.error(last_error)) # Allow sending of keep-alive message seems to prevent some servers # from closing SSL on us leading to deadlocks @@ -276,8 +276,8 @@ class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): else: self.sock.close() if last_error != 0: - # FIXME - raise socket.error(last_error) + raise Exception("can't open socket; error: %s"\ + % socket.error(last_error)) self.file = self.sock.makefile('rb') # imaplib2 uses this to poll() diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 147e84b..37fc553 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -289,6 +289,13 @@ class IMAPServer: "ver name correctly and that you are online."\ % (self.hostname, self.reposname) raise OfflineImapError(reason, severity) + # Could not acquire connection to the remote; + # socket.error(last_error) raised + if str(e)[:24] == "can't open socket; error": + raise OfflineImapError("Could not connect to remote server '%s' "\ + "for repository '%s'. Remote does not answer." + % (self.hostname, self.reposname), + OfflineImapError.ERROR.REPO) else: # re-raise all other errors raise From e87f213046fafd2ad053050b5d60d9e5bdd841a9 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 25 May 2011 21:09:09 +0200 Subject: [PATCH 123/817] fix magicline import commit 0318c6a [Create LocalStatus or LocalStatusSQLite folders] changes import of LocalStatus but doesn't preserve magicline. Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/LocalStatus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index cc22dd2..022be21 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -17,7 +17,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from Base import BaseRepository -from offlineimap.folder.LocalStatus import LocalStatusFolder +from offlineimap.folder.LocalStatus import LocalStatusFolder, magicline from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder import os import re @@ -67,7 +67,7 @@ class LocalStatusRepository(BaseRepository): filename = self.getfolderfilename(foldername) file = open(filename + ".tmp", "wt") - file.write(offlineimap.folder.LocalStatus.magicline + '\n') + file.write(magicline + '\n') file.close() os.rename(filename + ".tmp", filename) # Invalidate the cache. From 3dc9fc519a6fbbb5aad57744de570bdba2bcdfb4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 6 Jun 2011 11:04:55 +0200 Subject: [PATCH 124/817] Throw errors on connection refused and on non-standard SSL ports We were "crashing" with tracebacks when we could not connect to a host, (e.g. because no service was on the port) and we were getting mysterious SSL tracebacks when someone tried to connect via SSL to a non-ssl port. In these cases, we will now throw an nice error message. On python<2.6 where no ssl module exists, we simply won't throw those errors. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 37fc553..e7892cc 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -24,9 +24,14 @@ import offlineimap.accounts import hmac import socket import base64 +import errno from socket import gaierror - +try: + from ssl import SSLError +except ImportError: + # Protect against python<2.6, use dummy and won't get SSL errors. + SSLError = None try: # do we have a recent pykerberos? have_gss = False @@ -280,14 +285,37 @@ class IMAPServer: if(self.connectionlock.locked()): self.connectionlock.release() - if type(e) == gaierror: + # now, check for known errors and throw OfflineImapErrors + severity = OfflineImapError.ERROR.REPO + if isinstance(e, gaierror): #DNS related errors. Abort Repo sync - severity = OfflineImapError.ERROR.REPO #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ - "ver name correctly and that you are online."\ - % (self.hostname, self.reposname) + "ver name correctly and that you are online."%\ + (self.hostname, self.reposname) + raise OfflineImapError(reason, severity) + + elif isinstance(e, SSLError) and e.errno == 1: + # SSL unknown protocol error + # happens e.g. when connecting via SSL to a non-SSL service + if self.port != 443: + reason = "Could not connect via SSL to host '%s' and non-s"\ + "tandard ssl port %d configured. Make sure you connect"\ + " to the correct port." % (self.hostname, self.port) + else: + reason = "Unknown SSL protocol connecting to host '%s' for"\ + "repository '%s'. OpenSSL responded:\n%s"\ + % (self.hostname, self.reposname, e) + raise OfflineImapError(reason, severity) + + elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: + # "Connection refused", can be a non-existing port, or an unauthorized + # webproxy (open WLAN?) + reason = "Connection to host '%s:%d' for repository '%s' was "\ + "refused. Make sure you have the right host and port "\ + "configured and that you are actually able to access the "\ + "network." % (self.hostname, self.port, self.reposname) raise OfflineImapError(reason, severity) # Could not acquire connection to the remote; # socket.error(last_error) raised From f3ec6d9c7d65d258d54952e89a6efa3d1933649a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 6 Jun 2011 11:37:24 +0200 Subject: [PATCH 125/817] Assert error hint when trying to create MailDir root http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=499755 shows the cryptic output we have when, e.g. trying to put somethin in our mailDir root via the nametrans options. This commit adds at least some hint as to what went wrong using an "assert" message, although the correct thing is to allow the creation of a maildir in the root folder. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Maildir.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 0af4d1f..18fa591 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -97,6 +97,7 @@ class MaildirRepository(BaseRepository): (foldername, subdir) else: self.debug("makefolder: calling makedirs %s" % foldername) + assert foldername != '', "Can not create root MailDir." os.makedirs(foldername, 0700) self.debug("makefolder: creating cur, new, tmp") for subdir in ['cur', 'new', 'tmp']: From 1754bf4110da251f4aa1dc0803889899b02bfcff Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 6 Jun 2011 11:37:25 +0200 Subject: [PATCH 126/817] Allow to create the root MailDir directory We currently do not allow nametrans rules such as nametrans = lambda foldername: re.sub('^INBOX$', '', foldername) because we crash with a traceback when running: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=499755 The underlying reason is that we cannot create the "top level" root directory of the Maildir in the function makefolders(), it will bail out. John Goerzen intentionally prevented offlineimap from creating the top-level dir, so that a misconfiguration could not arbitrarily create folders on the file system. I believe that it should be perfectly possible to automatically create the root dirctory of the maildir. We still protect against folder creations at arbitrary places in the file system though. This patch cleans up makefolders(), adds documentation, allows to automatically create rootfolders if needed (using absolute paths) and adds some robustness in case the folders already exist that we want to create (rather than simply crapping out). Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Maildir.py | 78 +++++++++++++++---------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 18fa591..2a10a93 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -65,46 +65,46 @@ class MaildirRepository(BaseRepository): return self.getconf('sep', '.').strip() def makefolder(self, foldername): - self.debug("makefolder called with arg " + repr(foldername)) - # Do the chdir thing so the call to makedirs does not make the - # self.root directory (we'd prefer to raise an error in that case), - # but will make the (relative) paths underneath it. Need to use - # makedirs to support a / separator. - self.debug("Is dir? " + repr(os.path.isdir(foldername))) - if self.getsep() == '/': - for invalid in ['new', 'cur', 'tmp', 'offlineimap.uidvalidity']: - for component in foldername.split('/'): - assert component != invalid, "When using nested folders (/ as a separator in the account config), your folder names may not contain 'new', 'cur', 'tmp', or 'offlineimap.uidvalidity'." + """Create new Maildir folder if necessary - assert foldername.find('./') == -1, "Folder names may not contain ../" + :param foldername: A relative mailbox name. The maildir will be + created in self.root+'/'+foldername. All intermediate folder + levels will be created if they do not exist yet. 'cur', + 'tmp', and 'new' subfolders will be created in the maildir. + """ + self.debug("makefolder called with arg '%s'" % (foldername)) + full_path = os.path.abspath(os.path.join(self.root, foldername)) + + # sanity tests + if self.getsep() == '/': + for component in foldername.split('/'): + assert not component in ['new', 'cur', 'tmp'],\ + "When using nested folders (/ as a Maildir separator), "\ + "folder names may not contain 'new', 'cur', 'tmp'." + assert foldername.find('../') == -1, "Folder names may not contain ../" assert not foldername.startswith('/'), "Folder names may not begin with /" - oldcwd = os.getcwd() - os.chdir(self.root) - - # If we're using hierarchical folders, it's possible that sub-folders - # may be created before higher-up ones. If this is the case, - # makedirs will fail because the higher-up dir already exists. - # So, check to see if this is indeed the case. - - if (self.getsep() == '/' or self.getconfboolean('existsok', 0) or foldername == '.') \ - and os.path.isdir(foldername): - self.debug("makefolder: %s already is a directory" % foldername) - # Already exists. Sanity-check that it's not a Maildir. - for subdir in ['cur', 'new', 'tmp']: - assert not os.path.isdir(os.path.join(foldername, subdir)), \ - "Tried to create folder %s but it already had dir %s" %\ - (foldername, subdir) - else: - self.debug("makefolder: calling makedirs %s" % foldername) - assert foldername != '', "Can not create root MailDir." - os.makedirs(foldername, 0700) - self.debug("makefolder: creating cur, new, tmp") + # If we're using hierarchical folders, it's possible that + # sub-folders may be created before higher-up ones. + self.debug("makefolder: calling makedirs '%s'" % full_path) + try: + os.makedirs(full_path, 0700) + except OSError, e: + if e.errno == 17 and os.path.isdir(full_path): + self.debug("makefolder: '%s' already a directory" % foldername) + else: + raise for subdir in ['cur', 'new', 'tmp']: - os.mkdir(os.path.join(foldername, subdir), 0700) - # Invalidate the cache + try: + os.mkdir(os.path.join(full_path, subdir), 0700) + except OSError, e: + if e.errno == 17 and os.path.isdir(full_path): + self.debug("makefolder: '%s' already has subdir %s" % + (foldername, sudir)) + else: + raise + # Invalidate the folder cache self.folders = None - os.chdir(oldcwd) def deletefolder(self, foldername): self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername) @@ -132,11 +132,11 @@ class MaildirRepository(BaseRepository): self.debug(" toppath = %s" % toppath) - # Iterate over directories in top. - for dirname in os.listdir(toppath) + ['.']: + # Iterate over directories in top & top itself. + for dirname in os.listdir(toppath) + [toppath]: self.debug(" *** top of loop") self.debug(" dirname = %s" % dirname) - if dirname in ['cur', 'new', 'tmp', 'offlineimap.uidvalidity']: + if dirname in ['cur', 'new', 'tmp']: self.debug(" skipping this dir (Maildir special)") # Bypass special files. continue @@ -162,7 +162,7 @@ class MaildirRepository(BaseRepository): retval.append(folder.Maildir.MaildirFolder(self.root, foldername, self.getsep(), self, self.accountname, self.config)) - if self.getsep() == '/' and dirname != '.': + if self.getsep() == '/' and dirname: # Check sub-directories for folders. retval.extend(self._getfolders_scandir(root, foldername)) self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ From b20f5192fbef6d82ff4aef7d2beaacbb0a9d8864 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 6 Jun 2011 13:12:23 +0200 Subject: [PATCH 127/817] Allow to specify remote hostname even for the Gmail case Previously we hard-coded the imap server name in the case of Gmail repositories, but often we need a different host name. So, allow people to specify the hostname via the regular "remotehosteval" and "remotehost" settings, and only falling back to imap.gmail.com when nothing has been specified. Cache the hostname, so we don't evaluate the whole thing each time we query the host name. Make the remotehosteval processing more robust, by catching any Exceptions that occur, and throw a OfflineImapError, that explains where exactly the error had occured. You can test this, e.g. by setting remotehosteval to 1/"n" or some other invalid expression. The whole IMAP.gethost() function has been documented code wise while going through. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 2 +- offlineimap/repository/Gmail.py | 28 ++++++++++++++++------------ offlineimap/repository/IMAP.py | 32 +++++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 594f864..34f2f4a 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -21,7 +21,7 @@ import traceback from offlineimap import CustomConfig from offlineimap.ui import getglobalui -class BaseRepository(CustomConfig.ConfigHelperMixin): +class BaseRepository(object, CustomConfig.ConfigHelperMixin): def __init__(self, reposname, account): self.ui = getglobalui() self.account = account diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 97637b8..fd95361 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -16,33 +16,37 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap.repository.IMAP import IMAPRepository -from offlineimap import folder +from offlineimap import folder, OfflineImapError class GmailRepository(IMAPRepository): """Gmail IMAP repository. - Uses hard-coded host name and port, see: - http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814 + Falls back to hard-coded gmail host name and port, if none were specified: + http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814 """ - - #: Gmail IMAP server hostname + # Gmail IMAP server hostname HOSTNAME = "imap.gmail.com" - - #: Gmail IMAP server port + # Gmail IMAP server port PORT = 993 def __init__(self, reposname, account): """Initialize a GmailRepository object.""" - account.getconfig().set('Repository ' + reposname, - 'remotehost', GmailRepository.HOSTNAME) - account.getconfig().set('Repository ' + reposname, - 'remoteport', GmailRepository.PORT) + # Enforce SSL usage account.getconfig().set('Repository ' + reposname, 'ssl', 'yes') IMAPRepository.__init__(self, reposname, account) def gethost(self): - return GmailRepository.HOSTNAME + """Return the server name to connect to. + + Gmail implementation first checks for the usual IMAP settings + and falls back to imap.gmail.com if not specified.""" + try: + return super(GmailRepository, self).gethost() + except OfflineImapError: + # nothing was configured, cache and return hardcoded one + self._host = GmailRepository.HOSTNAME + return self._host def getport(self): return GmailRepository.PORT diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 3517d88..82d9e32 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -17,7 +17,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap.repository.Base import BaseRepository -from offlineimap import folder, imaputil, imapserver +from offlineimap import folder, imaputil, imapserver, OfflineImapError from offlineimap.folder.UIDMaps import MappedIMAPFolder from offlineimap.threadutil import ExitNotifyThread from threading import Event @@ -32,6 +32,7 @@ class IMAPRepository(BaseRepository): """Initialize an IMAPRepository object.""" BaseRepository.__init__(self, reposname, account) # self.ui is being set by the BaseRepository + self._host = None self.imapserver = imapserver.ConfigedIMAPServer(self) self.folders = None self.nametrans = lambda foldername: foldername @@ -94,17 +95,34 @@ class IMAPRepository(BaseRepository): return self.imapserver.delim def gethost(self): - host = None - localeval = self.localeval + """Return the configured hostname to connect to + :returns: hostname as string or throws Exception""" + if self._host: # use cached value if possible + return self._host + + # 1) check for remotehosteval setting if self.config.has_option(self.getsection(), 'remotehosteval'): host = self.getconf('remotehosteval') + try: + host = self.localeval.eval(host) + except Exception, e: + raise OfflineImapError("remotehosteval option for repository "\ + "'%s' failed:\n%s" % (self, e), + OfflineImapError.ERROR.REPO) + if host: + self._host = host + return self._host + # 2) check for plain remotehost setting + host = self.getconf('remotehost', None) if host != None: - return localeval.eval(host) + self._host = host + return self._host - host = self.getconf('remotehost') - if host != None: - return host + # no success + raise OfflineImapError("No remote host for repository "\ + "'%s' specified." % self, + OfflineImapError.ERROR.REPO) def getuser(self): user = None From 846070b240b095f7c56e13b34d9a7ecd12e75e18 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 2 Jun 2011 16:34:20 +0200 Subject: [PATCH 128/817] Fix typos in months names All months names are 3-letter abbreviated, but accidentally June and July slipped through. Thanks to the heads up by Philipp Kern . Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index b8d91c7..8dfca7a 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -127,7 +127,7 @@ class IMAPFolder(BaseFolder): #format this manually - otherwise locales could cause problems monthnames_standard = ["Jan", "Feb", "Mar", "Apr", "May", \ - "June", "July", "Aug", "Sep", "Oct", "Nov", "Dec"] + "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] our_monthname = monthnames_standard[oldest_time_struct[1]-1] daystr = "%(day)02d" % {'day' : oldest_time_struct[2]} From e8b633b88459db3512e85d444a95f283c254a5f9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 6 Jun 2011 23:21:47 +0200 Subject: [PATCH 129/817] folder/IMAP: Remove buggy duplicate assignment we do: for msgid in imapdata: maxmsgid = max(long(msgid), maxmsgid) and then basically immediately: maxmsgid = long(imapdata[0]) throwing away the first assignment although the first method of assigning is the correct one. The second had been forgotten to be removed when we introduced the above iteration. This bug would fix a regression with those broken ZIMBRA servers that send multiple EXISTS replies. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8dfca7a..8851b5b 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -164,13 +164,11 @@ class IMAPFolder(BaseFolder): maxmsgid = 0 for msgid in imapdata: maxmsgid = max(long(msgid), maxmsgid) - - maxmsgid = long(imapdata[0]) - messagesToFetch = '1:%d' % maxmsgid; - if maxmsgid < 1: #no messages; return return + messagesToFetch = '1:%d' % maxmsgid; + # Now, get the flags and UIDs for these. # We could conceivably get rid of maxmsgid and just say # '1:*' here. From 520e39d35536172d03b22d012cf4d8828687f2ff Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 9 Jun 2011 15:26:14 +0200 Subject: [PATCH 130/817] Update imaplib2 to 2.24 In some cases we had offlineimap trying to delete emails that shouldn't be deleted. E.g. https://bugzilla.redhat.com/show_bug.cgi?id=708898. It turns out that imaplib2 does not like FETCH responses that are interrupted by other unsolicited server responses, e.g. * OK Searched 43% of the mailbox, ETA 0:12\r\n Bump imaplib2 to a version that can cope with these (legal) responses by the IMAP server. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 325 +++++++++++++++++++++++++--------------- 1 file changed, 207 insertions(+), 118 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index be34025..ec6cd0d 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.22" +__version__ = "2.24" __release__ = "2" -__revision__ = "22" +__revision__ = "24" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -36,9 +36,11 @@ Improved untagged responses handling suggested by Dave Baggett June 2010. Improved timeout handling contributed by Ivan Vovnenko October 2010. Timeout handling further improved by Ethan Glasser-Camp December 2010. -Time2Internaldate() patch to match RFC2060 specification of English month names from http://bugs.python.org/issue11024 March 2011.""" +Time2Internaldate() patch to match RFC2060 specification of English month names from bugs.python.org/issue11024 March 2011. +starttls() bug fixed with the help of Sebastian Spaeth April 2011. +Threads now set the "daemon" flag (suggested by offlineimap-project).""" __author__ = "Piers Lauder " -__URL__ = "http://janeelix.com/piers/python/imaplib2" +__URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" import binascii, errno, os, Queue, random, re, select, socket, sys, time, threading, zlib @@ -55,6 +57,9 @@ IMAP4_SSL_PORT = 993 IDLE_TIMEOUT_RESPONSE = '* IDLE TIMEOUT\r\n' IDLE_TIMEOUT = 60*29 # Don't stay in IDLE state longer READ_POLL_TIMEOUT = 30 # Without this timeout interrupted network connections can hang reader +READ_SIZE = 32768 # Consume all available in socket + +DFLT_DEBUG_BUF_LVL = 3 # Level above which the logging output goes directly to stderr AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first @@ -133,6 +138,7 @@ class Request(object): """Private class to represent a request awaiting response.""" def __init__(self, parent, name=None, callback=None, cb_arg=None): + self.parent = parent self.name = name self.callback = callback # Function called to process result self.callback_arg = cb_arg # Optional arg passed to "callback" @@ -147,12 +153,16 @@ class Request(object): def abort(self, typ, val): + """Called whenever we abort a command + + Sets self.aborted reason, and deliver()s nothing""" self.aborted = (typ, val) self.deliver(None) def get_response(self, exc_fmt=None): self.callback = None + if __debug__: self.parent._log(3, '%s:%s.ready.wait' % (self.name, self.tag)) self.ready.wait() if self.aborted is not None: @@ -171,6 +181,7 @@ class Request(object): self.response = response self.ready.set() + if __debug__: self.parent._log(3, '%s:%s.ready.set' % (self.name, self.tag)) @@ -180,14 +191,15 @@ class IMAP4(object): """Threaded IMAP4 client class. Instantiate with: - IMAP4(host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None) + IMAP4(host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None) - host - host's name (default: localhost); - port - port number (default: standard IMAP4 port); - debug - debug level (default: 0 - no debug); - debug_file - debug stream (default: sys.stderr); - identifier - thread identifier prefix (default: host); - timeout - timeout in seconds when expecting a command response (default: no timeout). + host - host's name (default: localhost); + port - port number (default: standard IMAP4 port); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response (default: no timeout), + debug_buf_lvl - debug level at which buffering is turned off. All IMAP4rev1 commands are supported by methods of the same name. @@ -267,7 +279,7 @@ class IMAP4(object): untagged_status_cre = re.compile(r'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?') - def __init__(self, host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None): + def __init__(self, host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): self.state = NONAUTH # IMAP4 protocol state self.literal = None # A literal argument to a command @@ -295,7 +307,7 @@ class IMAP4(object): + self.tagpre + r'\d+) (?P[A-Z]+) (?P.*)') - if __debug__: self._init_debug(debug, debug_file) + if __debug__: self._init_debug(debug, debug_file, debug_buf_lvl) self.resp_timeout = timeout # Timeout waiting for command response @@ -303,6 +315,7 @@ class IMAP4(object): self.read_poll_timeout = timeout else: self.read_poll_timeout = READ_POLL_TIMEOUT + self.read_size = READ_SIZE # Open socket to server. @@ -321,20 +334,26 @@ class IMAP4(object): if self.identifier: self.identifier += ' ' - self.Terminate = False + self.Terminate = self.TerminateReader = False self.state_change_free = threading.Event() self.state_change_pending = threading.Lock() self.commands_lock = threading.Lock() + """commands_lock prevents self.untagged_responses to be + manipulated concurrently""" + self.idle_lock = threading.Lock() self.ouq = Queue.Queue(10) self.inq = Queue.Queue() self.wrth = threading.Thread(target=self._writer) + self.wrth.setDaemon(True) self.wrth.start() self.rdth = threading.Thread(target=self._reader) + self.rdth.setDaemon(True) self.rdth.start() self.inth = threading.Thread(target=self._handler) + self.inth.setDaemon(True) self.inth.start() # Get server welcome message, @@ -387,8 +406,8 @@ class IMAP4(object): This connection will be used by the routines: read, send, shutdown, socket.""" - self.host = host is not None and host or '' - self.port = port is not None and port or IMAP4_PORT + self.host = self._choose_nonull_or_dflt('', host) + self.port = self._choose_nonull_or_dflt(IMAP4_PORT, port) self.sock = self.open_socket() self.read_fd = self.sock.fileno() @@ -490,7 +509,7 @@ class IMAP4(object): self.start_compressing() if __debug__: self._log(1, 'Enabled COMPRESS=DEFLATE') finally: - self.state_change_pending.release() + self._release_state_change() def pop_untagged_responses(self): @@ -515,16 +534,9 @@ class IMAP4(object): else list of RECENT responses, most recent last.""" name = 'RECENT' - - data = [] - while True: - dat = self._get_untagged_response(name) - if not dat: - break - data += dat - - if data: - return self._deliver_dat(name, data, kw) + typ, dat = self._untagged_response(None, [None], name) + if dat != [None]: + return self._deliver_dat(typ, dat, kw) kw['untagged_response'] = name return self.noop(**kw) # Prod server for response @@ -564,7 +576,7 @@ class IMAP4(object): try: return self._simple_command(name, mailbox, flags, date_time, **kw) finally: - self.state_change_pending.release() + self._release_state_change() def authenticate(self, mechanism, authobject, **kw): @@ -592,7 +604,7 @@ class IMAP4(object): self.state = AUTH if __debug__: self._log(1, 'state => AUTH') finally: - self.state_change_pending.release() + self._release_state_change() return self._deliver_dat(typ, dat, kw) @@ -626,7 +638,7 @@ class IMAP4(object): finally: self.state = AUTH if __debug__: self._log(1, 'state => AUTH') - self.state_change_pending.release() + self._release_state_change() return self._deliver_dat(typ, dat, kw) @@ -753,7 +765,7 @@ class IMAP4(object): try: return self._simple_command(name, **kw) finally: - self.state_change_pending.release() + self._release_state_change() def list(self, directory='""', pattern='*', **kw): @@ -782,7 +794,7 @@ class IMAP4(object): self.state = AUTH if __debug__: self._log(1, 'state => AUTH') finally: - self.state_change_pending.release() + self._release_state_change() return self._deliver_dat(typ, dat, kw) @@ -810,14 +822,15 @@ class IMAP4(object): if __debug__: self._log(1, 'state => LOGOUT') try: - typ, dat = self._simple_command('LOGOUT') - except: - typ, dat = 'NO', ['%s: %s' % sys.exc_info()[:2]] - if __debug__: self._log(1, dat) + try: + typ, dat = self._simple_command('LOGOUT') + except: + typ, dat = 'NO', ['%s: %s' % sys.exc_info()[:2]] + if __debug__: self._log(1, dat) - self._close_threads() - - self.state_change_pending.release() + self._close_threads() + finally: + self._release_state_change() if __debug__: self._log(1, 'connection closed') @@ -882,7 +895,7 @@ class IMAP4(object): try: return self._simple_command('PROXYAUTH', user, **kw) finally: - self.state_change_pending.release() + self._release_state_change() def rename(self, oldmailbox, newmailbox, **kw): @@ -937,7 +950,7 @@ class IMAP4(object): self.state = SELECTED if __debug__: self._log(1, 'state => SELECTED') finally: - self.state_change_pending.release() + self._release_state_change() if self._get_untagged_response('READ-ONLY', leave=True) and not readonly: if __debug__: self._dump_ur(1) @@ -953,7 +966,7 @@ class IMAP4(object): try: return self._simple_command('SETACL', mailbox, who, what, **kw) finally: - self.state_change_pending.release() + self._release_state_change() def setannotation(self, *args, **kw): @@ -972,7 +985,7 @@ class IMAP4(object): try: return self._simple_command('SETQUOTA', root, limits, **kw) finally: - self.state_change_pending.release() + self._release_state_change() def sort(self, sort_criteria, charset, *search_criteria, **kw): @@ -998,24 +1011,47 @@ class IMAP4(object): if hasattr(self, '_tls_established') and self._tls_established: raise self.abort('TLS session already established') + # Must now shutdown reader thread after next response, and restart after changing read_fd + + self.read_size = 1 # Don't consume TLS handshake + self.TerminateReader = True + try: typ, dat = self._simple_command(name) finally: - self.state_change_pending.release() + self._release_state_change() + self.rdth.join() + self.TerminateReader = False + self.read_size = READ_SIZE - if typ == 'OK': - import ssl - self.sock = ssl.wrap_socket(self.sock, keyfile, certfile) - self.read_fd = self.sock.fileno() - - typ, dat = self.capability() - if dat == [None]: - raise self.error('no CAPABILITY response from server') - self.capabilities = tuple(dat[-1].upper().split()) - self._tls_established = True - else: + if typ != 'OK': + # Restart reader thread and error + self.rdth = threading.Thread(target=self._reader) + self.rdth.setDaemon(True) + self.rdth.start() raise self.error("Couldn't establish TLS session: %s" % dat) + try: + try: + import ssl + self.sock = ssl.wrap_socket(self.sock, keyfile, certfile) + except ImportError: + self.sock = socket.ssl(self.sock, keyfile, certfile) + + self.read_fd = self.sock.fileno() + finally: + # Restart reader thread + self.rdth = threading.Thread(target=self._reader) + self.rdth.setDaemon(True) + self.rdth.start() + + typ, dat = self.capability() + if dat == [None]: + raise self.error('no CAPABILITY response from server') + self.capabilities = tuple(dat[-1].upper().split()) + + self._tls_established = True + typ, dat = self._untagged_response(typ, dat, name) return self._deliver_dat(typ, dat, kw) @@ -1046,7 +1082,7 @@ class IMAP4(object): try: return self._simple_command('SUBSCRIBE', mailbox, **kw) finally: - self.state_change_pending.release() + self._release_state_change() def thread(self, threading_algorithm, charset, *search_criteria, **kw): @@ -1081,7 +1117,7 @@ class IMAP4(object): try: return self._simple_command('UNSUBSCRIBE', mailbox, **kw) finally: - self.state_change_pending.release() + self._release_state_change() def xatom(self, name, *args, **kw): @@ -1096,8 +1132,7 @@ class IMAP4(object): try: return self._simple_command(name, *args, **kw) finally: - if self.state_change_pending.locked(): - self.state_change_pending.release() + self._release_state_change() @@ -1105,34 +1140,29 @@ class IMAP4(object): def _append_untagged(self, typ, dat): + """Append new untagged response - # Append new 'dat' to end of last untagged response if same 'typ', - # else append new response. - + Append new 'dat' to end of last untagged response if same 'typ', + else append new response.""" if dat is None: dat = '' + ur_data = [] - self.commands_lock.acquire() + self.commands_lock.acquire() # protect untagged_responses - if self.untagged_responses: - urn, urd = self.untagged_responses[-1] - if urn != typ: - urd = None + if self.untagged_responses and self.untagged_responses[-1][0] == typ: + # last respons is of type 'typ', get ur_data for appending + ur_data = self.untagged_responses[-1][1] else: - urd = None - - if urd is None: - urd = [] - self.untagged_responses.append([typ, urd]) - - urd.append(dat) + # need to create new untagged response of this type + self.untagged_responses.append([typ, ur_data]) + ur_data.append(dat) self.commands_lock.release() - - if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%s"]' % (typ, len(urd)-1, dat)) + if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%s"]' % (typ, len(ur_data)-1, dat)) def _check_bye(self): - + """raise Exception if untagged responses contains a 'BYE'""" bye = self._get_untagged_response('BYE', leave=True) if bye: raise self.abort(bye[-1]) @@ -1152,6 +1182,16 @@ class IMAP4(object): return self._quote(arg) + def _choose_nonull_or_dflt(self, dflt, *args): + dflttyp = type(dflt) + for arg in args: + if arg is not None: + if type(arg) is dflttyp: + return arg + if __debug__: self._log(1, 'bad arg type is %s, expecting %s' % (type(arg), dflttyp)) + return dflt + + def _command(self, name, *args, **kw): if Commands[name][CMD_VAL_ASYNC]: @@ -1161,12 +1201,14 @@ class IMAP4(object): if __debug__: self._log(1, '[%s] %s %s' % (cmdtyp, name, args)) + if __debug__: self._log(3, 'state_change_pending.acquire') self.state_change_pending.acquire() self._end_idle() if cmdtyp == 'async': self.state_change_pending.release() + if __debug__: self._log(3, 'state_change_pending.release') else: # Need to wait for all async commands to complete self._check_bye() @@ -1178,9 +1220,9 @@ class IMAP4(object): need_event = False self.commands_lock.release() if need_event: - if __debug__: self._log(4, 'sync command %s waiting for empty commands Q' % name) + if __debug__: self._log(3, 'sync command %s waiting for empty commands Q' % name) self.state_change_free.wait() - if __debug__: self._log(4, 'sync command %s proceeding' % name) + if __debug__: self._log(3, 'sync command %s proceeding' % name) if self.state not in Commands[name][CMD_VAL_STATES]: self.literal = None @@ -1232,7 +1274,7 @@ class IMAP4(object): # Wait for continuation response ok, data = crqb.get_response('command: %s => %%s' % name) - if __debug__: self._log(3, 'continuation => %s, %s' % (ok, data)) + if __debug__: self._log(4, 'continuation => %s, %s' % (ok, data)) # NO/BAD response? @@ -1316,18 +1358,25 @@ class IMAP4(object): def _end_idle(self): + self.idle_lock.acquire() irqb = self.idle_rqb if irqb is None: + self.idle_lock.release() return self.idle_rqb = None self.idle_timeout = None + self.idle_lock.release() irqb.data = 'DONE%s' % CRLF self.ouq.put(irqb) if __debug__: self._log(2, 'server IDLE finished') def _get_untagged_response(self, name, leave=False): + """Return an untagged response of type 'name' + :param leave: If leave (default: False) is True, we keep the + fetched responsem; otherwise it will be deleted. Returns + None if no such response found.""" self.commands_lock.acquire() for i, (typ, dat) in enumerate(self.untagged_responses): @@ -1434,7 +1483,7 @@ class IMAP4(object): self._append_untagged(typ, dat) - if typ != 'OK': + if typ != 'OK': # NO, BYE, IDLE self._end_idle() # Bracketed response information? @@ -1460,14 +1509,22 @@ class IMAP4(object): return '"%s"' % arg.replace('\\', '\\\\').replace('"', '\\"') + def _release_state_change(self): + + if self.state_change_pending.locked(): + self.state_change_pending.release() + if __debug__: self._log(3, 'state_change_pending.release') + + def _request_pop(self, name, data): - if __debug__: self._log(4, '_request_pop(%s, %s)' % (name, data)) self.commands_lock.acquire() rqb = self.tagged_commands.pop(name) if not self.tagged_commands: + if __debug__: self._log(3, 'state_change_free.set') self.state_change_free.set() self.commands_lock.release() + if __debug__: self._log(4, '_request_pop(%s, %s) = %s' % (name, data, rqb.tag)) rqb.deliver(data) @@ -1479,7 +1536,7 @@ class IMAP4(object): tag = rqb.tag self.tagged_commands[tag] = rqb self.commands_lock.release() - if __debug__: self._log(4, '_request_push(%s, %s, %s)' % (tag, name, `kw`)) + if __debug__: self._log(4, '_request_push(%s, %s, %s) = %s' % (tag, name, `kw`, rqb.tag)) return rqb @@ -1493,12 +1550,28 @@ class IMAP4(object): def _untagged_response(self, typ, dat, name): + """Returns an untagged response for 'name' of type 'typ' + :param typ: 'OK, 'NO', etc... which will be used for the type of + the response. + :param dat: The fallback data to be used in case `typ` is + 'NO'. Otherwise the data from the existing untagged + responses will be searched for data to be returned. If there + is no such response, we return `[None]` as data. + :param name: The name of the response. + :returns: (typ, data) + """ if typ == 'NO': return typ, dat data = self._get_untagged_response(name) if not data: return typ, [None] + while True: + dat = self._get_untagged_response(name) + if not dat: + break + data += dat + if __debug__: self._log(4, '_untagged_response(%s, ?, %s) => %s' % (typ, name, data)) return typ, data @@ -1588,6 +1661,7 @@ class IMAP4(object): rqb.abort(typ, val) self.state_change_free.set() self.commands_lock.release() + if __debug__: self._log(3, 'state_change_free.set') if __debug__: self._log(1, 'finished') @@ -1615,9 +1689,10 @@ class IMAP4(object): poll.register(self.read_fd, select.POLLIN) rxzero = 0 + terminate = False read_poll_timeout = self.read_poll_timeout * 1000 # poll() timeout is in millisecs - while not self.Terminate: + while not (terminate or self.Terminate): if self.state == LOGOUT: timeout = 1 else: @@ -1631,7 +1706,7 @@ class IMAP4(object): fd,state = r[0] if state & select.POLLIN: - data = self.read(32768) # Drain ssl buffer if present + data = self.read(self.read_size) # Drain ssl buffer if present start = 0 dlen = len(data) if __debug__: self._log(5, 'rcvd %s' % dlen) @@ -1652,6 +1727,8 @@ class IMAP4(object): '', stop, line_part + data[start:stop] if __debug__: self._log(4, '< %s' % line) self.inq.put(line) + if self.TerminateReader: + terminate = True if state & ~(select.POLLIN): raise IOError(poll_error(state)) @@ -1682,20 +1759,20 @@ class IMAP4(object): line_part = '' rxzero = 0 - read_poll_timeout = self.read_poll_timeout + terminate = False - while not self.Terminate: + while not (terminate or self.Terminate): if self.state == LOGOUT: timeout = 1 else: - timeout = read_poll_timeout + timeout = self.read_poll_timeout try: r,w,e = select.select([self.read_fd], [], [], timeout) if __debug__: self._log(5, 'select => %s, %s, %s' % (r,w,e)) if not r: # Timeout continue - data = self.read(32768) # Drain ssl buffer if present + data = self.read(self.read_size) # Drain ssl buffer if present start = 0 dlen = len(data) if __debug__: self._log(5, 'rcvd %s' % dlen) @@ -1716,6 +1793,8 @@ class IMAP4(object): '', stop, line_part + data[start:stop] if __debug__: self._log(4, '< %s' % line) self.inq.put(line) + if self.TerminateReader: + terminate = True except: reason = 'socket error: %s - %s' % sys.exc_info()[:2] if __debug__: @@ -1766,9 +1845,10 @@ class IMAP4(object): if __debug__: - def _init_debug(self, debug=None, debug_file=None): - self.debug = debug is not None and debug or Debug is not None and Debug or 0 - self.debug_file = debug_file is not None and debug_file or sys.stderr + def _init_debug(self, debug=None, debug_file=None, debug_buf_lvl=None): + self.debug = self._choose_nonull_or_dflt(0, debug, Debug) + self.debug_file = self._choose_nonull_or_dflt(sys.stderr, debug_file) + self.debug_buf_lvl = self._choose_nonull_or_dflt(DFLT_DEBUG_BUF_LVL, debug_buf_lvl) self.debug_lock = threading.Lock() self._cmd_log_len = 20 @@ -1776,7 +1856,7 @@ class IMAP4(object): self._cmd_log = {} # Last `_cmd_log_len' interactions if self.debug: self._mesg('imaplib2 version %s' % __version__) - self._mesg('imaplib2 debug level %s' % self.debug) + self._mesg('imaplib2 debug level %s, buffer level %s' % (self.debug, self.debug_buf_lvl)) def _dump_ur(self, lvl): @@ -1803,11 +1883,12 @@ class IMAP4(object): tn = threading.currentThread().getName() - if lvl == 1 or self.debug >= 4: + if lvl <= 1 or self.debug > self.debug_buf_lvl: self.debug_lock.acquire() self._mesg(line, tn) self.debug_lock.release() - return + if lvl != 1: + return # Keep log of last `_cmd_log_len' interactions for debugging. self.debug_lock.acquire() @@ -1824,8 +1905,11 @@ class IMAP4(object): if tn is None: tn = threading.currentThread().getName() tm = time.strftime('%M:%S', time.localtime(secs)) - self.debug_file.write(' %s.%02d %s %s\n' % (tm, (secs*100)%100, tn, s)) - self.debug_file.flush() + try: + self.debug_file.write(' %s.%02d %s %s\n' % (tm, (secs*100)%100, tn, s)) + self.debug_file.flush() + finally: + pass def _print_log(self): @@ -1865,10 +1949,10 @@ class IMAP4_SSL(IMAP4): """ - def __init__(self, host=None, port=None, keyfile=None, certfile=None, debug=None, debug_file=None, identifier=None, timeout=None): + def __init__(self, host=None, port=None, keyfile=None, certfile=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): self.keyfile = keyfile self.certfile = certfile - IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout) + IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout, debug_buf_lvl) def open(self, host=None, port=None): @@ -1878,15 +1962,15 @@ class IMAP4_SSL(IMAP4): This connection will be used by the routines: read, send, shutdown, socket, ssl.""" - self.host = host is not None and host or '' - self.port = port is not None and port or IMAP4_SSL_PORT + self.host = self._choose_nonull_or_dflt('', host) + self.port = self._choose_nonull_or_dflt(IMAP4_SSL_PORT, port) self.sock = self.open_socket() try: - import ssl - self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile) + import ssl + self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile) except ImportError: - self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile) + self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile) self.read_fd = self.sock.fileno() @@ -1949,14 +2033,14 @@ class IMAP4_stream(IMAP4): """ - def __init__(self, command, debug=None, debug_file=None, identifier=None, timeout=None): + def __init__(self, command, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): self.command = command self.host = command self.port = None self.sock = None self.writefile, self.readfile = None, None self.read_fd = None - IMAP4.__init__(self, None, None, debug, debug_file, identifier, timeout) + IMAP4.__init__(self, None, None, debug, debug_file, identifier, timeout, debug_buf_lvl) def open(self, host=None, port=None): @@ -2055,12 +2139,14 @@ class _IdleCont(object): def __init__(self, parent, timeout): self.parent = parent - self.timeout = timeout is not None and timeout or IDLE_TIMEOUT + self.timeout = parent._choose_nonull_or_dflt(IDLE_TIMEOUT, timeout) self.parent.idle_timeout = self.timeout + time.time() def process(self, data, rqb): + self.parent.idle_lock.acquire() self.parent.idle_rqb = rqb self.parent.idle_timeout = self.timeout + time.time() + self.parent.idle_lock.release() if __debug__: self.parent._log(2, 'server IDLE started, timeout in %.2f secs' % self.timeout) return None @@ -2175,10 +2261,11 @@ if __name__ == '__main__': except getopt.error, val: optlist, args = (), () - debug, port, stream_command, keyfile, certfile = (None,)*5 + debug, debug_buf_lvl, port, stream_command, keyfile, certfile = (None,)*6 for opt,val in optlist: if opt == '-d': debug = int(val) + debug_buf_lvl = debug - 1 elif opt == '-l': try: keyfile,certfile = val.split(':') @@ -2235,31 +2322,33 @@ if __name__ == '__main__': cmd, args = cb_arg if error is not None: AsyncError = error - M._mesg('[cb] ERROR %s %.100s => %s' % (cmd, args, error)) + M._log(0, '[cb] ERROR %s %.100s => %s' % (cmd, args, error)) return typ, dat = response - M._mesg('[cb] %s %.100s => %s %.100s' % (cmd, args, typ, dat)) + M._log(0, '[cb] %s %.100s => %s %.100s' % (cmd, args, typ, dat)) if typ == 'NO': AsyncError = (Exception, dat[0]) def run(cmd, args, cb=True): if AsyncError: + M._log(1, 'AsyncError') M.logout() typ, val = AsyncError raise typ(val) - M._mesg('%s %.100s' % (cmd, args)) + if not M.debug: M._log(0, '%s %.100s' % (cmd, args)) try: if cb: typ, dat = getattr(M, cmd)(callback=responder, cb_arg=(cmd, args), *args) - if M.debug: - M._mesg('%s %.100s => %s %.100s' % (cmd, args, typ, dat)) + M._log(1, '%s %.100s => %s %.100s' % (cmd, args, typ, dat)) else: typ, dat = getattr(M, cmd)(*args) - M._mesg('%s %.100s => %s %.100s' % (cmd, args, typ, dat)) + M._log(1, '%s %.100s => %s %.100s' % (cmd, args, typ, dat)) except: + M._log(1, '%s - %s' % sys.exc_info()[:2]) M.logout() raise if typ == 'NO': + M._log(1, 'NO') M.logout() raise Exception(dat[0]) return dat @@ -2270,15 +2359,15 @@ if __name__ == '__main__': if keyfile is not None: if not keyfile: keyfile = None if not certfile: certfile = None - M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, debug=debug, identifier='', timeout=10) + M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) elif stream_command: - M = IMAP4_stream(stream_command, debug=debug, identifier='', timeout=10) + M = IMAP4_stream(stream_command, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) else: - M = IMAP4(host=host, port=port, debug=debug, identifier='', timeout=10) + M = IMAP4(host=host, port=port, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) if M.state != 'AUTH': # Login needed PASSWD = getpass.getpass("IMAP password for %s on %s: " % (USER, host or "localhost")) test_seq1.insert(0, ('login', (USER, PASSWD))) - M._mesg('PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) + M._log(0, 'PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) if 'COMPRESS=DEFLATE' in M.capabilities: M.enable_compression() From c1c200a487cc36381d17275f2f439665c3f11cb0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 10 Jun 2011 17:32:39 +0200 Subject: [PATCH 131/817] folder/Maildir: cache getfullname() value We use getfullname() very often (thousands to millions), yet we dynamically calculate the very same value over and over. Optimize this by caching the value once and be done with it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index e360619..b9f9d05 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -63,12 +63,15 @@ class MaildirFolder(BaseFolder): self.accountname = accountname BaseFolder.__init__(self) #self.ui is set in BaseFolder.init() + # Cache the full folder path, as we use getfullname() very often + self._fullname = os.path.join(self.getroot(), self.getname()) def getaccountname(self): return self.accountname def getfullname(self): - return os.path.join(self.getroot(), self.getname()) + """Return the absolute file path to the Maildir folder (sans cur|new)""" + return self._fullname def getuidvalidity(self): """Maildirs have no notion of uidvalidity, so we just return a magic From e023f190b0eed84fefc10e28bfe5e4e0467ff257 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 10 Jun 2011 17:32:40 +0200 Subject: [PATCH 132/817] folder/Maildir: Store only relative filename components MaildirFolder.messagelist[*]['filename'] was storing the absolute file paths for all stored emails. While this is convenient, it wastes much space, as the folder prefix is always the same and it is known to the MaildirFolder. Just 40 chars in a folder with 100k mails waste >4MB of space. Adapt the few locations where we need the full path to construct it dynamically. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 47 +++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index b9f9d05..a798b36 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -185,27 +185,32 @@ class MaildirFolder(BaseFolder): return self.messagelist def getmessage(self, uid): + """Return the content of the message""" filename = self.messagelist[uid]['filename'] - file = open(filename, 'rt') + filepath = os.path.join(self.getfullname(), filename) + file = open(filepath, 'rt') retval = file.read() file.close() + #TODO: WHY are we replacing \r\n with \n here? And why do we + # read it as text? return retval.replace("\r\n", "\n") def getmessagetime( self, uid ): filename = self.messagelist[uid]['filename'] - st = os.stat(filename) + filepath = os.path.join(self.getfullname(), filename) + st = os.stat(filepath) return st.st_mtime def savemessage(self, uid, content, flags, rtime): # This function only ever saves to tmp/, # but it calls savemessageflags() to actually save to cur/ or new/. - self.ui.debug('maildir', 'savemessage: called to write with flags %s and content %s' % \ - (repr(flags), repr(content))) + self.ui.debug('maildir', 'savemessage: called to write with flags %s ' + 'and content %s' % (repr(flags), repr(content))) if uid < 0: # We cannot assign a new uid. return uid if uid in self.messagelist: - # We already have it. + # We already have it, just update flags. self.savemessageflags(uid, flags) return uid @@ -231,7 +236,8 @@ class MaildirFolder(BaseFolder): else: break tmpmessagename = messagename.split(',')[0] - self.ui.debug('maildir', 'savemessage: using temporary name %s' % tmpmessagename) + self.ui.debug('maildir', 'savemessage: using temporary name %s' %\ + tmpmessagename) file = open(os.path.join(tmpdir, tmpmessagename), "wt") file.write(content) @@ -239,10 +245,10 @@ class MaildirFolder(BaseFolder): file.flush() if self.dofsync: os.fsync(file.fileno()) - file.close() + if rtime != None: - os.utime(os.path.join(tmpdir,tmpmessagename), (rtime,rtime)) + os.utime(os.path.join(tmpdir, tmpmessagename), (rtime, rtime)) self.ui.debug('maildir', 'savemessage: moving from %s to %s' % \ (tmpmessagename, messagename)) if tmpmessagename != messagename: # then rename it @@ -259,7 +265,8 @@ class MaildirFolder(BaseFolder): pass self.messagelist[uid] = {'uid': uid, 'flags': [], - 'filename': os.path.join(tmpdir, messagename)} + 'filename': os.path.join('tmp', messagename)} + # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) self.ui.debug('maildir', 'savemessage: returning uid %d' % uid) return uid @@ -269,14 +276,14 @@ class MaildirFolder(BaseFolder): def savemessageflags(self, uid, flags): oldfilename = self.messagelist[uid]['filename'] - newpath, newname = os.path.split(oldfilename) + dir_prefix, newname = os.path.split(oldfilename) tmpdir = os.path.join(self.getfullname(), 'tmp') if 'S' in flags: # If a message has been seen, it goes into the cur - # directory. CR debian#152482, [complete.org #4] - newpath = os.path.join(self.getfullname(), 'cur') + # directory. CR debian#152482 + dir_prefix = 'cur' else: - newpath = os.path.join(self.getfullname(), 'new') + dir_prefix = 'new' infostr = ':' infomatch = re.search('(:.*)$', newname) if infomatch: # If the info string is present.. @@ -287,15 +294,16 @@ class MaildirFolder(BaseFolder): infostr += '2,' + ''.join(flags) newname += infostr - newfilename = os.path.join(newpath, newname) + newfilename = os.path.join(dir_prefix, newname) if (newfilename != oldfilename): - os.rename(oldfilename, newfilename) + os.rename(os.path.join(self.getfullname(), oldfilename), + os.path.join(self.getfullname(), newfilename)) self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename # By now, the message had better not be in tmp/ land! final_dir, final_name = os.path.split(self.messagelist[uid]['filename']) - assert final_dir != tmpdir + assert final_dir != 'tmp' def deletemessage(self, uid): """Unlinks a message file from the Maildir. @@ -309,13 +317,16 @@ class MaildirFolder(BaseFolder): return filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) try: - os.unlink(filename) + os.unlink(filepath) except OSError: # Can't find the file -- maybe already deleted? newmsglist = self._scanfolder() if uid in newmsglist: # Nope, try new filename. - os.unlink(newmsglist[uid]['filename']) + filename = newmsglist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + os.unlink(filepath) # Yep -- return. del(self.messagelist[uid]) From e18bb4b2b249b04406aaf029160d9570aa2704e7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 10 Jun 2011 21:22:25 +0200 Subject: [PATCH 133/817] Add Changelog recommendation for commit 520e39d35536 Recommend to upgrade from the previous release. I forgot to change the Changelog previously. [Update imaplib2 to 2.24] Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index fd445ac..238a592 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,8 +19,13 @@ Changes Bug Fixes --------- +* A bug in the underlying imap library has been fixed that could + potentially lead to data loss if the server interrupted responses with + unexpected but legal server status responses. This would mainly occur + in folders with many thousands of emails. Upgrading from the previous + release is strongly recommended. -Pending for the next major release +Peanding for the next major release ================================== * UIs get shorter and nicer names. (API changing) From d8026c5308d25aeafd8ef6c6804e1ea757da1ff2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 13 Jun 2011 15:22:37 +0200 Subject: [PATCH 134/817] Simplify Maildir message saving Previously we were attempting to save out mails according to http://www.qmail.org/man/man5/maildir.html in 4 steps: 1 Create a unique filename 2 Do stat(tmp/). If it found a file, wait 2 sec and go back to 1. 3 Create and write the message to the tmp/. 4 Link from tmp/* to new/* (we did step 2 up to 15 times) But as stated by http://wiki1.dovecot.org/MailboxFormat/Maildir (see section 'Issues with the specification'), this is a pointless approach, e.g. there are race issues between stating that the filename does not exist and the actual moving (when it might exist). So, we can simplify the steps as suggested in the dovecot wiki and tighten up our safety at the same time. One improvement that we do is to open the file, guaranteeing that it did not exist before in an atomic manner, thus our simplified approach is really more secure than what we had before. Also, we throw an OfflineImapError at MESSAGE level when the supposedly unique filename already exists, so that we can skip this message and still continue with other messages. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 ++ offlineimap/folder/Maildir.py | 63 ++++++++++++++--------------------- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index fd445ac..2035aa8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,9 +16,12 @@ New Features Changes ------- +* Maildirs use less memory while syncing. + Bug Fixes --------- +* Saving to Maildirs now checks for file existence without race conditions. Pending for the next major release ================================== diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index a798b36..b576507 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -28,6 +28,8 @@ try: except ImportError: from md5 import md5 +from offlineimap import OfflineImapError + uidmatchre = re.compile(',U=(\d+)') flagmatchre = re.compile(':.*2,([A-Z]+)') timestampmatchre = re.compile('(\d+)'); @@ -217,52 +219,37 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - messagename = None - attempts = 0 - while 1: - if attempts > 15: - raise IOError, "Couldn't write to file %s" % messagename - timeval, timeseq = gettimeseq() - messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \ - (timeval, - timeseq, - os.getpid(), - socket.gethostname(), - uid, - md5(self.getvisiblename()).hexdigest()) - if os.path.exists(os.path.join(tmpdir, messagename)): - time.sleep(2) - attempts += 1 + timeval, timeseq = gettimeseq() + messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \ + (timeval, + timeseq, + os.getpid(), + socket.gethostname(), + uid, + md5(self.getvisiblename()).hexdigest()) + # open file and write it out + try: + fd = os.open(os.path.join(tmpdir, messagename), + os.O_EXCL|os.O_CREAT|os.O_WRONLY) + except OSError, e: + if e.errno == 17: + #FILE EXISTS ALREADY + severity = OfflineImapError.ERROR.MESSAGE + raise OfflineImapError("Unique filename %s already existing." %\ + messagename, severity) else: - break - tmpmessagename = messagename.split(',')[0] - self.ui.debug('maildir', 'savemessage: using temporary name %s' %\ - tmpmessagename) - file = open(os.path.join(tmpdir, tmpmessagename), "wt") - file.write(content) + raise + file = os.fdopen(fd, 'wt') + file.write(content) # Make sure the data hits the disk file.flush() if self.dofsync: - os.fsync(file.fileno()) + os.fsync(fd) file.close() if rtime != None: - os.utime(os.path.join(tmpdir, tmpmessagename), (rtime, rtime)) - self.ui.debug('maildir', 'savemessage: moving from %s to %s' % \ - (tmpmessagename, messagename)) - if tmpmessagename != messagename: # then rename it - os.rename(os.path.join(tmpdir, tmpmessagename), - os.path.join(tmpdir, messagename)) - - if self.dofsync: - try: - # fsync the directory (safer semantics in Linux) - fd = os.open(tmpdir, os.O_RDONLY) - os.fsync(fd) - os.close(fd) - except: - pass + os.utime(os.path.join(tmpdir, messagename), (rtime, rtime)) self.messagelist[uid] = {'uid': uid, 'flags': [], 'filename': os.path.join('tmp', messagename)} From d122dde93f4edcaf6f5f6e1d628f0df803b9782e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 11 Jun 2011 21:35:38 +0200 Subject: [PATCH 135/817] Check SSL certificate for expiration We currently don't care about expiration dates of the servers SSL certificate. This patch adds a check that fails Cert verification when it is past its due date. There is no way or option to override this check. Unfortunately we only seem to be able to get SSL certificate data when we passed in a CA cert file? How do we get that date when we don't have a ca cert file? Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index f501af6..20ab336 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -229,6 +229,13 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): dnsname = hostname.lower() certnames = [] + # cert expired? + notafter = cert.get('notAfter') + if notafter: + if time.time() >= ssl.cert_time_to_seconds(notafter): + return ('server certificate error: certificate expired %s' + ) % notafter + # First read commonName for s in cert.get('subject', []): key, value = s[0] From 5328f10b39d54b86cdb3a0273985f2d16fe256bc Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 13 Jun 2011 16:53:35 +0200 Subject: [PATCH 136/817] v6.3.4-rc2 Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 8 -------- Changelog.rst | 19 +++++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 83aa357..fd445ac 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,17 +16,9 @@ New Features Changes ------- -* Maildirs use less memory while syncing. - Bug Fixes --------- -* Saving to Maildirs now checks for file existence without race conditions. -* A bug in the underlying imap library has been fixed that could - potentially lead to data loss if the server interrupted responses with - unexpected but legal server status responses. This would mainly occur - in folders with many thousands of emails. Upgrading from the previous - release is strongly recommended. Pending for the next major release ================================== diff --git a/Changelog.rst b/Changelog.rst index 0f5d6d1..ac9bf47 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,25 @@ ChangeLog releases announces. +OfflineIMAP v6.3.4-rc2 (2011-06-15) +=================================== + +Changes +------- + +* Maildirs use less memory while syncing. + +Bug Fixes +--------- + +* Saving to Maildirs now checks for file existence without race conditions. +* A bug in the underlying imap library has been fixed that could + potentially lead to data loss if the server interrupted responses with + unexpected but legal server status responses. This would mainly occur + in folders with many thousands of emails. Upgrading from the previous + release is strongly recommended. + + OfflineIMAP v6.3.4-rc1 (2011-05-16) =================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index d7313cc..f64f93d 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.4-rc1" +__version__ = "6.3.4-rc2" __copyright__ = "Copyright (C) 2002 - 2010 John Goerzen" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 962a36e14fe4a3af712a5e08e79ee0a88210f959 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 15 Jun 2011 18:25:38 +0200 Subject: [PATCH 137/817] Changelog: add notes about v6.3.4-rc2 Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index ac9bf47..717e6f8 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -15,6 +15,18 @@ ChangeLog OfflineIMAP v6.3.4-rc2 (2011-06-15) =================================== +Notes +----- + +This was a very active rc1 and we could expect a lot of new fixes for the next +release. + +The most important fix is about a bug that could lead to data loss. Find more +information about his bug here: + + http://permalink.gmane.org/gmane.mail.imap.offlineimap.general/3803 + + Changes ------- From 41fad171257bb997b309faa7b51af1a1b6edbdd6 Mon Sep 17 00:00:00 2001 From: Scott Henson Date: Thu, 2 Jun 2011 17:04:52 -0400 Subject: [PATCH 138/817] Fix gssapi with multiple connections Fix a gssapi issue where threads beyond the first would not be able to authenticate against the imap server. This is done by using the connection lock around the gssapi authentication code and resetting (and releasing) the kerberos state after success so that subsequent connections may make use of kerberos. Signed-off-by: Scott Henson Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index e7892cc..0aaeca6 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -219,6 +219,7 @@ class IMAPServer: try: # Try GSSAPI and continue if it fails if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss: + self.connectionlock.acquire() self.ui.debug('imap', 'Attempting GSSAPI authentication') try: @@ -229,8 +230,12 @@ class IMAPServer: 'GSSAPI Authentication failed') else: self.gssapi = True + kerberos.authGSSClientClean(self.gss_vc) + self.gss_vc = None + self.gss_step = self.GSS_STATE_STEP #if we do self.password = None then the next attempt cannot try... #self.password = None + self.connectionlock.release() if not self.gssapi: if 'AUTH=CRAM-MD5' in imapobj.capabilities: From 856982a4e6cbb7d75ddf489ed8fe741922a57de6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 15 Jun 2011 10:59:00 +0200 Subject: [PATCH 139/817] Throw OfflineImapError when we try to request an inexistant message During a sync run, someone might remove or move IMAP messages. As we only cache the list of UIDs in the beginning, we might be requesting UIDs that don't exist anymore. Protect folder.IMAP.getmessage() against the response that we get when we ask for unknown UIDs. Also, if the server responds with anything else than "OK", (eg. Gmail seems to be saying frequently ['NO', 'Dave I can't let you do that now'] :-) so we should also be throwing OfflineImapErrors here rather than AssertionErrors. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +++ offlineimap/folder/IMAP.py | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index fd445ac..3d20bcf 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -20,6 +20,9 @@ Bug Fixes --------- +* We protect more robustly against asking for inexistent messages from the + IMAP server, when someone else deletes or moves messages while we sync. + Pending for the next major release ================================== diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8851b5b..62f220f 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -23,7 +23,7 @@ import re import time from copy import copy from Base import BaseFolder -from offlineimap import imaputil, imaplibutil +from offlineimap import imaputil, imaplibutil, OfflineImapError class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, visiblename, accountname, repository): @@ -195,13 +195,24 @@ class IMAPFolder(BaseFolder): def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body) - :returns: the message body + :returns: the message body or throws and OfflineImapError + (probably severity MESSAGE) if e.g. no message with + this UID could be found. """ imapobj = self.imapserver.acquireconnection() try: imapobj.select(self.getfullname(), readonly = 1) - res_type, data = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])') - assert res_type == 'OK', "Fetching message with UID '%d' failed" % uid + res_type, data = imapobj.uid('fetch', str(uid), '(BODY.PEEK[])') + if data == [None] or res_type != 'OK': + #IMAP server says bad request or UID does not exist + severity = OfflineImapError.ERROR.MESSAGE + reason = "IMAP server '%s' responded with '%s' to fetching "\ + "message UID '%d'" % (self.getrepository(), res_type, uid) + if data == [None]: + #IMAP server did not find a message with this UID + reason = "IMAP server '%s' does not have a message "\ + "with UID '%s'" % (self.getrepository(), uid) + raise OfflineImapError(reason, severity) # data looks now e.g. [('320 (UID 17061 BODY[] # {2565}','msgbody....')] we only asked for one message, # and that msg is in data[0]. msbody is in [0][1] From 2180f5fbf4d49c5f54b18918943bae22c079ae04 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Jun 2011 17:22:29 +0200 Subject: [PATCH 140/817] UIDMappedFolder.savemessage() returned nothing, breaking API conventions Folder.savemessage() is supposed to return the new UID that a backend assigned, and it BaseFolder.copymessageto() fails if we don't return a non-negative number in the savemessage() there. For some reason, the UIDMappedFolder was not returning anything in savemessage, despite clearly stating in the code docs that it is supposed to return a UID. Not sure how long this has already been the case. This patch fixes the UIDMappedFolder to behave as it should. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/UIDMaps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 76c5ee4..5f770ce 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -197,6 +197,7 @@ class MappingFolderMixIn: self._savemaps(dolock = 0) finally: self.maplock.release() + return uid def getmessageflags(self, uid): return self._mb.getmessageflags(self, self.r2l[uid]) From c701e4824d9dfd9a06d900d90404c9d916af1343 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Jun 2011 17:09:29 +0200 Subject: [PATCH 141/817] Fix recursively scanning Maildir folders Commit 1754bf4110da251f4aa1dc0803889899b02bfcff introduced a blunder: - for dirname in os.listdir(toppath) + ['.']: + for dirname in os.listdir(toppath) + [toppath]: ... - if self.getsep() == '/' and dirname != '.': + if self.getsep() == '/' and dirname: This change was plainly wrong and would never have worked, so this commit reverts above bit. While touching the function, some minor code documentation, cleanup and limiting line length to 80 chars. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Maildir.py | 36 +++++++++++++++++-------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 2a10a93..86716e1 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -117,23 +117,23 @@ class MaildirRepository(BaseRepository): self.accountname, self.config) def _getfolders_scandir(self, root, extension = None): + """Recursively scan folder 'root'; return a list of MailDirFolder + + :param root: (absolute) path to Maildir root + :param extension: (relative) subfolder to examine within root""" self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s" \ % (root, extension)) - # extension willl only be non-None when called recursively when - # getsep() returns '/'. retval = [] # Configure the full path to this repository -- "toppath" - - if extension == None: - toppath = root - else: + if extension: toppath = os.path.join(root, extension) - + else: + toppath = root self.debug(" toppath = %s" % toppath) # Iterate over directories in top & top itself. - for dirname in os.listdir(toppath) + [toppath]: + for dirname in os.listdir(toppath) + ['.']: self.debug(" *** top of loop") self.debug(" dirname = %s" % dirname) if dirname in ['cur', 'new', 'tmp']: @@ -153,17 +153,21 @@ class MaildirRepository(BaseRepository): os.path.isdir(os.path.join(fullname, 'new')) and os.path.isdir(os.path.join(fullname, 'tmp'))): # This directory has maildir stuff -- process - self.debug(" This is a maildir folder.") + self.debug(" This is maildir folder '%s'." % foldername) - self.debug(" foldername = %s" % foldername) - - if self.config.has_option('Repository ' + self.name, 'restoreatime') and self.config.getboolean('Repository ' + self.name, 'restoreatime'): + if self.config.has_option('Repository %s' % self, + 'restoreatime') and \ + self.config.getboolean('Repository %s' % self, + 'restoreatime'): self._append_folder_atimes(foldername) - retval.append(folder.Maildir.MaildirFolder(self.root, foldername, - self.getsep(), self, self.accountname, + retval.append(folder.Maildir.MaildirFolder(self.root, + foldername, + self.getsep(), + self, + self.accountname, self.config)) - if self.getsep() == '/' and dirname: - # Check sub-directories for folders. + if self.getsep() == '/' and dirname != '.': + # Recursively check sub-directories for folders too. retval.extend(self._getfolders_scandir(root, foldername)) self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ repr([x.getname() for x in retval])) From 335b320d9a3cefabc586c69ddfe62454752f3525 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Jun 2011 17:09:29 +0200 Subject: [PATCH 142/817] Fix recursively scanning Maildir folders Commit 1754bf4110da251f4aa1dc0803889899b02bfcff introduced a blunder: - for dirname in os.listdir(toppath) + ['.']: + for dirname in os.listdir(toppath) + [toppath]: ... - if self.getsep() == '/' and dirname != '.': + if self.getsep() == '/' and dirname: This change was plainly wrong and would never have worked, so this commit reverts above bit. While touching the function, some minor code documentation, cleanup and limiting line length to 80 chars. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Maildir.py | 36 +++++++++++++++++-------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 2a10a93..86716e1 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -117,23 +117,23 @@ class MaildirRepository(BaseRepository): self.accountname, self.config) def _getfolders_scandir(self, root, extension = None): + """Recursively scan folder 'root'; return a list of MailDirFolder + + :param root: (absolute) path to Maildir root + :param extension: (relative) subfolder to examine within root""" self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s" \ % (root, extension)) - # extension willl only be non-None when called recursively when - # getsep() returns '/'. retval = [] # Configure the full path to this repository -- "toppath" - - if extension == None: - toppath = root - else: + if extension: toppath = os.path.join(root, extension) - + else: + toppath = root self.debug(" toppath = %s" % toppath) # Iterate over directories in top & top itself. - for dirname in os.listdir(toppath) + [toppath]: + for dirname in os.listdir(toppath) + ['.']: self.debug(" *** top of loop") self.debug(" dirname = %s" % dirname) if dirname in ['cur', 'new', 'tmp']: @@ -153,17 +153,21 @@ class MaildirRepository(BaseRepository): os.path.isdir(os.path.join(fullname, 'new')) and os.path.isdir(os.path.join(fullname, 'tmp'))): # This directory has maildir stuff -- process - self.debug(" This is a maildir folder.") + self.debug(" This is maildir folder '%s'." % foldername) - self.debug(" foldername = %s" % foldername) - - if self.config.has_option('Repository ' + self.name, 'restoreatime') and self.config.getboolean('Repository ' + self.name, 'restoreatime'): + if self.config.has_option('Repository %s' % self, + 'restoreatime') and \ + self.config.getboolean('Repository %s' % self, + 'restoreatime'): self._append_folder_atimes(foldername) - retval.append(folder.Maildir.MaildirFolder(self.root, foldername, - self.getsep(), self, self.accountname, + retval.append(folder.Maildir.MaildirFolder(self.root, + foldername, + self.getsep(), + self, + self.accountname, self.config)) - if self.getsep() == '/' and dirname: - # Check sub-directories for folders. + if self.getsep() == '/' and dirname != '.': + # Recursively check sub-directories for folders too. retval.extend(self._getfolders_scandir(root, foldername)) self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ repr([x.getname() for x in retval])) From 8634b0030d7aba5f421d00a17aef9440895b8253 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 14 Jun 2011 11:52:05 +0200 Subject: [PATCH 143/817] Always call ui.terminate() I discovered that we do not run ui.terminate in all circumstances, so make sure that we call with properly at the end of each run (whether in threaded or single-thread mode). Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/init.py b/offlineimap/init.py index 7258365..93b7224 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -343,6 +343,7 @@ class OfflineImap: t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) + ui.terminate() except KeyboardInterrupt: ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') return From 89cbdc9244689ff05b86aae8bf2fbeedfcd14eed Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 16 Jun 2011 23:38:55 +0200 Subject: [PATCH 144/817] Revert "Throw errors on connection refused and on non-standard SSL ports" This reverts commit 3dc9fc519a6fbbb5aad57744de570bdba2bcdfb4. --- offlineimap/imapserver.py | 38 +++++--------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 0aaeca6..10492cd 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -24,14 +24,9 @@ import offlineimap.accounts import hmac import socket import base64 -import errno from socket import gaierror -try: - from ssl import SSLError -except ImportError: - # Protect against python<2.6, use dummy and won't get SSL errors. - SSLError = None + try: # do we have a recent pykerberos? have_gss = False @@ -290,37 +285,14 @@ class IMAPServer: if(self.connectionlock.locked()): self.connectionlock.release() - # now, check for known errors and throw OfflineImapErrors - severity = OfflineImapError.ERROR.REPO - if isinstance(e, gaierror): + if type(e) == gaierror: #DNS related errors. Abort Repo sync + severity = OfflineImapError.ERROR.REPO #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ - "ver name correctly and that you are online."%\ - (self.hostname, self.reposname) - raise OfflineImapError(reason, severity) - - elif isinstance(e, SSLError) and e.errno == 1: - # SSL unknown protocol error - # happens e.g. when connecting via SSL to a non-SSL service - if self.port != 443: - reason = "Could not connect via SSL to host '%s' and non-s"\ - "tandard ssl port %d configured. Make sure you connect"\ - " to the correct port." % (self.hostname, self.port) - else: - reason = "Unknown SSL protocol connecting to host '%s' for"\ - "repository '%s'. OpenSSL responded:\n%s"\ - % (self.hostname, self.reposname, e) - raise OfflineImapError(reason, severity) - - elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: - # "Connection refused", can be a non-existing port, or an unauthorized - # webproxy (open WLAN?) - reason = "Connection to host '%s:%d' for repository '%s' was "\ - "refused. Make sure you have the right host and port "\ - "configured and that you are actually able to access the "\ - "network." % (self.hostname, self.port, self.reposname) + "ver name correctly and that you are online."\ + % (self.hostname, self.reposname) raise OfflineImapError(reason, severity) # Could not acquire connection to the remote; # socket.error(last_error) raised From 5ffff9cf208fe756747a5f167a906541899b250f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 6 Jun 2011 11:37:25 +0200 Subject: [PATCH 145/817] Allow to create the root MailDir directory We currently do not allow nametrans rules such as nametrans = lambda foldername: re.sub('^INBOX$', '', foldername) because we crash with a traceback when running: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=499755 The underlying reason is that we cannot create the "top level" root directory of the Maildir in the function makefolders(), it will bail out. John Goerzen intentionally prevented offlineimap from creating the top-level dir, so that a misconfiguration could not arbitrarily create folders on the file system. I believe that it should be perfectly possible to automatically create the root dirctory of the maildir. We still protect against folder creations at arbitrary places in the file system though. This patch cleans up makefolders(), adds documentation, allows to automatically create rootfolders if needed (using absolute paths) and adds some robustness in case the folders already exist that we want to create (rather than simply crapping out). Signed-off-by: Sebastian Spaeth Tested-by: Mark Foxwell Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Maildir.py | 78 +++++++++++++++---------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 18fa591..2a10a93 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -65,46 +65,46 @@ class MaildirRepository(BaseRepository): return self.getconf('sep', '.').strip() def makefolder(self, foldername): - self.debug("makefolder called with arg " + repr(foldername)) - # Do the chdir thing so the call to makedirs does not make the - # self.root directory (we'd prefer to raise an error in that case), - # but will make the (relative) paths underneath it. Need to use - # makedirs to support a / separator. - self.debug("Is dir? " + repr(os.path.isdir(foldername))) - if self.getsep() == '/': - for invalid in ['new', 'cur', 'tmp', 'offlineimap.uidvalidity']: - for component in foldername.split('/'): - assert component != invalid, "When using nested folders (/ as a separator in the account config), your folder names may not contain 'new', 'cur', 'tmp', or 'offlineimap.uidvalidity'." + """Create new Maildir folder if necessary - assert foldername.find('./') == -1, "Folder names may not contain ../" + :param foldername: A relative mailbox name. The maildir will be + created in self.root+'/'+foldername. All intermediate folder + levels will be created if they do not exist yet. 'cur', + 'tmp', and 'new' subfolders will be created in the maildir. + """ + self.debug("makefolder called with arg '%s'" % (foldername)) + full_path = os.path.abspath(os.path.join(self.root, foldername)) + + # sanity tests + if self.getsep() == '/': + for component in foldername.split('/'): + assert not component in ['new', 'cur', 'tmp'],\ + "When using nested folders (/ as a Maildir separator), "\ + "folder names may not contain 'new', 'cur', 'tmp'." + assert foldername.find('../') == -1, "Folder names may not contain ../" assert not foldername.startswith('/'), "Folder names may not begin with /" - oldcwd = os.getcwd() - os.chdir(self.root) - - # If we're using hierarchical folders, it's possible that sub-folders - # may be created before higher-up ones. If this is the case, - # makedirs will fail because the higher-up dir already exists. - # So, check to see if this is indeed the case. - - if (self.getsep() == '/' or self.getconfboolean('existsok', 0) or foldername == '.') \ - and os.path.isdir(foldername): - self.debug("makefolder: %s already is a directory" % foldername) - # Already exists. Sanity-check that it's not a Maildir. - for subdir in ['cur', 'new', 'tmp']: - assert not os.path.isdir(os.path.join(foldername, subdir)), \ - "Tried to create folder %s but it already had dir %s" %\ - (foldername, subdir) - else: - self.debug("makefolder: calling makedirs %s" % foldername) - assert foldername != '', "Can not create root MailDir." - os.makedirs(foldername, 0700) - self.debug("makefolder: creating cur, new, tmp") + # If we're using hierarchical folders, it's possible that + # sub-folders may be created before higher-up ones. + self.debug("makefolder: calling makedirs '%s'" % full_path) + try: + os.makedirs(full_path, 0700) + except OSError, e: + if e.errno == 17 and os.path.isdir(full_path): + self.debug("makefolder: '%s' already a directory" % foldername) + else: + raise for subdir in ['cur', 'new', 'tmp']: - os.mkdir(os.path.join(foldername, subdir), 0700) - # Invalidate the cache + try: + os.mkdir(os.path.join(full_path, subdir), 0700) + except OSError, e: + if e.errno == 17 and os.path.isdir(full_path): + self.debug("makefolder: '%s' already has subdir %s" % + (foldername, sudir)) + else: + raise + # Invalidate the folder cache self.folders = None - os.chdir(oldcwd) def deletefolder(self, foldername): self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername) @@ -132,11 +132,11 @@ class MaildirRepository(BaseRepository): self.debug(" toppath = %s" % toppath) - # Iterate over directories in top. - for dirname in os.listdir(toppath) + ['.']: + # Iterate over directories in top & top itself. + for dirname in os.listdir(toppath) + [toppath]: self.debug(" *** top of loop") self.debug(" dirname = %s" % dirname) - if dirname in ['cur', 'new', 'tmp', 'offlineimap.uidvalidity']: + if dirname in ['cur', 'new', 'tmp']: self.debug(" skipping this dir (Maildir special)") # Bypass special files. continue @@ -162,7 +162,7 @@ class MaildirRepository(BaseRepository): retval.append(folder.Maildir.MaildirFolder(self.root, foldername, self.getsep(), self, self.accountname, self.config)) - if self.getsep() == '/' and dirname != '.': + if self.getsep() == '/' and dirname: # Check sub-directories for folders. retval.extend(self._getfolders_scandir(root, foldername)) self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ From 7570f718809b4883b03934837f639bf6c2af6971 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Jun 2011 08:56:30 +0200 Subject: [PATCH 146/817] Throw OfflineImapError when we try to request an inexistant message During a sync run, someone might remove or move IMAP messages. As we only cache the list of UIDs in the beginning, we might be requesting UIDs that don't exist anymore. Protect folder.IMAP.getmessage() against the response that we get when we ask for unknown UIDs. Also, if the server responds with anything else than "OK", (eg. Gmail seems to be saying frequently ['NO', 'Dave I can't let you do that now'] :-) so we should also be throwing OfflineImapErrors here rather than AssertionErrors. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +++ offlineimap/folder/IMAP.py | 19 +++++++++++++++---- offlineimap/imapserver.py | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index fd445ac..3d20bcf 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -20,6 +20,9 @@ Bug Fixes --------- +* We protect more robustly against asking for inexistent messages from the + IMAP server, when someone else deletes or moves messages while we sync. + Pending for the next major release ================================== diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8851b5b..62f220f 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -23,7 +23,7 @@ import re import time from copy import copy from Base import BaseFolder -from offlineimap import imaputil, imaplibutil +from offlineimap import imaputil, imaplibutil, OfflineImapError class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, visiblename, accountname, repository): @@ -195,13 +195,24 @@ class IMAPFolder(BaseFolder): def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body) - :returns: the message body + :returns: the message body or throws and OfflineImapError + (probably severity MESSAGE) if e.g. no message with + this UID could be found. """ imapobj = self.imapserver.acquireconnection() try: imapobj.select(self.getfullname(), readonly = 1) - res_type, data = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])') - assert res_type == 'OK', "Fetching message with UID '%d' failed" % uid + res_type, data = imapobj.uid('fetch', str(uid), '(BODY.PEEK[])') + if data == [None] or res_type != 'OK': + #IMAP server says bad request or UID does not exist + severity = OfflineImapError.ERROR.MESSAGE + reason = "IMAP server '%s' responded with '%s' to fetching "\ + "message UID '%d'" % (self.getrepository(), res_type, uid) + if data == [None]: + #IMAP server did not find a message with this UID + reason = "IMAP server '%s' does not have a message "\ + "with UID '%s'" % (self.getrepository(), uid) + raise OfflineImapError(reason, severity) # data looks now e.g. [('320 (UID 17061 BODY[] # {2565}','msgbody....')] we only asked for one message, # and that msg is in data[0]. msbody is in [0][1] diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 0aaeca6..1eb8f6b 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -301,7 +301,7 @@ class IMAPServer: (self.hostname, self.reposname) raise OfflineImapError(reason, severity) - elif isinstance(e, SSLError) and e.errno == 1: + elif SSLError and isinstance(e, SSLError) and e.errno == 1: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 443: From b40d02e8012f4b86113c80033594fe9e1be0e7dd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 22 Jun 2011 21:38:19 +0200 Subject: [PATCH 147/817] repository/Maildir.py: Fix typo 'sudir' sudir->subdir in a debug statement. Thanks ccxCZ on IRC for the heads up. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Maildir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 86716e1..069748a 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -100,7 +100,7 @@ class MaildirRepository(BaseRepository): except OSError, e: if e.errno == 17 and os.path.isdir(full_path): self.debug("makefolder: '%s' already has subdir %s" % - (foldername, sudir)) + (foldername, subdir)) else: raise # Invalidate the folder cache From 3ce514e92ba7801d43bef1ee3911902975661cde Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Jun 2011 11:28:30 +0200 Subject: [PATCH 148/817] Simplify MappedIMAPFolder, fixing bugs Previously, we instanciated an MappedImapFolder, and would cleverly (too cleverly?) invoke methods on it casting it to an IMAPFolder by calling methods such as: self._mb.cachemessages(self) where self._MB is the class IMAPFolder and self and instance of MappedImapFolder. If e.g. cachemessages() invokes a method uidexists() which exists for MappedImapFolder, but not directly in IMAPFolder, I am not sure if Python would at some point attempt to use the method of the wrong class. Also, this leads to some twisted thinking as our class would in same cases act as an IMAPFolder and in some cases as an MappedImapFOlder and it is not always clear if we mean REMOTE UID or LOCAL UID. This commit simplifies the class, by a)doing away with the complex Mixin construct and directly inheriting from IMAPFOlder (so we get all the IMAPFOlder methods that we can inherit). We instantiate self._mb as a new instance of IMAPFolder which represents the local IMAP using local UIDs, separating the MappedIMAPFolder construct logically from the IMAPFolder somewhat. In the long run, I would like to remove self._mb completely and simply override any method that needs overriding, but let us take small and understandable baby steps here. Reported-and-tested-by: Vincent Beffara Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/UIDMaps.py | 46 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 5f770ce..9233662 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -20,8 +20,11 @@ from threading import * from IMAP import IMAPFolder import os.path -class MappingFolderMixIn: - """Helper class to map between Folder() instances where both side assign a uid +class MappedIMAPFolder(IMAPFolder): + """IMAP class to map between Folder() instances where both side assign a uid + + This Folder is used on the local side, while the remote side should + be an IMAPFolder. Instance variables (self.): r2l: dict mapping message uids: self.r2l[remoteuid]=localuid @@ -29,10 +32,13 @@ class MappingFolderMixIn: #TODO: what is the difference, how are they used? diskr2l: dict mapping message uids: self.r2l[remoteuid]=localuid diskl2r: dict mapping message uids: self.r2l[localuid]=remoteuid""" - def _initmapping(self): + + def __init__(self, *args, **kwargs): + IMAPFolder.__init__(self, *args, **kwargs) self.maplock = Lock() (self.diskr2l, self.diskl2r) = self._loadmaps() - self._mb = self.__class__.__bases__[1] + self._mb = IMAPFolder(*args, **kwargs) + """Representing the local IMAP Folder using local UIDs""" def _getmapfilename(self): return os.path.join(self.repository.getmapdir(), @@ -81,8 +87,8 @@ class MappingFolderMixIn: return [mapping[x] for x in items] def cachemessagelist(self): - self._mb.cachemessagelist(self) - reallist = self._mb.getmessagelist(self) + self._mb.cachemessagelist() + reallist = self._mb.getmessagelist() self.maplock.acquire() try: @@ -137,7 +143,7 @@ class MappingFolderMixIn: cachemessagelist() before calling this function!""" retval = {} - localhash = self._mb.getmessagelist(self) + localhash = self._mb.getmessagelist() self.maplock.acquire() try: for key, value in localhash.items(): @@ -158,7 +164,7 @@ class MappingFolderMixIn: def getmessage(self, uid): """Returns the content of the specified message.""" - return self._mb.getmessage(self, self.r2l[uid]) + return self._mb.getmessage(self.r2l[uid]) def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. @@ -185,7 +191,7 @@ class MappingFolderMixIn: self.savemessageflags(uid, flags) return uid - newluid = self._mb.savemessage(self, -1, content, flags, rtime) + newluid = self._mb.savemessage(-1, content, flags, rtime) if newluid < 1: raise ValueError("Backend could not find uid for message") self.maplock.acquire() @@ -200,19 +206,19 @@ class MappingFolderMixIn: return uid def getmessageflags(self, uid): - return self._mb.getmessageflags(self, self.r2l[uid]) + return self._mb.getmessageflags(self.r2l[uid]) def getmessagetime(self, uid): return None def savemessageflags(self, uid, flags): - self._mb.savemessageflags(self, self.r2l[uid], flags) + self._mb.savemessageflags(self.r2l[uid], flags) def addmessageflags(self, uid, flags): - self._mb.addmessageflags(self, self.r2l[uid], flags) + self._mb.addmessageflags(self.r2l[uid], flags) def addmessagesflags(self, uidlist, flags): - self._mb.addmessagesflags(self, self._uidlist(self.r2l, uidlist), + self._mb.addmessagesflags(self._uidlist(self.r2l, uidlist), flags) def _mapped_delete(self, uidlist): @@ -233,22 +239,16 @@ class MappingFolderMixIn: self.maplock.release() def deletemessageflags(self, uid, flags): - self._mb.deletemessageflags(self, self.r2l[uid], flags) + self._mb.deletemessageflags(self.r2l[uid], flags) def deletemessagesflags(self, uidlist, flags): - self._mb.deletemessagesflags(self, self._uidlist(self.r2l, uidlist), + self._mb.deletemessagesflags(self._uidlist(self.r2l, uidlist), flags) def deletemessage(self, uid): - self._mb.deletemessage(self, self.r2l[uid]) + self._mb.deletemessage(self.r2l[uid]) self._mapped_delete([uid]) def deletemessages(self, uidlist): - self._mb.deletemessages(self, self._uidlist(self.r2l, uidlist)) + self._mb.deletemessages(self._uidlist(self.r2l, uidlist)) self._mapped_delete(uidlist) - -# Define a class for local part of IMAP. -class MappedIMAPFolder(MappingFolderMixIn, IMAPFolder): - def __init__(self, *args, **kwargs): - IMAPFolder.__init__(self, *args, **kwargs) - self._initmapping() From 416df0fc44749b9667d574e7f0ee093f53e12f28 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Jun 2011 15:38:55 +0200 Subject: [PATCH 149/817] imaputil.imapsplit: Remove overzealous debug statement In an IMAP run where we did not have to sync anything, I spend nearly a fulls second in imaputil.debug() without even having debug output enabled. imapsplit is mainly called by flagsplit() which will also do debug output, so we get TONS of nearly duplicate debug output in the log which makes it really hard to analyze. Cut down the debug logging in imapsplit, we should be debug logging stuff at a slightly higher level than here anyway. This one-line change sped up my folder sync (without having to sync anything) by 0.5 seconds even when debugging is disabled. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaputil.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 101f9a6..3905851 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -143,7 +143,6 @@ def imapsplit(imapstring): elif splitslen == 0: # There was not even an unquoted word. break - debug("imapsplit() returning:", retval) return retval flagmap = [('\\Seen', 'S'), From 6311716edbf05e730790151bb5c3b83e0f2cf9b7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Jun 2011 16:06:07 +0200 Subject: [PATCH 150/817] imapserver.py: Implement STARTTLS If we do not use a SSL connection anyway and if the server supports it, authenticate automatically with STARTTLS. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +++ offlineimap/imapserver.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 3d20bcf..fecce26 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,9 @@ others. New Features ------------ +* Added StartTLS support, it will automatically be used if the server + supports it. + Changes ------- diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 1aabcab..77a3c38 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -233,11 +233,18 @@ class IMAPServer: self.connectionlock.release() if not self.gssapi: + if 'STARTTLS' in imapobj.capabilities and not\ + self.usessl: + self.ui.debug('imap', + 'Using STARTTLS connection') + imapobj.starttls() + if 'AUTH=CRAM-MD5' in imapobj.capabilities: self.ui.debug('imap', - 'Attempting CRAM-MD5 authentication') + 'Attempting CRAM-MD5 authentication') try: - imapobj.authenticate('CRAM-MD5', self.md5handler) + imapobj.authenticate('CRAM-MD5', + self.md5handler) except imapobj.error, val: self.plainauth(imapobj) else: From 9d95d7bc62bc56af0f1d5c39a5489d26fdbb85f6 Mon Sep 17 00:00:00 2001 From: Haojun Bao Date: Mon, 27 Jun 2011 09:13:35 +0800 Subject: [PATCH 151/817] folder/IMAP: fix typo with maxsize and maxage. Signed-off-by: Bao Haojun Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 62f220f..9297a7b 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -137,7 +137,7 @@ class IMAPFolder(BaseFolder): search_condition += date_search_str if(maxsize != -1): - if(maxage != 1): #There are two conditions - add a space + if(maxage != -1): #There are two conditions - add a space search_condition += " " search_condition += "SMALLER " + self.config.getdefault("Account " + self.accountname, "maxsize", -1) From 928c363044510edfb5520e14975a065f9dc20165 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 27 Jun 2011 11:08:54 +0200 Subject: [PATCH 152/817] imapserver.py: Make severity var available where it is needed We we using the variable 'severity' in a few places to throw OfflineImapErrorrs of severity REPO. Somehow, that variable is now not accessible in all places that refer to it, so we move where it is defined to before all the 'if' checks which might make use of it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 77a3c38..0713eb8 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -292,9 +292,9 @@ class IMAPServer: if(self.connectionlock.locked()): self.connectionlock.release() + severity = OfflineImapError.ERROR.REPO if type(e) == gaierror: #DNS related errors. Abort Repo sync - severity = OfflineImapError.ERROR.REPO #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ @@ -328,8 +328,7 @@ class IMAPServer: if str(e)[:24] == "can't open socket; error": raise OfflineImapError("Could not connect to remote server '%s' "\ "for repository '%s'. Remote does not answer." - % (self.hostname, self.reposname), - OfflineImapError.ERROR.REPO) + % (self.hostname, self.reposname), severity) else: # re-raise all other errors raise From 8e2589982c9f2300fa15af7c387194cd4dd3bf9c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 14 Jun 2011 10:23:39 +0200 Subject: [PATCH 153/817] Don't use CStringIO to format a traceback The traceback module has format_exc() for this purpose so let's use that. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 4c60b2a..5524579 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -21,7 +21,6 @@ import time import sys import traceback import threading -from StringIO import StringIO import offlineimap debugtypes = {'':'Other offlineimap related sync messages', @@ -309,10 +308,8 @@ class UIBase: s.terminate(100) def getMainExceptionString(s): - sbuf = StringIO() - traceback.print_exc(file = sbuf) - return "Main program terminated with exception:\n" + \ - sbuf.getvalue() + "\n" + \ + return "Main program terminated with exception:\n%s\n" %\ + traceback.format_exc() + \ s.getThreadDebugLog(threading.currentThread()) def mainException(s): From aec35eebd37c4ea608bfbe9ec1bfc164d9b566c6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 1 Jul 2011 13:15:06 +0200 Subject: [PATCH 154/817] Reduce initial license blurb Rather than the extremly verbose NO WARRANTY blurb, we output a somewhat smaller initial text which should still make the GPL happy. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/__init__.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index f64f93d..688e50a 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,21 +2,20 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.3.4-rc2" -__copyright__ = "Copyright (C) 2002 - 2010 John Goerzen" +__copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" +__license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" __bigcopyright__ = """%(__productname__)s %(__version__)s -%(__copyright__)s <%(__author_email__)s>""" % locals() - -banner = __bigcopyright__ + """ - -This software comes with ABSOLUTELY NO WARRANTY; see the file -COPYING for details. This is free software, and you are welcome -to distribute it under the conditions laid out in COPYING.""" - +%(__copyright__)s. +%(__license__)s. +""" % locals() __homepage__ = "http://github.com/nicolas33/offlineimap" -__license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)." + + +banner = __bigcopyright__ + from offlineimap.error import OfflineImapError # put this last, so we don't run into circular dependencies using From 09ba269576ddb969e4d294211324fd4ae69daa12 Mon Sep 17 00:00:00 2001 From: Arnaud Fontaine Date: Wed, 6 Jul 2011 23:08:07 +0200 Subject: [PATCH 155/817] Add missing import of SSLError exception. In commit 89cbdc9, usage of SSLError was dropped but later reintroduced without importing SSLError exception. Signed-off-by: Arnaud Fontaine Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 0713eb8..a235fd5 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -26,6 +26,11 @@ import socket import base64 from socket import gaierror +try: + from ssl import SSLError +except ImportError: + # Protect against python<2.6, use dummy and won't get SSL errors. + SSLError = None try: # do we have a recent pykerberos? From c293e64119c3217347b48a1c0564af6dbf9b3a7c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 6 Jul 2011 23:15:33 +0200 Subject: [PATCH 156/817] Fix caching inconsistencies in getselectedfolder() getselectedfolder was using a cached variable that we were setting in select(), but sometimes the IMAP4 instance got into the SELECTED state without explicitely select()ing, it seems, and our variable was unset. Let us just use the self.mailbox variable that imaplib2 is setting when select()ing rather than doing our own caching. Also remove the part where we were setting the cache. Just access self.state rather than looking up self.state via self.getstate() every time, it is just an unnecessary layer of redirection. Original-patch-by: Arnaud Fontaine Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 20ab336..6492086 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -33,11 +33,9 @@ except ImportError: pass class UsefulIMAPMixIn: - def getstate(self): - return self.state def getselectedfolder(self): - if self.getstate() == 'SELECTED': - return self.selectedfolder + if self.state == 'SELECTED': + return self.mailbox return None def select(self, mailbox='INBOX', readonly=None, force = 0): @@ -58,10 +56,6 @@ class UsefulIMAPMixIn: (mailbox, result) severity = OfflineImapError.ERROR.FOLDER raise OfflineImapError(errstr, severity) - if self.getstate() == 'SELECTED': - self.selectedfolder = mailbox - else: - self.selectedfolder = None return result def _mesg(self, s, tn=None, secs=None): From 129b703bf20a154e3a16f7aa4c19877236336e32 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 7 Jul 2011 18:43:18 +0200 Subject: [PATCH 157/817] v6.3.4-rc3 Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 5 ----- Changelog.rst | 32 ++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index fecce26..87df0cd 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,9 +13,6 @@ others. New Features ------------ -* Added StartTLS support, it will automatically be used if the server - supports it. - Changes ------- @@ -23,8 +20,6 @@ Bug Fixes --------- -* We protect more robustly against asking for inexistent messages from the - IMAP server, when someone else deletes or moves messages while we sync. Pending for the next major release ================================== diff --git a/Changelog.rst b/Changelog.rst index 717e6f8..9e4bf8a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,38 @@ ChangeLog releases announces. +OfflineIMAP v6.3.4-rc3 (2011-07-07) +=================================== + +Notes +----- + +Here is a surprising release. :-) + +As expected we have a lot bug fixes in this round (see git log for details), +including a fix for a bug we had for ages (details below) which is a very good +news. + +What makes this cycle so unusual is that I merged a feature to support StartTLS +automatically (thanks Sebastian!). Another very good news. + +We usually don't do much changes so late in a cycle. Now, things are highly +calming down and I hope a lot of people will test this release. Next one could +be the stable! + +New Features +------------ + +* Added StartTLS support, it will automatically be used if the server + supports it. + +Bug Fixes +--------- + +* We protect more robustly against asking for inexistent messages from the + IMAP server, when someone else deletes or moves messages while we sync. + + OfflineIMAP v6.3.4-rc2 (2011-06-15) =================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 688e50a..4a0e52c 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.4-rc2" +__version__ = "6.3.4-rc3" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From beef888ed70814e043f197a3d777aae5a4cf07dc Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 7 Jul 2011 22:32:42 +0200 Subject: [PATCH 158/817] Changelog: fix missing note about iddle support Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 1 - Changelog.rst | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 87df0cd..e840717 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -25,4 +25,3 @@ Pending for the next major release ================================== * UIs get shorter and nicer names. (API changing) -* Implement IDLE feature. (delayed until next major release) diff --git a/Changelog.rst b/Changelog.rst index 9e4bf8a..b5fb686 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -58,6 +58,12 @@ information about his bug here: http://permalink.gmane.org/gmane.mail.imap.offlineimap.general/3803 +The IDLE support is merged as experimental feature. + +New Features +------------ + +* Implement experimental IDLE feature. Changes ------- From d5020c1d82153d99d7f08950a2ed15398b770ca3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 10 Jul 2011 09:51:01 +0200 Subject: [PATCH 159/817] MANUAL.rst: Improve with Perf tips and SSL notes Write up some tips in the manual on how to improve performance and some notes on how we currently use SSL, to be clear and transparent on what level of security users get by the various means of connecting via SSL/TLS. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 91 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 26f31a6..571704c 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -300,5 +300,92 @@ KNOWN BUGS * IDLE may only work "once" per refresh. If you encounter this bug, please send a report to the list! -SEE ALSO -======== + +Synchronization Performance +=========================== + +By default, we use fairly conservative settings that are good for +syncing but that might not be the best performing one. Once you got +everything set up and running, you might want to look into speeding up +your synchronization. Here are a couple of hints and tips on how to +achieve this. + + 1) Use maxconnections > 1. By default we only use one connection to an + IMAP server. Using 2 or even 3 speeds things up considerably in most + cases. This setting goes into the [Repository XXX] section. + + 2) Use folderfilters. The quickest sync is a sync that can ignore some + folders. I sort my inbox into monthly folders, and ignore every + folder that is more than 2-3 months old, this lets me only inspect a + fraction of my Mails on every sync. If you haven't done this yet, do + it :). See the folderfilter section the example offlineimap.conf. + + 3) The default status cache is a plain text file that will write out + the complete file for each single new message (or even changed flag) + to a temporary file. If you have plenty of files in a folder, this + is a few hundred kilo to megabytes for each mail and is bound to + make things slower. I recommend to use the sqlite backend for + that. See the status_backend = sqlite setting in the example + offlineimap.conf. You will need to have python-sqlite installed in + order to use this. This will save you plenty of disk activity. Do + note that the sqlite backend is still considered experimental as it + has only been included recently (although a loss of your status + cache should not be a tragedy as that file can be rebuild + automatically) + + 4) Use quick sync. A regular sync will request all flags and all UIDs + of all mails in each folder which takes quite some time. A 'quick' + sync only compares the number of messages in a folder on the IMAP + side (it will detect flag changes on the Maildir side of things + though). A quick sync on my smallish account will take 7 seconds + rather than 40 seconds. Eg, I run a cron script that does a regular + sync once a day, and does quick syncs inbetween. + + 5) Turn off fsync. In the [general] section you can set fsync to True + or False. If you want to play 110% safe and wait for all operations + to hit the disk before continueing, you can set this to True. If you + set it to False, you lose some of that safety trading it for speed. + +Security and SSL +================ + +Some words on OfflineImap and its use of SSL/TLS. By default, we will +connect using any method that openssl supports, that is SSLv2, SSLv3, or +TLSv1. Do note that SSLv2 is notoriously insecure and deprecated. +Unfortunately, python2 does not offer easy ways to disable SSLv2. It is +recommended you test your setup and make sure that the mail server does +not use an SSLv2 connection. Use e.g. "openssl s_client -host +mail.server -port 443" to find out the connection that is used by +default. + +Certificate checking +^^^^^^^^^^^^^^^^^^^^ + +Unfortunately, by default we will not verify the certificate of an IMAP +TLS/SSL server we connect to, so connecting by SSL is no guarantee +against man-in-the-middle attacks. While verifying a server certificate +fingerprint is being planned, it is not implemented yet. There is +currently only one safe way to ensure that you connect to the correct +server in an encrypted manner: You can specify a 'sslcacertfile' setting +in your repository section of offlineimap.conf pointing to a file that +contains (among others) a CA Certificate in PEM format which validating +your server certificate. In this case, we will check that: 1) The server +SSL certificate is validated by the CA Certificate 2) The server host +name matches the SSL certificate 3) The server certificate is not past +its expiration date. The FAQ contains an entry on how to create your own +certificate and CA certificate. + +StartTLS +^^^^^^^^ + +If you have not configured your account to connect via SSL anyway, +OfflineImap will still attempt to set up an SSL connection via the +STARTTLS function, in case the imap server supports it. Do note, that +there is no certificate or fingerprint checking involved at all, when +using STARTTLS (the underlying imaplib library does not support this +yet). This means that you will be protected against passively listening +eavesdroppers and they will not be able to see your password or email +contents. However, this will not protect you from active attacks, such +as Man-In-The-Middle attacks which cause you to connect to the wrong +server and pretend to be your mail server. DO NOT RELY ON STARTTLS AS A +SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF YOUR IMAP SERVER! From 6995eeb92e20b46e8f2d8b984d9d6ffbc4e2b875 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Tue, 12 Jul 2011 09:34:02 +0200 Subject: [PATCH 160/817] Support maildir for windows That makes OfflineIMAP to use exclamation mark (!) instead of colon for storing messages. Such files can be written to windows partitions. But you will probably loose compatibility with other programs trying to read the same Maildir. Signed-off-by: Vladimir Marek Reviewed-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 21 +++++++++++++++++++++ offlineimap.conf | 10 ++++++++++ offlineimap/folder/Maildir.py | 21 ++++++++++++++++----- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 26f31a6..e6a32e2 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -300,5 +300,26 @@ KNOWN BUGS * IDLE may only work "once" per refresh. If you encounter this bug, please send a report to the list! +* Maildir support in Windows drive + Maildir uses colon caracter (:) in message file names. Colon is however + forbidden character in windows drives. There are several workarounds for + that situation: + + * Use "maildir-windows-compatible = yes" account OfflineIMAP configuration. + - That makes OfflineIMAP to use exclamation mark (!) instead of colon for + storing messages. Such files can be written to windows partitions. But + you will probably loose compatibility with other programs trying to + read the same Maildir. + - Exclamation mark was choosed because of the note in + http://docs.python.org/library/mailbox.html + - If you have some messages already stored without this option, you will + have to re-sync them again + + * Enable file name character translation in windows registry (not tested) + - http://support.microsoft.com/kb/289627 + + * Use cygwin managed mount (not tested) + - not available anymore since cygwin 1.7 + SEE ALSO ======== diff --git a/offlineimap.conf b/offlineimap.conf index 9329c66..2a4b1a2 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -239,6 +239,16 @@ remoterepository = RemoteExample # maxage = 3 + +# Maildir format uses colon (:) separator between uniq name and info. +# Unfortunatelly colon is not allowed character in windows file name. If you +# enable maildir-windows-compatible option, offlineimap will be able to store +# messages on windows drive, but you will probably loose compatibility with +# other programs working with the maildir + +# maildir-windows-compatible = no + + [Repository LocalExample] # This is one of the two repositories that you'll work with given the diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index b576507..3b53156 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -31,7 +31,6 @@ except ImportError: from offlineimap import OfflineImapError uidmatchre = re.compile(',U=(\d+)') -flagmatchre = re.compile(':.*2,([A-Z]+)') timestampmatchre = re.compile('(\d+)'); timeseq = 0 @@ -63,6 +62,17 @@ class MaildirFolder(BaseFolder): self.messagelist = None self.repository = repository self.accountname = accountname + + self.wincompatible = self.config.getdefaultboolean( + "Account "+self.accountname, "maildir-windows-compatible", False) + + if self.wincompatible == False: + self.infosep = ':' + else: + self.infosep = '!' + + self.flagmatchre = re.compile(self.infosep + '.*2,([A-Z]+)') + BaseFolder.__init__(self) #self.ui is set in BaseFolder.init() # Cache the full folder path, as we use getfullname() very often @@ -156,7 +166,7 @@ class MaildirFolder(BaseFolder): nouidcounter -= 1 else: uid = long(uidmatch.group(1)) - flagmatch = flagmatchre.search(messagename) + flagmatch = self.flagmatchre.search(messagename) flags = [] if flagmatch: flags = [x for x in flagmatch.group(1)] @@ -271,11 +281,12 @@ class MaildirFolder(BaseFolder): dir_prefix = 'cur' else: dir_prefix = 'new' - infostr = ':' - infomatch = re.search('(:.*)$', newname) + + infostr = self.infosep + infomatch = re.search('(' + self.infosep + '.*)$', newname) if infomatch: # If the info string is present.. infostr = infomatch.group(1) - newname = newname.split(':')[0] # Strip off the info string. + newname = newname.split(self.infosep)[0] # Strip off the info string. infostr = re.sub('2,[A-Z]*', '', infostr) flags.sort() infostr += '2,' + ''.join(flags) From 5db1633afea1402e335059cb06f7532022cdba17 Mon Sep 17 00:00:00 2001 From: Luca Capello Date: Wed, 13 Jul 2011 15:34:17 +0200 Subject: [PATCH 161/817] docs/MANUAL.rst: remove leftover reference to old interface name This fixes Debian bug #629502. Signed-off-by: Luca Capello Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index d3542d0..af5683f 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -251,9 +251,9 @@ Quiet Quiet is designed for non-attended running in situations where normal status messages are not desired. It will output nothing except errors -and serious warnings. Like Noninteractive.Basic, this user interface is -not capable of reading a password from the keyboard; account passwords -must be specified using one of the configuration file options. +and serious warnings. Like Basic, this user interface is not capable +of reading a password from the keyboard; account passwords must be +specified using one of the configuration file options. MachineUI --------- From 86cfd31ce96456c9e8ec60c7722b045914031e6a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 25 Jul 2011 18:46:41 +0200 Subject: [PATCH 162/817] v6.3.4-rc4 Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 20 ++++++++++++++++++++ offlineimap/__init__.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index b5fb686..c872055 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,26 @@ ChangeLog releases announces. +OfflineIMAP v6.3.4-rc4 (2011-07-27) +=================================== + +Notes +----- + +There is nothing exciting in this release. This is somewhat expected due to the +late merge on -rc3. + +New Features +------------ + +* Support maildir for Windows. + +Changes +------- + +* Manual improved. + + OfflineIMAP v6.3.4-rc3 (2011-07-07) =================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 4a0e52c..d35df7f 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.4-rc3" +__version__ = "6.3.4-rc4" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 86e9c7442bed8703e059cdcbd18bbea71c513ae8 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Tue, 26 Jul 2011 10:59:53 +0200 Subject: [PATCH 163/817] Include message header at better place It's not enough to place header after first newline, since this might break multiline rfc0822 folded long header lines. Those are difined as CRLF followed by white space. Instead we'll search for two successive CRLF sequences which mark end of mail headers and place our header just before that. Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 9297a7b..19a0340 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -276,7 +276,7 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage_addheader: called to add %s: %s' % (headername, headervalue)) - insertionpoint = content.find("\r\n") + insertionpoint = content.find("\r\n\r\n") self.ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint) leader = content[0:insertionpoint] self.ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader)) From d5cbdc4c0e51842af4979c63297e22f7971ae62b Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Tue, 26 Jul 2011 10:59:54 +0200 Subject: [PATCH 164/817] Handle when UID can't be found on saved messages Message was stored to dstfolder, but we can't find it's UID. This means we can't link current message to the one created in IMAP. So we just delete local message and on next run we'll sync it back. Also fixed imap.savemessage description. This was broken by e20d8b967942934ddbf4659b5ec328a9a18da6bc. Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 8 ++++++++ offlineimap/folder/IMAP.py | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 608d361..0d5ddae 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -264,6 +264,14 @@ class BaseFolder(object): uid = newuid # Save uploaded status in the statusfolder statusfolder.savemessage(uid, message, flags, rtime) + elif newuid == 0: + # Message was stored to dstfolder, but we can't find it's UID + # This means we can't link current message to the one created + # in IMAP. So we just delete local message and on next run + # we'll sync it back + # XXX This could cause infinite loop on syncing between two + # IMAP servers ... + self.deletemessage(uid) else: raise UserWarning("Trying to save msg (uid %d) on folder " "%s returned invalid uid %d" % \ diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 19a0340..1fa35f7 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -409,8 +409,10 @@ class IMAPFolder(BaseFolder): the new message after sucessfully saving it. :param rtime: A timestamp to be used as the mail date - :returns: the UID of the new message as assigned by the - server. If the folder is read-only it will return 0.""" + :returns: the UID of the new message as assigned by the server. If the + message is saved, but it's UID can not be found, it will + return 0. If the message can't be written (folder is + read-only for example) it will return -1.""" self.ui.debug('imap', 'savemessage: called') # already have it, just save modified flags From e58399ac0b8f7542bf6cb6e25a87004af25f38f3 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Tue, 26 Jul 2011 10:59:55 +0200 Subject: [PATCH 165/817] Fixed typo Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 1fa35f7..3c702f4 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -506,7 +506,7 @@ class IMAPFolder(BaseFolder): return result = imapobj.uid('store', '%d' % uid, 'FLAGS', imaputil.flagsmaildir2imap(flags)) - assert result[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) + assert result[0] == 'OK', 'Error with store: ' + '. '.join(result[1]) finally: self.imapserver.releaseconnection(imapobj) result = result[1][0] From 08cd110830ba3696fa38684bce1b2faa7f627458 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Sun, 31 Jul 2011 10:48:24 +0200 Subject: [PATCH 166/817] FAQ: add commant about UID not found on saved messages This add missing doc as requested on the mailing list for commit d5cbdc4c0e5. Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- docs/FAQ.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index cebe766..3181a8a 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -189,6 +189,11 @@ Then, on your next sync, the message will be re-downloaded with the proper UID. `OfflineIMAP`_ makes sure that the message was properly uploaded before deleting it, so there should be no risk of data loss. +But if you try to sync between two IMAP servers, where both are unable to +provide you with UID of the new message, then this will lead to infinite loop. +`OfflineIMAP`_ will upload the message to one server and delete on second. On +next run it will upload the message to second server and delete on first, etc. + Does OfflineIMAP support POP? ----------------------------- From 9d5f805981b2db9d9265c41c8a28bcf8f5784dc0 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Sun, 31 Jul 2011 10:48:24 +0200 Subject: [PATCH 167/817] FAQ: add commant about UID not found on saved messages This add missing doc as requested on the mailing list for commit d5cbdc4c0e5. Also, add an entry in the changelog. Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ docs/FAQ.rst | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index e840717..938a5e6 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,8 @@ others. New Features ------------ +* Handle when UID can't be found on saved messages. + Changes ------- diff --git a/docs/FAQ.rst b/docs/FAQ.rst index cebe766..3181a8a 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -189,6 +189,11 @@ Then, on your next sync, the message will be re-downloaded with the proper UID. `OfflineIMAP`_ makes sure that the message was properly uploaded before deleting it, so there should be no risk of data loss. +But if you try to sync between two IMAP servers, where both are unable to +provide you with UID of the new message, then this will lead to infinite loop. +`OfflineIMAP`_ will upload the message to one server and delete on second. On +next run it will upload the message to second server and delete on first, etc. + Does OfflineIMAP support POP? ----------------------------- From 61e50b3b1a81a564697a74384f528b22dfd32d8b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 10 Aug 2011 21:44:58 +0200 Subject: [PATCH 168/817] v6.3.4 Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 -- Changelog.rst | 14 ++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 938a5e6..e840717 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,8 +13,6 @@ others. New Features ------------ -* Handle when UID can't be found on saved messages. - Changes ------- diff --git a/Changelog.rst b/Changelog.rst index c872055..95a13b8 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,20 @@ ChangeLog releases announces. +OfflineIMAP v6.3.4 (2011-08-10) +=============================== + +Notes +----- + +Here we are. A nice release since v6.3.3, I think. + +Changes +------- + +* Handle when UID can't be found on saved messages. + + OfflineIMAP v6.3.4-rc4 (2011-07-27) =================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index d35df7f..5070ac7 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.4-rc4" +__version__ = "6.3.4" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From f937a86571ff9d3ada7fb9d6f3badc29ebc2d15c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 11 Aug 2011 12:22:34 +0200 Subject: [PATCH 169/817] Implement ui.error() and output them at end of offlinimap sync Output all raised Exceptions error strings to the error log. If we are in debug mode, we also output the traceback of the exception. Save all exceptions that occur during the run time to a Queue and output them all again when the offlineimap sync is finished. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 58 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 5524579..c5f897f 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -21,6 +21,7 @@ import time import sys import traceback import threading +from Queue import Queue import offlineimap debugtypes = {'':'Other offlineimap related sync messages', @@ -47,7 +48,9 @@ class UIBase: s.debugmsglen = 50 s.threadaccounts = {} s.logfile = None - + s.exc_queue = Queue() + """saves all occuring exceptions, so we can output them at the end""" + ################################################## UTILS def _msg(s, msg): """Generic tool called when no other works.""" @@ -82,6 +85,39 @@ class UIBase: else: s._msg("WARNING: " + msg) + def error(self, exc, exc_traceback=None, msg=None): + """Log a message at severity level ERROR + + Log Exception 'exc' to error log, possibly prepended by a preceding + error "msg", detailing at what point the error occurred. + + In debug mode, we also output the full traceback that occurred + if one has been passed in via sys.exc_traceback. + + Also save the Exception to a stack that can be output at the end + of the sync run when offlineiamp exits. It is recommended to + always pass in exceptions if possible, so we can give the user + the best debugging info. + + One example of such a call might be: + + ui.error(exc, sys.exc_traceback, msg="While syncing Folder %s in " + "repo %s") + """ + cur_thread = threading.currentThread() + if msg: + self._msg("ERROR [%s]: %s\n %s" % (cur_thread, msg, exc)) + else: + self._msg("ERROR [%s]: %s" % (cur_thread, exc)) + + if not self.debuglist: + # only output tracebacks in debug mode + exc_traceback = None + # push exc on the queue for later output + self.exc_queue.put((msg, exc, exc_traceback)) + if exc_traceback: + self._msg(traceback.format_tb(exc_traceback)) + def registerthread(s, account): """Provides a hint to UIs about which account this particular thread is processing.""" @@ -315,12 +351,24 @@ class UIBase: def mainException(s): s._msg(s.getMainExceptionString()) - def terminate(s, exitstatus = 0, errortitle = None, errormsg = None): + def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" - if errormsg <> None: - if errortitle <> None: - sys.stderr.write('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) + #print any exceptions that have occurred over the run + if not self.exc_queue.empty(): + self._msg("\nERROR: Exceptions occurred during the run!") + while not self.exc_queue.empty(): + msg, exc, exc_traceback = self.exc_queue.get() + if msg: + self._msg("ERROR: %s\n %s" % (msg, exc)) else: + self._msg("ERROR: %s" % (exc)) + if exc_traceback: + self._msg("\nTraceback:\n%s" %"".join( + traceback.format_tb(exc_traceback))) + + if errormsg and errortitle: + sys.stderr.write('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) + elif errormsg: sys.stderr.write('%s\n' % errormsg) sys.exit(exitstatus) From b47bc4478320a0a0cd7a2a56298509d016e49fb1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 11 Aug 2011 12:22:35 +0200 Subject: [PATCH 170/817] accounts.py: Use ui.error when raising exceptions Rather than using ui.warn, use ui.error() which outputs Exceptions to the error log, saving them to a stack, so we get notified again at the end of the sync run. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 4 + offlineimap/accounts.py | 23 +++--- offlineimap/folder/Base.py | 148 +++++++++++++++++++------------------ 3 files changed, 95 insertions(+), 80 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index e840717..53fb381 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,10 @@ others. New Features ------------ +* When a message upload/download fails, we do not abort the whole folder + synchronization, but only skip that message, informing the user at the + end of the sync run. + Changes ------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index a989441..8653d48 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -22,6 +22,7 @@ from offlineimap.threadutil import InstanceLimitedThread from subprocess import Popen, PIPE from threading import Event import os +from sys import exc_info import traceback def getaccountlist(customconfig): @@ -178,16 +179,16 @@ class SyncableAccount(Account): except (KeyboardInterrupt, SystemExit): raise except OfflineImapError, e: - self.ui.warn(e.reason) # Stop looping and bubble up Exception if needed. if e.severity >= OfflineImapError.ERROR.REPO: if looping: looping -= 1 if e.severity >= OfflineImapError.ERROR.CRITICAL: raise - except: - self.ui.warn("Error occured attempting to sync account "\ - "'%s':\n%s"% (self, traceback.format_exc())) + self.ui.error(e, exc_info()[2]) + except Exception, e: + self.ui.error(e, msg = "While attempting to sync " + "account %s:\n %s"% (self, traceback.format_exc())) else: # after success sync, reset the looping counter to 3 if self.refreshperiod: @@ -276,8 +277,10 @@ class SyncableAccount(Account): r = p.communicate() self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n" % r) self.ui.callhook("Hook return code: %d" % p.returncode) - except: - self.ui.warn("Exception occured while calling hook") + except (KeyboardInterrupt, SystemExit): + raise + except Exception, e: + self.ui.error(e, exc_info()[2], msg = "Calling hook") def syncfolder(accountname, remoterepos, remotefolder, localrepos, @@ -366,9 +369,9 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, if e.severity > OfflineImapError.ERROR.FOLDER: raise else: - ui.warn("Aborting folder sync '%s' [acc: '%s']\nReason was: %s" %\ - (localfolder.name, accountname, e.reason)) - except: - ui.warn("ERROR in syncfolder for %s folder %s: %s" % \ + ui.error(e, exc_info()[2], msg = "Aborting folder sync '%s' " + "[acc: '%s']" % (localfolder, accountname)) + except Exception, e: + ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ (accountname,remotefolder.getvisiblename(), traceback.format_exc())) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 0d5ddae..5a67775 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -19,6 +19,7 @@ from offlineimap import threadutil from offlineimap.ui import getglobalui import os.path import re +from sys import exc_info import traceback class BaseFolder(object): @@ -229,62 +230,55 @@ class BaseFolder(object): # synced to the status cache. This is only a problem with # self.getmessage(). So, don't call self.getmessage unless # really needed. - try: - if register: # output that we start a new thread - self.ui.registerthread(self.getaccountname()) + if register: # output that we start a new thread + self.ui.registerthread(self.getaccountname()) - message = None - flags = self.getmessageflags(uid) - rtime = self.getmessagetime(uid) + message = None + flags = self.getmessageflags(uid) + rtime = self.getmessagetime(uid) - if uid > 0 and dstfolder.uidexists(uid): - # dst has message with that UID already, only update status - statusfolder.savemessage(uid, None, flags, rtime) - return + if uid > 0 and dstfolder.uidexists(uid): + # dst has message with that UID already, only update status + statusfolder.savemessage(uid, None, flags, rtime) + return - self.ui.copyingmessage(uid, self, [dstfolder]) - # If any of the destinations actually stores the message body, - # load it up. - if dstfolder.storesmessages(): - - message = self.getmessage(uid) - #Succeeded? -> IMAP actually assigned a UID. If newid - #remained negative, no server was willing to assign us an - #UID. If newid is 0, saving succeeded, but we could not - #retrieve the new UID. Ignore message in this case. - newuid = dstfolder.savemessage(uid, message, flags, rtime) - if newuid > 0: - if newuid != uid: - # Got new UID, change the local uid. - #TODO: Maildir could do this with a rename rather than - #load/save/del operation, IMPLEMENT a changeuid() - #function or so. - self.savemessage(newuid, message, flags, rtime) - self.deletemessage(uid) - uid = newuid - # Save uploaded status in the statusfolder - statusfolder.savemessage(uid, message, flags, rtime) - elif newuid == 0: - # Message was stored to dstfolder, but we can't find it's UID - # This means we can't link current message to the one created - # in IMAP. So we just delete local message and on next run - # we'll sync it back - # XXX This could cause infinite loop on syncing between two - # IMAP servers ... + self.ui.copyingmessage(uid, self, [dstfolder]) + # If any of the destinations actually stores the message body, + # load it up. + if dstfolder.storesmessages(): + message = self.getmessage(uid) + #Succeeded? -> IMAP actually assigned a UID. If newid + #remained negative, no server was willing to assign us an + #UID. If newid is 0, saving succeeded, but we could not + #retrieve the new UID. Ignore message in this case. + newuid = dstfolder.savemessage(uid, message, flags, rtime) + if newuid > 0: + if newuid != uid: + # Got new UID, change the local uid. + #TODO: Maildir could do this with a rename rather than + #load/save/del operation, IMPLEMENT a changeuid() + #function or so. + self.savemessage(newuid, message, flags, rtime) self.deletemessage(uid) - else: - raise UserWarning("Trying to save msg (uid %d) on folder " - "%s returned invalid uid %d" % \ - (uid, - dstfolder.getvisiblename(), - newuid)) - except (KeyboardInterrupt): - raise - except: - self.ui.warn("ERROR attempting to copy message " + str(uid) \ - + " for account " + self.getaccountname() + ":" \ - + traceback.format_exc()) - raise + uid = newuid + # Save uploaded status in the statusfolder + statusfolder.savemessage(uid, message, flags, rtime) + + elif newuid == 0: + # Message was stored to dstfolder, but we can't find it's UID + # This means we can't link current message to the one created + # in IMAP. So we just delete local message and on next run + # we'll sync it back + # XXX This could cause infinite loop on syncing between two + # IMAP servers ... + self.deletemessage(uid) + else: + raise OfflineImapError("Trying to save msg (uid %d) on folder " + "%s returned invalid uid %d" % \ + (uid, + dstfolder.getvisiblename(), + newuid), + OfflineImapError.ERROR.MESSAGE) def syncmessagesto_copy(self, dstfolder, statusfolder): """Pass1: Copy locally existing messages not on the other side @@ -303,20 +297,30 @@ class BaseFolder(object): statusfolder.uidexists(uid), self.getmessageuidlist()) for uid in copylist: - if self.suggeststhreads(): - self.waitforthread() - thread = threadutil.InstanceLimitedThread(\ - self.getcopyinstancelimit(), - target = self.copymessageto, - name = "Copy message %d from %s" % (uid, + try: + if self.suggeststhreads(): + self.waitforthread() + thread = threadutil.InstanceLimitedThread(\ + self.getcopyinstancelimit(), + target = self.copymessageto, + name = "Copy message %d from %s" % (uid, self.getvisiblename()), - args = (uid, dstfolder, statusfolder)) - thread.setDaemon(1) - thread.start() - threads.append(thread) - else: - self.copymessageto(uid, dstfolder, statusfolder, register = 0) - + args = (uid, dstfolder, statusfolder)) + thread.setDaemon(1) + thread.start() + threads.append(thread) + else: + self.copymessageto(uid, dstfolder, statusfolder, + register = 0) + except OfflineImapError, e: + if e.severity > OfflineImapError.ERROR.Message: + raise # buble severe errors up + self.ui.error(e, exc_info()[2]) + except Exception, e: + self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ + (uid, self.getaccountname(), + traceback.format_exc())) + raise #raise on unknown errors, so we can fix those for thread in threads: thread.join() @@ -421,8 +425,12 @@ class BaseFolder(object): action(dstfolder, statusfolder) except (KeyboardInterrupt): raise - except: - self.ui.warn("ERROR attempting to sync flags " \ - + "for account " + self.getaccountname() \ - + ":" + traceback.format_exc()) - raise + except OfflineImap, e: + if e.severity > OfflineImapError.ERROR.FOLDER: + raise + self.ui.error(e, exc_info()[2]) + except Exception, e: + self.ui.error(e, msg = "ERROR attempting to sync folder %s " + "[acc: %s]:\n %s" (self, self.getaccountname(), + traceback.format_exc())) + raise # raise unknown Exceptions so we can fix them From f6b9c683338c72349bbec903ac3c51a2ff03c0e3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 11 Aug 2011 12:22:36 +0200 Subject: [PATCH 171/817] IMAP: Don't use assert() in folder.savemessage() We simply assert()ed that APPENDing a message returned OK, but in some cases (e.g. Google chat messages) APPEND might return BAD or NO too. We should be throwing an OfflineImapError here at MESSAGE level, so that we can continue to sync all other messages, and still give the user some details on what went wrong at the end of the sync run. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 3c702f4..6bb9cfb 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -450,16 +450,20 @@ class IMAPFolder(BaseFolder): content[-50:]) else: dbg_output = content - self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % (date, dbg_output)) - (typ,dat) = imapobj.append(self.getfullname(), + #Do the APPEND + (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) - assert(typ == 'OK') - - # Checkpoint. Let it write out the messages, etc. + if typ != 'OK': #APPEND failed + raise OfflineImapError("Saving msg in folder '%s', repository " + "'%s' failed. Server reponded; %s %s\nMessage content was:" + " %s" % (self, self.getrepository(), typ, dat, dbg_output), + OfflineImapError.ERROR.MESSAGE) + # Checkpoint. Let it write out stuff, etc. Eg searches for + # just uploaded messages won't work if we don't do this. (typ,dat) = imapobj.check() assert(typ == 'OK') From 131298c2b1fa7ab57c8260bbfc53ef7ada3d7f13 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 12 Aug 2011 08:31:09 +0200 Subject: [PATCH 172/817] Remove superfluous class ConfigedIMAPServer Remove a level of wrapper abstraction that is not needed. Just use IMAPserver and be done with it. We do this by passing in the IMAPRepository() instance rather than a long list of single paramters to the IMAPServer instanciation. This way we can retrieve all repository parameters ourselves, rather than passing a dozen paramters into IMAPServer. Also, this enables us to pass the repository() object into our WrappedIMAP4() instance, so that it can query, e.g. the SSL fingerprint configuration. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 6 +- offlineimap/imapserver.py | 117 ++++++++++----------------------- offlineimap/repository/IMAP.py | 2 +- 3 files changed, 40 insertions(+), 85 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 53fb381..e57b757 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -20,10 +20,14 @@ New Features Changes ------- +* Refactor our IMAPServer class. Background work without user-visible + changes. + Bug Fixes --------- - +* We protect more robustly against asking for inexistent messages from the + IMAP server, when someone else deletes or moves messages while we sync. Pending for the next major release ================================== diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index a235fd5..6566ad5 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -42,58 +42,56 @@ except ImportError: pass class IMAPServer: + """Initializes all variables from an IMAPRepository() instance + + Various functions, such as acquireconnection() return an IMAP4 + object on which we can operate.""" GSS_STATE_STEP = 0 GSS_STATE_WRAP = 1 - def __init__(self, config, reposname, - username = None, password = None, hostname = None, - port = None, ssl = 1, maxconnections = 1, tunnel = None, - reference = '""', sslclientcert = None, sslclientkey = None, - sslcacertfile = None, idlefolders = []): + def __init__(self, repos): self.ui = getglobalui() - self.reposname = reposname - self.config = config - self.username = username - self.password = password + self.repos = repos + self.config = repos.getconfig() + self.tunnel = repos.getpreauthtunnel() + self.usessl = repos.getssl() + self.username = repos.getuser() + self.password = None self.passworderror = None self.goodpassword = None - self.hostname = hostname - self.tunnel = tunnel - self.port = port - self.usessl = ssl - self.sslclientcert = sslclientcert - self.sslclientkey = sslclientkey - self.sslcacertfile = sslcacertfile + self.hostname = repos.gethost() + self.port = repos.getport() + if self.port == None: + self.port = 993 if self.usessl else 143 + self.sslclientcert = repos.getsslclientcert() + self.sslclientkey = repos.getsslclientkey() + self.sslcacertfile = repos.getsslcacertfile() self.delim = None self.root = None - if port == None: - if ssl: - self.port = 993 - else: - self.port = 143 - self.maxconnections = maxconnections + self.maxconnections = repos.getmaxconnections() self.availableconnections = [] self.assignedconnections = [] self.lastowner = {} self.semaphore = BoundedSemaphore(self.maxconnections) self.connectionlock = Lock() - self.reference = reference - self.idlefolders = idlefolders + self.reference = repos.getreference() + self.idlefolders = repos.getidlefolders() self.gss_step = self.GSS_STATE_STEP self.gss_vc = None self.gssapi = False def getpassword(self): - if self.goodpassword != None: + """Returns the server password or None""" + if self.goodpassword != None: # use cached good one first return self.goodpassword if self.password != None and self.passworderror == None: - return self.password + return self.password # non-failed preconfigured one - self.password = self.ui.getpass(self.reposname, - self.config, - self.passworderror) + # get 1) configured password first 2) fall back to asking via UI + self.password = self.repos.getpassword() or \ + self.ui.getpass(self.repos.getname(), self.config, + self.passworderror) self.passworderror = None - return self.password def getdelim(self): @@ -204,7 +202,8 @@ class IMAPServer: success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) - imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, self.port, + imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, + self.port, self.sslclientkey, self.sslclientcert, timeout=socket.getdefaulttimeout(), cacertfile = self.sslcacertfile) @@ -260,7 +259,6 @@ class IMAPServer: except imapobj.error, val: self.passworderror = str(val) raise - #self.password = None if self.delim == None: listres = imapobj.list(self.reference, '""')[1] @@ -292,8 +290,6 @@ class IMAPServer: error...""" self.semaphore.release() - #Make sure that this can be retried the next time... - self.passworderror = None if(self.connectionlock.locked()): self.connectionlock.release() @@ -304,7 +300,7 @@ class IMAPServer: reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ "ver name correctly and that you are online."%\ - (self.hostname, self.reposname) + (self.hostname, self.repos) raise OfflineImapError(reason, severity) elif SSLError and isinstance(e, SSLError) and e.errno == 1: @@ -317,7 +313,7 @@ class IMAPServer: else: reason = "Unknown SSL protocol connecting to host '%s' for"\ "repository '%s'. OpenSSL responded:\n%s"\ - % (self.hostname, self.reposname, e) + % (self.hostname, self.repos, e) raise OfflineImapError(reason, severity) elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: @@ -333,7 +329,8 @@ class IMAPServer: if str(e)[:24] == "can't open socket; error": raise OfflineImapError("Could not connect to remote server '%s' "\ "for repository '%s'. Remote does not answer." - % (self.hostname, self.reposname), severity) + % (self.hostname, self.repos), + OfflineImapError.ERROR.REPO) else: # re-raise all other errors raise @@ -484,49 +481,3 @@ class IdleThread(object): if self.needsync: self.event.clear() self.dosync() - -class ConfigedIMAPServer(IMAPServer): - """This class is designed for easier initialization given a ConfigParser - object and an account name. The passwordhash is used if - passwords for certain accounts are known. If the password for this - account is listed, it will be obtained from there.""" - def __init__(self, repository, passwordhash = {}): - """Initialize the object. If the account is not a tunnel, - the password is required.""" - self.repos = repository - self.config = self.repos.getconfig() - usetunnel = self.repos.getpreauthtunnel() - if not usetunnel: - host = self.repos.gethost() - user = self.repos.getuser() - port = self.repos.getport() - ssl = self.repos.getssl() - sslclientcert = self.repos.getsslclientcert() - sslclientkey = self.repos.getsslclientkey() - sslcacertfile = self.repos.getsslcacertfile() - reference = self.repos.getreference() - idlefolders = self.repos.getidlefolders() - server = None - password = None - - if repository.getname() in passwordhash: - password = passwordhash[repository.getname()] - - # Connect to the remote server. - if usetunnel: - IMAPServer.__init__(self, self.config, self.repos.getname(), - tunnel = usetunnel, - reference = reference, - idlefolders = idlefolders, - maxconnections = self.repos.getmaxconnections()) - else: - if not password: - password = self.repos.getpassword() - IMAPServer.__init__(self, self.config, self.repos.getname(), - user, password, host, port, ssl, - self.repos.getmaxconnections(), - reference = reference, - idlefolders = idlefolders, - sslclientcert = sslclientcert, - sslclientkey = sslclientkey, - sslcacertfile = sslcacertfile) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 82d9e32..f120d19 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -33,7 +33,7 @@ class IMAPRepository(BaseRepository): BaseRepository.__init__(self, reposname, account) # self.ui is being set by the BaseRepository self._host = None - self.imapserver = imapserver.ConfigedIMAPServer(self) + self.imapserver = imapserver.IMAPServer(self) self.folders = None self.nametrans = lambda foldername: foldername self.folderfilter = lambda foldername: 1 From fa18968642c527600837b692ea11cc9b80c0a256 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 12 Aug 2011 08:56:56 +0200 Subject: [PATCH 173/817] Catch folderfilter errors in repository.IMAP.getfolders() Rather than throwing ValueError, we now properly throw OfflineImapError when selecting a folder in folderincludes. So we also need to catch OfflineImapErrors here. If they are of severity FOLDER, just ignore the invalid folder and continue. If the error is more severe, bubble it up. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 4 +++- offlineimap/repository/IMAP.py | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 53fb381..bffe3b4 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -23,7 +23,9 @@ Changes Bug Fixes --------- - +* Selecting inexistent folders specified in folderincludes now throws + nice errors and continues to sync with all other folders rather than + exiting offlineimap with a traceback. Pending for the next major release ================================== diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 82d9e32..7b23d8e 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -24,6 +24,7 @@ from threading import Event import re import types import os +from sys import exc_info import netrc import errno @@ -307,7 +308,12 @@ class IMAPRepository(BaseRepository): for foldername in self.folderincludes: try: imapobj.select(foldername, readonly = 1) - except ValueError: + except OfflineImapError, e: + # couldn't select this folderinclude, so ignore folder. + if e.severity > OfflineImapError.ERROR.FOLDER: + raise + self.ui.error(e, exc_info()[2], + 'Invalid folderinclude:') continue retval.append(self.getfoldertype()(self.imapserver, foldername, From 97dbd18e12094820be5985b9daec4ceab4d86116 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 12 Aug 2011 09:12:04 +0200 Subject: [PATCH 174/817] LocalStatus.py: Fix getfolders() This code was unused and broken. It is still unused but this commit fixes it. (We should retain the method in case we ever start calling getfolders() on LocalStatus. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/LocalStatus.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 022be21..50dfdc8 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -80,14 +80,13 @@ class LocalStatusRepository(BaseRepository): self.config) def getfolders(self): - """Returns a list of ALL folders on this server. - - This is currently nowhere used in the code.""" + """Returns a list of all cached folders.""" if self._folders != None: return self._folders + self._folders = [] for folder in os.listdir(self.directory): - self._folders = retval.append(self.getfolder(folder)) + self._folders.append(self.getfolder(folder)) return self._folders def forgetfolders(self): From c4320786ab78fca8bea5cd079c0596092c6ab602 Mon Sep 17 00:00:00 2001 From: Thomas Kahle Date: Sat, 13 Aug 2011 13:05:11 +0100 Subject: [PATCH 175/817] Fix manpage build failures from inconsistent heading Signed-off-by: Mark Foxwell Signed-off-by: Thomas Kahle Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index af5683f..8f09f92 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -380,7 +380,7 @@ mail.server -port 443" to find out the connection that is used by default. Certificate checking -^^^^^^^^^^^^^^^^^^^^ +-------------------- Unfortunately, by default we will not verify the certificate of an IMAP TLS/SSL server we connect to, so connecting by SSL is no guarantee @@ -397,7 +397,7 @@ its expiration date. The FAQ contains an entry on how to create your own certificate and CA certificate. StartTLS -^^^^^^^^ +-------- If you have not configured your account to connect via SSL anyway, OfflineImap will still attempt to set up an SSL connection via the @@ -410,4 +410,3 @@ contents. However, this will not protect you from active attacks, such as Man-In-The-Middle attacks which cause you to connect to the wrong server and pretend to be your mail server. DO NOT RELY ON STARTTLS AS A SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF YOUR IMAP SERVER! -======= From 306f584c86a7ed9a352a0814de34b810e7775b0e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 14 Aug 2011 10:42:17 +0200 Subject: [PATCH 176/817] Remove custom Gmail/folder/processmessagesflags() This function was overridden as the IMAP version apparently had been using imapobj.myrights() at some point in time, which was not implemented in the Gmail version. However, IMAP is not using myrights() anymore, and as that is an extension that needs to be advertised in CAPABILITIES we should not unconditionally use it anyway. So remove the function that is identical to it's ancestor's function. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Gmail.py | 54 ------------------------------------- 1 file changed, 54 deletions(-) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 2a82833..7cbff4f 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -65,57 +65,3 @@ class GmailFolder(IMAPFolder): del self.messagelist[uid] else: IMAPFolder.deletemessages_noconvert(self, uidlist) - - def processmessagesflags(self, operation, uidlist, flags): - # XXX: the imapobj.myrights(...) calls dies with an error - # report from Gmail server stating that IMAP command - # 'MYRIGHTS' is not implemented. So, this - # `processmessagesflags` is just a copy from `IMAPFolder`, - # with the references to `imapobj.myrights()` deleted This - # shouldn't hurt, however, Gmail users always have full - # control over all their mailboxes (apparently). - if len(uidlist) > 101: - # Hack for those IMAP ervers with a limited line length - self.processmessagesflags(operation, uidlist[:100], flags) - self.processmessagesflags(operation, uidlist[100:], flags) - return - - imapobj = self.imapserver.acquireconnection() - try: - imapobj.select(self.getfullname()) - r = imapobj.uid('store', - imaputil.listjoin(uidlist), - operation + 'FLAGS', - imaputil.flagsmaildir2imap(flags)) - assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) - r = r[1] - finally: - self.imapserver.releaseconnection(imapobj) - - needupdate = copy(uidlist) - for result in r: - if result == None: - # Compensate for servers that don't return anything from - # STORE. - continue - attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1]) - if not ('UID' in attributehash and 'FLAGS' in attributehash): - # Compensate for servers that don't return a UID attribute. - continue - flags = attributehash['FLAGS'] - uid = long(attributehash['UID']) - self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags) - try: - needupdate.remove(uid) - except ValueError: # Let it slide if it's not in the list - pass - for uid in needupdate: - if operation == '+': - for flag in flags: - if not flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].append(flag) - self.messagelist[uid]['flags'].sort() - elif operation == '-': - for flag in flags: - if flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].remove(flag) From 3c89a72be91ea1e9a9edbb510de2ddd890db2f79 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 14 Aug 2011 10:42:18 +0200 Subject: [PATCH 177/817] remove unused 'mustquote' regex We set an imapobj.mustquote which apparently was used in previous incarnations of imaplib or imaplib2, however, nothing in our codebase makes use of that. So let us remove it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 2 -- offlineimap/imapserver.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 6492086..c56b634 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -284,8 +284,6 @@ class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): # imaplib2 uses this to poll() self.read_fd = self.sock.fileno() -mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]") - def Internaldate2epoch(resp): """Convert IMAP4 INTERNALDATE to UT. diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 6566ad5..28511bb 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -212,8 +212,6 @@ class IMAPServer: imapobj = imaplibutil.WrappedIMAP4(self.hostname, self.port, timeout=socket.getdefaulttimeout()) - imapobj.mustquote = imaplibutil.mustquote - if not self.tunnel: try: # Try GSSAPI and continue if it fails From 51629b8ffe4ec4f3af6b214ba86519c576b0fe7e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 14 Aug 2011 10:42:21 +0200 Subject: [PATCH 178/817] Update example in code documentation of ui.error() sys.exc_traceback is long deprecated and is seems removed in python2.7, so document the legitimate use of sys.exc_info()[2] instead. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index c5f897f..c08d34c 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -92,7 +92,7 @@ class UIBase: error "msg", detailing at what point the error occurred. In debug mode, we also output the full traceback that occurred - if one has been passed in via sys.exc_traceback. + if one has been passed in via sys.info()[2]. Also save the Exception to a stack that can be output at the end of the sync run when offlineiamp exits. It is recommended to @@ -101,7 +101,7 @@ class UIBase: One example of such a call might be: - ui.error(exc, sys.exc_traceback, msg="While syncing Folder %s in " + ui.error(exc, sys.exc_info()[2], msg="While syncing Folder %s in " "repo %s") """ cur_thread = threading.currentThread() From 194aa1db3c261bc581b466dd9898ebadce024246 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 15 Aug 2011 10:18:37 +0200 Subject: [PATCH 179/817] Catch 'BAD' replies on append()ing a message append() raises an Exception, in case the IMAP server replies with 'BAD' (but not when it responds with 'NO') but we were not catching that. Do catch the situation and also raise an OfflineImapError at MESSAGE severity, so that we can continue with the next message. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 6bb9cfb..b4120a6 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -454,9 +454,14 @@ class IMAPFolder(BaseFolder): (date, dbg_output)) #Do the APPEND - (typ, dat) = imapobj.append(self.getfullname(), + try: + (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) + except Exception, e: + # If the server responds with 'BAD', append() raise()s directly. + # So we need to prepare a response ourselves. + typ, dat = 'BAD', str(e) if typ != 'OK': #APPEND failed raise OfflineImapError("Saving msg in folder '%s', repository " "'%s' failed. Server reponded; %s %s\nMessage content was:" From 634b6cd49ef02ab81720635f0fe1cb2f06f9abaf Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 15 Aug 2011 09:59:56 +0200 Subject: [PATCH 180/817] Don't make Blinkenlight statuschar configurable Allowing to specify the char to use in the BLinkenlights is a bit over the top and bloats our default offlineimap.conf. The dot is just fine, so let us settle for it and cut the example config file by an unneeded section. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 3 +++ offlineimap.conf | 4 ---- offlineimap/ui/Curses.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 6b5b18e..96558c5 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -23,6 +23,9 @@ Changes * Refactor our IMAPServer class. Background work without user-visible changes. +* Remove the configurability of the Blinkenlights statuschar. It + cluttered the main configuration file for little gain. + Bug Fixes --------- diff --git a/offlineimap.conf b/offlineimap.conf index 2a4b1a2..8cc2882 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -141,10 +141,6 @@ footer = "\n" # Note that this filter can be used only to further restrict mbnames # to a subset of folders that pass the account's folderfilter. -[ui.Curses.Blinkenlights] -# Character used to indicate thread status. - -statuschar = . ################################################## # Accounts diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 9e0b9d5..afe3acb 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -230,7 +230,7 @@ class CursesThreadFrame: if self.getcolor() == 'black': self.window.addstr(self.y, self.x, ' ', self.color) else: - self.window.addstr(self.y, self.x, self.ui.config.getdefault("ui.Curses.Blinkenlights", "statuschar", '.'), self.color) + self.window.addstr(self.y, self.x, '.', self.color) self.c.stdscr.move(self.c.height - 1, self.c.width - 1) self.window.refresh() self.c.locked(lockedstuff) From 9f23fea74edd6e50604628e1de042fd90eae5a7f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 15 Aug 2011 11:00:06 +0200 Subject: [PATCH 181/817] Fix error handling in folder.Base.copymessageto() This is a bug fix on several levels. 1) We were lacking the import of OfflineImapError. 2) OfflineImap.ERROR.MESSAGE was misspelled as ERROR.Message. 3) COntinuing with the next message only worked in single-thread mode (using debug) and not in multi-thread mode. The reason is that we were invoking a new thread and catching Exceptions in the main thread. But python immediately aborts if an Exception bubbles up to a thread start. This was fixed by catching exceptions directly in copymessageto() which is the new thread, rather than catching them in the main thread. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 135 +++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 66 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 5a67775..3da0de1 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -17,6 +17,7 @@ from offlineimap import threadutil from offlineimap.ui import getglobalui +from offlineimap.error import OfflineImapError import os.path import re from sys import exc_info @@ -233,52 +234,63 @@ class BaseFolder(object): if register: # output that we start a new thread self.ui.registerthread(self.getaccountname()) - message = None - flags = self.getmessageflags(uid) - rtime = self.getmessagetime(uid) + try: + message = None + flags = self.getmessageflags(uid) + rtime = self.getmessagetime(uid) - if uid > 0 and dstfolder.uidexists(uid): - # dst has message with that UID already, only update status - statusfolder.savemessage(uid, None, flags, rtime) - return + if uid > 0 and dstfolder.uidexists(uid): + # dst has message with that UID already, only update status + statusfolder.savemessage(uid, None, flags, rtime) + return - self.ui.copyingmessage(uid, self, [dstfolder]) - # If any of the destinations actually stores the message body, - # load it up. - if dstfolder.storesmessages(): - message = self.getmessage(uid) - #Succeeded? -> IMAP actually assigned a UID. If newid - #remained negative, no server was willing to assign us an - #UID. If newid is 0, saving succeeded, but we could not - #retrieve the new UID. Ignore message in this case. - newuid = dstfolder.savemessage(uid, message, flags, rtime) - if newuid > 0: - if newuid != uid: - # Got new UID, change the local uid. - #TODO: Maildir could do this with a rename rather than - #load/save/del operation, IMPLEMENT a changeuid() - #function or so. - self.savemessage(newuid, message, flags, rtime) + self.ui.copyingmessage(uid, self, [dstfolder]) + # If any of the destinations actually stores the message body, + # load it up. + if dstfolder.storesmessages(): + message = self.getmessage(uid) + #Succeeded? -> IMAP actually assigned a UID. If newid + #remained negative, no server was willing to assign us an + #UID. If newid is 0, saving succeeded, but we could not + #retrieve the new UID. Ignore message in this case. + newuid = dstfolder.savemessage(uid, message, flags, rtime) + if newuid > 0: + if newuid != uid: + # Got new UID, change the local uid. + #TODO: Maildir could do this with a rename rather than + #load/save/del operation, IMPLEMENT a changeuid() + #function or so. + self.savemessage(newuid, message, flags, rtime) + self.deletemessage(uid) + uid = newuid + # Save uploaded status in the statusfolder + statusfolder.savemessage(uid, message, flags, rtime) + + elif newuid == 0: + # Message was stored to dstfolder, but we can't find it's UID + # This means we can't link current message to the one created + # in IMAP. So we just delete local message and on next run + # we'll sync it back + # XXX This could cause infinite loop on syncing between two + # IMAP servers ... self.deletemessage(uid) - uid = newuid - # Save uploaded status in the statusfolder - statusfolder.savemessage(uid, message, flags, rtime) + else: + raise OfflineImapError("Trying to save msg (uid %d) on folder " + "%s returned invalid uid %d" % \ + (uid, + dstfolder.getvisiblename(), + newuid), + OfflineImapError.ERROR.MESSAGE) + except OfflineImapError, e: + if e.severity > OfflineImapError.ERROR.MESSAGE: + raise # buble severe errors up + self.ui.error(e, exc_info()[2]) + except Exception, e: + self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ + (uid, self.getaccountname(), + traceback.format_exc())) + raise #raise on unknown errors, so we can fix those - elif newuid == 0: - # Message was stored to dstfolder, but we can't find it's UID - # This means we can't link current message to the one created - # in IMAP. So we just delete local message and on next run - # we'll sync it back - # XXX This could cause infinite loop on syncing between two - # IMAP servers ... - self.deletemessage(uid) - else: - raise OfflineImapError("Trying to save msg (uid %d) on folder " - "%s returned invalid uid %d" % \ - (uid, - dstfolder.getvisiblename(), - newuid), - OfflineImapError.ERROR.MESSAGE) def syncmessagesto_copy(self, dstfolder, statusfolder): """Pass1: Copy locally existing messages not on the other side @@ -297,30 +309,21 @@ class BaseFolder(object): statusfolder.uidexists(uid), self.getmessageuidlist()) for uid in copylist: - try: - if self.suggeststhreads(): - self.waitforthread() - thread = threadutil.InstanceLimitedThread(\ - self.getcopyinstancelimit(), - target = self.copymessageto, - name = "Copy message %d from %s" % (uid, - self.getvisiblename()), - args = (uid, dstfolder, statusfolder)) - thread.setDaemon(1) - thread.start() - threads.append(thread) - else: - self.copymessageto(uid, dstfolder, statusfolder, - register = 0) - except OfflineImapError, e: - if e.severity > OfflineImapError.ERROR.Message: - raise # buble severe errors up - self.ui.error(e, exc_info()[2]) - except Exception, e: - self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ - (uid, self.getaccountname(), - traceback.format_exc())) - raise #raise on unknown errors, so we can fix those + # exceptions are caught in copymessageto() + if self.suggeststhreads(): + self.waitforthread() + thread = threadutil.InstanceLimitedThread(\ + self.getcopyinstancelimit(), + target = self.copymessageto, + name = "Copy message %d from %s" % (uid, + self.getvisiblename()), + args = (uid, dstfolder, statusfolder)) + thread.setDaemon(1) + thread.start() + threads.append(thread) + else: + self.copymessageto(uid, dstfolder, statusfolder, + register = 0) for thread in threads: thread.join() From fe388400c45bec0114679f0423c2a600fdee82e7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 15 Aug 2011 11:00:07 +0200 Subject: [PATCH 182/817] Better error message when FETCH fails. We were not including the full server reply into our error message. Fix that so we get better error logs. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index b4120a6..7b11238 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -206,8 +206,9 @@ class IMAPFolder(BaseFolder): if data == [None] or res_type != 'OK': #IMAP server says bad request or UID does not exist severity = OfflineImapError.ERROR.MESSAGE - reason = "IMAP server '%s' responded with '%s' to fetching "\ - "message UID '%d'" % (self.getrepository(), res_type, uid) + reason = "IMAP server '%s' failed to fetch message UID '%d'."\ + "Server responded: %s %s" % (self.getrepository(), uid, + res_type, data) if data == [None]: #IMAP server did not find a message with this UID reason = "IMAP server '%s' does not have a message "\ From 1c0c19ad861d6f7dffeedb6e6c271d5725e77732 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 15 Aug 2011 11:55:41 +0200 Subject: [PATCH 183/817] imaplib2: bump to upstream version 2.28 (rev 8228a0f) Imaplib2 2.28 can deal with ID sequences, such as 1:*, so we need to bump upstream in order to make use of these features. Note that this revision will not run correctly as it requires adaptations to our code, which happens in the next commit. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 241 +++++++++++++++++++++++----------------- 1 file changed, 141 insertions(+), 100 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index ec6cd0d..cf3c480 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -2,7 +2,7 @@ """Threaded IMAP4 client. -Based on RFC 2060 and original imaplib module. +Based on RFC 3501 and original imaplib module. Public classes: IMAP4 IMAP4_SSL @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.24" +__version__ = "2.28" __release__ = "2" -__revision__ = "24" +__revision__ = "28" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -38,7 +38,8 @@ Improved timeout handling contributed by Ivan Vovnenko Oct Timeout handling further improved by Ethan Glasser-Camp December 2010. Time2Internaldate() patch to match RFC2060 specification of English month names from bugs.python.org/issue11024 March 2011. starttls() bug fixed with the help of Sebastian Spaeth April 2011. -Threads now set the "daemon" flag (suggested by offlineimap-project).""" +Threads now set the "daemon" flag (suggested by offlineimap-project) April 2011. +Single quoting introduced with the help of Vladimir Marek August 2011.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -57,7 +58,7 @@ IMAP4_SSL_PORT = 993 IDLE_TIMEOUT_RESPONSE = '* IDLE TIMEOUT\r\n' IDLE_TIMEOUT = 60*29 # Don't stay in IDLE state longer READ_POLL_TIMEOUT = 30 # Without this timeout interrupted network connections can hang reader -READ_SIZE = 32768 # Consume all available in socket +READ_SIZE = 32768 # Consume all available in socket DFLT_DEBUG_BUF_LVL = 3 # Level above which the logging output goes directly to stderr @@ -88,7 +89,7 @@ Commands = { 'GETANNOTATION':((AUTH, SELECTED), True), 'GETQUOTA': ((AUTH, SELECTED), True), 'GETQUOTAROOT': ((AUTH, SELECTED), True), - 'ID': ((NONAUTH, AUTH, SELECTED), True), + 'ID': ((NONAUTH, AUTH, LOGOUT, SELECTED), True), 'IDLE': ((SELECTED,), False), 'LIST': ((AUTH, SELECTED), True), 'LOGIN': ((NONAUTH,), False), @@ -137,11 +138,14 @@ class Request(object): """Private class to represent a request awaiting response.""" - def __init__(self, parent, name=None, callback=None, cb_arg=None): + def __init__(self, parent, name=None, callback=None, cb_arg=None, cb_self=False): self.parent = parent self.name = name - self.callback = callback # Function called to process result - self.callback_arg = cb_arg # Optional arg passed to "callback" + self.callback = callback # Function called to process result + if not cb_self: + self.callback_arg = cb_arg # Optional arg passed to "callback" + else: + self.callback_arg = (self, cb_arg) # Self reference required in callback arg self.tag = '%s%s' % (parent.tagpre, parent.tagnum) parent.tagnum += 1 @@ -153,9 +157,6 @@ class Request(object): def abort(self, typ, val): - """Called whenever we abort a command - - Sets self.aborted reason, and deliver()s nothing""" self.aborted = (typ, val) self.deliver(None) @@ -238,12 +239,17 @@ class IMAP4(object): All (non-callback) arguments to commands are converted to strings, except for AUTHENTICATE, and the last argument to APPEND which is passed as an IMAP4 literal. If necessary (the string contains any - non-printing characters or white-space and isn't enclosed with either - parentheses or double quotes) each string is quoted. However, the - 'password' argument to the LOGIN command is always quoted. If you - want to avoid having an argument string quoted (eg: the 'flags' - argument to STORE) then enclose the string in parentheses (eg: - "(\Deleted)"). + non-printing characters or white-space and isn't enclosed with + either parentheses or double or single quotes) each string is + quoted. However, the 'password' argument to the LOGIN command is + always quoted. If you want to avoid having an argument string + quoted (eg: the 'flags' argument to STORE) then enclose the string + in parentheses (eg: "(\Deleted)"). If you are using "sequence sets" + containing the wildcard character '*', then enclose the argument + in single quotes: the quotes will be removed and the resulting + string passed unquoted. Note also that you can pass in an argument + with a type that doesn't evaluate to 'basestring' (eg: 'bytearray') + and it will be converted to a string without quoting. There is one instance variable, 'state', that is useful for tracking whether the client needs to login to the server. If it has the @@ -275,6 +281,7 @@ class IMAP4(object): # so match not the inverse set mustquote_cre = re.compile(r"[^!#$&'+,./0-9:;<=>?@A-Z\[^_`a-z|}~-]") response_code_cre = re.compile(r'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') + # sequence_set_cre = re.compile(r"^[0-9]+(:([0-9]+|\*))?(,[0-9]+(:([0-9]+|\*))?)*$") untagged_response_cre = re.compile(r'\* (?P[A-Z-]+)( (?P.*))?') untagged_status_cre = re.compile(r'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?') @@ -339,8 +346,6 @@ class IMAP4(object): self.state_change_free = threading.Event() self.state_change_pending = threading.Lock() self.commands_lock = threading.Lock() - """commands_lock prevents self.untagged_responses to be - manipulated concurrently""" self.idle_lock = threading.Lock() self.ouq = Queue.Queue(10) @@ -368,7 +373,7 @@ class IMAP4(object): elif self._get_untagged_response('OK'): if __debug__: self._log(1, 'state => NONAUTH') else: - raise self.error(self.welcome) + raise self.error('unrecognised server welcome message: %s' % `self.welcome`) typ, dat = self.capability() if dat == [None]: @@ -443,6 +448,35 @@ class IMAP4(object): return s + def ssl_wrap_socket(self): + + # Allow sending of keep-alive messages - seems to prevent some servers + # from closing SSL, leading to deadlocks. + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + try: + import ssl + if self.ca_certs is not None: + cert_reqs = ssl.CERT_REQUIRED + else: + cert_reqs = ssl.CERT_NONE + self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs) + ssl_exc = ssl.SSLError + except ImportError: + # No ssl module, and socket.ssl does not allow certificate verification + if self.ca_certs is not None: + raise socket.sslerror("SSL CA certificates cannot be checked without ssl module") + self.sock = socket.ssl(self.sock, self.keyfile, self.certfile) + ssl_exc = socket.sslerror + + if self.cert_verify_cb is not None: + cert_err = self.cert_verify_cb(self.sock.getpeercert(), self.host) + if cert_err: + raise ssl_exc(cert_err) + + self.read_fd = self.sock.fileno() + + def start_compressing(self): """start_compressing() Enable deflate compression on the socket (RFC 4978).""" @@ -671,7 +705,7 @@ class IMAP4(object): def examine(self, mailbox='INBOX', **kw): - """(typ, [data]) = examine(mailbox='INBOX', readonly=False) + """(typ, [data]) = examine(mailbox='INBOX') Select a mailbox for READ-ONLY access. (Flushes all untagged responses.) 'data' is count of messages in mailbox ('EXISTS' response). Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so @@ -745,13 +779,23 @@ class IMAP4(object): def id(self, *kv_pairs, **kw): """(typ, [data]) = .id(kv_pairs) - 'data' is list of ID key value pairs. - Request information for problem analysis and determination. + 'kv_pairs' is a possibly empty list of keys and values. + 'data' is a list of ID key value pairs or NIL. + NB: a single argument is assumed to be correctly formatted and is passed through unchanged + (for backward compatibility with earlier version). + Exchange information for problem analysis and determination. The ID extension is defined in RFC 2971. """ name = 'ID' kw['untagged_response'] = name - return self._simple_command(name, *kv_pairs, **kw) + + if not kv_pairs: + data = 'NIL' + elif len(kv_pairs) == 1: + data = kv_pairs[0] # Assume invoker passing correctly formatted string (back-compat) + else: + data = '(%s)' % ' '.join([(arg and self._quote(arg) or 'NIL') for arg in kv_pairs]) + return self._simple_command(name, (data,), **kw) def idle(self, timeout=None, **kw): @@ -999,8 +1043,8 @@ class IMAP4(object): return self._simple_command(name, sort_criteria, charset, *search_criteria, **kw) - def starttls(self, keyfile=None, certfile=None, **kw): - """(typ, [data]) = starttls(keyfile=None, certfile=None) + def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, **kw): + """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None) Start TLS negotiation as per RFC 2595.""" name = 'STARTTLS' @@ -1013,7 +1057,7 @@ class IMAP4(object): # Must now shutdown reader thread after next response, and restart after changing read_fd - self.read_size = 1 # Don't consume TLS handshake + self.read_size = 1 # Don't consume TLS handshake self.TerminateReader = True try: @@ -1031,14 +1075,13 @@ class IMAP4(object): self.rdth.start() raise self.error("Couldn't establish TLS session: %s" % dat) - try: - try: - import ssl - self.sock = ssl.wrap_socket(self.sock, keyfile, certfile) - except ImportError: - self.sock = socket.ssl(self.sock, keyfile, certfile) + self.keyfile = keyfile + self.certfile = certfile + self.ca_certs = ca_certs + self.cert_verify_cb = cert_verify_cb - self.read_fd = self.sock.fileno() + try: + self.ssl_wrap_socket() finally: # Restart reader thread self.rdth = threading.Thread(target=self._reader) @@ -1140,29 +1183,34 @@ class IMAP4(object): def _append_untagged(self, typ, dat): - """Append new untagged response - Append new 'dat' to end of last untagged response if same 'typ', - else append new response.""" + # Append new 'dat' to end of last untagged response if same 'typ', + # else append new response. + if dat is None: dat = '' - ur_data = [] - self.commands_lock.acquire() # protect untagged_responses + self.commands_lock.acquire() - if self.untagged_responses and self.untagged_responses[-1][0] == typ: - # last respons is of type 'typ', get ur_data for appending - ur_data = self.untagged_responses[-1][1] + if self.untagged_responses: + urn, urd = self.untagged_responses[-1] + if urn != typ: + urd = None else: - # need to create new untagged response of this type - self.untagged_responses.append([typ, ur_data]) + urd = None + + if urd is None: + urd = [] + self.untagged_responses.append([typ, urd]) + + urd.append(dat) - ur_data.append(dat) self.commands_lock.release() - if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%s"]' % (typ, len(ur_data)-1, dat)) + + if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%s"]' % (typ, len(urd)-1, dat)) def _check_bye(self): - """raise Exception if untagged responses contains a 'BYE'""" + bye = self._get_untagged_response('BYE', leave=True) if bye: raise self.abort(bye[-1]) @@ -1171,12 +1219,14 @@ class IMAP4(object): def _checkquote(self, arg): # Must quote command args if "atom-specials" present, - # and not already quoted. + # and not already quoted. NB: single quotes are removed. if not isinstance(arg, basestring): return arg if len(arg) >= 2 and (arg[0],arg[-1]) in (('(',')'),('"','"')): return arg + if len(arg) >= 2 and (arg[0],arg[-1]) in (("'","'"),): + return arg[1:-1] if arg and self.mustquote_cre.search(arg) is None: return arg return self._quote(arg) @@ -1372,11 +1422,7 @@ class IMAP4(object): def _get_untagged_response(self, name, leave=False): - """Return an untagged response of type 'name' - :param leave: If leave (default: False) is True, we keep the - fetched responsem; otherwise it will be deleted. Returns - None if no such response found.""" self.commands_lock.acquire() for i, (typ, dat) in enumerate(self.untagged_responses): @@ -1543,24 +1589,13 @@ class IMAP4(object): def _simple_command(self, name, *args, **kw): if 'callback' in kw: - rqb = self._command(name, callback=self._command_completer, *args) - rqb.callback_arg = (rqb, kw) + self._command(name, *args, callback=self._command_completer, cb_arg=kw, cb_self=True) return (None, None) return self._command_complete(self._command(name, *args), kw) def _untagged_response(self, typ, dat, name): - """Returns an untagged response for 'name' of type 'typ' - :param typ: 'OK, 'NO', etc... which will be used for the type of - the response. - :param dat: The fallback data to be used in case `typ` is - 'NO'. Otherwise the data from the existing untagged - responses will be searched for data to be returned. If there - is no such response, we return `[None]` as data. - :param name: The name of the response. - :returns: (typ, data) - """ if typ == 'NO': return typ, dat data = self._get_untagged_response(name) @@ -1936,22 +1971,27 @@ class IMAP4_SSL(IMAP4): Instantiate with: IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, debug=None, debug_file=None, identifier=None, timeout=None) - host - host's name (default: localhost); - port - port number (default: standard IMAP4 SSL port); - keyfile - PEM formatted file that contains your private key (default: None); - certfile - PEM formatted certificate chain file (default: None); - debug - debug level (default: 0 - no debug); - debug_file - debug stream (default: sys.stderr); - identifier - thread identifier prefix (default: host); - timeout - timeout in seconds when expecting a command response. + host - host's name (default: localhost); + port - port number (default: standard IMAP4 SSL port); + keyfile - PEM formatted file that contains your private key (default: None); + certfile - PEM formatted certificate chain file (default: None); + ca_certs - PEM formatted certificate chain file used to validate server certificates (default: None); + cert_verify_cb - function to verify authenticity of server certificates (default: None); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response. + debug_buf_lvl - debug level at which buffering is turned off. For more documentation see the docstring of the parent class IMAP4. """ - def __init__(self, host=None, port=None, keyfile=None, certfile=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): + def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): self.keyfile = keyfile self.certfile = certfile + self.ca_certs = ca_certs + self.cert_verify_cb = cert_verify_cb IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout, debug_buf_lvl) @@ -1965,14 +2005,7 @@ class IMAP4_SSL(IMAP4): self.host = self._choose_nonull_or_dflt('', host) self.port = self._choose_nonull_or_dflt(IMAP4_SSL_PORT, port) self.sock = self.open_socket() - - try: - import ssl - self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile) - except ImportError: - self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile) - - self.read_fd = self.sock.fileno() + self.ssl_wrap_socket() def read(self, size): @@ -1980,12 +2013,12 @@ class IMAP4_SSL(IMAP4): Read at most 'size' bytes from remote.""" if self.decompressor is None: - return self.sslobj.read(size) + return self.sock.read(size) if self.decompressor.unconsumed_tail: data = self.decompressor.unconsumed_tail else: - data = self.sslobj.read(8192) + data = self.sock.read(8192) return self.decompressor.decompress(data, size) @@ -1998,21 +2031,23 @@ class IMAP4_SSL(IMAP4): data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) - # NB: socket.ssl needs a "sendall" method to match socket objects. - bytes = len(data) - while bytes > 0: - sent = self.sslobj.write(data) - if sent == bytes: - break # avoid copy - data = data[sent:] - bytes = bytes - sent + if hasattr(self.sock, "sendall"): + self.sock.sendall(data) + else: + bytes = len(data) + while bytes > 0: + sent = self.sock.write(data) + if sent == bytes: + break # avoid copy + data = data[sent:] + bytes = bytes - sent def ssl(self): """ssl = ssl() Return socket.ssl instance used to communicate with the IMAP4 server.""" - return self.sslobj + return self.sock @@ -2021,13 +2056,14 @@ class IMAP4_stream(IMAP4): """IMAP4 client class over a stream Instantiate with: - IMAP4_stream(command, debug=None, debug_file=None, identifier=None, timeout=None) + IMAP4_stream(command, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None) - command - string that can be passed to subprocess.Popen(); - debug - debug level (default: 0 - no debug); - debug_file - debug stream (default: sys.stderr); - identifier - thread identifier prefix (default: host); - timeout - timeout in seconds when expecting a command response. + command - string that can be passed to subprocess.Popen(); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response. + debug_buf_lvl - debug level at which buffering is turned off. For more documentation see the docstring of the parent class IMAP4. """ @@ -2296,7 +2332,7 @@ if __name__ == '__main__': ('list', ('/tmp', 'imaplib2_test*')), ('select', ('/tmp/imaplib2_test.2',)), ('search', (None, 'SUBJECT', 'IMAP4 test')), - ('fetch', ('1', '(FLAGS INTERNALDATE RFC822)')), + ('fetch', ("'1:*'", '(FLAGS INTERNALDATE RFC822)')), ('store', ('1', 'FLAGS', '(\Deleted)')), ('namespace', ()), ('expunge', ()), @@ -2380,6 +2416,11 @@ if __name__ == '__main__': else: path = ml.split()[-1] run('delete', (path,)) + if 'ID' in M.capabilities: + run('id', ()) + run('id', ('("name", "imaplib2")',)) + run('id', ("version", __version__, "os", os.uname()[0])) + for cmd,args in test_seq2: if (cmd,args) != ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')): run(cmd, args) From 3a91e296f0138b2acb62b2abe37c73073decb3c5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 15 Aug 2011 11:55:42 +0200 Subject: [PATCH 184/817] Adapt the code to work with the new imaplib2 imaplib renamed self.sslobj to self.sock and our overriden open() functions were failing for that reason when updating imaplib2 to v2.28. It turns out that all of our custom initializations are being done by stock imaplib2 now anyway, so there is no need to override them anymore. This lets us simplify the code we have to worry about. Move the verifycert() function to the imapserver.py file, it is now a callback function that is being handed to imaplib from there, so it makes sense to also define it in our imapserver function... (this also lets us easily make use of the verifycert function in the starttls case in the future) TODO: we need to examine if and why we still need to override the select() function, it is the only reason why we still wrap the IMAP4 classes. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 + offlineimap/imaplibutil.py | 156 ++----------------------------------- offlineimap/imapserver.py | 51 +++++++++++- 3 files changed, 55 insertions(+), 154 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 6b5b18e..3b2243e 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -23,6 +23,8 @@ Changes * Refactor our IMAPServer class. Background work without user-visible changes. +* Updated bundled imaplib2 to version 2.28 + Bug Fixes --------- diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index c56b634..fa3a303 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -127,162 +127,16 @@ def new_mesg(self, s, tn=None, secs=None): tm = time.strftime('%M:%S', time.localtime(secs)) getglobalui().debug('imap', ' %s.%02d %s %s' % (tm, (secs*100)%100, tn, s)) + class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): - """Provides an improved version of the standard IMAP4_SSL + """Improved version of imaplib.IMAP4_SSL overriding select()""" + pass - It provides a better readline() implementation as impaplib's - readline() is extremly inefficient. It can also connect to IPv6 - addresses.""" - def __init__(self, *args, **kwargs): - self._readbuf = '' - self._cacertfile = kwargs.get('cacertfile', None) - if kwargs.has_key('cacertfile'): - del kwargs['cacertfile'] - IMAP4_SSL.__init__(self, *args, **kwargs) - - def open(self, host=None, port=None): - """Do whatever IMAP4_SSL would do in open, but call sslwrap - with cert verification""" - #IMAP4_SSL.open(self, host, port) uses the below 2 lines: - self.host = host - self.port = port - - #rather than just self.sock = socket.create_connection((host, port)) - #we use the below part to be able to connect to ipv6 addresses too - #This connects to the first ip found ipv4/ipv6 - #Added by Adriaan Peeters based on a socket - #example from the python documentation: - #http://www.python.org/doc/lib/socket-example.html - res = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM) - # Try all the addresses in turn until we connect() - last_error = 0 - for remote in res: - af, socktype, proto, canonname, sa = remote - self.sock = socket.socket(af, socktype, proto) - last_error = self.sock.connect_ex(sa) - if last_error == 0: - break - else: - self.sock.close() - if last_error != 0: - raise Exception("can't open socket; error: %s"\ - % socket.error(last_error)) - - # Allow sending of keep-alive message seems to prevent some servers - # from closing SSL on us leading to deadlocks - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - - #connected to socket, now wrap it in SSL - try: - if self._cacertfile: - requirecert = ssl.CERT_REQUIRED - else: - requirecert = ssl.CERT_NONE - - self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, - self.certfile, - ca_certs = self._cacertfile, - cert_reqs = requirecert) - except NameError: - #Python 2.4/2.5 don't have the ssl module, we need to - #socket.ssl() here but that doesn't allow cert - #verification!!! - if self._cacertfile: - #user configured a CA certificate, but python 2.4/5 doesn't - #allow us to easily check it. So bail out here. - raise Exception("SSL CA Certificates cannot be checked with python <=2.6. Abort") - self.sslobj = socket.ssl(self.sock, self.keyfile, - self.certfile) - - else: - #ssl.wrap_socket worked and cert is verified (if configured), - #now check that hostnames also match if we have a CA cert. - if self._cacertfile: - error = self._verifycert(self.sslobj.getpeercert(), host) - if error: - raise ssl.SSLError("SSL Certificate host name mismatch: %s" % error) - - # imaplib2 uses this to poll() - self.read_fd = self.sock.fileno() - - #TODO: Done for now. We should implement a mutt-like behavior - #that offers the users to accept a certificate (presenting a - #fingerprint of it) (get via self.sslobj.getpeercert()), and - #save that, and compare on future connects, rather than having - #to trust what the CA certs say. - - def _verifycert(self, cert, hostname): - '''Verify that cert (in socket.getpeercert() format) matches hostname. - CRLs are not handled. - - Returns error message if any problems are found and None on success. - ''' - if not cert: - return ('no certificate received') - dnsname = hostname.lower() - certnames = [] - - # cert expired? - notafter = cert.get('notAfter') - if notafter: - if time.time() >= ssl.cert_time_to_seconds(notafter): - return ('server certificate error: certificate expired %s' - ) % notafter - - # First read commonName - for s in cert.get('subject', []): - key, value = s[0] - if key == 'commonName': - certnames.append(value.lower()) - if len(certnames) == 0: - return ('no commonName found in certificate') - - # Then read subjectAltName - for key, value in cert.get('subjectAltName', []): - if key == 'DNS': - certnames.append(value.lower()) - - # And finally try to match hostname with one of these names - for certname in certnames: - if (certname == dnsname or - '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): - return None - - return ('no matching domain name found in certificate') class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): - """Improved version of imaplib.IMAP4 that can also connect to IPv6""" + """Improved version of imaplib.IMAP4 overriding select()""" + pass - def open(self, host = '', port = IMAP4_PORT): - """Setup connection to remote server on "host:port" - (default: localhost:standard IMAP4 port). - """ - #self.host and self.port are needed by the parent IMAP4 class - self.host = host - self.port = port - res = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM) - - # Try each address returned by getaddrinfo in turn until we - # manage to connect to one. - # Try all the addresses in turn until we connect() - last_error = 0 - for remote in res: - af, socktype, proto, canonname, sa = remote - self.sock = socket.socket(af, socktype, proto) - last_error = self.sock.connect_ex(sa) - if last_error == 0: - break - else: - self.sock.close() - if last_error != 0: - raise Exception("can't open socket; error: %s"\ - % socket.error(last_error)) - self.file = self.sock.makefile('rb') - - # imaplib2 uses this to poll() - self.read_fd = self.sock.fileno() def Internaldate2epoch(resp): """Convert IMAP4 INTERNALDATE to UT. diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 28511bb..ccd94ee 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -24,10 +24,11 @@ import offlineimap.accounts import hmac import socket import base64 +import time from socket import gaierror try: - from ssl import SSLError + from ssl import SSLError, cert_time_to_seconds except ImportError: # Protect against python<2.6, use dummy and won't get SSL errors. SSLError = None @@ -204,9 +205,12 @@ class IMAPServer: self.ui.connecting(self.hostname, self.port) imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, self.port, - self.sslclientkey, self.sslclientcert, + self.sslclientkey, + self.sslclientcert, + self.sslcacertfile, + self.verifycert, timeout=socket.getdefaulttimeout(), - cacertfile = self.sslcacertfile) + ) else: self.ui.connecting(self.hostname, self.port) imapobj = imaplibutil.WrappedIMAP4(self.hostname, self.port, @@ -403,6 +407,47 @@ class IMAPServer: self.ui.debug('imap', 'keepalive: bottom of loop') + + def verifycert(self, cert, hostname): + '''Verify that cert (in socket.getpeercert() format) matches hostname. + CRLs are not handled. + + Returns error message if any problems are found and None on success. + ''' + errstr = "CA Cert verifying failed: " + if not cert: + return ('%s no certificate received' % errstr) + dnsname = hostname.lower() + certnames = [] + + # cert expired? + notafter = cert.get('notAfter') + if notafter: + if time.time() >= cert_time_to_seconds(notafter): + return '%s certificate expired %s' % (errstr, notafter) + + # First read commonName + for s in cert.get('subject', []): + key, value = s[0] + if key == 'commonName': + certnames.append(value.lower()) + if len(certnames) == 0: + return ('%s no commonName found in certificate' % errstr) + + # Then read subjectAltName + for key, value in cert.get('subjectAltName', []): + if key == 'DNS': + certnames.append(value.lower()) + + # And finally try to match hostname with one of these names + for certname in certnames: + if (certname == dnsname or + '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): + return None + + return ('%s no matching domain name found in certificate' % errstr) + + class IdleThread(object): def __init__(self, parent, folder=None): self.parent = parent From 4a0d17b566509aac293446161a6c1a667cd9c2c5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 15 Aug 2011 13:15:36 +0200 Subject: [PATCH 185/817] Import errno we need errno.CONNREFUSED, but through some merging mishaps(?) the part that actually imported errno was missing. Import the errno module. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 28511bb..6534f13 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -24,6 +24,7 @@ import offlineimap.accounts import hmac import socket import base64 +import errno from socket import gaierror try: From 8d858a0b7848a9a79a05b7e090b98033b4dc7923 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 16 Aug 2011 10:48:37 +0200 Subject: [PATCH 186/817] Disable certificate verification if there is no certificate imaplib2 always attempts to verify a certificate if a verification callback function is passed in, even the certificate is None specified. Disable the verification excplictly by setting the verification function to None in that case. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index c1763ce..707464d 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -67,6 +67,8 @@ class IMAPServer: self.sslclientcert = repos.getsslclientcert() self.sslclientkey = repos.getsslclientkey() self.sslcacertfile = repos.getsslcacertfile() + if self.sslcacertfile is None: + self.verifycert = None # disable cert verification self.delim = None self.root = None self.maxconnections = repos.getmaxconnections() From 38b1d7b08515106c7dcd85ee0d5ae8ede5985b49 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Tue, 16 Aug 2011 10:55:13 +0200 Subject: [PATCH 187/817] Replaced tabs with spaces to unify python sources Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/UIDMaps.py | 2 +- offlineimap/repository/Base.py | 12 +++++------ offlineimap/repository/Gmail.py | 2 +- offlineimap/repository/Maildir.py | 34 +++++++++++++++---------------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 9233662..fa1924d 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -34,7 +34,7 @@ class MappedIMAPFolder(IMAPFolder): diskl2r: dict mapping message uids: self.r2l[localuid]=remoteuid""" def __init__(self, *args, **kwargs): - IMAPFolder.__init__(self, *args, **kwargs) + IMAPFolder.__init__(self, *args, **kwargs) self.maplock = Lock() (self.diskr2l, self.diskl2r) = self._loadmaps() self._mb = IMAPFolder(*args, **kwargs) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 34f2f4a..ae4d32c 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -42,14 +42,14 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): # The 'restoreatime' config parameter only applies to local Maildir # mailboxes. def restore_atime(self): - if self.config.get('Repository ' + self.name, 'type').strip() != \ - 'Maildir': - return + if self.config.get('Repository ' + self.name, 'type').strip() != \ + 'Maildir': + return - if not self.config.has_option('Repository ' + self.name, 'restoreatime') or not self.config.getboolean('Repository ' + self.name, 'restoreatime'): - return + if not self.config.has_option('Repository ' + self.name, 'restoreatime') or not self.config.getboolean('Repository ' + self.name, 'restoreatime'): + return - return self.restore_folder_atimes() + return self.restore_folder_atimes() def connect(self): """Establish a connection to the remote, if necessary. This exists diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index fd95361..20ed1f1 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -73,7 +73,7 @@ class GmailRepository(IMAPRepository): def gettrashfolder(self, foldername): #: Where deleted mail should be moved return self.getconf('trashfolder','[Gmail]/Trash') - + def getspamfolder(self): #: Gmail also deletes messages upon EXPUNGE in the Spam folder return self.getconf('spamfolder','[Gmail]/Spam') diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 069748a..ef3a723 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -32,28 +32,28 @@ class MaildirRepository(BaseRepository): self.folders = None self.ui = getglobalui() self.debug("MaildirRepository initialized, sep is " + repr(self.getsep())) - self.folder_atimes = [] + self.folder_atimes = [] # Create the top-level folder if it doesn't exist if not os.path.isdir(self.root): os.mkdir(self.root, 0700) def _append_folder_atimes(self, foldername): - p = os.path.join(self.root, foldername) - new = os.path.join(p, 'new') - cur = os.path.join(p, 'cur') - f = p, os.stat(new)[ST_ATIME], os.stat(cur)[ST_ATIME] - self.folder_atimes.append(f) + p = os.path.join(self.root, foldername) + new = os.path.join(p, 'new') + cur = os.path.join(p, 'cur') + f = p, os.stat(new)[ST_ATIME], os.stat(cur)[ST_ATIME] + self.folder_atimes.append(f) def restore_folder_atimes(self): - if not self.folder_atimes: - return + if not self.folder_atimes: + return - for f in self.folder_atimes: - t = f[1], os.stat(os.path.join(f[0], 'new'))[ST_MTIME] - os.utime(os.path.join(f[0], 'new'), t) - t = f[2], os.stat(os.path.join(f[0], 'cur'))[ST_MTIME] - os.utime(os.path.join(f[0], 'cur'), t) + for f in self.folder_atimes: + t = f[1], os.stat(os.path.join(f[0], 'new'))[ST_MTIME] + os.utime(os.path.join(f[0], 'new'), t) + t = f[2], os.stat(os.path.join(f[0], 'cur'))[ST_MTIME] + os.utime(os.path.join(f[0], 'cur'), t) def getlocalroot(self): return os.path.expanduser(self.getconf('localfolders')) @@ -110,8 +110,8 @@ class MaildirRepository(BaseRepository): self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername) def getfolder(self, foldername): - if self.config.has_option('Repository ' + self.name, 'restoreatime') and self.config.getboolean('Repository ' + self.name, 'restoreatime'): - self._append_folder_atimes(foldername) + if self.config.has_option('Repository ' + self.name, 'restoreatime') and self.config.getboolean('Repository ' + self.name, 'restoreatime'): + self._append_folder_atimes(foldername) return folder.Maildir.MaildirFolder(self.root, foldername, self.getsep(), self, self.accountname, self.config) @@ -155,11 +155,11 @@ class MaildirRepository(BaseRepository): # This directory has maildir stuff -- process self.debug(" This is maildir folder '%s'." % foldername) - if self.config.has_option('Repository %s' % self, + if self.config.has_option('Repository %s' % self, 'restoreatime') and \ self.config.getboolean('Repository %s' % self, 'restoreatime'): - self._append_folder_atimes(foldername) + self._append_folder_atimes(foldername) retval.append(folder.Maildir.MaildirFolder(self.root, foldername, self.getsep(), From 6f9b171ffd3bb038401d0101e3f65d677e8e80b2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 15 Aug 2011 09:58:46 +0200 Subject: [PATCH 188/817] Don't pass a list to ui.copyingmessage() We only copy to a single folder anyway, so clean up the code to only pass in a single folder. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 3 ++- offlineimap/ui/Blinkenlights.py | 4 ++-- offlineimap/ui/Machine.py | 8 ++++---- offlineimap/ui/UIBase.py | 11 ++++++----- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 3da0de1..ab8f998 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -244,7 +244,7 @@ class BaseFolder(object): statusfolder.savemessage(uid, None, flags, rtime) return - self.ui.copyingmessage(uid, self, [dstfolder]) + self.ui.copyingmessage(uid, self, dstfolder) # If any of the destinations actually stores the message body, # load it up. if dstfolder.storesmessages(): @@ -254,6 +254,7 @@ class BaseFolder(object): #UID. If newid is 0, saving succeeded, but we could not #retrieve the new UID. Ignore message in this case. newuid = dstfolder.savemessage(uid, message, flags, rtime) + if newuid > 0: if newuid != uid: # Got new UID, change the local uid. diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py index 2160100..f0dfec1 100644 --- a/offlineimap/ui/Blinkenlights.py +++ b/offlineimap/ui/Blinkenlights.py @@ -54,9 +54,9 @@ class BlinkenBase: s.gettf().setcolor('blue') s.__class__.__bases__[-1].syncingmessages(s, sr, sf, dr, df) - def copyingmessage(s, uid, src, destlist): + def copyingmessage(s, uid, src, destfolder): s.gettf().setcolor('orange') - s.__class__.__bases__[-1].copyingmessage(s, uid, src, destlist) + s.__class__.__bases__[-1].copyingmessage(s, uid, src, destfolder) def deletingmessages(s, uidlist, destlist): s.gettf().setcolor('red') diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 37b5e9b..bb8c7fa 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -108,10 +108,10 @@ class MachineUI(UIBase): (s.getnicename(sr), sf.getname(), s.getnicename(dr), df.getname())) - def copyingmessage(s, uid, src, destlist): - ds = s.folderlist(destlist) - s._printData('copyingmessage', "%d\n%s\n%s\n%s" % \ - (uid, s.getnicename(src), src.getname(), ds)) + def copyingmessage(s, uid, src, destfolder): + s._printData('copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ + (uid, s.getnicename(src), src.getname(), + destfolder.getnicename(), destfolder)) def folderlist(s, list): return ("\f".join(["%s\t%s" % (s.getnicename(x), x.getname()) for x in list])) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index c08d34c..16e10bc 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -285,11 +285,12 @@ class UIBase: s.getnicename(dr), df.getname())) - def copyingmessage(s, uid, src, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Copy message %d %s[%s] -> %s" % (uid, s.getnicename(src), - src.getname(), ds)) + def copyingmessage(self, uid, src, destfolder): + """Output a log line stating which message we copy""" + if self.verbose >= 0: + self._msg("Copy message %d %s[%s] -> %s[%s]" % \ + (uid, self.getnicename(src), src, + self.getnicename(destfolder), destfolder)) def deletingmessage(s, uid, destlist): if s.verbose >= 0: From b6ac1aecb1684b664028813f86cd309d72d2a380 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Tue, 16 Aug 2011 10:55:14 +0200 Subject: [PATCH 189/817] Another way of locating UID of just saved message It works by fetching all headers of new messages from IMAP server and searching for our X-OfflineIMAP marker by using regular expression. Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 75 +++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 7b11238..20940a2 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -318,6 +318,74 @@ class IMAPFolder(BaseFolder): matchinguids.sort() return long(matchinguids[0]) + def savemessage_fetchheaders(self, imapobj, headername, headervalue): + """ We fetch all new mail headers and search for the right + X-OfflineImap line by hand. The response from the server has form: + ( + 'OK', + [ + ( + '185 (RFC822.HEADER {1789}', + '... mail headers ...' + ), + ' UID 2444)', + ( + '186 (RFC822.HEADER {1789}', + '... 2nd mail headers ...' + ), + ' UID 2445)' + ] + ) + We need to locate the UID just after mail headers containing our + X-OfflineIMAP line. + + Returns UID when found, 0 when not found. + """ + self.ui.debug('imap', 'savemessage_fetchheaders called for %s: %s' % \ + (headername, headervalue)) + + # run "fetch X:* rfc822.header" + # since we stored the mail we are looking for just recently, it would + # not be optimal to fetch all messages. So we'll find highest message + # UID in our local messagelist and search from there (exactly from + # UID+1). That works because UIDs are guaranteed to be unique and + # ascending. + + if self.getmessagelist(): + start = 1+max(self.getmessagelist().keys()) + else: + # Folder was empty - start from 1 + start = 1 + + # Imaplib quotes all parameters of a string type. That must not happen + # with the range X:*. So we use bytearray to stop imaplib from getting + # in our way + + result = imapobj.uid('FETCH', bytearray('%d:*' % start), 'rfc822.header') + if result[0] != 'OK': + raise OfflineImapError('Error fetching mail headers: ' + '. '.join(result[1]), + OfflineImapError.ERROR.MESSAGE) + + result = result[1] + + found = 0 + for item in result: + if found == 0 and type(item) == type( () ): + # Walk just tuples + if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)" % (headername, headervalue), + item[1], flags=re.IGNORECASE): + found = 1 + elif found == 1: + if type(item) == type (""): + uid = re.search("UID\s+(\d+)", item, flags=re.IGNORECASE) + if uid: + return int(uid.group(1)) + else: + self.ui.warn("Can't parse FETCH response, can't find UID: %s", result.__repr__()) + else: + self.ui.warn("Can't parse FETCH response, we awaited string: %s", result.__repr__()) + + return 0 def getmessageinternaldate(self, content, rtime=None): """Parses mail and returns an INTERNALDATE string @@ -491,10 +559,15 @@ class IMAPFolder(BaseFolder): headervalue) # See docs for savemessage in Base.py for explanation of this and other return values if uid == 0: - self.ui.debug('imap', 'savemessage: first attempt to get new UID failed. Going to run a NOOP and try again.') + self.ui.debug('imap', 'savemessage: first attempt to get new UID failed. \ + Going to run a NOOP and try again.') assert(imapobj.noop()[0] == 'OK') uid = self.savemessage_searchforheader(imapobj, headername, headervalue) + if uid == 0: + self.ui.debug('imap', 'savemessage: second attempt to get new UID failed. \ + Going to try search headers manually') + uid = self.savemessage_fetchheaders(imapobj, headername, headervalue) finally: self.imapserver.releaseconnection(imapobj) From c2fc81dd3d080adf666bc80f7fc741502b3eba3b Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Tue, 16 Aug 2011 10:55:16 +0200 Subject: [PATCH 190/817] Make syncfoldersto to accept a single folder only It is just historic relict Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 2 +- offlineimap/repository/Base.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 8653d48..31ea2b7 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -233,7 +233,7 @@ class SyncableAccount(Account): # replicate the folderstructure from REMOTE to LOCAL if not localrepos.getconf('readonly', False): self.ui.syncfolders(remoterepos, localrepos) - remoterepos.syncfoldersto(localrepos, [statusrepos]) + remoterepos.syncfoldersto(localrepos, statusrepos) # iterate through all folders on the remote repo and sync for remotefolder in remoterepos.getfolders(): diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index ae4d32c..f96a33d 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -114,12 +114,12 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): def getfolder(self, foldername): raise NotImplementedError - def syncfoldersto(self, dest, copyfolders): + def syncfoldersto(self, dest, status): """Syncs the folders in this repository to those in dest. It does NOT sync the contents of those folders. - For every time dest.makefolder() is called, also call makefolder() - on each folder in copyfolders.""" + Whenever makefolder() is called, also call makefolder() on status + folder.""" src = self srcfolders = src.getfolders() destfolders = dest.getfolders() @@ -143,8 +143,7 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): if not key in desthash: try: dest.makefolder(key) - for copyfolder in copyfolders: - copyfolder.makefolder(key.replace(dest.getsep(), copyfolder.getsep())) + status.makefolder(key.replace(dest.getsep(), status.getsep())) except (KeyboardInterrupt): raise except: From 4f1cd05fdc8f9abccacbdd5c6a911fb574a8bcce Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 18 Aug 2011 09:05:45 +0200 Subject: [PATCH 191/817] Fix copyingmessage() in Machine UI I changed the API to pass in a folder rather than a list of folders, but used getnicename() on the wrong object. It is not used on the folder but on the ui object. Fix this and give the variable somewhat better names. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Machine.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index bb8c7fa..7a480ae 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -108,10 +108,10 @@ class MachineUI(UIBase): (s.getnicename(sr), sf.getname(), s.getnicename(dr), df.getname())) - def copyingmessage(s, uid, src, destfolder): - s._printData('copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ - (uid, s.getnicename(src), src.getname(), - destfolder.getnicename(), destfolder)) + def copyingmessage(self, uid, srcfolder, destfolder): + self._printData('copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ + (uid, self.getnicename(srcfolder), srcfolder.getname(), + self.getnicename(destfolder), destfolder)) def folderlist(s, list): return ("\f".join(["%s\t%s" % (s.getnicename(x), x.getname()) for x in list])) From 373e7cdbc1b7b6f1eb57365a1a63c25375cbdf7a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 17 Aug 2011 16:11:00 +0200 Subject: [PATCH 192/817] Allow empty foldernames Empty foldernames (as they could be created through nametrans) were failing as the uidvalidity and status files names as determined by folder/Base.py:getfolderbasename() lead to invalid file names ''. Fix this by handling empty file names and translating them to '.' which leads to the special file name 'dot'. (this special value existed before and was not invented by this patch) Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 14 +++++++++----- offlineimap/folder/LocalStatus.py | 2 +- offlineimap/folder/LocalStatusSQLite.py | 2 +- offlineimap/repository/LocalStatus.py | 15 +++++++++++---- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index ab8f998..626156a 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -72,11 +72,15 @@ class BaseFolder(object): return self.getname() def getfolderbasename(self): - foldername = self.getname() - foldername = foldername.replace(self.repository.getsep(), '.') - foldername = re.sub('/\.$', '/dot', foldername) - foldername = re.sub('^\.$', 'dot', foldername) - return foldername + """Return base file name of file to store Status/UID info in""" + if not self.name: + basename = '.' + else: #avoid directory hierarchies and file names such as '/' + basename = self.name.replace('/', '.') + # replace with literal 'dot' if final path name is '.' as '.' is + # an invalid file name. + basename = re.sub('(^|\/)\.$','\\1dot', basename) + return basename def isuidvalidityok(self): """Does the cached UID match the real UID diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 5113345..375777e 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -28,7 +28,7 @@ class LocalStatusFolder(BaseFolder): self.root = root self.sep = '.' self.config = config - self.filename = repository.getfolderfilename(name) + self.filename = os.path.join(root, self.getfolderbasename()) self.messagelist = {} self.repository = repository self.savelock = threading.Lock() diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index a57992b..77cdff3 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -115,7 +115,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): plaintextfilename = os.path.join( self.repository.account.getaccountmeta(), 'LocalStatus', - re.sub('(^|\/)\.$','\\1dot', self.name)) + self.getfolderbasename(self.name)) # MIGRATE from plaintext if needed if os.path.exists(plaintextfilename): self.ui._msg('Migrating LocalStatus cache from plain text ' diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 50dfdc8..a392dcf 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -50,10 +50,17 @@ class LocalStatusRepository(BaseRepository): return '.' def getfolderfilename(self, foldername): - """Return the full path of the status file""" - # replace with 'dot' if final path name is '.' - foldername = re.sub('(^|\/)\.$','\\1dot', foldername) - return os.path.join(self.directory, foldername) + """Return the full path of the status file + + This mimics the path that Folder().getfolderbasename() would return""" + if not foldername: + basename = '.' + else: #avoid directory hierarchies and file names such as '/' + basename = foldername.replace('/', '.') + # replace with literal 'dot' if final path name is '.' as '.' is + # an invalid file name. + basename = re.sub('(^|\/)\.$','\\1dot', basename) + return os.path.join(self.directory, basename) def makefolder(self, foldername): """Create a LocalStatus Folder From 466ded04d94396823bccdc56f70f51647ba61124 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 16 Aug 2011 12:16:46 +0200 Subject: [PATCH 193/817] Make flags a set rather than a list As this is essentially what it is, a set of values. This allows as to do set arithmetics to see, e.g. the intersection of 2 flag sets rather than clunkily having to do: for flag in newflags: if flag not in oldflags: oldflags.append(flag) Also some more code documenting. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 48 ++++++++++++------------- offlineimap/folder/Gmail.py | 3 +- offlineimap/folder/IMAP.py | 23 ++++++------ offlineimap/folder/LocalStatus.py | 15 ++++---- offlineimap/folder/LocalStatusSQLite.py | 8 +++-- offlineimap/folder/Maildir.py | 17 +++++---- offlineimap/imaputil.py | 39 +++++++++++++------- 7 files changed, 89 insertions(+), 64 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 626156a..6d7e093 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -22,6 +22,10 @@ import os.path import re from sys import exc_info import traceback +try: # python 2.6 has set() built in + set +except NameError: + from sets import Set as set class BaseFolder(object): def __init__(self): @@ -189,12 +193,9 @@ class BaseFolder(object): def addmessageflags(self, uid, flags): """Adds the specified flags to the message's flag set. If a given - flag is already present, it will not be duplicated.""" - newflags = self.getmessageflags(uid) - for flag in flags: - if not flag in newflags: - newflags.append(flag) - newflags.sort() + flag is already present, it will not be duplicated. + :param flags: A set() of flags""" + newflags = self.getmessageflags(uid) | flags self.savemessageflags(uid, newflags) def addmessagesflags(self, uidlist, flags): @@ -204,11 +205,7 @@ class BaseFolder(object): def deletemessageflags(self, uid, flags): """Removes each flag given from the message's flag set. If a given flag is already removed, no action will be taken for that flag.""" - newflags = self.getmessageflags(uid) - for flag in flags: - if flag in newflags: - newflags.remove(flag) - newflags.sort() + newflags = self.getmessageflags(uid) - flags self.savemessageflags(uid, newflags) def deletemessagesflags(self, uidlist, flags): @@ -362,8 +359,8 @@ class BaseFolder(object): addflaglist = {} delflaglist = {} for uid in self.getmessageuidlist(): - # Ignore messages with negative UIDs missed by pass 1 - # also don't do anything if the message has been deleted remotely + # Ignore messages with negative UIDs missed by pass 1 and + # don't do anything if the message has been deleted remotely if uid < 0 or not dstfolder.uidexists(uid): continue @@ -371,30 +368,31 @@ class BaseFolder(object): statusflags = statusfolder.getmessageflags(uid) #if we could not get message flags from LocalStatus, assume empty. if statusflags is None: - statusflags = [] - addflags = [x for x in selfflags if x not in statusflags] + statusflags = set() + + addflags = selfflags - statusflags + delflags = statusflags - selfflags for flag in addflags: if not flag in addflaglist: addflaglist[flag] = [] addflaglist[flag].append(uid) - delflags = [x for x in statusflags if x not in selfflags] for flag in delflags: if not flag in delflaglist: delflaglist[flag] = [] delflaglist[flag].append(uid) - for flag in addflaglist.keys(): - self.ui.addingflags(addflaglist[flag], flag, dstfolder) - dstfolder.addmessagesflags(addflaglist[flag], [flag]) - statusfolder.addmessagesflags(addflaglist[flag], [flag]) - - for flag in delflaglist.keys(): - self.ui.deletingflags(delflaglist[flag], flag, dstfolder) - dstfolder.deletemessagesflags(delflaglist[flag], [flag]) - statusfolder.deletemessagesflags(delflaglist[flag], [flag]) + for flag, uids in addflaglist.items(): + self.ui.addingflags(uids, flag, dstfolder) + dstfolder.addmessagesflags(uids, set(flag)) + statusfolder.addmessagesflags(uids, set(flag)) + for flag,uids in delflaglist.items(): + self.ui.deletingflags(uids, flag, dstfolder) + dstfolder.deletemessagesflags(uids, set(flag)) + statusfolder.deletemessagesflags(uids, set(flag)) + def syncmessagesto(self, dstfolder, statusfolder): """Syncs messages in this folder to the destination dstfolder. diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 7cbff4f..6cd3446 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -21,7 +21,6 @@ from IMAP import IMAPFolder from offlineimap import imaputil -from copy import copy class GmailFolder(IMAPFolder): @@ -45,7 +44,7 @@ class GmailFolder(IMAPFolder): def deletemessages_noconvert(self, uidlist): uidlist = [uid for uid in uidlist if uid in self.messagelist] if not len(uidlist): - return + return if self.realdelete and not (self.getname() in self.real_delete_folders): # IMAP expunge is just "remove label" in this folder, diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 20940a2..40eb31c 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -24,6 +24,11 @@ import time from copy import copy from Base import BaseFolder from offlineimap import imaputil, imaplibutil, OfflineImapError +try: # python 2.6 has set() built in + set +except NameError: + from sets import Set as set + class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, visiblename, accountname, repository): @@ -176,7 +181,8 @@ class IMAPFolder(BaseFolder): finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: - # Discard the message number. + # looks like: '1 (FLAGS (\\Seen Old) UID 4807)' + # Discard initial message number. messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not options.has_key('UID'): @@ -652,23 +658,18 @@ class IMAPFolder(BaseFolder): if not ('UID' in attributehash and 'FLAGS' in attributehash): # Compensate for servers that don't return a UID attribute. continue - lflags = attributehash['FLAGS'] + flagstr = attributehash['FLAGS'] uid = long(attributehash['UID']) - self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(lflags) + self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flagstr) try: needupdate.remove(uid) except ValueError: # Let it slide if it's not in the list pass for uid in needupdate: if operation == '+': - for flag in flags: - if not flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].append(flag) - self.messagelist[uid]['flags'].sort() + self.messagelist[uid]['flags'] |= flags elif operation == '-': - for flag in flags: - if flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].remove(flag) + self.messagelist[uid]['flags'] -= flags def deletemessage(self, uid): self.deletemessages_noconvert([uid]) @@ -682,7 +683,7 @@ class IMAPFolder(BaseFolder): if not len(uidlist): return - self.addmessagesflags_noconvert(uidlist, ['T']) + self.addmessagesflags_noconvert(uidlist, set('T')) imapobj = self.imapserver.acquireconnection() try: try: diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 375777e..be9d1e3 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -1,6 +1,5 @@ # Local status cache virtual folder -# Copyright (C) 2002 - 2008 John Goerzen -# +# Copyright (C) 2002 - 2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,6 +18,10 @@ from Base import BaseFolder import os import threading +try: # python 2.6 has set() built in + set +except NameError: + from sets import Set as set magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" @@ -80,11 +83,12 @@ class LocalStatusFolder(BaseFolder): try: uid, flags = line.split(':') uid = long(uid) + flags = set(flags) except ValueError, e: - errstr = "Corrupt line '%s' in cache file '%s'" % (line, self.filename) + errstr = "Corrupt line '%s' in cache file '%s'" % \ + (line, self.filename) self.ui.warn(errstr) raise ValueError(errstr) - flags = [x for x in flags] self.messagelist[uid] = {'uid': uid, 'flags': flags} file.close() @@ -95,8 +99,7 @@ class LocalStatusFolder(BaseFolder): file.write(magicline + "\n") for msg in self.messagelist.values(): flags = msg['flags'] - flags.sort() - flags = ''.join(flags) + flags = ''.join(sorted(flags)) file.write("%s:%s\n" % (msg['uid'], flags)) file.flush() if self.doautosave: diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 77cdff3..11791d0 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -23,6 +23,11 @@ try: except: pass #fail only if needed later on, not on import +try: # python 2.6 has set() built in + set +except NameError: + from sets import Set as set + class LocalStatusSQLiteFolder(LocalStatusFolder): """LocalStatus backend implemented with an SQLite database @@ -127,7 +132,6 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): for line in file.xreadlines(): uid, flags = line.strip().split(':') uid = long(uid) - flags = list(flags) flags = ''.join(sorted(flags)) data.append((uid,flags)) self.connection.executemany('INSERT INTO status (id,flags) VALUES (?,?)', @@ -167,7 +171,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): self.messagelist = {} cursor = self.connection.execute('SELECT id,flags from status') for row in cursor: - flags = [x for x in row[1]] + flags = set(row[1]) self.messagelist[row[0]] = {'uid': row[0], 'flags': flags} def save(self): diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 3b53156..3b564dc 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -28,6 +28,11 @@ try: except ImportError: from md5 import md5 +try: # python 2.6 has set() built in + set +except NameError: + from sets import Set as set + from offlineimap import OfflineImapError uidmatchre = re.compile(',U=(\d+)') @@ -166,11 +171,12 @@ class MaildirFolder(BaseFolder): nouidcounter -= 1 else: uid = long(uidmatch.group(1)) + #identify flags in the path name flagmatch = self.flagmatchre.search(messagename) - flags = [] if flagmatch: - flags = [x for x in flagmatch.group(1)] - flags.sort() + flags = set(flagmatch.group(1)) + else: + flags = set() retval[uid] = {'uid': uid, 'flags': flags, 'filename': file} @@ -261,7 +267,7 @@ class MaildirFolder(BaseFolder): if rtime != None: os.utime(os.path.join(tmpdir, messagename), (rtime, rtime)) - self.messagelist[uid] = {'uid': uid, 'flags': [], + self.messagelist[uid] = {'uid': uid, 'flags': set(), 'filename': os.path.join('tmp', messagename)} # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) @@ -288,8 +294,7 @@ class MaildirFolder(BaseFolder): infostr = infomatch.group(1) newname = newname.split(self.infosep)[0] # Strip off the info string. infostr = re.sub('2,[A-Z]*', '', infostr) - flags.sort() - infostr += '2,' + ''.join(flags) + infostr += '2,' + ''.join(sorted(flags)) newname += infostr newfilename = os.path.join(dir_prefix, newname) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 3905851..80a2486 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -20,6 +20,11 @@ import re import string import types from offlineimap.ui import getglobalui +try: # python 2.6 has set() built in + set +except NameError: + from sets import Set as set + quotere = re.compile('^("(?:[^"]|\\\\")*")') def debug(*args): @@ -42,11 +47,21 @@ def dequote(string): return string def flagsplit(string): + """Converts a string of IMAP flags to a list + + :returns: E.g. '(\\Draft \\Deleted)' returns ['\\Draft','\\Deleted']. + (FLAGS (\\Seen Old) UID 4807) returns + ['FLAGS,'(\\Seen Old)','UID', '4807'] + """ if string[0] != '(' or string[-1] != ')': raise ValueError, "Passed string '%s' is not a flag list" % string return imapsplit(string[1:-1]) def options2hash(list): + """convert list [1,2,3,4,5,6] to {1:2, 3:4, 5:6}""" + # effectively this does dict(zip(l[::2],l[1::2])), however + # measurements seemed to have indicated that the manual variant is + # faster for mosly small lists. retval = {} counter = 0 while (counter < len(list)): @@ -55,8 +70,12 @@ def options2hash(list): debug("options2hash returning:", retval) return retval -def flags2hash(string): - return options2hash(flagsplit(string)) +def flags2hash(flags): + """Converts IMAP response string from eg IMAP4.fetch() to a hash. + + E.g. '(FLAGS (\\Seen Old) UID 4807)' leads to + {'FLAGS': '(\\Seen Old)', 'UID': '4807'}""" + return options2hash(flagsplit(flags)) def imapsplit(imapstring): """Takes a string from an IMAP conversation and returns a list containing @@ -152,15 +171,16 @@ flagmap = [('\\Seen', 'S'), ('\\Draft', 'D')] def flagsimap2maildir(flagstring): - retval = [] - imapflaglist = [x.lower() for x in flagstring[1:-1].split()] + """Convert string '(\\Draft \\Deleted)' into a flags set(DR)""" + retval = set() + imapflaglist = flagstring[1:-1].split() for imapflag, maildirflag in flagmap: - if imapflag.lower() in imapflaglist: - retval.append(maildirflag) - retval.sort() + if imapflag in imapflaglist: + retval.add(maildirflag) return retval def flagsmaildir2imap(maildirflaglist): + """Convert set of flags ([DR]) into a string '(\\Draft \\Deleted)'""" retval = [] for imapflag, maildirflag in flagmap: if maildirflag in maildirflaglist: @@ -198,8 +218,3 @@ def listjoin(list): retval.append(getlist(start, end)) return ",".join(retval) - - - - - From df62bb61a5efd7505e51712fd7cee37c0cc84132 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Wed, 17 Aug 2011 10:01:39 +0200 Subject: [PATCH 194/817] Create exception when file rename fails Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 3b564dc..f72f4ea 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -299,8 +299,14 @@ class MaildirFolder(BaseFolder): newfilename = os.path.join(dir_prefix, newname) if (newfilename != oldfilename): - os.rename(os.path.join(self.getfullname(), oldfilename), - os.path.join(self.getfullname(), newfilename)) + try: + os.rename(os.path.join(self.getfullname(), oldfilename), + os.path.join(self.getfullname(), newfilename)) + except OSError, e: + raise OfflineImapError("Can't rename file '%s' to '%s': %s" % ( + oldfilename, newfilename, e[1]), + OfflineImapError.ERROR.FOLDER) + self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename From 8de326b29f249dcbd2498368cf0db246b67f041e Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Wed, 17 Aug 2011 10:01:37 +0200 Subject: [PATCH 195/817] Catch correct type of exception Signed-off-by: Vladimir Marek Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 6d7e093..09837ff 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -431,7 +431,7 @@ class BaseFolder(object): action(dstfolder, statusfolder) except (KeyboardInterrupt): raise - except OfflineImap, e: + except OfflineImapError, e: if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2]) From 0a254081991a29a000991465b18edb2d97602eba Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 22 Aug 2011 12:21:11 +0200 Subject: [PATCH 196/817] Rework undocumented listjoin to create UID sequences This function was badly named and completely undocumented. Rework it to avoid copying the full UID list using an iterator. Make it possible to hand it a list of UIDs as strings rather than implicitely relying on the fact that they are numeric already. Document the code. The behavior off the function itself remained otherwise unchanged. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Gmail.py | 2 +- offlineimap/folder/IMAP.py | 2 +- offlineimap/imaputil.py | 44 ++++++++++++++++++------------------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 6cd3446..3ca11cd 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -54,7 +54,7 @@ class GmailFolder(IMAPFolder): try: imapobj.select(self.getfullname()) result = imapobj.uid('copy', - imaputil.listjoin(uidlist), + imaputil.uid_sequence(uidlist), self.trash_folder) assert result[0] == 'OK', \ "Bad IMAPlib result: %s" % result[0] diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 40eb31c..dec14ae 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -638,7 +638,7 @@ class IMAPFolder(BaseFolder): self.ui.flagstoreadonly(self, uidlist, flags) return r = imapobj.uid('store', - imaputil.listjoin(uidlist), + imaputil.uid_sequence(uidlist), operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 80a2486..94d6ffc 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -188,33 +188,31 @@ def flagsmaildir2imap(maildirflaglist): retval.sort() return '(' + ' '.join(retval) + ')' -def listjoin(list): - start = None - end = None - retval = [] +def uid_sequence(uidlist): + """Collapse UID lists into shorter sequence sets - def getlist(start, end): + [1,2,3,4,5,10,12,13] will return "1:5,10,12:13". This function does + not sort the list, and only collapses if subsequent entries form a + range. + :returns: The collapsed UID list as string""" + def getrange(start, end): if start == end: return(str(start)) - else: - return(str(start) + ":" + str(end)) - + return "%s:%s" % (start, end) - for item in list: - if start == None: - # First item. - start = item - end = item - elif item == end + 1: - # An addition to the list. - end = item - else: - # Here on: starting a new list. - retval.append(getlist(start, end)) - start = item - end = item + if not len(uidlist): return '' # Empty list, return + start, end = None, None + retval = [] - if start != None: - retval.append(getlist(start, end)) + for item in iter(uidlist): + item = int(item) + if start == None: # First item + start, end = item, item + elif item == end + 1: # Next item in a range + end = item + else: # Starting a new range + retval.append(getrange(start, end)) + start, end = item, item + retval.append(getrange(start, end)) # Add final range/item return ",".join(retval) From 6295d64d545094ce6983c464c79d1d1dc6ac4ebd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 22 Aug 2011 12:21:12 +0200 Subject: [PATCH 197/817] Shorten list of messages to be deleted in UI output Rather than output the full list of messages, coalesce it into number ranges wherever possible. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 16e10bc..f5dca57 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -302,7 +302,7 @@ class UIBase: ds = s.folderlist(destlist) s._msg("Deleting %d messages (%s) in %s" % \ (len(uidlist), - ", ".join([str(u) for u in uidlist]), + offlineimap.imaputil.uid_sequence(uidlist), ds)) def addingflags(s, uidlist, flags, dest): From 9ecac29aaa6d17d4fd5561c993e5ec8ca4a1f6f4 Mon Sep 17 00:00:00 2001 From: Vladimir Marek Date: Mon, 22 Aug 2011 11:09:07 +0200 Subject: [PATCH 198/817] Maildir relative paths change was not complete Commit e023f190b0eed84fefc10e28bfe5e4e0467ff257 changed the storing of file paths in the messagelist variable to be relative paths, but we were using the full absolute path anyway as we missed one spot. Adapt this and construct the full file path in the one place where we need it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index f72f4ea..cedcb5d 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -133,7 +133,7 @@ class MaildirFolder(BaseFolder): folderstr = ',FMD5=' + foldermd5 for dirannex in ['new', 'cur']: fulldirname = os.path.join(self.getfullname(), dirannex) - files.extend(os.path.join(fulldirname, filename) for + files.extend(os.path.join(dirannex, filename) for filename in os.listdir(fulldirname)) for file in files: messagename = os.path.basename(file) @@ -151,10 +151,9 @@ class MaildirFolder(BaseFolder): #Check and see if the message is too big if the maxsize for this account is set if(maxsize != -1): - filesize = os.path.getsize(file) - if(filesize > maxsize): + size = os.path.getsize(os.path.join(self.getfullname(), file)) + if(size > maxsize): continue - foldermatch = messagename.find(folderstr) != -1 if not foldermatch: @@ -177,6 +176,7 @@ class MaildirFolder(BaseFolder): flags = set(flagmatch.group(1)) else: flags = set() + # 'filename' is 'dirannex/filename', e.g. cur/123_U=1_FMD5=1:2,S retval[uid] = {'uid': uid, 'flags': flags, 'filename': file} From 2b9b6be6beabe0013df6eb236272d8d12378b378 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 22 Aug 2011 10:33:55 +0200 Subject: [PATCH 199/817] Fix standard port for SSL/TLS 443 is of course the https and not the IMAPS standard port. Fix. Thanks to Daniel Shahaf for the heads up. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 707464d..3ce751c 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -311,7 +311,7 @@ class IMAPServer: elif SSLError and isinstance(e, SSLError) and e.errno == 1: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service - if self.port != 443: + if self.port != 993: reason = "Could not connect via SSL to host '%s' and non-s"\ "tandard ssl port %d configured. Make sure you connect"\ " to the correct port." % (self.hostname, self.port) From a6480d4959d2747680e75b2351dac8f1aca85fe1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 22 Aug 2011 17:22:41 +0200 Subject: [PATCH 200/817] Fix string formatting We were omitting an '%' where we needed it. Also include the traceback information where it belongs in the new ui.error infrastructure. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 09837ff..5bd8061 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -436,7 +436,6 @@ class BaseFolder(object): raise self.ui.error(e, exc_info()[2]) except Exception, e: - self.ui.error(e, msg = "ERROR attempting to sync folder %s " - "[acc: %s]:\n %s" (self, self.getaccountname(), - traceback.format_exc())) + self.ui.error(e, exc_info()[2], "Syncing folder %s [acc: %s]" %\ + (self, self.getaccountname())) raise # raise unknown Exceptions so we can fix them From dcc50efc4f3d66e5b04c4a04a1b4346959190c9b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 22 Aug 2011 14:58:42 +0200 Subject: [PATCH 201/817] Don't call set().sort() as that method doesn't exist on sets. Rather call the inbuilt sorted(flags). This fixes Exceptions being thrown when using the sqlite backend. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 11791d0..eae5d0c 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -231,8 +231,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): def savemessageflags(self, uid, flags): self.messagelist[uid] = {'uid': uid, 'flags': flags} - flags.sort() - flags = ''.join(flags) + flags = ''.join(sorted(flags)) self.sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) def deletemessages(self, uidlist): From 1fc64f4c198eb0de98c3cba1e8cddddf8c4099b3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 25 Aug 2011 10:18:03 +0200 Subject: [PATCH 202/817] Improve folderfilters documentation The text was confusing and not very helpful. Do away with the bad folderfilters examples and describe a bit more what they are for. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 8cc2882..e055974 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -443,10 +443,12 @@ subscribedonly = no # # nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) -# You can specify which folders to sync. You can do it several ways. -# I'll provide some examples. The folderfilter operates on the -# *UNTRANSLATED* name, if you specify nametrans. It should return -# true if the folder is to be included; false otherwise. +# You can specify which folders to sync using the folderfilter +# setting. You can provide any python function (e.g. a lambda function) +# which will be invoked for each foldername. If the filter function +# returns True, the folder will be synced, if it returns False, it. The +# folderfilter operates on the *UNTRANSLATED* name (before any nametrans +# translation takes place). # # Example 1: synchronizing only INBOX and Sent. # @@ -470,14 +472,7 @@ subscribedonly = no # folderfilter = lambda foldername: foldername in # ['INBOX', 'Sent Mail', 'Deleted Items', # 'Received'] -# -# FYI, you could also include every folder with: -# -# folderfilter = lambda foldername: 1 -# -# And exclude every folder with: -# -# folderfilter = lambda foldername: 0 + # You can specify folderincludes to include additional folders. # It should return a Python list. This might be used to include a From 654e3ab9dc6e0c877b10e3874d6c420bbfc4aaf3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 25 Aug 2011 10:18:04 +0200 Subject: [PATCH 203/817] De-EXPERIMENTALIZE the sqlite backend and IDLE folders Both have been in a stable release and we have never gotten negative feedback about them. I have been using sqlite exclusively for some time and people seem actively to use the IDLE folders without major problems. This patch removes the scary: "THIS IS EXPERIMENTAL" warning, that I know has kept some people from using it. Do note that the plaintext backend is still the default even with this patch. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index e055974..9d85377 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -201,8 +201,7 @@ remoterepository = RemoteExample # state in plain text files. On Repositories with large numbers of # mails, the performance might not be optimal, as we write out the # complete file for each change. Another new backend 'sqlite' is -# available which stores the status in sqlite databases. BE AWARE THIS -# IS EXPERIMENTAL STUFF. +# available which stores the status in sqlite databases. # # If you switch the backend, you may want to delete the old cache # directory in ~/.offlineimap/Account-/LocalStatus manually @@ -379,8 +378,8 @@ remoteuser = username # holdconnectionopen - to be true # keepalive - to be 29 minutes unless you specify otherwise # -# This feature isn't complete and may well have problems. BE AWARE THIS -# IS EXPERIMENTAL STUFF. See the manual for more details. +# This feature isn't complete and may well have problems. See the manual +# for more details. # # This option should return a Python list. For example # From 3333723ce79e12e8bb8f83ec33d2fa5a59cca7ce Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 25 Aug 2011 10:18:05 +0200 Subject: [PATCH 204/817] Improve our MANUAL Make it contain real use cases and more explanations. We probably need to add more of the FAQ entries to the MANUAL and point to relevant FAQ entries from the manual. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 383 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 264 insertions(+), 119 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 8f09f92..cf54985 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -14,39 +14,44 @@ Powerful IMAP/Maildir synchronization and reader support .. TODO: :Manual group: -SYNOPSIS -======== - - offlineimap [-h|--help] - - offlineimap [OPTIONS] - -| -1 -| -P profiledir -| -a accountlist -| -c configfile -| -d debugtype[,...] -| -f foldername[,...] -| -k [section:]option=value -| -l filename -| -o -| -u interface - - DESCRIPTION =========== -Most configuration is done via the configuration file. Nevertheless, there are -a few command-line options that you may set for OfflineIMAP. +OfflineImap operates on a REMOTE and a LOCAL repository and synchronizes +emails between them, so that you can read the same mailbox from multiple +computers. The REMOTE repository is some IMAP server, while LOCAL can be +either a local Maildir or another IMAP server. + +Missing folders will be automatically created on the LOCAL side, however +NO folders will currently be created on the REMOTE repository +automatically (it will sync your emails from local folders if +corresponding REMOTE folders already exist). + +Configuring OfflineImap in basic mode is quite easy, however it provides +an amazing amount of flexibility for those with special needs. You can +specify the number of connections to your IMAP server, use arbitrary +python functions (including regular expressions) to limit the number of +folders being synchronized. You can transpose folder names between +repositories using any python function, to mangle and modify folder +names on the LOCAL repository. There are six different ways to hand the +IMAP password to OfflineImap from console input, specifying in the +configuration file, .netrc support, specifying in a separate file, to +using arbitrary python functions that somehow return the +password. Finally, you can use IMAPs IDLE infrastructure to always keep +a connection to your IMAP server open and immediately be notified (and +synchronized) when a new mail arrives (aka Push mail). + +Most configuration is done via the configuration file. However, any setting can also be overriden by command line options handed to OfflineIMAP. + +OfflineImap is well suited to be frequently invoked by cron jobs, or can run in daemon mode to periodically check your email (however, it will exit in some error situations). + +Check out the `Use Cases`_ section for some example configurations. OPTIONS ======= - - - -1 Disable most multithreading operations Use solely a single-connection sync. This effectively sets the @@ -152,8 +157,7 @@ Blinkenlights --------------- Blinkenlights is an interface designed to be sleek, fun to watch, and -informative of the overall picture of what OfflineIMAP is doing. I consider it -to be the best general-purpose interface in OfflineIMAP. +informative of the overall picture of what OfflineIMAP is doing. Blinkenlights contains a row of "LEDs" with command buttons and a log. @@ -230,30 +234,27 @@ English-speaking world. One version ran in its entirety as follows: TTYUI --------- -TTYUI interface is for people running in basic, non-color terminals. It -prints out basic status messages and is generally friendly to use on a console -or xterm. +TTYUI interface is for people running in terminals. It prints out basic +status messages and is generally friendly to use on a console or xterm. Basic --------------------- +------ Basic is designed for situations in which OfflineIMAP will be run -non-attended and the status of its execution will be logged. You might use it, -for instance, to have the system run automatically and e-mail you the results of -the synchronization. This user interface is not capable of reading a password -from the keyboard; account passwords must be specified using one of the -configuration file options. +non-attended and the status of its execution will be logged. This user +interface is not capable of reading a password from the keyboard; +account passwords must be specified using one of the configuration file +options. Quiet ----- -Quiet is designed for non-attended running in situations where normal -status messages are not desired. It will output nothing except errors -and serious warnings. Like Basic, this user interface is not capable -of reading a password from the keyboard; account passwords must be -specified using one of the configuration file options. +It will output nothing except errors and serious warnings. Like Basic, +this user interface is not capable of reading a password from the +keyboard; account passwords must be specified using one of the +configuration file options. MachineUI --------- @@ -262,8 +263,98 @@ MachineUI generates output in a machine-parsable format. It is designed for other programs that will interface to OfflineIMAP. -Signals -======= +Synchronization Performance +=========================== + +By default, we use fairly conservative settings that are safe for +syncing but that might not be the best performing one. Once you got +everything set up and running, you might want to look into speeding up +your synchronization. Here are a couple of hints and tips on how to +achieve this. + + 1) Use maxconnections > 1. By default we only use one connection to an + IMAP server. Using 2 or even 3 speeds things up considerably in most + cases. This setting goes into the [Repository XXX] section. + + 2) Use folderfilters. The quickest sync is a sync that can ignore some + folders. I sort my inbox into monthly folders, and ignore every + folder that is more than 2-3 months old, this lets me only inspect a + fraction of my Mails on every sync. If you haven't done this yet, do + it :). See the folderfilter section the example offlineimap.conf. + + 3) The default status cache is a plain text file that will write out + the complete file for each single new message (or even changed flag) + to a temporary file. If you have plenty of files in a folder, this + is a few hundred kilo to megabytes for each mail and is bound to + make things slower. I recommend to use the sqlite backend for + that. See the status_backend = sqlite setting in the example + offlineimap.conf. You will need to have python-sqlite installed in + order to use this. This will save you plenty of disk activity. Do + note that the sqlite backend is still considered experimental as it + has only been included recently (although a loss of your status + cache should not be a tragedy as that file can be rebuild + automatically) + + 4) Use quick sync. A regular sync will request all flags and all UIDs + of all mails in each folder which takes quite some time. A 'quick' + sync only compares the number of messages in a folder on the IMAP + side (it will detect flag changes on the Maildir side of things + though). A quick sync on my smallish account will take 7 seconds + rather than 40 seconds. Eg, I run a cron script that does a regular + sync once a day, and does quick syncs (-q) only synchronizing the + "-f INBOX" in between. + + 5) Turn off fsync. In the [general] section you can set fsync to True + or False. If you want to play 110% safe and wait for all operations + to hit the disk before continueing, you can set this to True. If you + set it to False, you lose some of that safety, trading it for speed. + +Security and SSL +================ + +Some words on OfflineImap and its use of SSL/TLS. By default, we will +connect using any method that openssl supports, that is SSLv2, SSLv3, or +TLSv1. Do note that SSLv2 is notoriously insecure and deprecated. +Unfortunately, python2 does not offer easy ways to disable SSLv2. It is +recommended you test your setup and make sure that the mail server does +not use an SSLv2 connection. Use e.g. "openssl s_client -host +mail.server -port 443" to find out the connection that is used by +default. + +Certificate checking +-------------------- + +Unfortunately, by default we will not verify the certificate of an IMAP +TLS/SSL server we connect to, so connecting by SSL is no guarantee +against man-in-the-middle attacks. While verifying a server certificate +fingerprint is being planned, it is not implemented yet. There is +currently only one safe way to ensure that you connect to the correct +server in an encrypted manner: You can specify a 'sslcacertfile' setting +in your repository section of offlineimap.conf pointing to a file that +contains (among others) a CA Certificate in PEM format which validating +your server certificate. In this case, we will check that: 1) The server +SSL certificate is validated by the CA Certificate 2) The server host +name matches the SSL certificate 3) The server certificate is not past +its expiration date. The FAQ contains an entry on how to create your own +certificate and CA certificate. + +StartTLS +-------- + +If you have not configured your account to connect via SSL anyway, +OfflineImap will still attempt to set up an SSL connection via the +STARTTLS function, in case the imap server supports it. Do note, that +there is no certificate or fingerprint checking involved at all, when +using STARTTLS (the underlying imaplib library does not support this +yet). This means that you will be protected against passively listening +eavesdroppers and they will not be able to see your password or email +contents. However, this will not protect you from active attacks, such +as Man-In-The-Middle attacks which cause you to connect to the wrong +server and pretend to be your mail server. DO NOT RELY ON STARTTLS AS A +SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF YOUR IMAP SERVER! + +UNIX Signals +============ OfflineImap listens to the unix signals SIGUSR1 and SIGUSR2. @@ -310,7 +401,7 @@ KNOWN BUGS storing messages. Such files can be written to windows partitions. But you will probably loose compatibility with other programs trying to read the same Maildir. - - Exclamation mark was choosed because of the note in + - Exclamation mark was chosen because of the note in http://docs.python.org/library/mailbox.html - If you have some messages already stored without this option, you will have to re-sync them again @@ -322,91 +413,145 @@ KNOWN BUGS - not available anymore since cygwin 1.7 -Synchronization Performance -=========================== +PITFALLS & ISSUES +================= -By default, we use fairly conservative settings that are good for -syncing but that might not be the best performing one. Once you got -everything set up and running, you might want to look into speeding up -your synchronization. Here are a couple of hints and tips on how to -achieve this. +Sharing a maildir with multiple IMAP servers +-------------------------------------------- - 1) Use maxconnections > 1. By default we only use one connection to an - IMAP server. Using 2 or even 3 speeds things up considerably in most - cases. This setting goes into the [Repository XXX] section. + Generally a word of caution mixing IMAP repositories on the same + Maildir root. You have to be careful that you *never* use the same + maildir folder for 2 IMAP servers. In the best case, the folder MD5 + will be different, and you will get a loop where it will upload your + mails to both servers in turn (infinitely!) as it thinks you have + placed new mails in the local Maildir. In the worst case, the MD5 is + the same (likely) and mail UIDs overlap (likely too!) and it will fail to + sync some mails as it thinks they are already existent. - 2) Use folderfilters. The quickest sync is a sync that can ignore some - folders. I sort my inbox into monthly folders, and ignore every - folder that is more than 2-3 months old, this lets me only inspect a - fraction of my Mails on every sync. If you haven't done this yet, do - it :). See the folderfilter section the example offlineimap.conf. + I would create a new local Maildir Repository for the Personal Gmail and + use a different root to be on the safe side here. You could e.g. use + `~/mail/Pro` as Maildir root for the ProGmail and + `~/mail/Personal` as root for the personal one. - 3) The default status cache is a plain text file that will write out - the complete file for each single new message (or even changed flag) - to a temporary file. If you have plenty of files in a folder, this - is a few hundred kilo to megabytes for each mail and is bound to - make things slower. I recommend to use the sqlite backend for - that. See the status_backend = sqlite setting in the example - offlineimap.conf. You will need to have python-sqlite installed in - order to use this. This will save you plenty of disk activity. Do - note that the sqlite backend is still considered experimental as it - has only been included recently (although a loss of your status - cache should not be a tragedy as that file can be rebuild - automatically) + If you then point your local mutt, or whatever MUA you use to `~/mail/` + as root, it should still recognize all folders. (see the 2 IMAP setup + in the `Use Cases`_ section. - 4) Use quick sync. A regular sync will request all flags and all UIDs - of all mails in each folder which takes quite some time. A 'quick' - sync only compares the number of messages in a folder on the IMAP - side (it will detect flag changes on the Maildir side of things - though). A quick sync on my smallish account will take 7 seconds - rather than 40 seconds. Eg, I run a cron script that does a regular - sync once a day, and does quick syncs inbetween. +USE CASES +========= - 5) Turn off fsync. In the [general] section you can set fsync to True - or False. If you want to play 110% safe and wait for all operations - to hit the disk before continueing, you can set this to True. If you - set it to False, you lose some of that safety trading it for speed. +Sync from GMail to another IMAP server +-------------------------------------- -Security and SSL -================ +This is an example of a setup where "TheOtherImap" requires all folders to be under INBOX:: -Some words on OfflineImap and its use of SSL/TLS. By default, we will -connect using any method that openssl supports, that is SSLv2, SSLv3, or -TLSv1. Do note that SSLv2 is notoriously insecure and deprecated. -Unfortunately, python2 does not offer easy ways to disable SSLv2. It is -recommended you test your setup and make sure that the mail server does -not use an SSLv2 connection. Use e.g. "openssl s_client -host -mail.server -port 443" to find out the connection that is used by -default. + [Repository Gmailserver-foo] + #This is the remote repository + type = Gmail + remotepass = XXX + remoteuser = XXX + # The below will put all GMAIL folders as sub-folders of the 'local' INBOX, + # assuming that your path separator on 'local' is a dot. + nametrans = lambda x: 'INBOX.' + x + + [Repository TheOtherImap] + #This is the 'local' repository + type = IMAP + remotehost = XXX + remotepass = XXX + remoteuser = XXX + #Do not use nametrans here. -Certificate checking --------------------- +Selecting only a few folders to sync +------------------------------------ +Add this to the remote gmail repository section to only sync mails which are in a certain folder:: -Unfortunately, by default we will not verify the certificate of an IMAP -TLS/SSL server we connect to, so connecting by SSL is no guarantee -against man-in-the-middle attacks. While verifying a server certificate -fingerprint is being planned, it is not implemented yet. There is -currently only one safe way to ensure that you connect to the correct -server in an encrypted manner: You can specify a 'sslcacertfile' setting -in your repository section of offlineimap.conf pointing to a file that -contains (among others) a CA Certificate in PEM format which validating -your server certificate. In this case, we will check that: 1) The server -SSL certificate is validated by the CA Certificate 2) The server host -name matches the SSL certificate 3) The server certificate is not past -its expiration date. The FAQ contains an entry on how to create your own -certificate and CA certificate. + folderfilter = lambda folder: folder.startswith('MyLabel') -StartTLS --------- +To only get the All Mail folder from a Gmail account, you would e.g. do:: -If you have not configured your account to connect via SSL anyway, -OfflineImap will still attempt to set up an SSL connection via the -STARTTLS function, in case the imap server supports it. Do note, that -there is no certificate or fingerprint checking involved at all, when -using STARTTLS (the underlying imaplib library does not support this -yet). This means that you will be protected against passively listening -eavesdroppers and they will not be able to see your password or email -contents. However, this will not protect you from active attacks, such -as Man-In-The-Middle attacks which cause you to connect to the wrong -server and pretend to be your mail server. DO NOT RELY ON STARTTLS AS A -SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF YOUR IMAP SERVER! + folderfilter = lambda folder: folder.startswith('[Gmail]/All Mail') + + +Another nametrans transpose example +----------------------------------- + +Put everything in a GMX. subfolder except for the boxes INBOX, Draft, and Sent which should keep the same name:: + + folderfilter = lambda folder: re.sub(r'^(?!INBOX$|Draft$|Sent$)',r'GMX.', folder) + +2 IMAP using name translations +------------------------------ + +Synchronizing 2 IMAP accounts to local Maildirs that are "next to each other", so that mutt can work on both. Full email setup described by Thomas Kahle at `http://dev.gentoo.org/~tomka/mail.html`_ + +offlineimap.conf:: + + [general] + accounts = acc1, acc2 + maxsyncaccounts = 2 + ui = ttyui + pythonfile=~/bin/offlineimap-helpers.py + socktimeout = 90 + + [Account acc1] + localrepository = acc1local + remoterepository = acc1remote + autorefresh = 2 + + [Account acc2] + localrepository = acc2local + remoterepository = acc2remote + autorefresh = 4 + + [Repository acc1local] + type = Maildir + localfolders = ~/Mail/acc1 + + [Repository acc2local] + type = Maildir + localfolders = ~/Mail/acc2 + + [Repository acc1remote] + type = IMAP + remotehost = imap.acc1.com + remoteusereval = get_username("imap.acc1.net") + remotepasseval = get_password("imap.acc1.net") + nametrans = oimaptransfolder_acc1 + ssl = yes + maxconnections = 2 + # Folders to get: + folderfilter = lambda foldername: foldername in [ + 'INBOX', 'Drafts', 'Sent', 'archiv'] + + [Repository acc2remote] + type = IMAP + remotehost = imap.acc2.net + remoteusereval = get_username("imap.acc2.net") + remotepasseval = get_password("imap.acc2.net") + nametrans = oimaptransfolder_acc2 + ssl = yes + maxconnections = 2 + +One of the coolest things about offlineimap is that you can inject arbitrary python code. The file specified with:: + + pythonfile=~/bin/offlineimap-helpers.py + +contains python functions that I used for two purposes: Fetching passwords from the gnome-keyring and translating folder names on the server to local foldernames. The python file should contain all the functions that are called here. get_username and get_password are part of the interaction with gnome-keyring and not printed here. Find them in the example file that is in the tarball or here. The folderfilter is a lambda term that, well, filters which folders to get. `oimaptransfolder_acc2` translates remote folders into local folders with a very simple logic. The `INBOX` folder will simply have the same name as the account while any other folder will have the account name and a dot as a prefix. offlineimap handles the renaming correctly in both directions:: + + import re + def oimaptransfolder_acc1(foldername): + if(foldername == "INBOX"): + retval = "acc1" + else: + retval = "acc1." + foldername + retval = re.sub("/", ".", retval) + return retval + + def oimaptransfolder_acc2(foldername): + if(foldername == "INBOX"): + retval = "acc2" + else: + retval = "acc2." + foldername + retval = re.sub("/", ".", retval) + return retval From dcfdf2ade7434d0d529c5a8b331b73af9a706d6d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 16:10:50 +0200 Subject: [PATCH 205/817] MANUAL.rst: folderfilter->nametrans typo We really wanted nametrans here, also simplify the lambda per Dan Christensen's suggestion. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index cf54985..6c29041 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -478,7 +478,8 @@ Another nametrans transpose example Put everything in a GMX. subfolder except for the boxes INBOX, Draft, and Sent which should keep the same name:: - folderfilter = lambda folder: re.sub(r'^(?!INBOX$|Draft$|Sent$)',r'GMX.', folder) + nametrans = lambda folder: folder if folder in ['INBOX', 'Drafts', 'Sent'] \ + else re.sub(r'^', r'GMX.', folder) 2 IMAP using name translations ------------------------------ From f10e3a58fc42a5f4fb0179072d50ab52996309aa Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 30 Aug 2011 22:05:44 +0200 Subject: [PATCH 206/817] MANUAL: fix typo about variable declaration in configuration file I included a wrong change in commit dcfdf2ade7434d0d529c writen by Sebastian. As he said: > I think the first colon should be an equals sign... :-) Actually no, this was on purpose. http://docs.python.org/library/configparser.html: [My Section] foodir: %(dir)s/whatever dir=frob long: this value continues in the next line Reported-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 6c29041..928ae40 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -478,7 +478,7 @@ Another nametrans transpose example Put everything in a GMX. subfolder except for the boxes INBOX, Draft, and Sent which should keep the same name:: - nametrans = lambda folder: folder if folder in ['INBOX', 'Drafts', 'Sent'] \ + nametrans: lambda folder: folder if folder in ['INBOX', 'Drafts', 'Sent'] \ else re.sub(r'^', r'GMX.', folder) 2 IMAP using name translations From 4db591349295708e2a2846f5a1045119430db620 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 12:03:26 +0200 Subject: [PATCH 207/817] Use SafeConfigParser for the configuration SafeConfigParser is very similar to the currently used ConfigParser but it supports interpolation. This means values can contain format strings which refer to other values in the same section, or values in a special DEFAULT section. For example: [My Section] foodir: %(dir)s/whatever dir=frob would resolve the %(dir)s to the value of dir (frob in this case). All reference expansions are done on demand. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 18 ++++++++++++++++++ offlineimap/CustomConfig.py | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 9d85377..3cd8dd5 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -15,8 +15,26 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# This file documents all possible options and can be quite scary. # Looking for a quick start? Take a look at offlineimap.conf.minimal. +# Settings support interpolation. This means values can contain python +# format strings which refer to other values in the same section, or +# values in a special DEFAULT section. This allows you for example to +# use common settings for multiple accounts: +# +# [Repository Gmail1] +# trashfolder: %(gmailtrashfolder)s +# +# [Repository Gmail2] +# trashfolder: %(gmailtrashfolder)s +# +# [DEFAULT] +# gmailtrashfolder = [Google Mail]/Papierkorb +# +# would set the trashfolder setting for your German gmail accounts. + + ################################################## # General definitions diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 0c10271..42132f8 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -15,11 +15,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from ConfigParser import ConfigParser +from ConfigParser import SafeConfigParser from offlineimap.localeval import LocalEval import os -class CustomConfigParser(ConfigParser): +class CustomConfigParser(SafeConfigParser): def getdefault(self, section, option, default, *args, **kwargs): """Same as config.get, but returns the "default" option if there is no such option specified.""" From 8cce178fb8770e7345e72a9e1362ba13f0a36a68 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 14:31:52 +0200 Subject: [PATCH 208/817] Improve developer documentation Well, this patch... improves it. :) Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- docs/dev-doc-src/index.rst | 15 ++++++++------- docs/dev-doc-src/repository.rst | 4 ++-- docs/dev-doc-src/ui.rst | 2 +- offlineimap/error.py | 15 +++++++++------ 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/docs/dev-doc-src/index.rst b/docs/dev-doc-src/index.rst index f4d7067..7cf27e2 100644 --- a/docs/dev-doc-src/index.rst +++ b/docs/dev-doc-src/index.rst @@ -5,9 +5,9 @@ Welcome to :mod:`offlineimaps`'s documentation ============================================== -The :mod:`offlineimap` module provides the user interface for synchronization between IMAP servers and MailDirs or between IMAP servers. The homepage containing the source code repository can be found at the `offlineimap homepage `_. +The :mod:`offlineimap` module provides the user interface for synchronization between IMAP servers and MailDirs or between IMAP servers. The homepage containing the source code repository can be found at the `offlineimap homepage `_. The following provides the developer documentation for those who are interested in modifying the source code or otherwise peek into the OfflineImap internals. End users might want to check the MANUAL, our INSTALLation instructions, and the FAQ. -Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. A folder is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative :mod:`offlineimap.folder`. +Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. Email repositories are represented by a :class:`offlineimap.repository.Base.BaseRepository` or derivatives (see :mod:`offlineimap.repository` for details). A folder within a repository is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative from :mod:`offlineimap.folder`. .. moduleauthor:: John Goerzen, and many others. See AUTHORS and the git history for a full list. @@ -15,7 +15,7 @@ Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-le This page contains the main API overview of OfflineImap |release|. -Notmuch can be imported as:: +OfflineImap can be imported as:: from offlineimap import OfflineImap @@ -58,11 +58,12 @@ An :class:`accounts.Account` connects two email repositories that are to be sync Contains the current :mod:`offlineimap.ui`, and can be used for logging etc. - -:exc:`OfflineImapException` -- A Notmuch execution error +:exc:`OfflineImapError` -- A Notmuch execution error -------------------------------------------------------- -.. autoexception:: offlineimap.OfflineImapException +.. autoexception:: offlineimap.error.OfflineImapError :members: - This execption inherits directly from :exc:`Exception` and is raised on errors during the offlineimap execution. + This execption inherits directly from :exc:`Exception` and is raised + on errors during the offlineimap execution. It has an attribute + `severity` that denotes the severity level of the error. diff --git a/docs/dev-doc-src/repository.rst b/docs/dev-doc-src/repository.rst index 04025cd..1594a68 100644 --- a/docs/dev-doc-src/repository.rst +++ b/docs/dev-doc-src/repository.rst @@ -25,8 +25,8 @@ on your configuration. So when you want to instanciate a new -:mod:`offlineimap.repository` -- Basic representation of a mail repository --------------------------------------------------------------------------- +:mod:`offlineimap.repository.Base.BaseRepository` -- Representation of a mail repository +------------------------------------------------------------------------------------------ .. autoclass:: offlineimap.repository.Base.BaseRepository :members: :inherited-members: diff --git a/docs/dev-doc-src/ui.rst b/docs/dev-doc-src/ui.rst index 03da19b..27be221 100644 --- a/docs/dev-doc-src/ui.rst +++ b/docs/dev-doc-src/ui.rst @@ -1,4 +1,4 @@ -:mod:`offlineimap.ui` -- A pluggable logging system +:mod:`offlineimap.ui` -- A flexible logging system -------------------------------------------------------- .. currentmodule:: offlineimap.ui diff --git a/offlineimap/error.py b/offlineimap/error.py index d4d16be..9e55162 100644 --- a/offlineimap/error.py +++ b/offlineimap/error.py @@ -2,11 +2,14 @@ class OfflineImapError(Exception): """An Error during offlineimap synchronization""" class ERROR: - """Severity levels""" - MESSAGE = 0 - FOLDER = 10 - REPO = 20 - CRITICAL = 30 + """Severity level of an Exception + + * **MESSAGE**: Abort the current message, but continue with folder + * **FOLDER**: Abort folder sync, but continue with next folder + * **REPO**: Abort repository sync, continue with next account + * **CRITICAL**: Immediately exit offlineimap + """ + MESSAGE, FOLDER, REPO, CRITICAL = 0, 10, 20, 30 def __init__(self, reason, severity, errcode=None): """ @@ -17,7 +20,7 @@ class OfflineImapError(Exception): message, but a ERROR.REPO occurs when the server is offline. - :param errcode: optional number denoting a predefined error + :param errcode: optional number denoting a predefined error situation (which let's us exit with a predefined exit value). So far, no errcodes have been defined yet. From f0869e5c0f36ed8cb19b15bcfc440c0fac4dd586 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 14:26:49 +0200 Subject: [PATCH 209/817] Fix sqlite upgrade code to use correct API call We were using self.getfolderbasename(self.name) but the API is simply getfolderbasename(). Fix this glitch. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index eae5d0c..bd38930 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -111,7 +111,8 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): if hasattr(self, 'connection'): self.connection.close() #close old connections first - self.connection = sqlite.connect(self.filename, check_same_thread = False) + self.connection = sqlite.connect(self.filename, + check_same_thread = False) if from_ver == 0: # from_ver==0: no db existent: plain text migration? @@ -120,7 +121,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): plaintextfilename = os.path.join( self.repository.account.getaccountmeta(), 'LocalStatus', - self.getfolderbasename(self.name)) + self.getfolderbasename()) # MIGRATE from plaintext if needed if os.path.exists(plaintextfilename): self.ui._msg('Migrating LocalStatus cache from plain text ' From 54d3307499449ab83e3eecf015fcf833896670b1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 09:22:33 +0200 Subject: [PATCH 210/817] imaputil: make uid_sequence sort all items to improve collapsing To preserve previous behavior and save a few CPU cycles, we were not sorting UID lists and only collapsed them if they were alreay sorted. Vincent pointed out that this is not always the case and unsorted lists lead to non-optimally collapsing. Force lists to numeric types and sort them before collapsing. Reported-by: Beffara Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaputil.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 94d6ffc..e628df5 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -191,9 +191,10 @@ def flagsmaildir2imap(maildirflaglist): def uid_sequence(uidlist): """Collapse UID lists into shorter sequence sets - [1,2,3,4,5,10,12,13] will return "1:5,10,12:13". This function does - not sort the list, and only collapses if subsequent entries form a - range. + [1,2,3,4,5,10,12,13] will return "1:5,10,12:13". This function + converts items to numeric type and sorts the list to always produce + the minimal collapsed set. + :returns: The collapsed UID list as string""" def getrange(start, end): if start == end: @@ -203,9 +204,10 @@ def uid_sequence(uidlist): if not len(uidlist): return '' # Empty list, return start, end = None, None retval = [] + # Force items to be longs and sort them + sorted_uids = sorted(map(int, uidlist)) - for item in iter(uidlist): - item = int(item) + for item in iter(sorted_uids): if start == None: # First item start, end = item, item elif item == end + 1: # Next item in a range From fe1391084be4a0d482a1524b3b620c559a630c2b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 16:00:11 +0200 Subject: [PATCH 211/817] Improve repo.Base.py:syncfoldersto parameter/var names Name parameter that hands us a Status Repository 'status_repo' and not 'copyfolders' as was before: a) make it clear that we pass in a repository and not folder instances. That was very confusing before. b) We were always only using one 'copyfolders' item anyway, so let us not make it a list. Go through the list and make the variable nameing consistent: dst_repo rather than dest (is it a folder?) to match the status_repo naming scheme. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 38 +++++++++++++++------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index f96a33d..184bc9a 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -114,36 +114,32 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): def getfolder(self, foldername): raise NotImplementedError - def syncfoldersto(self, dest, status): + def syncfoldersto(self, dst_repo, status_repo): """Syncs the folders in this repository to those in dest. - It does NOT sync the contents of those folders. - Whenever makefolder() is called, also call makefolder() on status - folder.""" - src = self - srcfolders = src.getfolders() - destfolders = dest.getfolders() + It does NOT sync the contents of those folders.""" + src_repo = self + src_folders = src_repo.getfolders() + dst_folders = dst_repo.getfolders() # Create hashes with the names, but convert the source folders # to the dest folder's sep. - - srchash = {} - for folder in srcfolders: - srchash[folder.getvisiblename().replace(src.getsep(), dest.getsep())] = \ - folder - desthash = {} - for folder in destfolders: - desthash[folder.getvisiblename()] = folder + src_hash = {} + for folder in src_folders: + src_hash[folder.getvisiblename().replace( + src_repo.getsep(), dst_repo.getsep())] = folder + dst_hash = {} + for folder in dst_folders: + dst_hash[folder.getvisiblename()] = folder # # Find new folders. - # - - for key in srchash.keys(): - if not key in desthash: + for key in src_hash.keys(): + if not key in dst_hash: try: - dest.makefolder(key) - status.makefolder(key.replace(dest.getsep(), status.getsep())) + dst_repo.makefolder(key) + status_repo.makefolder(key.replace(dst_repo.getsep(), + status_repo.getsep())) except (KeyboardInterrupt): raise except: From 7aa4c49ba48f1c262dc3a4d666dd20ee682e7a44 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 16:00:10 +0200 Subject: [PATCH 212/817] Improve documentation on folderincludes Just minor wording improvements. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 3cd8dd5..12a09ba 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -491,25 +491,15 @@ subscribedonly = no # 'Received'] -# You can specify folderincludes to include additional folders. -# It should return a Python list. This might be used to include a -# folder that was excluded by your folderfilter rule, to include a -# folder that your server does not specify with its LIST option, or -# to include a folder that is outside your basic reference. Some examples: -# -# To include debian.user and debian.personal: -# + +# You can specify folderincludes to include additional folders. It +# should return a Python list. This might be used to include a folder +# that was excluded by your folderfilter rule, to include a folder that +# your server does not specify with its LIST option, or to include a +# folder that is outside your basic reference. The 'reference' value +# will not be prefixed to this folder name, even if you have specified +# one. For example: # folderincludes = ['debian.user', 'debian.personal'] -# -# To include your INBOX (UW IMAPd users will find this useful if they -# specify a reference): -# -# folderincludes = ['INBOX'] -# -# To specify a long list: -# -# folderincludes = ['box1', 'box2', 'box3', 'box4', -# 'box5', 'box6'] # You can specify foldersort to determine how folders are sorted. # This affects order of synchronization and mbnames. The expression From 7bc45507cbcca54b35628152bc3d7b68127822f6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Sep 2011 11:00:58 +0200 Subject: [PATCH 213/817] Introduce new error level FOLDER_RETRY In cases where processing a folder failed, but a retry might well succeed e.g. when the server connection had simply timed out and was disconnected, we can throw a FOLDER_RETRY (which is less severe than FOLDER). Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/error.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/error.py b/offlineimap/error.py index 9e55162..aa7f535 100644 --- a/offlineimap/error.py +++ b/offlineimap/error.py @@ -5,11 +5,12 @@ class OfflineImapError(Exception): """Severity level of an Exception * **MESSAGE**: Abort the current message, but continue with folder + * **FOLDER_RETRY**: Error syncing folder, but do retry * **FOLDER**: Abort folder sync, but continue with next folder * **REPO**: Abort repository sync, continue with next account * **CRITICAL**: Immediately exit offlineimap """ - MESSAGE, FOLDER, REPO, CRITICAL = 0, 10, 20, 30 + MESSAGE, FOLDER_RETRY, FOLDER, REPO, CRITICAL = 0, 10, 15, 20, 30 def __init__(self, reason, severity, errcode=None): """ From 61fffd5c5f16abda5af003eba3692f698a17f489 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Sep 2011 11:00:59 +0200 Subject: [PATCH 214/817] imaplibutil: Have SELECT throw an OfflineImapError when the connection died MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Imapserver.acquireconnection will succeed even whent the server connection has been terminated and the first IMAP operation will throw an exception. Often this is the folder SELECT operation (e.g. after an idle timeout), as has been reported by John Wiegley. Catch this case and throw an OfflineImapError with severity FOLDER_RETRY to notify consumers that they are supposed to retry. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index fa3a303..529af6f 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -49,7 +49,14 @@ class UsefulIMAPMixIn: return # Wipe out all old responses, to maintain semantics with old imaplib2 del self.untagged_responses[:] - result = self.__class__.__bases__[1].select(self, mailbox, readonly) + try: + result = self.__class__.__bases__[1].select(self, mailbox, readonly) + except self.abort, e: + # self.abort is raised when we are supposed to retry + errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ + "ver said: %s" % (self.host, mailbox, e.args[0]) + severity = OfflineImapError.ERROR.FOLDER_RETRY + raise OfflineImapError(errstr, severity) if result[0] != 'OK': #in case of error, bail out with OfflineImapError errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\ From f1555ec89311c1142f06a7085f9a097b0b2dccc8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Sep 2011 11:01:00 +0200 Subject: [PATCH 215/817] Catch terminated connections on the IDLE SELECT operation Handle the case gracefully where a server has closed an IMAP connection that we want to use for IDLEing. Simply have it dropped and get a new one in this case. THis should get rid of the errors reported by John Wiegley in mail id:"m2sjohd16t.fsf@gmail.com". Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 3ce751c..e630f83 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -26,7 +26,7 @@ import socket import base64 import time import errno - +from sys import exc_info from socket import gaierror try: from ssl import SSLError, cert_time_to_seconds @@ -456,6 +456,7 @@ class IdleThread(object): self.parent = parent self.folder = folder self.event = Event() + self.ui = getglobalui() if folder is None: self.thread = Thread(target=self.noop) else: @@ -505,13 +506,24 @@ class IdleThread(object): self.imapaborted = True self.stop() - imapobj = self.parent.acquireconnection() - imapobj.select(self.folder) + success = False # successfully selected FOLDER? + while not success: + imapobj = self.parent.acquireconnection() + try: + imapobj.select(self.folder) + except OfflineImapError, e: + if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: + # Connection closed, release connection and retry + self.ui.error(e, exc_info()[2]) + self.parent.releaseconnection(imapobj) + else: + raise e + else: + success = True if "IDLE" in imapobj.capabilities: imapobj.idle(callback=callback) else: - ui = getglobalui() - ui.warn("IMAP IDLE not supported on connection to %s." + self.ui.warn("IMAP IDLE not supported on connection to %s." "Falling back to old behavior: sleeping until next" "refresh cycle." %(imapobj.identifier,)) From dc103ab9ce26a6d2ef4d7585e430804b65156e27 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 5 Sep 2011 10:27:47 +0200 Subject: [PATCH 216/817] Protect IMAP.getmessage() against dropped connections If a connection is dropped for some reason while fetching a message, the imapobj.uid command throws an imapbj.abort() Exception which means we are supposed to retry. Implement a fail loop that drops the connection, gets a new one and attempts the command another time. Remove obsolete comment that we need to catch nonexisting messages. We do now. GMail seems to drop connections left and right. This patch is a response to the reported mail "4E5F8D8C.1020005@gmail.com" by zeek . Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index dec14ae..ac18d69 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -207,8 +207,21 @@ class IMAPFolder(BaseFolder): """ imapobj = self.imapserver.acquireconnection() try: - imapobj.select(self.getfullname(), readonly = 1) - res_type, data = imapobj.uid('fetch', str(uid), '(BODY.PEEK[])') + fails_left = 2 # retry on dropped connection + while fails_left: + try: + imapobj.select(self.getfullname(), readonly = 1) + res_type, data = imapobj.uid('fetch', str(uid), + '(BODY.PEEK[])') + fails_left = 0 + except imapobj.abort(), e: + # Release dropped connection, and get a new one + self.imapserver.releaseconnection(imapobj) + imapobj = self.imapserver.acquireconnection() + self.ui.error(e, exc_info()[2]) + fails_left -= 1 + if not fails_left: + raise e if data == [None] or res_type != 'OK': #IMAP server says bad request or UID does not exist severity = OfflineImapError.ERROR.MESSAGE @@ -223,16 +236,6 @@ class IMAPFolder(BaseFolder): # data looks now e.g. [('320 (UID 17061 BODY[] # {2565}','msgbody....')] we only asked for one message, # and that msg is in data[0]. msbody is in [0][1] - - #NB & TODO: When the message on the IMAP server has been - #deleted in the mean time, it will respond with an 'OK' - #res_type, but it will simply not send any data. This will - #lead to a crash in the below line. We need urgently to - #detect this, protect from this and need to think about what - #to return in this case. Probably returning `None` in this - #case would be good. But we need to make sure that all - #Backends behave the same, and that we actually check the - #return value and behave accordingly. data = data[0][1].replace("\r\n", "\n") if len(data)>200: From 0906d0db7062b37d68a9998045880d09d5222f70 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 5 Sep 2011 14:03:09 +0200 Subject: [PATCH 217/817] IMAP.cachemessagelist(): Protect against empty folders When invoked with FETCH 1:* (UID), imaplib returns [None] for empty folders. We need to protect against this case and simply 'continue' here. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index ac18d69..2774136 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -181,8 +181,10 @@ class IMAPFolder(BaseFolder): finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: - # looks like: '1 (FLAGS (\\Seen Old) UID 4807)' + # looks like: '1 (FLAGS (\\Seen Old) UID 4807)' or None if no msg # Discard initial message number. + if messagestr == None: + continue messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not options.has_key('UID'): From 9c678c9d6baee266c4772a4c197cd35ceb00ebc8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 6 Sep 2011 13:19:25 +0200 Subject: [PATCH 218/817] Allow Imapserver.releaseconnection() to drop a connection If a connection is broken, we want to have it really dropped and not be reused. So far, we are checking the .Terminate attribute for this, but according to the imaplib2 author, it is only set on normal shutdown and it is an undocumented attribute whose meaning could change any time. This patch introduces the parameter drop_conn which allows to tell releaseconnection() that we really want to connection being dropped from the pool of available connections and properly destroy it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index e630f83..05d8111 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -109,12 +109,15 @@ class IMAPServer: return self.root - def releaseconnection(self, connection): - """Releases a connection, returning it to the pool.""" + def releaseconnection(self, connection, drop_conn=False): + """Releases a connection, returning it to the pool. + + :param drop_conn: If True, the connection will be released and + not be reused. This can be used to indicate broken connections.""" self.connectionlock.acquire() self.assignedconnections.remove(connection) # Don't reuse broken connections - if connection.Terminate: + if connection.Terminate or drop_conn: connection.logout() else: self.availableconnections.append(connection) From 2cf6155282bfd5b7835a5c45a01d7de9a7512afe Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 6 Sep 2011 13:19:26 +0200 Subject: [PATCH 219/817] Error proof IMAP.APPEND against dropped connections Make sure that when a connection is dropped during append, we really discard the broken connection and get a new one, retrying. We retry indefinitely on the specific abort() Exception, as this is what imaplib2 suggests us to do. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 88 +++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 2774136..eb359af 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -21,6 +21,7 @@ import random import binascii import re import time +from sys import exc_info from copy import copy from Base import BaseFolder from offlineimap import imaputil, imaplibutil, OfflineImapError @@ -500,53 +501,62 @@ class IMAPFolder(BaseFolder): self.savemessageflags(uid, flags) return uid + imapobj = self.imapserver.acquireconnection() try: - imapobj = self.imapserver.acquireconnection() + success = False # succeeded in APPENDING? + while not success: - try: - imapobj.select(self.getfullname()) # Needed for search and making the box READ-WRITE - except imapobj.readonly: - # readonly exception. Return original uid to notify that - # we did not save the message. (see savemessage in Base.py) - self.ui.msgtoreadonly(self, uid, content, flags) - return uid + # UIDPLUS extension provides us with an APPENDUID response. + use_uidplus = 'UIDPLUS' in imapobj.capabilities - # UIDPLUS extension provides us with an APPENDUID response to our append() - use_uidplus = 'UIDPLUS' in imapobj.capabilities + # get the date of the message, so we can pass it to the server. + date = self.getmessageinternaldate(content, rtime) + content = re.sub("(?200: + dbg_output = "%s...%s" % (content[:150], content[-50:]) + else: + dbg_output = content + self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % + (date, dbg_output)) - if not use_uidplus: - # insert a random unique header that we can fetch later - (headername, headervalue) = self.generate_randomheader(content) - self.ui.debug('imap', 'savemessage: new header is: %s: %s' % \ - (headername, headervalue)) - content = self.savemessage_addheader(content, headername, - headervalue) - if len(content)>200: - dbg_output = "%s...%s" % (content[:150], - content[-50:]) - else: - dbg_output = content - self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % - (date, dbg_output)) + try: + # Select folder for append and make the box READ-WRITE + imapobj.select(self.getfullname()) + except imapobj.readonly: + # readonly exception. Return original uid to notify that + # we did not save the message. (see savemessage in Base.py) + self.ui.msgtoreadonly(self, uid, content, flags) + return uid - #Do the APPEND - try: - (typ, dat) = imapobj.append(self.getfullname(), + #Do the APPEND + try: + (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) - except Exception, e: - # If the server responds with 'BAD', append() raise()s directly. - # So we need to prepare a response ourselves. - typ, dat = 'BAD', str(e) - if typ != 'OK': #APPEND failed - raise OfflineImapError("Saving msg in folder '%s', repository " - "'%s' failed. Server reponded; %s %s\nMessage content was:" - " %s" % (self, self.getrepository(), typ, dat, dbg_output), - OfflineImapError.ERROR.MESSAGE) + success = True + except imapobj.abort, e: + # connection has been reset, release connection and retry. + self.ui.error(e, exc_info()[2]) + self.imapserver.releaseconnection(imapobj, True) + imapobj = self.imapserver.acquireconnection() + except imapobj.error, e: + # If the server responds with 'BAD', append() raise()s directly. + # So we need to prepare a response ourselves. + typ, dat = 'BAD', str(e) + if typ != 'OK': #APPEND failed + raise OfflineImapError("Saving msg in folder '%s', repository " + "'%s' failed. Server reponded; %s %s\nMessage content was:" + " %s" % (self, self.getrepository(), typ, dat, dbg_output), + OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. (typ,dat) = imapobj.check() From 63b9dcd8964927661966db5ac918ee023865ff6f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 6 Sep 2011 13:27:48 +0200 Subject: [PATCH 220/817] folder.IMAP: Don't import copy It is not needed. list(ALIST) will create a new copy of the list just fine. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 2774136..896ab05 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -21,7 +21,6 @@ import random import binascii import re import time -from copy import copy from Base import BaseFolder from offlineimap import imaputil, imaplibutil, OfflineImapError try: # python 2.6 has set() built in @@ -653,7 +652,7 @@ class IMAPFolder(BaseFolder): # Some IMAP servers do not always return a result. Therefore, # only update the ones that it talks about, and manually fix # the others. - needupdate = copy(uidlist) + needupdate = list(uidlist) for result in r: if result == None: # Compensate for servers that don't return anything from From 06bfff7c04ce74892368a7bda229ad397aa5b921 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 18 Aug 2011 09:08:53 +0200 Subject: [PATCH 221/817] Coalesce SEARCH uid list into sequence set Rather than passing in huge lists of continuous numbers which eventually overflow the maximum command line length, we coalesce number ranges before passing the UID sequence to SEARCH. This should do away with the error that has been reported with busy mailing lists and 'maxage'. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 20940a2..f930bc5 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -143,14 +143,13 @@ class IMAPFolder(BaseFolder): search_condition += "SMALLER " + self.config.getdefault("Account " + self.accountname, "maxsize", -1) search_condition += ")" - searchresult = imapobj.search(None, search_condition) - #result would come back seperated by space - to change into a fetch - #statement we need to change space to comma - messagesToFetch = searchresult[1][0].replace(" ", ",") + res_type, res_data = imapobj.search(None, search_condition) + #result UIDs come back seperated by space + messagesToFetch = imaputil.uid_sequence(res_data.split()) except KeyError: return - if len(messagesToFetch) < 1: + if not messagesToFetch: # No messages; return return else: @@ -172,7 +171,7 @@ class IMAPFolder(BaseFolder): # Now, get the flags and UIDs for these. # We could conceivably get rid of maxmsgid and just say # '1:*' here. - response = imapobj.fetch(messagesToFetch, '(FLAGS UID)')[1] + response = imapobj.fetch("'%s'" % messagesToFetch, '(FLAGS UID)')[1] finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: From 8532a69458f5256ba73be2d46dc5200dd81dcb17 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 18 Aug 2011 09:08:54 +0200 Subject: [PATCH 222/817] Clean up the maxsize maxage code Do away with the wrapping of this code in a try...except KeyError, as this code cannot conceivably throw a KeyError. Even if it could, it should be documented why we should simply return() in this case. Shorten some of the variable names and minor code cleanup while taking the git blame anyway. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 57 ++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index f930bc5..fa2c377 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -107,51 +107,48 @@ class IMAPFolder(BaseFolder): # TODO: Make this so that it can define a date that would be the oldest messages etc. def cachemessagelist(self): - imapobj = self.imapserver.acquireconnection() + maxage = self.config.getdefaultint("Account %s" % self.accountname, + "maxage", -1) + maxsize = self.config.getdefaultint("Account %s" % self.accountname, + "maxsize", -1) self.messagelist = {} + imapobj = self.imapserver.acquireconnection() + try: # Primes untagged_responses imaptype, imapdata = imapobj.select(self.getfullname(), readonly = 1, force = 1) - maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1) - maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1) - if (maxage != -1) | (maxsize != -1): - try: - search_condition = "("; + search_cond = "("; - if(maxage != -1): - #find out what the oldest message is that we should look at - oldest_time_struct = time.gmtime(time.time() - (60*60*24*maxage)) + if(maxage != -1): + #find out what the oldest message is that we should look at + oldest_struct = time.gmtime(time.time() - (60*60*24*maxage)) - #format this manually - otherwise locales could cause problems - monthnames_standard = ["Jan", "Feb", "Mar", "Apr", "May", \ - "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] + #format months manually - otherwise locales cause problems + monthnames = ["Jan", "Feb", "Mar", "Apr", "May", \ + "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] - our_monthname = monthnames_standard[oldest_time_struct[1]-1] - daystr = "%(day)02d" % {'day' : oldest_time_struct[2]} - date_search_str = "SINCE " + daystr + "-" + our_monthname \ - + "-" + str(oldest_time_struct[0]) + month = monthnames[oldest_struct[1]-1] + daystr = "%(day)02d" % {'day' : oldest_struct[2]} - search_condition += date_search_str + search_cond += "SINCE %s-%s-%s" % (daystr, month, + oldest_struct[0]) - if(maxsize != -1): - if(maxage != -1): #There are two conditions - add a space - search_condition += " " + if(maxsize != -1): + if(maxage != -1): # There are two conditions, add space + search_cond += " " + search_cond += "SMALLER %d" % maxsize - search_condition += "SMALLER " + self.config.getdefault("Account " + self.accountname, "maxsize", -1) + search_cond += ")" - search_condition += ")" - - res_type, res_data = imapobj.search(None, search_condition) - #result UIDs come back seperated by space - messagesToFetch = imaputil.uid_sequence(res_data.split()) - except KeyError: - return + res_type, res_data = imapobj.search(None, search_cond) + # Result UIDs seperated by space, coalesce into ranges + messagesToFetch = imaputil.uid_sequence(res_data.split()) if not messagesToFetch: - # No messages; return - return + return # No messages to sync + else: # 1. Some mail servers do not return an EXISTS response # if the folder is empty. 2. ZIMBRA servers can return From f755c8b4231c9ac01eae0d534dac56d4f96ae475 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 18 Aug 2011 09:08:55 +0200 Subject: [PATCH 223/817] Use range 1:* if we want to examine all messages in a folder Some code cleanup. If we want to examine all messages of a folder, don't try to find out how many there are and request a long list of all of them, but simply request 1:*. This obliviates us from the need to force a select even if we already had the folder selected and it requires us to send a few less bytes over the wire. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index fa2c377..8c42419 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -105,7 +105,6 @@ class IMAPFolder(BaseFolder): self.imapserver.releaseconnection(imapobj) return False - # TODO: Make this so that it can define a date that would be the oldest messages etc. def cachemessagelist(self): maxage = self.config.getdefaultint("Account %s" % self.accountname, "maxage", -1) @@ -114,10 +113,11 @@ class IMAPFolder(BaseFolder): self.messagelist = {} imapobj = self.imapserver.acquireconnection() - try: - # Primes untagged_responses - imaptype, imapdata = imapobj.select(self.getfullname(), readonly = 1, force = 1) + res_type, imapdata = imapobj.select(self.getfullname(), True) + + # By default examine all UIDs in this folder + msgsToFetch = '1:*' if (maxage != -1) | (maxsize != -1): search_cond = "("; @@ -149,28 +149,13 @@ class IMAPFolder(BaseFolder): if not messagesToFetch: return # No messages to sync - else: - # 1. Some mail servers do not return an EXISTS response - # if the folder is empty. 2. ZIMBRA servers can return - # multiple EXISTS replies in the form 500, 1000, 1500, - # 1623 so check for potentially multiple replies. - if imapdata == [None]: - return - - maxmsgid = 0 - for msgid in imapdata: - maxmsgid = max(long(msgid), maxmsgid) - if maxmsgid < 1: - #no messages; return - return - messagesToFetch = '1:%d' % maxmsgid; - - # Now, get the flags and UIDs for these. - # We could conceivably get rid of maxmsgid and just say - # '1:*' here. - response = imapobj.fetch("'%s'" % messagesToFetch, '(FLAGS UID)')[1] + # Get the flags and UIDs for these. single-quotes prevent + # imaplib2 from quoting the sequence. + res_type, response = imapobj.fetch("'%s'" % msgsToFetch, + '(FLAGS UID)') finally: self.imapserver.releaseconnection(imapobj) + for messagestr in response: # Discard the message number. messagestr = messagestr.split(' ', 1)[1] From 33029403821a1e862786e559bb04cfb9b97f59d4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 18 Aug 2011 09:08:56 +0200 Subject: [PATCH 224/817] Proper error handling for SEARCH and FETCH failures from the server SEARCH and FETCH were never checking that the IMAP server actually returned OK. Throw OfflineImapErrors at severity FOLDER in case one of them fails. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8c42419..417b80a 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -144,15 +144,28 @@ class IMAPFolder(BaseFolder): search_cond += ")" res_type, res_data = imapobj.search(None, search_cond) - # Result UIDs seperated by space, coalesce into ranges - messagesToFetch = imaputil.uid_sequence(res_data.split()) - if not messagesToFetch: + if res_type != 'OK': + raise OfflineImapError("SEARCH in folder [%s]%s failed. " + "Search string was '%s'. Server responded '[%s] %s'" % ( + self.getrepository(), self, + search_cond, res_type, res_data), + OfflineImapError.ERROR.FOLDER) + + # Result UIDs are seperated by space, coalesce into ranges + msgsToFetch = imaputil.uid_sequence(res_data.split()) + if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. res_type, response = imapobj.fetch("'%s'" % msgsToFetch, '(FLAGS UID)') + if res_type != 'OK': + raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " + "Server responded '[%s] %s'" % ( + self.getrepository(), self, + res_type, response), + OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) From fb8017991c3de0fd3111f47338e4648e1abb586d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 22 Aug 2011 12:21:11 +0200 Subject: [PATCH 225/817] Rework undocumented listjoin to create UID sequences This function was badly named and completely undocumented. Rework it to avoid copying the full UID list using an iterator. Make it possible to hand it a list of UIDs as strings rather than implicitely relying on the fact that they are numeric already. Document the code. The behavior off the function itself remained otherwise unchanged. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Gmail.py | 2 +- offlineimap/folder/IMAP.py | 2 +- offlineimap/imaputil.py | 44 ++++++++++++++++++------------------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 7cbff4f..587a00f 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -55,7 +55,7 @@ class GmailFolder(IMAPFolder): try: imapobj.select(self.getfullname()) result = imapobj.uid('copy', - imaputil.listjoin(uidlist), + imaputil.uid_sequence(uidlist), self.trash_folder) assert result[0] == 'OK', \ "Bad IMAPlib result: %s" % result[0] diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 417b80a..80f144d 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -626,7 +626,7 @@ class IMAPFolder(BaseFolder): self.ui.flagstoreadonly(self, uidlist, flags) return r = imapobj.uid('store', - imaputil.listjoin(uidlist), + imaputil.uid_sequence(uidlist), operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 3905851..b336876 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -168,35 +168,33 @@ def flagsmaildir2imap(maildirflaglist): retval.sort() return '(' + ' '.join(retval) + ')' -def listjoin(list): - start = None - end = None - retval = [] +def uid_sequence(uidlist): + """Collapse UID lists into shorter sequence sets - def getlist(start, end): + [1,2,3,4,5,10,12,13] will return "1:5,10,12:13". This function does + not sort the list, and only collapses if subsequent entries form a + range. + :returns: The collapsed UID list as string""" + def getrange(start, end): if start == end: return(str(start)) - else: - return(str(start) + ":" + str(end)) - + return "%s:%s" % (start, end) - for item in list: - if start == None: - # First item. - start = item - end = item - elif item == end + 1: - # An addition to the list. - end = item - else: - # Here on: starting a new list. - retval.append(getlist(start, end)) - start = item - end = item + if not len(uidlist): return '' # Empty list, return + start, end = None, None + retval = [] - if start != None: - retval.append(getlist(start, end)) + for item in iter(uidlist): + item = int(item) + if start == None: # First item + start, end = item, item + elif item == end + 1: # Next item in a range + end = item + else: # Starting a new range + retval.append(getrange(start, end)) + start, end = item, item + retval.append(getrange(start, end)) # Add final range/item return ",".join(retval) From 1917be8e83da4c4a5abb83874405a2795c176b60 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 22 Aug 2011 12:21:12 +0200 Subject: [PATCH 226/817] Shorten list of messages to be deleted in UI output Rather than output the full list of messages, coalesce it into number ranges wherever possible. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 16e10bc..f5dca57 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -302,7 +302,7 @@ class UIBase: ds = s.folderlist(destlist) s._msg("Deleting %d messages (%s) in %s" % \ (len(uidlist), - ", ".join([str(u) for u in uidlist]), + offlineimap.imaputil.uid_sequence(uidlist), ds)) def addingflags(s, uidlist, flags, dest): From 971ed3adacc037e3e1899419a18fa2aeedf79e01 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 6 Sep 2011 13:19:25 +0200 Subject: [PATCH 227/817] Allow Imapserver.releaseconnection() to drop a connection If a connection is broken, we want to have it really dropped and not be reused. So far, we are checking the .Terminate attribute for this, but according to the imaplib2 author, it is only set on normal shutdown and it is an undocumented attribute whose meaning could change any time. This patch introduces the parameter drop_conn which allows to tell releaseconnection() that we really want to connection being dropped from the pool of available connections and properly destroy it. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 707464d..40bbb8d 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -109,12 +109,15 @@ class IMAPServer: return self.root - def releaseconnection(self, connection): - """Releases a connection, returning it to the pool.""" + def releaseconnection(self, connection, drop_conn=False): + """Releases a connection, returning it to the pool. + + :param drop_conn: If True, the connection will be released and + not be reused. This can be used to indicate broken connections.""" self.connectionlock.acquire() self.assignedconnections.remove(connection) # Don't reuse broken connections - if connection.Terminate: + if connection.Terminate or drop_conn: connection.logout() else: self.availableconnections.append(connection) From 4c558c1b69bf16d399631c6be28ceb9d386c28f9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 6 Sep 2011 13:19:26 +0200 Subject: [PATCH 228/817] Error proof IMAP.APPEND against dropped connections Make sure that when a connection is dropped during append, we really discard the broken connection and get a new one, retrying. We retry indefinitely on the specific abort() Exception, as this is what imaplib2 suggests us to do. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 88 +++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 80f144d..8049eb1 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -21,6 +21,7 @@ import random import binascii import re import time +from sys import exc_info from copy import copy from Base import BaseFolder from offlineimap import imaputil, imaplibutil, OfflineImapError @@ -483,53 +484,62 @@ class IMAPFolder(BaseFolder): self.savemessageflags(uid, flags) return uid + imapobj = self.imapserver.acquireconnection() try: - imapobj = self.imapserver.acquireconnection() + success = False # succeeded in APPENDING? + while not success: - try: - imapobj.select(self.getfullname()) # Needed for search and making the box READ-WRITE - except imapobj.readonly: - # readonly exception. Return original uid to notify that - # we did not save the message. (see savemessage in Base.py) - self.ui.msgtoreadonly(self, uid, content, flags) - return uid + # UIDPLUS extension provides us with an APPENDUID response. + use_uidplus = 'UIDPLUS' in imapobj.capabilities - # UIDPLUS extension provides us with an APPENDUID response to our append() - use_uidplus = 'UIDPLUS' in imapobj.capabilities + # get the date of the message, so we can pass it to the server. + date = self.getmessageinternaldate(content, rtime) + content = re.sub("(?200: + dbg_output = "%s...%s" % (content[:150], content[-50:]) + else: + dbg_output = content + self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % + (date, dbg_output)) - if not use_uidplus: - # insert a random unique header that we can fetch later - (headername, headervalue) = self.generate_randomheader(content) - self.ui.debug('imap', 'savemessage: new header is: %s: %s' % \ - (headername, headervalue)) - content = self.savemessage_addheader(content, headername, - headervalue) - if len(content)>200: - dbg_output = "%s...%s" % (content[:150], - content[-50:]) - else: - dbg_output = content - self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % - (date, dbg_output)) + try: + # Select folder for append and make the box READ-WRITE + imapobj.select(self.getfullname()) + except imapobj.readonly: + # readonly exception. Return original uid to notify that + # we did not save the message. (see savemessage in Base.py) + self.ui.msgtoreadonly(self, uid, content, flags) + return uid - #Do the APPEND - try: - (typ, dat) = imapobj.append(self.getfullname(), + #Do the APPEND + try: + (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) - except Exception, e: - # If the server responds with 'BAD', append() raise()s directly. - # So we need to prepare a response ourselves. - typ, dat = 'BAD', str(e) - if typ != 'OK': #APPEND failed - raise OfflineImapError("Saving msg in folder '%s', repository " - "'%s' failed. Server reponded; %s %s\nMessage content was:" - " %s" % (self, self.getrepository(), typ, dat, dbg_output), - OfflineImapError.ERROR.MESSAGE) + success = True + except imapobj.abort, e: + # connection has been reset, release connection and retry. + self.ui.error(e, exc_info()[2]) + self.imapserver.releaseconnection(imapobj, True) + imapobj = self.imapserver.acquireconnection() + except imapobj.error, e: + # If the server responds with 'BAD', append() raise()s directly. + # So we need to prepare a response ourselves. + typ, dat = 'BAD', str(e) + if typ != 'OK': #APPEND failed + raise OfflineImapError("Saving msg in folder '%s', repository " + "'%s' failed. Server reponded; %s %s\nMessage content was:" + " %s" % (self, self.getrepository(), typ, dat, dbg_output), + OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. (typ,dat) = imapobj.check() From 26721c60d496c0782dabe0676ddcc8be4d8f97be Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 7 Sep 2011 15:23:28 +0200 Subject: [PATCH 229/817] Don't cache empty IMAP folders If a folder is empty, most servers will return EXISTS 0 and imaplib2 passes back ['0'] as return value to a select(). It returns [None] if no EXISTS response was given by the server at all. Attempting to fetch the UIDs of 0 emails which leads to various error messages (One server responds with "NO No matching messages", Gmail seems to say "BAD Bad message sequence 1:*" for some (although it is working fine for me with Gmail, so it might behave different for different people). In case we get an None or 0 back, we simply stop caching messages as the folder is empty. This should fix the various error reports that have popped up. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 5a14776..d860e85 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -120,7 +120,9 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: res_type, imapdata = imapobj.select(self.getfullname(), True) - + if imapdata == [None] or imapdata[0] == '0': + # Empty folder, no need to populate message list + return # By default examine all UIDs in this folder msgsToFetch = '1:*' From 1b99c019e53c4d95319ade03f0948657fdc968b8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 7 Sep 2011 18:21:46 +0200 Subject: [PATCH 230/817] Fix handling the search results Results are delivered in a 1-element list, and somehow I managed to drop a [0] in the previous patches. We need to look at the element of course, or our string splitting will fail horribly. Sorry this somehow slipped through. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index d860e85..cc04262 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -159,7 +159,7 @@ class IMAPFolder(BaseFolder): OfflineImapError.ERROR.FOLDER) # Result UIDs are seperated by space, coalesce into ranges - msgsToFetch = imaputil.uid_sequence(res_data.split()) + msgsToFetch = imaputil.uid_sequence(res_data[0].split()) if not msgsToFetch: return # No messages to sync From 135f8c45cfb24cc202d2589bfb13fdbaedbfd5f1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 7 Sep 2011 18:21:44 +0200 Subject: [PATCH 231/817] Simplify constructing the SEARCH date We can use Imaplib's monthnames and shorten the construction of the date by using them rather than hardcoding them again. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index cc04262..339f271 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -24,6 +24,7 @@ import time from sys import exc_info from Base import BaseFolder from offlineimap import imaputil, imaplibutil, OfflineImapError +from offlineimap.imaplib2 import MonthNames try: # python 2.6 has set() built in set except NameError: @@ -132,16 +133,10 @@ class IMAPFolder(BaseFolder): if(maxage != -1): #find out what the oldest message is that we should look at oldest_struct = time.gmtime(time.time() - (60*60*24*maxage)) - - #format months manually - otherwise locales cause problems - monthnames = ["Jan", "Feb", "Mar", "Apr", "May", \ - "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] - - month = monthnames[oldest_struct[1]-1] - daystr = "%(day)02d" % {'day' : oldest_struct[2]} - - search_cond += "SINCE %s-%s-%s" % (daystr, month, - oldest_struct[0]) + search_cond += "SINCE %02d-%s-%d" % ( + oldest_struct[2], + MonthNames[oldest_struct[1]], + oldest_struct[0]) if(maxsize != -1): if(maxage != -1): # There are two conditions, add space From 5cbec30b3ea2ef8abd8555630842ff4ebdbc5e03 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 7 Sep 2011 18:21:45 +0200 Subject: [PATCH 232/817] Sanity check for maxage setting If maxage is set too large, we would even SEARCH for negative years. With devastating results. So implement some sanity check and err out in case the year does not make sense. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 339f271..f3d1ad8 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -133,6 +133,10 @@ class IMAPFolder(BaseFolder): if(maxage != -1): #find out what the oldest message is that we should look at oldest_struct = time.gmtime(time.time() - (60*60*24*maxage)) + if oldest_struct[0] < 1900: + raise OfflineImapError("maxage setting led to year %d. " + "Abort syncing." % oldest_struct[0], + OfflineImapError.ERROR.REPO) search_cond += "SINCE %02d-%s-%d" % ( oldest_struct[2], MonthNames[oldest_struct[1]], From 8800fa37a3f75506fb7ab57d70ea3579111ff5ce Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 09:50:41 +0200 Subject: [PATCH 233/817] Implement Server SSL fingerprint check If we connect to a SSL server (not STARTTLS) and no CA cert has been specified for verification, we check the configured SSL fingerprint and bail out in case it has not been set yet, or it does not match. This means one more mandatory option for SSL configuration, but it improves security a lot. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 4 ++++ offlineimap.conf | 10 ++++++++++ offlineimap/imaplibutil.py | 22 ++++++++++++++++++++-- offlineimap/imapserver.py | 2 ++ offlineimap/repository/IMAP.py | 3 +++ 5 files changed, 39 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 74b6368..edd0f75 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -17,6 +17,10 @@ New Features synchronization, but only skip that message, informing the user at the end of the sync run. +* If you connect via ssl and 'cert_fingerprint' is configured, we check + that the server certificate is actually known and identical by + comparing the stored sha1 fingerprint with the current one. + Changes ------- diff --git a/offlineimap.conf b/offlineimap.conf index 12a09ba..2d5532d 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -327,6 +327,16 @@ ssl = yes # The certificate should be in PEM format. # sslcacertfile = /path/to/cacertfile.crt +# If you connect via SSL/TLS (ssl=true) and you have no CA certificate +# specified, offlineimap will refuse to sync as it connects to a server +# with an unknown "fingerprint". If you are sure you connect to the +# correct server, you can then configure the presented server +# fingerprint here. OfflineImap will verify that the server fingerprint +# has not changed on each connect and refuse to connect otherwise. +# You can also configure this in addition to CA certificate validation +# above and it will check both ways. cert_fingerprint = +# + # Specify the port. If not specified, use a default port. # remoteport = 993 diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 529af6f..ca556f0 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -21,8 +21,10 @@ import re import socket import time import subprocess -from offlineimap.ui import getglobalui import threading +from hashlib import sha1 + +from offlineimap.ui import getglobalui from offlineimap import OfflineImapError from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, IMAP4_PORT, InternalDate, Mon2num @@ -137,7 +139,23 @@ def new_mesg(self, s, tn=None, secs=None): class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): """Improved version of imaplib.IMAP4_SSL overriding select()""" - pass + def __init__(self, *args, **kwargs): + self._fingerprint = kwargs.get('fingerprint', None) + if kwargs.has_key('fingerprint'): + del kwargs['fingerprint'] + super(WrappedIMAP4_SSL, self).__init__(*args, **kwargs) + + def open(self, host=None, port=None): + super(WrappedIMAP4_SSL, self).open(host, port) + if self._fingerprint or not self.ca_certs: + # compare fingerprints + fingerprint = sha1(self.sslobj.getpeercert(True)).hexdigest() + if fingerprint != self._fingerprint: + raise OfflineImapError("Server SSL fingerprint '%s' for hostnam" + "e '%s' does not match configured fingerprint. Please ver" + "ify and set 'cert_fingerprint' accordingly if not set ye" + "t." % (fingerprint, host), + OfflineImapError.ERROR.REPO) class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 05d8111..5989ff1 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -209,6 +209,7 @@ class IMAPServer: success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) + fingerprint = self.repos.get_ssl_fingerprint() imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, self.port, self.sslclientkey, @@ -216,6 +217,7 @@ class IMAPServer: self.sslcacertfile, self.verifycert, timeout=socket.getdefaulttimeout(), + fingerprint=fingerprint ) else: self.ui.connecting(self.hostname, self.port) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 68eb637..76d0870 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -182,6 +182,9 @@ class IMAPRepository(BaseRepository): % (self.name, cacertfile)) return cacertfile + def get_ssl_fingerprint(self): + return self.getconf('cert_fingerprint', None) + def getpreauthtunnel(self): return self.getconf('preauthtunnel', None) From eafea0c8802aaef28588fb693ae2d81f00387ffc Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 09:50:42 +0200 Subject: [PATCH 234/817] IMAP4.sslobj -> sock imaplib2 has changed internally to use self.sock for its ssl socket when it used to be sslobj. Reflect that change. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index ca556f0..609fd4d 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -149,7 +149,7 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): super(WrappedIMAP4_SSL, self).open(host, port) if self._fingerprint or not self.ca_certs: # compare fingerprints - fingerprint = sha1(self.sslobj.getpeercert(True)).hexdigest() + fingerprint = sha1(self.sock.getpeercert(True)).hexdigest() if fingerprint != self._fingerprint: raise OfflineImapError("Server SSL fingerprint '%s' for hostnam" "e '%s' does not match configured fingerprint. Please ver" From bc10e056004764e819318fd52e687490e4fc964e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 09:50:43 +0200 Subject: [PATCH 235/817] Check for SSL module existence and only do fingerprint check then Python 2.5 has no ssl module, and we can therefor not get the server certificate for fingerprint verification. Add a check that disables fingerprint verification for python 2.5. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 609fd4d..5c94592 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -147,7 +147,8 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): def open(self, host=None, port=None): super(WrappedIMAP4_SSL, self).open(host, port) - if self._fingerprint or not self.ca_certs: + if (self._fingerprint or not self.ca_certs) and\ + 'ssl' in locals(): # <--disable for python 2.5 # compare fingerprints fingerprint = sha1(self.sock.getpeercert(True)).hexdigest() if fingerprint != self._fingerprint: From f369961a87d2476066447d3dcdd41d0f515bafd2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 10:37:53 +0200 Subject: [PATCH 236/817] IMAP IDLE cleanup(1): Move idle callback out of loop Don't redefine the idle callback function on every run in the while loop, define it once when we enter the function. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 05d8111..a07cdc8 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -493,21 +493,22 @@ class IdleThread(object): ui.unregisterthread(currentThread()) def idle(self): + def callback(args): + result, cb_arg, exc_data = args + if exc_data is None: + if not self.event.isSet(): + self.needsync = True + self.event.set() + else: + # We got an "abort" signal. + self.imapaborted = True + self.stop() + while True: if self.event.isSet(): return self.needsync = False self.imapaborted = False - def callback(args): - result, cb_arg, exc_data = args - if exc_data is None: - if not self.event.isSet(): - self.needsync = True - self.event.set() - else: - # We got an "abort" signal. - self.imapaborted = True - self.stop() success = False # successfully selected FOLDER? while not success: From 0bebd65ba00c9d77aedcb354b6513168406bdb24 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 10:37:54 +0200 Subject: [PATCH 237/817] IMAP IDLE cleanup(2): Add code documentation Add code documentation throughout the idle() function. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index a07cdc8..20424f2 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -456,6 +456,9 @@ class IMAPServer: class IdleThread(object): def __init__(self, parent, folder=None): + """If invoked without 'folder', perform a NOOP and wait for + self.stop() to be called. If invoked with folder, switch to IDLE + mode and synchronize once we have a new message""" self.parent = parent self.folder = folder self.event = Event() @@ -493,7 +496,14 @@ class IdleThread(object): ui.unregisterthread(currentThread()) def idle(self): + """Invoke IDLE mode until timeout or self.stop() is invoked""" def callback(args): + """IDLE callback function invoked by imaplib2 + + This is invoked when a) The IMAP server tells us something + while in IDLE mode, b) we get an Exception (e.g. on dropped + connections, or c) the standard imaplib IDLE timeout of 29 + minutes kicks in.""" result, cb_arg, exc_data = args if exc_data is None: if not self.event.isSet(): @@ -527,10 +537,8 @@ class IdleThread(object): if "IDLE" in imapobj.capabilities: imapobj.idle(callback=callback) else: - self.ui.warn("IMAP IDLE not supported on connection to %s." - "Falling back to old behavior: sleeping until next" - "refresh cycle." - %(imapobj.identifier,)) + self.ui.warn("IMAP IDLE not supported on server '%s'." + "Sleep until next refresh cycle." % imapobj.identifier) imapobj.noop() self.event.wait() if self.event.isSet(): @@ -541,5 +549,7 @@ class IdleThread(object): # of the loop next time around. self.parent.releaseconnection(imapobj) if self.needsync: + # here not via self.stop, but because IDLE responded. Do + # another round and invoke actual syncing. self.event.clear() self.dosync() From 6ad0de08ef66401426c03a58e6e5636f72ec7799 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 10:37:55 +0200 Subject: [PATCH 238/817] IMAP IDLE cleanup(3): Rename self.event to self.stop_sig Variable name 'event' is as bad as it gets. Rename it to something that actually describes what it is about. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 20424f2..df2318c 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -461,7 +461,7 @@ class IdleThread(object): mode and synchronize once we have a new message""" self.parent = parent self.folder = folder - self.event = Event() + self.stop_sig = Event() self.ui = getglobalui() if folder is None: self.thread = Thread(target=self.noop) @@ -473,7 +473,7 @@ class IdleThread(object): self.thread.start() def stop(self): - self.event.set() + self.stop_sig.set() def join(self): self.thread.join() @@ -481,7 +481,7 @@ class IdleThread(object): def noop(self): imapobj = self.parent.acquireconnection() imapobj.noop() - self.event.wait() + self.stop_sig.wait() self.parent.releaseconnection(imapobj) def dosync(self): @@ -506,16 +506,16 @@ class IdleThread(object): minutes kicks in.""" result, cb_arg, exc_data = args if exc_data is None: - if not self.event.isSet(): + if not self.stop_sig.isSet(): self.needsync = True - self.event.set() + self.stop_sig.set() else: # We got an "abort" signal. self.imapaborted = True self.stop() while True: - if self.event.isSet(): + if self.stop_sig.isSet(): return self.needsync = False self.imapaborted = False @@ -540,8 +540,8 @@ class IdleThread(object): self.ui.warn("IMAP IDLE not supported on server '%s'." "Sleep until next refresh cycle." % imapobj.identifier) imapobj.noop() - self.event.wait() - if self.event.isSet(): + self.stop_sig.wait() # self.stop() or IDLE callback are invoked + if self.stop_sig.isSet(): # Can't NOOP on a bad connection. if not self.imapaborted: imapobj.noop() @@ -551,5 +551,5 @@ class IdleThread(object): if self.needsync: # here not via self.stop, but because IDLE responded. Do # another round and invoke actual syncing. - self.event.clear() + self.stop_sig.clear() self.dosync() From 59753fc06f083504cd671235bcf37731a1c6870d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 10:37:56 +0200 Subject: [PATCH 239/817] IMAP IDLE cleanup(4): Simplify code while True: if a: return is equivalent to while not a: Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index df2318c..f4fc7fa 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -514,9 +514,7 @@ class IdleThread(object): self.imapaborted = True self.stop() - while True: - if self.stop_sig.isSet(): - return + while not self.stop_sig.isSet(): self.needsync = False self.imapaborted = False From ce8471a0111041d145137d5bff98a0c78349347b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 10:37:57 +0200 Subject: [PATCH 240/817] IMAP IDLE cleanup(5): Really discard connections when they are dropped Finally, actually discard dropped connections when we detect them as an imapobj.abort() has been thrown. In this case, invoke releaseconnection with drop_conn=True. We don't need the self.aborted attribute to get signified of dropped connections. An Execption during the noop will do. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index f4fc7fa..b9b30a6 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -505,18 +505,13 @@ class IdleThread(object): connections, or c) the standard imaplib IDLE timeout of 29 minutes kicks in.""" result, cb_arg, exc_data = args - if exc_data is None: - if not self.stop_sig.isSet(): - self.needsync = True - self.stop_sig.set() - else: - # We got an "abort" signal. - self.imapaborted = True - self.stop() + if exc_data is None and not self.stop_sig.isSet(): + # No Exception, and we are not supposed to stop: + self.needsync = True + self.stop_sig.set() # continue to sync while not self.stop_sig.isSet(): self.needsync = False - self.imapaborted = False success = False # successfully selected FOLDER? while not success: @@ -527,7 +522,7 @@ class IdleThread(object): if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: # Connection closed, release connection and retry self.ui.error(e, exc_info()[2]) - self.parent.releaseconnection(imapobj) + self.parent.releaseconnection(imapobj, True) else: raise e else: @@ -539,13 +534,16 @@ class IdleThread(object): "Sleep until next refresh cycle." % imapobj.identifier) imapobj.noop() self.stop_sig.wait() # self.stop() or IDLE callback are invoked - if self.stop_sig.isSet(): - # Can't NOOP on a bad connection. - if not self.imapaborted: - imapobj.noop() - # We don't do event.clear() so that we'll fall out - # of the loop next time around. - self.parent.releaseconnection(imapobj) + try: + # End IDLE mode with noop, imapobj can point to a dropped conn. + imapobj.noop() + except imapobj.abort(): + self.ui.warn('Attempting NOOP on dropped connection %s' % \ + imapobj.identifier) + self.parent.releaseconnection(imapobj, True) + else: + self.parent.releaseconnection(imapobj) + if self.needsync: # here not via self.stop, but because IDLE responded. Do # another round and invoke actual syncing. From 1a4b7c337c6321cf6f41d3801f7aba631b6b4d6d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 12 Sep 2011 20:10:44 +0200 Subject: [PATCH 241/817] v6.3.5-rc1 Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 20 -------------------- Changelog.rst | 40 ++++++++++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index edd0f75..9063d5e 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,32 +13,12 @@ others. New Features ------------ -* When a message upload/download fails, we do not abort the whole folder - synchronization, but only skip that message, informing the user at the - end of the sync run. - -* If you connect via ssl and 'cert_fingerprint' is configured, we check - that the server certificate is actually known and identical by - comparing the stored sha1 fingerprint with the current one. - Changes ------- -* Refactor our IMAPServer class. Background work without user-visible - changes. -* Remove the configurability of the Blinkenlights statuschar. It - cluttered the main configuration file for little gain. -* Updated bundled imaplib2 to version 2.28. - Bug Fixes --------- -* We protect more robustly against asking for inexistent messages from the - IMAP server, when someone else deletes or moves messages while we sync. -* Selecting inexistent folders specified in folderincludes now throws - nice errors and continues to sync with all other folders rather than - exiting offlineimap with a traceback. - Pending for the next major release ================================== diff --git a/Changelog.rst b/Changelog.rst index 95a13b8..4bdff19 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,45 @@ ChangeLog releases announces. +OfflineIMAP v6.3.5-rc1 (2011-09-12) +=================================== + +Notes +----- + +Idle feature and SQLite backend leave the experimental stage! ,-) + +New Features +------------ + +* When a message upload/download fails, we do not abort the whole folder + synchronization, but only skip that message, informing the user at the + end of the sync run. + +* If you connect via ssl and 'cert_fingerprint' is configured, we check + that the server certificate is actually known and identical by + comparing the stored sha1 fingerprint with the current one. + +Changes +------- + +* Refactor our IMAPServer class. Background work without user-visible + changes. +* Remove the configurability of the Blinkenlights statuschar. It + cluttered the main configuration file for little gain. +* Updated bundled imaplib2 to version 2.28. + +Bug Fixes +--------- + +* We protect more robustly against asking for inexistent messages from the + IMAP server, when someone else deletes or moves messages while we sync. +* Selecting inexistent folders specified in folderincludes now throws + nice errors and continues to sync with all other folders rather than + exiting offlineimap with a traceback. + + + OfflineIMAP v6.3.4 (2011-08-10) =============================== @@ -26,6 +65,7 @@ Changes * Handle when UID can't be found on saved messages. + OfflineIMAP v6.3.4-rc4 (2011-07-27) =================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 5070ac7..4eeb648 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.4" +__version__ = "6.3.5-rc1" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 24db42916c418b336dec6310ce8cae1d9a1ea4cb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 13 Sep 2011 16:27:54 +0200 Subject: [PATCH 242/817] IMAP savemessage(): Don't loop indefinitely on failure We were retrying indefinitely on imapobj.abort() (as that is what imaplib2 suggests), but if the failure occurs repeatedly, we'll never quit this loop. So implement a counter that errs out after unsuccessful retries. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index f3d1ad8..67603dd 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -495,11 +495,10 @@ class IMAPFolder(BaseFolder): self.savemessageflags(uid, flags) return uid + retry_left = 2 # succeeded in APPENDING? imapobj = self.imapserver.acquireconnection() try: - success = False # succeeded in APPENDING? - while not success: - + while retry_left: # UIDPLUS extension provides us with an APPENDUID response. use_uidplus = 'UIDPLUS' in imapobj.capabilities @@ -536,12 +535,20 @@ class IMAPFolder(BaseFolder): (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) - success = True + retry_left = 0 # Mark as success except imapobj.abort, e: # connection has been reset, release connection and retry. self.ui.error(e, exc_info()[2]) self.imapserver.releaseconnection(imapobj, True) imapobj = self.imapserver.acquireconnection() + if not retry_left: + raise OfflineImapError("Saving msg in folder '%s', " + "repository '%s' failed. Server reponded; %s %s\n" + "Message content was: %s" % + (self, self.getrepository(), + typ, dat, dbg_output), + OfflineImapError.ERROR.MESSAGE) + retry_left -= 1 except imapobj.error, e: # If the server responds with 'BAD', append() raise()s directly. # So we need to prepare a response ourselves. From 7c83d505f8707192f81a6e74c272a668eb62c1c7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 13 Sep 2011 15:17:55 +0200 Subject: [PATCH 243/817] IMAP cachefolder: Fix returning None on select We rely on the number of mails being returned by the imapobj.select() call, however that only happens if we "force" a real select() to occur. Pass in the force parameter that I dropped earlier (we did not make use of the return value when I dropped it, that is how it slipped through). Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 67603dd..4dcae7d 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -120,7 +120,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: - res_type, imapdata = imapobj.select(self.getfullname(), True) + res_type, imapdata = imapobj.select(self.getfullname(), True, True) if imapdata == [None] or imapdata[0] == '0': # Empty folder, no need to populate message list return From fe5714022471ed221fc39a7e0784ece344bfc93e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 12:06:06 +0200 Subject: [PATCH 244/817] Fix repository 'readonly' configuration The readonly feature was introduced to safeguard repositories from accidental modifications. Unfortunately, my patch treated the readonly setting as a string and not as a boolean, so if *anything* was set in the configuration file as 'readonly', this value evaluated to True Fortunately this was safe, we never treated a repository that we wanted read-only as read-write. We always treated them readonly if something was configured as "readonly=..." even if that was False. The fix is simply to use getconfboolean() rather than getconf() which checks for True/False/On/Off/yes/no/1/0 and hands back the correct boolean. Reported-by: Tomasz Nowak Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 31ea2b7..428cbd8 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -231,7 +231,7 @@ class SyncableAccount(Account): localrepos = self.localrepos statusrepos = self.statusrepos # replicate the folderstructure from REMOTE to LOCAL - if not localrepos.getconf('readonly', False): + if not localrepos.getconfboolean('readonly', False): self.ui.syncfolders(remoterepos, localrepos) remoterepos.syncfoldersto(localrepos, statusrepos) @@ -345,7 +345,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, remotefolder.getmessagecount()) # Synchronize remote changes. - if not localrepos.getconf('readonly', False): + if not localrepos.getconfboolean('readonly', False): ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) remotefolder.syncmessagesto(localfolder, statusfolder) else: @@ -353,7 +353,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, % localrepos.getname()) # Synchronize local changes - if not remoterepos.getconf('readonly', False): + if not remoterepos.getconfboolean('readonly', False): ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) localfolder.syncmessagesto(remotefolder, statusfolder) else: From d2af21d57d3241b1888f8f7d7c2c878e57da7ee3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 15:45:09 +0200 Subject: [PATCH 245/817] Simplify testing for 'subscribedonly' setting We only explicitly tested for 'yes' when we have a nice function to get boolean settings which also works with Treu/False/NO, etc... Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/IMAP.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 76d0870..daa31c9 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -280,8 +280,7 @@ class IMAPRepository(BaseRepository): imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list - if self.config.has_option(self.getsection(), 'subscribedonly'): - if self.getconf('subscribedonly') == "yes": + if self.getconfboolean('subscribedonly', False): listfunction = imapobj.lsub try: listresult = listfunction(directory = self.imapserver.reference)[1] From c1a2b1559de9be285ecb7254ea755d19693cd363 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 15:06:30 +0200 Subject: [PATCH 246/817] repository: Simplify restore_atime code repository.BaseRepository().restore_atime() was testing in complex ways that it only operates on a Maildir and that the 'restoreatime' setting is set. This is unecessary, we can simply make the base implementation a NoOp, and move the implementation to MaildirRepository(). This will save a tad of work for everyone doing IMAP<->IMAP synchronization and simplify the code. Also document the functions. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 14 +++++--------- offlineimap/repository/Maildir.py | 10 +++++++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 184bc9a..72a5f88 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -39,17 +39,13 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): if not os.path.exists(self.uiddir): os.mkdir(self.uiddir, 0700) - # The 'restoreatime' config parameter only applies to local Maildir - # mailboxes. def restore_atime(self): - if self.config.get('Repository ' + self.name, 'type').strip() != \ - 'Maildir': - return + """Sets folders' atime back to their values after a sync - if not self.config.has_option('Repository ' + self.name, 'restoreatime') or not self.config.getboolean('Repository ' + self.name, 'restoreatime'): - return - - return self.restore_folder_atimes() + Controlled by the 'restoreatime' config parameter (default + False), applies only to local Maildir mailboxes and does nothing + on all other repository types.""" + pass def connect(self): """Establish a connection to the remote, if necessary. This exists diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index ef3a723..75e0fe6 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -39,15 +39,19 @@ class MaildirRepository(BaseRepository): os.mkdir(self.root, 0700) def _append_folder_atimes(self, foldername): + """Store the atimes of a folder's new|cur in self.folder_atimes""" p = os.path.join(self.root, foldername) new = os.path.join(p, 'new') cur = os.path.join(p, 'cur') f = p, os.stat(new)[ST_ATIME], os.stat(cur)[ST_ATIME] self.folder_atimes.append(f) - def restore_folder_atimes(self): - if not self.folder_atimes: - return + def restore_atime(self): + """Sets folders' atime back to their values after a sync + + Controlled by the 'restoreatime' config parameter.""" + if not self.getconfboolean('restoreatime', False): + return # not configured for f in self.folder_atimes: t = f[1], os.stat(os.path.join(f[0], 'new'))[ST_MTIME] From ee1706fa90e514f40d27d5d950b211bd4725b079 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 15:06:32 +0200 Subject: [PATCH 247/817] documentation: Clarify restoreatime setting Better document what this actually does and that most people won't be needing it. (Especially as mount setting such as relatime|noatime now reduce the amount of atime changes anyway) Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- Changelog.draft.rst | 2 ++ offlineimap.conf | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 9063d5e..e4b8055 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,8 @@ New Features Changes ------- +* Documentation improvements concerning 'restoreatime' and some code cleanup + Bug Fixes --------- diff --git a/offlineimap.conf b/offlineimap.conf index 2d5532d..224e933 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -285,12 +285,12 @@ localfolders = ~/Test sep = . -# Some users on *nix platforms may not want the atime (last access -# time) to be modified by OfflineIMAP. In these cases, they would -# want to set restoreatime to yes. OfflineIMAP will make an effort -# to not touch the atime if you do that. +# Some users may not want the atime (last access time) of folders to be +# modified by OfflineIMAP. If 'restoreatime' is set to yes, OfflineIMAP +# will restore the atime of the "new" and "cur" folders in each maildir +# folder to their original value after each sync. # -# In most cases, the default of no should be sufficient. +# In nearly all cases, the default should be fine. restoreatime = no From 5bcfbc152593feb4d6a134b706ef3cf1f6314e72 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 15:06:31 +0200 Subject: [PATCH 248/817] MaildirRepository: Beautify restore_atime code Beauty of code is probably a subjective measure, but this patch hopefully is an improvement over the previous incarnation without changing functionality. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Maildir.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 75e0fe6..aeb9776 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -43,21 +43,21 @@ class MaildirRepository(BaseRepository): p = os.path.join(self.root, foldername) new = os.path.join(p, 'new') cur = os.path.join(p, 'cur') - f = p, os.stat(new)[ST_ATIME], os.stat(cur)[ST_ATIME] - self.folder_atimes.append(f) + atimes = (p, os.path.getatime(new), os.path.getatime(cur)) + self.folder_atimes.append(atimes) def restore_atime(self): """Sets folders' atime back to their values after a sync Controlled by the 'restoreatime' config parameter.""" if not self.getconfboolean('restoreatime', False): - return # not configured + return # not configured to restore - for f in self.folder_atimes: - t = f[1], os.stat(os.path.join(f[0], 'new'))[ST_MTIME] - os.utime(os.path.join(f[0], 'new'), t) - t = f[2], os.stat(os.path.join(f[0], 'cur'))[ST_MTIME] - os.utime(os.path.join(f[0], 'cur'), t) + for (dirpath, new_atime, cur_atime) in self.folder_atimes: + new_dir = os.path.join(dirpath, 'new') + cur_dir = os.path.join(dirpath, 'cur') + os.utime(new_dir, (new_atime, os.path.getmtime(new_dir))) + os.utime(cur_dir, (cur_atime, os.path.getmtime(cur_dir))) def getlocalroot(self): return os.path.expanduser(self.getconf('localfolders')) From fe4f385e2c38676ccdb635546756b38b6a7aae79 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 10:26:42 +0200 Subject: [PATCH 249/817] folder.IMAP: Improve dropped connection handling in quickchanged() The quickchanged() function was not handling dropped connections yet. If IMAP4.select() throws a FOLDER_RETRY error, we will now discard the connection, reconnect and retry. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 50 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 4dcae7d..25ebe38 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -89,27 +89,35 @@ class IMAPFolder(BaseFolder): # An IMAP folder has definitely changed if the number of # messages or the UID of the last message have changed. Otherwise # only flag changes could have occurred. - imapobj = self.imapserver.acquireconnection() - try: - # Primes untagged_responses - imaptype, imapdata = imapobj.select(self.getfullname(), readonly = 1, force = 1) - # 1. Some mail servers do not return an EXISTS response - # if the folder is empty. 2. ZIMBRA servers can return - # multiple EXISTS replies in the form 500, 1000, 1500, - # 1623 so check for potentially multiple replies. - if imapdata == [None]: - return True - maxmsgid = 0 - for msgid in imapdata: - maxmsgid = max(long(msgid), maxmsgid) - - # Different number of messages than last time? - if maxmsgid != statusfolder.getmessagecount(): - return True - - finally: - self.imapserver.releaseconnection(imapobj) - return False + retry = True # Should we attempt another round or exit? + while retry: + retry = False + imapobj = self.imapserver.acquireconnection() + try: + # Select folder and get number of messages + restype, imapdata = imapobj.select(self.getfullname(), True, + True) + except OfflineImapError, e: + # retry on dropped connections, raise otherwise + self.imapserver.releaseconnection(imapobj, true) + if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: + retry = True + else: raise + finally: + self.imapserver.releaseconnection(imapobj) + # 1. Some mail servers do not return an EXISTS response + # if the folder is empty. 2. ZIMBRA servers can return + # multiple EXISTS replies in the form 500, 1000, 1500, + # 1623 so check for potentially multiple replies. + if imapdata == [None]: + return True + maxmsgid = 0 + for msgid in imapdata: + maxmsgid = max(long(msgid), maxmsgid) + # Different number of messages than last time? + if maxmsgid != statusfolder.getmessagecount(): + return True + return False def cachemessagelist(self): maxage = self.config.getdefaultint("Account %s" % self.accountname, From 7941ea7e7d6baf5549689946cbe1b8e476fb93fc Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 12 Sep 2011 10:26:43 +0200 Subject: [PATCH 250/817] folder.IMAP: Make use of the new connection discarding In getmessage() we were releaseing a connection when we detected a dropped connection, but it turns out that this was not enough, we need to explicitely discard it when we detect a dropped one. So add the drop_conn=True parameter that was recently introduced to force the discarding of the dead conection. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 25ebe38..034d80d 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -99,7 +99,7 @@ class IMAPFolder(BaseFolder): True) except OfflineImapError, e: # retry on dropped connections, raise otherwise - self.imapserver.releaseconnection(imapobj, true) + self.imapserver.releaseconnection(imapobj, True) if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: retry = True else: raise @@ -117,7 +117,7 @@ class IMAPFolder(BaseFolder): # Different number of messages than last time? if maxmsgid != statusfolder.getmessagecount(): return True - return False + return False def cachemessagelist(self): maxage = self.config.getdefaultint("Account %s" % self.accountname, @@ -221,7 +221,7 @@ class IMAPFolder(BaseFolder): fails_left = 0 except imapobj.abort(), e: # Release dropped connection, and get a new one - self.imapserver.releaseconnection(imapobj) + self.imapserver.releaseconnection(imapobj, True) imapobj = self.imapserver.acquireconnection() self.ui.error(e, exc_info()[2]) fails_left -= 1 From c93cd9bb1aa8dec871a9202c37bd74372bf748fe Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 16 Sep 2011 10:54:22 +0200 Subject: [PATCH 251/817] BaseFolder(): Save name and repository As all Folders share these parameters, we can safely handle them in BaseFolder. This makes sense, as BaseFolder has a getname() function that returns self.name but nothing actually set self.name. It also saves a few lines of code. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 8 +++++++- offlineimap/folder/Gmail.py | 4 ++-- offlineimap/folder/IMAP.py | 5 ++--- offlineimap/folder/LocalStatus.py | 4 +--- offlineimap/folder/Maildir.py | 5 +---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 5bd8061..9c8feeb 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -28,8 +28,14 @@ except NameError: from sets import Set as set class BaseFolder(object): - def __init__(self): + def __init__(self, name, repository): + """ + :para name: Path & name of folder minus root or reference + :para repository: Repository() in which the folder is. + """ self.ui = getglobalui() + self.name = name + self.repository = repository def getname(self): """Returns name""" diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 3ca11cd..e65793d 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -34,12 +34,12 @@ class GmailFolder(IMAPFolder): """ def __init__(self, imapserver, name, visiblename, accountname, repository): + super(GmailFolder, self).__init__(imapserver, name, visiblename, + accountname, repository) self.realdelete = repository.getrealdelete(name) self.trash_folder = repository.gettrashfolder(name) #: Gmail will really delete messages upon EXPUNGE in these folders self.real_delete_folders = [ self.trash_folder, repository.getspamfolder() ] - IMAPFolder.__init__(self, imapserver, name, visiblename, \ - accountname, repository) def deletemessages_noconvert(self, uidlist): uidlist = [uid for uid in uidlist if uid in self.messagelist] diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 034d80d..f17e2bc 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -33,18 +33,17 @@ except NameError: class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, visiblename, accountname, repository): + name = imaputil.dequote(name) + super(IMAPFolder, self).__init__(name, repository) self.config = imapserver.config self.expunge = repository.getexpunge() - self.name = imaputil.dequote(name) self.root = None # imapserver.root self.sep = imapserver.delim self.imapserver = imapserver self.messagelist = None self.visiblename = visiblename self.accountname = accountname - self.repository = repository self.randomgenerator = random.Random() - BaseFolder.__init__(self) #self.ui is set in BaseFolder def selectro(self, imapobj): diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index be9d1e3..732c5a6 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -27,18 +27,16 @@ magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" class LocalStatusFolder(BaseFolder): def __init__(self, root, name, repository, accountname, config): - self.name = name + super(LocalStatusFolder, self).__init__(name, repository) self.root = root self.sep = '.' self.config = config self.filename = os.path.join(root, self.getfolderbasename()) self.messagelist = {} - self.repository = repository self.savelock = threading.Lock() self.doautosave = config.getdefaultboolean("general", "fsync", False) """Should we perform fsyncs as often as possible?""" self.accountname = accountname - super(LocalStatusFolder, self).__init__() def getaccountname(self): return self.accountname diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index cedcb5d..20b306b 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -59,13 +59,12 @@ def gettimeseq(): class MaildirFolder(BaseFolder): def __init__(self, root, name, sep, repository, accountname, config): - self.name = name + super(MaildirFolder, self).__init__(name, repository) self.config = config self.dofsync = config.getdefaultboolean("general", "fsync", True) self.root = root self.sep = sep self.messagelist = None - self.repository = repository self.accountname = accountname self.wincompatible = self.config.getdefaultboolean( @@ -77,8 +76,6 @@ class MaildirFolder(BaseFolder): self.infosep = '!' self.flagmatchre = re.compile(self.infosep + '.*2,([A-Z]+)') - - BaseFolder.__init__(self) #self.ui is set in BaseFolder.init() # Cache the full folder path, as we use getfullname() very often self._fullname = os.path.join(self.getroot(), self.getname()) From 410e2d35e97ce6a0fcbc3519a7725d3bfb8702f4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 16 Sep 2011 10:54:23 +0200 Subject: [PATCH 252/817] Set accountname in BaseFolder, and don't pass it in initialization We passed in the accountname to all derivatives of BaseFolder, such as IMAPFolder(...,repository,...,accountname), although it is perfectly possible to get the accountname from the Repository(). So remove this unneeded parameter. Each backend had to define getaccountname() (although the function is hardly used and most accessed .accountname directly). On the other hand BaseFolder was using getaccountname but it never defined the function. So make the sane thing, remove all definitions from backends and define accountname() once in Basefolder. It was made a property and not just a (public) attribute, so it will show up in our developer documentation as public API. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 11 ++++++++--- offlineimap/folder/Gmail.py | 4 ++-- offlineimap/folder/IMAP.py | 6 +----- offlineimap/folder/LocalStatus.py | 6 +----- offlineimap/folder/LocalStatusSQLite.py | 3 +-- offlineimap/folder/Maildir.py | 6 +----- offlineimap/repository/Base.py | 11 +++++++---- offlineimap/repository/IMAP.py | 6 +++--- offlineimap/repository/LocalStatus.py | 2 +- offlineimap/repository/Maildir.py | 3 +-- 10 files changed, 26 insertions(+), 32 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 9c8feeb..3339424 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -44,6 +44,11 @@ class BaseFolder(object): def __str__(self): return self.name + @property + def accountname(self): + """Account name as string""" + return self.repository.accountname + def suggeststhreads(self): """Returns true if this folder suggests using threads for actions; false otherwise. Probably only IMAP will return true.""" @@ -239,7 +244,7 @@ class BaseFolder(object): # self.getmessage(). So, don't call self.getmessage unless # really needed. if register: # output that we start a new thread - self.ui.registerthread(self.getaccountname()) + self.ui.registerthread(self.accountname) try: message = None @@ -295,7 +300,7 @@ class BaseFolder(object): self.ui.error(e, exc_info()[2]) except Exception, e: self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ - (uid, self.getaccountname(), + (uid, self.accountname, traceback.format_exc())) raise #raise on unknown errors, so we can fix those @@ -443,5 +448,5 @@ class BaseFolder(object): self.ui.error(e, exc_info()[2]) except Exception, e: self.ui.error(e, exc_info()[2], "Syncing folder %s [acc: %s]" %\ - (self, self.getaccountname())) + (self, self.accountname)) raise # raise unknown Exceptions so we can fix them diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index e65793d..dc301d0 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -33,9 +33,9 @@ class GmailFolder(IMAPFolder): http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 """ - def __init__(self, imapserver, name, visiblename, accountname, repository): + def __init__(self, imapserver, name, visiblename, repository): super(GmailFolder, self).__init__(imapserver, name, visiblename, - accountname, repository) + repository) self.realdelete = repository.getrealdelete(name) self.trash_folder = repository.gettrashfolder(name) #: Gmail will really delete messages upon EXPUNGE in these folders diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index f17e2bc..32625fa 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -32,7 +32,7 @@ except NameError: class IMAPFolder(BaseFolder): - def __init__(self, imapserver, name, visiblename, accountname, repository): + def __init__(self, imapserver, name, visiblename, repository): name = imaputil.dequote(name) super(IMAPFolder, self).__init__(name, repository) self.config = imapserver.config @@ -42,7 +42,6 @@ class IMAPFolder(BaseFolder): self.imapserver = imapserver self.messagelist = None self.visiblename = visiblename - self.accountname = accountname self.randomgenerator = random.Random() #self.ui is set in BaseFolder @@ -60,9 +59,6 @@ class IMAPFolder(BaseFolder): except imapobj.readonly: imapobj.select(self.getfullname(), readonly = 1) - def getaccountname(self): - return self.accountname - def suggeststhreads(self): return 1 diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 732c5a6..ad7ccb6 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -26,7 +26,7 @@ except NameError: magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" class LocalStatusFolder(BaseFolder): - def __init__(self, root, name, repository, accountname, config): + def __init__(self, root, name, repository, config): super(LocalStatusFolder, self).__init__(name, repository) self.root = root self.sep = '.' @@ -36,10 +36,6 @@ class LocalStatusFolder(BaseFolder): self.savelock = threading.Lock() self.doautosave = config.getdefaultboolean("general", "fsync", False) """Should we perform fsyncs as often as possible?""" - self.accountname = accountname - - def getaccountname(self): - return self.accountname def storesmessages(self): return 0 diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index bd38930..6eccbaa 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -46,10 +46,9 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): #current version of our db format cur_version = 1 - def __init__(self, root, name, repository, accountname, config): + def __init__(self, root, name, repository, config): super(LocalStatusSQLiteFolder, self).__init__(root, name, repository, - accountname, config) # dblock protects against concurrent writes in same connection diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 20b306b..5d75121 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -58,14 +58,13 @@ def gettimeseq(): timelock.release() class MaildirFolder(BaseFolder): - def __init__(self, root, name, sep, repository, accountname, config): + def __init__(self, root, name, sep, repository, config): super(MaildirFolder, self).__init__(name, repository) self.config = config self.dofsync = config.getdefaultboolean("general", "fsync", True) self.root = root self.sep = sep self.messagelist = None - self.accountname = accountname self.wincompatible = self.config.getdefaultboolean( "Account "+self.accountname, "maildir-windows-compatible", False) @@ -80,9 +79,6 @@ class MaildirFolder(BaseFolder): # Cache the full folder path, as we use getfullname() very often self._fullname = os.path.join(self.getroot(), self.getname()) - def getaccountname(self): - return self.accountname - def getfullname(self): """Return the absolute file path to the Maildir folder (sans cur|new)""" return self._fullname diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 72a5f88..ae17ee8 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -22,13 +22,14 @@ from offlineimap import CustomConfig from offlineimap.ui import getglobalui class BaseRepository(object, CustomConfig.ConfigHelperMixin): + def __init__(self, reposname, account): self.ui = getglobalui() self.account = account self.config = account.getconfig() self.name = reposname self.localeval = account.getlocaleval() - self.accountname = self.account.getname() + self._accountname = self.account.getname() self.uiddir = os.path.join(self.config.getmetadatadir(), 'Repository-' + self.name) if not os.path.exists(self.uiddir): os.mkdir(self.uiddir, 0700) @@ -71,15 +72,17 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): def __str__(self): return self.name + @property + def accountname(self): + """Account name as string""" + return self._accountname + def getuiddir(self): return self.uiddir def getmapdir(self): return self.mapdir - def getaccountname(self): - return self.accountname - def getsection(self): return 'Repository ' + self.name diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index daa31c9..0de6044 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -261,7 +261,7 @@ class IMAPRepository(BaseRepository): def getfolder(self, foldername): return self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), - self.accountname, self) + self) def getfoldertype(self): return folder.IMAP.IMAPFolder @@ -303,7 +303,7 @@ class IMAPRepository(BaseRepository): continue retval.append(self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), - self.accountname, self)) + self)) if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: @@ -320,7 +320,7 @@ class IMAPRepository(BaseRepository): retval.append(self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), - self.accountname, self)) + self)) finally: self.imapserver.releaseconnection(imapobj) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index a392dcf..20291a1 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -83,7 +83,7 @@ class LocalStatusRepository(BaseRepository): def getfolder(self, foldername): """Return the Folder() object for a foldername""" return self.LocalStatusFolderClass(self.directory, foldername, - self, self.accountname, + self, self.config) def getfolders(self): diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index aeb9776..a7f7c79 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -118,7 +118,7 @@ class MaildirRepository(BaseRepository): self._append_folder_atimes(foldername) return folder.Maildir.MaildirFolder(self.root, foldername, self.getsep(), self, - self.accountname, self.config) + self.config) def _getfolders_scandir(self, root, extension = None): """Recursively scan folder 'root'; return a list of MailDirFolder @@ -168,7 +168,6 @@ class MaildirRepository(BaseRepository): foldername, self.getsep(), self, - self.accountname, self.config)) if self.getsep() == '/' and dirname != '.': # Recursively check sub-directories for folders too. From ee75e0921ff105d19f619973dba188fc6900ff45 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 16 Sep 2011 10:54:24 +0200 Subject: [PATCH 253/817] Remove 'config' as parameter from BaseFolder & derivatives It is possible to get the config parameter from the Repository() which is set in BaseFolder, so we set self.config there and remove the various methods and 'config' parameters that are superfluous. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 1 + offlineimap/folder/IMAP.py | 1 - offlineimap/folder/LocalStatus.py | 6 +++--- offlineimap/folder/LocalStatusSQLite.py | 5 ++--- offlineimap/folder/Maildir.py | 5 ++--- offlineimap/repository/LocalStatus.py | 3 +-- offlineimap/repository/Maildir.py | 6 ++---- 7 files changed, 11 insertions(+), 16 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 3339424..c8ed108 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -36,6 +36,7 @@ class BaseFolder(object): self.ui = getglobalui() self.name = name self.repository = repository + self.config = repository.getconfig() def getname(self): """Returns name""" diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 32625fa..afaaa0d 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -35,7 +35,6 @@ class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, visiblename, repository): name = imaputil.dequote(name) super(IMAPFolder, self).__init__(name, repository) - self.config = imapserver.config self.expunge = repository.getexpunge() self.root = None # imapserver.root self.sep = imapserver.delim diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index ad7ccb6..821ad88 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -26,15 +26,15 @@ except NameError: magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" class LocalStatusFolder(BaseFolder): - def __init__(self, root, name, repository, config): + def __init__(self, root, name, repository): super(LocalStatusFolder, self).__init__(name, repository) self.root = root self.sep = '.' - self.config = config self.filename = os.path.join(root, self.getfolderbasename()) self.messagelist = {} self.savelock = threading.Lock() - self.doautosave = config.getdefaultboolean("general", "fsync", False) + self.doautosave = self.config.getdefaultboolean("general", "fsync", + False) """Should we perform fsyncs as often as possible?""" def storesmessages(self): diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 6eccbaa..ba80aa1 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -46,10 +46,9 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): #current version of our db format cur_version = 1 - def __init__(self, root, name, repository, config): + def __init__(self, root, name, repository): super(LocalStatusSQLiteFolder, self).__init__(root, name, - repository, - config) + repository) # dblock protects against concurrent writes in same connection self._dblock = Lock() diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 5d75121..894328c 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -58,10 +58,9 @@ def gettimeseq(): timelock.release() class MaildirFolder(BaseFolder): - def __init__(self, root, name, sep, repository, config): + def __init__(self, root, name, sep, repository): super(MaildirFolder, self).__init__(name, repository) - self.config = config - self.dofsync = config.getdefaultboolean("general", "fsync", True) + self.dofsync = self.config.getdefaultboolean("general", "fsync", True) self.root = root self.sep = sep self.messagelist = None diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 20291a1..d555064 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -83,8 +83,7 @@ class LocalStatusRepository(BaseRepository): def getfolder(self, foldername): """Return the Folder() object for a foldername""" return self.LocalStatusFolderClass(self.directory, foldername, - self, - self.config) + self) def getfolders(self): """Returns a list of all cached folders.""" diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index a7f7c79..2e185ee 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -117,8 +117,7 @@ class MaildirRepository(BaseRepository): if self.config.has_option('Repository ' + self.name, 'restoreatime') and self.config.getboolean('Repository ' + self.name, 'restoreatime'): self._append_folder_atimes(foldername) return folder.Maildir.MaildirFolder(self.root, foldername, - self.getsep(), self, - self.config) + self.getsep(), self) def _getfolders_scandir(self, root, extension = None): """Recursively scan folder 'root'; return a list of MailDirFolder @@ -167,8 +166,7 @@ class MaildirRepository(BaseRepository): retval.append(folder.Maildir.MaildirFolder(self.root, foldername, self.getsep(), - self, - self.config)) + self)) if self.getsep() == '/' and dirname != '.': # Recursively check sub-directories for folders too. retval.extend(self._getfolders_scandir(root, foldername)) From 80e87d0d99d00f5c3998cf249e12d117b892fbbd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 16 Sep 2011 10:54:25 +0200 Subject: [PATCH 254/817] Don't pass in 'root' as para to LocalStatusFolders They have the Repository() which contains the root, so no need to pass it in as an extra parameter. Rename repository.LocalStatus()'s self.directory to self.root for consistency with other backends. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatus.py | 7 +++---- offlineimap/folder/LocalStatusSQLite.py | 6 ++---- offlineimap/repository/LocalStatus.py | 19 +++++++++---------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 821ad88..fbe2e24 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -26,11 +26,10 @@ except NameError: magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" class LocalStatusFolder(BaseFolder): - def __init__(self, root, name, repository): + def __init__(self, name, repository): super(LocalStatusFolder, self).__init__(name, repository) - self.root = root self.sep = '.' - self.filename = os.path.join(root, self.getfolderbasename()) + self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self.messagelist = {} self.savelock = threading.Lock() self.doautosave = self.config.getdefaultboolean("general", "fsync", @@ -47,7 +46,7 @@ class LocalStatusFolder(BaseFolder): return self.name def getroot(self): - return self.root + return self.repository.root def getsep(self): return self.sep diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index ba80aa1..08af807 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -46,10 +46,8 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): #current version of our db format cur_version = 1 - def __init__(self, root, name, repository): - super(LocalStatusSQLiteFolder, self).__init__(root, name, - repository) - + def __init__(self, name, repository): + super(LocalStatusSQLiteFolder, self).__init__(name, repository) # dblock protects against concurrent writes in same connection self._dblock = Lock() #Try to establish connection, no need for threadsafety in __init__ diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index d555064..fdd1e19 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -25,14 +25,14 @@ import re class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): BaseRepository.__init__(self, reposname, account) - self.directory = os.path.join(account.getaccountmeta(), 'LocalStatus') - - #statusbackend can be 'plain' or 'sqlite' + # Root directory in which the LocalStatus folders reside + self.root = os.path.join(account.getaccountmeta(), 'LocalStatus') + # statusbackend can be 'plain' or 'sqlite' backend = self.account.getconf('status_backend', 'plain') if backend == 'sqlite': self._backend = 'sqlite' self.LocalStatusFolderClass = LocalStatusSQLiteFolder - self.directory += '-sqlite' + self.root += '-sqlite' elif backend == 'plain': self._backend = 'plain' self.LocalStatusFolderClass = LocalStatusFolder @@ -40,8 +40,8 @@ class LocalStatusRepository(BaseRepository): raise SyntaxWarning("Unknown status_backend '%s' for account '%s'" \ % (backend, account.name)) - if not os.path.exists(self.directory): - os.mkdir(self.directory, 0700) + if not os.path.exists(self.root): + os.mkdir(self.root, 0700) # self._folders is a list of LocalStatusFolders() self._folders = None @@ -60,7 +60,7 @@ class LocalStatusRepository(BaseRepository): # replace with literal 'dot' if final path name is '.' as '.' is # an invalid file name. basename = re.sub('(^|\/)\.$','\\1dot', basename) - return os.path.join(self.directory, basename) + return os.path.join(self.root, basename) def makefolder(self, foldername): """Create a LocalStatus Folder @@ -82,8 +82,7 @@ class LocalStatusRepository(BaseRepository): def getfolder(self, foldername): """Return the Folder() object for a foldername""" - return self.LocalStatusFolderClass(self.directory, foldername, - self) + return self.LocalStatusFolderClass(foldername, self) def getfolders(self): """Returns a list of all cached folders.""" @@ -91,7 +90,7 @@ class LocalStatusRepository(BaseRepository): return self._folders self._folders = [] - for folder in os.listdir(self.directory): + for folder in os.listdir(self.root): self._folders.append(self.getfolder(folder)) return self._folders From 0d3303ec12a0a528fed8c528fa67e5c1f1ba587f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 16 Sep 2011 10:54:26 +0200 Subject: [PATCH 255/817] Remove visiblename as parameter to IMAPFolder creation IMAPFolder has the repository and foldername values so it can get the transposed (aka visiblename) of a folder itself just fine. There is no need to pass it in as an separate parameter. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Gmail.py | 5 ++--- offlineimap/folder/IMAP.py | 4 ++-- offlineimap/repository/IMAP.py | 6 +----- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index dc301d0..8d9c0bc 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -33,9 +33,8 @@ class GmailFolder(IMAPFolder): http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 """ - def __init__(self, imapserver, name, visiblename, repository): - super(GmailFolder, self).__init__(imapserver, name, visiblename, - repository) + def __init__(self, imapserver, name, repository): + super(GmailFolder, self).__init__(imapserver, name, repository) self.realdelete = repository.getrealdelete(name) self.trash_folder = repository.gettrashfolder(name) #: Gmail will really delete messages upon EXPUNGE in these folders diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index afaaa0d..ea86d15 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -32,7 +32,7 @@ except NameError: class IMAPFolder(BaseFolder): - def __init__(self, imapserver, name, visiblename, repository): + def __init__(self, imapserver, name, repository): name = imaputil.dequote(name) super(IMAPFolder, self).__init__(name, repository) self.expunge = repository.getexpunge() @@ -40,7 +40,7 @@ class IMAPFolder(BaseFolder): self.sep = imapserver.delim self.imapserver = imapserver self.messagelist = None - self.visiblename = visiblename + self.visiblename = repository.nametrans(name) self.randomgenerator = random.Random() #self.ui is set in BaseFolder diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 0de6044..7392655 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -259,9 +259,7 @@ class IMAPRepository(BaseRepository): def getfolder(self, foldername): - return self.getfoldertype()(self.imapserver, foldername, - self.nametrans(foldername), - self) + return self.getfoldertype()(self.imapserver, foldername, self) def getfoldertype(self): return folder.IMAP.IMAPFolder @@ -302,7 +300,6 @@ class IMAPRepository(BaseRepository): foldername) continue retval.append(self.getfoldertype()(self.imapserver, foldername, - self.nametrans(foldername), self)) if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() @@ -319,7 +316,6 @@ class IMAPRepository(BaseRepository): continue retval.append(self.getfoldertype()(self.imapserver, foldername, - self.nametrans(foldername), self)) finally: self.imapserver.releaseconnection(imapobj) From 82e47896cfa8aefea212daa8c0263a617d9ccf08 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 08:27:18 +0200 Subject: [PATCH 256/817] Don't ask for username in the preauthtunnel case repos.getuesr() asks for a username if none is specified, but in the case of a tunnel connection, we don't need one, so we need to skip the repos.getuser() call here. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 7c740c7..5d2d108 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -56,7 +56,7 @@ class IMAPServer: self.config = repos.getconfig() self.tunnel = repos.getpreauthtunnel() self.usessl = repos.getssl() - self.username = repos.getuser() + self.username = None if self.tunnel else repos.getuser() self.password = None self.passworderror = None self.goodpassword = None From f5366343b98fd2ab388d13a40955f55b8559ba9b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 09:25:48 +0200 Subject: [PATCH 257/817] Fix default Maildir File permissions. open() and os.open() lead to different file permissions by default, and while we have not changed the os.open that had been used, some code changes led to these permissions slipping through. Fix this by setting the permissions explicitly to 0666 (minus the users umask). Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/folder/Maildir.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index e4b8055..909ebd6 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -21,6 +21,9 @@ Changes Bug Fixes --------- +* New emails are not created with "-rwxr-xr-x" but as "-rw-r--r--" + anymore, fixing a regression in 6.3.4. + Pending for the next major release ================================== diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 894328c..2bfd394 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -238,7 +238,7 @@ class MaildirFolder(BaseFolder): # open file and write it out try: fd = os.open(os.path.join(tmpdir, messagename), - os.O_EXCL|os.O_CREAT|os.O_WRONLY) + os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0666) except OSError, e: if e.errno == 17: #FILE EXISTS ALREADY From c7938dc0819e0a14d1a97e2362e46daa86a7ce85 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 8 Jul 2011 12:23:16 +0200 Subject: [PATCH 258/817] Per-account locking Previously, we were simply locking offlineimap whenever it was running. Howver there is no reason why we shouldn't be able to invoke it in parallel, e.g. to synchronize several accounts in one offlineimap each. This patch implements the locking per-account, so that it is possible to sync different accounts at the same time. If in refresh mode, we will attempt to loop three times before giving up. This also fixes Debian bug #586655 Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 7 +++++++ offlineimap/accounts.py | 36 ++++++++++++++++++++++++++++++++++++ offlineimap/init.py | 20 -------------------- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 909ebd6..1b1e644 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,13 @@ others. New Features ------------ +* Implement per-account locking, so that it will possible to sync + different accounts at the same time. The old global lock is still in + place for backward compatibility reasons (to be able to run old and + new versions of OfflineImap concurrently) and will be removed in the + future. Starting with this version, OfflineImap will be + forward-compatible with the per-account locking style. + Changes ------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 428cbd8..5a2a120 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -25,6 +25,11 @@ import os from sys import exc_info import traceback +try: + import fcntl +except: + pass # ok if this fails, we can do without + def getaccountlist(customconfig): return customconfig.getsectionlist('Account') @@ -159,6 +164,35 @@ class SyncableAccount(Account): functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`, used for syncing.""" + def __init__(self, *args, **kwargs): + Account.__init__(self, *args, **kwargs) + self._lockfd = None + self._lockfilepath = os.path.join(self.config.getmetadatadir(), + "%s.lock" % self) + + def lock(self): + """Lock the account, throwing an exception if it is locked already""" + self._lockfd = open(self._lockfilepath, 'w') + try: + fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB) + except NameError: + #fcntl not available (Windows), disable file locking... :( + pass + except IOError: + self._lockfd.close() + raise OfflineImapError("Could not lock account %s." % self, + OfflineImapError.ERROR.REPO) + + def unlock(self): + """Unlock the account, deleting the lock file""" + #If we own the lock file, delete it + if self._lockfd and not self._lockfd.closed: + self._lockfd.close() + try: + os.unlink(self._lockfilepath) + except OSError: + pass #Failed to delete for some reason. + def syncrunner(self): self.ui.registerthread(self.name) self.ui.acct(self.name) @@ -175,6 +209,7 @@ class SyncableAccount(Account): while looping: try: try: + self.lock() self.sync() except (KeyboardInterrupt, SystemExit): raise @@ -194,6 +229,7 @@ class SyncableAccount(Account): if self.refreshperiod: looping = 3 finally: + self.unlock() if looping and self.sleeper() >= 2: looping = 0 self.ui.acctdone(self.name) diff --git a/offlineimap/init.py b/offlineimap/init.py index 93b7224..f7b2eef 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -30,14 +30,6 @@ from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser -try: - import fcntl - hasfcntl = 1 -except: - hasfcntl = 0 - -lockfd = None - class OfflineImap: """The main class that encapsulates the high level use of OfflineImap. @@ -46,17 +38,6 @@ class OfflineImap: oi = OfflineImap() oi.run() """ - def lock(self, config, ui): - global lockfd, hasfcntl - if not hasfcntl: - return - lockfd = open(config.getmetadatadir() + "/lock", "w") - try: - fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - ui.locked() - ui.terminate(1) - def run(self): """Parse the commandline and invoke everything""" @@ -253,7 +234,6 @@ class OfflineImap: config.set(section, "folderfilter", folderfilter) config.set(section, "folderincludes", folderincludes) - self.lock(config, ui) self.config = config def sigterm_handler(signum, frame): From 89c705bb2656b8edf028ce569329a012b4515c3e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 16 Sep 2011 15:09:20 +0200 Subject: [PATCH 259/817] init.py: Import OfflineImapError The next commit will make use of OfflineImapError but is transient (the old-style lock). The commit is supposed to be reverted after a few releases. So add the new import in a separate commit, because we might need this even when reverting the commit. Signed-off-by: Sebastian Spaeth --- offlineimap/init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/init.py b/offlineimap/init.py index f7b2eef..88baf9f 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -26,6 +26,7 @@ import logging from optparse import OptionParser import offlineimap from offlineimap import accounts, threadutil, syncmaster +from offlineimap.error import OfflineImapError from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser From 0d9565141765b8b23c1c723d325cf494e47cc80d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 8 Jul 2011 12:29:00 +0200 Subject: [PATCH 260/817] Perform legacy global lock in addition to new per-account lock We simply lock OfflineImap the same global way that we have always done in addition to the previously implemented per-account lock. We can keep both systems in parallel and then after a few stable releases, drop the old-style global lock. by reverting this patch Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 1 + offlineimap/init.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 5a2a120..5138523 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -172,6 +172,7 @@ class SyncableAccount(Account): def lock(self): """Lock the account, throwing an exception if it is locked already""" + # Take a new-style per-account lock self._lockfd = open(self._lockfilepath, 'w') try: fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB) diff --git a/offlineimap/init.py b/offlineimap/init.py index 88baf9f..e5560a9 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -24,6 +24,10 @@ import signal import socket import logging from optparse import OptionParser +try: + import fcntl +except ImportError: + pass #it's OK import offlineimap from offlineimap import accounts, threadutil, syncmaster from offlineimap.error import OfflineImapError @@ -311,6 +315,18 @@ class OfflineImap: #various initializations that need to be performed: offlineimap.mbnames.init(config, syncaccounts) + #TODO: keep legacy lock for a few versions, then remove. + self._legacy_lock = open(self.config.getmetadatadir() + "/lock", + 'w') + try: + fcntl.lockf(self._legacy_lock, fcntl.LOCK_EX|fcntl.LOCK_NB) + except NameError: + #fcntl not available (Windows), disable file locking... :( + pass + except IOError: + raise OfflineImapError("Could not take global lock.", + OfflineImapError.ERROR.REPO) + if options.singlethreading: #singlethreaded self.sync_singlethreaded(syncaccounts, config) @@ -330,7 +346,7 @@ class OfflineImap: return except (SystemExit): raise - except: + except Exception, e: ui.mainException() def sync_singlethreaded(self, accs, config): From 19ff636390a06fd6c82f6087cb1647e00d34856c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 22 Aug 2011 11:49:57 +0200 Subject: [PATCH 261/817] Properly output errors when the main thread receives some Use the ui.error infrastructure that has been put in place and use ui.terminate even if we received an Exception, so that we can output the list of errors that we have. This does away with 2 now unused functions in ui/UIBase.py Signed-off-by: Sebastian Spaeth --- offlineimap/init.py | 3 ++- offlineimap/ui/UIBase.py | 8 -------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index e5560a9..063a2c0 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -347,7 +347,8 @@ class OfflineImap: except (SystemExit): raise except Exception, e: - ui.mainException() + ui.error(e) + ui.terminate() def sync_singlethreaded(self, accs, config): """Executed if we do not want a separate syncmaster thread diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index f5dca57..5017934 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -344,14 +344,6 @@ class UIBase: s.delThreadDebugLog(thread) s.terminate(100) - def getMainExceptionString(s): - return "Main program terminated with exception:\n%s\n" %\ - traceback.format_exc() + \ - s.getThreadDebugLog(threading.currentThread()) - - def mainException(s): - s._msg(s.getMainExceptionString()) - def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" #print any exceptions that have occurred over the run From 1ac9dc7fb42f797e94f5c8c30b767b1e111fc945 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 16 Sep 2011 13:57:55 +0200 Subject: [PATCH 262/817] Implement RFC 2595 LOGINDISABLED Warn the user and abort when we attempt a plaintext login, but the server has explicitly disabled plaintext logins. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 9 ++++----- offlineimap/imapserver.py | 6 ++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 1b1e644..632586b 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -20,6 +20,10 @@ New Features future. Starting with this version, OfflineImap will be forward-compatible with the per-account locking style. +* Implement RFC 2595 LOGINDISABLED. Warn the user and abort when we + attempt a plaintext login but the server has explicitly disabled + plaintext logins rather than crashing. + Changes ------- @@ -30,8 +34,3 @@ Bug Fixes * New emails are not created with "-rwxr-xr-x" but as "-rw-r--r--" anymore, fixing a regression in 6.3.4. - -Pending for the next major release -================================== - -* UIs get shorter and nicer names. (API changing) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 5d2d108..506f886 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -262,6 +262,12 @@ class IMAPServer: except imapobj.error, val: self.plainauth(imapobj) else: + # Use plaintext login, unless + # LOGINDISABLED (RFC2595) + if 'LOGINDISABLED' in imapobj.capabilities: + raise OfflineImapError("Plaintext login " + "disabled by server. Need to use SSL?", + OfflineImapError.ERROR.REPO) self.plainauth(imapobj) # Would bail by here if there was a failure. success = 1 From a43677a3e61cc25631cc050175791a13b2f9cf61 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 11:01:17 +0200 Subject: [PATCH 263/817] Shorten self.infosep assignment A more pythonic and less verbose way to do the same. Add a comment what the variable is all about. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 2bfd394..f26b07e 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -68,11 +68,8 @@ class MaildirFolder(BaseFolder): self.wincompatible = self.config.getdefaultboolean( "Account "+self.accountname, "maildir-windows-compatible", False) - if self.wincompatible == False: - self.infosep = ':' - else: - self.infosep = '!' - + self.infosep = '!' if self.wincompatible else ':' + """infosep is the separator between maildir name and flag appendix""" self.flagmatchre = re.compile(self.infosep + '.*2,([A-Z]+)') #self.ui is set in BaseFolder.init() # Cache the full folder path, as we use getfullname() very often From dfdc4d457be0f411c0e2d0ddc12a6169959f26df Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 12:36:55 +0200 Subject: [PATCH 264/817] Folder.Maildir: Simplify getmessagetime No need to use os.stat()['st_mtime'] when there is os.path.getmtime() Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index f26b07e..cd3da7d 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -202,11 +202,10 @@ class MaildirFolder(BaseFolder): # read it as text? return retval.replace("\r\n", "\n") - def getmessagetime( self, uid ): + def getmessagetime(self, uid): filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) - st = os.stat(filepath) - return st.st_mtime + return os.path.getmtime(filepath) def savemessage(self, uid, content, flags, rtime): # This function only ever saves to tmp/, From 32ca20d0da97b2ed2b377c06f12f4365af37538c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 11:19:53 +0200 Subject: [PATCH 265/817] Folder.Maildir: No need to store 'uid' in messagelist dict. The Message UID is already the key to self.messagelist, so we have that information. It is redundant to save the UID again as self.messagelist[uid]{'uid': uid} and we never made use of the information anyway. The same thing should be done with the other 2 backends. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index cd3da7d..df5dd2e 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -165,10 +165,8 @@ class MaildirFolder(BaseFolder): flags = set(flagmatch.group(1)) else: flags = set() - # 'filename' is 'dirannex/filename', e.g. cur/123_U=1_FMD5=1:2,S - retval[uid] = {'uid': uid, - 'flags': flags, - 'filename': file} + # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S + retval[uid] = {'flags': flags, 'filename': file} return retval def quickchanged(self, statusfolder): @@ -255,7 +253,7 @@ class MaildirFolder(BaseFolder): if rtime != None: os.utime(os.path.join(tmpdir, messagename), (rtime, rtime)) - self.messagelist[uid] = {'uid': uid, 'flags': set(), + self.messagelist[uid] = {'flags': set(), 'filename': os.path.join('tmp', messagename)} # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) From 8ba17c5bd1610bcf242922841f9449618f9f461a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 14 Aug 2011 13:38:13 +0200 Subject: [PATCH 266/817] add sync_this variable to all Folder() instances This variable shows if this folder should be synced or is disabled due to a folderfilter statement. This lets us distinguish between a non-existent folder and one that has been filtered out. Previously any filtered folder would simply appear to be non-existing. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 16 +++++++++++++++- offlineimap/folder/Base.py | 2 ++ offlineimap/repository/IMAP.py | 12 +++++++----- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 5138523..13bb12e 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -274,6 +274,10 @@ class SyncableAccount(Account): # iterate through all folders on the remote repo and sync for remotefolder in remoterepos.getfolders(): + if not remotefolder.sync_this: + self.ui.debug('', "Not syncing filtered remote folder '%s'" + "[%s]" % (remotefolder, remoterepos)) + continue # Filtered out remote folder thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, @@ -323,7 +327,9 @@ class SyncableAccount(Account): def syncfolder(accountname, remoterepos, remotefolder, localrepos, statusrepos, quick): """This function is called as target for the - InstanceLimitedThread invokation in SyncableAccount.""" + InstanceLimitedThread invokation in SyncableAccount. + + Filtered folders on the remote side will not invoke this function.""" ui = getglobalui() ui.registerthread(accountname) try: @@ -331,6 +337,14 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, localfolder = localrepos.\ getfolder(remotefolder.getvisiblename().\ replace(remoterepos.getsep(), localrepos.getsep())) + + #Filtered folders on the remote side will not invoke this + #function, but we need to NOOP if the local folder is filtered + #out too: + if not localfolder.sync_this: + ui.debug('', "Not syncing filtered local folder '%s'" \ + % localfolder) + return # Write the mailboxes mbnames.add(accountname, localfolder.getvisiblename()) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index c8ed108..3cfb18f 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -33,6 +33,8 @@ class BaseFolder(object): :para name: Path & name of folder minus root or reference :para repository: Repository() in which the folder is. """ + self.sync_this = True + """Should this folder be included in syncing?""" self.ui = getglobalui() self.name = name self.repository = repository diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 7392655..3bd7300 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -295,12 +295,14 @@ class IMAPRepository(BaseRepository): if '\\noselect' in flaglist: continue foldername = imaputil.dequote(name) - if not self.folderfilter(foldername): - self.ui.debug('imap',"Filtering out '%s' due to folderfilter" %\ - foldername) - continue retval.append(self.getfoldertype()(self.imapserver, foldername, self)) + # filter out the folder? + if not self.folderfilter(foldername): + self.ui.debug('imap', "Filtering out '%s'[%s] due to folderfilt" + "er" % (foldername, self)) + retval[-1].sync_this = False + # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: @@ -322,7 +324,7 @@ class IMAPRepository(BaseRepository): retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename())) self.folders = retval - return retval + return self.folders def makefolder(self, foldername): #if self.getreference() != '""': From c7420e6ad46ea3b7a8dea02f017c748037851830 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 14:18:53 +0200 Subject: [PATCH 267/817] Debug log/error proof the creation of new IMAP folders Output a debug log line whenever we create a new folder on an IMAP server. Also raise an OfflineImap Error in case we failed to create it. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/IMAP.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 3bd7300..03dc5ec 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -327,16 +327,27 @@ class IMAPRepository(BaseRepository): return self.folders def makefolder(self, foldername): + """Create a folder on the IMAP server + + :param foldername: Full path of the folder to be created.""" + #TODO: IMHO this existing commented out code is correct and + #should be enabled, but this would change the behavior for + #existing configurations who have a 'reference' set on a Mapped + #IMAP server....: #if self.getreference() != '""': # newname = self.getreference() + self.getsep() + foldername #else: # newname = foldername - newname = foldername imapobj = self.imapserver.acquireconnection() try: - result = imapobj.create(newname) + self.ui._msg("Creating new IMAP folder '%s' on server %s" %\ + (foldername, self)) + result = imapobj.create(foldername) if result[0] != 'OK': - raise RuntimeError, "Repository %s could not create folder %s: %s" % (self.getname(), foldername, str(result)) + raise OfflineImapError("Folder '%s'[%s] could not be created. " + "Server responded: %s" % \ + (foldername, self, str(result)), + OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) From 792243c78f6138904863f60059db051a6c4e4934 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 15:33:58 +0200 Subject: [PATCH 268/817] Improve repository/Maildir.getfolder() to use cached objects Previously, getfolder() would always construct new MaildirFolder() objects, independent of whether the folder exists or not. Improve the function to: 1) Scan and cache the folders if not already done 2) Return the same cached object if we ask for the same foldername twice 3) Reduce a tiny bit of code duplication This is important because we handle stuff like folderfilter in the scandir function and if we discard the scanned dir and create a new object on folderget(), we will lose the folderfilter information. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Maildir.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 2e185ee..637969f 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -19,6 +19,7 @@ from Base import BaseRepository from offlineimap import folder from offlineimap.ui import getglobalui +from offlineimap.error import OfflineImapError import os from stat import * @@ -114,11 +115,20 @@ class MaildirRepository(BaseRepository): self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername) def getfolder(self, foldername): - if self.config.has_option('Repository ' + self.name, 'restoreatime') and self.config.getboolean('Repository ' + self.name, 'restoreatime'): - self._append_folder_atimes(foldername) - return folder.Maildir.MaildirFolder(self.root, foldername, - self.getsep(), self) - + """Return a Folder instance of this Maildir + + If necessary, scan and cache all foldernames to make sure that + we only return existing folders and that 2 calls with the same + name will return the same object.""" + # getfolders() will scan and cache the values *if* necessary + folders = self.getfolders() + for folder in folders: + if foldername == folder.name: + return folder + raise OfflineImapError("getfolder() asked for a nonexisting " + "folder '%s'." % foldername, + OfflineImapError.ERROR.FOLDER) + def _getfolders_scandir(self, root, extension = None): """Recursively scan folder 'root'; return a list of MailDirFolder @@ -157,11 +167,7 @@ class MaildirRepository(BaseRepository): os.path.isdir(os.path.join(fullname, 'tmp'))): # This directory has maildir stuff -- process self.debug(" This is maildir folder '%s'." % foldername) - - if self.config.has_option('Repository %s' % self, - 'restoreatime') and \ - self.config.getboolean('Repository %s' % self, - 'restoreatime'): + if self.getconfboolean('restoreatime', False): self._append_folder_atimes(foldername) retval.append(folder.Maildir.MaildirFolder(self.root, foldername, From a279aa7307ca036f93b660ade84099550ff1a4f2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 14:14:44 +0200 Subject: [PATCH 269/817] Maildir: Call top-level directory '', not '.' If nametrans translates to an empty directory we want to find the top-level directory by name '' and not by name '.'. This unbreaks nametrans rules that result in empty folder names. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Maildir.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 637969f..afdfb76 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -146,7 +146,7 @@ class MaildirRepository(BaseRepository): self.debug(" toppath = %s" % toppath) # Iterate over directories in top & top itself. - for dirname in os.listdir(toppath) + ['.']: + for dirname in os.listdir(toppath) + ['']: self.debug(" *** top of loop") self.debug(" dirname = %s" % dirname) if dirname in ['cur', 'new', 'tmp']: @@ -173,7 +173,7 @@ class MaildirRepository(BaseRepository): foldername, self.getsep(), self)) - if self.getsep() == '/' and dirname != '.': + if self.getsep() == '/' and dirname != '': # Recursively check sub-directories for folders too. retval.extend(self._getfolders_scandir(root, foldername)) self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ From 3157a8d793876d050bff29a3830c26a43ed8a78a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 14:22:03 +0200 Subject: [PATCH 270/817] repository.Gmail.getfolder() Also remove the removed parameters in the Gmail folder initialization. This is one spot where I had forgotten to also strip the parameters. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Gmail.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 20ed1f1..ada2146 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -59,8 +59,7 @@ class GmailRepository(IMAPRepository): def getfolder(self, foldername): return self.getfoldertype()(self.imapserver, foldername, - self.nametrans(foldername), - self.accountname, self) + self) def getfoldertype(self): return folder.Gmail.GmailFolder From 9c5f73d3b34f29eb5e4ea5ca157f8b058684c9bc Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 15:08:26 +0200 Subject: [PATCH 271/817] Move self.folderfilter|nametrans|sort from IMAPFolder to BaseFolder We want to have these functions available for Maildir folders too, so we can folderfilter a Maildir repository too (which is currently not possible) This commit only move the corresponding functions from the IMAP to the Base implementation. It should not change behavior in any way yet. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 18 ++++++++++++++++++ offlineimap/repository/IMAP.py | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index ae17ee8..76fa4a3 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -16,6 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import re import os.path import traceback from offlineimap import CustomConfig @@ -40,6 +41,23 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): if not os.path.exists(self.uiddir): os.mkdir(self.uiddir, 0700) + self.nametrans = lambda foldername: foldername + self.folderfilter = lambda foldername: 1 + self.folderincludes = [] + self.foldersort = cmp + if self.config.has_option(self.getsection(), 'nametrans'): + self.nametrans = self.localeval.eval( + self.getconf('nametrans'), {'re': re}) + if self.config.has_option(self.getsection(), 'folderfilter'): + self.folderfilter = self.localeval.eval( + self.getconf('folderfilter'), {'re': re}) + if self.config.has_option(self.getsection(), 'folderincludes'): + self.folderincludes = self.localeval.eval( + self.getconf('folderincludes'), {'re': re}) + if self.config.has_option(self.getsection(), 'foldersort'): + self.foldersort = self.localeval.eval( + self.getconf('foldersort'), {'re': re}) + def restore_atime(self): """Sets folders' atime back to their values after a sync diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 03dc5ec..5ddf5b9 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -21,7 +21,6 @@ from offlineimap import folder, imaputil, imapserver, OfflineImapError from offlineimap.folder.UIDMaps import MappedIMAPFolder from offlineimap.threadutil import ExitNotifyThread from threading import Event -import re import types import os from sys import exc_info @@ -36,23 +35,6 @@ class IMAPRepository(BaseRepository): self._host = None self.imapserver = imapserver.IMAPServer(self) self.folders = None - self.nametrans = lambda foldername: foldername - self.folderfilter = lambda foldername: 1 - self.folderincludes = [] - self.foldersort = cmp - localeval = self.localeval - if self.config.has_option(self.getsection(), 'nametrans'): - self.nametrans = localeval.eval(self.getconf('nametrans'), - {'re': re}) - if self.config.has_option(self.getsection(), 'folderfilter'): - self.folderfilter = localeval.eval(self.getconf('folderfilter'), - {'re': re}) - if self.config.has_option(self.getsection(), 'folderincludes'): - self.folderincludes = localeval.eval(self.getconf('folderincludes'), - {'re': re}) - if self.config.has_option(self.getsection(), 'foldersort'): - self.foldersort = localeval.eval(self.getconf('foldersort'), - {'re': re}) def startkeepalive(self): keepalivetime = self.getkeepalive() From 7d34060217ba7ea0577d9140abf21b49f13a4413 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 29 Aug 2011 15:09:16 +0200 Subject: [PATCH 272/817] Enable folderfiltering for MailDir repositories too Currently we only filtered IMAP repositories, this patch enables filtering for Maildir repositories too. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/repository/Maildir.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 632586b..f83735f 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -29,6 +29,8 @@ Changes * Documentation improvements concerning 'restoreatime' and some code cleanup +* Maildir repositories now also respond to folderfilter= configurations. + Bug Fixes --------- diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index afdfb76..7c3bb1e 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -173,6 +173,12 @@ class MaildirRepository(BaseRepository): foldername, self.getsep(), self)) + # filter out the folder? + if not self.folderfilter(foldername): + self.debug("Filtering out '%s'[%s] due to folderfilt" + "er" % (foldername, self)) + retval[-1].sync_this = False + if self.getsep() == '/' and dirname != '': # Recursively check sub-directories for folders too. retval.extend(self._getfolders_scandir(root, foldername)) From 6b2ec956cfe8e356d3ffd54eee34773deb73279f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 13:41:38 +0200 Subject: [PATCH 273/817] Apply nametrans to all Foldertypes getvisiblename() was only defined on IMAP(derived) foldertypes, but we want it on eg. Maildirs too, so we define it centrally in Folder.Base.py rather than only in folder.IMAP.py. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 4 +++- offlineimap/folder/IMAP.py | 4 ---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 3cfb18f..d9fed70 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -38,6 +38,7 @@ class BaseFolder(object): self.ui = getglobalui() self.name = name self.repository = repository + self.visiblename = repository.nametrans(name) self.config = repository.getconfig() def getname(self): @@ -69,7 +70,8 @@ class BaseFolder(object): return 1 def getvisiblename(self): - return self.name + """The nametrans-transposed name of the folder's name""" + return self.visiblename def getrepository(self): """Returns the repository object that this folder is within.""" diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index ea86d15..041f5a0 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -40,7 +40,6 @@ class IMAPFolder(BaseFolder): self.sep = imapserver.delim self.imapserver = imapserver self.messagelist = None - self.visiblename = repository.nametrans(name) self.randomgenerator = random.Random() #self.ui is set in BaseFolder @@ -67,9 +66,6 @@ class IMAPFolder(BaseFolder): def getcopyinstancelimit(self): return 'MSGCOPY_' + self.repository.getname() - def getvisiblename(self): - return self.visiblename - def getuidvalidity(self): imapobj = self.imapserver.acquireconnection() try: From fd6261a50f039c21895d4fa9780d00bd0db19474 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 14 Aug 2011 13:38:54 +0200 Subject: [PATCH 274/817] Create new folders on srcrepo if needed This will ignore any nametrans rules, so we might want to limit this only to cases where no nametrans has been specified, or we might want to use the nametrans setting of the dest repo. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 7 ++++ offlineimap/repository/Base.py | 62 +++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index f83735f..16655a1 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -24,6 +24,13 @@ New Features attempt a plaintext login but the server has explicitly disabled plaintext logins rather than crashing. +* Folders will now also be automatically created on the REMOTE side of + an account if they exist on the local side. Use the folderfilters + setting on the local side to prevent some folders from migrating to + the remote side. Also, if you have a nametrans setting on the remote + repository, you might need a nametrans setting on the local repository + that leads to the original name (reverse nametrans). + Changes ------- diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 76fa4a3..59bc3cb 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -19,8 +19,10 @@ import re import os.path import traceback +from sys import exc_info from offlineimap import CustomConfig from offlineimap.ui import getglobalui +from offlineimap.error import OfflineImapError class BaseRepository(object, CustomConfig.ConfigHelperMixin): @@ -134,7 +136,11 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): def syncfoldersto(self, dst_repo, status_repo): """Syncs the folders in this repository to those in dest. - It does NOT sync the contents of those folders.""" + It does NOT sync the contents of those folders. nametrans rules + in both directions will be honored, but there are NO checks yet + that forward and backward nametrans actually match up! + Configuring nametrans on BOTH repositories therefore could lead + to infinite folder creation cycles.""" src_repo = self src_folders = src_repo.getfolders() dst_folders = dst_repo.getfolders() @@ -149,31 +155,47 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): for folder in dst_folders: dst_hash[folder.getvisiblename()] = folder - # - # Find new folders. - for key in src_hash.keys(): - if not key in dst_hash: + # Find new folders on src_repo. + for src_name, src_folder in src_hash.iteritems(): + if src_folder.sync_this and not src_name in dst_hash: try: - dst_repo.makefolder(key) - status_repo.makefolder(key.replace(dst_repo.getsep(), - status_repo.getsep())) - except (KeyboardInterrupt): + dst_repo.makefolder(src_name) + except OfflineImapError, e: + self.ui.error(e, exc_info()[2], + "Creating folder %s on repository %s" %\ + (src_name, dst_repo)) raise - except: - self.ui.warn("ERROR Attempting to create folder " \ - + key + ":" +traceback.format_exc()) + status_repo.makefolder(src_name.replace(dst_repo.getsep(), + status_repo.getsep())) + # Find new folders on dst_repo. + for dst_name, dst_folder in dst_hash.iteritems(): + if dst_folder.sync_this and not dst_name in src_hash: + # Check that back&forth nametrans lead to identical names + #src_name is the unmodified full src_name + newsrc_name = dst_name.replace(dst_repo.getsep(), + src_repo.getsep()) + folder = self.getfolder(newsrc_name) + newdst_name = folder.getvisiblename().replace( + src_repo.getsep(), dst_repo.getsep()) + if newsrc_name == newdst_name: + assert False, "newdstname %s equals newsrcname %s %s %s" % (newdst_name, newsrc_name, dst_name, folder) + else: + assert False, "newdstname %s Does not equal newsrcname %s %s %s" % (newdst_name, newsrc_name, dst_name, folder) + # end sanity check, actually create the folder - # + try: + src_repo.makefolder(dst_name.replace( + dst_repo.getsep(), src_repo.getsep())) + except OfflineImapError, e: + self.ui.error(e, exc_info()[2], + "Creating folder %s on repository %s" %\ + (src_name, dst_repo)) + raise + status_repo.makefolder(dst_name.replace( + dst_repo.getsep(), status_repo.getsep())) # Find deleted folders. - # # We don't delete folders right now. - #for key in desthash.keys(): - # if not key in srchash: - # dest.deletefolder(key) - - ##### Keepalive - def startkeepalive(self): """The default implementation will do nothing.""" pass From b92e453f0ea7c26c5eceebfb45083338564281e6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 14:01:48 +0200 Subject: [PATCH 275/817] sanity check so that nametrans does not lead to infinite folder creation --- offlineimap/repository/Base.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 59bc3cb..ebd024f 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -170,17 +170,25 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): # Find new folders on dst_repo. for dst_name, dst_folder in dst_hash.iteritems(): if dst_folder.sync_this and not dst_name in src_hash: - # Check that back&forth nametrans lead to identical names + # nametrans sanity check! + # Does nametrans back&forth lead to identical names? #src_name is the unmodified full src_name newsrc_name = dst_name.replace(dst_repo.getsep(), src_repo.getsep()) folder = self.getfolder(newsrc_name) newdst_name = folder.getvisiblename().replace( - src_repo.getsep(), dst_repo.getsep()) - if newsrc_name == newdst_name: - assert False, "newdstname %s equals newsrcname %s %s %s" % (newdst_name, newsrc_name, dst_name, folder) - else: - assert False, "newdstname %s Does not equal newsrcname %s %s %s" % (newdst_name, newsrc_name, dst_name, folder) + src_repo.getsep(), dst_repo.getsep()) + if newsrc_name != newdst_name: + raise OfflineImapError("INFINITE FOLDER CREATION DETECTED! " + "Folder '%s' (repository '%s') would be created as fold" + "er '%s' (repository '%s'). The atter becomes '%s' in r" + "eturn, leading to infinite folder creation cycles.\n S" + "OLUTION: 1) Do set your nametrans rules on both reposi" + "tories so they lead to identical names if applied back" + " and forth. 2) Use folderfilter settings on a reposito" + "ry to prevent some folders from being created on the o" "ther side." % (dst_folder, dst_repo, newsrc_name, + src_repo, newdst_name), + OfflineImapError.ERROR.REPO) # end sanity check, actually create the folder try: From eeef8f4bab057e1e73ab7d1d17898643dee003aa Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 15:07:34 +0200 Subject: [PATCH 276/817] Don't ask for hostname if using a tunnel In tunnel mode, we don't need to ask for a hostname because there is no need for one. Fix this. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 506f886..dc0bc50 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -60,7 +60,7 @@ class IMAPServer: self.password = None self.passworderror = None self.goodpassword = None - self.hostname = repos.gethost() + self.hostname = None if self.tunnel else repos.gethost() self.port = repos.getport() if self.port == None: self.port = 993 if self.usessl else 143 From 050bbb3d2d759da8b2785c2a5efd0ee529a1c42b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 15:21:27 +0200 Subject: [PATCH 277/817] Reset draft Changelog post-release Empty it, so new stuff can pour in. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 16655a1..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,33 +13,8 @@ others. New Features ------------ -* Implement per-account locking, so that it will possible to sync - different accounts at the same time. The old global lock is still in - place for backward compatibility reasons (to be able to run old and - new versions of OfflineImap concurrently) and will be removed in the - future. Starting with this version, OfflineImap will be - forward-compatible with the per-account locking style. - -* Implement RFC 2595 LOGINDISABLED. Warn the user and abort when we - attempt a plaintext login but the server has explicitly disabled - plaintext logins rather than crashing. - -* Folders will now also be automatically created on the REMOTE side of - an account if they exist on the local side. Use the folderfilters - setting on the local side to prevent some folders from migrating to - the remote side. Also, if you have a nametrans setting on the remote - repository, you might need a nametrans setting on the local repository - that leads to the original name (reverse nametrans). - Changes ------- -* Documentation improvements concerning 'restoreatime' and some code cleanup - -* Maildir repositories now also respond to folderfilter= configurations. - Bug Fixes --------- - -* New emails are not created with "-rwxr-xr-x" but as "-rw-r--r--" - anymore, fixing a regression in 6.3.4. From 48fdb1441892e1fdade1e13cd5679755bee53a81 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 19:59:39 +0200 Subject: [PATCH 278/817] Don't output thread ID in log The thread ID is not really useful and looks ugly. It also makes lines longer than needed, there is more useful information we can put in the log. So do away with it. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 5017934..52b1788 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -104,11 +104,10 @@ class UIBase: ui.error(exc, sys.exc_info()[2], msg="While syncing Folder %s in " "repo %s") """ - cur_thread = threading.currentThread() if msg: - self._msg("ERROR [%s]: %s\n %s" % (cur_thread, msg, exc)) + self._msg("ERROR: %s\n %s" % (msg, exc)) else: - self._msg("ERROR [%s]: %s" % (cur_thread, exc)) + self._msg("ERROR: %s" % (exc)) if not self.debuglist: # only output tracebacks in debug mode From ee82cd135b51059400d03255c83690500286d33d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 20:07:56 +0200 Subject: [PATCH 279/817] Fix typo atter->latter Simply typo in user error message. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index ebd024f..e5c3f1c 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -181,9 +181,9 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): if newsrc_name != newdst_name: raise OfflineImapError("INFINITE FOLDER CREATION DETECTED! " "Folder '%s' (repository '%s') would be created as fold" - "er '%s' (repository '%s'). The atter becomes '%s' in r" - "eturn, leading to infinite folder creation cycles.\n S" - "OLUTION: 1) Do set your nametrans rules on both reposi" + "er '%s' (repository '%s'). The latter becomes '%s' in " + "return, leading to infinite folder creation cycles.\n " + "SOLUTION: 1) Do set your nametrans rules on both reposi" "tories so they lead to identical names if applied back" " and forth. 2) Use folderfilter settings on a reposito" "ry to prevent some folders from being created on the o" "ther side." % (dst_folder, dst_repo, newsrc_name, From b0e88622c4b734dc87d2cc44aba576de62ff40f1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 21:06:54 +0200 Subject: [PATCH 280/817] Fix think on how to compare folder names on src and dest It is not easy to think through when to use visiblenames() and whatnot. It seems I managed to not think it through properly. Which might be fixed now. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index e5c3f1c..b5e91d8 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -153,7 +153,7 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): src_repo.getsep(), dst_repo.getsep())] = folder dst_hash = {} for folder in dst_folders: - dst_hash[folder.getvisiblename()] = folder + dst_hash[folder.name] = folder # Find new folders on src_repo. for src_name, src_folder in src_hash.iteritems(): @@ -173,21 +173,23 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): # nametrans sanity check! # Does nametrans back&forth lead to identical names? #src_name is the unmodified full src_name - newsrc_name = dst_name.replace(dst_repo.getsep(), - src_repo.getsep()) + newsrc_name = dst_folder.getvisiblename().replace( + dst_repo.getsep(), + src_repo.getsep()) folder = self.getfolder(newsrc_name) newdst_name = folder.getvisiblename().replace( src_repo.getsep(), dst_repo.getsep()) - if newsrc_name != newdst_name: + if dst_name != newdst_name: raise OfflineImapError("INFINITE FOLDER CREATION DETECTED! " "Folder '%s' (repository '%s') would be created as fold" "er '%s' (repository '%s'). The latter becomes '%s' in " "return, leading to infinite folder creation cycles.\n " - "SOLUTION: 1) Do set your nametrans rules on both reposi" - "tories so they lead to identical names if applied back" - " and forth. 2) Use folderfilter settings on a reposito" - "ry to prevent some folders from being created on the o" "ther side." % (dst_folder, dst_repo, newsrc_name, - src_repo, newdst_name), + "SOLUTION: 1) Do set your nametrans rules on both repos" + "itories so they lead to identical names if applied bac" + "k and forth. 2) Use folderfilter settings on a reposit" + "ory to prevent some folders from being created on the " + "other side." % (dst_name, dst_repo, newsrc_name, + src_repo, newdst_name), OfflineImapError.ERROR.REPO) # end sanity check, actually create the folder From 1c58ebe3484fa72e541acba14ff5498df63c77a7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 19 Sep 2011 21:25:50 +0200 Subject: [PATCH 281/817] Do not create folder on REMOTE if it would fall in REMOTE's folderfilter Previously, we only checked if a LOCAL folder falls under the local repositories folderfilter rule when deciding whether a folder should be created on REMOTE. However, we also do not want to create the folder on REMOTE if it would fall under a folderfilter rule there. This patch prevents us from doing so. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index b5e91d8..a08ef32 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -177,6 +177,13 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): dst_repo.getsep(), src_repo.getsep()) folder = self.getfolder(newsrc_name) + # would src repo filter out the new folder name? In this + # case don't create it on it: + if not self.folderfilter(newsrc_name): + self.ui.debug('', "Not creating folder '%s' (repository '%s" + "') as it would be filtered out on that repository." % + (newsrc_name, self)) + continue newdst_name = folder.getvisiblename().replace( src_repo.getsep(), dst_repo.getsep()) if dst_name != newdst_name: From db28cb77e7a555e889a3c6057f58697859f94ecc Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 20 Sep 2011 22:25:45 +0200 Subject: [PATCH 282/817] Fix another visiblename() glitch in determining the foldername to be created Commit b0e88622c4b changed dst_hash[folder.visiblename] to dst_hash[folder.name] but we did not adapt all places where it is needed to use visiblename again. This led to attempting to create a name on REMOTE ignoring the nametrans setting on the LOCAL repository. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index a08ef32..0da307f 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -172,7 +172,7 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): if dst_folder.sync_this and not dst_name in src_hash: # nametrans sanity check! # Does nametrans back&forth lead to identical names? - #src_name is the unmodified full src_name + #src_name is the unmodified full src_name that would be created newsrc_name = dst_folder.getvisiblename().replace( dst_repo.getsep(), src_repo.getsep()) @@ -184,6 +184,7 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): "') as it would be filtered out on that repository." % (newsrc_name, self)) continue + # apply reverse nametrans to see if we end up with the same name newdst_name = folder.getvisiblename().replace( src_repo.getsep(), dst_repo.getsep()) if dst_name != newdst_name: @@ -201,15 +202,14 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): # end sanity check, actually create the folder try: - src_repo.makefolder(dst_name.replace( - dst_repo.getsep(), src_repo.getsep())) + src_repo.makefolder(newsrc_name) except OfflineImapError, e: self.ui.error(e, exc_info()[2], "Creating folder %s on repository %s" %\ (src_name, dst_repo)) raise - status_repo.makefolder(dst_name.replace( - dst_repo.getsep(), status_repo.getsep())) + status_repo.makefolder(newsrc_name.replace( + src_repo.getsep(), status_repo.getsep())) # Find deleted folders. # We don't delete folders right now. From 347e1eaa3228a5aeac8d5375189646a5fb564f31 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 21 Sep 2011 02:16:33 +0200 Subject: [PATCH 283/817] update CAPABILITIES after login Some Webservers (I am looking at you Gmail) send different capabilities before and after login, so they can tailor their server capabilities to the user. While legal, this is uncommon and we were not updating our server capabilities. Doing so allows us to detect that Gmail actually supports the UIDPLUS extension, and we will stop mangling headers when uploading to Gmail. This could lead to some performance gains when we upload many messages to Gmail. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ offlineimap/imapserver.py | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..9bae1d8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,5 +16,9 @@ New Features Changes ------- +* Refresh server capabilities after login, so we know that Gmail + supports UIDPLUS (it only announces that after login, not + before). This prevents us from adding custom headers to Gmail uploads. + Bug Fixes --------- diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index dc0bc50..ead3280 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -1,6 +1,5 @@ # IMAP server support -# Copyright (C) 2002 - 2007 John Goerzen -# +# Copyright (C) 2002 - 2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -276,6 +275,11 @@ class IMAPServer: self.passworderror = str(val) raise + # update capabilities after login, e.g. gmail serves different ones + typ, dat = imapobj.capability() + if dat != [None]: + imapobj.capabilities = tuple(dat[-1].upper().split()) + if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: From 275dbfa3fb1349c01f813bf8fe4bd812db671433 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 21 Sep 2011 02:08:09 +0200 Subject: [PATCH 284/817] Release 6.3.5-rc3 Update Changelog and __VERSION__ for 6.3.5-rc3. Also add in the notes for rc2 to the Changelog. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ---- Changelog.rst | 52 +++++++++++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 9bae1d8..b532734 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -15,10 +15,6 @@ New Features Changes ------- - -* Refresh server capabilities after login, so we know that Gmail - supports UIDPLUS (it only announces that after login, not - before). This prevents us from adding custom headers to Gmail uploads. Bug Fixes --------- diff --git a/Changelog.rst b/Changelog.rst index 4bdff19..05c9a57 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,58 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.3.5-rc3 (2011-09-21) +=================================== + +Changes +------- + +* Refresh server capabilities after login, so we know that Gmail + supports UIDPLUS (it only announces that after login, not + before). This prevents us from adding custom headers to Gmail uploads. + +Bug Fixes +--------- + +* Fix the creation of folders on remote repositories, which was still + botched on rc2. + +OfflineIMAP v6.3.5-rc2 (2011-09-19) +=================================== + +New Features +------------ + +* Implement per-account locking, so that it will possible to sync + different accounts at the same time. The old global lock is still in + place for backward compatibility reasons (to be able to run old and + new versions of OfflineImap concurrently) and will be removed in the + future. Starting with this version, OfflineImap will be + forward-compatible with the per-account locking style. + +* Implement RFC 2595 LOGINDISABLED. Warn the user and abort when we + attempt a plaintext login but the server has explicitly disabled + plaintext logins rather than crashing. + +* Folders will now also be automatically created on the REMOTE side of + an account if they exist on the local side. Use the folderfilters + setting on the local side to prevent some folders from migrating to + the remote side. Also, if you have a nametrans setting on the remote + repository, you might need a nametrans setting on the local repository + that leads to the original name (reverse nametrans). + +Changes +------- + +* Documentation improvements concerning 'restoreatime' and some code cleanup + +* Maildir repositories now also respond to folderfilter= configurations. + +Bug Fixes +--------- + +* New emails are not created with "-rwxr-xr-x" but as "-rw-r--r--" + anymore, fixing a regression in 6.3.4. OfflineIMAP v6.3.5-rc1 (2011-09-12) =================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 4eeb648..f5f094c 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.5-rc1" +__version__ = "6.3.5-rc3" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From e30ae53b2a6d8f30fbe536c5425297e3de29754c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 23 Sep 2011 14:27:25 +0200 Subject: [PATCH 285/817] Folder: Force top-level folder visiblename to '' nametrans rules can lead to different visiblename names for the top-level directory, specifically both '.' and '' (the latter was recently introduced). However, we need to be able to compare folder names to see if we need to create a new directory or whether a directory already exists, so we need to be able to compare a repositories visiblename (=transposed via nametrans rule) with another folder. To make the top-level directory comparison happen, we enforce a top-level name of '', so that comparisons work. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index d9fed70..909db30 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -39,6 +39,11 @@ class BaseFolder(object): self.name = name self.repository = repository self.visiblename = repository.nametrans(name) + # In case the visiblename becomes '.' (top-level) we use '' as + # that is the name that e.g. the Maildir scanning will return + # for the top-level dir. + if self.visiblename == '.': + self.visiblename = '' self.config = repository.getconfig() def getname(self): From 3d00a8bb4ad8101661736bb0ef9a2d43212ee84d Mon Sep 17 00:00:00 2001 From: Thomas Kahle Date: Fri, 23 Sep 2011 01:20:36 +0100 Subject: [PATCH 286/817] MANUAL: Improve documentation on using python code in the config Hi, the following patch updates the documentation. It was a bit out of context since Sebastian originally just copied it from my webpage. Also, the code for the helper functions is available there under "Download". I did not include it because it would leave larger chunks of python code in the manual which seems weird. Signed-off-by: Thomas Kahle Signed-off-by: Sebastian Spaeth --- docs/MANUAL.rst | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 928ae40..a5a1fb3 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -476,7 +476,8 @@ To only get the All Mail folder from a Gmail account, you would e.g. do:: Another nametrans transpose example ----------------------------------- -Put everything in a GMX. subfolder except for the boxes INBOX, Draft, and Sent which should keep the same name:: +Put everything in a GMX. subfolder except for the boxes INBOX, Draft, +and Sent which should keep the same name:: nametrans: lambda folder: folder if folder in ['INBOX', 'Drafts', 'Sent'] \ else re.sub(r'^', r'GMX.', folder) @@ -484,7 +485,9 @@ Put everything in a GMX. subfolder except for the boxes INBOX, Draft, and Sent w 2 IMAP using name translations ------------------------------ -Synchronizing 2 IMAP accounts to local Maildirs that are "next to each other", so that mutt can work on both. Full email setup described by Thomas Kahle at `http://dev.gentoo.org/~tomka/mail.html`_ +Synchronizing 2 IMAP accounts to local Maildirs that are "next to each +other", so that mutt can work on both. Full email setup described by +Thomas Kahle at `http://dev.gentoo.org/~tomka/mail.html`_ offlineimap.conf:: @@ -534,11 +537,25 @@ offlineimap.conf:: ssl = yes maxconnections = 2 -One of the coolest things about offlineimap is that you can inject arbitrary python code. The file specified with:: +One of the coolest things about offlineimap is that you can call +arbitrary python code from your configuration. To do this, specify a +pythonfile with:: pythonfile=~/bin/offlineimap-helpers.py -contains python functions that I used for two purposes: Fetching passwords from the gnome-keyring and translating folder names on the server to local foldernames. The python file should contain all the functions that are called here. get_username and get_password are part of the interaction with gnome-keyring and not printed here. Find them in the example file that is in the tarball or here. The folderfilter is a lambda term that, well, filters which folders to get. `oimaptransfolder_acc2` translates remote folders into local folders with a very simple logic. The `INBOX` folder will simply have the same name as the account while any other folder will have the account name and a dot as a prefix. offlineimap handles the renaming correctly in both directions:: +Your pythonfile needs to contain implementations for the functions +that you want to use in offflineimaprc. The example uses it for two +purposes: Fetching passwords from the gnome-keyring and translating +folder names on the server to local foldernames. An example +implementation of get_username and get_password showing how to query +gnome-keyring is contained in +`http://dev.gentoo.org/~tomka/mail-setup.tar.bz2`_ The folderfilter is +a lambda term that, well, filters which folders to get. The function +`oimaptransfolder_acc2` translates remote folders into local folders +with a very simple logic. The `INBOX` folder will have the same name +as the account while any other folder will have the account name and a +dot as a prefix. This is useful for hierarchichal display in mutt. +Offlineimap handles the renaming correctly in both directions:: import re def oimaptransfolder_acc1(foldername): From 248f23afd6cf6f96c82a12e49d32ff3b00223f87 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 26 Sep 2011 15:14:10 +0200 Subject: [PATCH 287/817] Do not fail calling dequote() with empty string A report by Dave Abrahams showed that the dequote() function failed when invoked with an empty string. This fixes the function to be more robust. Signed-off-by: Sebastian Spaeth --- offlineimap/imaputil.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index eae9a76..f583312 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -34,16 +34,15 @@ def debug(*args): getglobalui().debug('imap', " ".join(msg)) def dequote(string): - """Takes a string which may or may not be quoted and returns it, unquoted. - This function does NOT consider parenthised lists to be quoted. - """ + """Takes string which may or may not be quoted and unquotes it. - if not (string[0] == '"' and string[-1] == '"'): - return string - string = string[1:-1] # Strip off quotes. - string = string.replace('\\"', '"') - string = string.replace('\\\\', '\\') - debug("dequote() returning:", string) + It only considers double quotes. This function does NOT consider + parenthised lists to be quoted. + """ + if string and string.startswith('"') and string.endswith('"'): + string = string[1:-1] # Strip off the surrounding quotes. + string = string.replace('\\"', '"') + string = string.replace('\\\\', '\\') return string def flagsplit(string): From 1b6d76345a772d53bde806a2ef71c81b2d29b8c9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 26 Sep 2011 15:27:04 +0200 Subject: [PATCH 288/817] Robustify error msg against more failure When syncfolder() fails, we output an error message containing the foldername per the localfolder variable. However, the localfolder variable is assigned inside our try: block and when the error occurs there, we will have no localfolder variable to use for output. This caused the errormsg to cause an Exception itself which unhelpfully distracts from the root cause of the error. Reconstruct the folder name in a bit more complex way, but in a way so it is guaranteed to work (by relying on parameters passed in to the function). Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 13bb12e..d28bd34 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -420,8 +420,15 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, if e.severity > OfflineImapError.ERROR.FOLDER: raise else: - ui.error(e, exc_info()[2], msg = "Aborting folder sync '%s' " - "[acc: '%s']" % (localfolder, accountname)) + #if the initial localfolder assignement bailed out, the localfolder var will not be available, so we need + ui.error(e, exc_info()[2], msg = "Aborting sync, folder '%s' " + "[acc: '%s']" % ( + remotefolder.getvisiblename().\ + replace(remoterepos.getsep(), localrepos.getsep()), + accountname)) + # we reconstruct foldername above rather than using + # localfolder, as the localfolder var is not + # available if assignment fails. except Exception, e: ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ (accountname,remotefolder.getvisiblename(), From 953c58a9c917f467d7256945e207e72f59227a13 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 26 Sep 2011 15:57:35 +0200 Subject: [PATCH 289/817] Robustify (&fix) error throwing on APPEND If APPEND raises abort(), the (typ, dat) variables will not be set, so we should not be using it for the OfflineImapError Exception string. Fixing and prettifying the string formatting a bit at the same time. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 041f5a0..69b202c 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -541,21 +541,18 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() if not retry_left: raise OfflineImapError("Saving msg in folder '%s', " - "repository '%s' failed. Server reponded; %s %s\n" + "repository '%s' failed. Server reponded: %s\n" "Message content was: %s" % - (self, self.getrepository(), - typ, dat, dbg_output), + (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) retry_left -= 1 - except imapobj.error, e: - # If the server responds with 'BAD', append() raise()s directly. - # So we need to prepare a response ourselves. - typ, dat = 'BAD', str(e) - if typ != 'OK': #APPEND failed - raise OfflineImapError("Saving msg in folder '%s', repository " - "'%s' failed. Server reponded; %s %s\nMessage content was:" - " %s" % (self, self.getrepository(), typ, dat, dbg_output), - OfflineImapError.ERROR.MESSAGE) + except imapobj.error, e: # APPEND failed + # If the server responds with 'BAD', append() + # raise()s directly. So we catch that too. + raise OfflineImapError("Saving msg folder '%s', repo '%s'" + "failed. Server reponded: %s\nMessage content was: " + "%s" % (self, self.getrepository(), str(e), dbg_output), + OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. (typ,dat) = imapobj.check() From e145beb394b84f533b1762dc9a1779463a9502cd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 26 Sep 2011 16:04:00 +0200 Subject: [PATCH 290/817] Fix "command CHECK illegal in state AUTH" Dave identified a case where our new dropped connection handling did not work out correctly: we use the retry_left variable to signify success (0=success if no exception occured). However, we were decrementing the variable AFTER all the exception checks, so if there was one due to a dropped connection, it could well be that we 1) did not raise an exception (because we want to retry), and 2) then DECREMENTED retry_left, which indicated "all is well, no need to retry". The code then continued to check() the append, which failed with the above message (because we obtained a new connection which had not even selected the current folder and we were still in mode AUTH). The fix is of course, to fix our logic: Decrement retry_left first, THEN decide whether to raise() (retry_left==0) or retry (retry_left>0) which would then correctly attempt another loop. I am sorry for this newbie type of logic error. The retry count loop was too hastily slipped in, it seems. Reported-by: Dave Abrahams Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 69b202c..00361b0 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -536,7 +536,7 @@ class IMAPFolder(BaseFolder): retry_left = 0 # Mark as success except imapobj.abort, e: # connection has been reset, release connection and retry. - self.ui.error(e, exc_info()[2]) + retry_left -= 1 self.imapserver.releaseconnection(imapobj, True) imapobj = self.imapserver.acquireconnection() if not retry_left: @@ -545,7 +545,8 @@ class IMAPFolder(BaseFolder): "Message content was: %s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) - retry_left -= 1 + self.ui.error(e, exc_info()[2]) + except imapobj.error, e: # APPEND failed # If the server responds with 'BAD', append() # raise()s directly. So we catch that too. From ae8a1cb79f0027e7ef6e842c8c60cec002dc02b6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 27 Sep 2011 12:58:23 +0200 Subject: [PATCH 291/817] Remove deprecated calls to apply() apply() has been deprecated since Python 2.3, and won't be working in python 3 anymore. Use the functional equivalent throughout. Signed-off-by: Sebastian Spaeth --- offlineimap/CustomConfig.py | 13 ++++++------- offlineimap/threadutil.py | 3 +-- offlineimap/ui/Curses.py | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 42132f8..dae20b6 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -24,26 +24,25 @@ class CustomConfigParser(SafeConfigParser): """Same as config.get, but returns the "default" option if there is no such option specified.""" if self.has_option(section, option): - return apply(self.get, [section, option] + list(args), kwargs) + return self.get(*(section, option) + args, **kwargs) else: return default def getdefaultint(self, section, option, default, *args, **kwargs): if self.has_option(section, option): - return apply(self.getint, [section, option] + list(args), kwargs) + return self.getint (*(section, option) + args, **kwargs) else: return default def getdefaultfloat(self, section, option, default, *args, **kwargs): if self.has_option(section, option): - return apply(self.getfloat, [section, option] + list(args), kwargs) + return self.getfloat(*(section, option) + args, **kwargs) else: return default def getdefaultboolean(self, section, option, default, *args, **kwargs): if self.has_option(section, option): - return apply(self.getboolean, [section, option] + list(args), - kwargs) + return self.getboolean(*(section, option) + args, **kwargs) else: return default @@ -91,9 +90,9 @@ class ConfigHelperMixin: def _confighelper_runner(self, option, default, defaultfunc, mainfunc): """Return config value for getsection()""" if default == CustomConfigDefault: - return apply(mainfunc, [self.getsection(), option]) + return mainfunc(*[self.getsection(), option]) else: - return apply(defaultfunc, [self.getsection(), option, default]) + return defaultfunc(*[self.getsection(), option, default]) def getconf(self, option, diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index bb3b63c..38387e9 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -214,8 +214,7 @@ def initInstanceLimit(instancename, instancemax): class InstanceLimitedThread(ExitNotifyThread): def __init__(self, instancename, *args, **kwargs): self.instancename = instancename - - apply(ExitNotifyThread.__init__, (self,) + args, kwargs) + super(InstanceLimitedThread, self).__init__(*args, **kwargs) def start(self): instancelimitedsems[self.instancename].acquire() diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index afe3acb..84b0e9a 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -66,7 +66,7 @@ class CursesUtil: """Perform an operation with full locking.""" self.lock() try: - apply(target, args, kwargs) + target(*args, **kwargs) finally: self.unlock() From 4bfc1e82275233ff11a45bc01e8fdd3bf5f92319 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 27 Sep 2011 14:08:20 +0200 Subject: [PATCH 292/817] except imapobj.abort() -> except imapobj.abort When checking for the IMAP4.abort() exception, we need of course to perform: except imapobj.abort: and not except imapobj.abort(): Thanks to Johannes Stezenbach for pointing to the glitch. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 2 +- offlineimap/imapserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 00361b0..8dbd7aa 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -209,7 +209,7 @@ class IMAPFolder(BaseFolder): res_type, data = imapobj.uid('fetch', str(uid), '(BODY.PEEK[])') fails_left = 0 - except imapobj.abort(), e: + except imapobj.abort, e: # Release dropped connection, and get a new one self.imapserver.releaseconnection(imapobj, True) imapobj = self.imapserver.acquireconnection() diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index ead3280..ef54e31 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -549,7 +549,7 @@ class IdleThread(object): try: # End IDLE mode with noop, imapobj can point to a dropped conn. imapobj.noop() - except imapobj.abort(): + except imapobj.abort: self.ui.warn('Attempting NOOP on dropped connection %s' % \ imapobj.identifier) self.parent.releaseconnection(imapobj, True) From c1120c9158f273fe691f89b9eefc34b2623d5a7a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 11:20:57 +0200 Subject: [PATCH 293/817] 6.4.0 release Signed-off-by: Sebastian Spaeth --- Changelog.rst | 7 +++++++ offlineimap/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 05c9a57..b301c20 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,13 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.0 (2011-09-29) +=============================== + +This is the first stable release to support the forward-compatible per-account locks and remote folder creation that has been introduced in the 6.3.5 series. + +* Various regression and bug fixes from the last couple of RCs + OfflineIMAP v6.3.5-rc3 (2011-09-21) =================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index f5f094c..46ff1d7 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.5-rc3" +__version__ = "6.4.0" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From b50668434e56dcff3babe2e2dc8ba2b7366449be Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 11:45:38 +0200 Subject: [PATCH 294/817] docs/*: Documentation enhancements Improve wordings and descriptions all around. Signed-off-by: Sebastian Spaeth --- docs/FAQ.rst | 153 +++++++++++++++++++++--------------------------- docs/MANUAL.rst | 115 +++++++++++++++++++++++++++++++++--- 2 files changed, 173 insertions(+), 95 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 3181a8a..e3a586b 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -49,50 +49,40 @@ increase performance of the sync. Don’t set the number too high. If you do that, things might actually slow down as your link gets saturated. Also, too many connections can cause mail servers to have excessive load. Administrators might take unkindly to this, and the -server might bog down. There are many variables in the optimal setting; -experimentation may help. +server might bog down. There are many variables in the optimal setting; experimentation may help. -An informal benchmark yields these results for my setup:: - - * 10 minutes with MacOS X Mail.app “manual cache” - * 5 minutes with GNUS agent sync - * 20 seconds with OfflineIMAP 1.x - * 9 seconds with OfflineIMAP 2.x - * 3 seconds with OfflineIMAP 3.x “cold start” - * 2 seconds with OfflineIMAP 3.x “held connection” +See the Performance section in the MANUAL for some tips. What platforms does OfflineIMAP support? ---------------------------------------- -It should run on most platforms supported by Python, which are quite a few. I -do not support Windows myself, but some have made it work there. Use on -Windows +It should run on most platforms supported by Python, with one exception: we do not support Windows, but some have made it work there. -These answers have been reported by OfflineIMAP users. I do not run OfflineIMAP -on Windows myself, so I can’t directly address their accuracy. +The following has been reported by OfflineIMAP users. We do not test +OfflineIMAP on Windows, so we can’t directly address their accuracy. The basic answer is that it’s possible and doesn’t require hacking OfflineIMAP source code. However, it’s not necessarily trivial. The information below is -based in instructions submitted by Chris Walker. +based in instructions submitted by Chris Walker:: -First, you must run OfflineIMAP in the Cygwin environment. The Windows -filesystem is not powerful enough to accomodate Maildir by itself. - -Next, you’ll need to mount your Maildir directory in a special way. There is -information for doing that at http://barnson.org/node/295. That site gives this -example:: - - mount -f -s -b -o managed "d:/tmp/mail" "/home/of/mail" - -That URL also has more details on making OfflineIMAP work with Windows. + First, you must run OfflineIMAP in the Cygwin environment. The Windows + filesystem is not powerful enough to accomodate Maildir by itself. + + Next, you’ll need to mount your Maildir directory in a special + way. There is information for doing that at + http://barnson.org/node/295. That site gives this example:: + + mount -f -s -b -o managed "d:/tmp/mail" "/home/of/mail" + + That URL also has more details on making OfflineIMAP work with Windows. Does OfflineIMAP support mbox, mh, or anything else other than Maildir? ----------------------------------------------------------------------- -Not directly. Maildir was the easiest to implement. I’m not planning to write -mbox code for OfflineIMAP, though if someone sent me well-written mbox support -and pledged to support it, I’d commit it to the tree. +Not directly. Maildir was the easiest to implement. We are not planning +to write an mbox-backend, though if someone sent me well-written mbox +support and pledged to support it, it would be committed it to the tree. However, OfflineIMAP can directly sync accounts on two different IMAP servers together. So you could install an IMAP server on your local machine that @@ -105,22 +95,25 @@ point your mail readers to that IMAP server on localhost. What is the UID validity problem for folder? -------------------------------------------- -IMAP servers use a unique ID (UID) to refer to a specific message. This number -is guaranteed to be unique to a particular message forever. No other message in -the same folder will ever get the same UID. UIDs are an integral part of -`OfflineIMAP`_'s synchronization scheme; they are used to match up messages on -your computer to messages on the server. +IMAP servers use a folders UIDVALIDITY value in combination with a +unique ID (UID) to refer to a specific message. This is guaranteed to +be unique to a particular message forever. No other message in the same +folder will ever get the same UID as long as UIDVALIDITY remains +unchanged. UIDs are an integral part of `OfflineIMAP`_'s +synchronization scheme; they are used to match up messages on your +computer to messages on the server. -Sometimes, the UIDs on the server might get reset. Usually this will happen if -you delete and then recreate a folder. When you create a folder, the server -will often start the UID back from 1. But `OfflineIMAP`_ might still have the -UIDs from the previous folder by the same name stored. `OfflineIMAP`_ will -detect this condition and skip the folder. This is GOOD, because it prevents -data loss. +Sometimes, the UIDs on the server might get reset. Usually this will +happen if you delete and then recreate a folder. When you create a +folder, the server will often start the UID back from 1. But +`OfflineIMAP`_ might still have the UIDs from the previous folder by the +same name stored. `OfflineIMAP`_ will detect this condition because of +the changed UIDVALIDITY value and skip the folder. This is GOOD, +because it prevents data loss. -You can fix it by removing your local folder and cache data. For instance, if -your folders are under `~/Folders` and the folder with the problem is INBOX, -you'd type this:: +In the IMAP<->Maildir case, you can fix it by removing your local folder +and cache data. For instance, if your folders are under `~/Folders` and +the folder with the problem is INBOX, you'd type this:: rm -r ~/Folders/INBOX rm -r ~/.offlineimap/Account-AccountName/LocalStatus/INBOX @@ -146,12 +139,10 @@ This question comes up frequently on the `mailing list`_. You can find a detail discussion of the problem there http://lists.complete.org/offlineimap@complete.org/2003/04/msg00012.html.gz. -How do I add or delete a folder? --------------------------------- +How do I automatically delete a folder? +--------------------------------------- -OfflineIMAP does not currently provide this feature. However, if you create a -new folder on the remote server, OfflineIMAP will detect this and create the -corresponding folder locally automatically. +OfflineIMAP does not currently provide this feature. You will have to delete folders manually. See next entry too. May I delete local folders? --------------------------- @@ -175,6 +166,7 @@ It will perform a check on startup and abort if another `OfflineIMAP`_ is already running. If you need to schedule synchronizations, you'll probably find autorefresh settings more convenient than cron. Alternatively, you can set a separate metadata directory for each instance. +In the future, we will lock each account individually rather than having one global lock. Can I copy messages between folders? --------------------------------------- @@ -197,10 +189,7 @@ next run it will upload the message to second server and delete on first, etc. Does OfflineIMAP support POP? ----------------------------- -No. POP is not robust enough to do a completely reliable multi-machine sync -like OfflineIMAP can do. - -OfflineIMAP will never support POP. +No. How is OfflineIMAP conformance? ------------------------------- @@ -214,17 +203,21 @@ How is OfflineIMAP conformance? Can I force OfflineIMAP to sync a folder right now? --------------------------------------------------- -Yes, if you use the `Blinkenlights` UI. That UI shows the active accounts +Yes, + 1) if you use the `Blinkenlights` UI. That UI shows the active accounts as follows:: - 4: [active] *Control: . - 3: [ 4:36] personal: - 2: [ 3:37] work: . - 1: [ 6:28] uni: + 4: [active] *Control: . + 3: [ 4:36] personal: + 2: [ 3:37] work: . + 1: [ 6:28] uni: -Simply press the appropriate digit (`3` for `personal`, etc.) to resync that -account immediately. This will be ignored if a resync is already in progress -for that account. + Simply press the appropriate digit (`3` for `personal`, etc.) to + resync that account immediately. This will be ignored if a resync is + already in progress for that account. + + 2) while in sleep mode, you can also send a SIGUSR1. See the `Signals + on UNIX`_ section in the MANUAL for details. Configuration Questions ======================= @@ -248,10 +241,12 @@ what folders are present on the IMAP server and synchronize them. You can use the folderfilter and nametrans configuration file options to request only certain folders and rename them as they come in if you like. +Also you can configure OfflineImap to only synchronize "subscribed" folders. + How do I prevent certain folders from being synced? --------------------------------------------------- -Use the folderfilter option. +Use the folderfilter option. See the MANUAL for details and examples. What is the mailbox name recorder (mbnames) for? ------------------------------------------------ @@ -261,9 +256,11 @@ Some mail readers, such as mutt, are not capable of automatically determining th Does OfflineIMAP verify SSL certificates? ----------------------------------------- -By default, no. However, as of version 6.3.2, it is possible to enforce verification -of SSL certificate on a per-repository basis by setting the `sslcacertfile` option in the -config file. (See the example offlineimap.conf for details.) +You can verify an imapserver's certificate by specifying the CA +certificate on a per-repository basis by setting the `sslcacertfile` +option in the config file. (See the example offlineimap.conf for +details.) If you do not specify any CA certificate, you will be presented with the server's certificate fingerprint and add that to the configuration file, to make sure it remains unchanged. +No verification happens if connecting via STARTTLS. How do I generate an `sslcacertfile` file? ------------------------------------------ @@ -293,23 +290,6 @@ with the IMAP RFCs. Some servers provide imperfect compatibility that may be good enough for general clients. OfflineIMAP needs more features, specifically support for UIDs, in order to do its job accurately and completely. -Microsoft Exchange ------------------- - -Several users have reported problems with Microsoft Exchange servers in -conjunction with OfflineIMAP. This generally seems to be related to the -Exchange servers not properly following the IMAP standards. - -Mark Biggers has posted some information to the OfflineIMAP `mailing list`_ -about how he made it work. - -Other users have indicated that older (5.5) releases of Exchange are so bad -that they will likely not work at all. - -I do not have access to Exchange servers for testing, so any problems with it, -if they can even be solved at all, will require help from OfflineIMAP users to -find and fix. - Client Notes ============ @@ -494,12 +474,13 @@ To send a patch, we recommend using 'git send-email'. Where from should my patches be based on? ----------------------------------------- -Depends. If you're not sure, it should start off of the master branch. master is -the branch where new patches should be based on by default. +Depends. If you're not sure, it should start off of the master +branch. master is the branch where new patches should be based on by +default. -Obvious materials for next release (e.g. new features) start off of current -next. Also, next is the natural branch to write patches on top of commits not -already in master. +Obvious materials for next release (e.g. new features) start off of +current next. Also, next is the natural branch to write patches on top +of commits not already in master. A fix for a very old bug or security issue may start off of maint. This isn't needed since such fix are backported by the maintainer, though. diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index a5a1fb3..22ddc95 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -6,7 +6,7 @@ Powerful IMAP/Maildir synchronization and reader support -------------------------------------------------------- -:Author: John Goerzen +:Author: John Goerzen & contributors :Date: 2011-01-15 :Copyright: GPL v2 :Manual section: 1 @@ -22,10 +22,8 @@ emails between them, so that you can read the same mailbox from multiple computers. The REMOTE repository is some IMAP server, while LOCAL can be either a local Maildir or another IMAP server. -Missing folders will be automatically created on the LOCAL side, however -NO folders will currently be created on the REMOTE repository -automatically (it will sync your emails from local folders if -corresponding REMOTE folders already exist). +Missing folders will be automatically created on both sides if +needed. No folders will be deleted at the moment. Configuring OfflineImap in basic mode is quite easy, however it provides an amazing amount of flexibility for those with special needs. You can @@ -159,12 +157,10 @@ Blinkenlights Blinkenlights is an interface designed to be sleek, fun to watch, and informative of the overall picture of what OfflineIMAP is doing. - Blinkenlights contains a row of "LEDs" with command buttons and a log. The log shows more detail about what is happening and is color-coded to match the color of the lights. - Each light in the Blinkenlights interface represents a thread of execution -- that is, a particular task that OfflineIMAP is performing right now. The colors indicate what task the particular thread is performing, and are as follows: @@ -232,7 +228,7 @@ English-speaking world. One version ran in its entirety as follows: TTYUI ---------- +------ TTYUI interface is for people running in terminals. It prints out basic status messages and is generally friendly to use on a console or xterm. @@ -245,7 +241,7 @@ Basic is designed for situations in which OfflineIMAP will be run non-attended and the status of its execution will be logged. This user interface is not capable of reading a password from the keyboard; account passwords must be specified using one of the configuration file -options. +options. For example, it will not print periodic sleep announcements and tends to be a tad less verbose, in general. Quiet @@ -367,6 +363,107 @@ accounts will abort any current sleep and will exit after a currently running synchronization has finished. This signal can be used to gracefully exit out of a running offlineimap "daemon". +Folder filtering and Name translation +===================================== + +OfflineImap provides advanced and potentially complex possibilities for +filtering and translating folder names. If you don't need this, you can +safely skip this section. + +folderfilter +------------ + +If you do not want to synchronize all your filters, you can specify a folderfilter function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. + +If the filter function returns True, the folder will be synced, if it +returns False, it. The folderfilter operates on the *UNTRANSLATED* name +(before any nametrans translation takes place). + +Example 1: synchronizing only INBOX and Sent:: + + folderfilter = lambda foldername: foldername in ['INBOX', 'Sent'] + +Example 2: synchronizing everything except Trash:: + + folderfilter = lambda foldername: foldername not in ['Trash'] + +Example 3: Using a regular expression to exclude Trash and all folders +containing the characters "Del":: + + folderfilter = lambda foldername: not re.search('(^Trash$|Del)', foldername) + +If folderfilter is not specified, ALL remote folders will be +synchronized. + +You can span multiple lines by indenting the others. (Use backslashes +at the end when required by Python syntax) For instance:: + + folderfilter = lambda foldername: foldername in + ['INBOX', 'Sent Mail', 'Deleted Items', + 'Received'] + +You only need a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. + +Even if you filtered out folders, You can specify folderincludes to +include additional folders. It should return a Python list. This might +be used to include a folder that was excluded by your folderfilter rule, +to include a folder that your server does not specify with its LIST +option, or to include a folder that is outside your basic reference. The +'reference' value will not be prefixed to this folder name, even if you +have specified one. For example:: + + folderincludes = ['debian.user', 'debian.personal'] + +nametrans +---------- + +Sometimes, folders need to have different names on the remote and the +local repositories. To achieve this you can specify a folder name +translator. This must be a eval-able Python expression that takes a +foldername arg and returns the new value. I suggest a lambda. This +example below will remove "INBOX." from the leading edge of folders +(great for Courier IMAP users):: + + nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) + +Using Courier remotely and want to duplicate its mailbox naming +locally? Try this:: + + nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) + + +WARNING: you MUST construct nametrans rules such that it NEVER returns +the same value for two folders, UNLESS the second values are +filtered out by folderfilter below. That is, two filters on one side may never point to the same folder on the other side. Failure to follow this rule +will result in undefined behavior. See also *Sharing a maildir with multiple IMAP servers* in the `PITFALLS & ISSUES`_ section. + +Where to put nametrans rules, on the remote and/or local repository? +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +If you never intend to create new folders on the LOCAL repository that need to be synced to the REMOTE repository, it is sufficient to create a nametrans rule on the remote Repository section. This will be used to determine the names of new folder names on the LOCAL repository, and to match existing folders that correspond. + +*IF* you create folders on the local repository, that are supposed to be automatically created on the remote repository, you will need to create a nametrans rule that provides the reverse name translation. + +(A nametrans rule provides only a one-way translation of names and in order to know which names folders on the LOCAL side would have on the REMOTE side, you need to specify the reverse nametrans rule on the local repository) + +OfflineImap will complain if it needs to create a new folder on the +remote side and a back-and-forth nametrans-lation does not yield the +original foldername (as that could potentially lead to infinite folder +creation cycles). + +What folder separators do I need to use in nametrans rules? ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +**Q:** If I sync from an IMAP server with folder separator '/' to a Maildir using the default folder separator '.' which do I need to use in nametrans rules?:: + nametrans = lambda f: "INBOX/" + f +or:: + nametrans = lambda f: "INBOX." + f + +**A:** Generally use the folder separator as defined in the repository you write the nametrans rule for. That is, use '/' in the above case. We will pass in the untranslated name of the IMAP folder as parameter (here `f`). The translated name will ultimately have all folder separators be replaced with the destination repositories' folder separator. + +So if ̀f` was "Sent", the first nametrans yields the translated name "INBOX/Sent" to be used on the other side. As that repository uses the folder separator '.' rather than '/', the ultimate name to be used will be "INBOX.Sent". + +(As a final note, the smart will see that both variants of the above nametrans rule would have worked identically in this case) KNOWN BUGS ========== From 8970a1500b6b85c7f0c3a9e2c112e63261d2d120 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 15:43:01 +0200 Subject: [PATCH 295/817] UIBase: Fix and cleanup register/unregisterthread Registering a thread (associating it with a certain account name) would fail if it was already registered. However, as we a) never unregister most threads (bad) and b) single-threaded mode reuses threads, we failed when syncing multiple accounts in single-threading mode. This commit cleans up the functions to not make re-registering a thread fatal (it could be legitimate, however it *should* not occur). Future work needs to be done to unregister new threads at the appropriate places. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ offlineimap/ui/UIBase.py | 42 ++++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index b532734..7ced439 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,7 @@ Changes Bug Fixes --------- + +* Syncing multiple accounts in single-threaded mode would fail as we try + to "register" a thread as belonging to two accounts which was + fatal. Make it non-fatal (it can be legitimate). diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 52b1788..77594f2 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -47,6 +47,7 @@ class UIBase: s.debugmessages = {} s.debugmsglen = 50 s.threadaccounts = {} + """dict linking active threads (k) to account names (v)""" s.logfile = None s.exc_queue = Queue() """saves all occuring exceptions, so we can output them at the end""" @@ -117,29 +118,32 @@ class UIBase: if exc_traceback: self._msg(traceback.format_tb(exc_traceback)) - def registerthread(s, account): - """Provides a hint to UIs about which account this particular - thread is processing.""" - if s.threadaccounts.has_key(threading.currentThread()): - raise ValueError, "Thread %s already registered (old %s, new %s)" %\ - (threading.currentThread().getName(), - s.getthreadaccount(s), account) - s.threadaccounts[threading.currentThread()] = account - s.debug('thread', "Register new thread '%s' (account '%s')" %\ - (threading.currentThread().getName(), account)) + def registerthread(self, account): + """Register current thread as being associated with an account name""" + cur_thread = threading.currentThread() + if cur_thread in self.threadaccounts: + # was already associated with an old account, update info + self.debug('thread', "Register thread '%s' (previously '%s', now " + "'%s')" % (cur_thread.getName(), + self.getthreadaccount(cur_thread), account)) + else: + self.debug('thread', "Register new thread '%s' (account '%s')" %\ + (cur_thread.getName(), account)) + self.threadaccounts[cur_thread] = account - def unregisterthread(s, thr): - """Recognizes a thread has exited.""" - if s.threadaccounts.has_key(thr): - del s.threadaccounts[thr] - s.debug('thread', "Unregister thread '%s'" % thr.getName()) + def unregisterthread(self, thr): + """Unregister a thread as being associated with an account name""" + if self.threadaccounts.has_key(thr): + del self.threadaccounts[thr] + self.debug('thread', "Unregister thread '%s'" % thr.getName()) - def getthreadaccount(s, thr = None): + def getthreadaccount(self, thr = None): + """Get name of account for a thread (current if None)""" if not thr: thr = threading.currentThread() - if s.threadaccounts.has_key(thr): - return s.threadaccounts[thr] - return '*Control' + if thr in self.threadaccounts: + return self.threadaccounts[thr] + return '*Control' # unregistered thread is '*Control' def debug(s, debugtype, msg): thisthread = threading.currentThread() From 642880b4040ee362727738cbe3e913f0000ad867 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 16:02:51 +0200 Subject: [PATCH 296/817] Don't use global variable profiledir Rather than setting a global threadutil/profiledir variable, we make set_profiledir a class function that sets the class variable profiledir. While touching theprofiledir code, add warning to the user if the profile directory existed before. Signed-off-by: Sebastian Spaeth --- offlineimap/init.py | 14 ++++++++------ offlineimap/threadutil.py | 29 +++++++++++++++-------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 063a2c0..27aeebd 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -1,6 +1,5 @@ # OfflineIMAP initialization code -# Copyright (C) 2002-2007 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -157,11 +156,14 @@ class OfflineImap: if not options.singlethreading: logging.warn("Profile mode: Forcing to singlethreaded.") options.singlethreading = True - profiledir = options.profiledir - os.mkdir(profiledir) - threadutil.setprofiledir(profiledir) + if os.path.exists(options.profiledir): + logging.warn("Profile mode: Directory '%s' already exists!" % + options.profiledir) + else: + os.mkdir(options.profiledir) + threadutil.ExitNotifyThread.set_profiledir(options.profiledir) logging.warn("Profile mode: Potentially large data will be " - "created in '%s'" % profiledir) + "created in '%s'" % options.profiledir) #override a config value if options.configoverride: diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 38387e9..463752f 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -1,6 +1,5 @@ -# Copyright (C) 2002, 2003 John Goerzen +# Copyright (C) 2002-2011 John Goerzen & contributors # Thread support module -# # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,15 +19,10 @@ from threading import Lock, Thread, BoundedSemaphore from Queue import Queue, Empty import traceback from thread import get_ident # python < 2.6 support +import os.path import sys from offlineimap.ui import getglobalui -profiledir = None - -def setprofiledir(newdir): - global profiledir - profiledir = newdir - ###################################################################### # General utilities ###################################################################### @@ -132,11 +126,14 @@ def threadexited(thread): class ExitNotifyThread(Thread): """This class is designed to alert a "monitor" to the fact that a thread has exited and to provide for the ability for it to find out why.""" + profiledir = None + """class variable that is set to the profile directory if required""" + def run(self): - global exitthreads, profiledir + global exitthreads self.threadid = get_ident() try: - if not profiledir: # normal case + if not ExitNotifyThread.profiledir: # normal case Thread.run(self) else: try: @@ -148,9 +145,8 @@ class ExitNotifyThread(Thread): prof = prof.runctx("Thread.run(self)", globals(), locals()) except SystemExit: pass - prof.dump_stats( \ - profiledir + "/" + str(self.threadid) + "_" + \ - self.getName() + ".prof") + prof.dump_stats(os.path.join(ExitNotifyThread.profiledir, + "%s_%s.prof" % (self.threadid, self.getName()))) except: self.setExitCause('EXCEPTION') if sys: @@ -194,7 +190,12 @@ class ExitNotifyThread(Thread): a call to setExitMessage(), or None if there was no such message set.""" return self.exitmessage - + + @classmethod + def set_profiledir(cls, directory): + """If set, will output profile information to 'directory'""" + cls.profiledir = directory + ###################################################################### # Instance-limited threads From eb0b546927c6eac01105e0278e5f92f099dae21e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 16:51:48 +0200 Subject: [PATCH 297/817] Output progress indication when copying files Output (2 of 500) when logging message copying. This required moving of self.ui.copyingmessage into a different function where we actually have the information about the progress handy. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 8 ++++---- offlineimap/ui/Blinkenlights.py | 5 +++-- offlineimap/ui/Machine.py | 2 +- offlineimap/ui/TTY.py | 6 +----- offlineimap/ui/UIBase.py | 9 ++++----- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 909db30..91ee08a 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -266,7 +266,6 @@ class BaseFolder(object): statusfolder.savemessage(uid, None, flags, rtime) return - self.ui.copyingmessage(uid, self, dstfolder) # If any of the destinations actually stores the message body, # load it up. if dstfolder.storesmessages(): @@ -331,15 +330,16 @@ class BaseFolder(object): copylist = filter(lambda uid: not \ statusfolder.uidexists(uid), self.getmessageuidlist()) - for uid in copylist: + num_to_copy = len(copylist) + for num, uid in enumerate(copylist): + self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # exceptions are caught in copymessageto() if self.suggeststhreads(): self.waitforthread() thread = threadutil.InstanceLimitedThread(\ self.getcopyinstancelimit(), target = self.copymessageto, - name = "Copy message %d from %s" % (uid, - self.getvisiblename()), + name = "Copy message from %s:%s" % (self.repository, self), args = (uid, dstfolder, statusfolder)) thread.setDaemon(1) thread.start() diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py index f0dfec1..330b9a8 100644 --- a/offlineimap/ui/Blinkenlights.py +++ b/offlineimap/ui/Blinkenlights.py @@ -54,9 +54,10 @@ class BlinkenBase: s.gettf().setcolor('blue') s.__class__.__bases__[-1].syncingmessages(s, sr, sf, dr, df) - def copyingmessage(s, uid, src, destfolder): + def copyingmessage(s, uid, num, num_to_copy, src, destfolder): s.gettf().setcolor('orange') - s.__class__.__bases__[-1].copyingmessage(s, uid, src, destfolder) + s.__class__.__bases__[-1].copyingmessage(s, uid, num, num_to_copy, src, + destfolder) def deletingmessages(s, uidlist, destlist): s.gettf().setcolor('red') diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 7a480ae..50a7f99 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -108,7 +108,7 @@ class MachineUI(UIBase): (s.getnicename(sr), sf.getname(), s.getnicename(dr), df.getname())) - def copyingmessage(self, uid, srcfolder, destfolder): + def copyingmessage(self, uid, num, num_to_copy, srcfolder, destfolder): self._printData('copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ (uid, self.getnicename(srcfolder), srcfolder.getname(), self.getnicename(destfolder), destfolder)) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index cba573f..48c468f 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - from UIBase import UIBase from getpass import getpass import sys @@ -37,10 +36,7 @@ class TTYUI(UIBase): #if the next output comes from a different thread than our last one #add the info. #Most look like 'account sync foo' or 'Folder sync foo'. - try: - threadname = currentThread().name - except AttributeError: - threadname = currentThread().getName() + threadname = currentThread().getName() if (threadname == s._lastThreaddisplay \ or threadname == 'MainThread'): print " %s" % msg diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 77594f2..587b055 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -288,12 +288,11 @@ class UIBase: s.getnicename(dr), df.getname())) - def copyingmessage(self, uid, src, destfolder): + def copyingmessage(self, uid, num, num_to_copy, src, destfolder): """Output a log line stating which message we copy""" - if self.verbose >= 0: - self._msg("Copy message %d %s[%s] -> %s[%s]" % \ - (uid, self.getnicename(src), src, - self.getnicename(destfolder), destfolder)) + if self.verbose < 0: return + self._msg("Copy message %s (%d of %d) %s:%s -> %s" % (uid, num, + num_to_copy, src.repository, src, destfolder.repository)) def deletingmessage(s, uid, destlist): if s.verbose >= 0: From ba396cf0ef3c688d34a00baa083055c9bc5e0ba9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 16:59:36 +0200 Subject: [PATCH 298/817] More verbose loggin Thread names are used to determine the logging header in the TTY ui. A recent change made them too terse (basically only changing the account name and not the folder names). Unbreak. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 +++- offlineimap/accounts.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 7ced439..ae84931 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -15,7 +15,9 @@ New Features Changes ------- - + +* Indicate progress when copying many messages (slightly change log format) + Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index d28bd34..6ab451f 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -281,7 +281,7 @@ class SyncableAccount(Account): thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, - name = "Folder sync [%s]" % self, + name = "Folder %s [acc: %s]" % (remotefolder, self), args = (self.name, remoterepos, remotefolder, localrepos, statusrepos, quick)) thread.setDaemon(1) From 3b647d65bedbbd3dc86ad79ce9becae84d58e78b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 17:22:02 +0200 Subject: [PATCH 299/817] Output sync timing Modify the UI:acct and acctdone functions to keep tab of the time inbetween. Put self.ui.acct() and acctdone() at the right places in accounts.py so that the timing happens at the right places. While modifying that loop, flatten the nested try: try: except: finally: constructs, we require python 2.5 now which copes with that. At the end of each account sync you will now see something like: *** Finished account 'test' in 0:05 Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/accounts.py | 46 +++++++++++++++++++--------------------- offlineimap/ui/UIBase.py | 22 ++++++++++++------- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index ae84931..44048b9 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,6 +18,8 @@ Changes * Indicate progress when copying many messages (slightly change log format) +* Output how long an account sync took (min:sec). + Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 6ab451f..2f674c9 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -1,5 +1,4 @@ -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -196,7 +195,6 @@ class SyncableAccount(Account): def syncrunner(self): self.ui.registerthread(self.name) - self.ui.acct(self.name) accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): os.mkdir(accountmetadata, 0700) @@ -208,32 +206,32 @@ class SyncableAccount(Account): # Loop account sync if needed (bail out after 3 failures) looping = 3 while looping: + self.ui.acct(self) try: - try: - self.lock() - self.sync() - except (KeyboardInterrupt, SystemExit): - raise - except OfflineImapError, e: - # Stop looping and bubble up Exception if needed. - if e.severity >= OfflineImapError.ERROR.REPO: - if looping: - looping -= 1 - if e.severity >= OfflineImapError.ERROR.CRITICAL: - raise - self.ui.error(e, exc_info()[2]) - except Exception, e: - self.ui.error(e, msg = "While attempting to sync " - "account %s:\n %s"% (self, traceback.format_exc())) - else: - # after success sync, reset the looping counter to 3 - if self.refreshperiod: - looping = 3 + self.lock() + self.sync() + except (KeyboardInterrupt, SystemExit): + raise + except OfflineImapError, e: + # Stop looping and bubble up Exception if needed. + if e.severity >= OfflineImapError.ERROR.REPO: + if looping: + looping -= 1 + if e.severity >= OfflineImapError.ERROR.CRITICAL: + raise + self.ui.error(e, exc_info()[2]) + except Exception, e: + self.ui.error(e, exc_info()[2], msg = "While attempting to sync" + " account '%s'" % self) + else: + # after success sync, reset the looping counter to 3 + if self.refreshperiod: + looping = 3 finally: + self.ui.acctdone(self) self.unlock() if looping and self.sleeper() >= 2: looping = 0 - self.ui.acctdone(self.name) def getaccountmeta(self): return os.path.join(self.metadatadir, 'Account-' + self.name) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 587b055..dc95c30 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -1,6 +1,5 @@ # UI base class -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,6 +47,8 @@ class UIBase: s.debugmsglen = 50 s.threadaccounts = {} """dict linking active threads (k) to account names (v)""" + s.acct_startimes = {} + """linking active accounts with the time.time() when sync started""" s.logfile = None s.exc_queue = Queue() """saves all occuring exceptions, so we can output them at the end""" @@ -238,13 +239,18 @@ class UIBase: displaystr = '.' s._msg("Establishing connection" + displaystr) - def acct(s, accountname): - if s.verbose >= 0: - s._msg("***** Processing account %s" % accountname) + def acct(self, account): + """Output that we start syncing an account (and start counting)""" + self.acct_startimes[account] = time.time() + if self.verbose >= 0: + self._msg("*** Processing account %s" % account) - def acctdone(s, accountname): - if s.verbose >= 0: - s._msg("***** Finished processing account " + accountname) + def acctdone(self, account): + """Output that we finished syncing an account (in which time)""" + sec = time.time() - self.acct_startimes[account] + del self.acct_startimes[account] + self._msg("*** Finished account '%s' in %d:%02d" % + (account, sec // 60, sec % 60)) def syncfolders(s, srcrepos, destrepos): if s.verbose >= 0: From 93b05215fba364564018c52ab257a323c25277d4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 17:46:46 +0200 Subject: [PATCH 300/817] Condense ui.connecting() function a bit Just tiny non-functional code cleanup. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index dc95c30..b5a75dd 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -228,16 +228,14 @@ class UIBase: s._msg(offlineimap.banner) def connecting(s, hostname, port): - if s.verbose < 0: - return - if hostname == None: - hostname = '' - if port != None: - port = ":%s" % str(port) - displaystr = ' to %s%s.' % (hostname, port) - if hostname == '' and port == None: - displaystr = '.' - s._msg("Establishing connection" + displaystr) + """Log 'Establishing connection to'""" + if s.verbose < 0: return + displaystr = '' + hostname = hostname if hostname else '' + port = "%d" % port if port else '' + if hostname: + displaystr = ' to %s:%s' % (hostname, port) + s._msg("Establishing connection%s" % displaystr) def acct(self, account): """Output that we start syncing an account (and start counting)""" From ac94de3449a5322e08ff22403a65318708acc123 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 29 Sep 2011 17:58:07 +0200 Subject: [PATCH 301/817] Only output a sleeping notice once every minute Rather than every ten seconds, which is a bit chatty. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index b5a75dd..4e53d32 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -407,14 +407,14 @@ class UIBase: """Sleep for sleepsecs, display remainingsecs to go. Does nothing if sleepsecs <= 0. - Display a message on the screen every 10 seconds. + Display a message on the screen if we pass a full minute. This implementation in UIBase does not support this, but some implementations return 0 for successful sleep and 1 for an 'abort', ie a request to sync immediately. """ if sleepsecs > 0: - if remainingsecs % 10 == 0: - s._msg("Next refresh in %d seconds" % remainingsecs) + if remainingsecs//60 != (remainingsecs-sleepsecs)//60: + s._msg("Next refresh in %.1f minutes" % (remainingsecs/60.0)) time.sleep(sleepsecs) return 0 From 3f7749016f0794a36b6df51189e21a2cce495dd3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 08:41:05 +0200 Subject: [PATCH 302/817] Reformat MANUAL line breaks Signed-off-by: Sebastian Spaeth --- docs/MANUAL.rst | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 22ddc95..bb13478 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -440,11 +440,20 @@ will result in undefined behavior. See also *Sharing a maildir with multiple IMA Where to put nametrans rules, on the remote and/or local repository? ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -If you never intend to create new folders on the LOCAL repository that need to be synced to the REMOTE repository, it is sufficient to create a nametrans rule on the remote Repository section. This will be used to determine the names of new folder names on the LOCAL repository, and to match existing folders that correspond. +If you never intend to create new folders on the LOCAL repository that +need to be synced to the REMOTE repository, it is sufficient to create a +nametrans rule on the remote Repository section. This will be used to +determine the names of new folder names on the LOCAL repository, and to +match existing folders that correspond. -*IF* you create folders on the local repository, that are supposed to be automatically created on the remote repository, you will need to create a nametrans rule that provides the reverse name translation. +*IF* you create folders on the local repository, that are supposed to be + automatically created on the remote repository, you will need to create + a nametrans rule that provides the reverse name translation. -(A nametrans rule provides only a one-way translation of names and in order to know which names folders on the LOCAL side would have on the REMOTE side, you need to specify the reverse nametrans rule on the local repository) +(A nametrans rule provides only a one-way translation of names and in +order to know which names folders on the LOCAL side would have on the +REMOTE side, you need to specify the reverse nametrans rule on the local +repository) OfflineImap will complain if it needs to create a new folder on the remote side and a back-and-forth nametrans-lation does not yield the @@ -454,16 +463,28 @@ creation cycles). What folder separators do I need to use in nametrans rules? +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -**Q:** If I sync from an IMAP server with folder separator '/' to a Maildir using the default folder separator '.' which do I need to use in nametrans rules?:: +**Q:** If I sync from an IMAP server with folder separator '/' to a + Maildir using the default folder separator '.' which do I need to use + in nametrans rules?:: + nametrans = lambda f: "INBOX/" + f or:: nametrans = lambda f: "INBOX." + f -**A:** Generally use the folder separator as defined in the repository you write the nametrans rule for. That is, use '/' in the above case. We will pass in the untranslated name of the IMAP folder as parameter (here `f`). The translated name will ultimately have all folder separators be replaced with the destination repositories' folder separator. +**A:** Generally use the folder separator as defined in the repository + you write the nametrans rule for. That is, use '/' in the above + case. We will pass in the untranslated name of the IMAP folder as + parameter (here `f`). The translated name will ultimately have all + folder separators be replaced with the destination repositories' + folder separator. -So if ̀f` was "Sent", the first nametrans yields the translated name "INBOX/Sent" to be used on the other side. As that repository uses the folder separator '.' rather than '/', the ultimate name to be used will be "INBOX.Sent". +So if ̀f` was "Sent", the first nametrans yields the translated name +"INBOX/Sent" to be used on the other side. As that repository uses the +folder separator '.' rather than '/', the ultimate name to be used will +be "INBOX.Sent". -(As a final note, the smart will see that both variants of the above nametrans rule would have worked identically in this case) +(As a final note, the smart will see that both variants of the above +nametrans rule would have worked identically in this case) KNOWN BUGS ========== From 4e89bbfe6e1a95742630191800dfd9cfd76150a0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 08:58:08 +0200 Subject: [PATCH 303/817] Recache folder list after creating a new folder If we create a new folder we would previously not update our folder list, which led to us skipping the synchronization of those new folders during the initial run (subsequent runs would pick it up). Invalidate the folder cache when we create a folder during folder structure sync. Regetting the whole list from an IMAP server might be slightly suboptimal from a performance point, but it is easy and will lead to consistent results. Hopefully we will not have to create new folders on each new run. Reported-by: Daniel Shahaf Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/accounts.py | 2 +- offlineimap/repository/Base.py | 13 +++++++++++-- offlineimap/repository/IMAP.py | 4 ++++ offlineimap/repository/Maildir.py | 6 ++++-- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 44048b9..5095b2e 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -26,3 +26,6 @@ Bug Fixes * Syncing multiple accounts in single-threaded mode would fail as we try to "register" a thread as belonging to two accounts which was fatal. Make it non-fatal (it can be legitimate). + +* New folders on the remote would be skipped on the very sync run they + are created and only by synced in subsequent runs. Fixed. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 2f674c9..71f08fe 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -265,7 +265,7 @@ class SyncableAccount(Account): remoterepos = self.remoterepos localrepos = self.localrepos statusrepos = self.statusrepos - # replicate the folderstructure from REMOTE to LOCAL + # replicate the folderstructure between REMOTE to LOCAL if not localrepos.getconfboolean('readonly', False): self.ui.syncfolders(remoterepos, localrepos) remoterepos.syncfoldersto(localrepos, statusrepos) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 0da307f..972aefb 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -144,7 +144,8 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): src_repo = self src_folders = src_repo.getfolders() dst_folders = dst_repo.getfolders() - + # Do we need to refresh the folder list afterwards? + src_haschanged, dst_haschanged = False, False # Create hashes with the names, but convert the source folders # to the dest folder's sep. src_hash = {} @@ -160,6 +161,7 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): if src_folder.sync_this and not src_name in dst_hash: try: dst_repo.makefolder(src_name) + dst_haschanged = True # Need to refresh list except OfflineImapError, e: self.ui.error(e, exc_info()[2], "Creating folder %s on repository %s" %\ @@ -203,6 +205,7 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): try: src_repo.makefolder(newsrc_name) + src_haschanged = True # Need to refresh list except OfflineImapError, e: self.ui.error(e, exc_info()[2], "Creating folder %s on repository %s" %\ @@ -211,7 +214,13 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): status_repo.makefolder(newsrc_name.replace( src_repo.getsep(), status_repo.getsep())) # Find deleted folders. - # We don't delete folders right now. + # TODO: We don't delete folders right now. + + #Forget old list of cached folders so we get new ones if needed + if src_haschanged: + self.forgetfolders() + if dst_haschanged: + dst_repo.forgetfolders() def startkeepalive(self): """The default implementation will do nothing.""" diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 5ddf5b9..2d2962f 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -311,6 +311,10 @@ class IMAPRepository(BaseRepository): def makefolder(self, foldername): """Create a folder on the IMAP server + This will not update the list cached in :meth:`getfolders`. You + will need to invoke :meth:`forgetfolders` to force new caching + when you are done creating folders yourself. + :param foldername: Full path of the folder to be created.""" #TODO: IMHO this existing commented out code is correct and #should be enabled, but this would change the behavior for diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 7c3bb1e..ae09486 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -72,6 +72,10 @@ class MaildirRepository(BaseRepository): def makefolder(self, foldername): """Create new Maildir folder if necessary + This will not update the list cached in getfolders(). You will + need to invoke :meth:`forgetfolders` to force new caching when + you are done creating folders yourself. + :param foldername: A relative mailbox name. The maildir will be created in self.root+'/'+foldername. All intermediate folder levels will be created if they do not exist yet. 'cur', @@ -108,8 +112,6 @@ class MaildirRepository(BaseRepository): (foldername, subdir)) else: raise - # Invalidate the folder cache - self.folders = None def deletefolder(self, foldername): self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername) From a5eebd4b6bec419e96b01b6b72fe301c14ffbd59 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 09:21:03 +0200 Subject: [PATCH 304/817] Make "syncing folder structure" a debug level log entry It happens quick, and clutters the log. So we can usually skip this. We will output a log entry when we actually create a new folder anyway. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 4e53d32..6d92152 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -250,10 +250,11 @@ class UIBase: self._msg("*** Finished account '%s' in %d:%02d" % (account, sec // 60, sec % 60)) - def syncfolders(s, srcrepos, destrepos): - if s.verbose >= 0: - s._msg("Copying folder structure from %s to %s" % \ - (s.getnicename(srcrepos), s.getnicename(destrepos))) + def syncfolders(self, src_repo, dst_repo): + """Log 'Copying folder structure...'""" + if self.verbose < 0: return + self.debug('', "Copying folder structure from %s to %s" % \ + (src_repo, dst_repo)) ############################## Folder syncing def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): From 6b3f429c81f985b23df144053c9f767cd0137a1f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 30 Jun 2011 14:04:18 +0200 Subject: [PATCH 305/817] Split offlineimap.run() Was getting too large, split into an parse_cmd_options and a sync() function. Moving config and ui to self.config and self.ui to make them available through the OfflineImap instance. Signed-off-by: Sebastian Spaeth --- offlineimap/init.py | 108 ++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 27aeebd..065807f 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -44,9 +44,13 @@ class OfflineImap: """ def run(self): """Parse the commandline and invoke everything""" + # next line also sets self.config + options = self.parse_cmd_options() + self.sync(options) + def parse_cmd_options(self): parser = OptionParser(version=offlineimap.__version__, - description="%s.\n\n%s" % + description="%s.\n\n%s" % (offlineimap.__copyright__, offlineimap.__license__)) parser.add_option("-1", @@ -103,7 +107,7 @@ class OfflineImap: "Only sync the specified folders. The folder names " "are the *untranslated* foldernames. This " "command-line option overrides any 'folderfilter' " - "and 'folderincludes' options in the configuration " + "and 'folderincludes' options in the configuration " "file.") parser.add_option("-k", dest="configoverride", @@ -187,19 +191,19 @@ class OfflineImap: 'of %s' % ', '.join(UI_LIST.keys())) try: # create the ui class - ui = UI_LIST[ui_type.lower()](config) + self.ui = UI_LIST[ui_type.lower()](config) except KeyError: logging.error("UI '%s' does not exist, choose one of: %s" % \ (ui_type,', '.join(UI_LIST.keys()))) sys.exit(1) - setglobalui(ui) + setglobalui(self.ui) #set up additional log files if options.logfile: - ui.setlogfd(open(options.logfile, 'wt')) + self.ui.setlogfd(open(options.logfile, 'wt')) #welcome blurb - ui.init_banner() + self.ui.init_banner() if options.debugtype: if options.debugtype.lower() == 'all': @@ -207,13 +211,13 @@ class OfflineImap: #force single threading? if not ('thread' in options.debugtype.split(',') \ and not options.singlethreading): - ui._msg("Debug mode: Forcing to singlethreaded.") + self.ui._msg("Debug mode: Forcing to singlethreaded.") options.singlethreading = True debugtypes = options.debugtype.split(',') + [''] for type in debugtypes: type = type.strip() - ui.add_debug(type) + self.ui.add_debug(type) if type.lower() == 'imap': imaplib.Debug = 5 @@ -241,47 +245,64 @@ class OfflineImap: config.set(section, "folderfilter", folderfilter) config.set(section, "folderincludes", folderincludes) - self.config = config + if options.logfile: + sys.stderr = self.ui.logfile + socktimeout = config.getdefaultint("general", "socktimeout", 0) + if socktimeout > 0: + socket.setdefaulttimeout(socktimeout) + + threadutil.initInstanceLimit('ACCOUNTLIMIT', + config.getdefaultint('general', 'maxsyncaccounts', 1)) + + for reposname in config.getsectionlist('Repository'): + for instancename in ["FOLDER_" + reposname, + "MSGCOPY_" + reposname]: + if options.singlethreading: + threadutil.initInstanceLimit(instancename, 1) + else: + threadutil.initInstanceLimit(instancename, + config.getdefaultint('Repository ' + reposname, + 'maxconnections', 2)) + self.config = config + return options + + def sync(self, options): + """Invoke the correct single/multithread syncing + + self.config is supposed to have been correctly initialized + already.""" def sigterm_handler(signum, frame): # die immediately - ui = getglobalui() - ui.terminate(errormsg="terminating...") - - signal.signal(signal.SIGTERM,sigterm_handler) + self.ui.terminate(errormsg="terminating...") + signal.signal(signal.SIGTERM, sigterm_handler) try: - pidfd = open(config.getmetadatadir() + "/pid", "w") + pidfd = open(self.config.getmetadatadir() + "/pid", "w") pidfd.write(str(os.getpid()) + "\n") pidfd.close() except: pass - try: - if options.logfile: - sys.stderr = ui.logfile - - socktimeout = config.getdefaultint("general", "socktimeout", 0) - if socktimeout > 0: - socket.setdefaulttimeout(socktimeout) - - activeaccounts = config.get("general", "accounts") + try: + activeaccounts = self.config.get("general", "accounts") if options.accounts: activeaccounts = options.accounts activeaccounts = activeaccounts.replace(" ", "") activeaccounts = activeaccounts.split(",") - allaccounts = accounts.AccountHashGenerator(config) + allaccounts = accounts.AccountHashGenerator(self.config) syncaccounts = [] for account in activeaccounts: if account not in allaccounts: if len(allaccounts) == 0: - errormsg = 'The account "%s" does not exist because no accounts are defined!'%account + errormsg = "The account '%s' does not exist because no"\ + " accounts are defined!" % account else: - errormsg = 'The account "%s" does not exist. Valid accounts are:'%account - for name in allaccounts.keys(): - errormsg += '\n%s'%name - ui.terminate(1, errortitle = 'Unknown Account "%s"'%account, errormsg = errormsg) + errormsg = "The account '%s' does not exist. Valid ac"\ + "counts are: " % account + errormsg += ", ".join(allaccounts.keys()) + self.ui.terminate(1, errormsg = errormsg) if account not in syncaccounts: syncaccounts.append(account) @@ -289,19 +310,6 @@ class OfflineImap: remoterepos = None localrepos = None - threadutil.initInstanceLimit('ACCOUNTLIMIT', - config.getdefaultint('general', - 'maxsyncaccounts', 1)) - - for reposname in config.getsectionlist('Repository'): - for instancename in ["FOLDER_" + reposname, - "MSGCOPY_" + reposname]: - if options.singlethreading: - threadutil.initInstanceLimit(instancename, 1) - else: - threadutil.initInstanceLimit(instancename, - config.getdefaultint('Repository ' + reposname, - 'maxconnections', 2)) def sig_handler(sig, frame): if sig == signal.SIGUSR1 or sig == signal.SIGHUP: # tell each account to stop sleeping @@ -315,7 +323,7 @@ class OfflineImap: signal.signal(signal.SIGUSR2,sig_handler) #various initializations that need to be performed: - offlineimap.mbnames.init(config, syncaccounts) + offlineimap.mbnames.init(self.config, syncaccounts) #TODO: keep legacy lock for a few versions, then remove. self._legacy_lock = open(self.config.getmetadatadir() + "/lock", @@ -331,20 +339,20 @@ class OfflineImap: if options.singlethreading: #singlethreaded - self.sync_singlethreaded(syncaccounts, config) + self.sync_singlethreaded(syncaccounts) else: # multithreaded t = threadutil.ExitNotifyThread(target=syncmaster.syncitall, name='Sync Runner', kwargs = {'accounts': syncaccounts, - 'config': config}) + 'config': self.config}) t.setDaemon(1) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) - ui.terminate() + self.ui.terminate() except KeyboardInterrupt: - ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') + self.ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') return except (SystemExit): raise @@ -352,13 +360,13 @@ class OfflineImap: ui.error(e) ui.terminate() - def sync_singlethreaded(self, accs, config): + def sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread :param accs: A list of accounts that should be synced - :param config: The CustomConfig object """ for accountname in accs: - account = offlineimap.accounts.SyncableAccount(config, accountname) + account = offlineimap.accounts.SyncableAccount(self.config, + accountname) threading.currentThread().name = "Account sync %s" % accountname account.syncrunner() From 3885acf87dd4e14ad28ca63c4818114a1dc95e15 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 30 Jun 2011 15:18:03 +0200 Subject: [PATCH 306/817] Implement server diagnostics This outputs a handy summary of your server configuration and version strings etc, which is useful for bug reporting. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/accounts.py | 18 ++++++++++----- offlineimap/init.py | 36 +++++++++++++++++++++--------- offlineimap/ui/UIBase.py | 47 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 15 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5095b2e..7982706 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,9 @@ others. New Features ------------ +* add a --info command line switch that outputs useful information about + the server and the configuration for all enabled accounts. + Changes ------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 71f08fe..1c5dd38 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -82,6 +82,9 @@ class Account(CustomConfig.ConfigHelperMixin): def __str__(self): return self.name + def getaccountmeta(self): + return os.path.join(self.metadatadir, 'Account-' + self.name) + def getsection(self): return 'Account ' + self.getname() @@ -154,8 +157,16 @@ class Account(CustomConfig.ConfigHelperMixin): self.quicknum = 0 return 1 return 0 - - + + def serverdiagnostics(self): + """Output diagnostics for all involved repositories""" + remote_repo = Repository(self, 'remote') + local_repo = Repository(self, 'local') + #status_repo = Repository(self, 'status') + self.ui.serverdiagnostics(remote_repo, 'Remote') + self.ui.serverdiagnostics(local_repo, 'Local') + #self.ui.serverdiagnostics(statusrepos, 'Status') + class SyncableAccount(Account): """A syncable email account connecting 2 repositories @@ -233,9 +244,6 @@ class SyncableAccount(Account): if looping and self.sleeper() >= 2: looping = 0 - def getaccountmeta(self): - return os.path.join(self.metadatadir, 'Account-' + self.name) - def sync(self): """Synchronize the account once, then return diff --git a/offlineimap/init.py b/offlineimap/init.py index 065807f..a7f82ca 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -44,9 +44,12 @@ class OfflineImap: """ def run(self): """Parse the commandline and invoke everything""" - # next line also sets self.config - options = self.parse_cmd_options() - self.sync(options) + # next line also sets self.config and self.ui + options, args = self.parse_cmd_options() + if options.diagnostics: + self.serverdiagnostics(options) + else: + self.sync(options) def parse_cmd_options(self): parser = OptionParser(version=offlineimap.__version__, @@ -143,6 +146,13 @@ class OfflineImap: "not usable. Possible interface choices are: %s " % ", ".join(UI_LIST.keys())) + parser.add_option("--info", + action="store_true", dest="diagnostics", + default=False, + help="Output information on the configured email repositories" + ". Useful for debugging and bug reporting. Use in conjunction wit" + "h the -a option to limit the output to a single account") + (options, args) = parser.parse_args() #read in configuration file @@ -265,7 +275,7 @@ class OfflineImap: config.getdefaultint('Repository ' + reposname, 'maxconnections', 2)) self.config = config - return options + return (options, args) def sync(self, options): """Invoke the correct single/multithread syncing @@ -276,7 +286,7 @@ class OfflineImap: # die immediately self.ui.terminate(errormsg="terminating...") signal.signal(signal.SIGTERM, sigterm_handler) - + try: pidfd = open(self.config.getmetadatadir() + "/pid", "w") pidfd.write(str(os.getpid()) + "\n") @@ -305,11 +315,7 @@ class OfflineImap: self.ui.terminate(1, errormsg = errormsg) if account not in syncaccounts: syncaccounts.append(account) - - server = None - remoterepos = None - localrepos = None - + def sig_handler(sig, frame): if sig == signal.SIGUSR1 or sig == signal.SIGHUP: # tell each account to stop sleeping @@ -370,3 +376,13 @@ class OfflineImap: accountname) threading.currentThread().name = "Account sync %s" % accountname account.syncrunner() + + def serverdiagnostics(self, options): + activeaccounts = self.config.get("general", "accounts") + if options.accounts: + activeaccounts = options.accounts + activeaccounts = activeaccounts.split(",") + allaccounts = accounts.AccountListGenerator(self.config) + for account in allaccounts: + if account.name not in activeaccounts: continue + account.serverdiagnostics() diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 6d92152..caf5efd 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -322,6 +322,53 @@ class UIBase: s._msg("Deleting flag %s from %d messages on %s" % \ (", ".join(flags), len(uidlist), dest)) + def serverdiagnostics(self, repository, type): + """Connect to repository and output useful information for debugging""" + conn = None + self._msg("%s repository '%s': type '%s'" % (type, repository.name, + self.getnicename(repository))) + try: + if hasattr(repository, 'gethost'): # IMAP + self._msg("Host: %s Port: %s SSL: %s" % (repository.gethost(), + repository.getport(), + repository.getssl())) + try: + conn = repository.imapserver.acquireconnection() + except OfflineImapError, e: + self._msg("Failed to connect. Reason %s" % e) + else: + if 'ID' in conn.capabilities: + self._msg("Server supports ID extension.") + #TODO: Debug and make below working, it hangs Gmail + #res_type, response = conn.id(( + # 'name', offlineimap.__productname__, + # 'version', offlineimap.__version__)) + #self._msg("Server ID: %s %s" % (res_type, response[0])) + self._msg("Server welcome string: %s" % str(conn.welcome)) + self._msg("Server capabilities: %s\n" % str(conn.capabilities)) + repository.imapserver.releaseconnection(conn) + if type != 'Status': + folderfilter = repository.getconf('folderfilter', None) + if folderfilter: + self._msg("folderfilter= %s\n" % folderfilter) + folderincludes = repository.getconf('folderincludes', None) + if folderincludes: + self._msg("folderincludes= %s\n" % folderincludes) + nametrans = repository.getconf('nametrans', None) + if nametrans: + self._msg("nametrans= %s\n" % nametrans) + + folders = repository.getfolders() + foldernames = [(f.name, f.getvisiblename()) for f in folders] + folders = [] + for name, visiblename in foldernames: + if name == visiblename: folders.append(name) + else: folders.append("%s -> %s" % (name, visiblename)) + self._msg("Folderlist: %s\n" % str(folders)) + finally: + if conn: #release any existing IMAP connection + repository.imapserver.close() + ################################################## Threads def getThreadDebugLog(s, thread): From 0ba9634205c005fba3915a76b2857dcb836afc9e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 10:59:07 +0200 Subject: [PATCH 307/817] Fix string formatting when port is not given. If port is None, we would try to format an empty string with %d wich fails. Fix it by using %s. Reported-by: Iain Dalton Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index caf5efd..b5dcdaa 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -232,7 +232,7 @@ class UIBase: if s.verbose < 0: return displaystr = '' hostname = hostname if hostname else '' - port = "%d" % port if port else '' + port = "%s" % port if port else '' if hostname: displaystr = ' to %s:%s' % (hostname, port) s._msg("Establishing connection%s" % displaystr) From 5f5aec6114e22be45bf80e417ed9a2d4a8ddeae4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 10:59:07 +0200 Subject: [PATCH 308/817] Fix string formatting when port is not given. If port is None, we would try to format an empty string with %d wich fails. Fix it by using %s. Reported-by: Iain Dalton Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 6d92152..b5094eb 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -232,7 +232,7 @@ class UIBase: if s.verbose < 0: return displaystr = '' hostname = hostname if hostname else '' - port = "%d" % port if port else '' + port = "%s" % port if port else '' if hostname: displaystr = ' to %s:%s' % (hostname, port) s._msg("Establishing connection%s" % displaystr) From 9578f291959a5762ed157a16a461853f2d51ed58 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 16:53:18 +0200 Subject: [PATCH 309/817] Use 'reference' value when creating an IMAP directory A repositories 'reference value is always prefixed to the full folder path, so we should do so when creating a new one. The code had existed but was commented out since 2003, I guess the "reference" option is not too often used. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 3 +-- offlineimap/repository/IMAP.py | 16 +++++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8dbd7aa..58ff633 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -1,6 +1,5 @@ # IMAP folder support -# Copyright (C) 2002-2007 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 2d2962f..4c2cadd 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -1,6 +1,5 @@ # IMAP repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -171,7 +170,7 @@ class IMAPRepository(BaseRepository): return self.getconf('preauthtunnel', None) def getreference(self): - return self.getconf('reference', '""') + return self.getconf('reference', '') def getidlefolders(self): localeval = self.localeval @@ -316,14 +315,9 @@ class IMAPRepository(BaseRepository): when you are done creating folders yourself. :param foldername: Full path of the folder to be created.""" - #TODO: IMHO this existing commented out code is correct and - #should be enabled, but this would change the behavior for - #existing configurations who have a 'reference' set on a Mapped - #IMAP server....: - #if self.getreference() != '""': - # newname = self.getreference() + self.getsep() + foldername - #else: - # newname = foldername + if self.getreference(): + foldername = self.getreference() + self.getsep() + foldername + imapobj = self.imapserver.acquireconnection() try: self.ui._msg("Creating new IMAP folder '%s' on server %s" %\ From e707bc530b52c5e8dd5f4862e40f996286ac179a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 3 Oct 2011 14:20:13 +0200 Subject: [PATCH 310/817] Fix quotation marks I guess they looked identical to whoever wrote that, but here the the first had a space with a combining grave accent where the second has a plain space and a plain backquote instead. Reported-by: Daniel Shahaf Signed-off-by: Sebastian Spaeth --- docs/MANUAL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index bb13478..49dedd6 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -478,7 +478,7 @@ or:: folder separators be replaced with the destination repositories' folder separator. -So if ̀f` was "Sent", the first nametrans yields the translated name +So if 'f' was "Sent", the first nametrans yields the translated name "INBOX/Sent" to be used on the other side. As that repository uses the folder separator '.' rather than '/', the ultimate name to be used will be "INBOX.Sent". From 76b0d7cf25ac3049529b3afda906b1207bd708db Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 5 Oct 2011 18:18:55 +0200 Subject: [PATCH 311/817] imapserver: Make noop() more resitent against dropped connections. Drop a connection, if the NOOP to keep a connection open fails due to broken connections. Note that I believe this function is not working as intended. We grab one random connection and send a NOOP. This is not enough to keep all connections open, and if we invoke this function multiple times, we might well always get the same connection to send a NOOP through. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/imapserver.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 7982706..1f6b46b 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -32,3 +32,6 @@ Bug Fixes * New folders on the remote would be skipped on the very sync run they are created and only by synced in subsequent runs. Fixed. + +* Make NOOPs to keep a server connection open more resistant against dropped + connections. diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index ef54e31..7006ef9 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -491,10 +491,24 @@ class IdleThread(object): self.thread.join() def noop(self): + #TODO: AFAIK this is not optimal, we will send a NOOP on one + #random connection (ie not enough to keep all connections + #open). In case we do the noop multiple times, we can well use + #the same connection every time, as we get a random one. This + #function should IMHO send a noop on ALL available connections + #to the server. imapobj = self.parent.acquireconnection() - imapobj.noop() - self.stop_sig.wait() - self.parent.releaseconnection(imapobj) + try: + imapobj.noop() + except imapobj.abort: + self.ui.warn('Attempting NOOP on dropped connection %s' % \ + imapobj.identifier) + self.parent.releaseconnection(imapobj, True) + imapobj = None + finally: + if imapobj: + self.parent.releaseconnection(imapobj) + self.stop_sig.wait() # wait until we are supposed to quit def dosync(self): remoterepos = self.parent.repos From b0fd6c6ab8caf8cb0f30af4c9df595a5fbddfbaf Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 21 Sep 2011 00:46:46 +0200 Subject: [PATCH 312/817] Do not import threading.* Only import the lock, that we actually need. Also import the with statement for use with python 2.5. We'll need it for sure in this file. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/UIDMaps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index fa1924d..5da8e00 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -15,8 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -from threading import * +from __future__ import with_statement # needed for python 2.5 +from threading import Lock from IMAP import IMAPFolder import os.path From 097b7ab2b03aaceba818e25d12856886333b10fd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 5 Oct 2011 18:41:38 +0200 Subject: [PATCH 313/817] createfolder: Fix wrong error output If folder creation failed, we would output the wrong repository and folder name (copy'n paste error). Fix this so we actually output the correct values. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 972aefb..8f400f0 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -207,9 +207,8 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): src_repo.makefolder(newsrc_name) src_haschanged = True # Need to refresh list except OfflineImapError, e: - self.ui.error(e, exc_info()[2], - "Creating folder %s on repository %s" %\ - (src_name, dst_repo)) + self.ui.error(e, exc_info()[2], "Creating folder %s on " + "repository %s" % (newsrc_name, src_repo)) raise status_repo.makefolder(newsrc_name.replace( src_repo.getsep(), status_repo.getsep())) From d4a11c62eaeff9726e7989d5fa2c589449957dce Mon Sep 17 00:00:00 2001 From: dtk Date: Wed, 12 Oct 2011 11:03:27 +0200 Subject: [PATCH 314/817] Fix typos in documentary comments in exemplary config file I stumbled upon a few typos in the config file coming with master and just patched them. Apply as you like. Signed-off-by: dtk Signed-off-by: Sebastian Spaeth --- offlineimap.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 224e933..b6ff6e6 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -55,7 +55,7 @@ metadata = ~/.offlineimap accounts = Test -# Offlineimap can synchronize more the one account at a time. If you +# Offlineimap can synchronize more than one account at a time. If you # want to enable this feature, set the below value to something # greater than 1. To force it to synchronize only one account at a # time, set it to 1. @@ -200,7 +200,7 @@ remoterepository = RemoteExample # quick = 10 # You can specify a pre and post sync hook to execute a external command. -# in this case a call to imapfilter to filter mail before the sync process +# In this case a call to imapfilter to filter mail before the sync process # starts and a custom shell script after the sync completes. # The pre sync script has to complete before a sync to the account will # start. @@ -445,7 +445,7 @@ holdconnectionopen = no # mark them deleted on the server, but not actually delete them. # You must use some other IMAP client to delete them if you use this # setting; otherwise, the messgaes will just pile up there forever. -# Therefore, this setting is definately NOT recommended. +# Therefore, this setting is definitely NOT recommended. # # expunge = no From cbec8bb5b20eb8051aaea2b20f81de9158dab975 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 26 Oct 2011 16:47:21 +0200 Subject: [PATCH 315/817] Rework UI system to make use of the logging module Logging was flawed as the output was e.g. heavily buffered and people complained about missing log entries. Fix this by making use of the standard logging facilities that offlineimap offers. This is one big ugly patch that does many things. It fixes the Blinkenlights backend to work again with the logging facilities. Resize windows and hotkeys are still not handled absolut correctly, this is left for future fixing. THe rest of the backends should be working fine. Signed-off-by: Sebastian Spaeth --- offlineimap/__init__.py | 6 +- offlineimap/accounts.py | 13 +- offlineimap/init.py | 31 +- offlineimap/syncmaster.py | 3 +- offlineimap/threadutil.py | 31 +- offlineimap/ui/Blinkenlights.py | 148 ----- offlineimap/ui/Curses.py | 964 ++++++++++++++++--------------- offlineimap/ui/Machine.py | 78 +-- offlineimap/ui/Noninteractive.py | 41 +- offlineimap/ui/TTY.py | 99 ++-- offlineimap/ui/UIBase.py | 358 ++++++------ offlineimap/ui/__init__.py | 2 +- 12 files changed, 829 insertions(+), 945 deletions(-) delete mode 100644 offlineimap/ui/Blinkenlights.py diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 46ff1d7..ea9aa0d 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -8,15 +8,11 @@ __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" __bigcopyright__ = """%(__productname__)s %(__version__)s -%(__copyright__)s. -%(__license__)s. -""" % locals() + %(__license__)s""" % locals() __homepage__ = "http://github.com/nicolas33/offlineimap" - banner = __bigcopyright__ - from offlineimap.error import OfflineImapError # put this last, so we don't run into circular dependencies using # e.g. offlineimap.__version__. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 1c5dd38..7387c51 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -143,7 +143,7 @@ class Account(CustomConfig.ConfigHelperMixin): for item in kaobjs: item.startkeepalive() - + refreshperiod = int(self.refreshperiod * 60) sleepresult = self.ui.sleep(refreshperiod, self) @@ -166,7 +166,8 @@ class Account(CustomConfig.ConfigHelperMixin): self.ui.serverdiagnostics(remote_repo, 'Remote') self.ui.serverdiagnostics(local_repo, 'Local') #self.ui.serverdiagnostics(statusrepos, 'Status') - + + class SyncableAccount(Account): """A syncable email account connecting 2 repositories @@ -208,7 +209,7 @@ class SyncableAccount(Account): self.ui.registerthread(self.name) accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): - os.mkdir(accountmetadata, 0700) + os.mkdir(accountmetadata, 0700) self.remoterepos = Repository(self, 'remote') self.localrepos = Repository(self, 'local') @@ -223,7 +224,7 @@ class SyncableAccount(Account): self.sync() except (KeyboardInterrupt, SystemExit): raise - except OfflineImapError, e: + except OfflineImapError, e: # Stop looping and bubble up Exception if needed. if e.severity >= OfflineImapError.ERROR.REPO: if looping: @@ -242,7 +243,7 @@ class SyncableAccount(Account): self.ui.acctdone(self) self.unlock() if looping and self.sleeper() >= 2: - looping = 0 + looping = 0 def sync(self): """Synchronize the account once, then return @@ -408,7 +409,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, else: ui.debug('imap', "Not syncing to read-only repository '%s'" \ % localrepos.getname()) - + # Synchronize local changes if not remoterepos.getconfboolean('readonly', False): ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) diff --git a/offlineimap/init.py b/offlineimap/init.py index a7f82ca..0e5b08d 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -157,9 +157,10 @@ class OfflineImap: #read in configuration file configfilename = os.path.expanduser(options.configfile) - + config = CustomConfigParser() if not os.path.exists(configfilename): + # TODO, initialize and make use of chosen ui for logging logging.error(" *** Config file '%s' does not exist; aborting!" % configfilename) sys.exit(1) @@ -168,14 +169,17 @@ class OfflineImap: #profile mode chosen? if options.profiledir: if not options.singlethreading: + # TODO, make use of chosen ui for logging logging.warn("Profile mode: Forcing to singlethreaded.") options.singlethreading = True if os.path.exists(options.profiledir): - logging.warn("Profile mode: Directory '%s' already exists!" % + # TODO, make use of chosen ui for logging + logging.warn("Profile mode: Directory '%s' already exists!" % options.profiledir) else: os.mkdir(options.profiledir) threadutil.ExitNotifyThread.set_profiledir(options.profiledir) + # TODO, make use of chosen ui for logging logging.warn("Profile mode: Potentially large data will be " "created in '%s'" % options.profiledir) @@ -197,6 +201,7 @@ class OfflineImap: if '.' in ui_type: #transform Curses.Blinkenlights -> Blinkenlights ui_type = ui_type.split('.')[-1] + # TODO, make use of chosen ui for logging logging.warning('Using old interface name, consider using one ' 'of %s' % ', '.join(UI_LIST.keys())) try: @@ -210,12 +215,13 @@ class OfflineImap: #set up additional log files if options.logfile: - self.ui.setlogfd(open(options.logfile, 'wt')) - + self.ui.setlogfile(options.logfile) + #welcome blurb self.ui.init_banner() if options.debugtype: + self.ui.logger.setLevel(logging.DEBUG) if options.debugtype.lower() == 'all': options.debugtype = 'imap,maildir,thread' #force single threading? @@ -293,15 +299,15 @@ class OfflineImap: pidfd.close() except: pass - - try: + + try: activeaccounts = self.config.get("general", "accounts") if options.accounts: activeaccounts = options.accounts activeaccounts = activeaccounts.replace(" ", "") activeaccounts = activeaccounts.split(",") allaccounts = accounts.AccountHashGenerator(self.config) - + syncaccounts = [] for account in activeaccounts: if account not in allaccounts: @@ -323,11 +329,11 @@ class OfflineImap: elif sig == signal.SIGUSR2: # tell each account to stop looping accounts.Account.set_abort_event(self.config, 2) - + signal.signal(signal.SIGHUP,sig_handler) signal.signal(signal.SIGUSR1,sig_handler) signal.signal(signal.SIGUSR2,sig_handler) - + #various initializations that need to be performed: offlineimap.mbnames.init(self.config, syncaccounts) @@ -352,10 +358,9 @@ class OfflineImap: name='Sync Runner', kwargs = {'accounts': syncaccounts, 'config': self.config}) - t.setDaemon(1) + t.setDaemon(True) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) - self.ui.terminate() except KeyboardInterrupt: self.ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') @@ -363,8 +368,8 @@ class OfflineImap: except (SystemExit): raise except Exception, e: - ui.error(e) - ui.terminate() + self.ui.error(e) + self.ui.terminate() def sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread diff --git a/offlineimap/syncmaster.py b/offlineimap/syncmaster.py index 3aea6d2..0d139dd 100644 --- a/offlineimap/syncmaster.py +++ b/offlineimap/syncmaster.py @@ -25,12 +25,11 @@ def syncaccount(threads, config, accountname): thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT', target = account.syncrunner, name = "Account sync %s" % accountname) - thread.setDaemon(1) + thread.setDaemon(True) thread.start() threads.add(thread) def syncitall(accounts, config): - currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE') threads = threadlist() for accountname in accounts: syncaccount(threads, config, accountname) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 463752f..39d3ae7 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -35,7 +35,7 @@ def semaphorereset(semaphore, originalstate): # Now release these. for i in range(originalstate): semaphore.release() - + class threadlist: def __init__(self): self.lock = Lock() @@ -70,7 +70,7 @@ class threadlist: if not thread: return thread.join() - + ###################################################################### # Exit-notify threads @@ -81,20 +81,21 @@ exitthreads = Queue(100) def exitnotifymonitorloop(callback): """An infinite "monitoring" loop watching for finished ExitNotifyThread's. - :param callback: the function to call when a thread terminated. That - function is called with a single argument -- the - ExitNotifyThread that has terminated. The monitor will + This one is supposed to run in the main thread. + :param callback: the function to call when a thread terminated. That + function is called with a single argument -- the + ExitNotifyThread that has terminated. The monitor will not continue to monitor for other threads until 'callback' returns, so if it intends to perform long calculations, it should start a new thread itself -- but - NOT an ExitNotifyThread, or else an infinite loop + NOT an ExitNotifyThread, or else an infinite loop may result. - Furthermore, the monitor will hold the lock all the + Furthermore, the monitor will hold the lock all the while the other thread is waiting. :type callback: a callable function """ global exitthreads - while 1: + while 1: # Loop forever and call 'callback' for each thread that exited try: # we need a timeout in the get() call, so that ctrl-c can throw @@ -124,11 +125,19 @@ def threadexited(thread): ui.threadExited(thread) class ExitNotifyThread(Thread): - """This class is designed to alert a "monitor" to the fact that a thread has - exited and to provide for the ability for it to find out why.""" + """This class is designed to alert a "monitor" to the fact that a + thread has exited and to provide for the ability for it to find out + why. All instances are made daemon threads (setDaemon(True), so we + bail out when the mainloop dies.""" profiledir = None """class variable that is set to the profile directory if required""" + def __init__(self, *args, **kwargs): + super(ExitNotifyThread, self).__init__(*args, **kwargs) + # These are all child threads that are supposed to go away when + # the main thread is killed. + self.setDaemon(True) + def run(self): global exitthreads self.threadid = get_ident() @@ -220,7 +229,7 @@ class InstanceLimitedThread(ExitNotifyThread): def start(self): instancelimitedsems[self.instancename].acquire() ExitNotifyThread.start(self) - + def run(self): try: ExitNotifyThread.run(self) diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py deleted file mode 100644 index 330b9a8..0000000 --- a/offlineimap/ui/Blinkenlights.py +++ /dev/null @@ -1,148 +0,0 @@ -# Blinkenlights base classes -# Copyright (C) 2003 John Goerzen -# -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -from threading import RLock, currentThread -from offlineimap.ui.UIBase import UIBase -from thread import get_ident # python < 2.6 support - -class BlinkenBase: - """This is a mix-in class that should be mixed in with either UIBase - or another appropriate base class. The Tk interface, for instance, - will probably mix it in with VerboseUI.""" - - def acct(s, accountname): - s.gettf().setcolor('purple') - s.__class__.__bases__[-1].acct(s, accountname) - - def connecting(s, hostname, port): - s.gettf().setcolor('gray') - s.__class__.__bases__[-1].connecting(s, hostname, port) - - def syncfolders(s, srcrepos, destrepos): - s.gettf().setcolor('blue') - s.__class__.__bases__[-1].syncfolders(s, srcrepos, destrepos) - - def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): - s.gettf().setcolor('cyan') - s.__class__.__bases__[-1].syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder) - - def skippingfolder(s, folder): - s.gettf().setcolor('cyan') - s.__class__.__bases__[-1].skippingfolder(s, folder) - - def loadmessagelist(s, repos, folder): - s.gettf().setcolor('green') - s._msg("Scanning folder [%s/%s]" % (s.getnicename(repos), - folder.getvisiblename())) - - def syncingmessages(s, sr, sf, dr, df): - s.gettf().setcolor('blue') - s.__class__.__bases__[-1].syncingmessages(s, sr, sf, dr, df) - - def copyingmessage(s, uid, num, num_to_copy, src, destfolder): - s.gettf().setcolor('orange') - s.__class__.__bases__[-1].copyingmessage(s, uid, num, num_to_copy, src, - destfolder) - - def deletingmessages(s, uidlist, destlist): - s.gettf().setcolor('red') - s.__class__.__bases__[-1].deletingmessages(s, uidlist, destlist) - - def deletingmessage(s, uid, destlist): - s.gettf().setcolor('red') - s.__class__.__bases__[-1].deletingmessage(s, uid, destlist) - - def addingflags(s, uidlist, flags, dest): - s.gettf().setcolor('yellow') - s.__class__.__bases__[-1].addingflags(s, uidlist, flags, dest) - - def deletingflags(s, uidlist, flags, dest): - s.gettf().setcolor('pink') - s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, dest) - - def warn(s, msg, minor = 0): - if minor: - s.gettf().setcolor('pink') - else: - s.gettf().setcolor('red') - s.__class__.__bases__[-1].warn(s, msg, minor) - - def init_banner(s): - s.availablethreadframes = {} - s.threadframes = {} - #tflock protects the s.threadframes manipulation to only happen from 1 thread - s.tflock = RLock() - - def threadExited(s, thread): - threadid = thread.threadid - accountname = s.getthreadaccount(thread) - s.tflock.acquire() - try: - if threadid in s.threadframes[accountname]: - tf = s.threadframes[accountname][threadid] - del s.threadframes[accountname][threadid] - s.availablethreadframes[accountname].append(tf) - tf.setthread(None) - finally: - s.tflock.release() - - UIBase.threadExited(s, thread) - - def gettf(s): - threadid = get_ident() - accountname = s.getthreadaccount() - - s.tflock.acquire() - - try: - if not accountname in s.threadframes: - s.threadframes[accountname] = {} - - if threadid in s.threadframes[accountname]: - return s.threadframes[accountname][threadid] - - if not accountname in s.availablethreadframes: - s.availablethreadframes[accountname] = [] - - if len(s.availablethreadframes[accountname]): - tf = s.availablethreadframes[accountname].pop(0) - tf.setthread(currentThread()) - else: - tf = s.getaccountframe().getnewthreadframe() - s.threadframes[accountname][threadid] = tf - return tf - finally: - s.tflock.release() - - def callhook(s, msg): - s.gettf().setcolor('white') - s.__class__.__bases__[-1].callhook(s, msg) - - def sleep(s, sleepsecs, account): - s.gettf().setcolor('red') - s.getaccountframe().startsleep(sleepsecs) - return UIBase.sleep(s, sleepsecs, account) - - def sleeping(s, sleepsecs, remainingsecs): - if remainingsecs and s.gettf().getcolor() == 'black': - s.gettf().setcolor('red') - else: - s.gettf().setcolor('black') - return s.getaccountframe().sleeping(sleepsecs, remainingsecs) - - diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 84b0e9a..3dc32f5 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -16,42 +16,67 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import RLock, Lock, Event +from __future__ import with_statement # needed for python 2.5 +from threading import RLock, currentThread, Lock, Event, Thread +from thread import get_ident # python < 2.6 support +from collections import deque import time import sys import os import signal import curses -from Blinkenlights import BlinkenBase -from UIBase import UIBase +import logging +from offlineimap.ui.UIBase import UIBase +from offlineimap.threadutil import ExitNotifyThread import offlineimap - acctkeys = '1234567890abcdefghijklmnoprstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-=;/.,' class CursesUtil: - def __init__(self): - self.pairlock = Lock() - # iolock protects access to the + + def __init__(self, *args, **kwargs): + # iolock protects access to the self.iolock = RLock() - self.start() + self.tframe_lock = RLock() + """tframe_lock protects the self.threadframes manipulation to + only happen from 1 thread""" + self.colormap = {} + """dict, translating color string to curses color pair number""" - def initpairs(self): - self.pairlock.acquire() - try: - self.pairs = {self._getpairindex(curses.COLOR_WHITE, - curses.COLOR_BLACK): 0} - self.nextpair = 1 - finally: - self.pairlock.release() + def curses_colorpair(self, col_name): + """Return the curses color pair, that corresponds to the color""" + return curses.color_pair(self.colormap[col_name]) - def lock(self): + def init_colorpairs(self): + """initialize the curses color pairs available""" + # set special colors 'gray' and 'banner' + self.colormap['white'] = 0 #hardcoded by curses + curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) + self.colormap['banner'] = 1 # color 'banner' for bannerwin + + bcol = curses.COLOR_BLACK + colors = ( # name, color, bold? + ('black', curses.COLOR_BLACK, False), + ('blue', curses.COLOR_BLUE,False), + ('red', curses.COLOR_RED, False), + ('purple', curses.COLOR_MAGENTA, False), + ('cyan', curses.COLOR_CYAN, False), + ('green', curses.COLOR_GREEN, False), + ('orange', curses.COLOR_YELLOW, False)) + #set the rest of all colors starting at pair 2 + i = 1 + for name, fcol, bold in colors: + i += 1 + self.colormap[name] = i + curses.init_pair(i, fcol, bcol) + + def lock(self, block=True): """Locks the Curses ui thread Can be invoked multiple times from the owning thread. Invoking from a non-owning thread blocks and waits until it has been unlocked by the owning thread.""" - self.iolock.acquire() + return self.iolock.acquire(block) def unlock(self): """Unlocks the Curses ui thread @@ -61,8 +86,8 @@ class CursesUtil: thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked.""" self.iolock.release() - - def locked(self, target, *args, **kwargs): + + def exec_locked(self, target, *args, **kwargs): """Perform an operation with full locking.""" self.lock() try: @@ -74,102 +99,56 @@ class CursesUtil: def lockedstuff(): curses.panel.update_panels() curses.doupdate() - self.locked(lockedstuff) + self.exec_locked(lockedstuff) def isactive(self): return hasattr(self, 'stdscr') - def _getpairindex(self, fg, bg): - return '%d/%d' % (fg,bg) - - def getpair(self, fg, bg): - if not self.has_color: - return 0 - pindex = self._getpairindex(fg, bg) - self.pairlock.acquire() - try: - if self.pairs.has_key(pindex): - return curses.color_pair(self.pairs[pindex]) - else: - self.pairs[pindex] = self.nextpair - curses.init_pair(self.nextpair, fg, bg) - self.nextpair += 1 - return curses.color_pair(self.nextpair - 1) - finally: - self.pairlock.release() - - def start(self): - self.stdscr = curses.initscr() - curses.noecho() - curses.cbreak() - self.stdscr.keypad(1) - try: - curses.start_color() - self.has_color = curses.has_colors() - except: - self.has_color = 0 - - self.oldcursor = None - try: - self.oldcursor = curses.curs_set(0) - except: - pass - - self.stdscr.clear() - self.stdscr.refresh() - (self.height, self.width) = self.stdscr.getmaxyx() - self.initpairs() - - def stop(self): - if not hasattr(self, 'stdscr'): - return - #self.stdscr.addstr(self.height - 1, 0, "\n", - # self.getpair(curses.COLOR_WHITE, - # curses.COLOR_BLACK)) - if self.oldcursor != None: - curses.curs_set(self.oldcursor) - self.stdscr.refresh() - self.stdscr.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - del self.stdscr - - def reset(self): - # dirty walkaround for bug http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 (fixed since #83743) - if (sys.version_info[0:3] >= (2,6) and sys.version_info[0:3] <= (2,6,5)): return - self.stop() - self.start() class CursesAccountFrame: - def __init__(s, master, accountname, ui): - s.c = master - s.children = [] - s.accountname = accountname - s.ui = ui + """Notable instance variables: - def drawleadstr(s, secs = None): + - accountname: String with associated account name + - children + - ui + - key + - window: curses window associated with an account + """ + + def __init__(self, ui, accountname): + self.children = [] + self.accountname = accountname + self.ui = ui + + def drawleadstr(self, secs = None): + #TODO: does what? if secs == None: - acctstr = '%s: [active] %13.13s: ' % (s.key, s.accountname) + acctstr = '%s: [active] %13.13s: ' % (self.key, self.accountname) else: - acctstr = '%s: [%3d:%02d] %13.13s: ' % (s.key, + acctstr = '%s: [%3d:%02d] %13.13s: ' % (self.key, secs / 60, secs % 60, - s.accountname) - s.c.locked(s.window.addstr, 0, 0, acctstr) - s.location = len(acctstr) + self.accountname) + self.ui.exec_locked(self.window.addstr, 0, 0, acctstr) + self.location = len(acctstr) - def setwindow(s, window, key): - s.window = window - s.key = key - s.drawleadstr() - for child in s.children: - child.update(window, 0, s.location) - s.location += 1 + def setwindow(self, curses_win, key): + #TODO: does what? + # the curses window associated with an account + self.window = curses_win + self.key = key + self.drawleadstr() + # Update the child ThreadFrames + for child in self.children: + child.update(curses_win, self.location, 0) + self.location += 1 - def getnewthreadframe(s): - tf = CursesThreadFrame(s.c, s.ui, s.window, 0, s.location) - s.location += 1 - s.children.append(tf) + def get_new_tframe(self): + """Create a new ThreadFrame and append it to self.children + + :returns: The new ThreadFrame""" + tf = CursesThreadFrame(self.ui, self.window, self.location, 0) + self.location += 1 + self.children.append(tf) return tf def startsleep(s, sleepsecs): @@ -197,413 +176,456 @@ class CursesAccountFrame: s.sleeping_abort = 1 class CursesThreadFrame: - def __init__(s, master, ui, window, y, x): - """master should be a CursesUtil object.""" - s.c = master - s.ui = ui - s.window = window - s.x = x - s.y = y - s.colors = [] - bg = curses.COLOR_BLACK - s.colormap = {'black': s.c.getpair(curses.COLOR_BLACK, bg), - 'gray': s.c.getpair(curses.COLOR_WHITE, bg), - 'white': curses.A_BOLD | s.c.getpair(curses.COLOR_WHITE, bg), - 'blue': s.c.getpair(curses.COLOR_BLUE, bg), - 'red': s.c.getpair(curses.COLOR_RED, bg), - 'purple': s.c.getpair(curses.COLOR_MAGENTA, bg), - 'cyan': s.c.getpair(curses.COLOR_CYAN, bg), - 'green': s.c.getpair(curses.COLOR_GREEN, bg), - 'orange': s.c.getpair(curses.COLOR_YELLOW, bg), - 'yellow': curses.A_BOLD | s.c.getpair(curses.COLOR_YELLOW, bg), - 'pink': curses.A_BOLD | s.c.getpair(curses.COLOR_RED, bg)} - #s.setcolor('gray') - s.setcolor('black') + """ + curses_color: current color pair for logging""" + def __init__(self, ui, acc_win, x, y): + """ + :param ui: is a Blinkenlights() instance + :param acc_win: curses Account window""" + self.ui = ui + self.window = acc_win + self.x = x + self.y = y + self.curses_color = curses.color_pair(0) #default color - def setcolor(self, color): - self.color = self.colormap[color] + def setcolor(self, color, modifier=0): + """Draw the thread symbol '.' in the specified color + :param modifier: Curses modified, such as curses.A_BOLD""" + self.curses_color = modifier | self.ui.curses_colorpair(color) self.colorname = color self.display() def display(self): - def lockedstuff(): - if self.getcolor() == 'black': - self.window.addstr(self.y, self.x, ' ', self.color) - else: - self.window.addstr(self.y, self.x, '.', self.color) - self.c.stdscr.move(self.c.height - 1, self.c.width - 1) + def locked_display(): + self.window.addch(self.y, self.x, '.', self.curses_color) self.window.refresh() - self.c.locked(lockedstuff) + # lock the curses IO while fudging stuff + self.ui.exec_locked(locked_display) - def getcolor(self): - return self.colorname - - def getcolorpair(self): - return self.color - - def update(self, window, y, x): - self.window = window + def update(self, acc_win, x, y): + """Update the xy position of the '.' (and possibly the aframe)""" + self.window = acc_win self.y = y self.x = x self.display() - def setthread(self, newthread): + def std_color(self): self.setcolor('black') - #if newthread: - # self.setcolor('gray') - #else: - # self.setcolor('black') -class InputHandler: - def __init__(s, util): - s.c = util - s.bgchar = None - s.inputlock = Lock() - s.lockheld = 0 - s.statuslock = Lock() - s.startup = Event() - s.startthread() - def startthread(s): - s.thread = offlineimap.threadutil.ExitNotifyThread(target = s.bgreaderloop, - name = "InputHandler loop") - s.thread.setDaemon(1) - s.thread.start() +class InputHandler(ExitNotifyThread): + """Listens for input via the curses interfaces""" + #TODO, we need to use the ugly exitnotifythread (rather than simply + #threading.Thread here, so exiting this thread via the callback + #handler, kills off all parents too. Otherwise, they would simply + #continue. + def __init__(self, ui): + super(InputHandler, self).__init__() + self.char_handler = None + self.ui = ui + self.enabled = Event() + """We will only parse input if we are enabled""" + self.inputlock = RLock() + """denotes whether we should be handling the next char.""" + self.start() #automatically start the thread - def bgreaderloop(s): - while 1: - s.statuslock.acquire() - if s.lockheld or s.bgchar == None: - s.statuslock.release() - s.startup.wait() - else: - s.statuslock.release() - ch = s.c.stdscr.getch() - s.statuslock.acquire() - try: - if s.lockheld or s.bgchar == None: - curses.ungetch(ch) - else: - s.bgchar(ch) - finally: - s.statuslock.release() + def get_next_char(self): + """return the key pressed or -1 - def set_bgchar(s, callback): - """Sets a "background" character handler. If a key is pressed - while not doing anything else, it will be passed to this handler. + Wait until `enabled` and loop internally every stdscr.timeout() + msecs, releasing the inputlock. + :returns: char or None if disabled while in here""" + self.enabled.wait() + while self.enabled.is_set(): + with self.inputlock: + char = self.ui.stdscr.getch() + if char != -1: yield char + + def run(self): + while True: + char_gen = self.get_next_char() + for char in char_gen: + self.char_handler(char) + #curses.ungetch(char) + + def set_char_hdlr(self, callback): + """Sets a character callback handler + + If a key is pressed it will be passed to this handler. Keys + include the curses.KEY_RESIZE key. callback is a function taking a single arg -- the char pressed. + If callback is None, input will be ignored.""" + with self.inputlock: + self.char_handler = callback + # start or stop the parsing of things + if callback is None: + self.enabled.clear() + else: + self.enabled.set() - If callback is None, clears the request.""" - s.statuslock.acquire() - oldhandler = s.bgchar - newhandler = callback - s.bgchar = callback - - if oldhandler and not newhandler: - pass - if newhandler and not oldhandler: - s.startup.set() - - s.statuslock.release() - - def input_acquire(s): + def input_acquire(self): """Call this method when you want exclusive input control. - Make sure to call input_release afterwards! + + Make sure to call input_release afterwards! While this lockis + held, input can go to e.g. the getpass input. """ + self.enabled.clear() + self.inputlock.acquire() - s.inputlock.acquire() - s.statuslock.acquire() - s.lockheld = 1 - s.statuslock.release() - - def input_release(s): + def input_release(self): """Call this method when you are done getting input.""" - s.statuslock.acquire() - s.lockheld = 0 - s.statuslock.release() - s.inputlock.release() - s.startup.set() - -class Blinkenlights(BlinkenBase, UIBase): - def init_banner(s): - s.af = {} - s.aflock = Lock() - s.c = CursesUtil() - s.text = [] - BlinkenBase.init_banner(s) - s.setupwindows() - s.inputhandler = InputHandler(s.c) - s.gettf().setcolor('red') - s._msg(offlineimap.banner) - s.inputhandler.set_bgchar(s.keypress) - signal.signal(signal.SIGWINCH, s.resizehandler) - s.resizelock = Lock() - s.resizecount = 0 - - def resizehandler(s, signum, frame): - s.resizeterm() - - def resizeterm(s, dosleep = 1): - if not s.resizelock.acquire(0): - s.resizecount += 1 - return - signal.signal(signal.SIGWINCH, signal.SIG_IGN) - s.aflock.acquire() - s.c.lock() - s.resizecount += 1 - while s.resizecount: - s.c.reset() - s.setupwindows() - s.resizecount -= 1 - s.c.unlock() - s.aflock.release() - s.resizelock.release() - signal.signal(signal.SIGWINCH, s.resizehandler) - if dosleep: - time.sleep(1) - s.resizeterm(0) - - def isusable(s): - # Not a terminal? Can't use curses. - if not sys.stdout.isatty() and sys.stdin.isatty(): - return 0 - - # No TERM specified? Can't use curses. - try: - if not len(os.environ['TERM']): - return 0 - except: return 0 - - # ncurses doesn't want to start? Can't use curses. - # This test is nasty because initscr() actually EXITS on error. - # grr. - - pid = os.fork() - if pid: - # parent - return not os.WEXITSTATUS(os.waitpid(pid, 0)[1]) - else: - # child - curses.initscr() - curses.endwin() - # If we didn't die by here, indicate success. - sys.exit(0) - - def keypress(s, key): - if key < 1 or key > 255: - return - - if chr(key) == 'q': - # Request to quit. - s.terminate() - - try: - index = acctkeys.index(chr(key)) - except ValueError: - # Key not a valid one: exit. - return - - if index >= len(s.hotkeys): - # Not in our list of valid hotkeys. - return - - # Trying to end sleep somewhere. - - s.getaccountframe(s.hotkeys[index]).syncnow() - - def getpass(s, accountname, config, errmsg = None): - s.inputhandler.input_acquire() - - # See comment on _msg for info on why both locks are obtained. - - s.tflock.acquire() - s.c.lock() - try: - s.gettf().setcolor('white') - s._addline(" *** Input Required", s.gettf().getcolorpair()) - s._addline(" *** Please enter password for account %s: " % accountname, - s.gettf().getcolorpair()) - s.logwindow.refresh() - password = s.logwindow.getstr() - finally: - s.tflock.release() - s.c.unlock() - s.inputhandler.input_release() - return password - - def setupwindows(s): - s.c.lock() - try: - s.bannerwindow = curses.newwin(1, s.c.width, 0, 0) - s.setupwindow_drawbanner() - s.logheight = s.c.height - 1 - len(s.af.keys()) - s.logwindow = curses.newwin(s.logheight, s.c.width, 1, 0) - s.logwindow.idlok(1) - s.logwindow.scrollok(1) - s.logwindow.move(s.logheight - 1, 0) - s.setupwindow_drawlog() - accounts = s.af.keys() - accounts.sort() - accounts.reverse() - - pos = s.c.height - 1 - index = 0 - s.hotkeys = [] - for account in accounts: - accountwindow = curses.newwin(1, s.c.width, pos, 0) - s.af[account].setwindow(accountwindow, acctkeys[index]) - s.hotkeys.append(account) - index += 1 - pos -= 1 - - curses.doupdate() - finally: - s.c.unlock() - - def setupwindow_drawbanner(s): - if s.c.has_color: - color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLUE) | \ - curses.A_BOLD - else: - color = curses.A_REVERSE - s.bannerwindow.bkgd(' ', color) # Fill background with that color - s.bannerwindow.addstr("%s %s" % (offlineimap.__productname__, - offlineimap.__version__)) - s.bannerwindow.addstr(0, s.bannerwindow.getmaxyx()[1] - len(offlineimap.__copyright__) - 1, - offlineimap.__copyright__) - - s.bannerwindow.noutrefresh() - - def setupwindow_drawlog(s): - if s.c.has_color: - color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLACK) - else: - color = curses.A_NORMAL - s.logwindow.bkgd(' ', color) - for line, color in s.text: - s.logwindow.addstr("\n" + line, color) - s.logwindow.noutrefresh() - - def getaccountframe(s, accountname = None): - if accountname == None: - accountname = s.getthreadaccount() - s.aflock.acquire() - try: - if accountname in s.af: - return s.af[accountname] - - # New one. - s.af[accountname] = CursesAccountFrame(s.c, accountname, s) - s.c.lock() - try: - s.c.reset() - s.setupwindows() - finally: - s.c.unlock() - finally: - s.aflock.release() - return s.af[accountname] + self.inputlock.release() + self.enabled.set() - def _display(s, msg, color = None): - if "\n" in msg: - for thisline in msg.split("\n"): - s._msg(thisline) - return +class CursesLogHandler(logging.StreamHandler): + def emit(self, record): + log_str = super(CursesLogHandler, self).format(record) + color = self.ui.gettf().curses_color # We must acquire both locks. Otherwise, deadlock can result. # This can happen if one thread calls _msg (locking curses, then # tf) and another tries to set the color (locking tf, then curses) # # By locking both up-front here, in this order, we prevent deadlock. - - s.tflock.acquire() - s.c.lock() + self.ui.tframe_lock.acquire() + self.ui.lock() try: - if not s.c.isactive(): - # For dumping out exceptions and stuff. - print msg - return - if color: - s.gettf().setcolor(color) - elif s.gettf().getcolor() == 'black': - s.gettf().setcolor('gray') - s._addline(msg, s.gettf().getcolorpair()) - s.logwindow.refresh() + for line in log_str.split("\n"): + self.ui.logwin.addstr("\n" + line, color) + self.ui.text.append((line, color)) + while len(self.ui.text) > self.ui.logheight: + self.ui.text.popleft() finally: - s.c.unlock() - s.tflock.release() + self.ui.unlock() + self.ui.tframe_lock.release() + self.ui.logwin.refresh() + self.ui.stdscr.refresh() - def _addline(s, msg, color): - s.c.lock() +class Blinkenlights(UIBase, CursesUtil): + """Curses-cased fancy UI + + Notable instance variables self. ....: + + - stdscr: THe curses std screen + - bannerwin: The top line banner window + - width|height: The total curses screen dimensions + - logheight: Available height for the logging part + - log_con_handler: The CursesLogHandler() + - threadframes: + - accframes[account]: 'Accountframe'""" + + def __init__(self, *args, **kwargs): + super(Blinkenlights, self).__init__(*args, **kwargs) + CursesUtil.__init__(self) + + ################################################## UTILS + def setup_consolehandler(self): + """Backend specific console handler + + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = CursesLogHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + # the handler is not usable yet. We still need all the + # intialization stuff currently done in init_banner. Move here? + return ch + + def isusable(s): + """Returns true if the backend is usable ie Curses works""" + # Not a terminal? Can't use curses. + if not sys.stdout.isatty() and sys.stdin.isatty(): + return False + # No TERM specified? Can't use curses. + if not os.environ.get('TERM', None): + return False + # Test if ncurses actually starts up fine. Only do so for + # python>=2.6.6 as calling initscr() twice messing things up. + # see http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 + if sys.version_info[0:3] < (2,6) or sys.version_info[0:3] >= (2,6,6): + try: + curses.initscr() + curses.endwin() + except: + return False + return True + + def init_banner(self): + self.availablethreadframes = {} + self.threadframes = {} + self.accframes = {} + self.aflock = Lock() + self.text = deque() + + self.stdscr = curses.initscr() + # turn off automatic echoing of keys to the screen + curses.noecho() + # react to keys instantly, without Enter key + curses.cbreak() + # return special key values, eg curses.KEY_LEFT + self.stdscr.keypad(1) + # wait 1s for input, so we don't block the InputHandler infinitely + self.stdscr.timeout(1000) + curses.start_color() + # turn off cursor and save original state + self.oldcursor = None try: - s.logwindow.addstr("\n" + msg, color) - s.text.append((msg, color)) - while len(s.text) > s.logheight: - s.text = s.text[1:] - finally: - s.c.unlock() + self.oldcursor = curses.curs_set(0) + except: + pass - def terminate(s, exitstatus = 0, errortitle = None, errormsg = None): - s.c.stop() - UIBase.terminate(s, exitstatus = exitstatus, errortitle = errortitle, errormsg = errormsg) + self.stdscr.clear() + self.stdscr.refresh() + self.init_colorpairs() + # set log handlers ui to ourself + self._log_con_handler.ui = self + self.setupwindows() + # Settup keyboard handler + self.inputhandler = InputHandler(self) + self.inputhandler.set_char_hdlr(self.on_keypressed) - def threadException(s, thread): - s.c.stop() - UIBase.threadException(s, thread) + self.gettf().setcolor('red') + self.info(offlineimap.banner) - def mainException(s): - s.c.stop() - UIBase.mainException(s) + def acct(self, *args): + """Output that we start syncing an account (and start counting)""" + self.gettf().setcolor('purple') + super(Blinkenlights, self).acct(*args) + + def connecting(self, *args): + self.gettf().setcolor('white') + super(Blinkenlights, self).connecting(*args) + + def syncfolders(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).syncfolders(*args) + + def syncingfolder(self, *args): + self.gettf().setcolor('cyan') + super(Blinkenlights, self).syncingfolder(*args) + + def skippingfolder(self, *args): + self.gettf().setcolor('cyan') + super(Blinkenlights, self).skippingfolder(*args) + + def loadmessagelist(self, *args): + self.gettf().setcolor('green') + super(Blinkenlights, self).loadmessagelist(*args) + + def syncingmessages(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).syncingmessages(*args) + + def copyingmessage(self, *args): + self.gettf().setcolor('orange') + super(Blinkenlights, self).copyingmessage(*args) + + def deletingmessages(self, *args): + self.gettf().setcolor('red') + super(Blinkenlights, self).deletingmessages(*args) + + def addingflags(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).addingflags(*args) + + def deletingflags(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).deletingflags(*args) + + def callhook(self, *args): + self.gettf().setcolor('white') + super(Blinkenlights, self).callhook(*args) + + ############ Generic logging functions ############################# + def warn(self, msg, minor=0): + self.gettf().setcolor('red', curses.A_BOLD) + super(Blinkenlights, self).warn(msg) + + def threadExited(self, thread): + acc_name = self.getthreadaccount(thread) + with self.tframe_lock: + if thread in self.threadframes[acc_name]: + tf = self.threadframes[acc_name][thread] + tf.setcolor('black') + self.availablethreadframes[acc_name].append(tf) + del self.threadframes[acc_name][thread] + super(Blinkenlights, self).threadExited(thread) + + def gettf(self): + """Return the ThreadFrame() of the current thread""" + cur_thread = currentThread() + acc_name = self.getthreadaccount() + + with self.tframe_lock: + # Ideally we already have self.threadframes[accountname][thread] + try: + if cur_thread in self.threadframes[acc_name]: + return self.threadframes[acc_name][cur_thread] + except KeyError: + # Ensure threadframes already has an account dict + self.threadframes[acc_name] = {} + self.availablethreadframes[acc_name] = deque() + + # If available, return a ThreadFrame() + if len(self.availablethreadframes[acc_name]): + tf = self.availablethreadframes[acc_name].popleft() + tf.std_color() + else: + tf = self.getaccountframe(acc_name).get_new_tframe() + self.threadframes[acc_name][cur_thread] = tf + return tf + + def on_keypressed(self, key): + # received special KEY_RESIZE, resize terminal + if key == curses.KEY_RESIZE: + self.resizeterm() + + if key < 1 or key > 255: + return + if chr(key) == 'q': + # Request to quit. + #TODO: this causes us to bail out in main loop when the thread exits + #TODO: review and rework this mechanism. + currentThread().setExitCause('EXCEPTION') + self.terminate() + try: + index = acctkeys.index(chr(key)) + except ValueError: + # Key not a valid one: exit. + return + if index >= len(self.hotkeys): + # Not in our list of valid hotkeys. + return + # Trying to end sleep somewhere. + self.getaccountframe(self.hotkeys[index]).syncnow() def sleep(s, sleepsecs, account): s.gettf().setcolor('red') s._msg("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return BlinkenBase.sleep(s, sleepsecs, account) - -if __name__ == '__main__': - x = Blinkenlights(None) - x.init_banner() - import time - time.sleep(5) - x.c.stop() - fgs = {'black': curses.COLOR_BLACK, 'red': curses.COLOR_RED, - 'green': curses.COLOR_GREEN, 'yellow': curses.COLOR_YELLOW, - 'blue': curses.COLOR_BLUE, 'magenta': curses.COLOR_MAGENTA, - 'cyan': curses.COLOR_CYAN, 'white': curses.COLOR_WHITE} - - x = CursesUtil() - win1 = curses.newwin(x.height, x.width / 4 - 1, 0, 0) - win1.addstr("Black/normal\n") - for name, fg in fgs.items(): - win1.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLACK)) - win2 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1]) - win2.addstr("Blue/normal\n") - for name, fg in fgs.items(): - win2.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLUE)) - win3 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1] + - win2.getmaxyx()[1]) - win3.addstr("Black/bright\n") - for name, fg in fgs.items(): - win3.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLACK) | \ - curses.A_BOLD) - win4 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1] * 3) - win4.addstr("Blue/bright\n") - for name, fg in fgs.items(): - win4.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLUE) | \ - curses.A_BOLD) - - - win1.refresh() - win2.refresh() - win3.refresh() - win4.refresh() - x.stdscr.refresh() - import time - time.sleep(5) - x.stop() - print x.has_color - print x.height - print x.width + s.getaccountframe().startsleep(sleepsecs) + return UIBase.sleep(s, sleepsecs, account) + + def sleeping(self, sleepsecs, remainingsecs): + if remainingsecs and s.gettf().getcolor() == 'black': + self.gettf().setcolor('red') + else: + self.gettf().setcolor('black') + return self.getaccountframe().sleeping(sleepsecs, remainingsecs) + + def resizeterm(self): + """Resize the current windows""" + self.exec_locked(self.setupwindows(True)) + + def mainException(self): + UIBase.mainException(self) + + def getpass(self, accountname, config, errmsg = None): + # disable the hotkeys inputhandler + self.inputhandler.input_acquire() + + # See comment on _msg for info on why both locks are obtained. + self.lock() + try: + #s.gettf().setcolor('white') + self.warn(" *** Input Required") + self.warn(" *** Please enter password for account %s: " % \ + accountname) + self.logwin.refresh() + password = self.logwin.getstr() + finally: + self.unlock() + self.inputhandler.input_release() + return password + + def setupwindows(self, resize=False): + """Setup and draw bannerwin and logwin + + If `resize`, don't create new windows, just adapt size""" + self.height, self.width = self.stdscr.getmaxyx() + if resize: + raise Exception("resizehandler %d" % self.width) + + self.logheight = self.height - len(self.accframes) - 1 + if resize: + curses.resizeterm(self.height, self.width) + self.bannerwin.resize(1, self.width) + else: + self.bannerwin = curses.newwin(1, self.width, 0, 0) + self.logwin = curses.newwin(self.logheight, self.width, 1, 0) + + self.draw_bannerwin() + self.logwin.idlok(1) + self.logwin.scrollok(1) + self.logwin.move(self.logheight - 1, 0) + self.draw_logwin() + self.accounts = reversed(sorted(self.accframes.keys())) + + pos = self.height - 1 + index = 0 + self.hotkeys = [] + for account in self.accounts: + acc_win = curses.newwin(1, self.width, pos, 0) + self.accframes[account].setwindow(acc_win, acctkeys[index]) + self.hotkeys.append(account) + index += 1 + pos -= 1 + curses.doupdate() + + def draw_bannerwin(self): + """Draw the top-line banner line""" + if curses.has_colors(): + color = curses.A_BOLD | self.curses_colorpair('banner') + else: + color = curses.A_REVERSE + self.bannerwin.bkgd(' ', color) # Fill background with that color + string = "%s %s" % (offlineimap.__productname__, + offlineimap.__version__) + self.bannerwin.addstr(0, 0, string, color) + self.bannerwin.addstr(0, self.width -len(offlineimap.__copyright__) -1, + offlineimap.__copyright__, color) + self.bannerwin.noutrefresh() + + def draw_logwin(self): + #if curses.has_colors(): + # color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLACK) + #else: + color = curses.A_NORMAL + self.logwin.bkgd(' ', color) + for line, color in self.text: + self.logwin.addstr("\n" + line, color) + self.logwin.noutrefresh() + + def getaccountframe(self, acc_name = None): + """Return an AccountFrame()""" + if acc_name == None: + acc_name = self.getthreadaccount() + with self.aflock: + # 1) Return existing or 2) create a new CursesAccountFrame. + if acc_name in self.accframes: return self.accframes[acc_name] + self.accframes[acc_name] = CursesAccountFrame(self, acc_name) + self.setupwindows() + return self.accframes[acc_name] + + def terminate(self, *args, **kwargs): + curses.nocbreak(); + self.stdscr.keypad(0); + curses.echo() + curses.endwin() + # need to remove the Curses console handler now and replace with + # basic one, so exceptions and stuff are properly displayed + self.logger.removeHandler(self._log_con_handler) + UIBase.setup_consolehandler(self) + # finally call parent terminate which prints out exceptions etc + super(Blinkenlights, self).terminate(*args, **kwargs) + + def threadException(s, thread): + #self._log_con_handler.stop() + UIBase.threadException(s, thread) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 50a7f99..334121d 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -1,5 +1,4 @@ -# Copyright (C) 2007 John Goerzen -# +# Copyright (C) 2007-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,48 +17,30 @@ import urllib import sys import time +import logging from UIBase import UIBase -from threading import currentThread, Lock +from threading import currentThread import offlineimap -protocol = '6.0.0' +protocol = '7.0.0' class MachineUI(UIBase): - def __init__(s, config, verbose = 0): - UIBase.__init__(s, config, verbose) - s.safechars=" ;,./-_=+()[]" - s.iswaiting = 0 - s.outputlock = Lock() - s._printData('__init__', protocol) + def __init__(self, config, loglevel = logging.INFO): + super(MachineUI, self).__init__(config, loglevel) + self._log_con_handler.createLock() + """lock needed to block on password input""" - def isusable(s): - return True + def _printData(self, command, msg): + self.logger.info("%s:%s:%s:%s" % ( + 'msg', command, currentThread().getName(), msg)) - def _printData(s, command, data, dolock = True): - s._printDataOut('msg', command, data, dolock) - - def _printWarn(s, command, data, dolock = True): - s._printDataOut('warn', command, data, dolock) - - def _printDataOut(s, datatype, command, data, dolock = True): - if dolock: - s.outputlock.acquire() - try: - print "%s:%s:%s:%s" % \ - (datatype, - urllib.quote(command, s.safechars), - urllib.quote(currentThread().getName(), s.safechars), - urllib.quote(data, s.safechars)) - sys.stdout.flush() - finally: - if dolock: - s.outputlock.release() - - def _display(s, msg): + def _msg(s, msg): s._printData('_display', msg) def warn(s, msg, minor = 0): - s._printData('warn', '%s\n%d' % (msg, int(minor))) + # TODO, remove and cleanup the unused minor stuff + self.logger.warning("%s:%s:%s:%s" % ( + 'warn', '', currentThread().getName(), msg)) def registerthread(s, account): UIBase.registerthread(s, account) @@ -112,13 +93,10 @@ class MachineUI(UIBase): self._printData('copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ (uid, self.getnicename(srcfolder), srcfolder.getname(), self.getnicename(destfolder), destfolder)) - + def folderlist(s, list): return ("\f".join(["%s\t%s" % (s.getnicename(x), x.getname()) for x in list])) - def deletingmessage(s, uid, destlist): - s.deletingmessages(s, [uid], destlist) - def uidlist(s, list): return ("\f".join([str(u) for u in list])) @@ -161,19 +139,21 @@ class MachineUI(UIBase): return 0 - def getpass(s, accountname, config, errmsg = None): - s.outputlock.acquire() + def getpass(self, accountname, config, errmsg = None): + if errmsg: + self._printData('getpasserror', "%s\n%s" % (accountname, errmsg), + False) + + self._log_con_handler.acquire() # lock the console output try: - if errmsg: - s._printData('getpasserror', "%s\n%s" % (accountname, errmsg), - False) - s._printData('getpass', accountname, False) + self._printData('getpass', accountname, False) return (sys.stdin.readline()[:-1]) finally: - s.outputlock.release() + self._log_con_handler.release() - def init_banner(s): - s._printData('initbanner', offlineimap.banner) + def init_banner(self): + self._printData('protocol', protocol) + self._printData('initbanner', offlineimap.banner) - def callhook(s, msg): - s._printData('callhook', msg) + def callhook(self, msg): + self._printData('callhook', msg) diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index 33248dd..36bb92b 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -1,6 +1,5 @@ # Noninteractive UI -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,37 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import sys -import time +import logging from UIBase import UIBase class Basic(UIBase): - def getpass(s, accountname, config, errmsg = None): - raise NotImplementedError, "Prompting for a password is not supported in noninteractive mode." + """'Quiet' simply sets log level to INFO""" + def __init__(self, config, loglevel = logging.INFO): + return super(Basic, self).__init__(config, loglevel) - def _display(s, msg): - print msg - sys.stdout.flush() - - def warn(s, msg, minor = 0): - warntxt = 'WARNING' - if minor: - warntxt = 'warning' - sys.stderr.write(warntxt + ": " + str(msg) + "\n") - - def sleep(s, sleepsecs, siglistener): - if s.verbose >= 0: - s._msg("Sleeping for %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return UIBase.sleep(s, sleepsecs, siglistener) - - def sleeping(s, sleepsecs, remainingsecs): - if sleepsecs > 0: - time.sleep(sleepsecs) - return 0 - - def locked(s): - s.warn("Another OfflineIMAP is running with the same metadatadir; exiting.") - -class Quiet(Basic): - def __init__(s, config, verbose = -1): - Basic.__init__(s, config, verbose) +class Quiet(UIBase): + """'Quiet' simply sets log level to WARNING""" + def __init__(self, config, loglevel = logging.WARNING): + return super(Quiet, self).__init__(config, loglevel) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 48c468f..cedd8a6 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -1,6 +1,5 @@ # TTY UI -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,53 +14,73 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from UIBase import UIBase -from getpass import getpass + +import logging import sys -from threading import Lock, currentThread +from getpass import getpass +from offlineimap import banner +from offlineimap.ui.UIBase import UIBase + +class TTYFormatter(logging.Formatter): + """Specific Formatter that adds thread information to the log output""" + def __init__(self, *args, **kwargs): + super(TTYFormatter, self).__init__(*args, **kwargs) + self._last_log_thread = None + + def format(self, record): + """Override format to add thread information""" + log_str = super(TTYFormatter, self).format(record) + # If msg comes from a different thread than our last, prepend + # thread info. Most look like 'Account sync foo' or 'Folder + # sync foo'. + t_name = record.threadName + if t_name == 'MainThread': + return log_str # main thread doesn't get things prepended + if t_name != self._last_log_thread: + self._last_log_thread = t_name + log_str = "%s:\n %s" % (t_name, log_str) + else: + log_str = " %s" % log_str + return log_str class TTYUI(UIBase): - def __init__(s, config, verbose = 0): - UIBase.__init__(s, config, verbose) - s.iswaiting = 0 - s.outputlock = Lock() - s._lastThreaddisplay = None + def setup_consolehandler(self): + """Backend specific console handler - def isusable(s): + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = logging.StreamHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = TTYFormatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(banner) + # init lock for console output + ch.createLock() + return ch + + def isusable(self): + """TTYUI is reported as usable when invoked on a terminal""" return sys.stdout.isatty() and sys.stdin.isatty() - - def _display(s, msg): - s.outputlock.acquire() - try: - #if the next output comes from a different thread than our last one - #add the info. - #Most look like 'account sync foo' or 'Folder sync foo'. - threadname = currentThread().getName() - if (threadname == s._lastThreaddisplay \ - or threadname == 'MainThread'): - print " %s" % msg - else: - print "%s:\n %s" % (threadname, msg) - s._lastThreaddisplay = threadname - sys.stdout.flush() - finally: - s.outputlock.release() - - def getpass(s, accountname, config, errmsg = None): + def getpass(self, accountname, config, errmsg = None): + """TTYUI backend is capable of querying the password""" if errmsg: - s._msg("%s: %s" % (accountname, errmsg)) - s.outputlock.acquire() + self.warn("%s: %s" % (accountname, errmsg)) + self._log_con_handler.acquire() # lock the console output try: - return getpass("%s: Enter password: " % accountname) + return getpass("Enter password for account '%s': " % accountname) finally: - s.outputlock.release() + self._log_con_handler.release() - def mainException(s): - if isinstance(sys.exc_info()[1], KeyboardInterrupt) and \ - s.iswaiting: - sys.stdout.write("Timer interrupted at user request; program terminating. \n") - s.terminate() + def mainException(self): + if isinstance(sys.exc_info()[1], KeyboardInterrupt): + self.logger.warn("Timer interrupted at user request; program " + "terminating.\n") + self.terminate() else: - UIBase.mainException(s) + UIBase.mainException(self) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index b5dcdaa..9f5f4bc 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -15,12 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import logging import re import time import sys +import os import traceback import threading from Queue import Queue +from collections import deque import offlineimap debugtypes = {'':'Other offlineimap related sync messages', @@ -38,54 +41,72 @@ def getglobalui(): global globalui return globalui -class UIBase: - def __init__(s, config, verbose = 0): - s.verbose = verbose - s.config = config - s.debuglist = [] - s.debugmessages = {} - s.debugmsglen = 50 - s.threadaccounts = {} +class UIBase(object): + def __init__(self, config, loglevel = logging.INFO): + self.config = config + self.debuglist = [] + """list of debugtypes we are supposed to log""" + self.debugmessages = {} + """debugmessages in a deque(v) per thread(k)""" + self.debugmsglen = 50 + self.threadaccounts = {} """dict linking active threads (k) to account names (v)""" - s.acct_startimes = {} + self.acct_startimes = {} """linking active accounts with the time.time() when sync started""" - s.logfile = None - s.exc_queue = Queue() + self.logfile = None + self.exc_queue = Queue() """saves all occuring exceptions, so we can output them at the end""" + # create logger with 'OfflineImap' app + self.logger = logging.getLogger('OfflineImap') + self.logger.setLevel(loglevel) + self._log_con_handler = self.setup_consolehandler() + """The console handler (we need access to be able to lock it)""" ################################################## UTILS - def _msg(s, msg): - """Generic tool called when no other works.""" - s._log(msg) - s._display(msg) + def setup_consolehandler(self): + """Backend specific console handler - def _log(s, msg): - """Log it to disk. Returns true if it wrote something; false - otherwise.""" - if s.logfile: - s.logfile.write("%s: %s\n" % (threading.currentThread().getName(), - msg)) - return 1 - return 0 + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = logging.StreamHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(offlineimap.banner) + return ch - def setlogfd(s, logfd): - s.logfile = logfd - logfd.write("This is %s %s\n" % \ - (offlineimap.__productname__, - offlineimap.__version__)) - logfd.write("Python: %s\n" % sys.version) - logfd.write("Platform: %s\n" % sys.platform) - logfd.write("Args: %s\n" % sys.argv) + def setlogfile(self, logfile): + """Create file handler which logs to file""" + fh = logging.FileHandler(logfile, 'wt') + #fh.setLevel(logging.DEBUG) + file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " + "%(message)s", '%Y-%m-%d %H:%M:%S') + fh.setFormatter(file_formatter) + self.logger.addHandler(fh) + # write out more verbose initial info blurb on the log file + p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) + msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ + "Args: %s" % (offlineimap.__version__, p_ver, sys.platform, + " ".join(sys.argv)) + record = logging.LogRecord('OfflineImap', logging.INFO, __file__, + None, msg, None, None) + fh.emit(record) - def _display(s, msg): + def _msg(self, msg): """Display a message.""" - raise NotImplementedError + # TODO: legacy function, rip out. + self.info(msg) - def warn(s, msg, minor = 0): - if minor: - s._msg("warning: " + msg) - else: - s._msg("WARNING: " + msg) + def info(self, msg): + """Display a message.""" + self.logger.info(msg) + + def warn(self, msg, minor = 0): + self.logger.warning(msg) def error(self, exc, exc_traceback=None, msg=None): """Log a message at severity level ERROR @@ -146,40 +167,43 @@ class UIBase: return self.threadaccounts[thr] return '*Control' # unregistered thread is '*Control' - def debug(s, debugtype, msg): - thisthread = threading.currentThread() - if s.debugmessages.has_key(thisthread): - s.debugmessages[thisthread].append("%s: %s" % (debugtype, msg)) - else: - s.debugmessages[thisthread] = ["%s: %s" % (debugtype, msg)] + def debug(self, debugtype, msg): + cur_thread = threading.currentThread() + if not self.debugmessages.has_key(cur_thread): + # deque(..., self.debugmsglen) would be handy but was + # introduced in p2.6 only, so we'll need to work around and + # shorten our debugmsg list manually :-( + self.debugmessages[cur_thread] = deque() + self.debugmessages[cur_thread].append("%s: %s" % (debugtype, msg)) - while len(s.debugmessages[thisthread]) > s.debugmsglen: - s.debugmessages[thisthread] = s.debugmessages[thisthread][1:] + # Shorten queue if needed + if len(self.debugmessages[cur_thread]) > self.debugmsglen: + self.debugmessages[cur_thread].popleft() - if debugtype in s.debuglist: - if not s._log("DEBUG[%s]: %s" % (debugtype, msg)): - s._display("DEBUG[%s]: %s" % (debugtype, msg)) + if debugtype in self.debuglist: # log if we are supposed to do so + self.logger.debug("[%s]: %s" % (debugtype, msg)) - def add_debug(s, debugtype): + def add_debug(self, debugtype): global debugtypes if debugtype in debugtypes: - if not debugtype in s.debuglist: - s.debuglist.append(debugtype) - s.debugging(debugtype) + if not debugtype in self.debuglist: + self.debuglist.append(debugtype) + self.debugging(debugtype) else: - s.invaliddebug(debugtype) + self.invaliddebug(debugtype) - def debugging(s, debugtype): + def debugging(self, debugtype): global debugtypes - s._msg("Now debugging for %s: %s" % (debugtype, debugtypes[debugtype])) + self.logger.debug("Now debugging for %s: %s" % (debugtype, + debugtypes[debugtype])) - def invaliddebug(s, debugtype): - s.warn("Invalid debug type: %s" % debugtype) + def invaliddebug(self, debugtype): + self.warn("Invalid debug type: %s" % debugtype) def locked(s): raise Exception, "Another OfflineIMAP is running with the same metadatadir; exiting." - def getnicename(s, object): + def getnicename(self, object): """Return the type of a repository or Folder as string (IMAP, Gmail, Maildir, etc...)""" @@ -187,61 +211,73 @@ class UIBase: # Strip off extra stuff. return re.sub('(Folder|Repository)', '', prelimname) - def isusable(s): + def isusable(self): """Returns true if this UI object is usable in the current environment. For instance, an X GUI would return true if it's being run in X with a valid DISPLAY setting, and false otherwise.""" - return 1 + return True ################################################## INPUT - def getpass(s, accountname, config, errmsg = None): - raise NotImplementedError + def getpass(self, accountname, config, errmsg = None): + raise NotImplementedError("Prompting for a password is not supported"\ + " in this UI backend.") - def folderlist(s, list): - return ', '.join(["%s[%s]" % (s.getnicename(x), x.getname()) for x in list]) + def folderlist(self, list): + return ', '.join(["%s[%s]" % \ + (self.getnicename(x), x.getname()) for x in list]) ################################################## WARNINGS - def msgtoreadonly(s, destfolder, uid, content, flags): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to synchronize message %d to folder %s[%s], but that folder is read-only. The message will not be copied to that folder." % \ - (uid, s.getnicename(destfolder), destfolder.getname())) + def msgtoreadonly(self, destfolder, uid, content, flags): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to synchronize message %d to folder %s[%s], " + "but that folder is read-only. The message will not be " + "copied to that folder." % ( + uid, self.getnicename(destfolder), destfolder)) - def flagstoreadonly(s, destfolder, uidlist, flags): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to modify flags for messages %s in folder %s[%s], but that folder is read-only. No flags have been modified for that message." % \ - (str(uidlist), s.getnicename(destfolder), destfolder.getname())) + def flagstoreadonly(self, destfolder, uidlist, flags): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to modify flags for messages %s in folder %s[%s], " + "but that folder is read-only. No flags have been modified " + "for that message." % ( + str(uidlist), self.getnicename(destfolder), destfolder)) - def deletereadonly(s, destfolder, uidlist): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to delete messages %s in folder %s[%s], but that folder is read-only. No messages have been deleted in that folder." % \ - (str(uidlist), s.getnicename(destfolder), destfolder.getname())) + def deletereadonly(self, destfolder, uidlist): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to delete messages %s in folder %s[%s], but that " + "folder is read-only. No messages have been deleted in that " + "folder." % (str(uidlist), self.getnicename(destfolder), + destfolder)) ################################################## MESSAGES - def init_banner(s): + def init_banner(self): """Called when the UI starts. Must be called before any other UI call except isusable(). Displays the copyright banner. This is where the UI should do its setup -- TK, for instance, would create the application window here.""" - if s.verbose >= 0: - s._msg(offlineimap.banner) + pass - def connecting(s, hostname, port): + def connecting(self, hostname, port): """Log 'Establishing connection to'""" - if s.verbose < 0: return + if not self.logger.isEnabledFor(logging.info): return displaystr = '' hostname = hostname if hostname else '' port = "%s" % port if port else '' if hostname: displaystr = ' to %s:%s' % (hostname, port) - s._msg("Establishing connection%s" % displaystr) + self.logger.info("Establishing connection%s" % displaystr) def acct(self, account): """Output that we start syncing an account (and start counting)""" self.acct_startimes[account] = time.time() - if self.verbose >= 0: - self._msg("*** Processing account %s" % account) + self.logger.info("*** Processing account %s" % account) def acctdone(self, account): """Output that we finished syncing an account (in which time)""" @@ -252,75 +288,63 @@ class UIBase: def syncfolders(self, src_repo, dst_repo): """Log 'Copying folder structure...'""" - if self.verbose < 0: return - self.debug('', "Copying folder structure from %s to %s" % \ - (src_repo, dst_repo)) + if self.logger.isEnabledFor(logging.DEBUG): + self.debug('', "Copying folder structure from %s to %s" %\ + (src_repo, dst_repo)) ############################## Folder syncing - def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): + def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder): """Called when a folder sync operation is started.""" - if s.verbose >= 0: - s._msg("Syncing %s: %s -> %s" % (srcfolder.getname(), - s.getnicename(srcrepos), - s.getnicename(destrepos))) + self.logger.info("Syncing %s: %s -> %s" % (srcfolder, + self.getnicename(srcrepos), + self.getnicename(destrepos))) - def skippingfolder(s, folder): + def skippingfolder(self, folder): """Called when a folder sync operation is started.""" - if s.verbose >= 0: - s._msg("Skipping %s (not changed)" % folder.getname()) + self.logger.info("Skipping %s (not changed)" % folder) - def validityproblem(s, folder): - s.warn("UID validity problem for folder %s (repo %s) (saved %d; got %d); skipping it" % \ - (folder.getname(), folder.getrepository().getname(), + def validityproblem(self, folder): + self.logger.warning("UID validity problem for folder %s (repo %s) " + "(saved %d; got %d); skipping it. Please see FAQ " + "and manual how to handle this." % \ + (folder, folder.getrepository(), folder.getsaveduidvalidity(), folder.getuidvalidity())) - def loadmessagelist(s, repos, folder): - if s.verbose > 0: - s._msg("Loading message list for %s[%s]" % (s.getnicename(repos), - folder.getname())) + def loadmessagelist(self, repos, folder): + self.logger.debug("Loading message list for %s[%s]" % ( + self.getnicename(repos), + folder)) - def messagelistloaded(s, repos, folder, count): - if s.verbose > 0: - s._msg("Message list for %s[%s] loaded: %d messages" % \ - (s.getnicename(repos), folder.getname(), count)) + def messagelistloaded(self, repos, folder, count): + self.logger.debug("Message list for %s[%s] loaded: %d messages" % ( + self.getnicename(repos), folder, count)) ############################## Message syncing - def syncingmessages(s, sr, sf, dr, df): - if s.verbose > 0: - s._msg("Syncing messages %s[%s] -> %s[%s]" % (s.getnicename(sr), - sf.getname(), - s.getnicename(dr), - df.getname())) + def syncingmessages(self, sr, srcfolder, dr, dstfolder): + self.logger.debug("Syncing messages %s[%s] -> %s[%s]" % ( + self.getnicename(sr), srcfolder, + self.getnicename(dr), dstfolder)) def copyingmessage(self, uid, num, num_to_copy, src, destfolder): """Output a log line stating which message we copy""" - if self.verbose < 0: return - self._msg("Copy message %s (%d of %d) %s:%s -> %s" % (uid, num, - num_to_copy, src.repository, src, destfolder.repository)) + self.logger.info("Copy message %s (%d of %d) %s:%s -> %s" % ( + uid, num, num_to_copy, src.repository, src, + destfolder.repository)) - def deletingmessage(s, uid, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting message %d in %s" % (uid, ds)) + def deletingmessages(self, uidlist, destlist): + ds = self.folderlist(destlist) + self.logger.info("Deleting %d messages (%s) in %s" % ( + len(uidlist), + offlineimap.imaputil.uid_sequence(uidlist), ds)) - def deletingmessages(s, uidlist, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting %d messages (%s) in %s" % \ - (len(uidlist), - offlineimap.imaputil.uid_sequence(uidlist), - ds)) + def addingflags(self, uidlist, flags, dest): + self.logger.info("Adding flag %s to %d messages on %s" % ( + ", ".join(flags), len(uidlist), dest)) - def addingflags(s, uidlist, flags, dest): - if s.verbose >= 0: - s._msg("Adding flag %s to %d messages on %s" % \ - (", ".join(flags), len(uidlist), dest)) - - def deletingflags(s, uidlist, flags, dest): - if s.verbose >= 0: - s._msg("Deleting flag %s from %d messages on %s" % \ - (", ".join(flags), len(uidlist), dest)) + def deletingflags(self, uidlist, flags, dest): + self.logger.info("Deleting flag %s from %d messages on %s" % ( + ", ".join(flags), len(uidlist), dest)) def serverdiagnostics(self, repository, type): """Connect to repository and output useful information for debugging""" @@ -371,69 +395,68 @@ class UIBase: ################################################## Threads - def getThreadDebugLog(s, thread): - if s.debugmessages.has_key(thread): + def getThreadDebugLog(self, thread): + if self.debugmessages.has_key(thread): message = "\nLast %d debug messages logged for %s prior to exception:\n"\ - % (len(s.debugmessages[thread]), thread.getName()) - message += "\n".join(s.debugmessages[thread]) + % (len(self.debugmessages[thread]), thread.getName()) + message += "\n".join(self.debugmessages[thread]) else: message = "\nNo debug messages were logged for %s." % \ thread.getName() return message - def delThreadDebugLog(s, thread): - if s.debugmessages.has_key(thread): - del s.debugmessages[thread] + def delThreadDebugLog(self, thread): + if thread in self.debugmessages: + del self.debugmessages[thread] - def getThreadExceptionString(s, thread): + def getThreadExceptionString(self, thread): message = "Thread '%s' terminated with exception:\n%s" % \ (thread.getName(), thread.getExitStackTrace()) - message += "\n" + s.getThreadDebugLog(thread) + message += "\n" + self.getThreadDebugLog(thread) return message - def threadException(s, thread): + def threadException(self, thread): """Called when a thread has terminated with an exception. The argument is the ExitNotifyThread that has so terminated.""" - s._msg(s.getThreadExceptionString(thread)) - s.delThreadDebugLog(thread) - s.terminate(100) + self.warn(self.getThreadExceptionString(thread)) + self.delThreadDebugLog(thread) + self.terminate(100) def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" #print any exceptions that have occurred over the run if not self.exc_queue.empty(): - self._msg("\nERROR: Exceptions occurred during the run!") + self.warn("ERROR: Exceptions occurred during the run!") while not self.exc_queue.empty(): msg, exc, exc_traceback = self.exc_queue.get() if msg: - self._msg("ERROR: %s\n %s" % (msg, exc)) + self.warn("ERROR: %s\n %s" % (msg, exc)) else: - self._msg("ERROR: %s" % (exc)) + self.warn("ERROR: %s" % (exc)) if exc_traceback: - self._msg("\nTraceback:\n%s" %"".join( + self.warn("\nTraceback:\n%s" %"".join( traceback.format_tb(exc_traceback))) if errormsg and errortitle: - sys.stderr.write('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) + self.warn('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) elif errormsg: - sys.stderr.write('%s\n' % errormsg) + self.warn('%s\n' % errormsg) sys.exit(exitstatus) - def threadExited(s, thread): + def threadExited(self, thread): """Called when a thread has exited normally. Many UIs will just ignore this.""" - s.delThreadDebugLog(thread) - s.unregisterthread(thread) + self.delThreadDebugLog(thread) + self.unregisterthread(thread) ################################################## Hooks - def callhook(s, msg): - if s.verbose >= 0: - s._msg(msg) + def callhook(self, msg): + self.info(msg) ################################################## Other - def sleep(s, sleepsecs, account): + def sleep(self, sleepsecs, account): """This function does not actually output anything, but handles the overall sleep, dealing with updates as necessary. It will, however, call sleeping() which DOES output something. @@ -446,12 +469,12 @@ class UIBase: if account.get_abort_event(): abortsleep = True else: - abortsleep = s.sleeping(10, sleepsecs) - sleepsecs -= 10 - s.sleeping(0, 0) # Done sleeping. + abortsleep = self.sleeping(10, sleepsecs) + sleepsecs -= 10 + self.sleeping(0, 0) # Done sleeping. return abortsleep - def sleeping(s, sleepsecs, remainingsecs): + def sleeping(self, sleepsecs, remainingsecs): """Sleep for sleepsecs, display remainingsecs to go. Does nothing if sleepsecs <= 0. @@ -463,6 +486,7 @@ class UIBase: """ if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: - s._msg("Next refresh in %.1f minutes" % (remainingsecs/60.0)) + self.logger.info("Next refresh in %.1f minutes" % ( + remainingsecs/60.0)) time.sleep(sleepsecs) return 0 diff --git a/offlineimap/ui/__init__.py b/offlineimap/ui/__init__.py index 102cbfd..6ddbaf6 100644 --- a/offlineimap/ui/__init__.py +++ b/offlineimap/ui/__init__.py @@ -1,5 +1,5 @@ # UI module -# Copyright (C) 2010 Sebastian Spaeth +# Copyright (C) 2010-2011 Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From f6f8fc852809798823793d9800577321765e3be4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 27 Oct 2011 16:56:18 +0200 Subject: [PATCH 316/817] Simplify the keepalive code a bit Should not change behavior. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 25 +++++++++---------------- offlineimap/repository/Base.py | 2 +- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 7006ef9..e709125 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -389,47 +389,40 @@ class IMAPServer: to be invoked in a separate thread, which should be join()'d after the event is set.""" self.ui.debug('imap', 'keepalive thread started') - while 1: - self.ui.debug('imap', 'keepalive: top of loop') - if event.isSet(): - self.ui.debug('imap', 'keepalive: event is set; exiting') - return - self.ui.debug('imap', 'keepalive: acquiring connectionlock') + while not event.isSet(): self.connectionlock.acquire() numconnections = len(self.assignedconnections) + \ len(self.availableconnections) self.connectionlock.release() - self.ui.debug('imap', 'keepalive: connectionlock released') + threads = [] - for i in range(numconnections): self.ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections)) if len(self.idlefolders) > i: + # IDLE thread idler = IdleThread(self, self.idlefolders[i]) else: + # NOOP thread idler = IdleThread(self) idler.start() threads.append(idler) - self.ui.debug('imap', 'keepalive: thread started') self.ui.debug('imap', 'keepalive: waiting for timeout') event.wait(timeout) self.ui.debug('imap', 'keepalive: after wait') - self.ui.debug('imap', 'keepalive: joining threads') - for idler in threads: # Make sure all the commands have completed. idler.stop() idler.join() - - self.ui.debug('imap', 'keepalive: bottom of loop') - + self.ui.debug('imap', 'keepalive: all threads joined') + self.ui.debug('imap', 'keepalive: event is set; exiting') + return def verifycert(self, cert, hostname): '''Verify that cert (in socket.getpeercert() format) matches hostname. CRLs are not handled. - + Returns error message if any problems are found and None on success. ''' errstr = "CA Cert verifying failed: " @@ -508,7 +501,7 @@ class IdleThread(object): finally: if imapobj: self.parent.releaseconnection(imapobj) - self.stop_sig.wait() # wait until we are supposed to quit + self.stop_sig.wait() # wait until we are supposed to quit def dosync(self): remoterepos = self.parent.repos diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 8f400f0..8af1f03 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -229,4 +229,4 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): """Stop keep alive, but don't bother waiting for the threads to terminate.""" pass - + From 8195d1410f94de4956047f8dc004c0acc68ee386 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 27 Oct 2011 16:58:44 +0200 Subject: [PATCH 317/817] Prettify Blinkenlights.sleep() Prettify the function. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 3dc32f5..46b5157 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -506,11 +506,11 @@ class Blinkenlights(UIBase, CursesUtil): # Trying to end sleep somewhere. self.getaccountframe(self.hotkeys[index]).syncnow() - def sleep(s, sleepsecs, account): - s.gettf().setcolor('red') - s._msg("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - s.getaccountframe().startsleep(sleepsecs) - return UIBase.sleep(s, sleepsecs, account) + def sleep(self, sleepsecs, account): + self.gettf().setcolor('red') + self.info("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) + self.getaccountframe().startsleep(sleepsecs) + return super(Blinkenlights, self).sleep(sleepsecs, account) def sleeping(self, sleepsecs, remainingsecs): if remainingsecs and s.gettf().getcolor() == 'black': From d992c66156ccd15eac3d9b92a0a49a375e36b40d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 27 Oct 2011 17:23:43 +0200 Subject: [PATCH 318/817] Rework the whole unused get/setExitCause machinery It is basically unused by now. Rework to be able to make use of it later, no functional changes. Signed-off-by: Sebastian Spaeth --- offlineimap/threadutil.py | 72 ++++++++++++++++----------------------- offlineimap/ui/Curses.py | 2 +- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 39d3ae7..2dfb194 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -109,15 +109,15 @@ def exitnotifymonitorloop(callback): def threadexited(thread): """Called when a thread exits.""" ui = getglobalui() - if thread.getExitCause() == 'EXCEPTION': - if isinstance(thread.getExitException(), SystemExit): + if thread.exit_exception: + if isinstance(thread.exit_exception, SystemExit): # Bring a SystemExit into the main thread. # Do not send it back to UI layer right now. # Maybe later send it to ui.terminate? raise SystemExit ui.threadException(thread) # Expected to terminate sys.exit(100) # Just in case... - elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE': + elif thread.exit_message == 'SYNC_WITH_TIMER_TERMINATE': ui.terminate() # Just in case... sys.exit(100) @@ -128,7 +128,10 @@ class ExitNotifyThread(Thread): """This class is designed to alert a "monitor" to the fact that a thread has exited and to provide for the ability for it to find out why. All instances are made daemon threads (setDaemon(True), so we - bail out when the mainloop dies.""" + bail out when the mainloop dies. + + The thread can set instance variables self.exit_message for a human + readable reason of the thread exit.""" profiledir = None """class variable that is set to the profile directory if required""" @@ -137,6 +140,9 @@ class ExitNotifyThread(Thread): # These are all child threads that are supposed to go away when # the main thread is killed. self.setDaemon(True) + self.exit_message = None + self._exit_exc = None + self._exit_stacktrace = None def run(self): global exitthreads @@ -156,49 +162,31 @@ class ExitNotifyThread(Thread): pass prof.dump_stats(os.path.join(ExitNotifyThread.profiledir, "%s_%s.prof" % (self.threadid, self.getName()))) - except: - self.setExitCause('EXCEPTION') - if sys: - self.setExitException(sys.exc_info()[1]) - tb = traceback.format_exc() - self.setExitStackTrace(tb) - else: - self.setExitCause('NORMAL') - if not hasattr(self, 'exitmessage'): - self.setExitMessage(None) + except Exception, e: + # Thread exited with Exception, store it + tb = traceback.format_exc() + self.set_exit_exception(e, tb) if exitthreads: exitthreads.put(self, True) - def setExitCause(self, cause): - self.exitcause = cause - def getExitCause(self): + def set_exit_exception(self, exc, st=None): + """Sets Exception and stacktrace of a thread, so that other + threads can query its exit status""" + self._exit_exc = exc + self._exit_stacktrace = st + + @property + def exit_exception(self): """Returns the cause of the exit, one of: - 'EXCEPTION' -- the thread aborted because of an exception - 'NORMAL' -- normal termination.""" - return self.exitcause - def setExitException(self, exc): - self.exitexception = exc - def getExitException(self): - """If getExitCause() is 'EXCEPTION', holds the value from - sys.exc_info()[1] for this exception.""" - return self.exitexception - def setExitStackTrace(self, st): - self.exitstacktrace = st - def getExitStackTrace(self): - """If getExitCause() is 'EXCEPTION', returns a string representing - the stack trace for this exception.""" - return self.exitstacktrace - def setExitMessage(self, msg): - """Sets the exit message to be fetched by a subsequent call to - getExitMessage. This message may be any object or type except - None.""" - self.exitmessage = msg - def getExitMessage(self): - """For any exit cause, returns the message previously set by - a call to setExitMessage(), or None if there was no such message - set.""" - return self.exitmessage + Exception() -- the thread aborted with this exception + None -- normal termination.""" + return self._exit_exc + + @property + def exit_stacktrace(self): + """Returns a string representing the stack trace if set""" + return self._exit_stacktrace @classmethod def set_profiledir(cls, directory): diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 46b5157..2afb6cd 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -493,7 +493,7 @@ class Blinkenlights(UIBase, CursesUtil): # Request to quit. #TODO: this causes us to bail out in main loop when the thread exits #TODO: review and rework this mechanism. - currentThread().setExitCause('EXCEPTION') + currentThread().set_exit_exception(SystemExit("User requested shutdown")) self.terminate() try: index = acctkeys.index(chr(key)) From 7c45e05428e6e6a9437c2f3dc6597e19e864b111 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 27 Oct 2011 17:45:00 +0200 Subject: [PATCH 319/817] Fix getExitStackTrace call It was changed --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 9f5f4bc..44c920f 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -411,7 +411,7 @@ class UIBase(object): def getThreadExceptionString(self, thread): message = "Thread '%s' terminated with exception:\n%s" % \ - (thread.getName(), thread.getExitStackTrace()) + (thread.getName(), thread.exit_stacktrace) message += "\n" + self.getThreadDebugLog(thread) return message From d981d66305d3e8342e74cf5a130339aab1eb5396 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 28 Oct 2011 09:35:23 +0200 Subject: [PATCH 320/817] Add changelog entry for reworked UI backends Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 1f6b46b..778b52c 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -15,7 +15,7 @@ New Features * add a --info command line switch that outputs useful information about the server and the configuration for all enabled accounts. - + Changes ------- @@ -23,6 +23,14 @@ Changes * Output how long an account sync took (min:sec). +* Reworked logging which was reported to e.g. not flush output to files + often enough. User-visible changes: + a) console output goes to stderr (for now). + b) file output has timestamps and looks identical in the basic and + ttyui UIs. + c) File output should be flushed after logging by default (do + report if not). + Bug Fixes --------- From 33b4a16dac98e303322d2a0e56404e865c762d28 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 08:40:03 +0100 Subject: [PATCH 321/817] Fix mbox.select(foldername) readonly parameter comparison The default parameter value was "None", and we were comparing that directly to the imaplib2 value of is_readonly which is False or True, so the comparison always returned "False". Fix this by setting the default parameter to "False" and not "None". Also convert all users of that function to use False/True. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ offlineimap/folder/IMAP.py | 4 ++-- offlineimap/imaplibutil.py | 14 +++++++------- offlineimap/repository/IMAP.py | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 778b52c..ff9fc5b 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -43,3 +43,7 @@ Bug Fixes * Make NOOPs to keep a server connection open more resistant against dropped connections. + +* a readonly parameter to select() was not always treated correctly, + which could result in some folders being opened read-only when we + really needed read-write. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 58ff633..a7b8762 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -54,7 +54,7 @@ class IMAPFolder(BaseFolder): try: imapobj.select(self.getfullname()) except imapobj.readonly: - imapobj.select(self.getfullname(), readonly = 1) + imapobj.select(self.getfullname(), readonly = True) def suggeststhreads(self): return 1 @@ -204,7 +204,7 @@ class IMAPFolder(BaseFolder): fails_left = 2 # retry on dropped connection while fails_left: try: - imapobj.select(self.getfullname(), readonly = 1) + imapobj.select(self.getfullname(), readonly = True) res_type, data = imapobj.uid('fetch', str(uid), '(BODY.PEEK[])') fails_left = 0 diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 5c94592..c9a7715 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -34,37 +34,37 @@ except ImportError: #fails on python <2.6 pass -class UsefulIMAPMixIn: +class UsefulIMAPMixIn(object): def getselectedfolder(self): if self.state == 'SELECTED': return self.mailbox return None - def select(self, mailbox='INBOX', readonly=None, force = 0): + def select(self, mailbox='INBOX', readonly=False, force = 0): """Selects a mailbox on the IMAP server :returns: 'OK' on success, nothing if the folder was already selected or raises an :exc:`OfflineImapError`""" - if (not force) and self.getselectedfolder() == mailbox \ - and self.is_readonly == readonly: + if self.getselectedfolder() == mailbox and self.is_readonly == readonly \ + and not force: # No change; return. return # Wipe out all old responses, to maintain semantics with old imaplib2 del self.untagged_responses[:] try: - result = self.__class__.__bases__[1].select(self, mailbox, readonly) + result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) except self.abort, e: # self.abort is raised when we are supposed to retry errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ "ver said: %s" % (self.host, mailbox, e.args[0]) severity = OfflineImapError.ERROR.FOLDER_RETRY - raise OfflineImapError(errstr, severity) + raise OfflineImapError(errstr, severity) if result[0] != 'OK': #in case of error, bail out with OfflineImapError errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\ (mailbox, result) severity = OfflineImapError.ERROR.FOLDER - raise OfflineImapError(errstr, severity) + raise OfflineImapError(errstr, severity) return result def _mesg(self, s, tn=None, secs=None): diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 4c2cadd..8c29ee2 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -289,7 +289,7 @@ class IMAPRepository(BaseRepository): try: for foldername in self.folderincludes: try: - imapobj.select(foldername, readonly = 1) + imapobj.select(foldername, readonly = True) except OfflineImapError, e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: From 8b3ed8b00451e5f1145c93e6171cb8613903ac1c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 08:40:03 +0100 Subject: [PATCH 322/817] Fix mbox.select(foldername) readonly parameter comparison The default parameter value was "None", and we were comparing that directly to the imaplib2 value of is_readonly which is False or True, so the comparison always returned "False". Fix this by setting the default parameter to "False" and not "None". Also convert all users of that function to use False/True. Signed-off-by: Sebastian Spaeth Conflicts: Changelog.draft.rst --- Changelog.draft.rst | 4 ++++ offlineimap/folder/IMAP.py | 4 ++-- offlineimap/imaplibutil.py | 14 +++++++------- offlineimap/repository/IMAP.py | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5095b2e..67f72df 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -29,3 +29,7 @@ Bug Fixes * New folders on the remote would be skipped on the very sync run they are created and only by synced in subsequent runs. Fixed. + +* a readonly parameter to select() was not always treated correctly, + which could result in some folders being opened read-only when we + really needed read-write. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8dbd7aa..5eeba4c 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -55,7 +55,7 @@ class IMAPFolder(BaseFolder): try: imapobj.select(self.getfullname()) except imapobj.readonly: - imapobj.select(self.getfullname(), readonly = 1) + imapobj.select(self.getfullname(), readonly = True) def suggeststhreads(self): return 1 @@ -205,7 +205,7 @@ class IMAPFolder(BaseFolder): fails_left = 2 # retry on dropped connection while fails_left: try: - imapobj.select(self.getfullname(), readonly = 1) + imapobj.select(self.getfullname(), readonly = True) res_type, data = imapobj.uid('fetch', str(uid), '(BODY.PEEK[])') fails_left = 0 diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 5c94592..c9a7715 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -34,37 +34,37 @@ except ImportError: #fails on python <2.6 pass -class UsefulIMAPMixIn: +class UsefulIMAPMixIn(object): def getselectedfolder(self): if self.state == 'SELECTED': return self.mailbox return None - def select(self, mailbox='INBOX', readonly=None, force = 0): + def select(self, mailbox='INBOX', readonly=False, force = 0): """Selects a mailbox on the IMAP server :returns: 'OK' on success, nothing if the folder was already selected or raises an :exc:`OfflineImapError`""" - if (not force) and self.getselectedfolder() == mailbox \ - and self.is_readonly == readonly: + if self.getselectedfolder() == mailbox and self.is_readonly == readonly \ + and not force: # No change; return. return # Wipe out all old responses, to maintain semantics with old imaplib2 del self.untagged_responses[:] try: - result = self.__class__.__bases__[1].select(self, mailbox, readonly) + result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) except self.abort, e: # self.abort is raised when we are supposed to retry errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ "ver said: %s" % (self.host, mailbox, e.args[0]) severity = OfflineImapError.ERROR.FOLDER_RETRY - raise OfflineImapError(errstr, severity) + raise OfflineImapError(errstr, severity) if result[0] != 'OK': #in case of error, bail out with OfflineImapError errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\ (mailbox, result) severity = OfflineImapError.ERROR.FOLDER - raise OfflineImapError(errstr, severity) + raise OfflineImapError(errstr, severity) return result def _mesg(self, s, tn=None, secs=None): diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 2d2962f..f93510f 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -290,7 +290,7 @@ class IMAPRepository(BaseRepository): try: for foldername in self.folderincludes: try: - imapobj.select(foldername, readonly = 1) + imapobj.select(foldername, readonly = True) except OfflineImapError, e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: From 006f19aba90640d931f475635a8a69694d498e72 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:34:45 +0100 Subject: [PATCH 323/817] Make releaseconnection a NOOP when conn is None During cleanup we often call releaseconnection in a finally: block. But in cases of error, we might have dropped the connection earlier already and set it to "None". In this case don't fail releaseconnection() but make it a NOOP. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index e709125..de1cf95 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -113,6 +113,7 @@ class IMAPServer: :param drop_conn: If True, the connection will be released and not be reused. This can be used to indicate broken connections.""" + if connection is None: return #noop on bad connection self.connectionlock.acquire() self.assignedconnections.remove(connection) # Don't reuse broken connections From 656405616d77b75264a23a86e725884bfeb68632 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:27:08 +0100 Subject: [PATCH 324/817] Drop connection if it might be bad on APPEND 1) Differentiate error messages between imaplib.abort and imaplib.error exceptions in the log. 2) Drop connections in the case of imapobj.error, it also might denote a broken connection. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index a7b8762..4471387 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -540,17 +540,19 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() if not retry_left: raise OfflineImapError("Saving msg in folder '%s', " - "repository '%s' failed. Server reponded: %s\n" + "repository '%s' failed (abort). Server reponded: %s\n" "Message content was: %s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) self.ui.error(e, exc_info()[2]) - except imapobj.error, e: # APPEND failed # If the server responds with 'BAD', append() # raise()s directly. So we catch that too. + # drop conn, it might be bad. + self.imapserver.releaseconnection(imapobj, True) + imapobj = None raise OfflineImapError("Saving msg folder '%s', repo '%s'" - "failed. Server reponded: %s\nMessage content was: " + "failed (error). Server reponded: %s\nMessage content was: " "%s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for From 3aded16ed4de691d79581f3b27d4da8dd9bc684d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:34:45 +0100 Subject: [PATCH 325/817] Make releaseconnection a NOOP when conn is None During cleanup we often call releaseconnection in a finally: block. But in cases of error, we might have dropped the connection earlier already and set it to "None". In this case don't fail releaseconnection() but make it a NOOP. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index ef54e31..b7f062d 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -113,6 +113,7 @@ class IMAPServer: :param drop_conn: If True, the connection will be released and not be reused. This can be used to indicate broken connections.""" + if connection is None: return #noop on bad connection self.connectionlock.acquire() self.assignedconnections.remove(connection) # Don't reuse broken connections From dd02c307db3365ce64471f440aa7ddc58f5a2917 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:27:08 +0100 Subject: [PATCH 326/817] Drop connection if it might be bad on APPEND 1) Differentiate error messages between imaplib.abort and imaplib.error exceptions in the log. 2) Drop connections in the case of imapobj.error, it also might denote a broken connection. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 5eeba4c..2fdea30 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -541,17 +541,19 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() if not retry_left: raise OfflineImapError("Saving msg in folder '%s', " - "repository '%s' failed. Server reponded: %s\n" + "repository '%s' failed (abort). Server reponded: %s\n" "Message content was: %s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) self.ui.error(e, exc_info()[2]) - except imapobj.error, e: # APPEND failed # If the server responds with 'BAD', append() # raise()s directly. So we catch that too. + # drop conn, it might be bad. + self.imapserver.releaseconnection(imapobj, True) + imapobj = None raise OfflineImapError("Saving msg folder '%s', repo '%s'" - "failed. Server reponded: %s\nMessage content was: " + "failed (error). Server reponded: %s\nMessage content was: " "%s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for From 54117e702d78e549df82c0233a3a23a248a78349 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:55:42 +0100 Subject: [PATCH 327/817] Bump bundled imaplib2 to 2.29 It has fixed some bugs, so update to current upstream. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/imaplib2.py | 27 +++++++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index ff9fc5b..13b97e8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -31,6 +31,8 @@ Changes c) File output should be flushed after logging by default (do report if not). +* Bumped bundled imaplib2 to release 2.29 + Bug Fixes --------- diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index cf3c480..ffa2676 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.28" +__version__ = "2.29" __release__ = "2" -__revision__ = "28" +__revision__ = "29" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -1234,11 +1234,13 @@ class IMAP4(object): def _choose_nonull_or_dflt(self, dflt, *args): dflttyp = type(dflt) + if isinstance(dflttyp, basestring): + dflttyp = basestring # Allow any string type for arg in args: if arg is not None: - if type(arg) is dflttyp: + if isinstance(arg, dflttyp): return arg - if __debug__: self._log(1, 'bad arg type is %s, expecting %s' % (type(arg), dflttyp)) + if __debug__: self._log(0, 'bad arg is %s, expecting %s' % (type(arg), dflttyp)) return dflt @@ -2323,6 +2325,7 @@ if __name__ == '__main__': data = open(os.path.exists("test.data") and "test.data" or __file__).read(1000) test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)s%(data)s' \ % {'user':USER, 'lf':'\n', 'data':data} + test_seq1 = [ ('list', ('""', '%')), ('create', ('/tmp/imaplib2_test.0',)), @@ -2342,7 +2345,7 @@ if __name__ == '__main__': test_seq2 = ( ('select', ()), - ('response',('UIDVALIDITY',)), + ('response', ('UIDVALIDITY',)), ('response', ('EXISTS',)), ('append', (None, None, None, test_mesg)), ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')), @@ -2351,6 +2354,7 @@ if __name__ == '__main__': ('recent', ()), ) + AsyncError = None def responder((response, cb_arg, error)): @@ -2436,10 +2440,21 @@ if __name__ == '__main__': if 'IDLE' in M.capabilities: run('idle', (2,), cb=False) - run('idle', (99,), cb=True) # Asynchronous, to test interruption of 'idle' by 'noop' + run('idle', (99,)) # Asynchronous, to test interruption of 'idle' by 'noop' time.sleep(1) run('noop', (), cb=False) + run('append', (None, None, None, test_mesg), cb=False) + num = run('search', (None, 'ALL'), cb=False)[0].split()[0] + dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) + M._mesg('fetch %s => %s' % (num, `dat`)) + run('idle', (2,)) + run('store', (num, '-FLAGS', '(\Seen)'), cb=False), + dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) + M._mesg('fetch %s => %s' % (num, `dat`)) + run('uid', ('STORE', num, 'FLAGS', '(\Deleted)')) + run('expunge', ()) + run('logout', (), cb=False) if debug: From 4eeb88dd8fff60f11321192cf5ca264e8ddcb3e9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:56:42 +0100 Subject: [PATCH 328/817] Append, don't truncate, log file The new logging handler was truncating the log file on each start. Append by default. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 44c920f..75cb313 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -81,7 +81,7 @@ class UIBase(object): def setlogfile(self, logfile): """Create file handler which logs to file""" - fh = logging.FileHandler(logfile, 'wt') + fh = logging.FileHandler(logfile, 'at') #fh.setLevel(logging.DEBUG) file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " "%(message)s", '%Y-%m-%d %H:%M:%S') From d54859a9318baf6d6eb000a82793d3beff20f547 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:29:23 +0100 Subject: [PATCH 329/817] Don't setDaemon explicitly, it's done inherently All ExitNotifyThreads and InstanceLimitThreads are setDaemon(True) in their constructor, so there is no need to do that again in the code. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 1 - offlineimap/folder/Base.py | 1 - offlineimap/init.py | 1 - offlineimap/threadutil.py | 4 ++-- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 7387c51..7a87791 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -291,7 +291,6 @@ class SyncableAccount(Account): name = "Folder %s [acc: %s]" % (remotefolder, self), args = (self.name, remoterepos, remotefolder, localrepos, statusrepos, quick)) - thread.setDaemon(1) thread.start() folderthreads.append(thread) # wait for all threads to finish diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 91ee08a..f8cea25 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -341,7 +341,6 @@ class BaseFolder(object): target = self.copymessageto, name = "Copy message from %s:%s" % (self.repository, self), args = (uid, dstfolder, statusfolder)) - thread.setDaemon(1) thread.start() threads.append(thread) else: diff --git a/offlineimap/init.py b/offlineimap/init.py index 0e5b08d..36240ac 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -358,7 +358,6 @@ class OfflineImap: name='Sync Runner', kwargs = {'accounts': syncaccounts, 'config': self.config}) - t.setDaemon(True) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) self.ui.terminate() diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 2dfb194..e94910b 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -28,8 +28,8 @@ from offlineimap.ui import getglobalui ###################################################################### def semaphorereset(semaphore, originalstate): - """Wait until the semaphore gets back to its original state -- all acquired - resources released.""" + """Block until `semaphore` gets back to its original state, ie all acquired + resources have been released.""" for i in range(originalstate): semaphore.acquire() # Now release these. From ccfa747ce65583c2352e6546885a9a47ff59e5cb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:30:16 +0100 Subject: [PATCH 330/817] Use lock with 'with' statement To make sure, the lock gets released even if we raise an exception between acquire() and release() Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index de1cf95..721c646 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +from __future__ import with_statement # needed for python 2.5 from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui from threading import Lock, BoundedSemaphore, Thread, Event, currentThread @@ -355,7 +356,7 @@ class IMAPServer: else: # re-raise all other errors raise - + def connectionwait(self): """Waits until there is a connection available. Note that between the time that a connection becomes available and the time it is @@ -370,18 +371,18 @@ class IMAPServer: def close(self): # Make sure I own all the semaphores. Let the threads finish # their stuff. This is a blocking method. - self.connectionlock.acquire() - threadutil.semaphorereset(self.semaphore, self.maxconnections) - for imapobj in self.assignedconnections + self.availableconnections: - imapobj.logout() - self.assignedconnections = [] - self.availableconnections = [] - self.lastowner = {} - # reset kerberos state - self.gss_step = self.GSS_STATE_STEP - self.gss_vc = None - self.gssapi = False - self.connectionlock.release() + with self.connectionlock: + # first, wait till all + threadutil.semaphorereset(self.semaphore, self.maxconnections) + for imapobj in self.assignedconnections + self.availableconnections: + imapobj.logout() + self.assignedconnections = [] + self.availableconnections = [] + self.lastowner = {} + # reset kerberos state + self.gss_step = self.GSS_STATE_STEP + self.gss_vc = None + self.gssapi = False def keepalive(self, timeout, event): """Sends a NOOP to each connection recorded. It will wait a maximum From 52ca66f055f5f65a5533b5ede616f140d90b4b51 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:37:29 +0100 Subject: [PATCH 331/817] Reduce log verbosity while scanning Maildir no need to be that verbose here... Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Maildir.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index ae09486..6e93839 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -149,14 +149,12 @@ class MaildirRepository(BaseRepository): # Iterate over directories in top & top itself. for dirname in os.listdir(toppath) + ['']: - self.debug(" *** top of loop") self.debug(" dirname = %s" % dirname) if dirname in ['cur', 'new', 'tmp']: self.debug(" skipping this dir (Maildir special)") # Bypass special files. continue fullname = os.path.join(toppath, dirname) - self.debug(" fullname = %s" % fullname) if not os.path.isdir(fullname): self.debug(" skipping this entry (not a directory)") # Not a directory -- not a folder. From 74d580bc6892f0a22bbbc573c2418b8fbf839933 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:55:05 +0100 Subject: [PATCH 332/817] Exit "infinite" monitorloop when SyncRunner thread exits The huge UI rework patch removed some obscure logic and special handling of thread exit messages. It turns out that this was in fact still needed as a specific exit message of the SyncRunner thread signified the threatmonitor to quit. We will want a nicer machinery for this in the future I guess, but fix the eternal hang on exit by reintroducing a special exit message for the SyncRunner thread, and return from the infinite monitor loop if SyncRunner finishes. Signed-off-by: Sebastian Spaeth --- offlineimap/syncmaster.py | 2 ++ offlineimap/threadutil.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/offlineimap/syncmaster.py b/offlineimap/syncmaster.py index 0d139dd..5fd0dee 100644 --- a/offlineimap/syncmaster.py +++ b/offlineimap/syncmaster.py @@ -30,6 +30,8 @@ def syncaccount(threads, config, accountname): threads.add(thread) def syncitall(accounts, config): + # Special exit message for SyncRunner thread, so main thread can exit + currentThread().exit_message = 'SYNCRUNNER_EXITED_NORMALLY' threads = threadlist() for accountname in accounts: syncaccount(threads, config, accountname) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index e94910b..c102446 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -95,19 +95,23 @@ def exitnotifymonitorloop(callback): :type callback: a callable function """ global exitthreads - while 1: + do_loop = True + while do_loop: # Loop forever and call 'callback' for each thread that exited try: # we need a timeout in the get() call, so that ctrl-c can throw # a SIGINT (http://bugs.python.org/issue1360). A timeout with empty # Queue will raise `Empty`. thrd = exitthreads.get(True, 60) - callback(thrd) + # request to abort when callback returns true + do_loop = (callback(thrd) != True) except Empty: pass def threadexited(thread): - """Called when a thread exits.""" + """Called when a thread exits. + + Main thread is aborted when this returns True.""" ui = getglobalui() if thread.exit_exception: if isinstance(thread.exit_exception, SystemExit): @@ -117,12 +121,11 @@ def threadexited(thread): raise SystemExit ui.threadException(thread) # Expected to terminate sys.exit(100) # Just in case... - elif thread.exit_message == 'SYNC_WITH_TIMER_TERMINATE': - ui.terminate() - # Just in case... - sys.exit(100) + elif thread.exit_message == 'SYNCRUNNER_EXITED_NORMALLY': + return True else: ui.threadExited(thread) + return False class ExitNotifyThread(Thread): """This class is designed to alert a "monitor" to the fact that a From 8dbf62cfdba9924e5e82b041e118f6fe059637da Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:56:04 +0100 Subject: [PATCH 333/817] Add a TODO comment This function can IMHO lead to possible deadlocks when waiting for the connectionlock. Do add a comment to that regard, this will need to audit. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 721c646..2f41bde 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -372,7 +372,10 @@ class IMAPServer: # Make sure I own all the semaphores. Let the threads finish # their stuff. This is a blocking method. with self.connectionlock: - # first, wait till all + # first, wait till all connections had been released. + # TODO: won't work IMHO, as releaseconnection() also + # requires the connectionlock, leading to a potential + # deadlock! Audit & check! threadutil.semaphorereset(self.semaphore, self.maxconnections) for imapobj in self.assignedconnections + self.availableconnections: imapobj.logout() From 212e50eb4b53055e247feb29d34e8a007610c5cc Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 12:48:22 +0100 Subject: [PATCH 334/817] Fix sleeping in the Blinkenlights UI We were still referring to s.gettf() in sleeping(self, ...) causing each attempt to sleep to crash. Fix this, and the CursesAccountFrame.sleeping() method. I am sure, there is still wrong and broken but we are getting there. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 2afb6cd..42a8f01 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -151,26 +151,15 @@ class CursesAccountFrame: self.children.append(tf) return tf - def startsleep(s, sleepsecs): - s.sleeping_abort = 0 + def startsleep(self, sleepsecs): + self.sleeping_abort = 0 - def sleeping(s, sleepsecs, remainingsecs): - if remainingsecs: - s.c.lock() - try: - s.drawleadstr(remainingsecs) - s.window.refresh() - finally: - s.c.unlock() - time.sleep(sleepsecs) - else: - s.c.lock() - try: - s.drawleadstr() - s.window.refresh() - finally: - s.c.unlock() - return s.sleeping_abort + def sleeping(self, sleepsecs, remainingsecs): + # show how long we are going to sleep and sleep + self.drawleadstr(remainingsecs) + self.ui.exec_locked(self.window.refresh) + time.sleep(sleepsecs) + return 0 def syncnow(s): s.sleeping_abort = 1 @@ -513,10 +502,9 @@ class Blinkenlights(UIBase, CursesUtil): return super(Blinkenlights, self).sleep(sleepsecs, account) def sleeping(self, sleepsecs, remainingsecs): - if remainingsecs and s.gettf().getcolor() == 'black': - self.gettf().setcolor('red') - else: - self.gettf().setcolor('black') + if not sleepsecs: + # reset color to default if we are done sleeping. + self.gettf().setcolor('white') return self.getaccountframe().sleeping(sleepsecs, remainingsecs) def resizeterm(self): From bfbd37802525279fbfbd540a6e06eafa60b90921 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 15:56:33 +0100 Subject: [PATCH 335/817] Add FAQ item on "Mailbox already exists" error IMAP servers treating folder names as case insensitive can lead to that error. Signed-off-by: Sebastian Spaeth --- docs/FAQ.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index e3a586b..29b53d6 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -219,6 +219,27 @@ as follows:: 2) while in sleep mode, you can also send a SIGUSR1. See the `Signals on UNIX`_ section in the MANUAL for details. +I get a "Mailbox already exists" error +-------------------------------------- +**Q:** When synchronizing, I receive errors such as:: + + Folder 'sent'[main-remote] could not be created. Server responded: + ('NO', ['Mailbox already exists.']) + +**A:** IMAP folders are usually case sensitive. But some IMAP servers seem + to treat "special" folders as case insensitive (e.g. the initial + INBOX. part, or folders such as "Sent" or "Trash"). If you happen to + have a folder "sent" on one side of things and a folder called "Sent" + on the other side, offlineimap will try to create those folders on + both sides. If you server happens to treat those folders as + case-insensitive you can then see this warning. + + You can solve this by excluding the "sent" folder by filtering it from + the repository settings:: + + folderfilter= lambda f: f not in ['sent'] + + Configuration Questions ======================= From 70125d58e67de1fff83a3e59d9498668153a2d55 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 17:03:40 +0100 Subject: [PATCH 336/817] Curses UI: make resize behave better Resizing a Blinkenlights terminal doesn't crash anymore, and actually seems to be changing the size, with this patch. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 42a8f01..341d4e8 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -119,6 +119,7 @@ class CursesAccountFrame: self.children = [] self.accountname = accountname self.ui = ui + self.window = None def drawleadstr(self, secs = None): #TODO: does what? @@ -178,7 +179,7 @@ class CursesThreadFrame: self.curses_color = curses.color_pair(0) #default color def setcolor(self, color, modifier=0): - """Draw the thread symbol '.' in the specified color + """Draw the thread symbol '@' in the specified color :param modifier: Curses modified, such as curses.A_BOLD""" self.curses_color = modifier | self.ui.curses_colorpair(color) self.colorname = color @@ -186,7 +187,7 @@ class CursesThreadFrame: def display(self): def locked_display(): - self.window.addch(self.y, self.x, '.', self.curses_color) + self.window.addch(self.y, self.x, '@', self.curses_color) self.window.refresh() # lock the curses IO while fudging stuff self.ui.exec_locked(locked_display) @@ -289,7 +290,7 @@ class CursesLogHandler(logging.StreamHandler): finally: self.ui.unlock() self.ui.tframe_lock.release() - self.ui.logwin.refresh() + self.ui.logwin.noutrefresh() self.ui.stdscr.refresh() class Blinkenlights(UIBase, CursesUtil): @@ -509,7 +510,7 @@ class Blinkenlights(UIBase, CursesUtil): def resizeterm(self): """Resize the current windows""" - self.exec_locked(self.setupwindows(True)) + self.exec_locked(self.setupwindows, True) def mainException(self): UIBase.mainException(self) @@ -537,33 +538,30 @@ class Blinkenlights(UIBase, CursesUtil): If `resize`, don't create new windows, just adapt size""" self.height, self.width = self.stdscr.getmaxyx() - if resize: - raise Exception("resizehandler %d" % self.width) - self.logheight = self.height - len(self.accframes) - 1 if resize: curses.resizeterm(self.height, self.width) self.bannerwin.resize(1, self.width) + self.logwin.resize(self.logheight, self.width) else: self.bannerwin = curses.newwin(1, self.width, 0, 0) self.logwin = curses.newwin(self.logheight, self.width, 1, 0) self.draw_bannerwin() - self.logwin.idlok(1) - self.logwin.scrollok(1) + self.logwin.idlok(True) # needed for scrollok below + self.logwin.scrollok(True) # scroll window when too many lines added self.logwin.move(self.logheight - 1, 0) self.draw_logwin() self.accounts = reversed(sorted(self.accframes.keys())) - pos = self.height - 1 index = 0 self.hotkeys = [] for account in self.accounts: - acc_win = curses.newwin(1, self.width, pos, 0) - self.accframes[account].setwindow(acc_win, acctkeys[index]) - self.hotkeys.append(account) - index += 1 - pos -= 1 + acc_win = curses.newwin(1, self.width, pos, 0) + self.accframes[account].setwindow(acc_win, acctkeys[index]) + self.hotkeys.append(account) + index += 1 + pos -= 1 curses.doupdate() def draw_bannerwin(self): @@ -572,6 +570,7 @@ class Blinkenlights(UIBase, CursesUtil): color = curses.A_BOLD | self.curses_colorpair('banner') else: color = curses.A_REVERSE + self.bannerwin.clear() # Delete old content (eg before resizes) self.bannerwin.bkgd(' ', color) # Fill background with that color string = "%s %s" % (offlineimap.__productname__, offlineimap.__version__) @@ -581,10 +580,12 @@ class Blinkenlights(UIBase, CursesUtil): self.bannerwin.noutrefresh() def draw_logwin(self): - #if curses.has_colors(): - # color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLACK) - #else: - color = curses.A_NORMAL + """(Re)draw the current logwindow""" + if curses.has_colors(): + color = curses.color_pair(0) #default colors + else: + color = curses.A_NORMAL + self.logwin.clear() self.logwin.bkgd(' ', color) for line, color in self.text: self.logwin.addstr("\n" + line, color) @@ -598,7 +599,8 @@ class Blinkenlights(UIBase, CursesUtil): # 1) Return existing or 2) create a new CursesAccountFrame. if acc_name in self.accframes: return self.accframes[acc_name] self.accframes[acc_name] = CursesAccountFrame(self, acc_name) - self.setupwindows() + # update the window layout + self.setupwindows(resize= True) return self.accframes[acc_name] def terminate(self, *args, **kwargs): From ef28d5dac0efebcfaea6776e4f5988f1fb48e65b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Nov 2011 12:25:55 +0100 Subject: [PATCH 337/817] More Blinkenlights UI cleanup Rename some variables, simplify the hotkeys treatment. Refresh/exit signals still don't work as of yet, but will come. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 51 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 341d4e8..bc63f7e 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -1,6 +1,5 @@ # Curses-based interfaces -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from __future__ import with_statement # needed for python 2.5 -from threading import RLock, currentThread, Lock, Event, Thread +from threading import RLock, currentThread, Lock, Event from thread import get_ident # python < 2.6 support from collections import deque import time @@ -30,8 +29,6 @@ from offlineimap.ui.UIBase import UIBase from offlineimap.threadutil import ExitNotifyThread import offlineimap -acctkeys = '1234567890abcdefghijklmnoprstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-=;/.,' - class CursesUtil: def __init__(self, *args, **kwargs): @@ -115,28 +112,34 @@ class CursesAccountFrame: - window: curses window associated with an account """ - def __init__(self, ui, accountname): + def __init__(self, ui, acc_name): self.children = [] - self.accountname = accountname + self.acc_name = acc_name self.ui = ui self.window = None + """Curses window associated with this acc""" + self.acc_num = None + """Account number (& hotkey) associated with this acc""" + self.location = 0 + """length of the account prefix string""" - def drawleadstr(self, secs = None): - #TODO: does what? - if secs == None: - acctstr = '%s: [active] %13.13s: ' % (self.key, self.accountname) - else: - acctstr = '%s: [%3d:%02d] %13.13s: ' % (self.key, - secs / 60, secs % 60, - self.accountname) - self.ui.exec_locked(self.window.addstr, 0, 0, acctstr) - self.location = len(acctstr) + def drawleadstr(self, secs = 0): + """Draw the account status string - def setwindow(self, curses_win, key): - #TODO: does what? - # the curses window associated with an account + secs tells us how long we are going to sleep.""" + sleepstr = '%3d:%02d' % (secs // 60, secs % 60) if secs else 'active' + accstr = '%s: [%s] %12.12s: ' % (self.acc_num, sleepstr, self.acc_name) + + self.ui.exec_locked(self.window.addstr, 0, 0, accstr) + self.location = len(accstr) + + def setwindow(self, curses_win, acc_num): + """Register an curses win and a hotkey as Account window + + :param curses_win: the curses window associated with an account + :param acc_num: int denoting the hotkey associated with this account.""" self.window = curses_win - self.key = key + self.acc_num = acc_num self.drawleadstr() # Update the child ThreadFrames for child in self.children: @@ -486,9 +489,9 @@ class Blinkenlights(UIBase, CursesUtil): currentThread().set_exit_exception(SystemExit("User requested shutdown")) self.terminate() try: - index = acctkeys.index(chr(key)) + index = int(chr(key)) except ValueError: - # Key not a valid one: exit. + # Key not a valid number: exit. return if index >= len(self.hotkeys): # Not in our list of valid hotkeys. @@ -558,7 +561,7 @@ class Blinkenlights(UIBase, CursesUtil): self.hotkeys = [] for account in self.accounts: acc_win = curses.newwin(1, self.width, pos, 0) - self.accframes[account].setwindow(acc_win, acctkeys[index]) + self.accframes[account].setwindow(acc_win, index) self.hotkeys.append(account) index += 1 pos -= 1 From ab184d84e23a2280d6c501159759027180e07683 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Nov 2011 13:27:35 +0100 Subject: [PATCH 338/817] Reduce parameter list to account.syncfolder call The remote|local|statusrepo is an anttribute of each SyncableAccount() anyway, so we don't need to pass it in, we can simply get it from the Account(). Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 19 ++++++++++--------- offlineimap/imapserver.py | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 7a87791..51a3ab9 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -289,8 +289,7 @@ class SyncableAccount(Account): instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, name = "Folder %s [acc: %s]" % (remotefolder, self), - args = (self.name, remoterepos, remotefolder, localrepos, - statusrepos, quick)) + args = (self, remotefolder, quick)) thread.start() folderthreads.append(thread) # wait for all threads to finish @@ -329,15 +328,17 @@ class SyncableAccount(Account): except Exception, e: self.ui.error(e, exc_info()[2], msg = "Calling hook") - -def syncfolder(accountname, remoterepos, remotefolder, localrepos, - statusrepos, quick): +def syncfolder(account, remotefolder, quick): """This function is called as target for the InstanceLimitedThread invokation in SyncableAccount. Filtered folders on the remote side will not invoke this function.""" + remoterepos = account.remoterepos + localrepos = account.localrepos + statusrepos = account.statusrepos + ui = getglobalui() - ui.registerthread(accountname) + ui.registerthread(account.name) try: # Load local folder. localfolder = localrepos.\ @@ -352,7 +353,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, % localfolder) return # Write the mailboxes - mbnames.add(accountname, localfolder.getvisiblename()) + mbnames.add(account.name, localfolder.getvisiblename()) # Load status folder. statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ @@ -431,11 +432,11 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, "[acc: '%s']" % ( remotefolder.getvisiblename().\ replace(remoterepos.getsep(), localrepos.getsep()), - accountname)) + account)) # we reconstruct foldername above rather than using # localfolder, as the localfolder var is not # available if assignment fails. except Exception, e: ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ - (accountname,remotefolder.getvisiblename(), + (account, remotefolder.getvisiblename(), traceback.format_exc())) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 2f41bde..b8446d3 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -515,9 +515,9 @@ class IdleThread(object): remoterepos = account.remoterepos statusrepos = account.statusrepos remotefolder = remoterepos.getfolder(self.folder) - offlineimap.accounts.syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False) + offlineimap.accounts.syncfolder(account, remotefolder, quick=False) ui = getglobalui() - ui.unregisterthread(currentThread()) + ui.unregisterthread(currentThread()) #syncfolder registered the thread def idle(self): """Invoke IDLE mode until timeout or self.stop() is invoked""" From f4a32bafd64ee80050ee3aa85aab5a5d369693b6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Nov 2011 13:45:44 +0100 Subject: [PATCH 339/817] Pass ui.registerthread an Account() and not a name as string This way, we can use all the account functions such as set_abort_event() from the ui if needed. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 4 ++-- offlineimap/folder/Base.py | 2 +- offlineimap/ui/Curses.py | 11 +++++------ offlineimap/ui/Machine.py | 6 +++--- offlineimap/ui/UIBase.py | 8 +++++--- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 51a3ab9..bf3f327 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -206,7 +206,7 @@ class SyncableAccount(Account): pass #Failed to delete for some reason. def syncrunner(self): - self.ui.registerthread(self.name) + self.ui.registerthread(self) accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): os.mkdir(accountmetadata, 0700) @@ -338,7 +338,7 @@ def syncfolder(account, remotefolder, quick): statusrepos = account.statusrepos ui = getglobalui() - ui.registerthread(account.name) + ui.registerthread(account) try: # Load local folder. localfolder = localrepos.\ diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index f8cea25..53bc71c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -254,7 +254,7 @@ class BaseFolder(object): # self.getmessage(). So, don't call self.getmessage unless # really needed. if register: # output that we start a new thread - self.ui.registerthread(self.accountname) + self.ui.registerthread(self.repository.account) try: message = None diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index bc63f7e..71ffe3a 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -105,16 +105,16 @@ class CursesUtil: class CursesAccountFrame: """Notable instance variables: - - accountname: String with associated account name + - account: corresponding Account() - children - ui - key - window: curses window associated with an account """ - def __init__(self, ui, acc_name): + def __init__(self, ui, account): self.children = [] - self.acc_name = acc_name + self.account = account self.ui = ui self.window = None """Curses window associated with this acc""" @@ -128,7 +128,7 @@ class CursesAccountFrame: secs tells us how long we are going to sleep.""" sleepstr = '%3d:%02d' % (secs // 60, secs % 60) if secs else 'active' - accstr = '%s: [%s] %12.12s: ' % (self.acc_num, sleepstr, self.acc_name) + accstr = '%s: [%s] %12.12s: ' % (self.acc_num, sleepstr, self.account) self.ui.exec_locked(self.window.addstr, 0, 0, accstr) self.location = len(accstr) @@ -491,8 +491,7 @@ class Blinkenlights(UIBase, CursesUtil): try: index = int(chr(key)) except ValueError: - # Key not a valid number: exit. - return + return # Key not a valid number: exit. if index >= len(self.hotkeys): # Not in our list of valid hotkeys. return diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 334121d..868433a 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -42,9 +42,9 @@ class MachineUI(UIBase): self.logger.warning("%s:%s:%s:%s" % ( 'warn', '', currentThread().getName(), msg)) - def registerthread(s, account): - UIBase.registerthread(s, account) - s._printData('registerthread', account) + def registerthread(self, account): + super(MachineUI, self).registerthread(self, account) + self._printData('registerthread', account) def unregisterthread(s, thread): UIBase.unregisterthread(s, thread) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 75cb313..0176422 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -160,12 +160,14 @@ class UIBase(object): self.debug('thread', "Unregister thread '%s'" % thr.getName()) def getthreadaccount(self, thr = None): - """Get name of account for a thread (current if None)""" - if not thr: + """Get Account() for a thread (current if None) + + If no account has been registered with this thread, return 'None'""" + if thr == None: thr = threading.currentThread() if thr in self.threadaccounts: return self.threadaccounts[thr] - return '*Control' # unregistered thread is '*Control' + return None def debug(self, debugtype, msg): cur_thread = threading.currentThread() From f8d5f1890ca7725df629967a6ace80030f1c9dbf Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Nov 2011 14:21:25 +0100 Subject: [PATCH 340/817] Make sleep abort request working again for Curses UI 1) Rework the sleep abort request to set the skipsleep configuration setting that the sleep() code checks. 2) Only output 15 rather than 50 debug messages on abort... Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 63 ++++++++++++++++++++++------------------ offlineimap/ui/UIBase.py | 2 +- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 71ffe3a..35a17fb 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -113,8 +113,10 @@ class CursesAccountFrame: """ def __init__(self, ui, account): + """ + :param account: An Account() or None (for eg SyncrunnerThread)""" self.children = [] - self.account = account + self.account = account if account else '*Control' self.ui = ui self.window = None """Curses window associated with this acc""" @@ -155,18 +157,21 @@ class CursesAccountFrame: self.children.append(tf) return tf - def startsleep(self, sleepsecs): - self.sleeping_abort = 0 - def sleeping(self, sleepsecs, remainingsecs): # show how long we are going to sleep and sleep self.drawleadstr(remainingsecs) self.ui.exec_locked(self.window.refresh) time.sleep(sleepsecs) - return 0 + return self.account.abort_signal.is_set() - def syncnow(s): - s.sleeping_abort = 1 + def syncnow(self): + """Request that we stop sleeping asap and continue to sync""" + # if this belongs to an Account (and not *Control), set the + # skipsleep pref + if isinstance(self.account, offlineimap.accounts.Account): + self.ui.info("Requested synchronization for acc: %s" % self.account) + self.account.config.set('Account %s' % self.account.name, + 'skipsleep', '1') class CursesThreadFrame: """ @@ -442,37 +447,37 @@ class Blinkenlights(UIBase, CursesUtil): super(Blinkenlights, self).warn(msg) def threadExited(self, thread): - acc_name = self.getthreadaccount(thread) + acc = self.getthreadaccount(thread) with self.tframe_lock: - if thread in self.threadframes[acc_name]: - tf = self.threadframes[acc_name][thread] + if thread in self.threadframes[acc]: + tf = self.threadframes[acc][thread] tf.setcolor('black') - self.availablethreadframes[acc_name].append(tf) - del self.threadframes[acc_name][thread] + self.availablethreadframes[acc].append(tf) + del self.threadframes[acc][thread] super(Blinkenlights, self).threadExited(thread) def gettf(self): """Return the ThreadFrame() of the current thread""" cur_thread = currentThread() - acc_name = self.getthreadaccount() + acc = self.getthreadaccount() #Account() or None with self.tframe_lock: # Ideally we already have self.threadframes[accountname][thread] try: - if cur_thread in self.threadframes[acc_name]: - return self.threadframes[acc_name][cur_thread] + if cur_thread in self.threadframes[acc]: + return self.threadframes[acc][cur_thread] except KeyError: # Ensure threadframes already has an account dict - self.threadframes[acc_name] = {} - self.availablethreadframes[acc_name] = deque() + self.threadframes[acc] = {} + self.availablethreadframes[acc] = deque() # If available, return a ThreadFrame() - if len(self.availablethreadframes[acc_name]): - tf = self.availablethreadframes[acc_name].popleft() + if len(self.availablethreadframes[acc]): + tf = self.availablethreadframes[acc].popleft() tf.std_color() else: - tf = self.getaccountframe(acc_name).get_new_tframe() - self.threadframes[acc_name][cur_thread] = tf + tf = self.getaccountframe(acc).get_new_tframe() + self.threadframes[acc][cur_thread] = tf return tf def on_keypressed(self, key): @@ -501,14 +506,14 @@ class Blinkenlights(UIBase, CursesUtil): def sleep(self, sleepsecs, account): self.gettf().setcolor('red') self.info("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - self.getaccountframe().startsleep(sleepsecs) return super(Blinkenlights, self).sleep(sleepsecs, account) def sleeping(self, sleepsecs, remainingsecs): if not sleepsecs: # reset color to default if we are done sleeping. self.gettf().setcolor('white') - return self.getaccountframe().sleeping(sleepsecs, remainingsecs) + accframe = self.getaccountframe(self.getthreadaccount()) + return accframe.sleeping(sleepsecs, remainingsecs) def resizeterm(self): """Resize the current windows""" @@ -593,10 +598,10 @@ class Blinkenlights(UIBase, CursesUtil): self.logwin.addstr("\n" + line, color) self.logwin.noutrefresh() - def getaccountframe(self, acc_name = None): - """Return an AccountFrame()""" - if acc_name == None: - acc_name = self.getthreadaccount() + def getaccountframe(self, acc_name): + """Return an AccountFrame() corresponding to acc_name + + Note that the *control thread uses acc_name `None`.""" with self.aflock: # 1) Return existing or 2) create a new CursesAccountFrame. if acc_name in self.accframes: return self.accframes[acc_name] @@ -617,7 +622,7 @@ class Blinkenlights(UIBase, CursesUtil): # finally call parent terminate which prints out exceptions etc super(Blinkenlights, self).terminate(*args, **kwargs) - def threadException(s, thread): + def threadException(self, thread): #self._log_con_handler.stop() - UIBase.threadException(s, thread) + UIBase.threadException(self, thread) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 0176422..7dc23e5 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -48,7 +48,7 @@ class UIBase(object): """list of debugtypes we are supposed to log""" self.debugmessages = {} """debugmessages in a deque(v) per thread(k)""" - self.debugmsglen = 50 + self.debugmsglen = 15 self.threadaccounts = {} """dict linking active threads (k) to account names (v)""" self.acct_startimes = {} From a93c80292d0474420654eb9cfa873ce20a6fbdc8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 8 Nov 2011 13:51:36 +0100 Subject: [PATCH 341/817] Curses UI: Simplify text buffer handling Rather than keeping a separate queue of all logged lines in memory, we rely on the curses window scrolling functionality to scroll lines. On resizing the terminal this means, we'll clear the screen and start filling it afresh, but that should be acceptable. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 35a17fb..f1626c4 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -278,6 +278,7 @@ class InputHandler(ExitNotifyThread): class CursesLogHandler(logging.StreamHandler): + """self.ui has been set to the UI class before anything is invoked""" def emit(self, record): log_str = super(CursesLogHandler, self).format(record) @@ -290,11 +291,9 @@ class CursesLogHandler(logging.StreamHandler): self.ui.tframe_lock.acquire() self.ui.lock() try: - for line in log_str.split("\n"): - self.ui.logwin.addstr("\n" + line, color) - self.ui.text.append((line, color)) - while len(self.ui.text) > self.ui.logheight: - self.ui.text.popleft() + y,x = self.ui.logwin.getyx() + if y or x: self.ui.logwin.addch(10) # no \n before 1st item + self.ui.logwin.addstr(log_str, color) finally: self.ui.unlock() self.ui.tframe_lock.release() @@ -360,7 +359,6 @@ class Blinkenlights(UIBase, CursesUtil): self.threadframes = {} self.accframes = {} self.aflock = Lock() - self.text = deque() self.stdscr = curses.initscr() # turn off automatic echoing of keys to the screen @@ -543,7 +541,8 @@ class Blinkenlights(UIBase, CursesUtil): def setupwindows(self, resize=False): """Setup and draw bannerwin and logwin - If `resize`, don't create new windows, just adapt size""" + If `resize`, don't create new windows, just adapt size. This + function should be invoked with CursesUtils.locked().""" self.height, self.width = self.stdscr.getmaxyx() self.logheight = self.height - len(self.accframes) - 1 if resize: @@ -555,9 +554,8 @@ class Blinkenlights(UIBase, CursesUtil): self.logwin = curses.newwin(self.logheight, self.width, 1, 0) self.draw_bannerwin() - self.logwin.idlok(True) # needed for scrollok below + self.logwin.idlok(True) # needed for scrollok below self.logwin.scrollok(True) # scroll window when too many lines added - self.logwin.move(self.logheight - 1, 0) self.draw_logwin() self.accounts = reversed(sorted(self.accframes.keys())) pos = self.height - 1 @@ -592,11 +590,9 @@ class Blinkenlights(UIBase, CursesUtil): color = curses.color_pair(0) #default colors else: color = curses.A_NORMAL - self.logwin.clear() + self.logwin.move(0, 0) + self.logwin.erase() self.logwin.bkgd(' ', color) - for line, color in self.text: - self.logwin.addstr("\n" + line, color) - self.logwin.noutrefresh() def getaccountframe(self, acc_name): """Return an AccountFrame() corresponding to acc_name From 1a4b536a5032f0837b9a690c7e1233b2b0b62c02 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 17 Nov 2011 08:55:32 +0100 Subject: [PATCH 342/817] Release current master as 6.4.1 Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 15 --------------- Changelog.rst | 24 ++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 67f72df..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,20 +16,5 @@ New Features Changes ------- -* Indicate progress when copying many messages (slightly change log format) - -* Output how long an account sync took (min:sec). - Bug Fixes --------- - -* Syncing multiple accounts in single-threaded mode would fail as we try - to "register" a thread as belonging to two accounts which was - fatal. Make it non-fatal (it can be legitimate). - -* New folders on the remote would be skipped on the very sync run they - are created and only by synced in subsequent runs. Fixed. - -* a readonly parameter to select() was not always treated correctly, - which could result in some folders being opened read-only when we - really needed read-write. diff --git a/Changelog.rst b/Changelog.rst index b301c20..4c17c39 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,30 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.0 (2011-11-17) +=============================== + +Changes +------- + +* Indicate progress when copying many messages (slightly change log format) + +* Output how long an account sync took (min:sec). + +Bug Fixes +--------- + +* Syncing multiple accounts in single-threaded mode would fail as we try + to "register" a thread as belonging to two accounts which was + fatal. Make it non-fatal (it can be legitimate). + +* New folders on the remote would be skipped on the very sync run they + are created and only by synced in subsequent runs. Fixed. + +* a readonly parameter to select() was not always treated correctly, + which could result in some folders being opened read-only when we + really needed read-write. + OfflineIMAP v6.4.0 (2011-09-29) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 46ff1d7..38a517c 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.0" +__version__ = "6.4.1" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 98e82db505bf65db5fd33259f5a42d911647c716 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 17 Nov 2011 08:55:32 +0100 Subject: [PATCH 343/817] Release current master as 6.4.1 Signed-off-by: Sebastian Spaeth Conflicts: Changelog.draft.rst --- Changelog.draft.rst | 18 ------------------ Changelog.rst | 24 ++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 13b97e8..a9b7de4 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,10 +19,6 @@ New Features Changes ------- -* Indicate progress when copying many messages (slightly change log format) - -* Output how long an account sync took (min:sec). - * Reworked logging which was reported to e.g. not flush output to files often enough. User-visible changes: a) console output goes to stderr (for now). @@ -35,17 +31,3 @@ Changes Bug Fixes --------- - -* Syncing multiple accounts in single-threaded mode would fail as we try - to "register" a thread as belonging to two accounts which was - fatal. Make it non-fatal (it can be legitimate). - -* New folders on the remote would be skipped on the very sync run they - are created and only by synced in subsequent runs. Fixed. - -* Make NOOPs to keep a server connection open more resistant against dropped - connections. - -* a readonly parameter to select() was not always treated correctly, - which could result in some folders being opened read-only when we - really needed read-write. diff --git a/Changelog.rst b/Changelog.rst index b301c20..4c17c39 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,30 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.0 (2011-11-17) +=============================== + +Changes +------- + +* Indicate progress when copying many messages (slightly change log format) + +* Output how long an account sync took (min:sec). + +Bug Fixes +--------- + +* Syncing multiple accounts in single-threaded mode would fail as we try + to "register" a thread as belonging to two accounts which was + fatal. Make it non-fatal (it can be legitimate). + +* New folders on the remote would be skipped on the very sync run they + are created and only by synced in subsequent runs. Fixed. + +* a readonly parameter to select() was not always treated correctly, + which could result in some folders being opened read-only when we + really needed read-write. + OfflineIMAP v6.4.0 (2011-09-29) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index ea9aa0d..13a6f28 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.0" +__version__ = "6.4.1" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 51164c4974974d78ae3482b0517d4e2a2ce750a0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 17 Nov 2011 09:04:03 +0100 Subject: [PATCH 344/817] Fix version number typo in Changelog Signed-off-by: Sebastian Spaeth --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 4c17c39..7fe4d4b 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,7 +11,7 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. -OfflineIMAP v6.4.0 (2011-11-17) +OfflineIMAP v6.4.1 (2011-11-17) =============================== Changes From 464342e1dd416764598bd4e02b767310dac16371 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 17 Nov 2011 09:04:03 +0100 Subject: [PATCH 345/817] Fix version number typo in Changelog Signed-off-by: Sebastian Spaeth --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 4c17c39..7fe4d4b 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,7 +11,7 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. -OfflineIMAP v6.4.0 (2011-11-17) +OfflineIMAP v6.4.1 (2011-11-17) =============================== Changes From 20f2edfcecfbb14e27a1a9445751ccb0c451eb45 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 09:49:09 +0100 Subject: [PATCH 346/817] Sanity check to notify us when we call IMAPRepository.getsep() too early The folder delimiter is only initialized after a call to acquireconnection(), so we must never call this function too early. Include an assert() to make sure we get notified when we do. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/IMAP.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 8c29ee2..8635f7c 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -74,6 +74,13 @@ class IMAPRepository(BaseRepository): return num def getsep(self): + """Return the folder separator for the IMAP repository + + This requires that self.imapserver has been initialized with an + acquireconnection() or it will still be `None`""" + assert self.imapserver.delim != None, "'%s' " \ + "repository called getsep() before the folder separator was " \ + "queried from the server" % self return self.imapserver.delim def gethost(self): From bf4127c2d63dee6b87727a5eeeaed3f12c81ca02 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 10:12:54 +0100 Subject: [PATCH 347/817] Remove unused imapserver getdelim() imapserver.getdelim() was not used at all, so remove this function. The folder delimiter is available via the repository.getsep() call. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index b8446d3..0106782 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -47,7 +47,11 @@ class IMAPServer: """Initializes all variables from an IMAPRepository() instance Various functions, such as acquireconnection() return an IMAP4 - object on which we can operate.""" + object on which we can operate. + + Public instance variables are: self.: + delim The server's folder delimiter. Only valid after acquireconnection() + """ GSS_STATE_STEP = 0 GSS_STATE_WRAP = 1 def __init__(self, repos): @@ -98,11 +102,6 @@ class IMAPServer: self.passworderror = None return self.password - def getdelim(self): - """Returns this server's folder delimiter. Can only be called - after one or more calls to acquireconnection.""" - return self.delim - def getroot(self): """Returns this server's folder root. Can only be called after one or more calls to acquireconnection.""" From c93f8710a38cec1135b03b034bbc69e477782bd2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 10:27:12 +0100 Subject: [PATCH 348/817] Init folder list early enough We need the list of folders and the folder delimiter, but it was not always retrieved early enough. E.g. when doing IMAP<->IMAP sync and the local IMAP being readonly, we would bunk out with a mysterious error message become repository.getsel() would still return None. This commit fixes this error. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ offlineimap/accounts.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index a9b7de4..92c57a1 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -31,3 +31,8 @@ Changes Bug Fixes --------- + +* IMAP<->IMAP sync with a readonly local IMAP repository failed with a + rather mysterious "TypeError: expected a character buffer object" + error. Fix this my retrieving the list of folders early enough even + for readonly repositories. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index bf3f327..6c1d7f2 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -274,6 +274,13 @@ class SyncableAccount(Account): remoterepos = self.remoterepos localrepos = self.localrepos statusrepos = self.statusrepos + + # init repos with list of folders, so we have them (and the + # folder delimiter etc) + remoterepos.getfolders() + localrepos.getfolders() + statusrepos.getfolders() + # replicate the folderstructure between REMOTE to LOCAL if not localrepos.getconfboolean('readonly', False): self.ui.syncfolders(remoterepos, localrepos) From 9a654968fc6d73d3bc45b8191e5bae28ce51ac47 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 17:01:43 +0100 Subject: [PATCH 349/817] Don't append trailing slash to maildir foldernames When sep='/' in a Maildir, we were doing a os.path.join(dirname,'') on the top level maildir, which results in a "dirname/", so all our maildir folder names had slashes appended. Which is pretty much wrong, so this fixes it by only using os.path.join when we actually have something to append. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Maildir.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 6e93839..70a5ca3 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -160,7 +160,7 @@ class MaildirRepository(BaseRepository): # Not a directory -- not a folder. continue foldername = dirname - if extension != None: + if extension and dirname != '': foldername = os.path.join(extension, dirname) if (os.path.isdir(os.path.join(fullname, 'cur')) and os.path.isdir(os.path.join(fullname, 'new')) and @@ -185,7 +185,7 @@ class MaildirRepository(BaseRepository): self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ repr([x.getname() for x in retval])) return retval - + def getfolders(self): if self.folders == None: self.folders = self._getfolders_scandir(self.root) From 3bc68ecd6574d654d87acac912227b33853a4f99 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 17:01:43 +0100 Subject: [PATCH 350/817] Don't append trailing slash to maildir foldernames When sep='/' in a Maildir, we were doing a os.path.join(dirname,'') on the top level maildir, which results in a "dirname/", so all our maildir folder names had slashes appended. Which is pretty much wrong, so this fixes it by only using os.path.join when we actually have something to append. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Maildir.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index ae09486..0dd79a2 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -162,7 +162,7 @@ class MaildirRepository(BaseRepository): # Not a directory -- not a folder. continue foldername = dirname - if extension != None: + if extension and dirname != '': foldername = os.path.join(extension, dirname) if (os.path.isdir(os.path.join(fullname, 'cur')) and os.path.isdir(os.path.join(fullname, 'new')) and @@ -187,7 +187,7 @@ class MaildirRepository(BaseRepository): self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ repr([x.getname() for x in retval])) return retval - + def getfolders(self): if self.folders == None: self.folders = self._getfolders_scandir(self.root) From 416b1fe55145cc64de3432938a7ef87004768db8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 17:06:26 +0100 Subject: [PATCH 351/817] Add changelog entry for bug fixed in previous commit Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 92c57a1..3261465 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -36,3 +36,7 @@ Bug Fixes rather mysterious "TypeError: expected a character buffer object" error. Fix this my retrieving the list of folders early enough even for readonly repositories. + +* Fix regression from 6.4.0. When using local Maildirs with "/" as a + folder separator, all folder names would get a trailing slash + appended, which is plain wrong. From 50e78a8d41ea01c1f077438d9c2d6cf7367575b5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 17:14:52 +0100 Subject: [PATCH 352/817] Release v6.4.2 This is a bugfix release over 6.4.1. Upgrading is recommended. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 9 --------- Changelog.rst | 12 ++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 3261465..a9b7de4 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -31,12 +31,3 @@ Changes Bug Fixes --------- - -* IMAP<->IMAP sync with a readonly local IMAP repository failed with a - rather mysterious "TypeError: expected a character buffer object" - error. Fix this my retrieving the list of folders early enough even - for readonly repositories. - -* Fix regression from 6.4.0. When using local Maildirs with "/" as a - folder separator, all folder names would get a trailing slash - appended, which is plain wrong. diff --git a/Changelog.rst b/Changelog.rst index 7fe4d4b..f369aaa 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,18 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.2 (2011-12-01) +=============================== + +* IMAP<->IMAP sync with a readonly local IMAP repository failed with a + rather mysterious "TypeError: expected a character buffer object" + error. Fix this my retrieving the list of folders early enough even + for readonly repositories. + +* Fix regression from 6.4.0. When using local Maildirs with "/" as a + folder separator, all folder names would get a trailing slash + appended, which is plain wrong. + OfflineIMAP v6.4.1 (2011-11-17) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 13a6f28..5e7b1c9 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.1" +__version__ = "6.4.2" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 8c441158070194b53a416a574e4856709c732c60 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 10:27:12 +0100 Subject: [PATCH 353/817] Init folder list early enough We need the list of folders and the folder delimiter, but it was not always retrieved early enough. E.g. when doing IMAP<->IMAP sync and the local IMAP being readonly, we would bunk out with a mysterious error message become repository.getsel() would still return None. This commit fixes this error. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ offlineimap/accounts.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..d9aa0bf 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,8 @@ Changes Bug Fixes --------- + +* IMAP<->IMAP sync with a readonly local IMAP repository failed with a + rather mysterious "TypeError: expected a character buffer object" + error. Fix this my retrieving the list of folders early enough even + for readonly repositories. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 71f08fe..d476b3e 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -265,6 +265,13 @@ class SyncableAccount(Account): remoterepos = self.remoterepos localrepos = self.localrepos statusrepos = self.statusrepos + + # init repos with list of folders, so we have them (and the + # folder delimiter etc) + remoterepos.getfolders() + localrepos.getfolders() + statusrepos.getfolders() + # replicate the folderstructure between REMOTE to LOCAL if not localrepos.getconfboolean('readonly', False): self.ui.syncfolders(remoterepos, localrepos) From 5f3ac2c4a2f5b5267cc105cc22b903e96d56bfc2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 17:14:52 +0100 Subject: [PATCH 354/817] Release v6.4.2 This is a bugfix release over 6.4.1. Upgrading is recommended. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 12 ++++++++++++ offlineimap/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 7fe4d4b..f369aaa 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,18 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.2 (2011-12-01) +=============================== + +* IMAP<->IMAP sync with a readonly local IMAP repository failed with a + rather mysterious "TypeError: expected a character buffer object" + error. Fix this my retrieving the list of folders early enough even + for readonly repositories. + +* Fix regression from 6.4.0. When using local Maildirs with "/" as a + folder separator, all folder names would get a trailing slash + appended, which is plain wrong. + OfflineIMAP v6.4.1 (2011-11-17) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 38a517c..b7b3f53 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.1" +__version__ = "6.4.2" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 8ec6980c96313380609f2cc8af7c9daf0cc110e4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 23:57:54 +0100 Subject: [PATCH 355/817] Don't fail on empty LocalStatus cache files As reported in https://github.com/spaetz/offlineimap/pull/2, we would fail when files are empty because file.read() would throw attribute errors. Fix this by removing the superfluous read() check and additionally log some warning message. Reported-by: Ralf Schmitt Signed-off-by: Sebastian Spaeth --- offlineimap/folder/LocalStatus.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index fbe2e24..fd4228c 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -65,9 +65,11 @@ class LocalStatusFolder(BaseFolder): file = open(self.filename, "rt") self.messagelist = {} line = file.readline().strip() - if not line and not line.read(): + if not line: # The status file is empty - should not have happened, # but somehow did. + errstr = "Cache file '%s' is empty. Closing..." % self.filename + self.ui.warn(errstr) file.close() return assert(line == magicline) From 0ccf06d5e631db2a5e139d63335134d983591487 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 Jan 2012 19:24:19 +0100 Subject: [PATCH 356/817] Implement clean CTRL-C termination Previously, we would simply bail out in an ugly way, potentially leaving temporary files around etc, or while writing status files. Hand SIGINT and SIGTERM as an event to the Account class, and make that bail out cleanly at predefined points. Stopping on ctrl-c can take a few seconds (it will e.g. finish to transfer the ongoing message), but it will shut down cleanly. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ offlineimap/accounts.py | 23 ++++++++++++++++++----- offlineimap/folder/Base.py | 7 +++++++ offlineimap/init.py | 16 ++++++++-------- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index a9b7de4..8b9b8bd 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -29,5 +29,9 @@ Changes * Bumped bundled imaplib2 to release 2.29 +* Make ctrl-c exit cleanly rather aborting brutally (which could leave + around temporary files, half-written cache files, etc). Exiting on + SIGTERM and CTRL-C can take a little longer, but will be clean. + Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 6c1d7f2..857fbfb 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -50,7 +50,9 @@ class Account(CustomConfig.ConfigHelperMixin): :class:`accounts.SyncableAccount` which contains all functions used for syncing an account.""" #signal gets set when we should stop looping - abort_signal = Event() + abort_soon_signal = Event() + #signal gets set on CTRL-C/SIGTERM + abort_NOW_signal = Event() def __init__(self, config, name): """ @@ -97,7 +99,8 @@ class Account(CustomConfig.ConfigHelperMixin): set_abort_event() to send the corresponding signal. Signum = 1 implies that we want all accounts to abort or skip the current or next sleep phase. Signum = 2 will end the autorefresh loop, - ie all accounts will return after they finished a sync. + ie all accounts will return after they finished a sync. signum=3 + means, abort NOW, e.g. on SIGINT or SIGTERM. This is a class method, it will send the signal to all accounts. """ @@ -107,7 +110,10 @@ class Account(CustomConfig.ConfigHelperMixin): config.set('Account ' + acctsection, "skipsleep", '1') elif signum == 2: # don't autorefresh anymore - cls.abort_signal.set() + cls.abort_soon_signal.set() + elif signum == 3: + # abort ASAP + cls.abort_NOW_signal.set() def get_abort_event(self): """Checks if an abort signal had been sent @@ -122,7 +128,8 @@ class Account(CustomConfig.ConfigHelperMixin): skipsleep = self.getconfboolean("skipsleep", 0) if skipsleep: self.config.set(self.getsection(), "skipsleep", '0') - return skipsleep or Account.abort_signal.is_set() + return skipsleep or Account.abort_soon_signal.is_set() or \ + Account.abort_NOW_signal.is_set() def sleeper(self): """Sleep if the account is set to autorefresh @@ -152,7 +159,8 @@ class Account(CustomConfig.ConfigHelperMixin): item.stopkeepalive() if sleepresult: - if Account.abort_signal.is_set(): + if Account.abort_soon_signal.is_set() or \ + Account.abort_NOW_signal.is_set(): return 2 self.quicknum = 0 return 1 @@ -288,6 +296,8 @@ class SyncableAccount(Account): # iterate through all folders on the remote repo and sync for remotefolder in remoterepos.getfolders(): + # check for CTRL-C or SIGTERM + if Account.abort_NOW_signal.is_set(): break if not remotefolder.sync_this: self.ui.debug('', "Not syncing filtered remote folder '%s'" "[%s]" % (remotefolder, remoterepos)) @@ -320,6 +330,9 @@ class SyncableAccount(Account): self.callhook(hook) def callhook(self, cmd): + # check for CTRL-C or SIGTERM and run postsynchook + if Account.abort_NOW_signal.is_set(): + return if not cmd: return try: diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 53bc71c..f61e89d 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -18,6 +18,7 @@ from offlineimap import threadutil from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError +import offlineimap.accounts import os.path import re from sys import exc_info @@ -332,6 +333,9 @@ class BaseFolder(object): self.getmessageuidlist()) num_to_copy = len(copylist) for num, uid in enumerate(copylist): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # exceptions are caught in copymessageto() if self.suggeststhreads(): @@ -447,6 +451,9 @@ class BaseFolder(object): ('syncing flags' , self.syncmessagesto_flags)] for (passdesc, action) in passes: + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break try: action(dstfolder, statusfolder) except (KeyboardInterrupt): diff --git a/offlineimap/init.py b/offlineimap/init.py index 36240ac..4287b5e 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -288,11 +288,6 @@ class OfflineImap: self.config is supposed to have been correctly initialized already.""" - def sigterm_handler(signum, frame): - # die immediately - self.ui.terminate(errormsg="terminating...") - signal.signal(signal.SIGTERM, sigterm_handler) - try: pidfd = open(self.config.getmetadatadir() + "/pid", "w") pidfd.write(str(os.getpid()) + "\n") @@ -328,11 +323,19 @@ class OfflineImap: accounts.Account.set_abort_event(self.config, 1) elif sig == signal.SIGUSR2: # tell each account to stop looping + getglobalui().warn("Terminating after this sync...") accounts.Account.set_abort_event(self.config, 2) + elif sig == signal.SIGTERM or sig == signal.SIGINT: + # tell each account to ABORT ASAP (ctrl-c) + getglobalui().warn("Terminating NOW (this may "\ + "take a few seconds)...") + accounts.Account.set_abort_event(self.config, 3) signal.signal(signal.SIGHUP,sig_handler) signal.signal(signal.SIGUSR1,sig_handler) signal.signal(signal.SIGUSR2,sig_handler) + signal.signal(signal.SIGTERM, sig_handler) + signal.signal(signal.SIGINT, sig_handler) #various initializations that need to be performed: offlineimap.mbnames.init(self.config, syncaccounts) @@ -361,9 +364,6 @@ class OfflineImap: t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) self.ui.terminate() - except KeyboardInterrupt: - self.ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') - return except (SystemExit): raise except Exception, e: From 9a7c700248c7f8deaddb798d438787f5af19cd05 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 Jan 2012 19:34:13 +0100 Subject: [PATCH 357/817] Release 6.4.3 Merge in the new logging mechanism, and provide an --info feature that will help with debugging. There is still some unstableness, so there will be another release soon, but this should be no worse than 6.4.2 at least... Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 17 ----------------- Changelog.rst | 27 +++++++++++++++++++++++++++ offlineimap/__init__.py | 4 ++-- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 8b9b8bd..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,25 +13,8 @@ others. New Features ------------ -* add a --info command line switch that outputs useful information about - the server and the configuration for all enabled accounts. - Changes ------- -* Reworked logging which was reported to e.g. not flush output to files - often enough. User-visible changes: - a) console output goes to stderr (for now). - b) file output has timestamps and looks identical in the basic and - ttyui UIs. - c) File output should be flushed after logging by default (do - report if not). - -* Bumped bundled imaplib2 to release 2.29 - -* Make ctrl-c exit cleanly rather aborting brutally (which could leave - around temporary files, half-written cache files, etc). Exiting on - SIGTERM and CTRL-C can take a little longer, but will be clean. - Bug Fixes --------- diff --git a/Changelog.rst b/Changelog.rst index f369aaa..b713c40 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,33 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.3 (2012-01-04) +=============================== + +New Features +------------ + +* add a --info command line switch that outputs useful information about + the server and the configuration for all enabled accounts. + +Changes +------- + +* Reworked logging which was reported to e.g. not flush output to files + often enough. User-visible changes: + a) console output goes to stderr (for now). + b) file output has timestamps and looks identical in the basic and + ttyui UIs. + c) File output should be flushed after logging by default (do + report if not). + +* Bumped bundled imaplib2 to release 2.29 + +* Make ctrl-c exit cleanly rather aborting brutally (which could leave + around temporary files, half-written cache files, etc). Exiting on + SIGTERM and CTRL-C can take a little longer, but will be clean. + + OfflineIMAP v6.4.2 (2011-12-01) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 5e7b1c9..ea857e6 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,8 +1,8 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.2" -__copyright__ = "Copyright 2002-2011 John Goerzen & contributors" +__version__ = "6.4.3" +__copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" From 6f361c4d9a25f092fc2dc81b68bdba03c8b4ca67 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 Jan 2012 13:21:08 +0100 Subject: [PATCH 358/817] Don't output "Finished in x seconds" in quiet ui The quiet UI should only output errors, and the final "Finished account X in 2 seconds" clearly is none, so the message debug level needed to be reduced to INFO to suppress it in the quiet ui. Fixes https://github.com/spaetz/offlineimap/issues/6 Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 7dc23e5..5ef10cb 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -285,7 +285,7 @@ class UIBase(object): """Output that we finished syncing an account (in which time)""" sec = time.time() - self.acct_startimes[account] del self.acct_startimes[account] - self._msg("*** Finished account '%s' in %d:%02d" % + self.logger.info("*** Finished account '%s' in %d:%02d" % (account, sec // 60, sec % 60)) def syncfolders(self, src_repo, dst_repo): From 3e28073f98632f35fb46e96c3f5bda21fc36cb12 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 Jan 2012 14:05:51 +0100 Subject: [PATCH 359/817] Do not create folders on readonly repositories 1) Rename the unintuitive repository.syncfoldersto() to sync_folder_structure() 2) We were checking if the local repository is readonly and then turning off any folder creation. But as we can create folders on a remote repository too, we need to be more fine grained here. Just don't create a folder on the repository that is marked readonly=True. This still does not do away with the error message that one currently gets on missing local folders. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 2 +- offlineimap/repository/Base.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 857fbfb..09c6882 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -289,10 +289,10 @@ class SyncableAccount(Account): localrepos.getfolders() statusrepos.getfolders() + remoterepos.sync_folder_structure(localrepos, statusrepos) # replicate the folderstructure between REMOTE to LOCAL if not localrepos.getconfboolean('readonly', False): self.ui.syncfolders(remoterepos, localrepos) - remoterepos.syncfoldersto(localrepos, statusrepos) # iterate through all folders on the remote repo and sync for remotefolder in remoterepos.getfolders(): diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 8af1f03..084841f 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -132,8 +132,8 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): def getfolder(self, foldername): raise NotImplementedError - - def syncfoldersto(self, dst_repo, status_repo): + + def sync_folder_structure(self, dst_repo, status_repo): """Syncs the folders in this repository to those in dest. It does NOT sync the contents of those folders. nametrans rules @@ -158,6 +158,9 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): # Find new folders on src_repo. for src_name, src_folder in src_hash.iteritems(): + # Don't create on dst_repo, if it is readonly + if dst_repo.getconfboolean('readonly', False): + break if src_folder.sync_this and not src_name in dst_hash: try: dst_repo.makefolder(src_name) @@ -171,6 +174,10 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): status_repo.getsep())) # Find new folders on dst_repo. for dst_name, dst_folder in dst_hash.iteritems(): + if self.getconfboolean('readonly', False): + # Don't create missing folder on readonly repo. + break + if dst_folder.sync_this and not dst_name in src_hash: # nametrans sanity check! # Does nametrans back&forth lead to identical names? @@ -202,7 +209,6 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): src_repo, newdst_name), OfflineImapError.ERROR.REPO) # end sanity check, actually create the folder - try: src_repo.makefolder(newsrc_name) src_haschanged = True # Need to refresh list From a280a905ca115306344572dae56eb8509d5ba34f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 Jan 2012 14:18:29 +0100 Subject: [PATCH 360/817] Fix "getfolders() asked to get nonexisting folder" bug When a new remote folder was detected, we tried to create the folder locally on the Maildir and called repository.forgetfolders() to force a new scanning of the Maildir. However, that implementation used the inherited base function that did nothing. We simply needed to implement forgetfolders() to set self.folder=None, so we would force a new read in of the updated local folder structure. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ offlineimap/repository/Maildir.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..7842f51 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,7 @@ Changes Bug Fixes --------- + +* Fix the missing folder error that occured when a new remote folder was + detected (IMAP<->Maildir) + diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 70a5ca3..cdf054d 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -190,4 +190,8 @@ class MaildirRepository(BaseRepository): if self.folders == None: self.folders = self._getfolders_scandir(self.root) return self.folders - + + def forgetfolders(self): + """Forgets the cached list of folders, if any. Useful to run + after a sync run.""" + self.folders = None From 64f5e557bc54e04e6adbee0cede4a276a4a56b7b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 5 Jan 2012 14:39:09 +0100 Subject: [PATCH 361/817] Fix (harmless) regex flaw when determining maildir flags The regex for catching Maildir message flags was self.infosep + '.*2,([A-Z]+)' (infosep being ':'). The .* is bogus, as there is nothing between the : and the 2, per maildir name specification, so remove that unneeded piece. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index df5dd2e..47f76e8 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -70,7 +70,7 @@ class MaildirFolder(BaseFolder): self.infosep = '!' if self.wincompatible else ':' """infosep is the separator between maildir name and flag appendix""" - self.flagmatchre = re.compile(self.infosep + '.*2,([A-Z]+)') + self.flagmatchre = re.compile(self.infosep + '2,([A-Z]+)') #self.ui is set in BaseFolder.init() # Cache the full folder path, as we use getfullname() very often self._fullname = os.path.join(self.getroot(), self.getname()) From d5666ce91de13923f939edaf0f23ac227fb7fc06 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 13:05:08 +0100 Subject: [PATCH 362/817] Don't create invalid maildir names with lower case maildir flags If someone had a custom :2,a flag, adding a new flag would lead to the invalid maildir filename ...a:2,... due to regex deficiencies not coping with this. Fix this so we alway produce valid maildir names. Note that custom flags are still problematic: as the syncing to the remote IMAP server will fail, the next sync will assume that they have been removed from the remote IMAP side and they will be removed from the local Maildir then. We will need to think about how to handle this. At least, with this patch we won't lose standard flags and won't produce invalid maildir names. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 47f76e8..c9ea63d 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -64,13 +64,13 @@ class MaildirFolder(BaseFolder): self.root = root self.sep = sep self.messagelist = None - + # check if we should use a different infosep to support Win file systems self.wincompatible = self.config.getdefaultboolean( "Account "+self.accountname, "maildir-windows-compatible", False) self.infosep = '!' if self.wincompatible else ':' """infosep is the separator between maildir name and flag appendix""" - self.flagmatchre = re.compile(self.infosep + '2,([A-Z]+)') + self.flagmatchre = re.compile('(%s2,)(\w*)' % self.infosep) #self.ui is set in BaseFolder.init() # Cache the full folder path, as we use getfullname() very often self._fullname = os.path.join(self.getroot(), self.getname()) @@ -162,7 +162,7 @@ class MaildirFolder(BaseFolder): #identify flags in the path name flagmatch = self.flagmatchre.search(messagename) if flagmatch: - flags = set(flagmatch.group(1)) + flags = set(flagmatch.group(2)) else: flags = set() # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S @@ -185,7 +185,7 @@ class MaildirFolder(BaseFolder): def cachemessagelist(self): if self.messagelist is None: self.messagelist = self._scanfolder() - + def getmessagelist(self): return self.messagelist @@ -259,7 +259,7 @@ class MaildirFolder(BaseFolder): self.savemessageflags(uid, flags) self.ui.debug('maildir', 'savemessage: returning uid %d' % uid) return uid - + def getmessageflags(self, uid): return self.messagelist[uid]['flags'] @@ -274,15 +274,14 @@ class MaildirFolder(BaseFolder): else: dir_prefix = 'new' - infostr = self.infosep - infomatch = re.search('(' + self.infosep + '.*)$', newname) - if infomatch: # If the info string is present.. - infostr = infomatch.group(1) - newname = newname.split(self.infosep)[0] # Strip off the info string. - infostr = re.sub('2,[A-Z]*', '', infostr) - infostr += '2,' + ''.join(sorted(flags)) + # Strip off existing infostring (preserving small letter flags, that + # dovecot uses) + infomatch = self.flagmatchre.search(newname) + if infomatch: + newname = newname[:-len(infomatch.group())] #strip off + infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) newname += infostr - + newfilename = os.path.join(dir_prefix, newname) if (newfilename != oldfilename): try: @@ -292,7 +291,7 @@ class MaildirFolder(BaseFolder): raise OfflineImapError("Can't rename file '%s' to '%s': %s" % ( oldfilename, newfilename, e[1]), OfflineImapError.ERROR.FOLDER) - + self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename From 69838b81a5998bbdf473bde372f54b15220a7637 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 13:22:34 +0100 Subject: [PATCH 363/817] Release 6.4.4 This is a bugfix release that fixes regressions since 6.4.0. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ---- Changelog.rst | 17 +++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 7842f51..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,7 +18,3 @@ Changes Bug Fixes --------- - -* Fix the missing folder error that occured when a new remote folder was - detected (IMAP<->Maildir) - diff --git a/Changelog.rst b/Changelog.rst index b713c40..51bf175 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,23 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.4 (2012-01-06) +=============================== + +This is a bugfix release, fixing regressions occurring in or since 6.4.0. + +* Fix the missing folder error that occured when a new remote folder was + detected (IMAP<->Maildir) + +* Possibly fixed bug that prevented us from ever re-reading Maildir + folders, so flag changes and deletions were not detected when running + in a refresh loop. This is a regression that was introduced in about + 6.4.0. + +* Never mangle maildir file names when using nonstandard Maildir flags + (such as 'a'), note that they will still be deleted as they are not + supported in the sync to an IMAP server. + OfflineIMAP v6.4.3 (2012-01-04) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index ea857e6..8406401 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.3" +__version__ = "6.4.4" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From d75a7a862f2954af8fbaa49ceef4e92e8087e1ba Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 14:02:22 +0100 Subject: [PATCH 364/817] Fix regression of MachineUI The recent UI overhaul led to the breakage of the machineui. This bug fixes that. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/Machine.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..5a113b2 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,5 @@ Changes Bug Fixes --------- + +* Fix regression that broke MachineUI diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 868433a..0a51bcc 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -43,7 +43,7 @@ class MachineUI(UIBase): 'warn', '', currentThread().getName(), msg)) def registerthread(self, account): - super(MachineUI, self).registerthread(self, account) + super(MachineUI, self).registerthread(account) self._printData('registerthread', account) def unregisterthread(s, thread): From de4f8c8605f55388f2717e0989252dff01c3c352 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 14:06:03 +0100 Subject: [PATCH 365/817] Remove unneeded import in MachineUI The urllib module is not needed anymore. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Machine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 0a51bcc..e6d1877 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -14,7 +14,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import urllib import sys import time import logging From 9d79bd7b01b8a66a48de4f4b12148e81833b4022 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 14:02:22 +0100 Subject: [PATCH 366/817] Fix regression of MachineUI The recent UI overhaul led to the breakage of the machineui. This bug fixes that. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/Machine.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..5a113b2 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,5 @@ Changes Bug Fixes --------- + +* Fix regression that broke MachineUI diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 868433a..0a51bcc 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -43,7 +43,7 @@ class MachineUI(UIBase): 'warn', '', currentThread().getName(), msg)) def registerthread(self, account): - super(MachineUI, self).registerthread(self, account) + super(MachineUI, self).registerthread(account) self._printData('registerthread', account) def unregisterthread(s, thread): From 6fe808338c1826a66eb507830be023dc1db9978a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 11:01:49 +0200 Subject: [PATCH 367/817] Refactor parsing out maildirs filename components Create a helper function that retrieves the UID, folder MD5, and Flags from a message filename. We need these items when we simply want to rename (=new UID) a Maildir message file later. The new function can give us these components. Rework, so we cache the calculation of the folder's md5 value once, it never changes and we call it a lot. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 116 +++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index c9ea63d..dc2e368 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -1,6 +1,5 @@ # Maildir folder support -# Copyright (C) 2002 - 2007 John Goerzen -# +# Copyright (C) 2002 - 2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,8 +34,10 @@ except NameError: from offlineimap import OfflineImapError -uidmatchre = re.compile(',U=(\d+)') -timestampmatchre = re.compile('(\d+)'); +# Find the UID in a message filename +re_uidmatch = re.compile(',U=(\d+)') +# Find a numeric timestamp in a string (filename prefix) +re_timestampmatch = re.compile('(\d+)'); timeseq = 0 lasttime = long(0) @@ -67,11 +68,14 @@ class MaildirFolder(BaseFolder): # check if we should use a different infosep to support Win file systems self.wincompatible = self.config.getdefaultboolean( "Account "+self.accountname, "maildir-windows-compatible", False) - self.infosep = '!' if self.wincompatible else ':' """infosep is the separator between maildir name and flag appendix""" - self.flagmatchre = re.compile('(%s2,)(\w*)' % self.infosep) + self.re_flagmatch = re.compile('%s2,(\w*)' % self.infosep) #self.ui is set in BaseFolder.init() + # Everything up to the first comma or colon (or ! if Windows): + self.re_prefixmatch = re.compile('([^'+ self.infosep + ',]*)') + #folder's md, so we can match with recorded file md5 for validity + self._foldermd5 = md5(self.getvisiblename()).hexdigest() # Cache the full folder path, as we use getfullname() very often self._fullname = os.path.join(self.getroot(), self.getname()) @@ -97,7 +101,7 @@ class MaildirFolder(BaseFolder): + oldest_time_struct[5]) oldest_time_utc -= oldest_time_today_seconds - timestampmatch = timestampmatchre.search(messagename) + timestampmatch = re_timestampmatch.search(messagename) timestampstr = timestampmatch.group() timestamplong = long(timestampstr) if(timestamplong < oldest_time_utc): @@ -105,68 +109,80 @@ class MaildirFolder(BaseFolder): else: return True + def _parse_filename(self, filename): + """Returns a messages file name components + + Receives the file name (without path) of a msg. Usual format is + '<%d_%d.%d.%s>,U=<%d>,FMD5=<%s>:2,' (pointy brackets + denoting the various components). + + If FMD5 does not correspond with the current folder MD5, we will + return None for the UID & FMD5 (as it is not valid in this + folder). If UID or FMD5 can not be detected, we return `None` + for the respective element. If flags are empty or cannot be + detected, we return an empty flags list. + + :returns: (prefix, UID, FMD5, flags). UID is a numeric "long" + type. flags is a set() of Maildir flags""" + prefix, uid, fmd5, flags = None, None, None, set() + prefixmatch = self.re_prefixmatch.match(filename) + if prefixmatch: + prefix = prefixmatch.group(1) + folderstr = ',FMD5=%s' % self._foldermd5 + foldermatch = folderstr in filename + # If there was no folder MD5 specified, or if it mismatches, + # assume it is a foreign (new) message and ret: uid, fmd5 = None, None + if foldermatch: + uidmatch = re_uidmatch.search(filename) + if uidmatch: + uid = long(uidmatch.group(1)) + flagmatch = self.re_flagmatch.search(filename) + if flagmatch: + flags = set(flagmatch.group(1)) + return prefix, uid, fmd5, flags def _scanfolder(self): - """Cache the message list. Maildir flags are: - R (replied) - S (seen) - T (trashed) - D (draft) - F (flagged) - and must occur in ASCII order.""" + """Cache the message list from a Maildir. + + Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F + (flagged). + :returns: dict that can be used as self.messagelist""" + maxage = self.config.getdefaultint("Account " + self.accountname, + "maxage", None) + maxsize = self.config.getdefaultint("Account " + self.accountname, + "maxsize", None) retval = {} files = [] - nouidcounter = -1 # Messages without UIDs get - # negative UID numbers. - foldermd5 = md5(self.getvisiblename()).hexdigest() - folderstr = ',FMD5=' + foldermd5 + nouidcounter = -1 # Messages without UIDs get negative UIDs. for dirannex in ['new', 'cur']: fulldirname = os.path.join(self.getfullname(), dirannex) - files.extend(os.path.join(dirannex, filename) for + files.extend((dirannex, filename) for filename in os.listdir(fulldirname)) - for file in files: - messagename = os.path.basename(file) - #check if there is a parameter for maxage / maxsize - then see if this - #message should be considered or not - maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1) - maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1) + for dirannex, filename in files: + # We store just dirannex and filename, ie 'cur/123...' + filepath = os.path.join(dirannex, filename) + # check maxage/maxsize if this message should be considered + if maxage and not self._iswithinmaxage(filename, maxage): + continue + if maxsize and (os.path.getsize(os.path.join( + self.getfullname(), filepath)) > maxsize): + continue - if(maxage != -1): - isnewenough = self._iswithinmaxage(messagename, maxage) - if(isnewenough != True): - #this message is older than we should consider.... - continue - - #Check and see if the message is too big if the maxsize for this account is set - if(maxsize != -1): - size = os.path.getsize(os.path.join(self.getfullname(), file)) - if(size > maxsize): - continue - - foldermatch = messagename.find(folderstr) != -1 - if not foldermatch: - # If there is no folder MD5 specified, or if it mismatches, - # assume it is a foreign (new) message and generate a - # negative uid for it + (prefix, uid, fmd5, flags) = self._parse_filename(filename) + if uid is None: # assign negative uid to upload it. uid = nouidcounter nouidcounter -= 1 else: # It comes from our folder. - uidmatch = uidmatchre.search(messagename) + uidmatch = re_uidmatch.search(filename) uid = None if not uidmatch: uid = nouidcounter nouidcounter -= 1 else: uid = long(uidmatch.group(1)) - #identify flags in the path name - flagmatch = self.flagmatchre.search(messagename) - if flagmatch: - flags = set(flagmatch.group(2)) - else: - flags = set() # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S - retval[uid] = {'flags': flags, 'filename': file} + retval[uid] = {'flags': flags, 'filename': filepath} return retval def quickchanged(self, statusfolder): From 09ce56c5942db78193f1e44343759afb2d2d8396 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 10:51:33 +0200 Subject: [PATCH 368/817] Factor out creating a Maildir message filename Various functions (such as change_message_uid) will want to construct maildir filenames, so factor out the code into a helper. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index dc2e368..5b2de91 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -221,6 +221,17 @@ class MaildirFolder(BaseFolder): filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) + def new_message_filename(self, uid, flags=set()): + """Creates a new unique Maildir filename + + :param uid: The UID`None`, or a set of maildir flags + :param flags: A set of maildir flags + :returns: String containing unique message filename""" + timeval, timeseq = gettimeseq() + return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s' % \ + (timeval, timeseq, os.getpid(), socket.gethostname(), + uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) + def savemessage(self, uid, content, flags, rtime): # This function only ever saves to tmp/, # but it calls savemessageflags() to actually save to cur/ or new/. @@ -237,14 +248,7 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - timeval, timeseq = gettimeseq() - messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \ - (timeval, - timeseq, - os.getpid(), - socket.gethostname(), - uid, - md5(self.getvisiblename()).hexdigest()) + messagename = self.new_message_filename(uid, flags) # open file and write it out try: fd = os.open(os.path.join(tmpdir, messagename), From de537dc09c208afb1cbad4b024b2cec10f5e5123 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 10:52:11 +0200 Subject: [PATCH 369/817] Implement change_message_uid Previously, assigning a new UID to a mapped IMAP or Maildir repository was done by loading the "local" item, saving it under a new UID and deleting the old one. This involved lots of disk activity for nothing more than an effective file rename in Maildirs, and lots of network usage in the MappedUID cases. We do this on every upload from a local to a remote item, so that can potentially be quite expensive. This patch lets backends that support it (Maildir, MappedUID) efficiently rename the file rather than having to read the mail content, write it out as a new file and delete the old file. This speeds up uploads from Maildir and the MappedUID server. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 +++ offlineimap/folder/Base.py | 39 ++++++++++++++------------ offlineimap/folder/IMAP.py | 10 ++++++- offlineimap/folder/Maildir.py | 52 +++++++++++++++++++++++------------ offlineimap/folder/UIDMaps.py | 25 +++++++++++++++++ 5 files changed, 93 insertions(+), 37 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5a113b2..5fd29e4 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,10 @@ New Features Changes ------- +* Uploading of Messages from Maildir and IMAP<->IMAP has been made more + efficient by renaming files/mapping entries, rather than actually + loading and saving the message under a new UID. + Bug Fixes --------- diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index f61e89d..0e065e3 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -234,6 +234,15 @@ class BaseFolder(object): for uid in uidlist: self.deletemessageflags(uid, flags) + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + + If the backend supports it (IMAP does not). + :param new_uid: (optional) If given, the old UID will be changed + to a new UID. This allows backends efficient renaming of + messages if the UID has changed.""" + raise NotImplementedException + def deletemessage(self, uid): raise NotImplementedException @@ -275,20 +284,15 @@ class BaseFolder(object): #remained negative, no server was willing to assign us an #UID. If newid is 0, saving succeeded, but we could not #retrieve the new UID. Ignore message in this case. - newuid = dstfolder.savemessage(uid, message, flags, rtime) - - if newuid > 0: - if newuid != uid: + new_uid = dstfolder.savemessage(uid, message, flags, rtime) + if new_uid > 0: + if new_uid != uid: + # Got new UID, change the local uid to match the new one. + self.change_message_uid(uid, new_uid) + statusfolder.deletemessage(uid) # Got new UID, change the local uid. - #TODO: Maildir could do this with a rename rather than - #load/save/del operation, IMPLEMENT a changeuid() - #function or so. - self.savemessage(newuid, message, flags, rtime) - self.deletemessage(uid) - uid = newuid # Save uploaded status in the statusfolder - statusfolder.savemessage(uid, message, flags, rtime) - + statusfolder.savemessage(new_uid, message, flags, rtime) elif newuid == 0: # Message was stored to dstfolder, but we can't find it's UID # This means we can't link current message to the one created @@ -299,11 +303,11 @@ class BaseFolder(object): self.deletemessage(uid) else: raise OfflineImapError("Trying to save msg (uid %d) on folder " - "%s returned invalid uid %d" % \ - (uid, - dstfolder.getvisiblename(), - newuid), + "%s returned invalid uid %d" % (uid, + dstfolder.getvisiblename(), new_uid), OfflineImapError.ERROR.MESSAGE) + except (KeyboardInterrupt): # bubble up CTRL-C + raise except OfflineImapError, e: if e.severity > OfflineImapError.ERROR.MESSAGE: raise # buble severe errors up @@ -311,10 +315,9 @@ class BaseFolder(object): except Exception, e: self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ (uid, self.accountname, - traceback.format_exc())) + exc_info()[2])) raise #raise on unknown errors, so we can fix those - def syncmessagesto_copy(self, dstfolder, statusfolder): """Pass1: Copy locally existing messages not on the other side diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 4471387..5d8d1c0 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -597,8 +597,8 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage: returning new UID %d' % uid) return uid - def savemessageflags(self, uid, flags): + """Change a message's flags to `flags`.""" imapobj = self.imapserver.acquireconnection() try: try: @@ -684,6 +684,14 @@ class IMAPFolder(BaseFolder): elif operation == '-': self.messagelist[uid]['flags'] -= flags + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + + If the backend supports it. IMAP does not and will throw errors.""" + raise OfflineImapError('IMAP backend cannot change a messages UID from ' + '%d to %d' % (uid, new_uid), + OfflineImapError.ERROR.MESSAGE) + def deletemessage(self, uid): self.deletemessages_noconvert([uid]) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 5b2de91..22a5ef4 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -273,7 +273,7 @@ class MaildirFolder(BaseFolder): if rtime != None: os.utime(os.path.join(tmpdir, messagename), (rtime, rtime)) - self.messagelist[uid] = {'flags': set(), + self.messagelist[uid] = {'flags': flags, 'filename': os.path.join('tmp', messagename)} # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) @@ -284,25 +284,25 @@ class MaildirFolder(BaseFolder): return self.messagelist[uid]['flags'] def savemessageflags(self, uid, flags): + """Sets the specified message's flags to the given set. + + This function moves the message to the cur or new subdir, + depending on the Seen flag.""" + # TODO: This function could be improved to only parse the + # filenames if the flags actually changed. oldfilename = self.messagelist[uid]['filename'] - dir_prefix, newname = os.path.split(oldfilename) - tmpdir = os.path.join(self.getfullname(), 'tmp') - if 'S' in flags: - # If a message has been seen, it goes into the cur - # directory. CR debian#152482 - dir_prefix = 'cur' - else: - dir_prefix = 'new' - - # Strip off existing infostring (preserving small letter flags, that + dir_prefix, filename = os.path.split(oldfilename) + # If a message has been seen, it goes into 'cur' + dir_prefix = 'cur' if 'S' in flags else 'new' + # Strip off existing infostring (preserving small letter flags that # dovecot uses) - infomatch = self.flagmatchre.search(newname) + infomatch = self.flagmatchre.search(filename) if infomatch: - newname = newname[:-len(infomatch.group())] #strip off + filename = filename[:-len(infomatch.group())] #strip off infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) - newname += infostr + filename += infostr + newfilename = os.path.join(dir_prefix, filename) - newfilename = os.path.join(dir_prefix, newname) if (newfilename != oldfilename): try: os.rename(os.path.join(self.getfullname(), oldfilename), @@ -315,10 +315,26 @@ class MaildirFolder(BaseFolder): self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename - # By now, the message had better not be in tmp/ land! - final_dir, final_name = os.path.split(self.messagelist[uid]['filename']) - assert final_dir != 'tmp' + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + This will not update the statusfolder UID, you need to do that yourself. + :param new_uid: (optional) If given, the old UID will be changed + to a new UID. The Maildir backend can implement this as an efficient + rename.""" + if not uid in self.messagelist: + raise OfflineImapError("Cannot change unknown Maildir UID %s" % uid) + if uid == new_uid: return + + oldfilename = self.messagelist[uid]['filename'] + dir_prefix, filename = os.path.split(oldfilename) + flags = self.getmessageflags(uid) + filename = self.new_message_filename(new_uid, flags) + os.rename(os.path.join(self.getfullname(), oldfilename), + os.path.join(self.getfullname(), dir_prefix, filename)) + self.messagelist[new_uid] = self.messagelist[uid] + del self.messagelist[uid] + def deletemessage(self, uid): """Unlinks a message file from the Maildir. diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 5da8e00..6b361f2 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -221,6 +221,31 @@ class MappedIMAPFolder(IMAPFolder): self._mb.addmessagesflags(self._uidlist(self.r2l, uidlist), flags) + def change_message_uid(self, ruid, new_ruid): + """Change the message from existing ruid to new_ruid + + :param new_uid: The old remote UID will be changed to a new + UID. The UIDMaps case handles this efficiently by simply + changing the mappings file.""" + if ruid not in self.r2l: + raise OfflineImapError("Cannot change unknown Maildir UID %s" % ruid, + OfflineImapError.ERROR.MESSAGE) + if ruid == new_ruid: return # sanity check shortcut + self.maplock.acquire() + try: + luid = self.r2l[ruid] + self.l2r[luid] = new_ruid + del self.r2l[ruid] + self.r2l[new_ruid] = luid + #TODO: diskl2r|r2l are a pain to sync and should be done away with + #diskl2r only contains positive UIDs, so wrap in ifs + if luid>0: self.diskl2r[luid] = new_ruid + if ruid>0: del self.diskr2l[ruid] + if new_ruid > 0: self.diskr2l[new_ruid] = luid + self._savemaps(dolock = 0) + finally: + self.maplock.release() + def _mapped_delete(self, uidlist): self.maplock.acquire() try: From ff50585007759b9dfe480ddbbf7a214010c3ce4f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 10:43:10 +0200 Subject: [PATCH 370/817] Folder-Maildir.savemessageflags(): Only parse file name if flags changed Rather than always parsing the filename, we only need to do so if the flags have actually changed, otherwise we can keep the filename. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 22a5ef4..d7639f4 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -287,22 +287,24 @@ class MaildirFolder(BaseFolder): """Sets the specified message's flags to the given set. This function moves the message to the cur or new subdir, - depending on the Seen flag.""" - # TODO: This function could be improved to only parse the - # filenames if the flags actually changed. + depending on the 'S'een flag.""" + oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) # If a message has been seen, it goes into 'cur' dir_prefix = 'cur' if 'S' in flags else 'new' - # Strip off existing infostring (preserving small letter flags that - # dovecot uses) - infomatch = self.flagmatchre.search(filename) - if infomatch: - filename = filename[:-len(infomatch.group())] #strip off - infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) - filename += infostr - newfilename = os.path.join(dir_prefix, filename) + if flags != self.messagelist[uid]['flags']: + # Flags have actually changed, construct new filename + # Strip off existing infostring (preserving small letter flags that + # dovecot uses) + infomatch = self.flagmatchre.search(filename) + if infomatch: + filename = filename[:-len(infomatch.group())] #strip off + infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) + filename += infostr + + newfilename = os.path.join(dir_prefix, filename) if (newfilename != oldfilename): try: os.rename(os.path.join(self.getfullname(), oldfilename), From 78a37f27ef26f754eec56ae91cdaa3fc114a145c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 17:20:11 +0200 Subject: [PATCH 371/817] WIP, revamp how we treat top-level dirs Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 11 ++++++----- offlineimap/folder/IMAP.py | 2 +- offlineimap/folder/LocalStatus.py | 2 +- offlineimap/folder/Maildir.py | 2 +- offlineimap/repository/IMAP.py | 2 ++ 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 0e065e3..313728d 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -37,13 +37,14 @@ class BaseFolder(object): self.sync_this = True """Should this folder be included in syncing?""" self.ui = getglobalui() - self.name = name + # Top level dir name is always '' + self.name = name if not name == self.getsep() else '' self.repository = repository self.visiblename = repository.nametrans(name) - # In case the visiblename becomes '.' (top-level) we use '' as - # that is the name that e.g. the Maildir scanning will return - # for the top-level dir. - if self.visiblename == '.': + # In case the visiblename becomes '.' or '/' (top-level) we use + # '' as that is the name that e.g. the Maildir scanning will + # return for the top-level dir. + if self.visiblename == self.getsep(): self.visiblename = '' self.config = repository.getconfig() diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 5d8d1c0..d804766 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -33,10 +33,10 @@ except NameError: class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, repository): name = imaputil.dequote(name) + self.sep = imapserver.delim super(IMAPFolder, self).__init__(name, repository) self.expunge = repository.getexpunge() self.root = None # imapserver.root - self.sep = imapserver.delim self.imapserver = imapserver self.messagelist = None self.randomgenerator = random.Random() diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index fd4228c..3d2cd37 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -27,8 +27,8 @@ magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" class LocalStatusFolder(BaseFolder): def __init__(self, name, repository): + self.sep = '.' #needs to be set before super.__init__() super(LocalStatusFolder, self).__init__(name, repository) - self.sep = '.' self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self.messagelist = {} self.savelock = threading.Lock() diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index d7639f4..973bc92 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -60,10 +60,10 @@ def gettimeseq(): class MaildirFolder(BaseFolder): def __init__(self, root, name, sep, repository): + self.sep = sep # needs to be set before super().__init__ super(MaildirFolder, self).__init__(name, repository) self.dofsync = self.config.getdefaultboolean("general", "fsync", True) self.root = root - self.sep = sep self.messagelist = None # check if we should use a different infosep to support Win file systems self.wincompatible = self.config.getdefaultboolean( diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 8635f7c..a5ed84c 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -324,6 +324,8 @@ class IMAPRepository(BaseRepository): :param foldername: Full path of the folder to be created.""" if self.getreference(): foldername = self.getreference() + self.getsep() + foldername + if not foldername: # Create top level folder as folder separator + foldername = self.getsep() imapobj = self.imapserver.acquireconnection() try: From 00c67881a02200afb8dac9cec8149bf5ec5077d4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 19:35:56 +0100 Subject: [PATCH 372/817] docs/MANUAL: How to upgrade to sqlite storage Add section to the manual describing how to upgrade to the new cache backend. Signed-off-by: Sebastian Spaeth --- docs/MANUAL.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 49dedd6..da09141 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -305,6 +305,25 @@ achieve this. to hit the disk before continueing, you can set this to True. If you set it to False, you lose some of that safety, trading it for speed. + +Upgrading from plain text cache to SQLITE based cache +===================================================== + +OfflineImap uses a cache to store the last know status of mails (flags etc). Historically that has meant plain text files, but recently we introduced sqlite-based cache, which helps with performance and CPU usage on large folders. Here is how to upgrade existing plain text cache installations to sqlite based one: + + 1) Sync to make sure things are reasonably similar + 3) Change the account section to status_backend = sqlite + 4) A new sync will convert your plain text cache to an sqlite cache (but + leave the old plain text cache around for easy reverting) + This should be quick and not involve any mail up/downloading. + 5) See if it works :-) + 6a) If it does not work, go back to the old version or set + status_backend=plain + 6b) Or once you are sure it works, you can delete the + .offlineimap/Account-foo/LocalStatus folder (the new cache will be in + the LocalStatus-sqlite folder) + + Security and SSL ================ From 5240d3f36722c416f81c55774c7e7714c266aaf6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 19:51:58 +0100 Subject: [PATCH 373/817] docs/INSTALL: How to uninstall Add a blurb on how to uninstall system wide installations. Signed-off-by: Sebastian Spaeth --- docs/INSTALL.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index 684ca6f..e2ed87a 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -110,3 +110,23 @@ within the sample file. `OfflineIMAP`_ also ships a file named `offlineimap.conf.minimal` that you can also try. It's useful if you want to get started with the most basic feature set, and you can read about other features later with `offlineimap.conf`. + + +=============== +Uninstall +=============== + +If you installed a system-wide installation via "python setup.py +install", there are a few files to purge to uninstall it again. I assume +that /usr/local is the standard prefix that your system and you use +python 2.7. Adapt to your system. In that case you need to: + +1) Delete: + /usr/local/lib/python2.7/dist-packages/offlineimap-6.4.4.egg-info + /usr/local/lib/python2.7/dist-packages/offlineimap + +2) Delete the cache at (default location) ~/.offlineimap + Delete your manually created (default loc) ~/.offlineimaprc + (It is possible that you created those in different spots) + +That's it. Have fun without OfflineImap. From 0ea6c6ed471b7de14dd42a37f4b80d99df60028e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 21:41:04 +0100 Subject: [PATCH 374/817] Forgot to change a variable name in all cases the newuid var was renamed new_uid but there was one leftover. Fixed. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 313728d..9ff60e2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -294,7 +294,7 @@ class BaseFolder(object): # Got new UID, change the local uid. # Save uploaded status in the statusfolder statusfolder.savemessage(new_uid, message, flags, rtime) - elif newuid == 0: + elif new_uid == 0: # Message was stored to dstfolder, but we can't find it's UID # This means we can't link current message to the one created # in IMAP. So we just delete local message and on next run From 8fc72271895b7e46ad770417e2c5b8211f91eccd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 21:57:48 +0100 Subject: [PATCH 375/817] SEVERE: Fix getting wrong UID back on IMAP upload This change looks harmless, but it fixes a severe bugfix, potentially leading to data loss! It fixes the "on n new uploads, it will redownload n-1, n-2, n-3,... messages during the next syncs" condition, and this is what happens: If there are more than one Mails to upload to a server, we do that by repeatedly invoking folder.IMAP.savemessage(). If the server supports the UIDPLUS extension we query the resulting UID by doing a: imapobj._get_untagged_response('APPENDUID', True) and that is exactly the problem. The "True" part causes the reply to remain in the "response stack" of the imaplib2 library. When we do the same call on a subsequent message and the connection is still on the same folder, we will get the same UID response back (imaplib2 only looks for the first matching response and returns that). The only time we clear the response stack, is when the IMAP connection SELECTS a different folder. This means that when we upload 10 messages, the IMAP server gives us always the same UID (that of the first one) back. And trying to write out 10 different messages with the same UID will confuse OfflineIMAP. This is the reason why we saw the ongoing UPLOADING/DOWNLOADING behavior that people reported. And this is the reason why we saw the inconsistency in the UID mapping in the IMAP<->IMAP case. I urge everyone to upgrade ASAP. Sorry for that, I don't know why the problem only became prevalent in the recent few releases as this code has been there for quite a while. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index d804766..68d1216 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -570,7 +570,7 @@ class IMAPFolder(BaseFolder): self.ui.warn("Server supports UIDPLUS but got no APPENDUID " "appending a message.") return 0 - uid = long(imapobj._get_untagged_response('APPENDUID', True)[-1].split(' ')[1]) + uid = long(imapobj._get_untagged_response('APPENDUID')[-1].split(' ')[1]) else: # we don't support UIDPLUS From d22390506882f68b523e482af4e781191533b3e6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 22:23:39 +0100 Subject: [PATCH 376/817] BUG FIX: Release v6.5.0 This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. Please upgrade to avoid potential data loss! The version has been bumped to 6.5.0, please let everyone know to stay away from 6.5.x! I am sorry for this. See details in the Changelog and even more gory details in commit message for commit 8fc72271895b7e46ad770417e2c5b8211f91eccd. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 6 ------ Changelog.rst | 16 ++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5fd29e4..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,11 +16,5 @@ New Features Changes ------- -* Uploading of Messages from Maildir and IMAP<->IMAP has been made more - efficient by renaming files/mapping entries, rather than actually - loading and saving the message under a new UID. - Bug Fixes --------- - -* Fix regression that broke MachineUI diff --git a/Changelog.rst b/Changelog.rst index 51bf175..21f9d42 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,22 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.5.0 (2012-01-06) +=============================== + +This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. Please upgrade to avoid potential data loss! The version has been bumped to 6.5.0, please let everyone know that the 6.4.x series is problematic. + +* Uploading multiple emails to an IMAP server would lead to wrong UIDs + being returned (ie the same for all), which confused offlineimap and + led to recurrent upload/download loops and inconsistencies in the + IMAP<->IMAP uid mapping. + +* Uploading of Messages from Maildir and IMAP<->IMAP has been made more + efficient by renaming files/mapping entries, rather than actually + loading and saving the message under a new UID. + +* Fix regression that broke MachineUI + OfflineIMAP v6.4.4 (2012-01-06) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 8406401..b16a9d6 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.4" +__version__ = "6.5.0" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 17f60f7233b32860e3c79b30adf8d6a36daa52c9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 23:13:55 +0100 Subject: [PATCH 377/817] Remove from __future__ import with_statements These were needed for python <2.6 compatability, but since we depend on python 2.6 now, these can go. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/UIDMaps.py | 4 +--- offlineimap/imapserver.py | 1 - offlineimap/ui/Curses.py | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 6b361f2..1a6283c 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -1,6 +1,5 @@ # Base folder support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2012 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,7 +14,6 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from __future__ import with_statement # needed for python 2.5 from threading import Lock from IMAP import IMAPFolder import os.path diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 0106782..a355eaf 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -15,7 +15,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from __future__ import with_statement # needed for python 2.5 from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui from threading import Lock, BoundedSemaphore, Thread, Event, currentThread diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index f1626c4..9a2b97f 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -15,7 +15,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from __future__ import with_statement # needed for python 2.5 from threading import RLock, currentThread, Lock, Event from thread import get_ident # python < 2.6 support from collections import deque From 946386a8e7813355b51967c13fa714e576ef2655 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 23:40:31 +0100 Subject: [PATCH 378/817] Restore previous MachineUI output a bit more The logging rework led to multipline output as we stopped urlencoding the output lines. Urrg. Fixed this, so output is urlencoded again. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Machine.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index e6d1877..ab57ea2 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - +from urllib import urlencode import sys import time import logging @@ -23,15 +23,24 @@ import offlineimap protocol = '7.0.0' +class MachineLogFormatter(logging.Formatter): + """urlencodes any outputted line, to avoid multi-line output""" + def format(self, record): + # urlencode the "mesg" attribute and append to regular line... + line = super(MachineLogFormatter, self).format(record) + return line + urlencode([('', record.mesg)])[1:] + class MachineUI(UIBase): def __init__(self, config, loglevel = logging.INFO): super(MachineUI, self).__init__(config, loglevel) self._log_con_handler.createLock() """lock needed to block on password input""" + # Set up the formatter that urlencodes the strings... + self._log_con_handler.setFormatter(MachineLogFormatter()) def _printData(self, command, msg): - self.logger.info("%s:%s:%s:%s" % ( - 'msg', command, currentThread().getName(), msg)) + self.logger.info("%s:%s:%s" % ( + 'msg', command, currentThread().getName()), extra={'mesg': msg}) def _msg(s, msg): s._printData('_display', msg) From 5509691dcf1dcf91b1e508edf54929e9a8255a9c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 23:47:26 +0100 Subject: [PATCH 379/817] Add Changelog entry about restored MachineUI Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..9608369 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,7 @@ Changes Bug Fixes --------- + +* Fixed MachineUI to urlencode() output lines again, rather than + outputting multi-line items. It's ugly as hell, but it had been that + way for years. From 1b85e3525678e6038614e0510c0be7f7bc179f88 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 23:50:54 +0100 Subject: [PATCH 380/817] Disable old global lock system Remove the old global locking system. We lock only the accounts that we currently sync, so you can invoke OfflineImap multiple times now as long as you sync different accounts. This system is compatible with all releases >= 6.4.0, so don't run older releases simultanous to this one. This mostly reverts commit 0d9565141765b8b23c1c723d325cf494e47cc80d, disabling the old global lock system that we had in parallel to the new one. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 6 ++++++ offlineimap/accounts.py | 1 - offlineimap/init.py | 16 ---------------- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 9608369..c19a285 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,12 @@ others. New Features ------------ +* Remove the old global locking system. We lock only the accounts that + we currently sync, so you can invoke OfflineImap multiple times now as + long as you sync different accounts. This system is compatible with + all releases >= 6.4.0, so don't run older releases simultanous to this + one. + Changes ------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 09c6882..4644604 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -191,7 +191,6 @@ class SyncableAccount(Account): def lock(self): """Lock the account, throwing an exception if it is locked already""" - # Take a new-style per-account lock self._lockfd = open(self._lockfilepath, 'w') try: fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB) diff --git a/offlineimap/init.py b/offlineimap/init.py index 4287b5e..4dc01aa 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -23,10 +23,6 @@ import signal import socket import logging from optparse import OptionParser -try: - import fcntl -except ImportError: - pass #it's OK import offlineimap from offlineimap import accounts, threadutil, syncmaster from offlineimap.error import OfflineImapError @@ -340,18 +336,6 @@ class OfflineImap: #various initializations that need to be performed: offlineimap.mbnames.init(self.config, syncaccounts) - #TODO: keep legacy lock for a few versions, then remove. - self._legacy_lock = open(self.config.getmetadatadir() + "/lock", - 'w') - try: - fcntl.lockf(self._legacy_lock, fcntl.LOCK_EX|fcntl.LOCK_NB) - except NameError: - #fcntl not available (Windows), disable file locking... :( - pass - except IOError: - raise OfflineImapError("Could not take global lock.", - OfflineImapError.ERROR.REPO) - if options.singlethreading: #singlethreaded self.sync_singlethreaded(syncaccounts) From ec63b4fe6b5570465eb4943059fe478b538cd2c1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 01:28:20 +0100 Subject: [PATCH 381/817] replace imaplib internal _get_untagged_response with public functions We were using the internal imaplib2 _get_untagged_response() functions a few times. Replace 3 of these calls with 2 calls to the public function response() rather than fudging with internals that could change anytime. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 68d1216..7fde40b 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -68,9 +68,10 @@ class IMAPFolder(BaseFolder): def getuidvalidity(self): imapobj = self.imapserver.acquireconnection() try: - # Primes untagged_responses + # SELECT receives UIDVALIDITY response self.selectro(imapobj) - return long(imapobj._get_untagged_response('UIDVALIDITY', True)[0]) + typ, uidval = imapobj.response('UIDVALIDITY') + return long(uidval[0]) finally: self.imapserver.releaseconnection(imapobj) @@ -563,14 +564,15 @@ class IMAPFolder(BaseFolder): # get the new UID. Test for APPENDUID response even if the # server claims to not support it, as e.g. Gmail does :-( if use_uidplus or imapobj._get_untagged_response('APPENDUID', True): - # get the new UID from the APPENDUID response, it could look like - # OK [APPENDUID 38505 3955] APPEND completed - # with 38505 bein folder UIDvalidity and 3955 the new UID - if not imapobj._get_untagged_response('APPENDUID', True): + # get new UID from the APPENDUID response, it could look + # like OK [APPENDUID 38505 3955] APPEND completed with + # 38505 bein folder UIDvalidity and 3955 the new UID + typ, resp = imapobj.response('APPENDUID') + if resp == [None]: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " - "appending a message.") + "appending a message. %s" % imapobj._get_untagged_response('APPENDUID', True)) return 0 - uid = long(imapobj._get_untagged_response('APPENDUID')[-1].split(' ')[1]) + uid = long(resp[-1].split(' ')[1]) else: # we don't support UIDPLUS From 3a1eab73830b22bc682af32d3ee2aedff3b546a0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 01:31:24 +0100 Subject: [PATCH 382/817] Remove a stray debug output that slipped in the previous commit. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 7fde40b..93f0c56 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -570,7 +570,7 @@ class IMAPFolder(BaseFolder): typ, resp = imapobj.response('APPENDUID') if resp == [None]: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " - "appending a message. %s" % imapobj._get_untagged_response('APPENDUID', True)) + "appending a message.") return 0 uid = long(resp[-1].split(' ')[1]) From 9d6a2dec37f05fd15369c2351a841709384e2900 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 03:04:26 +0100 Subject: [PATCH 383/817] Have log output go to STDOUT by default THe new logging framwork spit putput to STDERR by default (as that is pythons default), but we used to have STDERR, so make it go there again. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/ui/UIBase.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index c19a285..91f1f17 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -22,6 +22,9 @@ New Features Changes ------- +* Have console output go by default to STDOUT and not STDERR (regression + in 6.5.0) + Bug Fixes --------- diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 5ef10cb..296159d 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -69,7 +69,7 @@ class UIBase(object): Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" # create console handler with a higher log level - ch = logging.StreamHandler() + ch = logging.StreamHandler(sys.stdout) #ch.setLevel(logging.DEBUG) # create formatter and add it to the handlers self.formatter = logging.Formatter("%(message)s") From 0b85a34fc7f2189c224c7ad4ce533665409ed0d6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 13:10:41 +0100 Subject: [PATCH 384/817] Maildir: flagmatchre --> re_flagmatch Overlooked a variable renaming when merging in a branch. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/folder/Maildir.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 91f1f17..38a90b0 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -31,3 +31,5 @@ Bug Fixes * Fixed MachineUI to urlencode() output lines again, rather than outputting multi-line items. It's ugly as hell, but it had been that way for years. + +* Fixed Maildir regression "flagmatchre" not found. (regressed in 6.5.0) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 973bc92..938d47b 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -298,7 +298,7 @@ class MaildirFolder(BaseFolder): # Flags have actually changed, construct new filename # Strip off existing infostring (preserving small letter flags that # dovecot uses) - infomatch = self.flagmatchre.search(filename) + infomatch = self.re_flagmatch.search(filename) if infomatch: filename = filename[:-len(infomatch.group())] #strip off infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) From 45782ca3ac72119ac3af276cbfc763c72fada86f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 13:15:44 +0100 Subject: [PATCH 385/817] Release v 6.5.1 OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" ======================================================= * Fixed Maildir regression "flagmatchre" not found. (regressed in 6.5.0) * Have console output go by default to STDOUT and not STDERR (regression in 6.5.0) * Fixed MachineUI to urlencode() output lines again, rather than outputting multi-line items. It's ugly as hell, but it had been that way for years. * Remove the old global locking system. We lock only the accounts that we currently sync, so you can invoke OfflineImap multiple times now as long as you sync different accounts. This system is compatible with all releases >= 6.4.0, so don't run older releases simultanous to this one. --- Changelog.draft.rst | 14 -------------- Changelog.rst | 19 +++++++++++++++++++ offlineimap/__init__.py | 4 ++-- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 38a90b0..1fb8079 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,23 +13,9 @@ others. New Features ------------ -* Remove the old global locking system. We lock only the accounts that - we currently sync, so you can invoke OfflineImap multiple times now as - long as you sync different accounts. This system is compatible with - all releases >= 6.4.0, so don't run older releases simultanous to this - one. - Changes ------- -* Have console output go by default to STDOUT and not STDERR (regression - in 6.5.0) - Bug Fixes --------- -* Fixed MachineUI to urlencode() output lines again, rather than - outputting multi-line items. It's ugly as hell, but it had been that - way for years. - -* Fixed Maildir regression "flagmatchre" not found. (regressed in 6.5.0) diff --git a/Changelog.rst b/Changelog.rst index 21f9d42..5f04e2e 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,25 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. + +OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" +======================================================= + +* Fixed Maildir regression "flagmatchre" not found. (regressed in 6.5.0) + +* Have console output go by default to STDOUT and not STDERR (regression + in 6.5.0) + +* Fixed MachineUI to urlencode() output lines again, rather than + outputting multi-line items. It's ugly as hell, but it had been that + way for years. + +* Remove the old global locking system. We lock only the accounts that + we currently sync, so you can invoke OfflineImap multiple times now as + long as you sync different accounts. This system is compatible with + all releases >= 6.4.0, so don't run older releases simultanous to this + one. + OfflineIMAP v6.5.0 (2012-01-06) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index b16a9d6..93cdb33 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.0" +__version__ = "6.5.1" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" @@ -9,7 +9,7 @@ __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Suppo __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" __bigcopyright__ = """%(__productname__)s %(__version__)s %(__license__)s""" % locals() -__homepage__ = "http://github.com/nicolas33/offlineimap" +__homepage__ = "http://offlineimap.org" banner = __bigcopyright__ From 415c7d797910e7cac56439bf7ef020df3eba3eb6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 14:54:16 +0100 Subject: [PATCH 386/817] Fix abort_sleep missing crash in Blinkenlights UI This is a regression introduced when renaming signals due to the improved CTRL-C handling. Regression in 6.5.0 Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/Curses.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 1fb8079..e97953b 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,3 +19,5 @@ Changes Bug Fixes --------- +* Abort sleep in blinkenlights UI led to crash ('abort_signal' not existing), + regression from 6.5.0. diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 9a2b97f..98055ba 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -157,11 +157,13 @@ class CursesAccountFrame: return tf def sleeping(self, sleepsecs, remainingsecs): - # show how long we are going to sleep and sleep + """show how long we are going to sleep and sleep + + :returns: Boolean, whether we want to abort the sleep""" self.drawleadstr(remainingsecs) self.ui.exec_locked(self.window.refresh) time.sleep(sleepsecs) - return self.account.abort_signal.is_set() + return self.account.get_abort_event() def syncnow(self): """Request that we stop sleeping asap and continue to sync""" From 968dcd57801799110f7749d041a49a17f5523ed1 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 14:59:42 +0100 Subject: [PATCH 387/817] Make exit via 'q' key work again in Blinkenlights UI With the new abort signal handler, we can send a signal that lets us exit cleanly. Make use of this, rather than crashing out in ugly ways. This affects only the Blinkenlights UI when pressing 'q'. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 ++++- offlineimap/ui/Curses.py | 8 +++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index e97953b..5c963f6 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,5 +19,8 @@ Changes Bug Fixes --------- -* Abort sleep in blinkenlights UI led to crash ('abort_signal' not existing), +* [Blinkenlights UI] Abort sleep led to crash ('abort_signal' not existing), + regression from 6.5.0. + +* [Blinkenlights UI] Make exit via 'q' key work again cleanly regression from 6.5.0. diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 98055ba..af41d35 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -487,11 +487,9 @@ class Blinkenlights(UIBase, CursesUtil): if key < 1 or key > 255: return if chr(key) == 'q': - # Request to quit. - #TODO: this causes us to bail out in main loop when the thread exits - #TODO: review and rework this mechanism. - currentThread().set_exit_exception(SystemExit("User requested shutdown")) - self.terminate() + # Request to quit completely. + self.warn("Requested shutdown via 'q'") + offlineimap.accounts.Account.set_abort_event(self.config, 3) try: index = int(chr(key)) except ValueError: From 9cba9b8f3a6db846257bd5cdbc7f111b13827a83 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 15:14:30 +0100 Subject: [PATCH 388/817] OfflineIMAP v6.5.1.1 Blinkenlights UI 6.5.0 regression fixes only. * Sleep led to crash ('abort_signal' not existing) * Make exit via 'q' key work again cleanly Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 6 ------ Changelog.rst | 8 ++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5c963f6..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,9 +18,3 @@ Changes Bug Fixes --------- - -* [Blinkenlights UI] Abort sleep led to crash ('abort_signal' not existing), - regression from 6.5.0. - -* [Blinkenlights UI] Make exit via 'q' key work again cleanly - regression from 6.5.0. diff --git a/Changelog.rst b/Changelog.rst index 5f04e2e..d52c928 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,14 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.5.1.1 (2012-01-07) - "Das machine control is nicht fur gerfinger-poken und mittengrabben" +================================================================================================================== + +Blinkenlights UI 6.5.0 regression fixes only. + +* Sleep led to crash ('abort_signal' not existing) + +* Make exit via 'q' key work again cleanly OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" ======================================================= diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 93cdb33..e4ba31b 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.1" +__version__ = "6.5.1.1" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From b85f36e52024a9fdba5436125773f7cf912a97ed Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 7 Jan 2012 15:17:52 +0100 Subject: [PATCH 389/817] doc: add link to the web page of autogenerated API Signed-off-by: Nicolas Sebrecht --- docs/HACKING.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/HACKING.rst b/docs/HACKING.rst index e21afd7..728f957 100644 --- a/docs/HACKING.rst +++ b/docs/HACKING.rst @@ -26,6 +26,9 @@ In order to involve into OfflineIMAP you need some knowledges about Git and our workflow. Don't be afraid if you don't know much, we would be pleased to help you. +You can find the API docs autogenerated on http://offlineimap.readthedocs.org, +(although it still does not work fully + Release cycles ============== From f88f8cf984a6b63bf938ac42438e64520897bff6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 18:03:21 +0100 Subject: [PATCH 390/817] offlineimap.conf wording tweaks Clarify and improve wording in minor ways. Signed-off-by: Sebastian Spaeth --- docs/HACKING.rst | 3 +-- offlineimap.conf | 56 ++++++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/docs/HACKING.rst b/docs/HACKING.rst index 728f957..71ec112 100644 --- a/docs/HACKING.rst +++ b/docs/HACKING.rst @@ -26,8 +26,7 @@ In order to involve into OfflineIMAP you need some knowledges about Git and our workflow. Don't be afraid if you don't know much, we would be pleased to help you. -You can find the API docs autogenerated on http://offlineimap.readthedocs.org, -(although it still does not work fully +You can find the API docs autogenerated on http://docs.offlineimap.org. Release cycles ============== diff --git a/offlineimap.conf b/offlineimap.conf index b6ff6e6..71ea2f7 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -238,37 +238,37 @@ remoterepository = RemoteExample # maxsize = 2000000 -# When you are starting to sync an already existing account yuo can tell offlineimap -# to sync messages from only the last x days. When you do this messages older than x -# days will be completely ignored. This can be useful for importing existing accounts -# when you do not want to download large amounts of archive email. - +# When you are starting to sync an already existing account you can tell +# offlineimap to sync messages from only the last x days. When you do +# this messages older than x days will be completely ignored. This can +# be useful for importing existing accounts when you do not want to +# download large amounts of archive email. +# # Messages older than maxage days will not be synced, their flags will -# not be changed, they will not be deleted etc. For offlineimap it will be like these -# messages do not exist. This will perform an IMAP search in the case of IMAP or Gmail -# and therefor requires that the server support server side searching. This will -# calculate the earliest day that would be included in the search and include all -# messages from that day until today. e.g. maxage = 3 to sync only the last 3 days mail - -# maxage = 3 +# not be changed, they will not be deleted etc. For offlineimap it will +# be like these messages do not exist. This will perform an IMAP search +# in the case of IMAP or Gmail and therefor requires that the server +# support server side searching. This will calculate the earliest day +# that would be included in the search and include all messages from +# that day until today. e.g. maxage = 3 to sync only the last 3 days +# mail +# +# maxage = -# Maildir format uses colon (:) separator between uniq name and info. +# Maildir file format uses colon (:) separator between uniq name and info. # Unfortunatelly colon is not allowed character in windows file name. If you # enable maildir-windows-compatible option, offlineimap will be able to store # messages on windows drive, but you will probably loose compatibility with # other programs working with the maildir - -# maildir-windows-compatible = no +# +#maildir-windows-compatible = no [Repository LocalExample] -# This is one of the two repositories that you'll work with given the -# above example. Each repository requires a "type" declaration. -# -# The types supported are Maildir and IMAP. -# +# Each repository requires a "type" declaration. The types supported for +# local repositories are Maildir and IMAP. type = Maildir @@ -278,12 +278,12 @@ type = Maildir localfolders = ~/Test -# You can specify the "path separator character" used for your Maildir -# folders. This is inserted in-between the components of the tree. -# It defaults to ".". If you want your Maildir folders to be nested, -# set it to "/". - -sep = . +# You can specify the "folder separator character" used for your Maildir +# folders. It is inserted in-between the components of the tree. If you +# want your folders to be nested directories, set it to "/". 'sep' is +# ignored for IMAP repositories, as it is queried automatically. +# +#sep = . # Some users may not want the atime (last access time) of folders to be # modified by OfflineIMAP. If 'restoreatime' is set to yes, OfflineIMAP @@ -291,11 +291,11 @@ sep = . # folder to their original value after each sync. # # In nearly all cases, the default should be fine. +# +#restoreatime = no -restoreatime = no [Repository RemoteExample] - # And this is the remote repository. We only support IMAP or Gmail here. type = IMAP From 679977e5d3f0290fb710da85b5444013e1f6fac0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 20:28:58 +0100 Subject: [PATCH 391/817] Need to import OfflineImapError for --info function An import was missing. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/UIBase.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..4a8eee8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,5 @@ Changes Bug Fixes --------- + +* Fix possible crash during --info run diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 296159d..ac3c160 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -24,6 +24,7 @@ import traceback import threading from Queue import Queue from collections import deque +from offlineimap.error import OfflineImapError import offlineimap debugtypes = {'':'Other offlineimap related sync messages', @@ -360,7 +361,7 @@ class UIBase(object): repository.getssl())) try: conn = repository.imapserver.acquireconnection() - except OfflineImapError, e: + except OfflineImapError as e: self._msg("Failed to connect. Reason %s" % e) else: if 'ID' in conn.capabilities: From e267dcd24d656830196372a0c5291929a7b765e7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 20:28:58 +0100 Subject: [PATCH 392/817] Need to import OfflineImapError for --info function An import was missing. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/UIBase.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..4a8eee8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,5 @@ Changes Bug Fixes --------- + +* Fix possible crash during --info run diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 296159d..ac3c160 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -24,6 +24,7 @@ import traceback import threading from Queue import Queue from collections import deque +from offlineimap.error import OfflineImapError import offlineimap debugtypes = {'':'Other offlineimap related sync messages', @@ -360,7 +361,7 @@ class UIBase(object): repository.getssl())) try: conn = repository.imapserver.acquireconnection() - except OfflineImapError, e: + except OfflineImapError as e: self._msg("Failed to connect. Reason %s" % e) else: if 'ID' in conn.capabilities: From 241e8f838bf84be9b93d3075829ec4f93c8a4177 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 21:17:36 +0100 Subject: [PATCH 393/817] Add filter information to --info output Prettify the folder listing in --info and add the "filtered" information to it. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/UIBase.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 4a8eee8..ddefc5e 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,8 @@ New Features Changes ------- +* Add folderfilter information to the --info output + Bug Fixes --------- diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index ac3c160..dc9e935 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -386,12 +386,16 @@ class UIBase(object): self._msg("nametrans= %s\n" % nametrans) folders = repository.getfolders() - foldernames = [(f.name, f.getvisiblename()) for f in folders] + foldernames = [(f.name, f.getvisiblename(), f.sync_this) \ + for f in folders] folders = [] - for name, visiblename in foldernames: - if name == visiblename: folders.append(name) - else: folders.append("%s -> %s" % (name, visiblename)) - self._msg("Folderlist: %s\n" % str(folders)) + for name, visiblename, sync_this in foldernames: + syncstr = "" if sync_this else " (disabled)" + if name == visiblename: folders.append("%s%s" % (name, + syncstr)) + else: folders.append("%s -> %s%s" % (name, + visiblename, syncstr)) + self._msg("Folderlist:\n %s\n" % "\n ".join(folders)) finally: if conn: #release any existing IMAP connection repository.imapserver.close() From d549cbdac9f93fdd921b32856b0c9687774139e4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 21:34:57 +0100 Subject: [PATCH 394/817] Fix code comment on flag stripping Fix code comment that we preserve custom dovecot flags. We discard them. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 938d47b..714636d 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -295,9 +295,9 @@ class MaildirFolder(BaseFolder): dir_prefix = 'cur' if 'S' in flags else 'new' if flags != self.messagelist[uid]['flags']: - # Flags have actually changed, construct new filename - # Strip off existing infostring (preserving small letter flags that - # dovecot uses) + # Flags have actually changed, construct new filename Strip + # off existing infostring (possibly discarding small letter + # flags that dovecot uses TODO) infomatch = self.re_flagmatch.search(filename) if infomatch: filename = filename[:-len(infomatch.group())] #strip off From 6e9e2569460b8913a59ba53f1bcc43fc7cd086ed Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 22:00:45 +0100 Subject: [PATCH 395/817] Ignore lower-case custom Maildir flags Do not read in custom maildir flags, or we would try to sync them over the wire. The next step will be to merge flag writes with existing custom flags, so we don't lose information. The long term goal will be to attempt to sync flags to the other side, of course. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 714636d..95258d9 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -138,7 +138,9 @@ class MaildirFolder(BaseFolder): uid = long(uidmatch.group(1)) flagmatch = self.re_flagmatch.search(filename) if flagmatch: - flags = set(flagmatch.group(1)) + # Filter out all lowercase (custom maildir) flags. We don't + # handle them yet. + flags = set((c for c in flagmatch.group(1) if not c.islower())) return prefix, uid, fmd5, flags def _scanfolder(self): From 87ebb3779a3bc203c78294f7886a01eaf9a9d910 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 22:15:31 +0100 Subject: [PATCH 396/817] Restore Changelog draft It had accidentally been reversed by the master->next merge Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index ddefc5e..73c0da0 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,7 +16,9 @@ New Features Changes ------- -* Add folderfilter information to the --info output +* Do not attempt to sync lower case custom Maildir flags. We do not + support them (yet) +* Add filter information to the filter list in --info output Bug Fixes --------- From 061712fe1cd37d423b1eaa5b824f735fdaa70d68 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 13 Dec 2011 12:04:19 -0500 Subject: [PATCH 397/817] Folder could not be created MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Tue, Dec 13, 2011 at 12:00:57PM -0500, W. Trevor King wrote: > I've attached a patch that does fix the problem… Oops, *now* I've attached the patch and logs ;). -- This email may be signed or encrypted with GnuPG (http://www.gnupg.org). For more information, see http://en.wikipedia.org/wiki/Pretty_Good_Privacy From 3067b1b4dfb00d165bd9480ea49f446adb12991d Mon Sep 17 00:00:00 2001 From: W. Trevor King Date: Tue, 13 Dec 2011 11:26:00 -0500 Subject: [PATCH] Only scan children in _getfolders_scandir if extension is set. When sep is '/', MaildirRepository._getfolders_scandir recursively checks sub-directories for additional maildirs. The old loop logic always checked the top directory and its children. This lead to children being found twice, once from their parent, with dirname matching their directory name, and once from themselves, with a dirname of ''. This patch fixes the problem by only checking the top directory when extension is not set (i.e. for the root directory). --- offlineimap/repository/Maildir.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index cdf054d..bb384c2 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -150,6 +150,9 @@ class MaildirRepository(BaseRepository): # Iterate over directories in top & top itself. for dirname in os.listdir(toppath) + ['']: self.debug(" dirname = %s" % dirname) + if dirname == '' and extension is not None: + self.debug(' skip this entry (extension set)') + continue if dirname in ['cur', 'new', 'tmp']: self.debug(" skipping this dir (Maildir special)") # Bypass special files. From 137e11c88e0af8bb4c5fd9392f8a5ebf4b254f39 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 23:26:02 +0100 Subject: [PATCH 398/817] Improve previous comment & Changelog Add a changelog to W. Trevor King's previous commit. Also make wording a bit more consistent and and remove a now unneeded comparison (dirname is always set when extension is set). Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/repository/Maildir.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 73c0da0..1b9f8f6 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -24,3 +24,5 @@ Bug Fixes --------- * Fix possible crash during --info run +* Fix reading in Maildirs, where we would attempt to create empty + directories on REMOTE. diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index bb384c2..66a3ebd 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -151,20 +151,23 @@ class MaildirRepository(BaseRepository): for dirname in os.listdir(toppath) + ['']: self.debug(" dirname = %s" % dirname) if dirname == '' and extension is not None: - self.debug(' skip this entry (extension set)') + self.debug(' skip this entry (already scanned)') continue if dirname in ['cur', 'new', 'tmp']: - self.debug(" skipping this dir (Maildir special)") + self.debug(" skip this entry (Maildir special)") # Bypass special files. continue fullname = os.path.join(toppath, dirname) if not os.path.isdir(fullname): - self.debug(" skipping this entry (not a directory)") + self.debug(" skip this entry (not a directory)") # Not a directory -- not a folder. continue - foldername = dirname - if extension and dirname != '': + if extension: + # extension can be None which fails. foldername = os.path.join(extension, dirname) + else: + foldername = dirname + if (os.path.isdir(os.path.join(fullname, 'cur')) and os.path.isdir(os.path.join(fullname, 'new')) and os.path.isdir(os.path.join(fullname, 'tmp'))): From 15b4a7a5b9d8c4a751c9f50c6e01d0d37ba59dfe Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 23:34:00 +0100 Subject: [PATCH 399/817] OfflineIMAP v6.5.1.2 See changelog for details Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 8 -------- Changelog.rst | 12 ++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 1b9f8f6..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,13 +16,5 @@ New Features Changes ------- -* Do not attempt to sync lower case custom Maildir flags. We do not - support them (yet) -* Add filter information to the filter list in --info output - Bug Fixes --------- - -* Fix possible crash during --info run -* Fix reading in Maildirs, where we would attempt to create empty - directories on REMOTE. diff --git a/Changelog.rst b/Changelog.rst index d52c928..59550b9 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,18 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.5.1.2 (2012-01-07) - "Baby steps" +================================================ + +Smallish bug fixes that deserve to be put out. + +* Fix possible crash during --info run +* Fix reading in Maildirs, where we would attempt to create empty + directories on REMOTE. +* Do not attempt to sync lower case custom Maildir flags. We do not + support them (yet) (this prevents many scary bogus sync messages) +* Add filter information to the filter list in --info output + OfflineIMAP v6.5.1.1 (2012-01-07) - "Das machine control is nicht fur gerfinger-poken und mittengrabben" ================================================================================================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index e4ba31b..2bf4637 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.1.1" +__version__ = "6.5.1.2" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 0a275b9532a8de3be445dfc3b78e5d3bdb61199b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 01:06:56 +0100 Subject: [PATCH 400/817] Add scary warnings about "realdelete" option WARNING: I consider the Gmail "realdelete" option as harmful with the potential for DATALOSS. Add scary warnings to offlineimap.conf. See the analysis at http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 Deleting a message from a Gmail folder via the IMAP interface will just remove that folder's label from the message: the message will continue to exist in the '[Gmail]/All Mail' folder. If `realdelete` is set to `True`, then deleted messages will be moved to the '[Gmail]/Trash' folder. BEWARE: this will immediately delete a messages from *all folders* it belongs to! AS OFFLINEIMAP IMPLEMENTS FOLDER MOVES AS 1) AN ADD and 2) A DELETE (the order can vary), THIS MEANS THAT A FOLDER MOVE CAN CAUSE DATALOSS. DO NOT USE IT AND MOVE MAIL TO "[Gmail]/Trash" TO DELETE MAIL FROM "[Gmail]/All Mail"! We will need to discuss whether to completely disable that feature. Signed-off-by: Sebastian Spaeth --- offlineimap.conf | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 71ea2f7..c7f7d9d 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -546,16 +546,20 @@ type = Gmail # Specify the Gmail user name. This is the only mandatory parameter. remoteuser = username@gmail.com -# Deleting a message from a Gmail folder via the IMAP interface will -# just remove that folder's label from the message: the message will -# continue to exist in the '[Gmail]/All Mail' folder. If `realdelete` -# is set to `True`, then deleted messages will really be deleted -# during `offlineimap` sync, by moving them to the '[Gmail]/Trash' -# folder. BEWARE: this will delete a messages from *all folders* it -# belongs to! -# -# See http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 -realdelete = no +# WARNING: READ THIS BEFORE CONSIDERING TO CHANGE IT! Deleting a +# message from a Gmail folder via the IMAP interface will just remove +# that folder's label from the message: the message will continue to +# exist in the '[Gmail]/All Mail' folder. If `realdelete` is set to +# `True`, then deleted messages will be moved to the '[Gmail]/Trash' +# folder. BEWARE: this will immediately delete a messages from *all +# folders* it belongs to! AS OFFLINEIMAP IMPLEMENTS FOLDER MOVES AS 1) +# AN ADD and 2) A DELETE (the order can vary), THIS MEANS THAT A FOLDER +# MOVE CAN CAUSE DATALOSS. DO NOT USE IT AND MOVE MAIL TO +# "[Gmail]/Trash" TO DELETE MAIL FROM "[Gmail]/All Mail"! See the +# analysis at +# http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 See +# http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 +# realdelete = no !!!READ ABOVE BEFORE USING # The trash folder name may be different from [Gmail]/Trash # for example on german googlemail, this setting should be From b5a13157e499c0a8749d110f6c7ad00b5e25cf85 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 01:06:56 +0100 Subject: [PATCH 401/817] Add scary warnings about "realdelete" option WARNING: I consider the Gmail "realdelete" option as harmful with the potential for DATALOSS. Add scary warnings to offlineimap.conf. See the analysis at http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 Deleting a message from a Gmail folder via the IMAP interface will just remove that folder's label from the message: the message will continue to exist in the '[Gmail]/All Mail' folder. If `realdelete` is set to `True`, then deleted messages will be moved to the '[Gmail]/Trash' folder. BEWARE: this will immediately delete a messages from *all folders* it belongs to! AS OFFLINEIMAP IMPLEMENTS FOLDER MOVES AS 1) AN ADD and 2) A DELETE (the order can vary), THIS MEANS THAT A FOLDER MOVE CAN CAUSE DATALOSS. DO NOT USE IT AND MOVE MAIL TO "[Gmail]/Trash" TO DELETE MAIL FROM "[Gmail]/All Mail"! We will need to discuss whether to completely disable that feature. Signed-off-by: Sebastian Spaeth --- offlineimap.conf | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 71ea2f7..c7f7d9d 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -546,16 +546,20 @@ type = Gmail # Specify the Gmail user name. This is the only mandatory parameter. remoteuser = username@gmail.com -# Deleting a message from a Gmail folder via the IMAP interface will -# just remove that folder's label from the message: the message will -# continue to exist in the '[Gmail]/All Mail' folder. If `realdelete` -# is set to `True`, then deleted messages will really be deleted -# during `offlineimap` sync, by moving them to the '[Gmail]/Trash' -# folder. BEWARE: this will delete a messages from *all folders* it -# belongs to! -# -# See http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 -realdelete = no +# WARNING: READ THIS BEFORE CONSIDERING TO CHANGE IT! Deleting a +# message from a Gmail folder via the IMAP interface will just remove +# that folder's label from the message: the message will continue to +# exist in the '[Gmail]/All Mail' folder. If `realdelete` is set to +# `True`, then deleted messages will be moved to the '[Gmail]/Trash' +# folder. BEWARE: this will immediately delete a messages from *all +# folders* it belongs to! AS OFFLINEIMAP IMPLEMENTS FOLDER MOVES AS 1) +# AN ADD and 2) A DELETE (the order can vary), THIS MEANS THAT A FOLDER +# MOVE CAN CAUSE DATALOSS. DO NOT USE IT AND MOVE MAIL TO +# "[Gmail]/Trash" TO DELETE MAIL FROM "[Gmail]/All Mail"! See the +# analysis at +# http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 See +# http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 +# realdelete = no !!!READ ABOVE BEFORE USING # The trash folder name may be different from [Gmail]/Trash # for example on german googlemail, this setting should be From ed718054764ab75c99abcfe37e291dafa4fd8118 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 01:13:08 +0100 Subject: [PATCH 402/817] Changelog entry about "realdelete" option Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..a9a73c7 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -10,6 +10,9 @@ others. `WIP (coming releases)` ======================= +* Gmail "realdelete" is considered harmful and has the potential for data loss. Analysis at http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 +Warnings were added to offlineimap.conf + New Features ------------ From 50de2174bfbf35dfcc1b0f9253e46218fb29b986 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 11:29:54 +0100 Subject: [PATCH 403/817] Allow to pass 'force' arg to selectro() to enforce a new select Pass through the 'force' argument from selectro() to select() so that it can also enforce a new SELECT even if we already are on that folder. Also change the default parameter from '0' to 'False' to make clear that this is a Bool. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 8 ++++---- offlineimap/imaplibutil.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 93f0c56..d23fb5f 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -42,19 +42,19 @@ class IMAPFolder(BaseFolder): self.randomgenerator = random.Random() #self.ui is set in BaseFolder - def selectro(self, imapobj): + def selectro(self, imapobj, force = False): """Select this folder when we do not need write access. Prefer SELECT to EXAMINE if we can, since some servers (Courier) do not stabilize UID validity until the folder is selected. .. todo: Still valid? Needs verification - + :param: Enforce new SELECT even if we are on that folder already. :returns: raises :exc:`OfflineImapError` severity FOLDER on error""" try: - imapobj.select(self.getfullname()) + imapobj.select(self.getfullname(), force = force) except imapobj.readonly: - imapobj.select(self.getfullname(), readonly = True) + imapobj.select(self.getfullname(), readonly = True, force = force) def suggeststhreads(self): return 1 diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index c9a7715..b4345fa 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -40,7 +40,7 @@ class UsefulIMAPMixIn(object): return self.mailbox return None - def select(self, mailbox='INBOX', readonly=False, force = 0): + def select(self, mailbox='INBOX', readonly=False, force = False): """Selects a mailbox on the IMAP server :returns: 'OK' on success, nothing if the folder was already From 7184ec28cced142c655c06d360250c7fb330809c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 12:26:47 +0100 Subject: [PATCH 404/817] Sanity check return value of UIDVALIDTY response We have a reported case where response('UIDVALIDITY') returned [None] which results in an ugly non-intuitive crash. Sanity check and report something nicer. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index d23fb5f..f04f871 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -71,6 +71,7 @@ class IMAPFolder(BaseFolder): # SELECT receives UIDVALIDITY response self.selectro(imapobj) typ, uidval = imapobj.response('UIDVALIDITY') + assert uidval != [None], "response('UIDVALIDITY') returned [None]!" return long(uidval[0]) finally: self.imapserver.releaseconnection(imapobj) From 81f194adca78da848f7c139a526945358750ac32 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 12:48:21 +0100 Subject: [PATCH 405/817] mbnames should write out local and not nametransformed box names Rather than to write out the nametrans'lated folder names for mbnames, we now write out the local untransformed box names. This is generally what we want. This became relevant since we support nametrans rules on the local side since only a short time. Reported by Paul Collignan. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ offlineimap/accounts.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index a9a73c7..41d3af8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,5 +19,10 @@ New Features Changes ------- +* Rather than to write out the nametrans'lated folder names for mbnames, + we now write out the local untransformed box names. This is generally + what we want. This became relevant since we support nametrans rules on + the local side since only a short time. Reported by Paul Collignan. + Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 4644604..fc2687f 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -372,7 +372,7 @@ def syncfolder(account, remotefolder, quick): % localfolder) return # Write the mailboxes - mbnames.add(account.name, localfolder.getvisiblename()) + mbnames.add(account.name, localfolder.getname()) # Load status folder. statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ From 0b103e86271b10bc2ae1602173d77295d54862da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Sun, 8 Jan 2012 19:57:03 +0100 Subject: [PATCH 406/817] Don't fail if /etc/netrc is not readable --- offlineimap/repository/IMAP.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index a5ed84c..c2633dc 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -138,7 +138,7 @@ class IMAPRepository(BaseRepository): try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) except IOError, inst: - if inst.errno != errno.ENOENT: + if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: @@ -235,7 +235,7 @@ class IMAPRepository(BaseRepository): try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) except IOError, inst: - if inst.errno != errno.ENOENT: + if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: From 3284e010ff12a8c6a80e8ea299af26e80edcd4c0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 9 Jan 2012 09:51:43 +0100 Subject: [PATCH 407/817] Revert "use .response() rather _get_untagged_response()" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recently the internal function use of imaplib2's _get_untagged_response() was switched to use the public documented .response() function (which should return the same data). However within a few fays we received reports that both uses of a) the UIDVALIDITY fetching and b) the APPENDUID fetching returned [None] as data although the IMAP log definitely shows that data was returned. Revert to using the undocumented internal imaplib2 function, that seemed to have worked without problems. This needs to be taken up to the imaplib2 developer. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index f04f871..75c731c 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -70,9 +70,12 @@ class IMAPFolder(BaseFolder): try: # SELECT receives UIDVALIDITY response self.selectro(imapobj) - typ, uidval = imapobj.response('UIDVALIDITY') + # note: we would want to use .response() here but that + # often seems to return [None], even though we have + # data. TODO + uidval = imapobj._get_untagged_response('UIDVALIDITY') assert uidval != [None], "response('UIDVALIDITY') returned [None]!" - return long(uidval[0]) + return long(uidval[-1]) finally: self.imapserver.releaseconnection(imapobj) @@ -567,8 +570,11 @@ class IMAPFolder(BaseFolder): if use_uidplus or imapobj._get_untagged_response('APPENDUID', True): # get new UID from the APPENDUID response, it could look # like OK [APPENDUID 38505 3955] APPEND completed with - # 38505 bein folder UIDvalidity and 3955 the new UID - typ, resp = imapobj.response('APPENDUID') + # 38505 bein folder UIDvalidity and 3955 the new UID. + # note: we would want to use .response() here but that + # often seems to return [None], even though we have + # data. TODO + resp = imapobj._get_untagged_response('APPENDUID') if resp == [None]: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " "appending a message.") From d72bb88729b95d1a6b8e377c286680fbd24b04c8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 9 Jan 2012 09:57:36 +0100 Subject: [PATCH 408/817] Improve error message Add *what* UID was returned in case savemessage did not return a UID>0 Signed-off-by: Sebastian Spaeth --- offlineimap/folder/UIDMaps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 1a6283c..ac0a9b4 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -191,7 +191,8 @@ class MappedIMAPFolder(IMAPFolder): newluid = self._mb.savemessage(-1, content, flags, rtime) if newluid < 1: - raise ValueError("Backend could not find uid for message") + raise ValueError("Backend could not find uid for message, returned " + "%s" % newluid) self.maplock.acquire() try: self.diskl2r[newluid] = uid From 9e65cfca2161c023d9772924647533841cf0cc89 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 9 Jan 2012 10:04:05 +0100 Subject: [PATCH 409/817] Release v6.5.2-rc1 OfflineIMAP v6.5.2-rc1 (2012-01-09) =================================== Commits v6.5.1.1 - v6.5.2-rc1: note: Proper Changelog still in Changelog-draft.rst d72bb88 Improve error message 3284e01 Revert "use .response() rather _get_untagged_response()" 81f194a mbnames should write out local and not nametransformed box names 7184ec2 Sanity check return value of UIDVALIDTY response 50de217 Allow to pass 'force' arg to selectro() to enforce a new select ed71805 Changelog entry about "realdelete" option 0a275b9 Add scary warnings about "realdelete" option Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ Changelog.rst | 12 ++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 41d3af8..bf0b460 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -24,5 +24,10 @@ Changes what we want. This became relevant since we support nametrans rules on the local side since only a short time. Reported by Paul Collignan. +* Some sanity checks and improved error messages. + +* Revert 6.5.1.1 change to use public imaplib2 function, it was reported to + not always work. + Bug Fixes --------- diff --git a/Changelog.rst b/Changelog.rst index 59550b9..f7a7dd3 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,18 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.5.2-rc1 (2012-01-09) +=================================== +Commits v6.5.1.1 - v6.5.2-rc1: +note: Proper Changelog still in Changelog-draft.rst +d72bb88 Improve error message +3284e01 Revert "use .response() rather _get_untagged_response()" +81f194a mbnames should write out local and not nametransformed box names +7184ec2 Sanity check return value of UIDVALIDTY response +50de217 Allow to pass 'force' arg to selectro() to enforce a new select +ed71805 Changelog entry about "realdelete" option +0a275b9 Add scary warnings about "realdelete" option + OfflineIMAP v6.5.1.2 (2012-01-07) - "Baby steps" ================================================ diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 2bf4637..99fc2b0 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.1.2" +__version__ = "6.5.2-rc1" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 3028648e8f03f2e7d49b980ef08c0d20e0162457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Sun, 8 Jan 2012 19:57:03 +0100 Subject: [PATCH 410/817] Don't fail if /etc/netrc is not readable --- offlineimap/repository/IMAP.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index a5ed84c..c2633dc 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -138,7 +138,7 @@ class IMAPRepository(BaseRepository): try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) except IOError, inst: - if inst.errno != errno.ENOENT: + if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: @@ -235,7 +235,7 @@ class IMAPRepository(BaseRepository): try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) except IOError, inst: - if inst.errno != errno.ENOENT: + if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: From 1b30ab896e68342600793799b1c195630864f5c4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 9 Jan 2012 16:39:50 +0100 Subject: [PATCH 411/817] Add changelog entry Changelog entry for commit 0b103e86271b10bc2ae1602173d77295d54862da Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index bf0b460..be30b27 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -29,5 +29,7 @@ Changes * Revert 6.5.1.1 change to use public imaplib2 function, it was reported to not always work. +* Don't fail when ~/netrc is not readable by us. + Bug Fixes --------- From f2a94af5227db133bd861823418a7456202e64d8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 10 Jan 2012 07:19:22 +0100 Subject: [PATCH 412/817] Don't emit regular sleeping announcements in Basic UI They are too noisy in the Basic UI, but are still included in TTYUI. Reported by John Wiegley. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 1 + offlineimap/ui/TTY.py | 16 ++++++++++++++++ offlineimap/ui/UIBase.py | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index be30b27..2b278ca 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -31,5 +31,6 @@ Changes * Don't fail when ~/netrc is not readable by us. +* Don't emit noisy regular sleeping announcements in Basic UI. Bug Fixes --------- diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index cedd8a6..2b3e834 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -84,3 +84,19 @@ class TTYUI(UIBase): else: UIBase.mainException(self) + def sleeping(self, sleepsecs, remainingsecs): + """Sleep for sleepsecs, display remainingsecs to go. + + Does nothing if sleepsecs <= 0. + Display a message on the screen if we pass a full minute. + + This implementation in UIBase does not support this, but some + implementations return 0 for successful sleep and 1 for an + 'abort', ie a request to sync immediately. + """ + if sleepsecs > 0: + if remainingsecs//60 != (remainingsecs-sleepsecs)//60: + self.logger.info("Next refresh in %.1f minutes" % ( + remainingsecs/60.0)) + time.sleep(sleepsecs) + return 0 diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index dc9e935..66ecd4f 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -493,7 +493,7 @@ class UIBase(object): """ if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: - self.logger.info("Next refresh in %.1f minutes" % ( + self.logger.debug("Next refresh in %.1f minutes" % ( remainingsecs/60.0)) time.sleep(sleepsecs) return 0 From 340c2f9a8934e00d5d9ff625365ffb54391f76eb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 17 Jan 2012 01:34:56 +0100 Subject: [PATCH 413/817] Improve documentation Improve wording, and integrate the MANUAL and INSTALL and FAQ documents into our complete user manual that is hosted online. Signed-off-by: Sebastian Spaeth --- docs/INSTALL.rst | 170 +++++++++++++++++------------------ docs/MANUAL.rst | 17 ++++ docs/dev-doc-src/API.rst | 55 ++++++++++++ docs/dev-doc-src/FAQ.rst | 1 + docs/dev-doc-src/INSTALL.rst | 1 + docs/dev-doc-src/MANUAL.rst | 1 + docs/dev-doc-src/conf.py | 2 +- docs/dev-doc-src/index.rst | 89 +++++++----------- 8 files changed, 192 insertions(+), 144 deletions(-) create mode 100644 docs/dev-doc-src/API.rst create mode 120000 docs/dev-doc-src/FAQ.rst create mode 120000 docs/dev-doc-src/INSTALL.rst create mode 120000 docs/dev-doc-src/MANUAL.rst diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index e2ed87a..d14c574 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -1,81 +1,90 @@ .. -*- coding: utf-8 -*- - -.. _OfflineIMAP: https://github.com/nicolas33/offlineimap - -.. contents:: -.. sectnum:: - -============= -Prerequisites -============= - -In order to use `OfflineIMAP`_, you need to have these conditions satisfied: - -1. Your mail server must support IMAP. Most Internet Service Providers and - corporate networks do, and most operating systems have an IMAP implementation - readily available. A special Gmail mailbox type is available to interface with - Gmail's IMAP front-end. - -2. You must have Python version 2.6 or above installed. If you are running on - Debian GNU/Linux, this requirement will automatically be taken care of for you. - If you do not have Python already, check with your system administrator or - operating system vendor; or, download it from the Python website. If you intend - to use the SSL interface, your Python must have been built with SSL support. - -3. Have a mail reader that supports the Maildir mailbox format. Most modern - mail readers have this support built-in, so you can choose from a wide variety - of mail servers. This format is also known as the "qmail" format, so any mail - reader compatible with it will work with `OfflineIMAP`_. If you do not have a - mail reader that supports Maildir, you can often install a local IMAP server and - point both `OfflineIMAP`_ and your mail reader at it. - +.. _OfflineIMAP: https://github.com/spaetz/offlineimap +.. _OLI_git_repo: git://github.com/spaetz/offlineimap.git ============ Installation ============ -You have three options: +.. contents:: +.. .. sectnum:: -1. a system-wide installation with Debian -2. a system-wide installation with other systems -3. a single-user installation. You can checkout the latest version of - `OfflineIMAP`_ from official `OfflineIMAP`_ repository. +------------- +Prerequisites +------------- + +In order to use `OfflineIMAP`_, you need to have these conditions satisfied: + +1. Your mail server must support IMAP. Mail access via POP is not + supported. A special Gmail mailbox type is available to interface + with Gmail's IMAP front-end, although Gmail has a very peculiar and + non-standard implementation of its IMAP interface. + +2. You must have Python version 2.6 or above installed. If you are + running on Debian GNU/Linux, this requirement will automatically be + taken care of for you. If you intend to use the SSL interface, + your Python must have been built with SSL support. + +3. If you use OfflineImap as an IMAP<->Maildir synchronizer, you will + obviously need to have a mail reader that supports the Maildir + mailbox format. Most modern mail readers have this support built-in, + so you can choose from a wide variety of mail servers. This format + is also known as the "qmail" format, so any mail reader compatible + with it will work with `OfflineIMAP`_. -System-Wide Installation, Debian -================================ +------------ +Installation +------------ -If you are tracking Debian unstable, you may install `OfflineIMAP`_ by simply -running the following command as root:: +Installing OfflineImap should usually be quite easy, as you can simply unpack and run OfflineImap in place if you wish to do so. There are a number of options though: - apt-get install offlineimap +#. system-wide :ref:`installation via your distribution package manager ` +#. system-wide or single user :ref:`installation from the source package ` +#. system-wide or single user :ref:`installation from a git checkout ` -If you are not tracking Debian unstable, download the Debian `.deb` package from -the `OfflineIMAP`_ website and then run ``dpkg -i`` to install the downloaded -package. Then, skip to below. You will type offlineimap to invoke the -program. +Having installed OfflineImap, you will need to configure it, to be actually useful. Please check the :ref:`Configuration` section in the :doc:`MANUAL` for more information on the configuration step. -System-Wide Installation, Other -=============================== +.. _inst_pkg_man: -Check your distribution packaging tool, OfflineIMAP may already be packaged for -you. +System-Wide Installation via distribution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The easiest way to install OfflineIMAP is via your distribution's package manager. OfflineImap is available under the name `offlineimap` in most Linux and BSD distributions. -System-Wide Installation, From source -===================================== -Get your own copy of the official git repository at `OfflineIMAP`_:: +.. _inst_src_tar: - git clone git://github.com/nicolas33/offlineimap.git +Installation from source package +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Download the latest source archive from our `download page `_. Simply click the "Download as .zip" or "Download as .tar.gz" buttons to get the latest "stable" code from the master branch. If you prefer command line, you will want to use: + wget https://github.com/spaetz/offlineimap/tarball/master -This will download all the sources with history. By default, git set up the -local master branch up which is most likely what you want. If not, you can -checkout a particular release:: +Unpack and continue with the :ref:`system-wide installation ` or the :ref:`single-user installation ` section. + + +.. _inst_git: + +Installation from git checkout +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Get your own copy of the `official git repository `_ at `OfflineIMAP`_:: + + git clone git://github.com/spaetz/offlineimap.git + +This will download the source with history. By default, git sets up the +`master` branch up, which is most likely what you want. If not, you can +checkout a particular release like this:: cd offlineimap - git checkout -b local_version v6.3.3 + git checkout v6.5.2.1 -The latter creates a local branch called "local_version" of the v6.3.3 release. +You have now a source tree available and proceed with either the :ref:`system-wide installation ` or the :ref:`single-user installation `. + + +.. _system_wide_inst: + +System-wide installation +++++++++++++++++++++++++ Then run these commands, to build the python package:: @@ -86,47 +95,38 @@ Finally, install the program (as root):: python setup.py install -Next, proceed to below. You will type offlineimap to invoke the program. +Next, proceed to below. You tofflineimap to invoke the program. -Single-Account Installation -=========================== + +.. _single_user_inst: + +Single-user installation +++++++++++++++++++++++++ Download the git repository as described above. Instead of installing the program as root, you type `./offlineimap.py`; there is no installation step necessary. -============= -Configuration -============= - -`OfflineIMAP`_ is regulated by a configuration file that is normally stored in -`~/.offlineimaprc`. `OfflineIMAP`_ ships with a file named `offlineimap.conf` -that you should copy to that location and then edit. This file is vital to -proper operation of the system; it sets everything you need to run -`OfflineIMAP`_. Full documentation for the configuration file is included -within the sample file. - - -`OfflineIMAP`_ also ships a file named `offlineimap.conf.minimal` that you can -also try. It's useful if you want to get started with the most basic feature -set, and you can read about other features later with `offlineimap.conf`. - - -=============== +--------- Uninstall -=============== +--------- If you installed a system-wide installation via "python setup.py -install", there are a few files to purge to uninstall it again. I assume -that /usr/local is the standard prefix that your system and you use -python 2.7. Adapt to your system. In that case you need to: +install", there are a few files to purge to cleanly uninstall +`OfflineImap`_ again. Assuming that `/usr/local` is the standard prefix of +your system and that you use python 2.7, you need to: + +#) Delete the OfflineImap installation itself:: -1) Delete: /usr/local/lib/python2.7/dist-packages/offlineimap-6.4.4.egg-info /usr/local/lib/python2.7/dist-packages/offlineimap -2) Delete the cache at (default location) ~/.offlineimap - Delete your manually created (default loc) ~/.offlineimaprc + In case, you did the single-user installation, simply delete your + offlineimap directory. + +#) Delete all files that OfflineImap creates during its operation. + - The cache at (default location) ~/.offlineimap + - Your manually created (default loc) ~/.offlineimaprc (It is possible that you created those in different spots) That's it. Have fun without OfflineImap. diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index da09141..21a5dd6 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -43,6 +43,23 @@ Most configuration is done via the configuration file. However, any setting can OfflineImap is well suited to be frequently invoked by cron jobs, or can run in daemon mode to periodically check your email (however, it will exit in some error situations). +.. _configuration: + +Configuration +============= + +`OfflineIMAP`_ is regulated by a configuration file that is normally stored in +`~/.offlineimaprc`. `OfflineIMAP`_ ships with a file named `offlineimap.conf` +that you should copy to that location and then edit. This file is vital to +proper operation of the system; it sets everything you need to run +`OfflineIMAP`_. Full documentation for the configuration file is included +within the sample file. + + +`OfflineIMAP`_ also ships a file named `offlineimap.conf.minimal` that you can +also try. It's useful if you want to get started with the most basic feature +set, and you can read about other features later with `offlineimap.conf`. + Check out the `Use Cases`_ section for some example configurations. diff --git a/docs/dev-doc-src/API.rst b/docs/dev-doc-src/API.rst new file mode 100644 index 0000000..c901938 --- /dev/null +++ b/docs/dev-doc-src/API.rst @@ -0,0 +1,55 @@ +.. OfflineImap API documentation + +.. currentmodule:: offlineimap + +Welcome to :mod:`offlineimaps`'s documentation +============================================== + +Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. Email repositories are represented by a :class:`offlineimap.repository.Base.BaseRepository` or derivatives (see :mod:`offlineimap.repository` for details). A folder within a repository is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative from :mod:`offlineimap.folder`. + +This page contains the main API overview of OfflineImap |release|. + +OfflineImap can be imported as:: + + from offlineimap import OfflineImap + +:mod:`offlineimap` -- The OfflineImap module +============================================= + +.. module:: offlineimap + +.. autoclass:: offlineimap.OfflineImap(cmdline_opts = None) + + + .. automethod:: run + + .. automethod:: parse_cmd_options + +.. .. autoattribute:: ui + + :todo: Document + +:class:`offlineimap.account` +============================ + +An :class:`accounts.Account` connects two email repositories that are to be synced. It comes in two flavors, normal and syncable. + +.. autoclass:: offlineimap.accounts.Account + +.. autoclass:: offlineimap.accounts.SyncableAccount + :members: + :inherited-members: + + .. autodata:: ui + + Contains the current :mod:`offlineimap.ui`, and can be used for logging etc. + +:exc:`OfflineImapError` -- A Notmuch execution error +-------------------------------------------------------- + +.. autoexception:: offlineimap.error.OfflineImapError + :members: + + This execption inherits directly from :exc:`Exception` and is raised + on errors during the offlineimap execution. It has an attribute + `severity` that denotes the severity level of the error. diff --git a/docs/dev-doc-src/FAQ.rst b/docs/dev-doc-src/FAQ.rst new file mode 120000 index 0000000..e3ca8b5 --- /dev/null +++ b/docs/dev-doc-src/FAQ.rst @@ -0,0 +1 @@ +../FAQ.rst \ No newline at end of file diff --git a/docs/dev-doc-src/INSTALL.rst b/docs/dev-doc-src/INSTALL.rst new file mode 120000 index 0000000..356deb7 --- /dev/null +++ b/docs/dev-doc-src/INSTALL.rst @@ -0,0 +1 @@ +../INSTALL.rst \ No newline at end of file diff --git a/docs/dev-doc-src/MANUAL.rst b/docs/dev-doc-src/MANUAL.rst new file mode 120000 index 0000000..a45b3ee --- /dev/null +++ b/docs/dev-doc-src/MANUAL.rst @@ -0,0 +1 @@ +../MANUAL.rst \ No newline at end of file diff --git a/docs/dev-doc-src/conf.py b/docs/dev-doc-src/conf.py index f583a9c..e961ab2 100644 --- a/docs/dev-doc-src/conf.py +++ b/docs/dev-doc-src/conf.py @@ -94,7 +94,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' - +#html_style = '' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. diff --git a/docs/dev-doc-src/index.rst b/docs/dev-doc-src/index.rst index 7cf27e2..e1b957e 100644 --- a/docs/dev-doc-src/index.rst +++ b/docs/dev-doc-src/index.rst @@ -1,69 +1,42 @@ .. OfflineImap documentation master file +.. _OfflineImap: http://offlineimap.org -.. currentmodule:: offlineimap Welcome to :mod:`offlineimaps`'s documentation ============================================== -The :mod:`offlineimap` module provides the user interface for synchronization between IMAP servers and MailDirs or between IMAP servers. The homepage containing the source code repository can be found at the `offlineimap homepage `_. The following provides the developer documentation for those who are interested in modifying the source code or otherwise peek into the OfflineImap internals. End users might want to check the MANUAL, our INSTALLation instructions, and the FAQ. +`OfflineImap`_ synchronizes email between an IMAP server and a MailDir or between two IMAP servers. It offers very powerful and flexible configuration options, that allow things such as the filtering of folders, transposing of names via static configuration or python scripting. It plays well with mutt and other MailDir consuming email clients. + +The documentation contains the end user documentation in a first part. It also contains use cases and example configurations. It is followed by the internal :doc:`API documentation ` for those interested in modifying the source code or otherwise peek into the OfflineImap internals in a second part. + + +If you just want to get started with minimal fuzz, have a look at our `online quick start guide `_. Do note though, that our configuration options are many and powerful. Perusing our precious documentation does often pay off! + +More information on specific topics can be found on the following pages: + +**User documentation** + * :doc:`installation/uninstall ` + * :doc:`user manual/Configuration ` + * :doc:`command line options ` + * :doc:`Frequently Asked Questions ` + +**Developer documentation** + * :doc:`API documentation ` for internal details on the + :mod:`offlineimap` module + +.. toctree:: + :hidden: + + INSTALL + MANUAL + offlineimap + FAQ + + API + repository + ui -Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. Email repositories are represented by a :class:`offlineimap.repository.Base.BaseRepository` or derivatives (see :mod:`offlineimap.repository` for details). A folder within a repository is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative from :mod:`offlineimap.folder`. .. moduleauthor:: John Goerzen, and many others. See AUTHORS and the git history for a full list. :License: This module is covered under the GNU GPL v2 (or later). - -This page contains the main API overview of OfflineImap |release|. - -OfflineImap can be imported as:: - - from offlineimap import OfflineImap - -More information on specific topics can be found on the following pages: - -.. toctree:: - :maxdepth: 1 - - repository - ui - offlineimap - -:mod:`offlineimap` -- The OfflineImap module -============================================= - -.. module:: offlineimap - -.. autoclass:: offlineimap.OfflineImap(cmdline_opts = None) - - .. automethod:: lock - - .. automethod:: run - -.. .. autoattribute:: ui - - :todo: Document - -:class:`offlineimap.account` -============================ - -An :class:`accounts.Account` connects two email repositories that are to be synced. It comes in two flavors, normal and syncable. - -.. autoclass:: offlineimap.accounts.Account - -.. autoclass:: offlineimap.accounts.SyncableAccount - :members: - :inherited-members: - - .. autodata:: ui - - Contains the current :mod:`offlineimap.ui`, and can be used for logging etc. - -:exc:`OfflineImapError` -- A Notmuch execution error --------------------------------------------------------- - -.. autoexception:: offlineimap.error.OfflineImapError - :members: - - This execption inherits directly from :exc:`Exception` and is raised - on errors during the offlineimap execution. It has an attribute - `severity` that denotes the severity level of the error. From 7a5768e471ddf561f907c335e1e2e2cd81f79040 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 17 Jan 2012 01:40:12 +0100 Subject: [PATCH 414/817] Release v6.5.2 Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 16 ---------------- Changelog.rst | 32 +++++++++++++++++++++----------- offlineimap/__init__.py | 2 +- 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 2b278ca..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -10,27 +10,11 @@ others. `WIP (coming releases)` ======================= -* Gmail "realdelete" is considered harmful and has the potential for data loss. Analysis at http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 -Warnings were added to offlineimap.conf - New Features ------------ Changes ------- -* Rather than to write out the nametrans'lated folder names for mbnames, - we now write out the local untransformed box names. This is generally - what we want. This became relevant since we support nametrans rules on - the local side since only a short time. Reported by Paul Collignan. - -* Some sanity checks and improved error messages. - -* Revert 6.5.1.1 change to use public imaplib2 function, it was reported to - not always work. - -* Don't fail when ~/netrc is not readable by us. - -* Don't emit noisy regular sleeping announcements in Basic UI. Bug Fixes --------- diff --git a/Changelog.rst b/Changelog.rst index f7a7dd3..53a8a0c 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,17 +11,27 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. -OfflineIMAP v6.5.2-rc1 (2012-01-09) -=================================== -Commits v6.5.1.1 - v6.5.2-rc1: -note: Proper Changelog still in Changelog-draft.rst -d72bb88 Improve error message -3284e01 Revert "use .response() rather _get_untagged_response()" -81f194a mbnames should write out local and not nametransformed box names -7184ec2 Sanity check return value of UIDVALIDTY response -50de217 Allow to pass 'force' arg to selectro() to enforce a new select -ed71805 Changelog entry about "realdelete" option -0a275b9 Add scary warnings about "realdelete" option +OfflineIMAP v6.5.2 (2012-01-17) +=============================== + +* Gmail "realdelete" option is considered harmful and has the potential + for data loss. Analysis at + http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 + Warnings were added to offlineimap.conf + +* Rather than write out the nametrans'lated folder names for mbnames, we + now write out the local untransformed box names. This is generally + what we want. This became relevant since we support nametrans rules on + the local side since only a short time. Reported by Paul Collignan. + +* Some sanity checks and improved error messages. + +* Revert 6.5.1.1 change to use public imaplib2 function, it was reported to + not always work. + +* Don't fail when ~/netrc is not readable by us. + +* Don't emit noisy regular sleeping announcements in Basic UI. OfflineIMAP v6.5.1.2 (2012-01-07) - "Baby steps" ================================================ diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 99fc2b0..6b7b404 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.2-rc1" +__version__ = "6.5.2" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 6dc74c9da5016a8fe7b30bcc438ae8b700084824 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 22:39:59 +0100 Subject: [PATCH 415/817] Improve delete performance with SQLITE backend When deleting many (eg 2000) mails using the SQLITE backend, this takes a long time durig which OfflineImap can not be aborted via CTRL-C. Thinking it had frozen permanently, I killed it hard, leaving a corrupted db journal (which leads to awkwards complaints by OLI on subsequent starts!). That shows that delete performance is critical and needs improvement. We were iterating through the list of messages to delete and deleted them one-by-one execute()'ing a new SQL Query for each message. This patch improves the situation by allowing us to use executemany(), which is -despite still being one SQL query per message- much faster. This is because rather than performing a commit() after each mail, we now do only one commit() after all mails have been deleted. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/LocalStatusSQLite.py | 39 ++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 08af807..6bfa667 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -73,24 +73,32 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): if version < LocalStatusSQLiteFolder.cur_version: self.upgrade_db(version) - def sql_write(self, sql, vars=None): - """execute some SQL retrying if the db was locked. + def sql_write(self, sql, vars=None, executemany=False): + """Execute some SQL, retrying if the db was locked. - :param sql: the SQL string passed to execute() :param args: the - variable values to `sql`. E.g. (1,2) or {uid:1, flags:'T'}. See - sqlite docs for possibilities. + :param sql: the SQL string passed to execute() + :param vars: the variable values to `sql`. E.g. (1,2) or {uid:1, + flags:'T'}. See sqlite docs for possibilities. + :param executemany: bool indicating whether we want to + perform conn.executemany() or conn.execute(). :returns: the Cursor() or raises an Exception""" success = False while not success: self._dblock.acquire() try: if vars is None: - cursor = self.connection.execute(sql) + if executemany: + cursor = self.connection.executemany(sql) + else: + cursor = self.connection.execute(sql) else: - cursor = self.connection.execute(sql, vars) + if executemany: + cursor = self.connection.executemany(sql, vars) + else: + cursor = self.connection.execute(sql, vars) success = True self.connection.commit() - except sqlite.OperationalError, e: + except sqlite.OperationalError as e: if e.args[0] == 'cannot commit - no transaction is active': pass elif e.args[0] == 'database is locked': @@ -231,12 +239,23 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): flags = ''.join(sorted(flags)) self.sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) + def deletemessage(self, uid): + if not uid in self.messagelist: + return + self.sql_write('DELETE FROM status WHERE id=?', (uid, )) + del(self.messagelist[uid]) + def deletemessages(self, uidlist): + """Delete list of UIDs from status cache + + This function uses sqlites executemany() function which is + much faster than iterating through deletemessage() when we have + many messages to delete.""" # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if uid in self.messagelist] if not len(uidlist): return + # arg2 needs to be an iterable of 1-tuples [(1,),(2,),...] + self.sql_write('DELETE FROM status WHERE id=?', zip(uidlist, ), True) for uid in uidlist: del(self.messagelist[uid]) - #TODO: we want a way to do executemany(.., uidlist) to delete all - self.sql_write('DELETE FROM status WHERE id=?', (uid, )) From 73b2b01dfa0c37e4003699393005c37757f3c856 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 17 Jan 2012 05:14:50 +0100 Subject: [PATCH 416/817] Changelog entry for previous commit Forgot to add a changelog entry. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..f17387a 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,5 +16,7 @@ New Features Changes ------- +* Improve delete msg performance with SQLITE backend + Bug Fixes --------- From 344b2f0b7a084855ebce0aad85fe7a7d0dfd2b15 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 18 Jan 2012 00:39:04 +0100 Subject: [PATCH 417/817] TTYUI: Fix python 2.6 compatibility We were using super() on a class derived from logging.Formatter() which worked fine in python 2.7. Apparently python 2.6 uses old-style classes for this, so the TTYUI broke and crashed OfflineImap. This was introduced in OLI 6.5.0, I think. Fix it by calling logging.Formatter.... directly, rather than the elegant super() (which I happen to like a lot more than is appropriate in the python world). Reported by Nik Reiman as github issue 23, should fix that issue. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/TTY.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index f17387a..21133ec 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -20,3 +20,5 @@ Changes Bug Fixes --------- + +* Fix python2.6 compatibility with the TTYUI backend (crash) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 2b3e834..862d7a4 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -24,12 +24,14 @@ from offlineimap.ui.UIBase import UIBase class TTYFormatter(logging.Formatter): """Specific Formatter that adds thread information to the log output""" def __init__(self, *args, **kwargs): - super(TTYFormatter, self).__init__(*args, **kwargs) + #super() doesn't work in py2.6 as 'logging' uses old-style class + logging.Formatter.__init__(self, *args, **kwargs) self._last_log_thread = None def format(self, record): """Override format to add thread information""" - log_str = super(TTYFormatter, self).format(record) + #super() doesn't work in py2.6 as 'logging' uses old-style class + log_str = logging.Formatter.format(self, record) # If msg comes from a different thread than our last, prepend # thread info. Most look like 'Account sync foo' or 'Folder # sync foo'. From 48802ae4b63c5887208a044261aeaace7959be20 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 02:00:09 +0100 Subject: [PATCH 418/817] TTYUI: Fix missing "time" import commit f2a94af5 introduced the use of time.sleep in ui/TTY.py without importing it. This caused a regression in 6.5.2, crashing OfflineIMap when in refresh mode. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 1 + offlineimap/ui/TTY.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 21133ec..10967c3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -22,3 +22,4 @@ Bug Fixes --------- * Fix python2.6 compatibility with the TTYUI backend (crash) +* Fix TTYUI regression from 6.5.2 in refresh loop (crash) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 862d7a4..efde74f 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -17,6 +17,7 @@ import logging import sys +import time from getpass import getpass from offlineimap import banner from offlineimap.ui.UIBase import UIBase From 3cd3edefca21724dddf873ed03eea889eb8ede10 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 18 Jan 2012 00:39:04 +0100 Subject: [PATCH 419/817] TTYUI: Fix python 2.6 compatibility We were using super() on a class derived from logging.Formatter() which worked fine in python 2.7. Apparently python 2.6 uses old-style classes for this, so the TTYUI broke and crashed OfflineImap. This was introduced in OLI 6.5.0, I think. Fix it by calling logging.Formatter.... directly, rather than the elegant super() (which I happen to like a lot more than is appropriate in the python world). Reported by Nik Reiman as github issue 23, should fix that issue. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/TTY.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..99c70bc 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,5 @@ Changes Bug Fixes --------- + +* Fix python2.6 compatibility with the TTYUI backend (crash) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 2b3e834..862d7a4 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -24,12 +24,14 @@ from offlineimap.ui.UIBase import UIBase class TTYFormatter(logging.Formatter): """Specific Formatter that adds thread information to the log output""" def __init__(self, *args, **kwargs): - super(TTYFormatter, self).__init__(*args, **kwargs) + #super() doesn't work in py2.6 as 'logging' uses old-style class + logging.Formatter.__init__(self, *args, **kwargs) self._last_log_thread = None def format(self, record): """Override format to add thread information""" - log_str = super(TTYFormatter, self).format(record) + #super() doesn't work in py2.6 as 'logging' uses old-style class + log_str = logging.Formatter.format(self, record) # If msg comes from a different thread than our last, prepend # thread info. Most look like 'Account sync foo' or 'Folder # sync foo'. From 075c1d2cdc0a8e25f8cb60583cabd2ffa1db1d13 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 02:00:09 +0100 Subject: [PATCH 420/817] TTYUI: Fix missing "time" import commit f2a94af5 introduced the use of time.sleep in ui/TTY.py without importing it. This caused a regression in 6.5.2, crashing OfflineIMap when in refresh mode. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 1 + offlineimap/ui/TTY.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 99c70bc..7f71597 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -20,3 +20,4 @@ Bug Fixes --------- * Fix python2.6 compatibility with the TTYUI backend (crash) +* Fix TTYUI regression from 6.5.2 in refresh loop (crash) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 862d7a4..efde74f 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -17,6 +17,7 @@ import logging import sys +import time from getpass import getpass from offlineimap import banner from offlineimap.ui.UIBase import UIBase From 2c19a63b9da87259c4df0dd798a0bb7d8303fcc0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 10:08:31 +0100 Subject: [PATCH 421/817] DOCS: Add SSL debugging info to FAQ Take the debugging instructions by Daniel Shahaf and add them to the FAQ. Signed-off-by: Sebastian Spaeth --- docs/FAQ.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 29b53d6..4acad4e 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -302,6 +302,20 @@ The path `/etc/ssl/certs` is not standardized; your system may store SSL certificates elsewhere. (On some systems it may be in `/usr/local/share/certs/`.) +If this does not work and you are getting error messages, you can test the certificate using a command like (credits to Daniel Shahaf for this):: + + % openssl s_client -CAfile $sslcacertfile -connect ${hostname}:imaps 2>&1 Date: Thu, 19 Jan 2012 10:15:04 +0100 Subject: [PATCH 422/817] DOCS: Tweak SSL FAQ a bit more Cleanup the previously added information. Signed-off-by: Sebastian Spaeth --- docs/FAQ.rst | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 4acad4e..8381681 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -288,33 +288,29 @@ How do I generate an `sslcacertfile` file? The `sslcacertfile` file must contain an SSL certificate (or a concatenated certificates chain) in PEM format. (See the documentation of -`ssl.wrap_socket`_'s `certfile` parameter for the gory details.) The following -command should generate a file in the proper format:: +`ssl.wrap_socket`_'s `certfile` parameter for the gory details.) You can use either openssl or gnutls to create a certificate file in the required format. +#. via openssl:: openssl s_client -CApath /etc/ssl/certs -connect ${hostname}:imaps -showcerts \ | perl -ne 'print if /BEGIN/../END/; print STDERR if /return/' > $sslcacertfile ^D -Before using the resulting file, ensure that openssl verified the certificate -successfully. +#. via gnutls:: + gnutls-cli --print-cert -p imaps ${host} $sslcacertfile The path `/etc/ssl/certs` is not standardized; your system may store SSL certificates elsewhere. (On some systems it may be in `/usr/local/share/certs/`.) -If this does not work and you are getting error messages, you can test the certificate using a command like (credits to Daniel Shahaf for this):: +Before using the resulting file, ensure that openssl verified the certificate +successfully. In case of problems, you can test the certificate using a command such as (credits to Daniel Shahaf for this) to verify the certificate:: % openssl s_client -CAfile $sslcacertfile -connect ${hostname}:imaps 2>&1 Date: Mon, 9 Jan 2012 13:12:28 +0100 Subject: [PATCH 423/817] Add initial skeleton for a testing framework This is the first revision that actually performs a test, in that it starts up OfflineImap and sees if there are any exceptions. Signed-off-by: Sebastian Spaeth --- test/.gitignore | 2 + test/OLItest/TestRunner.py | 102 +++++++++++++++++++++++++++++++++++ test/OLItest/__init__.py | 34 ++++++++++++ test/OLItest/globals.py | 37 +++++++++++++ test/README | 17 ++++++ test/credentials.conf.sample | 13 +++++ test/test | 9 ++++ test/tests/__init__.py | 0 test/tests/test_01_basic.py | 75 ++++++++++++++++++++++++++ 9 files changed, 289 insertions(+) create mode 100644 test/.gitignore create mode 100644 test/OLItest/TestRunner.py create mode 100644 test/OLItest/__init__.py create mode 100644 test/OLItest/globals.py create mode 100644 test/README create mode 100644 test/credentials.conf.sample create mode 100755 test/test create mode 100644 test/tests/__init__.py create mode 100644 test/tests/test_01_basic.py diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..08ea7df --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,2 @@ +credentials.conf +tmp_* \ No newline at end of file diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py new file mode 100644 index 0000000..f984471 --- /dev/null +++ b/test/OLItest/TestRunner.py @@ -0,0 +1,102 @@ +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import unittest +import logging +import os +import sys +import shutil +import subprocess +import tempfile +from ConfigParser import SafeConfigParser +from OLItest import default_conf + +class OLITestLib(): + cred_file = None + testdir = None + """Absolute path of the current temporary test directory""" + cmd = None + """command that will be executed to invoke offlineimap""" + + def __init__(self, cred_file = None, cmd='offlineimap'): + """ + + :param cred_file: file of the configuration + snippet for authenticating against the test IMAP server(s). + :param cmd: command that will be executed to invoke offlineimap""" + OLITestLib.cred_file = cred_file + OLITestLib.cmd = cmd + + @classmethod + def create_test_dir(cls, suffix=''): + """Creates a test directory and places OLI config there + + Note that this is a class method. There can only be one test + directory at a time. OLITestLib is not suited for running + several tests in parallel. The user is responsible for + cleaning that up herself.""" + # creating temporary directory for testing in current dir + cls.testdir = os.path.abspath( + tempfile.mkdtemp(prefix='tmp_%s_'%suffix, dir='.')) + cls.create_config_file() + return cls.testdir + + @classmethod + def create_config_file(cls): + """Creates a OLI configuration file + + It is created in testdir (so create_test_dir has to be called + earlier) using the credentials information given (so they had to + be set earlier). Failure to do either of them will raise an + AssertionException.""" + assert cls.cred_file != None + assert cls.testdir != None + config = SafeConfigParser() + config.readfp(default_conf) + config.read(cls.cred_file) + config.set("general", "metadata", cls.testdir) + localfolders = os.path.join(cls.testdir, 'mail') + config.set("Repository Maildir", "localfolders", localfolders) + with open(os.path.join(cls.testdir, 'offlineimap.conf'), "wa") as f: + config.write(f) + + @classmethod + def delete_test_dir(cls): + """Deletes the current test directory + + The users is responsible for cleaning that up herself.""" + if os.path.isdir(cls.testdir): + shutil.rmtree(cls.testdir) + + @classmethod + def run_OLI(cls): + """Runs OfflineImap + + :returns: (rescode, stdout) + """ + try: + output = subprocess.check_output( + [cls.cmd, + "-c%s" % os.path.join(cls.testdir, 'offlineimap.conf')], + shell=False) + except subprocess.CalledProcessError as e: + return (e.returncode, e.output) + return (0, output) + +class OLITextTestRunner(unittest.TextTestRunner): + + def __init__(self,*args, **kwargs): + logging.warning("OfflineImap testsuite") + return super(OLITextTestRunner, self).__init__(*args, **kwargs) diff --git a/test/OLItest/__init__.py b/test/OLItest/__init__.py new file mode 100644 index 0000000..c9bda69 --- /dev/null +++ b/test/OLItest/__init__.py @@ -0,0 +1,34 @@ +# OfflineImap test library +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +__all__ = ['OLITestLib', 'OLITextTestRunner','TestLoader'] + +__productname__ = 'OfflineIMAP Test suite' +__version__ = '0' +__copyright__ = "Copyright 2012- Sebastian Spaeth & contributors" +__author__ = 'Sebastian Spaeth' +__author_email__= 'Sebastian@SSpaeth.de' +__description__ = 'Moo' +__license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" +__homepage__ = "http://offlineimap.org" +banner = """%(__productname__)s %(__version__)s + %(__license__)s""" % locals() + +import unittest +from unittest import TestLoader +from globals import default_conf +from TestRunner import OLITestLib, OLITextTestRunner diff --git a/test/OLItest/globals.py b/test/OLItest/globals.py new file mode 100644 index 0000000..4009b9b --- /dev/null +++ b/test/OLItest/globals.py @@ -0,0 +1,37 @@ +#Constants, that don't rely on anything else in the module +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +from cStringIO import StringIO + +default_conf=StringIO("""[general] +#will be set automatically +metadata = +accounts = test +ui = quiet + +[Account test] +localrepository = Maildir +remoterepository = IMAP + +[Repository Maildir] +Type = Maildir +# will be set automatically during tests +localfolders = + +[Repository IMAP] +type=IMAP +folderfilter= lambda f: f.startswith('OLItest') +""") diff --git a/test/README b/test/README new file mode 100644 index 0000000..3bf6694 --- /dev/null +++ b/test/README @@ -0,0 +1,17 @@ +Documentation for the OfflineImap Test suite. + +How to run the tests +==================== + +- Copy the credentials.conf.sample to credentials.conf and insert + credentials for an IMAP account and a Gmail account. Delete the Gmail + section if you don't have a Gmail account. Do note, that the tests + will change the account and upload/delete/modify it's contents and + folder structure. So don't use a real used account here... + +- Execute './test' + +System requirements +=================== + +This test suite depend on python>=2.7 to run out of the box. If you want to run this with python 2.6 you will need to install the backport from http://pypi.python.org/pypi/unittest2 instead. \ No newline at end of file diff --git a/test/credentials.conf.sample b/test/credentials.conf.sample new file mode 100644 index 0000000..cd22f77 --- /dev/null +++ b/test/credentials.conf.sample @@ -0,0 +1,13 @@ +[Repository IMAP] +type = IMAP +remotehost = localhost +ssl = no +#sslcacertfile = +#cert_fingerprint = +remoteuser = user@domain +remotepass = SeKr3t + +[Repository Gmail] +type = Gmail +remoteuser = user@domain +remotepass = SeKr3t diff --git a/test/test b/test/test new file mode 100755 index 0000000..08803f9 --- /dev/null +++ b/test/test @@ -0,0 +1,9 @@ +#!/usr/bin/env python +import logging +from OLItest import OLITextTestRunner, TestLoader, OLITestLib + +if __name__ == '__main__': + logging.basicConfig(format='%(message)s') + OLITestLib(cred_file='./credentials.conf', cmd='../offlineimap.py') + suite = TestLoader().discover('./tests') + OLITextTestRunner(verbosity=2).run(suite) diff --git a/test/tests/__init__.py b/test/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py new file mode 100644 index 0000000..4456890 --- /dev/null +++ b/test/tests/test_01_basic.py @@ -0,0 +1,75 @@ +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import random +import unittest +import logging +import os, sys +# Insert ".." into the python search path +cmd_folder = os.path.dirname(os.path.abspath(__file__)) +if cmd_folder not in sys.path: + sys.path.insert(0, cmd_folder) +from OLItest import OLITestLib + +def setUpModule(): + logging.info("Set Up test module %s" % __name__) + tdir = OLITestLib.create_test_dir(suffix=__name__) + +def tearDownModule(): + logging.info("Tear Down test module") + OLITestLib.delete_test_dir() + +#Stuff that can be used +#self.assertEqual(self.seq, range(10)) +# should raise an exception for an immutable sequence +#self.assertRaises(TypeError, random.shuffle, (1,2,3)) +#self.assertTrue(element in self.seq) +#self.assertFalse(element in self.seq) + +class TestBasicFunctions(unittest.TestCase): + #@classmethod + #def setUpClass(cls): + #This is run before all tests in this class + # cls._connection = createExpensiveConnectionObject() + + #@classmethod + #This is run after all tests in this class + #def tearDownClass(cls): + # cls._connection.destroy() + + # This will be run before each test + #def setUp(self): + # self.seq = range(10) + + def test_01_olistartup(self): + """Tests if OLI can be invoked without exceptions + + It syncs all "OLItest* (specified in the default config) to our + local Maildir at keeps it there.""" + code, res = OLITestLib.run_OLI() + #logging.warn("%s %s "% (code, res)) + self.assertEqual(res, "") + #TODO implement OLITestLib.countmails() + logging.warn("synced %d boxes, %d mails" % (0,0)) + + def test_02_wipedir(self): + """Wipe OLItest* maildir, sync and see if it's still empty + + Wipes all "OLItest* (specified in the default config) to our + local Maildir at keeps it there.""" + + code, res = OLITestLib.run_OLI() + #logging.warn("%s %s "% (code, res)) + self.assertEqual(res, "") From 9a905188f231478f690ed95fd5d4356348835ced Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 9 Jan 2012 14:38:28 +0100 Subject: [PATCH 424/817] Add userfriendly warning if credentials.conf is missing Warn the user and advise what she should be doing to get going. Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index f984471..5235949 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -37,6 +37,9 @@ class OLITestLib(): snippet for authenticating against the test IMAP server(s). :param cmd: command that will be executed to invoke offlineimap""" OLITestLib.cred_file = cred_file + if not os.path.isfile(cred_file): + raise UserWarning("Please copy 'credentials.conf.sample' to '%s' " + "and set your credentials there." % cred_file) OLITestLib.cmd = cmd @classmethod From 1cdabcec4641478226cd6e490827792a11c28b25 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 18 Jan 2012 22:01:33 +0100 Subject: [PATCH 425/817] Add code comment Signed-off-by: Sebastian Spaeth --- test/test | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test b/test/test index 08803f9..1394058 100755 --- a/test/test +++ b/test/test @@ -4,6 +4,7 @@ from OLItest import OLITextTestRunner, TestLoader, OLITestLib if __name__ == '__main__': logging.basicConfig(format='%(message)s') + # set credentials and OfflineImap command to be executed: OLITestLib(cred_file='./credentials.conf', cmd='../offlineimap.py') suite = TestLoader().discover('./tests') OLITextTestRunner(verbosity=2).run(suite) From 1f91d0fd06180833ebe2adb28be4849bb8be4a12 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 18 Jan 2012 22:39:25 +0100 Subject: [PATCH 426/817] Adapt imports so that test/OLItest can be imported To run the test suite from the main setup.py we need to be able to import and run the test suite from a different folder than the "test" dir. Make "test" a package and fix the imports to still work when imported from another folder. Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 2 +- test/__init__.py | 0 test/tests/test_01_basic.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 test/__init__.py diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 5235949..bb7e1c6 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -21,7 +21,7 @@ import shutil import subprocess import tempfile from ConfigParser import SafeConfigParser -from OLItest import default_conf +from . import default_conf class OLITestLib(): cred_file = None diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 4456890..7f30052 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -17,8 +17,8 @@ import random import unittest import logging import os, sys -# Insert ".." into the python search path -cmd_folder = os.path.dirname(os.path.abspath(__file__)) +# Insert ".." into the python search path to get OLItest +cmd_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if cmd_folder not in sys.path: sys.path.insert(0, cmd_folder) from OLItest import OLITestLib From bf1f79c5712c9f4dc95a68af238b1857f70d977b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 18 Jan 2012 22:41:05 +0100 Subject: [PATCH 427/817] Add "test" command to setup.py "python setup.py test" will now run the complete test suite. Remove the previous ./test command. Signed-off-by: Sebastian Spaeth --- setup.py | 30 ++++++++++++++++++++++++++---- test/README | 3 ++- test/test | 10 ---------- test/tests/test_01_basic.py | 6 +----- 4 files changed, 29 insertions(+), 20 deletions(-) delete mode 100755 test/test diff --git a/setup.py b/setup.py index c87077b..e684cda 100644 --- a/setup.py +++ b/setup.py @@ -20,11 +20,32 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# END OF COPYRIGHT # - -from distutils.core import setup +import os +from distutils.core import setup, Command import offlineimap +import logging +from test.OLItest import OLITextTestRunner, TestLoader, OLITestLib + +class TestCommand(Command): + """runs the OLI testsuite""" + description = "Runs the test suite" + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + logging.basicConfig(format='%(message)s') + # set credentials and OfflineImap command to be executed: + OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') + suite = TestLoader().discover('./test/tests') + #TODO: failfast does not seem to exist in python2.6? + OLITextTestRunner(verbosity=2,failfast=True).run(suite) + setup(name = "offlineimap", version = offlineimap.__version__, @@ -36,6 +57,7 @@ setup(name = "offlineimap", 'offlineimap.repository', 'offlineimap.ui'], scripts = ['bin/offlineimap'], license = offlineimap.__copyright__ + \ - ", Licensed under the GPL version 2" + ", Licensed under the GPL version 2", + cmdclass = { 'test': TestCommand} ) diff --git a/test/README b/test/README index 3bf6694..a02e3b9 100644 --- a/test/README +++ b/test/README @@ -9,7 +9,8 @@ How to run the tests will change the account and upload/delete/modify it's contents and folder structure. So don't use a real used account here... -- Execute './test' +- go to the top level dir (one above this one) and execute: + 'python setup.py test' System requirements =================== diff --git a/test/test b/test/test deleted file mode 100755 index 1394058..0000000 --- a/test/test +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -import logging -from OLItest import OLITextTestRunner, TestLoader, OLITestLib - -if __name__ == '__main__': - logging.basicConfig(format='%(message)s') - # set credentials and OfflineImap command to be executed: - OLITestLib(cred_file='./credentials.conf', cmd='../offlineimap.py') - suite = TestLoader().discover('./tests') - OLITextTestRunner(verbosity=2).run(suite) diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 7f30052..cb6d293 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -17,11 +17,7 @@ import random import unittest import logging import os, sys -# Insert ".." into the python search path to get OLItest -cmd_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if cmd_folder not in sys.path: - sys.path.insert(0, cmd_folder) -from OLItest import OLITestLib +from test.OLItest import OLITestLib def setUpModule(): logging.info("Set Up test module %s" % __name__) From 2776ee197409e31c8ac90a47356d7890072868da Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 18 Jan 2012 23:12:27 +0100 Subject: [PATCH 428/817] Create temp directories in "test" dir Don't create the temp dirs in the current directory, but create them in the same directory as the credentials.conf file is. We still need to prevent deleting them on test failure, so that they can be inspected manually. Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index bb7e1c6..0d0730b 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -50,9 +50,10 @@ class OLITestLib(): directory at a time. OLITestLib is not suited for running several tests in parallel. The user is responsible for cleaning that up herself.""" - # creating temporary directory for testing in current dir + # creating temporary dir for testing in same dir as credentials.conf cls.testdir = os.path.abspath( - tempfile.mkdtemp(prefix='tmp_%s_'%suffix, dir='.')) + tempfile.mkdtemp(prefix='tmp_%s_'%suffix, + dir=os.path.dirname(cls.cred_file))) cls.create_config_file() return cls.testdir From 7c4fea906fd1a6ad9f3a597a8a2dcf7e30df22fb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 10:34:19 +0100 Subject: [PATCH 429/817] Changelog entry for the test suite Announce existence of a test suite, although it is still quite useless. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 10967c3..83cab97 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,9 @@ others. New Features ------------ +* Beginning of a test suite. So far there is only one test. Configure + test/credentials.conf and invoke with "python setup.py test" + Changes ------- From 9453e1d9551e56bc7c8d73c19a29e609de873f15 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 11:24:16 +0100 Subject: [PATCH 430/817] Fix getuidvalidity crash (UIDVALIDITY returning None) Rename getuidvalidity -> get_uidvalidity and cache the IMAP result. 1) Start modernizing our function names using more underscores 2) IMAPs implementation of get_uidvalidity was removing the UIDVALIDITY result from the imaplib2 result stack, so subsequent calls would return "None". As various functions can invoke this, this led to some errors that we avoid by caching the current UIDVALIDITY value in the Folder instance. There are more simplifications and improvements to be made. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 1 + offlineimap/accounts.py | 2 +- offlineimap/folder/Base.py | 17 +++++++++++------ offlineimap/folder/IMAP.py | 22 ++++++++++++++-------- offlineimap/folder/Maildir.py | 6 ++++-- offlineimap/ui/Machine.py | 2 +- offlineimap/ui/UIBase.py | 4 ++-- 7 files changed, 34 insertions(+), 20 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 83cab97..93f64f1 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -26,3 +26,4 @@ Bug Fixes * Fix python2.6 compatibility with the TTYUI backend (crash) * Fix TTYUI regression from 6.5.2 in refresh loop (crash) +* Fix crashes related to UIDVALIDITY returning "None" diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index fc2687f..ad2015a 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -378,7 +378,7 @@ def syncfolder(account, remotefolder, quick): statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ replace(remoterepos.getsep(), statusrepos.getsep())) - if localfolder.getuidvalidity() == None: + if localfolder.get_uidvalidity() == None: # This is a new folder, so delete the status cache to be sure # we don't have a conflict. statusfolder.deletemessagelist() diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 9ff60e2..cab22c2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -110,13 +110,14 @@ class BaseFolder(object): return basename def isuidvalidityok(self): - """Does the cached UID match the real UID + """Tests if the cached UIDVALIDITY match the real current one - If required it caches the UID. In this case the function is not - threadsafe. So don't attempt to call it from concurrent threads.""" + If required it saves the UIDVALIDITY value. In this case the + function is not threadsafe. So don't attempt to call it from + concurrent threads.""" if self.getsaveduidvalidity() != None: - return self.getsaveduidvalidity() == self.getuidvalidity() + return self.getsaveduidvalidity() == self.get_uidvalidity() else: self.saveuidvalidity() return 1 @@ -142,7 +143,7 @@ class BaseFolder(object): This function is not threadsafe, so don't attempt to call it from concurrent threads.""" - newval = self.getuidvalidity() + newval = self.get_uidvalidity() uidfilename = self._getuidfilename() file = open(uidfilename + ".tmp", "wt") @@ -151,7 +152,11 @@ class BaseFolder(object): os.rename(uidfilename + ".tmp", uidfilename) self._base_saved_uidvalidity = newval - def getuidvalidity(self): + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value + + This function needs to be implemented by each Backend + :returns: UIDVALIDITY as a (long) number""" raise NotImplementedException def cachemessagelist(self): diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 75c731c..ffbd6b1 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -65,17 +65,23 @@ class IMAPFolder(BaseFolder): def getcopyinstancelimit(self): return 'MSGCOPY_' + self.repository.getname() - def getuidvalidity(self): + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value + + UIDVALIDITY value will be cached on the first call. + :returns: The UIDVALIDITY as (long) number.""" + if hasattr(self, '_uidvalidity'): + # use cached value if existing + return self._uidvalidity imapobj = self.imapserver.acquireconnection() try: - # SELECT receives UIDVALIDITY response + # SELECT (if not already done) and get current UIDVALIDITY self.selectro(imapobj) - # note: we would want to use .response() here but that - # often seems to return [None], even though we have - # data. TODO - uidval = imapobj._get_untagged_response('UIDVALIDITY') - assert uidval != [None], "response('UIDVALIDITY') returned [None]!" - return long(uidval[-1]) + typ, uidval = imapobj.response('UIDVALIDITY') + assert uidval != [None] and uidval != None, \ + "response('UIDVALIDITY') returned [None]!" + self._uidvalidity = long(uidval[-1]) + return self._uidvalidity finally: self.imapserver.releaseconnection(imapobj) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 95258d9..fe85308 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -83,8 +83,10 @@ class MaildirFolder(BaseFolder): """Return the absolute file path to the Maildir folder (sans cur|new)""" return self._fullname - def getuidvalidity(self): - """Maildirs have no notion of uidvalidity, so we just return a magic + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value + + Maildirs have no notion of uidvalidity, so we just return a magic token.""" return 42 diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index ab57ea2..271629c 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -70,7 +70,7 @@ class MachineUI(UIBase): def validityproblem(s, folder): s._printData('validityproblem', "%s\n%s\n%s\n%s" % \ (folder.getname(), folder.getrepository().getname(), - folder.getsaveduidvalidity(), folder.getuidvalidity())) + folder.getsaveduidvalidity(), folder.get_uidvalidity())) def connecting(s, hostname, port): s._printData('connecting', "%s\n%s" % (hostname, str(port))) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 66ecd4f..1cdd94b 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -309,9 +309,9 @@ class UIBase(object): def validityproblem(self, folder): self.logger.warning("UID validity problem for folder %s (repo %s) " "(saved %d; got %d); skipping it. Please see FAQ " - "and manual how to handle this." % \ + "and manual on how to handle this." % \ (folder, folder.getrepository(), - folder.getsaveduidvalidity(), folder.getuidvalidity())) + folder.getsaveduidvalidity(), folder.get_uidvalidity())) def loadmessagelist(self, repos, folder): self.logger.debug("Loading message list for %s[%s]" % ( From 297a6184419f86954761acda0a0a1674e2139938 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 11:50:19 +0100 Subject: [PATCH 431/817] Rename folder.getsaveduidvalidity(), folder.getuidvalidity() Modernize names to: folder.get_saveduidvalidity(), folder.get_uidvalidity() Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 10 +++++++--- offlineimap/ui/Machine.py | 2 +- offlineimap/ui/UIBase.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index cab22c2..689c94b 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -116,8 +116,8 @@ class BaseFolder(object): function is not threadsafe. So don't attempt to call it from concurrent threads.""" - if self.getsaveduidvalidity() != None: - return self.getsaveduidvalidity() == self.get_uidvalidity() + if self.get_saveduidvalidity() != None: + return self.get_saveduidvalidity() == self.get_uidvalidity() else: self.saveuidvalidity() return 1 @@ -126,7 +126,11 @@ class BaseFolder(object): return os.path.join(self.repository.getuiddir(), self.getfolderbasename()) - def getsaveduidvalidity(self): + def get_saveduidvalidity(self): + """Return the previously cached UIDVALIDITY value + + :returns: UIDVALIDITY as (long) number or None, if None had been + saved yet.""" if hasattr(self, '_base_saved_uidvalidity'): return self._base_saved_uidvalidity uidfilename = self._getuidfilename() diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 271629c..4bf1418 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -70,7 +70,7 @@ class MachineUI(UIBase): def validityproblem(s, folder): s._printData('validityproblem', "%s\n%s\n%s\n%s" % \ (folder.getname(), folder.getrepository().getname(), - folder.getsaveduidvalidity(), folder.get_uidvalidity())) + folder.get_saveduidvalidity(), folder.get_uidvalidity())) def connecting(s, hostname, port): s._printData('connecting', "%s\n%s" % (hostname, str(port))) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 1cdd94b..7cd8895 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -311,7 +311,7 @@ class UIBase(object): "(saved %d; got %d); skipping it. Please see FAQ " "and manual on how to handle this." % \ (folder, folder.getrepository(), - folder.getsaveduidvalidity(), folder.get_uidvalidity())) + folder.get_saveduidvalidity(), folder.get_uidvalidity())) def loadmessagelist(self, repos, folder): self.logger.debug("Loading message list for %s[%s]" % ( From eb616da60266470d869968c2b034049c7addcb2a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 11:51:53 +0100 Subject: [PATCH 432/817] Rename folder.isuidvalidityok() Use nicer name folder.check_uidvalidity() and improve code documentation Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 13 +++++++------ offlineimap/folder/Base.py | 9 ++++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index ad2015a..2c44208 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -398,16 +398,17 @@ def syncfolder(account, remotefolder, quick): localfolder.cachemessagelist() ui.messagelistloaded(localrepos, localfolder, localfolder.getmessagecount()) - # If either the local or the status folder has messages and there is a UID - # validity problem, warn and abort. If there are no messages, UW IMAPd - # loses UIDVALIDITY. But we don't really need it if both local folders are - # empty. So, in that case, just save it off. + # If either the local or the status folder has messages and + # there is a UID validity problem, warn and abort. If there are + # no messages, UW IMAPd loses UIDVALIDITY. But we don't really + # need it if both local folders are empty. So, in that case, + # just save it off. if localfolder.getmessagecount() or statusfolder.getmessagecount(): - if not localfolder.isuidvalidityok(): + if not localfolder.check_uidvalidity(): ui.validityproblem(localfolder) localrepos.restore_atime() return - if not remotefolder.isuidvalidityok(): + if not remotefolder.check_uidvalidity(): ui.validityproblem(remotefolder) localrepos.restore_atime() return diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 689c94b..ae654c8 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -109,18 +109,21 @@ class BaseFolder(object): basename = re.sub('(^|\/)\.$','\\1dot', basename) return basename - def isuidvalidityok(self): + def check_uidvalidity(self): """Tests if the cached UIDVALIDITY match the real current one If required it saves the UIDVALIDITY value. In this case the function is not threadsafe. So don't attempt to call it from - concurrent threads.""" + concurrent threads. + + :returns: Boolean indicating the match. Returns True in case it + implicitely saved the UIDVALIDITY.""" if self.get_saveduidvalidity() != None: return self.get_saveduidvalidity() == self.get_uidvalidity() else: self.saveuidvalidity() - return 1 + return True def _getuidfilename(self): return os.path.join(self.repository.getuiddir(), From d3c693be7da08450acfbe20a6b2ec591dceb7ed9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 11:53:48 +0100 Subject: [PATCH 433/817] Rename folder.saveuidvalidity() Simply rename to folder.save_uidvalidity() Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 5 +++-- offlineimap/folder/Base.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 2c44208..42cc22c 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -413,8 +413,9 @@ def syncfolder(account, remotefolder, quick): localrepos.restore_atime() return else: - localfolder.saveuidvalidity() - remotefolder.saveuidvalidity() + # Both folders empty, just save new UIDVALIDITY + localfolder.save_uidvalidity() + remotefolder.save_uidvalidity() # Load remote folder. ui.loadmessagelist(remoterepos, remotefolder) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index ae654c8..e852018 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -122,7 +122,7 @@ class BaseFolder(object): if self.get_saveduidvalidity() != None: return self.get_saveduidvalidity() == self.get_uidvalidity() else: - self.saveuidvalidity() + self.save_uidvalidity() return True def _getuidfilename(self): @@ -145,8 +145,8 @@ class BaseFolder(object): file.close() return self._base_saved_uidvalidity - def saveuidvalidity(self): - """Save the UID value of the folder to the status + def save_uidvalidity(self): + """Save the UIDVALIDITY value of the folder to the cache This function is not threadsafe, so don't attempt to call it from concurrent threads.""" From de1e03b06524e8055d40b449e8b113c1723f791f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 11:54:16 +0100 Subject: [PATCH 434/817] Tweak code documentation Minor comment changes Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 6 ++++-- offlineimap/folder/Base.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 42cc22c..81edbe3 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -379,8 +379,10 @@ def syncfolder(account, remotefolder, quick): replace(remoterepos.getsep(), statusrepos.getsep())) if localfolder.get_uidvalidity() == None: - # This is a new folder, so delete the status cache to be sure - # we don't have a conflict. + # This is a new folder, so delete the status cache to be + # sure we don't have a conflict. + # TODO: This does not work. We always return a value, need + # to rework this... statusfolder.deletemessagelist() statusfolder.cachemessagelist() diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index e852018..30dfac2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -126,6 +126,7 @@ class BaseFolder(object): return True def _getuidfilename(self): + """provides UIDVALIDITY cache filename for class internal purposes""" return os.path.join(self.repository.getuiddir(), self.getfolderbasename()) From 165cf5cb4f044142a608397c71c6ec5caca154c7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 11:55:06 +0100 Subject: [PATCH 435/817] Use modern "with open() as file" Saves a LOC and guarantees that the file will be closed Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 30dfac2..5e3d1e5 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -154,9 +154,8 @@ class BaseFolder(object): newval = self.get_uidvalidity() uidfilename = self._getuidfilename() - file = open(uidfilename + ".tmp", "wt") - file.write("%d\n" % newval) - file.close() + with open(uidfilename + ".tmp", "wt") as file: + file.write("%d\n" % newval) os.rename(uidfilename + ".tmp", uidfilename) self._base_saved_uidvalidity = newval From d2f85047573a4eab71caf3132173ece875c698c9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Jan 2012 11:24:16 +0100 Subject: [PATCH 436/817] Fix getuidvalidity crash (UIDVALIDITY returning None) Rename getuidvalidity -> get_uidvalidity and cache the IMAP result. 1) Start modernizing our function names using more underscores 2) IMAPs implementation of get_uidvalidity was removing the UIDVALIDITY result from the imaplib2 result stack, so subsequent calls would return "None". As various functions can invoke this, this led to some errors that we avoid by caching the current UIDVALIDITY value in the Folder instance. There are more simplifications and improvements to be made. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 1 + offlineimap/accounts.py | 2 +- offlineimap/folder/Base.py | 17 +++++++++++------ offlineimap/folder/IMAP.py | 22 ++++++++++++++-------- offlineimap/folder/Maildir.py | 6 ++++-- offlineimap/ui/Machine.py | 2 +- offlineimap/ui/UIBase.py | 4 ++-- 7 files changed, 34 insertions(+), 20 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 7f71597..2e84162 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -21,3 +21,4 @@ Bug Fixes * Fix python2.6 compatibility with the TTYUI backend (crash) * Fix TTYUI regression from 6.5.2 in refresh loop (crash) +* Fix crashes related to UIDVALIDITY returning "None" diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index fc2687f..ad2015a 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -378,7 +378,7 @@ def syncfolder(account, remotefolder, quick): statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ replace(remoterepos.getsep(), statusrepos.getsep())) - if localfolder.getuidvalidity() == None: + if localfolder.get_uidvalidity() == None: # This is a new folder, so delete the status cache to be sure # we don't have a conflict. statusfolder.deletemessagelist() diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 9ff60e2..cab22c2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -110,13 +110,14 @@ class BaseFolder(object): return basename def isuidvalidityok(self): - """Does the cached UID match the real UID + """Tests if the cached UIDVALIDITY match the real current one - If required it caches the UID. In this case the function is not - threadsafe. So don't attempt to call it from concurrent threads.""" + If required it saves the UIDVALIDITY value. In this case the + function is not threadsafe. So don't attempt to call it from + concurrent threads.""" if self.getsaveduidvalidity() != None: - return self.getsaveduidvalidity() == self.getuidvalidity() + return self.getsaveduidvalidity() == self.get_uidvalidity() else: self.saveuidvalidity() return 1 @@ -142,7 +143,7 @@ class BaseFolder(object): This function is not threadsafe, so don't attempt to call it from concurrent threads.""" - newval = self.getuidvalidity() + newval = self.get_uidvalidity() uidfilename = self._getuidfilename() file = open(uidfilename + ".tmp", "wt") @@ -151,7 +152,11 @@ class BaseFolder(object): os.rename(uidfilename + ".tmp", uidfilename) self._base_saved_uidvalidity = newval - def getuidvalidity(self): + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value + + This function needs to be implemented by each Backend + :returns: UIDVALIDITY as a (long) number""" raise NotImplementedException def cachemessagelist(self): diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 75c731c..ffbd6b1 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -65,17 +65,23 @@ class IMAPFolder(BaseFolder): def getcopyinstancelimit(self): return 'MSGCOPY_' + self.repository.getname() - def getuidvalidity(self): + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value + + UIDVALIDITY value will be cached on the first call. + :returns: The UIDVALIDITY as (long) number.""" + if hasattr(self, '_uidvalidity'): + # use cached value if existing + return self._uidvalidity imapobj = self.imapserver.acquireconnection() try: - # SELECT receives UIDVALIDITY response + # SELECT (if not already done) and get current UIDVALIDITY self.selectro(imapobj) - # note: we would want to use .response() here but that - # often seems to return [None], even though we have - # data. TODO - uidval = imapobj._get_untagged_response('UIDVALIDITY') - assert uidval != [None], "response('UIDVALIDITY') returned [None]!" - return long(uidval[-1]) + typ, uidval = imapobj.response('UIDVALIDITY') + assert uidval != [None] and uidval != None, \ + "response('UIDVALIDITY') returned [None]!" + self._uidvalidity = long(uidval[-1]) + return self._uidvalidity finally: self.imapserver.releaseconnection(imapobj) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 95258d9..fe85308 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -83,8 +83,10 @@ class MaildirFolder(BaseFolder): """Return the absolute file path to the Maildir folder (sans cur|new)""" return self._fullname - def getuidvalidity(self): - """Maildirs have no notion of uidvalidity, so we just return a magic + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value + + Maildirs have no notion of uidvalidity, so we just return a magic token.""" return 42 diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index ab57ea2..271629c 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -70,7 +70,7 @@ class MachineUI(UIBase): def validityproblem(s, folder): s._printData('validityproblem', "%s\n%s\n%s\n%s" % \ (folder.getname(), folder.getrepository().getname(), - folder.getsaveduidvalidity(), folder.getuidvalidity())) + folder.getsaveduidvalidity(), folder.get_uidvalidity())) def connecting(s, hostname, port): s._printData('connecting', "%s\n%s" % (hostname, str(port))) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 66ecd4f..1cdd94b 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -309,9 +309,9 @@ class UIBase(object): def validityproblem(self, folder): self.logger.warning("UID validity problem for folder %s (repo %s) " "(saved %d; got %d); skipping it. Please see FAQ " - "and manual how to handle this." % \ + "and manual on how to handle this." % \ (folder, folder.getrepository(), - folder.getsaveduidvalidity(), folder.getuidvalidity())) + folder.getsaveduidvalidity(), folder.get_uidvalidity())) def loadmessagelist(self, repos, folder): self.logger.debug("Loading message list for %s[%s]" % ( From bf31accb78e300aa5e5832c8f744bf08e3f58dfd Mon Sep 17 00:00:00 2001 From: Daniel Shahaf Date: Thu, 19 Jan 2012 11:55:11 +0200 Subject: [PATCH 437/817] We have a (read-only) commit mailing list Sebastian Spaeth wrote on Thu, Jan 19, 2012 at 10:28:19 +0100: > Forgot to Cc the list... > > On Wed, 18 Jan 2012 08:17:44 +0200, Daniel Shahaf wrote: > > Sebastian Spaeth wrote on Wed, Jan 18, 2012 at 01:02:22 +0100: > >> http://docs.offlineimap.org > > > > I've subscribed. And if you tell me where the sources for docs.o.o are, > > I'll send a patch for them, too. :-) > > They are autogenerated from docs/dev-docs-src, and > docs/[INSTALL|MANUAL|FAQ].rst respectively. Patches are welcome... > > Sebastian From 84fcb9fa5de9eb2f95d588a7403d9c6d13f0c7f0 Mon Sep 17 00:00:00 2001 From: Daniel Shahaf Date: Thu, 19 Jan 2012 11:53:13 +0200 Subject: [PATCH] Document the commits@ list Mention the commits@ list in SubmittingPatches.rst. Mention SubmittingPatches.rst in the API part of the documentation. Signed-off-by: Daniel Shahaf --- SubmittingPatches.rst | 7 +++++++ docs/dev-doc-src/API.rst | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/SubmittingPatches.rst b/SubmittingPatches.rst index c200087..5161519 100644 --- a/SubmittingPatches.rst +++ b/SubmittingPatches.rst @@ -1,6 +1,7 @@ .. -*- coding: utf-8 -*- .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project +.. _commits mailing list: http://lists.offlineimap.org/listinfo.cgi/commits-offlineimap.org ================================================= Checklist (and a short version for the impatient) @@ -320,6 +321,12 @@ Know the status of your patch after submission tell you if your patch is merged in pu if you rebase on top of master). +* You can follow upstream commits on +`CIA.vc `, +`Ohloh `, +`GitHub `, +or on the `commits mailing list`_. + .. * Read the git mailing list, the maintainer regularly posts messages entitled "What's cooking in git.git" and "What's in git.git" giving the status of various proposed changes. diff --git a/docs/dev-doc-src/API.rst b/docs/dev-doc-src/API.rst index c901938..20f3d1f 100644 --- a/docs/dev-doc-src/API.rst +++ b/docs/dev-doc-src/API.rst @@ -13,6 +13,11 @@ OfflineImap can be imported as:: from offlineimap import OfflineImap +The file ``SubmittingPatches.rst`` in the source distribution documents a +number of resources and conventions you may find useful. It will eventually +be merged into the main documentation. +.. TODO: merge SubmittingPatches.rst to the main documentation + :mod:`offlineimap` -- The OfflineImap module ============================================= From c7df907cf69aad5b89b24516e398da44d0d63b93 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 11:01:27 +0100 Subject: [PATCH 438/817] Make folders containing quotes work imaputil.imapsplit did not cope with strings that contained encoded quotation marks, e.g. a folder name '"Make" Magazine' would fail and crash OfflineImap. Make it work by adapting the regex that we use to extract the first quote to also work with encoded \" quotes. (We do no sanity checks that there is an even number of such marks within a string though) This commit makes such folders work. This was reported and analyzed by Mark Eichin. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/imaputil.py | 22 +++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 2e84162..cfbd1b9 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,9 @@ others. New Features ------------ +* Make folders containing quotes work rather than crashing + (reported by Mark Eichin) + Changes ------- diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index f583312..ffe16fa 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -25,7 +25,12 @@ try: # python 2.6 has set() built in except NameError: from sets import Set as set -quotere = re.compile('^("(?:[^"]|\\\\")*")') +# find the first quote in a string +quotere = re.compile( + r"""(?P"(?:\\"|[^"])*") # Quote, possibly containing encoded + # quotation mark + \s*(?P.*)$ # Whitespace & remainder of string""", + re.VERBOSE) def debug(*args): msg = [] @@ -71,7 +76,7 @@ def options2hash(list): def flags2hash(flags): """Converts IMAP response string from eg IMAP4.fetch() to a hash. - + E.g. '(FLAGS (\\Seen Old) UID 4807)' leads to {'FLAGS': '(\\Seen Old)', 'UID': '4807'}""" return options2hash(flagsplit(flags)) @@ -124,10 +129,11 @@ def imapsplit(imapstring): retval.extend(imapsplit(arg)) debug("imapsplit() non-string: returning %s" % str(retval)) return retval - + workstr = imapstring.strip() retval = [] while len(workstr): + # handle parenthized fragments (...()...) if workstr[0] == '(': rparenc = 1 # count of right parenthesis to match rpareni = 1 # position to examine @@ -141,9 +147,10 @@ def imapsplit(imapstring): workstr = workstr[rpareni:].lstrip() retval.append(parenlist) elif workstr[0] == '"': - quotelist = quotere.search(workstr).group(1) - workstr = workstr[len(quotelist):].lstrip() - retval.append(quotelist) + # quoted fragments '"...\"..."' + m = quotere.match(workstr) + retval.append(m.group('quote')) + workstr = m.group('rest') else: splits = string.split(workstr, maxsplit = 1) splitslen = len(splits) @@ -161,8 +168,9 @@ def imapsplit(imapstring): elif splitslen == 0: # There was not even an unquoted word. break + getglobalui().warn("%s->%s" % (imapstring, retval)) return retval - + flagmap = [('\\Seen', 'S'), ('\\Answered', 'R'), ('\\Flagged', 'F'), From eb04036da5492be137f075f308d52feda1934343 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 11:01:27 +0100 Subject: [PATCH 439/817] Make folders containing quotes work imaputil.imapsplit did not cope with strings that contained encoded quotation marks, e.g. a folder name '"Make" Magazine' would fail and crash OfflineImap. Make it work by adapting the regex that we use to extract the first quote to also work with encoded \" quotes. (We do no sanity checks that there is an even number of such marks within a string though) This commit makes such folders work. This was reported and analyzed by Mark Eichin. Signed-off-by: Sebastian Spaeth Conflicts: Changelog.draft.rst --- Changelog.draft.rst | 3 +++ offlineimap/imaputil.py | 22 +++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 93f64f1..9316218 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,9 @@ New Features * Beginning of a test suite. So far there is only one test. Configure test/credentials.conf and invoke with "python setup.py test" +* Make folders containing quotes work rather than crashing + (reported by Mark Eichin) + Changes ------- diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index f583312..ffe16fa 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -25,7 +25,12 @@ try: # python 2.6 has set() built in except NameError: from sets import Set as set -quotere = re.compile('^("(?:[^"]|\\\\")*")') +# find the first quote in a string +quotere = re.compile( + r"""(?P"(?:\\"|[^"])*") # Quote, possibly containing encoded + # quotation mark + \s*(?P.*)$ # Whitespace & remainder of string""", + re.VERBOSE) def debug(*args): msg = [] @@ -71,7 +76,7 @@ def options2hash(list): def flags2hash(flags): """Converts IMAP response string from eg IMAP4.fetch() to a hash. - + E.g. '(FLAGS (\\Seen Old) UID 4807)' leads to {'FLAGS': '(\\Seen Old)', 'UID': '4807'}""" return options2hash(flagsplit(flags)) @@ -124,10 +129,11 @@ def imapsplit(imapstring): retval.extend(imapsplit(arg)) debug("imapsplit() non-string: returning %s" % str(retval)) return retval - + workstr = imapstring.strip() retval = [] while len(workstr): + # handle parenthized fragments (...()...) if workstr[0] == '(': rparenc = 1 # count of right parenthesis to match rpareni = 1 # position to examine @@ -141,9 +147,10 @@ def imapsplit(imapstring): workstr = workstr[rpareni:].lstrip() retval.append(parenlist) elif workstr[0] == '"': - quotelist = quotere.search(workstr).group(1) - workstr = workstr[len(quotelist):].lstrip() - retval.append(quotelist) + # quoted fragments '"...\"..."' + m = quotere.match(workstr) + retval.append(m.group('quote')) + workstr = m.group('rest') else: splits = string.split(workstr, maxsplit = 1) splitslen = len(splits) @@ -161,8 +168,9 @@ def imapsplit(imapstring): elif splitslen == 0: # There was not even an unquoted word. break + getglobalui().warn("%s->%s" % (imapstring, retval)) return retval - + flagmap = [('\\Seen', 'S'), ('\\Answered', 'R'), ('\\Flagged', 'F'), From fae48f309e67e96cb00d983a7651b5e047a6e009 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 11:10:42 +0100 Subject: [PATCH 440/817] Release 6.2.1-rc1 See Changelog.rst for details Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 12 ------------ Changelog.rst | 17 +++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 9316218..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,20 +13,8 @@ others. New Features ------------ -* Beginning of a test suite. So far there is only one test. Configure - test/credentials.conf and invoke with "python setup.py test" - -* Make folders containing quotes work rather than crashing - (reported by Mark Eichin) - Changes ------- -* Improve delete msg performance with SQLITE backend - Bug Fixes --------- - -* Fix python2.6 compatibility with the TTYUI backend (crash) -* Fix TTYUI regression from 6.5.2 in refresh loop (crash) -* Fix crashes related to UIDVALIDITY returning "None" diff --git a/Changelog.rst b/Changelog.rst index 53a8a0c..137fcbb 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,23 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.5.2.1-rc1 (2012-01-20) +===================================== + +* Fix python2.6 compatibility with the TTYUI backend (crash) + +* Fix TTYUI regression from 6.5.2 in refresh loop (crash) + +* Fix crashes related to UIDVALIDITY returning "None" + +* Beginning of a test suite. So far there is only one test. Configure + test/credentials.conf and invoke with "python setup.py test" + +* Make folders containing quotes work rather than crashing + (reported by Mark Eichin) + +* Improve delete msg performance with SQLITE backend + OfflineIMAP v6.5.2 (2012-01-17) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 6b7b404..ccdf898 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.2" +__version__ = "6.5.2.1-rc1" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 09ef5ab042b37c7fe59365e3e808a59605384b5d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 11:31:41 +0100 Subject: [PATCH 441/817] Remove stray debug line This line was accidentally left laying around. Signed-off-by: Sebastian Spaeth --- offlineimap/imaputil.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index ffe16fa..3955683 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -168,7 +168,6 @@ def imapsplit(imapstring): elif splitslen == 0: # There was not even an unquoted word. break - getglobalui().warn("%s->%s" % (imapstring, retval)) return retval flagmap = [('\\Seen', 'S'), From 36dd46abfc35ca435600c307f09f84ebb9b3add5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 11:33:30 +0100 Subject: [PATCH 442/817] Fix warn() output for MachineUI We used "self" but the paramter was called "s". Fixes a crash when we ui.warn() (only when using the MachineUI). Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 4bf1418..a87395d 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -45,7 +45,7 @@ class MachineUI(UIBase): def _msg(s, msg): s._printData('_display', msg) - def warn(s, msg, minor = 0): + def warn(self, msg, minor = 0): # TODO, remove and cleanup the unused minor stuff self.logger.warning("%s:%s:%s:%s" % ( 'warn', '', currentThread().getName(), msg)) From 4d47f7bf3c7e0af56e0bc1f74c3afe6c762beb3e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 14:33:16 +0100 Subject: [PATCH 443/817] Clean up and improve APPENDUID handling We were not cleaning out possibly existing APPENDUID messages before APPENDing a new message. In case an old message were still hanging around, this could *possibly* lead to retrieving and old UID. Things should have been fine, but we do want to play safe here. Also, make use of the "official" imaplib2 .response() command rather than the internal _get_untagged_response() function. Remove the hack that we would be looking for APPENDUID responses even if the server claimed not to support the UIDPLUS ext. We now poll server CAPABILITIES after login, and Gmail does provide us with the UIDPLUS capability after login. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 49 +++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index ffbd6b1..6f4cf92 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -538,12 +538,13 @@ class IMAPFolder(BaseFolder): self.ui.msgtoreadonly(self, uid, content, flags) return uid - #Do the APPEND + # Clean out existing APPENDUID responses and do APPEND try: - (typ, dat) = imapobj.append(self.getfullname(), + imapobj.response('APPENDUID') # flush APPENDUID responses + typ, dat = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) - retry_left = 0 # Mark as success + retry_left = 0 # Mark as success except imapobj.abort, e: # connection has been reset, release connection and retry. retry_left -= 1 @@ -568,41 +569,35 @@ class IMAPFolder(BaseFolder): OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. - (typ,dat) = imapobj.check() + typ, dat = imapobj.check() assert(typ == 'OK') - # get the new UID. Test for APPENDUID response even if the - # server claims to not support it, as e.g. Gmail does :-( - if use_uidplus or imapobj._get_untagged_response('APPENDUID', True): + # get the new UID, default to 0 (=unknown) + uid = 0 + if use_uidplus: # get new UID from the APPENDUID response, it could look # like OK [APPENDUID 38505 3955] APPEND completed with - # 38505 bein folder UIDvalidity and 3955 the new UID. - # note: we would want to use .response() here but that - # often seems to return [None], even though we have - # data. TODO - resp = imapobj._get_untagged_response('APPENDUID') - if resp == [None]: + # 38505 being folder UIDvalidity and 3955 the new UID. + typ, resp = imapobj.response('APPENDUID') + if resp == [None] or resp == None: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " "appending a message.") - return 0 - uid = long(resp[-1].split(' ')[1]) + else: + uid = long(resp[-1].split(' ')[1]) else: - # we don't support UIDPLUS + # Don't support UIDPLUS uid = self.savemessage_searchforheader(imapobj, headername, headervalue) - # See docs for savemessage in Base.py for explanation of this and other return values + # If everything failed up to here, search the message + # manually TODO: rather than inserting and searching for our + # custom header, we should be searching the Message-ID and + # compare the message size... if uid == 0: - self.ui.debug('imap', 'savemessage: first attempt to get new UID failed. \ - Going to run a NOOP and try again.') - assert(imapobj.noop()[0] == 'OK') - uid = self.savemessage_searchforheader(imapobj, headername, - headervalue) - if uid == 0: - self.ui.debug('imap', 'savemessage: second attempt to get new UID failed. \ - Going to try search headers manually') - uid = self.savemessage_fetchheaders(imapobj, headername, headervalue) - + self.ui.debug('imap', 'savemessage: attempt to get new UID ' + 'UID failed. Search headers manually.') + uid = self.savemessage_fetchheaders(imapobj, headername, + headervalue) finally: self.imapserver.releaseconnection(imapobj) From 47390e03d6a07b2ab73b65b74b668b7ce0663f57 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 14:39:53 +0100 Subject: [PATCH 444/817] Don't CHECK imapserver after each APPEND Most servers support the UIDPLUS extension, and we don't have to search headers after each uploaded message. There is no need to CHECK the imap server after each message when there is no need to search headers. I have not measured the performance impact on real world servers, but this lets us do less unneeded work in the common case. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/folder/IMAP.py | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index cfbd1b9..679ccef 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,6 +19,9 @@ New Features Changes ------- +* Slight performance enhancement uploading mails to an IMAP server in the + common case. + Bug Fixes --------- diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 6f4cf92..3dbcaa8 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -567,10 +567,6 @@ class IMAPFolder(BaseFolder): "failed (error). Server reponded: %s\nMessage content was: " "%s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) - # Checkpoint. Let it write out stuff, etc. Eg searches for - # just uploaded messages won't work if we don't do this. - typ, dat = imapobj.check() - assert(typ == 'OK') # get the new UID, default to 0 (=unknown) uid = 0 @@ -587,6 +583,11 @@ class IMAPFolder(BaseFolder): else: # Don't support UIDPLUS + # Checkpoint. Let it write out stuff, etc. Eg searches for + # just uploaded messages won't work if we don't do this. + typ, dat = imapobj.check() + assert(typ == 'OK') + uid = self.savemessage_searchforheader(imapobj, headername, headervalue) # If everything failed up to here, search the message From 58450fb22e731ffd78746c9287cae3fe5b9a4768 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 14:53:18 +0100 Subject: [PATCH 445/817] Enforce "Basic" UI when using --info switch Blinkenlights does not work well (at all) when using the --info switch. All we really want here is an output that can be pasted as debugging information. Enforce the usage of the "Basic" ui, when the --info switch is used. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 1 + offlineimap/init.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 679ccef..23839ed 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -21,6 +21,7 @@ Changes * Slight performance enhancement uploading mails to an IMAP server in the common case. +* Enforce basic UI when using the --info switch Bug Fixes --------- diff --git a/offlineimap/init.py b/offlineimap/init.py index 4dc01aa..5cd1894 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -200,6 +200,7 @@ class OfflineImap: # TODO, make use of chosen ui for logging logging.warning('Using old interface name, consider using one ' 'of %s' % ', '.join(UI_LIST.keys())) + if options.diagnostics: ui_type = 'basic' # enforce basic UI for --info try: # create the ui class self.ui = UI_LIST[ui_type.lower()](config) From 314a4e3b2c4a01668b3de9abfad06537e0a9857d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Jan 2012 15:55:44 +0100 Subject: [PATCH 446/817] Improve test suite and add test 1) Add helper functions to create and count maildirs and mails. 2) Add a second test that creates 2 maildirs, one of the including a quotation sign " in its folder name and sync. Signed-off-by: Sebastian Spaeth --- setup.py | 4 ++-- test/OLItest/TestRunner.py | 34 ++++++++++++++++++++++++++++++---- test/OLItest/__init__.py | 6 +++--- test/OLItest/globals.py | 2 +- test/tests/test_01_basic.py | 17 ++++++++++------- 5 files changed, 46 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index e684cda..3d04af2 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ import os from distutils.core import setup, Command import offlineimap import logging -from test.OLItest import OLITextTestRunner, TestLoader, OLITestLib +from test.OLItest import TextTestRunner, TestLoader, OLITestLib class TestCommand(Command): """runs the OLI testsuite""" @@ -44,7 +44,7 @@ class TestCommand(Command): OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') suite = TestLoader().discover('./test/tests') #TODO: failfast does not seem to exist in python2.6? - OLITextTestRunner(verbosity=2,failfast=True).run(suite) + TextTestRunner(verbosity=2,failfast=True).run(suite) setup(name = "offlineimap", diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 0d0730b..183f67a 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -99,8 +99,34 @@ class OLITestLib(): return (e.returncode, e.output) return (0, output) -class OLITextTestRunner(unittest.TextTestRunner): + @classmethod + def create_maildir(cls, folder): + """Create empty maildir 'folder' in our test maildir - def __init__(self,*args, **kwargs): - logging.warning("OfflineImap testsuite") - return super(OLITextTestRunner, self).__init__(*args, **kwargs) + Does not fail if it already exists""" + assert cls.testdir != None + maildir = os.path.join(cls.testdir, 'mail', folder) + for subdir in ('','tmp','cur','new'): + try: + os.mkdir(os.path.join(maildir, subdir)) + except OSError as e: + if e.errno != 17: # 'already exists' is ok. + raise + + @classmethod + def count_maildir_mails(cls, folder): + """Returns the number of mails in maildir 'folder' + + Counting only those in cur&new (ignoring tmp).""" + assert cls.testdir != None + maildir = os.path.join(cls.testdir, 'mail', folder) + + boxes, mails = 0, 0 + for dirpath, dirs, files in os.walk(maildir, False): + if set(dirs) == set(['cur', 'new', 'tmp']): + # New maildir folder + boxes += 1 + #raise RuntimeError("%s is not Maildir" % maildir) + if dirpath.endswith(('/cur', '/new')): + mails += len(files) + return boxes, mails diff --git a/test/OLItest/__init__.py b/test/OLItest/__init__.py index c9bda69..2f85210 100644 --- a/test/OLItest/__init__.py +++ b/test/OLItest/__init__.py @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -__all__ = ['OLITestLib', 'OLITextTestRunner','TestLoader'] +__all__ = ['OLITestLib', 'TextTestRunner','TestLoader'] __productname__ = 'OfflineIMAP Test suite' __version__ = '0' @@ -29,6 +29,6 @@ banner = """%(__productname__)s %(__version__)s %(__license__)s""" % locals() import unittest -from unittest import TestLoader +from unittest import TestLoader, TextTestRunner from globals import default_conf -from TestRunner import OLITestLib, OLITextTestRunner +from TestRunner import OLITestLib diff --git a/test/OLItest/globals.py b/test/OLItest/globals.py index 4009b9b..5d1f122 100644 --- a/test/OLItest/globals.py +++ b/test/OLItest/globals.py @@ -33,5 +33,5 @@ localfolders = [Repository IMAP] type=IMAP -folderfilter= lambda f: f.startswith('OLItest') +folderfilter= lambda f: f.startswith('INBOX.OLItest') """) diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index cb6d293..a0697ec 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -55,17 +55,20 @@ class TestBasicFunctions(unittest.TestCase): It syncs all "OLItest* (specified in the default config) to our local Maildir at keeps it there.""" code, res = OLITestLib.run_OLI() - #logging.warn("%s %s "% (code, res)) self.assertEqual(res, "") - #TODO implement OLITestLib.countmails() - logging.warn("synced %d boxes, %d mails" % (0,0)) - def test_02_wipedir(self): - """Wipe OLItest* maildir, sync and see if it's still empty + boxes, mails = OLITestLib.count_maildir_mails('') + logging.warn("%d boxes and %d mails" % (boxes, mails)) - Wipes all "OLItest* (specified in the default config) to our - local Maildir at keeps it there.""" + def test_02_createdir(self): + """Create local OLItest 1 & OLItest "1" maildir, sync + Folder names with quotes used to fail and have been fixed, so + one is included here as a small challenge.""" + OLITestLib.create_maildir('INBOX.OLItest 1') + OLITestLib.create_maildir('INBOX.OLItest "1"') code, res = OLITestLib.run_OLI() #logging.warn("%s %s "% (code, res)) self.assertEqual(res, "") + boxes, mails = OLITestLib.count_maildir_mails('') + logging.warn("%d boxes and %d mails" % (boxes, mails)) From e351ab736c9771d3ede38699c13c9118a3c78907 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 4 Feb 2012 20:35:31 +0100 Subject: [PATCH 447/817] Prettify default offlineimap.conf a bit Comment out all values that are the default values and make it a bit more consistent in general. We should have a man page for those values, really. Signed-off-by: Sebastian Spaeth --- offlineimap.conf | 62 +++++++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index c7f7d9d..66dd284 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -1,27 +1,11 @@ -# Sample configuration file -# Copyright (C) 2002-2011 John Goerzen & contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - +# Offlineimap sample configuration file # This file documents all possible options and can be quite scary. # Looking for a quick start? Take a look at offlineimap.conf.minimal. -# Settings support interpolation. This means values can contain python -# format strings which refer to other values in the same section, or -# values in a special DEFAULT section. This allows you for example to -# use common settings for multiple accounts: +# Settings generally support interpolation. This means values can +# contain python format strings which refer to other values in the same +# section, or values in a special DEFAULT section. This allows you for +# example to use common settings for multiple accounts: # # [Repository Gmail1] # trashfolder: %(gmailtrashfolder)s @@ -33,7 +17,6 @@ # gmailtrashfolder = [Google Mail]/Papierkorb # # would set the trashfolder setting for your German gmail accounts. - ################################################## @@ -45,13 +28,12 @@ # This specifies where offlineimap is to store its metadata. # This directory will be created if it does not already exist. -metadata = ~/.offlineimap +#metadata = ~/.offlineimap # This variable specifies which accounts are defined. Separate them # with commas. Account names should be alphanumeric only. # You will need to specify one section per account below. You may # not use "general" for an account name. -# accounts = Test @@ -65,7 +47,7 @@ accounts = Test # since any given sync run never "finishes" due to a timer, you will never # sync your additional accounts if this is 1. -maxsyncaccounts = 1 +#maxsyncaccounts = 1 # You can specify one or more user interface modules for OfflineIMAP # to use. OfflineIMAP will try the first in the list, and if it @@ -82,7 +64,7 @@ maxsyncaccounts = 1 # # You can override this with a command-line option -u. -ui = Blinkenlights +#ui = basic # If you try to synchronize messages to a folder which the IMAP server # considers read-only, OfflineIMAP will generate a warning. If you want @@ -92,7 +74,8 @@ ui = Blinkenlights # will prevent OfflineIMAP from propagating those changes to the IMAP # server. Note that ignore-readonly is unrelated to the "readonly" # setting which prevents a repository from being modified at all. -ignore-readonly = no + +#ignore-readonly = no ########## Advanced settings @@ -334,8 +317,9 @@ ssl = yes # fingerprint here. OfflineImap will verify that the server fingerprint # has not changed on each connect and refuse to connect otherwise. # You can also configure this in addition to CA certificate validation -# above and it will check both ways. cert_fingerprint = -# +# above and it will check both ways. + +#cert_fingerprint = # Specify the port. If not specified, use a default port. # remoteport = 993 @@ -421,7 +405,7 @@ remoteuser = username # cases, it may slow things down. The safe answer is 1. You should # probably never set it to a value more than 5. -maxconnections = 2 +#maxconnections = 2 # OfflineIMAP normally closes IMAP server connections between refreshes if # the global option autorefresh is specified. If you wish it to keep the @@ -429,15 +413,15 @@ maxconnections = 2 # false. Keeping the connection open means a faster sync start the # next time and may use fewer server resources on connection, but uses # more server memory. This setting has no effect if autorefresh is not set. - -holdconnectionopen = no +# +#holdconnectionopen = no # If you want to have "keepalives" sent while waiting between syncs, # specify the amount of time IN SECONDS between keepalives here. Note that # sometimes more than this amount of time might pass, so don't make it # tight. This setting has no effect if autorefresh and holdconnectionopen # are not both set. - +# # keepalive = 60 # Normally, OfflineIMAP will expunge deleted messages from the server. @@ -447,11 +431,12 @@ holdconnectionopen = no # setting; otherwise, the messgaes will just pile up there forever. # Therefore, this setting is definitely NOT recommended. # -# expunge = no +#expunge = no # Specify whether to process all mail folders on the server, or only # those listed as "subscribed". -subscribedonly = no +# +#subscribedonly = no # You can specify a folder translator. This must be a eval-able # Python expression that takes a foldername arg and returns the new @@ -525,7 +510,8 @@ subscribedonly = no # repository will not be modified during synchronization. Use to # e.g. backup an IMAP server. The readonly setting can be applied to any # type of Repository (Maildir, Imap, etc). -readonly = False +# +#readonly = False [Repository GmailExample] @@ -539,7 +525,6 @@ readonly = False # `remoteport`, `tunnel` and `ssl`. (See # http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814) # Any attempt to set those parameters will be silently ignored. -# type = Gmail @@ -571,4 +556,5 @@ remoteuser = username@gmail.com # spamfolder = [Google Mail]/Spam # Enable 1-way synchronization. See above for explanation. -readonly = False +# +#readonly = False From 51728ed815aa2ae399e80dcb85d8b6af619fafcb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 4 Feb 2012 21:08:44 +0100 Subject: [PATCH 448/817] Remove Gmail realdelete option It can lead to potential dataloss (see recent commit log where I added a scary warning about it to offlineimap.conf). Signed-off-by: Sebastian Spaeth --- offlineimap.conf | 15 ------------ offlineimap/folder/Gmail.py | 41 ++++++++------------------------- offlineimap/repository/Gmail.py | 5 ---- 3 files changed, 9 insertions(+), 52 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 66dd284..2656ef6 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -531,21 +531,6 @@ type = Gmail # Specify the Gmail user name. This is the only mandatory parameter. remoteuser = username@gmail.com -# WARNING: READ THIS BEFORE CONSIDERING TO CHANGE IT! Deleting a -# message from a Gmail folder via the IMAP interface will just remove -# that folder's label from the message: the message will continue to -# exist in the '[Gmail]/All Mail' folder. If `realdelete` is set to -# `True`, then deleted messages will be moved to the '[Gmail]/Trash' -# folder. BEWARE: this will immediately delete a messages from *all -# folders* it belongs to! AS OFFLINEIMAP IMPLEMENTS FOLDER MOVES AS 1) -# AN ADD and 2) A DELETE (the order can vary), THIS MEANS THAT A FOLDER -# MOVE CAN CAUSE DATALOSS. DO NOT USE IT AND MOVE MAIL TO -# "[Gmail]/Trash" TO DELETE MAIL FROM "[Gmail]/All Mail"! See the -# analysis at -# http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 See -# http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 -# realdelete = no !!!READ ABOVE BEFORE USING - # The trash folder name may be different from [Gmail]/Trash # for example on german googlemail, this setting should be # diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 8d9c0bc..5d11119 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -18,16 +18,18 @@ """Folder implementation to support features of the Gmail IMAP server. """ - from IMAP import IMAPFolder -from offlineimap import imaputil - class GmailFolder(IMAPFolder): """Folder implementation to support features of the Gmail IMAP server. - Specifically, deleted messages are moved to folder `Gmail.TRASH_FOLDER` - (by default: ``[Gmail]/Trash``) prior to expunging them, since - Gmail maps to IMAP ``EXPUNGE`` command to "remove label". + + Removing a message from a folder will only remove the "label" from + the message and keep it in the "All mails" folder. To really delete + a message it needs to be copied to the Trash folder. However, this + is dangerous as our folder moves are implemented as a 1) delete in + one folder and 2) append to the other. If 2 comes before 1, this + will effectively delete the message from all folders. So we cannot + do that until we have a smarter folder move mechanism. For more information on the Gmail IMAP server: http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 @@ -35,31 +37,6 @@ class GmailFolder(IMAPFolder): def __init__(self, imapserver, name, repository): super(GmailFolder, self).__init__(imapserver, name, repository) - self.realdelete = repository.getrealdelete(name) self.trash_folder = repository.gettrashfolder(name) - #: Gmail will really delete messages upon EXPUNGE in these folders + # Gmail will really delete messages upon EXPUNGE in these folders self.real_delete_folders = [ self.trash_folder, repository.getspamfolder() ] - - def deletemessages_noconvert(self, uidlist): - uidlist = [uid for uid in uidlist if uid in self.messagelist] - if not len(uidlist): - return - - if self.realdelete and not (self.getname() in self.real_delete_folders): - # IMAP expunge is just "remove label" in this folder, - # so map the request into a "move into Trash" - - imapobj = self.imapserver.acquireconnection() - try: - imapobj.select(self.getfullname()) - result = imapobj.uid('copy', - imaputil.uid_sequence(uidlist), - self.trash_folder) - assert result[0] == 'OK', \ - "Bad IMAPlib result: %s" % result[0] - finally: - self.imapserver.releaseconnection(imapobj) - for uid in uidlist: - del self.messagelist[uid] - else: - IMAPFolder.deletemessages_noconvert(self, uidlist) diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index ada2146..f4260c0 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -64,11 +64,6 @@ class GmailRepository(IMAPRepository): def getfoldertype(self): return folder.Gmail.GmailFolder - def getrealdelete(self, foldername): - # XXX: `foldername` is currently ignored - the `realdelete` - # setting is repository-wide - return self.getconfboolean('realdelete', 0) - def gettrashfolder(self, foldername): #: Where deleted mail should be moved return self.getconf('trashfolder','[Gmail]/Trash') From a242b985bfd6e1c7849834b5da60bfd7f57b204a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 4 Feb 2012 21:11:44 +0100 Subject: [PATCH 449/817] Release v6.5.2.1 See changelog for details Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 11 ----------- Changelog.rst | 7 ++++++- offlineimap/__init__.py | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 23839ed..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,19 +13,8 @@ others. New Features ------------ -* Make folders containing quotes work rather than crashing - (reported by Mark Eichin) - Changes ------- -* Slight performance enhancement uploading mails to an IMAP server in the - common case. -* Enforce basic UI when using the --info switch - Bug Fixes --------- - -* Fix python2.6 compatibility with the TTYUI backend (crash) -* Fix TTYUI regression from 6.5.2 in refresh loop (crash) -* Fix crashes related to UIDVALIDITY returning "None" diff --git a/Changelog.rst b/Changelog.rst index 137fcbb..c0fd517 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,7 +11,7 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. -OfflineIMAP v6.5.2.1-rc1 (2012-01-20) +OfflineIMAP v6.5.2.1 (2012-04-04) ===================================== * Fix python2.6 compatibility with the TTYUI backend (crash) @@ -28,6 +28,11 @@ OfflineIMAP v6.5.2.1-rc1 (2012-01-20) * Improve delete msg performance with SQLITE backend +* Enforce basic UI when using the --info switch + +* Remove the Gmail "realdelete" option, as it could lead to potential + data loss. + OfflineIMAP v6.5.2 (2012-01-17) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index ccdf898..0dc0943 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.2.1-rc1" +__version__ = "6.5.2.1" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 7f22d89872f1a47654d69fb7658e67b663826893 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 6 Feb 2012 17:40:02 +0100 Subject: [PATCH 450/817] fix type: logging.info --> logging.INFO We mean the (numeric) logging level here and not the info() function. logger.isEnabledFor() takes the logging level as argument, obviously. This was a stupid typo that failed under python3. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 7cd8895..06dbd01 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -269,7 +269,7 @@ class UIBase(object): def connecting(self, hostname, port): """Log 'Establishing connection to'""" - if not self.logger.isEnabledFor(logging.info): return + if not self.logger.isEnabledFor(logging.INFO): return displaystr = '' hostname = hostname if hostname else '' port = "%s" % port if port else '' From 0844d27f9f417c9ea3be09117566077d8fb08508 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 10:14:23 +0100 Subject: [PATCH 451/817] except Ex, e: --> except Ex as e: Nudge us towards python3 compatability by converting deprecated python2 syntax. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 10 +++++----- offlineimap/folder/Base.py | 8 ++++---- offlineimap/folder/IMAP.py | 10 +++++----- offlineimap/folder/LocalStatus.py | 2 +- offlineimap/folder/Maildir.py | 4 ++-- offlineimap/imaplibutil.py | 2 +- offlineimap/imapserver.py | 12 ++++++------ offlineimap/init.py | 2 +- offlineimap/repository/Base.py | 4 ++-- offlineimap/repository/IMAP.py | 12 ++++++------ offlineimap/repository/Maildir.py | 4 ++-- offlineimap/threadutil.py | 2 +- 12 files changed, 36 insertions(+), 36 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 81edbe3..780ad04 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -231,7 +231,7 @@ class SyncableAccount(Account): self.sync() except (KeyboardInterrupt, SystemExit): raise - except OfflineImapError, e: + except OfflineImapError as e: # Stop looping and bubble up Exception if needed. if e.severity >= OfflineImapError.ERROR.REPO: if looping: @@ -239,7 +239,7 @@ class SyncableAccount(Account): if e.severity >= OfflineImapError.ERROR.CRITICAL: raise self.ui.error(e, exc_info()[2]) - except Exception, e: + except Exception as e: self.ui.error(e, exc_info()[2], msg = "While attempting to sync" " account '%s'" % self) else: @@ -344,7 +344,7 @@ class SyncableAccount(Account): self.ui.callhook("Hook return code: %d" % p.returncode) except (KeyboardInterrupt, SystemExit): raise - except Exception, e: + except Exception as e: self.ui.error(e, exc_info()[2], msg = "Calling hook") def syncfolder(account, remotefolder, quick): @@ -445,7 +445,7 @@ def syncfolder(account, remotefolder, quick): localrepos.restore_atime() except (KeyboardInterrupt, SystemExit): raise - except OfflineImapError, e: + except OfflineImapError as e: # bubble up severe Errors, skip folder otherwise if e.severity > OfflineImapError.ERROR.FOLDER: raise @@ -459,7 +459,7 @@ def syncfolder(account, remotefolder, quick): # we reconstruct foldername above rather than using # localfolder, as the localfolder var is not # available if assignment fails. - except Exception, e: + except Exception as e: ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ (account, remotefolder.getvisiblename(), traceback.format_exc())) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 5e3d1e5..c9d86c7 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -321,11 +321,11 @@ class BaseFolder(object): OfflineImapError.ERROR.MESSAGE) except (KeyboardInterrupt): # bubble up CTRL-C raise - except OfflineImapError, e: + except OfflineImapError as e: if e.severity > OfflineImapError.ERROR.MESSAGE: raise # buble severe errors up self.ui.error(e, exc_info()[2]) - except Exception, e: + except Exception as e: self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ (uid, self.accountname, exc_info()[2])) @@ -474,11 +474,11 @@ class BaseFolder(object): action(dstfolder, statusfolder) except (KeyboardInterrupt): raise - except OfflineImapError, e: + except OfflineImapError as e: if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2]) - except Exception, e: + except Exception as e: self.ui.error(e, exc_info()[2], "Syncing folder %s [acc: %s]" %\ (self, self.accountname)) raise # raise unknown Exceptions so we can fix them diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 3dbcaa8..9cdf752 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -97,7 +97,7 @@ class IMAPFolder(BaseFolder): # Select folder and get number of messages restype, imapdata = imapobj.select(self.getfullname(), True, True) - except OfflineImapError, e: + except OfflineImapError as e: # retry on dropped connections, raise otherwise self.imapserver.releaseconnection(imapobj, True) if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: @@ -219,7 +219,7 @@ class IMAPFolder(BaseFolder): res_type, data = imapobj.uid('fetch', str(uid), '(BODY.PEEK[])') fails_left = 0 - except imapobj.abort, e: + except imapobj.abort as e: # Release dropped connection, and get a new one self.imapserver.releaseconnection(imapobj, True) imapobj = self.imapserver.acquireconnection() @@ -314,7 +314,7 @@ class IMAPFolder(BaseFolder): headervalue = imapobj._quote(headervalue) try: matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0] - except imapobj.error, err: + except imapobj.error as err: # IMAP server doesn't implement search or had a problem. self.ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername)) return 0 @@ -545,7 +545,7 @@ class IMAPFolder(BaseFolder): imaputil.flagsmaildir2imap(flags), date, content) retry_left = 0 # Mark as success - except imapobj.abort, e: + except imapobj.abort as e: # connection has been reset, release connection and retry. retry_left -= 1 self.imapserver.releaseconnection(imapobj, True) @@ -557,7 +557,7 @@ class IMAPFolder(BaseFolder): (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) self.ui.error(e, exc_info()[2]) - except imapobj.error, e: # APPEND failed + except imapobj.error as e: # APPEND failed # If the server responds with 'BAD', append() # raise()s directly. So we catch that too. # drop conn, it might be bad. diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 3d2cd37..9466d4f 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -79,7 +79,7 @@ class LocalStatusFolder(BaseFolder): uid, flags = line.split(':') uid = long(uid) flags = set(flags) - except ValueError, e: + except ValueError as e: errstr = "Corrupt line '%s' in cache file '%s'" % \ (line, self.filename) self.ui.warn(errstr) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index fe85308..e778bfd 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -257,7 +257,7 @@ class MaildirFolder(BaseFolder): try: fd = os.open(os.path.join(tmpdir, messagename), os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0666) - except OSError, e: + except OSError as e: if e.errno == 17: #FILE EXISTS ALREADY severity = OfflineImapError.ERROR.MESSAGE @@ -313,7 +313,7 @@ class MaildirFolder(BaseFolder): try: os.rename(os.path.join(self.getfullname(), oldfilename), os.path.join(self.getfullname(), newfilename)) - except OSError, e: + except OSError as e: raise OfflineImapError("Can't rename file '%s' to '%s': %s" % ( oldfilename, newfilename, e[1]), OfflineImapError.ERROR.FOLDER) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index b4345fa..4732446 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -53,7 +53,7 @@ class UsefulIMAPMixIn(object): del self.untagged_responses[:] try: result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) - except self.abort, e: + except self.abort as e: # self.abort is raised when we are supposed to retry errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ "ver said: %s" % (self.host, mailbox, e.args[0]) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index a355eaf..609b6fa 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -153,7 +153,7 @@ class IMAPServer: rc = kerberos.authGSSClientWrap(self.gss_vc, response, self.username) response = kerberos.authGSSClientResponse(self.gss_vc) - except kerberos.GSSError, err: + except kerberos.GSSError as err: # Kerberos errored out on us, respond with None to cancel the # authentication self.ui.debug('imap', '%s: %s' % (err[0][0], err[1][0])) @@ -232,7 +232,7 @@ class IMAPServer: 'Attempting GSSAPI authentication') try: imapobj.authenticate('GSSAPI', self.gssauth) - except imapobj.error, val: + except imapobj.error as val: self.gssapi = False self.ui.debug('imap', 'GSSAPI Authentication failed') @@ -258,7 +258,7 @@ class IMAPServer: try: imapobj.authenticate('CRAM-MD5', self.md5handler) - except imapobj.error, val: + except imapobj.error as val: self.plainauth(imapobj) else: # Use plaintext login, unless @@ -271,7 +271,7 @@ class IMAPServer: # Would bail by here if there was a failure. success = 1 self.goodpassword = self.password - except imapobj.error, val: + except imapobj.error as val: self.passworderror = str(val) raise @@ -304,7 +304,7 @@ class IMAPServer: self.lastowner[imapobj] = get_ident() self.connectionlock.release() return imapobj - except Exception, e: + except Exception as e: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" @@ -540,7 +540,7 @@ class IdleThread(object): imapobj = self.parent.acquireconnection() try: imapobj.select(self.folder) - except OfflineImapError, e: + except OfflineImapError as e: if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: # Connection closed, release connection and retry self.ui.error(e, exc_info()[2]) diff --git a/offlineimap/init.py b/offlineimap/init.py index 5cd1894..51d535d 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -351,7 +351,7 @@ class OfflineImap: self.ui.terminate() except (SystemExit): raise - except Exception, e: + except Exception as e: self.ui.error(e) self.ui.terminate() diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 084841f..bc2c15a 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -165,7 +165,7 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): try: dst_repo.makefolder(src_name) dst_haschanged = True # Need to refresh list - except OfflineImapError, e: + except OfflineImapError as e: self.ui.error(e, exc_info()[2], "Creating folder %s on repository %s" %\ (src_name, dst_repo)) @@ -212,7 +212,7 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): try: src_repo.makefolder(newsrc_name) src_haschanged = True # Need to refresh list - except OfflineImapError, e: + except OfflineImapError as e: self.ui.error(e, exc_info()[2], "Creating folder %s on " "repository %s" % (newsrc_name, src_repo)) raise diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index c2633dc..d4b6599 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -95,7 +95,7 @@ class IMAPRepository(BaseRepository): host = self.getconf('remotehosteval') try: host = self.localeval.eval(host) - except Exception, e: + except Exception as e: raise OfflineImapError("remotehosteval option for repository "\ "'%s' failed:\n%s" % (self, e), OfflineImapError.ERROR.REPO) @@ -128,7 +128,7 @@ class IMAPRepository(BaseRepository): try: netrcentry = netrc.netrc().authenticators(self.gethost()) - except IOError, inst: + except IOError as inst: if inst.errno != errno.ENOENT: raise else: @@ -137,7 +137,7 @@ class IMAPRepository(BaseRepository): try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) - except IOError, inst: + except IOError as inst: if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: @@ -223,7 +223,7 @@ class IMAPRepository(BaseRepository): # 4. read password from ~/.netrc try: netrcentry = netrc.netrc().authenticators(self.gethost()) - except IOError, inst: + except IOError as inst: if inst.errno != errno.ENOENT: raise else: @@ -234,7 +234,7 @@ class IMAPRepository(BaseRepository): # 5. read password from /etc/netrc try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) - except IOError, inst: + except IOError as inst: if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: @@ -297,7 +297,7 @@ class IMAPRepository(BaseRepository): for foldername in self.folderincludes: try: imapobj.select(foldername, readonly = True) - except OfflineImapError, e: + except OfflineImapError as e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: raise diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 66a3ebd..139ddb4 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -98,7 +98,7 @@ class MaildirRepository(BaseRepository): self.debug("makefolder: calling makedirs '%s'" % full_path) try: os.makedirs(full_path, 0700) - except OSError, e: + except OSError as e: if e.errno == 17 and os.path.isdir(full_path): self.debug("makefolder: '%s' already a directory" % foldername) else: @@ -106,7 +106,7 @@ class MaildirRepository(BaseRepository): for subdir in ['cur', 'new', 'tmp']: try: os.mkdir(os.path.join(full_path, subdir), 0700) - except OSError, e: + except OSError as e: if e.errno == 17 and os.path.isdir(full_path): self.debug("makefolder: '%s' already has subdir %s" % (foldername, subdir)) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index c102446..7499ec4 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -165,7 +165,7 @@ class ExitNotifyThread(Thread): pass prof.dump_stats(os.path.join(ExitNotifyThread.profiledir, "%s_%s.prof" % (self.threadid, self.getName()))) - except Exception, e: + except Exception as e: # Thread exited with Exception, store it tb = traceback.format_exc() self.set_exit_exception(e, tb) From 55da31c84bbf4e0d4b9ec474fbdd71b338d70969 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 11:31:54 +0100 Subject: [PATCH 452/817] octal notation 0700 -> 0o700 Use octal notation that python3 understands. Works >=python2.6 Signed-off-by: Sebastian Spaeth --- offlineimap/CustomConfig.py | 2 +- offlineimap/accounts.py | 2 +- offlineimap/folder/Maildir.py | 2 +- offlineimap/repository/Base.py | 6 +++--- offlineimap/repository/LocalStatus.py | 2 +- offlineimap/repository/Maildir.py | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index dae20b6..e00b59d 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -49,7 +49,7 @@ class CustomConfigParser(SafeConfigParser): def getmetadatadir(self): metadatadir = os.path.expanduser(self.getdefault("general", "metadata", "~/.offlineimap")) if not os.path.exists(metadatadir): - os.mkdir(metadatadir, 0700) + os.mkdir(metadatadir, 0o700) return metadatadir def getlocaleval(self): diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 780ad04..429dc19 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -216,7 +216,7 @@ class SyncableAccount(Account): self.ui.registerthread(self) accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): - os.mkdir(accountmetadata, 0700) + os.mkdir(accountmetadata, 0o700) self.remoterepos = Repository(self, 'remote') self.localrepos = Repository(self, 'local') diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index e778bfd..11541e4 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -256,7 +256,7 @@ class MaildirFolder(BaseFolder): # open file and write it out try: fd = os.open(os.path.join(tmpdir, messagename), - os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0666) + os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666) except OSError as e: if e.errno == 17: #FILE EXISTS ALREADY diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index bc2c15a..a096a15 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -35,13 +35,13 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): self._accountname = self.account.getname() self.uiddir = os.path.join(self.config.getmetadatadir(), 'Repository-' + self.name) if not os.path.exists(self.uiddir): - os.mkdir(self.uiddir, 0700) + os.mkdir(self.uiddir, 0o700) self.mapdir = os.path.join(self.uiddir, 'UIDMapping') if not os.path.exists(self.mapdir): - os.mkdir(self.mapdir, 0700) + os.mkdir(self.mapdir, 0o700) self.uiddir = os.path.join(self.uiddir, 'FolderValidity') if not os.path.exists(self.uiddir): - os.mkdir(self.uiddir, 0700) + os.mkdir(self.uiddir, 0o700) self.nametrans = lambda foldername: foldername self.folderfilter = lambda foldername: 1 diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index fdd1e19..b1e9fd0 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -41,7 +41,7 @@ class LocalStatusRepository(BaseRepository): % (backend, account.name)) if not os.path.exists(self.root): - os.mkdir(self.root, 0700) + os.mkdir(self.root, 0o700) # self._folders is a list of LocalStatusFolders() self._folders = None diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 139ddb4..67f659b 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -37,7 +37,7 @@ class MaildirRepository(BaseRepository): # Create the top-level folder if it doesn't exist if not os.path.isdir(self.root): - os.mkdir(self.root, 0700) + os.mkdir(self.root, 0o700) def _append_folder_atimes(self, foldername): """Store the atimes of a folder's new|cur in self.folder_atimes""" @@ -97,7 +97,7 @@ class MaildirRepository(BaseRepository): # sub-folders may be created before higher-up ones. self.debug("makefolder: calling makedirs '%s'" % full_path) try: - os.makedirs(full_path, 0700) + os.makedirs(full_path, 0o700) except OSError as e: if e.errno == 17 and os.path.isdir(full_path): self.debug("makefolder: '%s' already a directory" % foldername) @@ -105,7 +105,7 @@ class MaildirRepository(BaseRepository): raise for subdir in ['cur', 'new', 'tmp']: try: - os.mkdir(os.path.join(full_path, subdir), 0700) + os.mkdir(os.path.join(full_path, subdir), 0o700) except OSError as e: if e.errno == 17 and os.path.isdir(full_path): self.debug("makefolder: '%s' already has subdir %s" % From a8ab269ada0c9376e2eb3fe03662c86e60ec96fb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 11:38:07 +0100 Subject: [PATCH 453/817] Import configparser for python3 compatability Attempt to load first ConfigParser and then configparser. At some point this should be switched to do the python3 thing first. Signed-off-by: Sebastian Spaeth --- offlineimap/CustomConfig.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index e00b59d..b5869e7 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -1,5 +1,4 @@ -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2012 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,7 +14,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from ConfigParser import SafeConfigParser +try: + from ConfigParser import SafeConfigParser +except ImportError: #python3 + from configparser import SafeConfigParser from offlineimap.localeval import LocalEval import os From 9df7f34d4c7dab72de991416fff5d678e02abdab Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 11:49:28 +0100 Subject: [PATCH 454/817] Remove unused locked() function We do not use ui.locked() anymore to output an error message, the text comes directly from the exception. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 3 ++- offlineimap/ui/UIBase.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 429dc19..ed08cad 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -199,7 +199,8 @@ class SyncableAccount(Account): pass except IOError: self._lockfd.close() - raise OfflineImapError("Could not lock account %s." % self, + raise OfflineImapError("Could not lock account %s. Is another " + "instance using this account?" % self, OfflineImapError.ERROR.REPO) def unlock(self): diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 06dbd01..74ef22e 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -203,9 +203,6 @@ class UIBase(object): def invaliddebug(self, debugtype): self.warn("Invalid debug type: %s" % debugtype) - def locked(s): - raise Exception, "Another OfflineIMAP is running with the same metadatadir; exiting." - def getnicename(self, object): """Return the type of a repository or Folder as string From c5468ae599e9f008bedb06250537a89a11b960e9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 11:51:02 +0100 Subject: [PATCH 455/817] raise Exception, "text" --> raise Exception("text") To have the code work in python3, we need to convert all occurences of raise Exception, "text" to be proper functions. This style also adheres to PEP8. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 4 +++- offlineimap/imaputil.py | 2 +- offlineimap/repository/__init__.py | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 9cdf752..17364d3 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -328,7 +328,9 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \ repr(matchinguids)) if len(matchinguids) != 1 or matchinguids[0] == None: - raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids)) + raise ValueError("While attempting to find UID for message with " + "header %s, got wrong-sized matchinguids of %s" %\ + (headername, str(matchinguids))) matchinguids.sort() return long(matchinguids[0]) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 3955683..f9b021c 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -58,7 +58,7 @@ def flagsplit(string): ['FLAGS,'(\\Seen Old)','UID', '4807'] """ if string[0] != '(' or string[-1] != ')': - raise ValueError, "Passed string '%s' is not a flag list" % string + raise ValueError("Passed string '%s' is not a flag list" % string) return imapsplit(string[1:-1]) def options2hash(list): diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py index c63f6db..0059bdf 100644 --- a/offlineimap/repository/__init__.py +++ b/offlineimap/repository/__init__.py @@ -47,15 +47,15 @@ class Repository(object): return LocalStatusRepository(name, account) else: - raise ValueError, "Request type %s not supported" % reqtype + raise ValueError("Request type %s not supported" % reqtype) config = account.getconfig() repostype = config.get('Repository ' + name, 'type').strip() try: repo = typemap[repostype] except KeyError: - raise Exception, "'%s' repository not supported for %s repositories."%\ - (repostype, reqtype) + raise ValueError("'%s' repository not supported for %s repositories" + "." % (repostype, reqtype)) return repo(name, account) From 06a78b611275df9d7613d63dbd6547510bafba67 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 11:55:26 +0100 Subject: [PATCH 456/817] python3: Queue -> queue import queue (python3) if Queue is not available (python2) Signed-off-by: Sebastian Spaeth --- offlineimap/threadutil.py | 7 +++++-- offlineimap/ui/UIBase.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 7499ec4..92c54e2 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2012 John Goerzen & contributors # Thread support module # # This program is free software; you can redistribute it and/or modify @@ -16,7 +16,10 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import Lock, Thread, BoundedSemaphore -from Queue import Queue, Empty +try: + from Queue import Queue, Empty +except ImportError: # python3 + from queue import Queue, Empty import traceback from thread import get_ident # python < 2.6 support import os.path diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 74ef22e..bf46e6f 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -22,7 +22,10 @@ import sys import os import traceback import threading -from Queue import Queue +try: + from Queue import Queue +except ImportError: #python3 + from queue import Queue from collections import deque from offlineimap.error import OfflineImapError import offlineimap From 74fc90296702ad71b47727f1f761814adf21e8f4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 11:57:19 +0100 Subject: [PATCH 457/817] python3: import absolute package name This import failed in python3, we need to either specify "." (relative) or from OfflineImap.ui. (absolute). Done the latter. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/LocalStatus.py | 2 +- offlineimap/repository/Maildir.py | 2 +- offlineimap/ui/Machine.py | 2 +- offlineimap/ui/Noninteractive.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index b1e9fd0..30c8560 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -16,9 +16,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from Base import BaseRepository from offlineimap.folder.LocalStatus import LocalStatusFolder, magicline from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder +from offlineimap.repository.Base import BaseRepository import os import re diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 67f659b..7c08d64 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -16,10 +16,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from Base import BaseRepository from offlineimap import folder from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError +from offlineimap.repository.Base import BaseRepository import os from stat import * diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index a87395d..8bf77da 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -17,8 +17,8 @@ from urllib import urlencode import sys import time import logging -from UIBase import UIBase from threading import currentThread +from offlineimap.ui.UIBase import UIBase import offlineimap protocol = '7.0.0' diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index 36bb92b..de1e8df 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -1,5 +1,5 @@ # Noninteractive UI -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2012 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,7 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import logging -from UIBase import UIBase +from offlineimap.ui.UIBase import UIBase class Basic(UIBase): """'Quiet' simply sets log level to INFO""" From 8b6af63d830b826b4e21c42d0c272ce69bf0a3d4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 12:00:28 +0100 Subject: [PATCH 458/817] [MachineUI] Remove unneeded "print 't'" statement 1) it's print('t') now and 2) this was a superfluous statement. This broke python3 Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Machine.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 8bf77da..c7f1557 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -122,12 +122,11 @@ class MachineUI(UIBase): "\f".join(flags), dest)) - def threadException(s, thread): - print s.getThreadExceptionString(thread) - s._printData('threadException', "%s\n%s" % \ - (thread.getName(), s.getThreadExceptionString(thread))) - s.delThreadDebugLog(thread) - s.terminate(100) + def threadException(self, thread): + self._printData('threadException', "%s\n%s" % \ + (thread.getName(), self.getThreadExceptionString(thread))) + self.delThreadDebugLog(thread) + self.terminate(100) def terminate(s, exitstatus = 0, errortitle = '', errormsg = ''): s._printData('terminate', "%d\n%s\n%s" % (exitstatus, errortitle, errormsg)) From 93f4a19778b1a6c1977cc784a5b2706dfd32e1f4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 12:06:18 +0100 Subject: [PATCH 459/817] python3: urlencode is in a different module import urlencode from urllib.parse if neeeded for python3. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Machine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index c7f1557..069bb35 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -13,7 +13,10 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from urllib import urlencode +try: + from urllib import urlencode +except ImportError: # python3 + from urllib.parse import urlencode import sys import time import logging From 81fc20c7cadbc4dea221208479e29522610501f7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 12:17:02 +0100 Subject: [PATCH 460/817] Remove python<2.6 import workarounds (set & ssl) 'set' is builtin since python2.6, so remove the imports. Also 'ssl' exists since 2.6 and has everything we need, so no need for conditional import tests here anymore. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 5 +---- offlineimap/folder/IMAP.py | 4 ---- offlineimap/folder/LocalStatus.py | 5 +---- offlineimap/folder/LocalStatusSQLite.py | 4 ---- offlineimap/imaplibutil.py | 6 +----- offlineimap/imapserver.py | 8 ++------ offlineimap/imaputil.py | 5 +---- 7 files changed, 6 insertions(+), 31 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index c9d86c7..1c05370 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -23,10 +23,7 @@ import os.path import re from sys import exc_info import traceback -try: # python 2.6 has set() built in - set -except NameError: - from sets import Set as set + class BaseFolder(object): def __init__(self, name, repository): diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 17364d3..9d53e89 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -24,10 +24,6 @@ from sys import exc_info from Base import BaseFolder from offlineimap import imaputil, imaplibutil, OfflineImapError from offlineimap.imaplib2 import MonthNames -try: # python 2.6 has set() built in - set -except NameError: - from sets import Set as set class IMAPFolder(BaseFolder): diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 9466d4f..e637d94 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -18,13 +18,10 @@ from Base import BaseFolder import os import threading -try: # python 2.6 has set() built in - set -except NameError: - from sets import Set as set magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" + class LocalStatusFolder(BaseFolder): def __init__(self, name, repository): self.sep = '.' #needs to be set before super.__init__() diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 6bfa667..f59d4b2 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -23,10 +23,6 @@ try: except: pass #fail only if needed later on, not on import -try: # python 2.6 has set() built in - set -except NameError: - from sets import Set as set class LocalStatusSQLiteFolder(LocalStatusFolder): """LocalStatus backend implemented with an SQLite database diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 4732446..2d5fc39 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -19,6 +19,7 @@ import os import fcntl import re import socket +import ssl import time import subprocess import threading @@ -28,11 +29,6 @@ from offlineimap.ui import getglobalui from offlineimap import OfflineImapError from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, IMAP4_PORT, InternalDate, Mon2num -try: - import ssl -except ImportError: - #fails on python <2.6 - pass class UsefulIMAPMixIn(object): def getselectedfolder(self): diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 609b6fa..16d0cac 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -27,11 +27,7 @@ import time import errno from sys import exc_info from socket import gaierror -try: - from ssl import SSLError, cert_time_to_seconds -except ImportError: - # Protect against python<2.6, use dummy and won't get SSL errors. - SSLError = None +from ssl import SSLError, cert_time_to_seconds try: # do we have a recent pykerberos? @@ -323,7 +319,7 @@ class IMAPServer: (self.hostname, self.repos) raise OfflineImapError(reason, severity) - elif SSLError and isinstance(e, SSLError) and e.errno == 1: + elif isinstance(e, SSLError) and e.errno == 1: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 993: diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index f9b021c..f1be3c7 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -20,10 +20,7 @@ import re import string import types from offlineimap.ui import getglobalui -try: # python 2.6 has set() built in - set -except NameError: - from sets import Set as set + # find the first quote in a string quotere = re.compile( From dc67f515b61834a9ac4afe56f9c04d806e3a1787 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 12:20:28 +0100 Subject: [PATCH 461/817] no need for type(s) == types.StringType All we want to do here is to test whether we got a string'ish type or a list (literal), so testing for basestring will be fine. Signed-off-by: Sebastian Spaeth --- offlineimap/imaputil.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index f1be3c7..ba1f458 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -18,7 +18,6 @@ import re import string -import types from offlineimap.ui import getglobalui @@ -88,7 +87,7 @@ def imapsplit(imapstring): ['(\\HasNoChildren)', '"."', '"INBOX.Sent"']""" - if type(imapstring) != types.StringType: + if not isinstance(imapstring, basestring): debug("imapsplit() got a non-string input; working around.") # Sometimes, imaplib will throw us a tuple if the input # contains a literal. See Python bug From ba3a698a6741ebfc05052b2add720f8000cd0949 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 12:23:49 +0100 Subject: [PATCH 462/817] No need to test for types.StringType all we want to know is if we got some string'ish type and testing for isinstance 'basestring' is sufficient for that. Remove the import. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/IMAP.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index d4b6599..f008592 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -20,7 +20,6 @@ from offlineimap import folder, imaputil, imapserver, OfflineImapError from offlineimap.folder.UIDMaps import MappedIMAPFolder from offlineimap.threadutil import ExitNotifyThread from threading import Event -import types import os from sys import exc_info import netrc @@ -274,9 +273,9 @@ class IMAPRepository(BaseRepository): self.imapserver.releaseconnection(imapobj) for string in listresult: if string == None or \ - (type(string) == types.StringType and string == ''): + (isinstance(string, basestring) and string == ''): # Bug in imaplib: empty strings in results from - # literals. + # literals. TODO: still relevant? continue flags, delim, name = imaputil.imapsplit(string) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] From 03566b203723655daff1e3cd6d6ba378e5f41eae Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 12:34:27 +0100 Subject: [PATCH 463/817] Replace thread.get_ident() Replace low-level thread.get_ident() with threading.currentThread().ident. This works both in python2.6 and python3. (thread is renamed _thread and its direct use is not recommended) Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 8 ++++---- offlineimap/threadutil.py | 6 ++---- offlineimap/ui/Curses.py | 1 - 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 16d0cac..22c5c16 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -18,7 +18,6 @@ from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui from threading import Lock, BoundedSemaphore, Thread, Event, currentThread -from thread import get_ident # python < 2.6 support import offlineimap.accounts import hmac import socket @@ -167,6 +166,7 @@ class IMAPServer: self.semaphore.acquire() self.connectionlock.acquire() + curThread = currentThread() imapobj = None if len(self.availableconnections): # One is available. @@ -176,7 +176,7 @@ class IMAPServer: imapobj = None for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] - if self.lastowner[tryobj] == get_ident(): + if self.lastowner[tryobj] == curThread.ident: imapobj = tryobj del(self.availableconnections[i]) break @@ -184,7 +184,7 @@ class IMAPServer: imapobj = self.availableconnections[0] del(self.availableconnections[0]) self.assignedconnections.append(imapobj) - self.lastowner[imapobj] = get_ident() + self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj @@ -297,7 +297,7 @@ class IMAPServer: self.connectionlock.acquire() self.assignedconnections.append(imapobj) - self.lastowner[imapobj] = get_ident() + self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj except Exception as e: diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 92c54e2..054fa11 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -15,13 +15,12 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import Lock, Thread, BoundedSemaphore +from threading import Lock, Thread, BoundedSemaphore, currentThread try: from Queue import Queue, Empty except ImportError: # python3 from queue import Queue, Empty import traceback -from thread import get_ident # python < 2.6 support import os.path import sys from offlineimap.ui import getglobalui @@ -152,7 +151,6 @@ class ExitNotifyThread(Thread): def run(self): global exitthreads - self.threadid = get_ident() try: if not ExitNotifyThread.profiledir: # normal case Thread.run(self) @@ -167,7 +165,7 @@ class ExitNotifyThread(Thread): except SystemExit: pass prof.dump_stats(os.path.join(ExitNotifyThread.profiledir, - "%s_%s.prof" % (self.threadid, self.getName()))) + "%s_%s.prof" % (self.ident, self.getName()))) except Exception as e: # Thread exited with Exception, store it tb = traceback.format_exc() diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index af41d35..365be60 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -16,7 +16,6 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import RLock, currentThread, Lock, Event -from thread import get_ident # python < 2.6 support from collections import deque import time import sys From 014caddee60243d02904926c8d56d49151239f18 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 12:49:08 +0100 Subject: [PATCH 464/817] folder/Base: Create unambigous MRO inheritence class BaseRepository(object, CustomConfig.ConfigHelperMixin): led to TypeError: Cannot create a consistent method resolution order (MRO) for bases ConfigHelperMixin, object. Switching the inherited classes helps. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index a096a15..cbbc5f4 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -1,6 +1,5 @@ # Base repository support -# Copyright (C) 2002-2007 John Goerzen -# +# Copyright (C) 2002-2012 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,7 +23,7 @@ from offlineimap import CustomConfig from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError -class BaseRepository(object, CustomConfig.ConfigHelperMixin): +class BaseRepository(CustomConfig.ConfigHelperMixin, object): def __init__(self, reposname, account): self.ui = getglobalui() From b33f2452f07b9fcb891717b0046b82dac1b7267a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 12:52:12 +0100 Subject: [PATCH 465/817] Use "from . import" for relative imports Will fail in python3 otherwise. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Gmail.py | 2 +- offlineimap/folder/IMAP.py | 2 +- offlineimap/folder/LocalStatus.py | 2 +- offlineimap/folder/LocalStatusSQLite.py | 2 +- offlineimap/folder/Maildir.py | 2 +- offlineimap/folder/UIDMaps.py | 2 +- offlineimap/folder/__init__.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 5d11119..e3433c0 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -18,7 +18,7 @@ """Folder implementation to support features of the Gmail IMAP server. """ -from IMAP import IMAPFolder +from .IMAP import IMAPFolder class GmailFolder(IMAPFolder): """Folder implementation to support features of the Gmail IMAP server. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 9d53e89..4e7e885 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -21,7 +21,7 @@ import binascii import re import time from sys import exc_info -from Base import BaseFolder +from .Base import BaseFolder from offlineimap import imaputil, imaplibutil, OfflineImapError from offlineimap.imaplib2 import MonthNames diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index e637d94..7f27768 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from Base import BaseFolder +from .Base import BaseFolder import os import threading diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index f59d4b2..dbe3e66 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -17,7 +17,7 @@ import os.path import re from threading import Lock -from LocalStatus import LocalStatusFolder +from .LocalStatus import LocalStatusFolder try: import sqlite3 as sqlite except: diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 11541e4..8ca32e5 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -19,7 +19,7 @@ import socket import time import re import os -from Base import BaseFolder +from .Base import BaseFolder from threading import Lock try: diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index ac0a9b4..36e92af 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import Lock -from IMAP import IMAPFolder +from .IMAP import IMAPFolder import os.path class MappedIMAPFolder(IMAPFolder): diff --git a/offlineimap/folder/__init__.py b/offlineimap/folder/__init__.py index 425148b..2b54a71 100644 --- a/offlineimap/folder/__init__.py +++ b/offlineimap/folder/__init__.py @@ -1,2 +1,2 @@ -import Base, Gmail, IMAP, Maildir, LocalStatus +from . import Base, Gmail, IMAP, Maildir, LocalStatus From e6e708ec786bbdb3b94a8d4ef7bd10559ee2fd2c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 13:17:16 +0100 Subject: [PATCH 466/817] Don't use sort() on dict values() This won't work in python3 anymore, so just use sorted() when needed. In one case, we could remove the sort() completely as were were sanity checking one line above, that we only having one UID as response which makes sorting unneeded. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 3 +-- offlineimap/imaputil.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 4e7e885..0bbe746 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -1,5 +1,5 @@ # IMAP folder support -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2012 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -327,7 +327,6 @@ class IMAPFolder(BaseFolder): raise ValueError("While attempting to find UID for message with " "header %s, got wrong-sized matchinguids of %s" %\ (headername, str(matchinguids))) - matchinguids.sort() return long(matchinguids[0]) def savemessage_fetchheaders(self, imapobj, headername, headervalue): diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index ba1f458..7165e09 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -182,13 +182,12 @@ def flagsimap2maildir(flagstring): return retval def flagsmaildir2imap(maildirflaglist): - """Convert set of flags ([DR]) into a string '(\\Draft \\Deleted)'""" + """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'""" retval = [] for imapflag, maildirflag in flagmap: if maildirflag in maildirflaglist: retval.append(imapflag) - retval.sort() - return '(' + ' '.join(retval) + ')' + return '(' + ' '.join(sorted(retval)) + ')' def uid_sequence(uidlist): """Collapse UID lists into shorter sequence sets From 046210b93d3585066b6970b7bc2f246434f23b0c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 13:23:12 +0100 Subject: [PATCH 467/817] Fix mixed space/tabs Signed-off-by: Sebastian Spaeth --- offlineimap/imaputil.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 7165e09..557064b 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -133,12 +133,12 @@ def imapsplit(imapstring): if workstr[0] == '(': rparenc = 1 # count of right parenthesis to match rpareni = 1 # position to examine - while rparenc: # Find the end of the group. - if workstr[rpareni] == ')': # end of a group - rparenc -= 1 - elif workstr[rpareni] == '(': # start of a group - rparenc += 1 - rpareni += 1 # Move to next character. + while rparenc: # Find the end of the group. + if workstr[rpareni] == ')': # end of a group + rparenc -= 1 + elif workstr[rpareni] == '(': # start of a group + rparenc += 1 + rpareni += 1 # Move to next character. parenlist = workstr[0:rpareni] workstr = workstr[rpareni:].lstrip() retval.append(parenlist) From 8aba2800e6336e64b6aad0a64c23163a245ecc42 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 13:28:00 +0100 Subject: [PATCH 468/817] long(0) -> 0 There is no need to cast 0 to 'long' even if we want to compare it to long numbers in modern pythons. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 8ca32e5..16745ab 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -40,7 +40,7 @@ re_uidmatch = re.compile(',U=(\d+)') re_timestampmatch = re.compile('(\d+)'); timeseq = 0 -lasttime = long(0) +lasttime = 0 timelock = Lock() def gettimeseq(): From 5c598d7e74b11c3b6e171570164bdf3a17f7cc76 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 5 Feb 2012 13:40:06 +0100 Subject: [PATCH 469/817] dict.has_key(a) --> a in dict has_key() is gone in python3, so use a more modern syntax here. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 2 +- offlineimap/folder/UIDMaps.py | 4 ++-- offlineimap/imaplibutil.py | 2 +- offlineimap/threadutil.py | 2 +- offlineimap/ui/UIBase.py | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 0bbe746..d06c740 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -186,7 +186,7 @@ class IMAPFolder(BaseFolder): continue messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) - if not options.has_key('UID'): + if not 'UID' in options: self.ui.warn('No UID in message with options %s' %\ str(options), minor = 1) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 36e92af..f306459 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -94,7 +94,7 @@ class MappedIMAPFolder(IMAPFolder): # summary that have been deleted from the folder. for luid in self.diskl2r.keys(): - if not reallist.has_key(luid): + if not luid in reallist: ruid = self.diskl2r[luid] del self.diskr2l[ruid] del self.diskl2r[luid] @@ -107,7 +107,7 @@ class MappedIMAPFolder(IMAPFolder): self.l2r = self.diskl2r.copy() for luid in reallist.keys(): - if not self.l2r.has_key(luid): + if not luid in self.l2r: ruid = nextneg nextneg -= 1 self.l2r[luid] = ruid diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 2d5fc39..2aa81d9 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -137,7 +137,7 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): """Improved version of imaplib.IMAP4_SSL overriding select()""" def __init__(self, *args, **kwargs): self._fingerprint = kwargs.get('fingerprint', None) - if kwargs.has_key('fingerprint'): + if 'fingerprint' in kwargs: del kwargs['fingerprint'] super(WrappedIMAP4_SSL, self).__init__(*args, **kwargs) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 054fa11..a29e68f 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -209,7 +209,7 @@ def initInstanceLimit(instancename, instancemax): """Initialize the instance-limited thread implementation to permit up to intancemax threads with the given instancename.""" instancelimitedlock.acquire() - if not instancelimitedsems.has_key(instancename): + if not instancename in instancelimitedsems: instancelimitedsems[instancename] = BoundedSemaphore(instancemax) instancelimitedlock.release() diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index bf46e6f..a8aeeaf 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -159,7 +159,7 @@ class UIBase(object): def unregisterthread(self, thr): """Unregister a thread as being associated with an account name""" - if self.threadaccounts.has_key(thr): + if thr in self.threadaccounts: del self.threadaccounts[thr] self.debug('thread', "Unregister thread '%s'" % thr.getName()) @@ -175,7 +175,7 @@ class UIBase(object): def debug(self, debugtype, msg): cur_thread = threading.currentThread() - if not self.debugmessages.has_key(cur_thread): + if not cur_thread in self.debugmessages: # deque(..., self.debugmsglen) would be handy but was # introduced in p2.6 only, so we'll need to work around and # shorten our debugmsg list manually :-( @@ -403,7 +403,7 @@ class UIBase(object): ################################################## Threads def getThreadDebugLog(self, thread): - if self.debugmessages.has_key(thread): + if thread in self.debugmessages: message = "\nLast %d debug messages logged for %s prior to exception:\n"\ % (len(self.debugmessages[thread]), thread.getName()) message += "\n".join(self.debugmessages[thread]) From 19c014c6cdbe67fdeb727164767901e5e00ba18b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 6 Feb 2012 17:33:50 +0100 Subject: [PATCH 470/817] Implement foldersort in a python3 compatible way By default we sort folders alphabetically (for IMAP) according to their transposed names. For python3, we need to bend a bit backwards to still allow the use of a cmp() function for foldersort. While going through, I discovered that we never sort folders for Maildir. Signed-off-by: Sebastian Spaeth --- offlineimap.conf | 13 +++++++------ offlineimap/repository/Base.py | 2 +- offlineimap/repository/IMAP.py | 20 ++++++++++++++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 2656ef6..b4f6a99 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -496,13 +496,14 @@ remoteuser = username # one. For example: # folderincludes = ['debian.user', 'debian.personal'] -# You can specify foldersort to determine how folders are sorted. +# You can specify 'foldersort' to determine how folders are sorted. # This affects order of synchronization and mbnames. The expression -# should return -1, 0, or 1, as the default Python cmp() does. The -# two arguments, x and y, are strings representing the names of the folders -# to be sorted. The sorting is applied *AFTER* nametrans, if any. -# -# To reverse the sort: +# should return -1, 0, or 1, as the default Python cmp() does. The two +# arguments, x and y, are strings representing the names of the folders +# to be sorted. The sorting is applied *AFTER* nametrans, if any. The +# default is to sort IMAP folders alphabetically +# (case-insensitive). Usually, you should never have to modify this. To +# eg. reverse the sort: # # foldersort = lambda x, y: -cmp(x, y) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index cbbc5f4..cca5cf3 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -45,7 +45,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): self.nametrans = lambda foldername: foldername self.folderfilter = lambda foldername: 1 self.folderincludes = [] - self.foldersort = cmp + self.foldersort = None if self.config.has_option(self.getsection(), 'nametrans'): self.nametrans = self.localeval.eval( self.getconf('nametrans'), {'re': re}) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index f008592..1b13d79 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -308,8 +308,24 @@ class IMAPRepository(BaseRepository): self)) finally: self.imapserver.releaseconnection(imapobj) - - retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename())) + + if self.foldersort is None: + # default sorting by case insensitive transposed name + retval.sort(key=lambda x: str.lower(x.getvisiblename())) + else: + # do foldersort in a python3-compatible way + # http://bytes.com/topic/python/answers/844614-python-3-sorting-comparison-function + def cmp2key(mycmp): + """Converts a cmp= function into a key= function + We need to keep cmp functions for backward compatibility""" + class K: + def __init__(self, obj, *args): + self.obj = obj + def __cmp__(self, other): + return mycmp(self.obj, other.obj) + return K + retval.sort(key=cmp2key(self.foldersort)) + self.folders = retval return self.folders From da0ba2e26687a2f6b7d5149b133ebb66151c36f2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 13 Feb 2012 16:07:33 +0100 Subject: [PATCH 471/817] Improve nametrans user documentation Fill in more details on nametrans and folder filtering. Also give them a separate section in our user documentation. Everything will be immediately online at docs.offlineimap.org. The main change is to describe the reverse nametrans settings that are needed since 6.4.0 to support remote folder creation. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 + docs/MANUAL.rst | 151 ++++----------------------- docs/dev-doc-src/index.rst | 2 + docs/dev-doc-src/nametrans.rst | 182 +++++++++++++++++++++++++++++++++ offlineimap.conf | 19 ++-- offlineimap.conf.minimal | 2 +- 6 files changed, 221 insertions(+), 139 deletions(-) create mode 100644 docs/dev-doc-src/nametrans.rst diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..9a046ac 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,5 +16,9 @@ New Features Changes ------- +* internal code changes to prepare for Python3 + +* Improve user documentation of nametrans/folderfilter + Bug Fixes --------- diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 21a5dd6..8d7a8b7 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -328,18 +328,22 @@ Upgrading from plain text cache to SQLITE based cache OfflineImap uses a cache to store the last know status of mails (flags etc). Historically that has meant plain text files, but recently we introduced sqlite-based cache, which helps with performance and CPU usage on large folders. Here is how to upgrade existing plain text cache installations to sqlite based one: - 1) Sync to make sure things are reasonably similar - 3) Change the account section to status_backend = sqlite - 4) A new sync will convert your plain text cache to an sqlite cache (but - leave the old plain text cache around for easy reverting) - This should be quick and not involve any mail up/downloading. - 5) See if it works :-) - 6a) If it does not work, go back to the old version or set - status_backend=plain - 6b) Or once you are sure it works, you can delete the - .offlineimap/Account-foo/LocalStatus folder (the new cache will be in - the LocalStatus-sqlite folder) +1) Sync to make sure things are reasonably similar +2) Change the account section to status_backend = sqlite + +3) A new sync will convert your plain text cache to an sqlite cache + (but leave the old plain text cache around for easy reverting) This + should be quick and not involve any mail up/downloading. + +4) See if it works :-) + +5) If it does not work, go back to the old version or set + status_backend=plain + +6) Or, once you are sure it works, you can delete the + .offlineimap/Account-foo/LocalStatus folder (the new cache will be + in the LocalStatus-sqlite folder) Security and SSL ================ @@ -399,128 +403,10 @@ accounts will abort any current sleep and will exit after a currently running synchronization has finished. This signal can be used to gracefully exit out of a running offlineimap "daemon". -Folder filtering and Name translation -===================================== +Folder filtering and nametrans +============================== -OfflineImap provides advanced and potentially complex possibilities for -filtering and translating folder names. If you don't need this, you can -safely skip this section. - -folderfilter ------------- - -If you do not want to synchronize all your filters, you can specify a folderfilter function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. - -If the filter function returns True, the folder will be synced, if it -returns False, it. The folderfilter operates on the *UNTRANSLATED* name -(before any nametrans translation takes place). - -Example 1: synchronizing only INBOX and Sent:: - - folderfilter = lambda foldername: foldername in ['INBOX', 'Sent'] - -Example 2: synchronizing everything except Trash:: - - folderfilter = lambda foldername: foldername not in ['Trash'] - -Example 3: Using a regular expression to exclude Trash and all folders -containing the characters "Del":: - - folderfilter = lambda foldername: not re.search('(^Trash$|Del)', foldername) - -If folderfilter is not specified, ALL remote folders will be -synchronized. - -You can span multiple lines by indenting the others. (Use backslashes -at the end when required by Python syntax) For instance:: - - folderfilter = lambda foldername: foldername in - ['INBOX', 'Sent Mail', 'Deleted Items', - 'Received'] - -You only need a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. - -Even if you filtered out folders, You can specify folderincludes to -include additional folders. It should return a Python list. This might -be used to include a folder that was excluded by your folderfilter rule, -to include a folder that your server does not specify with its LIST -option, or to include a folder that is outside your basic reference. The -'reference' value will not be prefixed to this folder name, even if you -have specified one. For example:: - - folderincludes = ['debian.user', 'debian.personal'] - -nametrans ----------- - -Sometimes, folders need to have different names on the remote and the -local repositories. To achieve this you can specify a folder name -translator. This must be a eval-able Python expression that takes a -foldername arg and returns the new value. I suggest a lambda. This -example below will remove "INBOX." from the leading edge of folders -(great for Courier IMAP users):: - - nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) - -Using Courier remotely and want to duplicate its mailbox naming -locally? Try this:: - - nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) - - -WARNING: you MUST construct nametrans rules such that it NEVER returns -the same value for two folders, UNLESS the second values are -filtered out by folderfilter below. That is, two filters on one side may never point to the same folder on the other side. Failure to follow this rule -will result in undefined behavior. See also *Sharing a maildir with multiple IMAP servers* in the `PITFALLS & ISSUES`_ section. - -Where to put nametrans rules, on the remote and/or local repository? -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -If you never intend to create new folders on the LOCAL repository that -need to be synced to the REMOTE repository, it is sufficient to create a -nametrans rule on the remote Repository section. This will be used to -determine the names of new folder names on the LOCAL repository, and to -match existing folders that correspond. - -*IF* you create folders on the local repository, that are supposed to be - automatically created on the remote repository, you will need to create - a nametrans rule that provides the reverse name translation. - -(A nametrans rule provides only a one-way translation of names and in -order to know which names folders on the LOCAL side would have on the -REMOTE side, you need to specify the reverse nametrans rule on the local -repository) - -OfflineImap will complain if it needs to create a new folder on the -remote side and a back-and-forth nametrans-lation does not yield the -original foldername (as that could potentially lead to infinite folder -creation cycles). - -What folder separators do I need to use in nametrans rules? -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -**Q:** If I sync from an IMAP server with folder separator '/' to a - Maildir using the default folder separator '.' which do I need to use - in nametrans rules?:: - - nametrans = lambda f: "INBOX/" + f -or:: - nametrans = lambda f: "INBOX." + f - -**A:** Generally use the folder separator as defined in the repository - you write the nametrans rule for. That is, use '/' in the above - case. We will pass in the untranslated name of the IMAP folder as - parameter (here `f`). The translated name will ultimately have all - folder separators be replaced with the destination repositories' - folder separator. - -So if 'f' was "Sent", the first nametrans yields the translated name -"INBOX/Sent" to be used on the other side. As that repository uses the -folder separator '.' rather than '/', the ultimate name to be used will -be "INBOX.Sent". - -(As a final note, the smart will see that both variants of the above -nametrans rule would have worked identically in this case) +OfflineImap offers flexible (and complex) ways of filtering and transforming folder names. Please see the docs/dev-docs-src/folderfilters.rst document about details how to use folder filters and name transformations. The documentation will be autogenerated by a "make dev-doc" in the docs directory. It is also viewable at :ref:`folder_filtering_and_name_translation`. KNOWN BUGS ========== @@ -566,6 +452,7 @@ KNOWN BUGS * Use cygwin managed mount (not tested) - not available anymore since cygwin 1.7 +.. _pitfalls: PITFALLS & ISSUES ================= diff --git a/docs/dev-doc-src/index.rst b/docs/dev-doc-src/index.rst index e1b957e..d6aa939 100644 --- a/docs/dev-doc-src/index.rst +++ b/docs/dev-doc-src/index.rst @@ -17,6 +17,7 @@ More information on specific topics can be found on the following pages: **User documentation** * :doc:`installation/uninstall ` * :doc:`user manual/Configuration ` + * :doc:`Folder filtering & name transformation guide ` * :doc:`command line options ` * :doc:`Frequently Asked Questions ` @@ -29,6 +30,7 @@ More information on specific topics can be found on the following pages: INSTALL MANUAL + nametrans offlineimap FAQ diff --git a/docs/dev-doc-src/nametrans.rst b/docs/dev-doc-src/nametrans.rst new file mode 100644 index 0000000..fabf7a4 --- /dev/null +++ b/docs/dev-doc-src/nametrans.rst @@ -0,0 +1,182 @@ +.. _folder_filtering_and_name_translation: + +Folder filtering and Name translation +===================================== + +OfflineImap provides advanced and potentially complex possibilities for +filtering and translating folder names. If you don't need any of this, you can +safely skip this section. + +.. warning:: + Starting with v6.4.0, OfflineImap supports the creation of folders on the remote repostory. This change means that people that only had a nametrans option on the remote repository (everyone) will need to have a nametrans setting on the local repository too that will reverse the name transformation. See section `Reverse nametrans`_ for details. + +folderfilter +------------ + +If you do not want to synchronize all your filters, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. + +The only parameter to that function is the folder name. If the filter +function returns True, the folder will be synced, if it returns False, +it. will be skipped. The folderfilter operates on the *UNTRANSLATED* +name (before any `nametrans`_ fudging takes place). Consider the +examples below to get an idea of what they do. + +Example 1: synchronizing only INBOX and Sent:: + + folderfilter = lambda folder: folder in ['INBOX', 'Sent'] + +Example 2: synchronizing everything except Trash:: + + folderfilter = lambda folder: folder not in ['Trash'] + +Example 3: Using a regular expression to exclude Trash and all folders +containing the characters "Del":: + + folderfilter = lambda folder: not re.search('(^Trash$|Del)', folder) + +.. note:: + If folderfilter is not specified, ALL remote folders will be + synchronized. + +You can span multiple lines by indenting the others. (Use backslashes +at the end when required by Python syntax) For instance:: + + folderfilter = lambda foldername: foldername in + ['INBOX', 'Sent Mail', 'Deleted Items', + 'Received'] + +Usually it suffices to put a `folderfilter`_ setting in the remote repository section. You might want to put a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. (Even in this case, folder filters on the remote repository will prevent that) + +folderincludes +^^^^^^^^^^^^^^ + +You can specify `folderincludes`_ to manually include additional folders to be synced, even if they had been filtered out by a folderfilter setting. `folderincludes`_ should return a Python list. + +This can be used to 1) add a folder that was excluded by your +folderfilter rule, 2) to include a folder that your server does not specify +with its LIST option, or 3) to include a folder that is outside your basic +`reference`. The `reference` value will not be prefixed to this folder +name, even if you have specified one. For example:: + + folderincludes = ['debian.user', 'debian.personal'] + +This will add the "debian.user" and "debian.personal" folders even if you +have filtered out everything starting with "debian" in your folderfilter +settings. + + +nametrans +---------- + +Sometimes, folders need to have different names on the remote and the +local repositories. To achieve this you can specify a folder name +translator. This must be a eval-able Python expression that takes a +foldername arg and returns the new value. We suggest a lambda function, +but it could be any python function really. If you use nametrans rules, you will need to set them both on the remote and the local repository, see `Reverse nametrans`_ just below for details. The following examples are thought to be put in the remote repository section. + +The below will remove "INBOX." from the leading edge of folders (great +for Courier IMAP users):: + + nametrans = lambda folder: re.sub('^INBOX\.', '', folder) + +Using Courier remotely and want to duplicate its mailbox naming +locally? Try this:: + + nametrans = lambda folder: re.sub('^INBOX\.*', '.', folder) + +.. warning:: + You MUST construct nametrans rules such that it NEVER returns the + same value for two folders, UNLESS the second values are filtered + out by folderfilter below. That is, two filters on one side may + never point to the same folder on the other side. Failure to follow + this rule will result in undefined behavior. See also *Sharing a + maildir with multiple IMAP servers* in the :ref:`pitfalls` section. + +Reverse nametrans +^^^^^^^^^^^^^^^^^^ + +Since 6.4.0, OfflineImap supports the creation of folders on the remote repository and that complicates things. Previously, only one nametrans setting on the remote repository was needed and that transformed a remote to a local name. However, nametrans transformations are one-way, and OfflineImap has no way using those rules on the remote repository to back local names to remote names. + +Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts of any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. + +Take the above examples. If your remote nametrans setting was:: + + nametrans = lambda folder: re.sub('^INBOX\.', '', folder) + +then you will want to have this in your local repository, prepending "INBOX" to any local folder name:: + + nametrans = lambda folder: 'INBOX' + folder + +Failure to set the local nametrans rule will lead to weird-looking error messages of -for instance- this type:: + + ERROR: Creating folder moo.foo on repository remote + Folder 'moo.foo'[remote] could not be created. Server responded: ('NO', ['Unknown namespace.']) + +(This indicates that you attempted to create a folder "Sent" when all remote folders needed to be under the prefix of "INBOX."). + +OfflineImap will make some sanity checks if it needs to create a new +folder on the remote side and a back-and-forth nametrans-lation does not +yield the original foldername (as that could potentially lead to +infinite folder creation cycles). + +You can probably already see now that creating nametrans rules can be a pretty daunting and complex endeavour. Check out the Use cases in the manual. If you have some interesting use cases that we can present as examples here, please let us know. + +Debugging folderfilter and nametrans +------------------------------------ + +Given the complexity of the functions and regexes involved, it is easy to misconfigure things. One way to test your configuration without danger to corrupt anything or to create unwanted folders is to invoke offlineimap with the `--info` option. + +It will output a list of folders and their transformations on the screen (save them to a file with -l info.log), and will help you to tweak your rules as well as to understand your configuration. It also provides good output for bug reporting. + +FAQ on nametrans +---------------- + +Where to put nametrans rules, on the remote and/or local repository? +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +If you never intend to create new folders on the LOCAL repository that +need to be synced to the REMOTE repository, it is sufficient to create a +nametrans rule on the remote Repository section. This will be used to +determine the names of new folder names on the LOCAL repository, and to +match existing folders that correspond. + +*IF* you create folders on the local repository, that are supposed to be + automatically created on the remote repository, you will need to create + a nametrans rule that provides the reverse name translation. + +(A nametrans rule provides only a one-way translation of names and in +order to know which names folders on the LOCAL side would have on the +REMOTE side, you need to specify the reverse nametrans rule on the local +repository) + +OfflineImap will complain if it needs to create a new folder on the +remote side and a back-and-forth nametrans-lation does not yield the +original foldername (as that could potentially lead to infinite folder +creation cycles). + +What folder separators do I need to use in nametrans rules? ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +**Q:** If I sync from an IMAP server with folder separator '/' to a + Maildir using the default folder separator '.' which do I need to use + in nametrans rules?:: + + nametrans = lambda f: "INBOX/" + f +or:: + nametrans = lambda f: "INBOX." + f + +**A:** Generally use the folder separator as defined in the repository + you write the nametrans rule for. That is, use '/' in the above + case. We will pass in the untranslated name of the IMAP folder as + parameter (here `f`). The translated name will ultimately have all + folder separators be replaced with the destination repositories' + folder separator. + +So if 'f' was "Sent", the first nametrans yields the translated name +"INBOX/Sent" to be used on the other side. As that repository uses the +folder separator '.' rather than '/', the ultimate name to be used will +be "INBOX.Sent". + +(As a final note, the smart will see that both variants of the above +nametrans rule would have worked identically in this case) + diff --git a/offlineimap.conf b/offlineimap.conf index b4f6a99..627b237 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -1,11 +1,14 @@ # Offlineimap sample configuration file -# This file documents all possible options and can be quite scary. -# Looking for a quick start? Take a look at offlineimap.conf.minimal. -# Settings generally support interpolation. This means values can -# contain python format strings which refer to other values in the same -# section, or values in a special DEFAULT section. This allows you for -# example to use common settings for multiple accounts: +# This file documents *all* possible options and can be quite scary. +# Looking for a quick start? Take a look at offlineimap.conf.minimal. +# More details can be found in the included user documention, which is +# also available at: http://docs.offlineimap.org/en/latest/ + +# NOTE: Settings generally support python interpolation. This means +# values can contain python format strings which refer to other values +# in the same section, or values in a special DEFAULT section. This +# allows you for example to use common settings for multiple accounts: # # [Repository Gmail1] # trashfolder: %(gmailtrashfolder)s @@ -443,6 +446,10 @@ remoteuser = username # value. I suggest a lambda. This example below will remove "INBOX." from # the leading edge of folders (great for Courier IMAP users) # +# See the user documentation for details and use cases. They are also +# online at: +# http://docs.offlineimap.org/en/latest/nametrans.html +# # WARNING: you MUST construct this such that it NEVER returns # the same value for two folders, UNLESS the second values are # filtered out by folderfilter below. Failure to follow this rule diff --git a/offlineimap.conf.minimal b/offlineimap.conf.minimal index 2979d3d..3ca6e09 100644 --- a/offlineimap.conf.minimal +++ b/offlineimap.conf.minimal @@ -1,5 +1,5 @@ # Sample minimal config file. Copy this to ~/.offlineimaprc and edit to -# suit to get started fast. +# get started fast. [general] accounts = Test From f5b82ad76d40cc9ea900e7e887051c65370d0a0d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 13 Feb 2012 16:07:33 +0100 Subject: [PATCH 472/817] Improve nametrans user documentation Fill in more details on nametrans and folder filtering. Also give them a separate section in our user documentation. Everything will be immediately online at docs.offlineimap.org. The main change is to describe the reverse nametrans settings that are needed since 6.4.0 to support remote folder creation. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 + docs/MANUAL.rst | 151 ++++----------------------- docs/dev-doc-src/index.rst | 2 + docs/dev-doc-src/nametrans.rst | 182 +++++++++++++++++++++++++++++++++ offlineimap.conf | 19 ++-- offlineimap.conf.minimal | 2 +- 6 files changed, 221 insertions(+), 139 deletions(-) create mode 100644 docs/dev-doc-src/nametrans.rst diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..9a046ac 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,5 +16,9 @@ New Features Changes ------- +* internal code changes to prepare for Python3 + +* Improve user documentation of nametrans/folderfilter + Bug Fixes --------- diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 21a5dd6..8d7a8b7 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -328,18 +328,22 @@ Upgrading from plain text cache to SQLITE based cache OfflineImap uses a cache to store the last know status of mails (flags etc). Historically that has meant plain text files, but recently we introduced sqlite-based cache, which helps with performance and CPU usage on large folders. Here is how to upgrade existing plain text cache installations to sqlite based one: - 1) Sync to make sure things are reasonably similar - 3) Change the account section to status_backend = sqlite - 4) A new sync will convert your plain text cache to an sqlite cache (but - leave the old plain text cache around for easy reverting) - This should be quick and not involve any mail up/downloading. - 5) See if it works :-) - 6a) If it does not work, go back to the old version or set - status_backend=plain - 6b) Or once you are sure it works, you can delete the - .offlineimap/Account-foo/LocalStatus folder (the new cache will be in - the LocalStatus-sqlite folder) +1) Sync to make sure things are reasonably similar +2) Change the account section to status_backend = sqlite + +3) A new sync will convert your plain text cache to an sqlite cache + (but leave the old plain text cache around for easy reverting) This + should be quick and not involve any mail up/downloading. + +4) See if it works :-) + +5) If it does not work, go back to the old version or set + status_backend=plain + +6) Or, once you are sure it works, you can delete the + .offlineimap/Account-foo/LocalStatus folder (the new cache will be + in the LocalStatus-sqlite folder) Security and SSL ================ @@ -399,128 +403,10 @@ accounts will abort any current sleep and will exit after a currently running synchronization has finished. This signal can be used to gracefully exit out of a running offlineimap "daemon". -Folder filtering and Name translation -===================================== +Folder filtering and nametrans +============================== -OfflineImap provides advanced and potentially complex possibilities for -filtering and translating folder names. If you don't need this, you can -safely skip this section. - -folderfilter ------------- - -If you do not want to synchronize all your filters, you can specify a folderfilter function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. - -If the filter function returns True, the folder will be synced, if it -returns False, it. The folderfilter operates on the *UNTRANSLATED* name -(before any nametrans translation takes place). - -Example 1: synchronizing only INBOX and Sent:: - - folderfilter = lambda foldername: foldername in ['INBOX', 'Sent'] - -Example 2: synchronizing everything except Trash:: - - folderfilter = lambda foldername: foldername not in ['Trash'] - -Example 3: Using a regular expression to exclude Trash and all folders -containing the characters "Del":: - - folderfilter = lambda foldername: not re.search('(^Trash$|Del)', foldername) - -If folderfilter is not specified, ALL remote folders will be -synchronized. - -You can span multiple lines by indenting the others. (Use backslashes -at the end when required by Python syntax) For instance:: - - folderfilter = lambda foldername: foldername in - ['INBOX', 'Sent Mail', 'Deleted Items', - 'Received'] - -You only need a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. - -Even if you filtered out folders, You can specify folderincludes to -include additional folders. It should return a Python list. This might -be used to include a folder that was excluded by your folderfilter rule, -to include a folder that your server does not specify with its LIST -option, or to include a folder that is outside your basic reference. The -'reference' value will not be prefixed to this folder name, even if you -have specified one. For example:: - - folderincludes = ['debian.user', 'debian.personal'] - -nametrans ----------- - -Sometimes, folders need to have different names on the remote and the -local repositories. To achieve this you can specify a folder name -translator. This must be a eval-able Python expression that takes a -foldername arg and returns the new value. I suggest a lambda. This -example below will remove "INBOX." from the leading edge of folders -(great for Courier IMAP users):: - - nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) - -Using Courier remotely and want to duplicate its mailbox naming -locally? Try this:: - - nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) - - -WARNING: you MUST construct nametrans rules such that it NEVER returns -the same value for two folders, UNLESS the second values are -filtered out by folderfilter below. That is, two filters on one side may never point to the same folder on the other side. Failure to follow this rule -will result in undefined behavior. See also *Sharing a maildir with multiple IMAP servers* in the `PITFALLS & ISSUES`_ section. - -Where to put nametrans rules, on the remote and/or local repository? -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -If you never intend to create new folders on the LOCAL repository that -need to be synced to the REMOTE repository, it is sufficient to create a -nametrans rule on the remote Repository section. This will be used to -determine the names of new folder names on the LOCAL repository, and to -match existing folders that correspond. - -*IF* you create folders on the local repository, that are supposed to be - automatically created on the remote repository, you will need to create - a nametrans rule that provides the reverse name translation. - -(A nametrans rule provides only a one-way translation of names and in -order to know which names folders on the LOCAL side would have on the -REMOTE side, you need to specify the reverse nametrans rule on the local -repository) - -OfflineImap will complain if it needs to create a new folder on the -remote side and a back-and-forth nametrans-lation does not yield the -original foldername (as that could potentially lead to infinite folder -creation cycles). - -What folder separators do I need to use in nametrans rules? -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -**Q:** If I sync from an IMAP server with folder separator '/' to a - Maildir using the default folder separator '.' which do I need to use - in nametrans rules?:: - - nametrans = lambda f: "INBOX/" + f -or:: - nametrans = lambda f: "INBOX." + f - -**A:** Generally use the folder separator as defined in the repository - you write the nametrans rule for. That is, use '/' in the above - case. We will pass in the untranslated name of the IMAP folder as - parameter (here `f`). The translated name will ultimately have all - folder separators be replaced with the destination repositories' - folder separator. - -So if 'f' was "Sent", the first nametrans yields the translated name -"INBOX/Sent" to be used on the other side. As that repository uses the -folder separator '.' rather than '/', the ultimate name to be used will -be "INBOX.Sent". - -(As a final note, the smart will see that both variants of the above -nametrans rule would have worked identically in this case) +OfflineImap offers flexible (and complex) ways of filtering and transforming folder names. Please see the docs/dev-docs-src/folderfilters.rst document about details how to use folder filters and name transformations. The documentation will be autogenerated by a "make dev-doc" in the docs directory. It is also viewable at :ref:`folder_filtering_and_name_translation`. KNOWN BUGS ========== @@ -566,6 +452,7 @@ KNOWN BUGS * Use cygwin managed mount (not tested) - not available anymore since cygwin 1.7 +.. _pitfalls: PITFALLS & ISSUES ================= diff --git a/docs/dev-doc-src/index.rst b/docs/dev-doc-src/index.rst index e1b957e..d6aa939 100644 --- a/docs/dev-doc-src/index.rst +++ b/docs/dev-doc-src/index.rst @@ -17,6 +17,7 @@ More information on specific topics can be found on the following pages: **User documentation** * :doc:`installation/uninstall ` * :doc:`user manual/Configuration ` + * :doc:`Folder filtering & name transformation guide ` * :doc:`command line options ` * :doc:`Frequently Asked Questions ` @@ -29,6 +30,7 @@ More information on specific topics can be found on the following pages: INSTALL MANUAL + nametrans offlineimap FAQ diff --git a/docs/dev-doc-src/nametrans.rst b/docs/dev-doc-src/nametrans.rst new file mode 100644 index 0000000..fabf7a4 --- /dev/null +++ b/docs/dev-doc-src/nametrans.rst @@ -0,0 +1,182 @@ +.. _folder_filtering_and_name_translation: + +Folder filtering and Name translation +===================================== + +OfflineImap provides advanced and potentially complex possibilities for +filtering and translating folder names. If you don't need any of this, you can +safely skip this section. + +.. warning:: + Starting with v6.4.0, OfflineImap supports the creation of folders on the remote repostory. This change means that people that only had a nametrans option on the remote repository (everyone) will need to have a nametrans setting on the local repository too that will reverse the name transformation. See section `Reverse nametrans`_ for details. + +folderfilter +------------ + +If you do not want to synchronize all your filters, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. + +The only parameter to that function is the folder name. If the filter +function returns True, the folder will be synced, if it returns False, +it. will be skipped. The folderfilter operates on the *UNTRANSLATED* +name (before any `nametrans`_ fudging takes place). Consider the +examples below to get an idea of what they do. + +Example 1: synchronizing only INBOX and Sent:: + + folderfilter = lambda folder: folder in ['INBOX', 'Sent'] + +Example 2: synchronizing everything except Trash:: + + folderfilter = lambda folder: folder not in ['Trash'] + +Example 3: Using a regular expression to exclude Trash and all folders +containing the characters "Del":: + + folderfilter = lambda folder: not re.search('(^Trash$|Del)', folder) + +.. note:: + If folderfilter is not specified, ALL remote folders will be + synchronized. + +You can span multiple lines by indenting the others. (Use backslashes +at the end when required by Python syntax) For instance:: + + folderfilter = lambda foldername: foldername in + ['INBOX', 'Sent Mail', 'Deleted Items', + 'Received'] + +Usually it suffices to put a `folderfilter`_ setting in the remote repository section. You might want to put a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. (Even in this case, folder filters on the remote repository will prevent that) + +folderincludes +^^^^^^^^^^^^^^ + +You can specify `folderincludes`_ to manually include additional folders to be synced, even if they had been filtered out by a folderfilter setting. `folderincludes`_ should return a Python list. + +This can be used to 1) add a folder that was excluded by your +folderfilter rule, 2) to include a folder that your server does not specify +with its LIST option, or 3) to include a folder that is outside your basic +`reference`. The `reference` value will not be prefixed to this folder +name, even if you have specified one. For example:: + + folderincludes = ['debian.user', 'debian.personal'] + +This will add the "debian.user" and "debian.personal" folders even if you +have filtered out everything starting with "debian" in your folderfilter +settings. + + +nametrans +---------- + +Sometimes, folders need to have different names on the remote and the +local repositories. To achieve this you can specify a folder name +translator. This must be a eval-able Python expression that takes a +foldername arg and returns the new value. We suggest a lambda function, +but it could be any python function really. If you use nametrans rules, you will need to set them both on the remote and the local repository, see `Reverse nametrans`_ just below for details. The following examples are thought to be put in the remote repository section. + +The below will remove "INBOX." from the leading edge of folders (great +for Courier IMAP users):: + + nametrans = lambda folder: re.sub('^INBOX\.', '', folder) + +Using Courier remotely and want to duplicate its mailbox naming +locally? Try this:: + + nametrans = lambda folder: re.sub('^INBOX\.*', '.', folder) + +.. warning:: + You MUST construct nametrans rules such that it NEVER returns the + same value for two folders, UNLESS the second values are filtered + out by folderfilter below. That is, two filters on one side may + never point to the same folder on the other side. Failure to follow + this rule will result in undefined behavior. See also *Sharing a + maildir with multiple IMAP servers* in the :ref:`pitfalls` section. + +Reverse nametrans +^^^^^^^^^^^^^^^^^^ + +Since 6.4.0, OfflineImap supports the creation of folders on the remote repository and that complicates things. Previously, only one nametrans setting on the remote repository was needed and that transformed a remote to a local name. However, nametrans transformations are one-way, and OfflineImap has no way using those rules on the remote repository to back local names to remote names. + +Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts of any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. + +Take the above examples. If your remote nametrans setting was:: + + nametrans = lambda folder: re.sub('^INBOX\.', '', folder) + +then you will want to have this in your local repository, prepending "INBOX" to any local folder name:: + + nametrans = lambda folder: 'INBOX' + folder + +Failure to set the local nametrans rule will lead to weird-looking error messages of -for instance- this type:: + + ERROR: Creating folder moo.foo on repository remote + Folder 'moo.foo'[remote] could not be created. Server responded: ('NO', ['Unknown namespace.']) + +(This indicates that you attempted to create a folder "Sent" when all remote folders needed to be under the prefix of "INBOX."). + +OfflineImap will make some sanity checks if it needs to create a new +folder on the remote side and a back-and-forth nametrans-lation does not +yield the original foldername (as that could potentially lead to +infinite folder creation cycles). + +You can probably already see now that creating nametrans rules can be a pretty daunting and complex endeavour. Check out the Use cases in the manual. If you have some interesting use cases that we can present as examples here, please let us know. + +Debugging folderfilter and nametrans +------------------------------------ + +Given the complexity of the functions and regexes involved, it is easy to misconfigure things. One way to test your configuration without danger to corrupt anything or to create unwanted folders is to invoke offlineimap with the `--info` option. + +It will output a list of folders and their transformations on the screen (save them to a file with -l info.log), and will help you to tweak your rules as well as to understand your configuration. It also provides good output for bug reporting. + +FAQ on nametrans +---------------- + +Where to put nametrans rules, on the remote and/or local repository? +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +If you never intend to create new folders on the LOCAL repository that +need to be synced to the REMOTE repository, it is sufficient to create a +nametrans rule on the remote Repository section. This will be used to +determine the names of new folder names on the LOCAL repository, and to +match existing folders that correspond. + +*IF* you create folders on the local repository, that are supposed to be + automatically created on the remote repository, you will need to create + a nametrans rule that provides the reverse name translation. + +(A nametrans rule provides only a one-way translation of names and in +order to know which names folders on the LOCAL side would have on the +REMOTE side, you need to specify the reverse nametrans rule on the local +repository) + +OfflineImap will complain if it needs to create a new folder on the +remote side and a back-and-forth nametrans-lation does not yield the +original foldername (as that could potentially lead to infinite folder +creation cycles). + +What folder separators do I need to use in nametrans rules? ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +**Q:** If I sync from an IMAP server with folder separator '/' to a + Maildir using the default folder separator '.' which do I need to use + in nametrans rules?:: + + nametrans = lambda f: "INBOX/" + f +or:: + nametrans = lambda f: "INBOX." + f + +**A:** Generally use the folder separator as defined in the repository + you write the nametrans rule for. That is, use '/' in the above + case. We will pass in the untranslated name of the IMAP folder as + parameter (here `f`). The translated name will ultimately have all + folder separators be replaced with the destination repositories' + folder separator. + +So if 'f' was "Sent", the first nametrans yields the translated name +"INBOX/Sent" to be used on the other side. As that repository uses the +folder separator '.' rather than '/', the ultimate name to be used will +be "INBOX.Sent". + +(As a final note, the smart will see that both variants of the above +nametrans rule would have worked identically in this case) + diff --git a/offlineimap.conf b/offlineimap.conf index 2656ef6..3836895 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -1,11 +1,14 @@ # Offlineimap sample configuration file -# This file documents all possible options and can be quite scary. -# Looking for a quick start? Take a look at offlineimap.conf.minimal. -# Settings generally support interpolation. This means values can -# contain python format strings which refer to other values in the same -# section, or values in a special DEFAULT section. This allows you for -# example to use common settings for multiple accounts: +# This file documents *all* possible options and can be quite scary. +# Looking for a quick start? Take a look at offlineimap.conf.minimal. +# More details can be found in the included user documention, which is +# also available at: http://docs.offlineimap.org/en/latest/ + +# NOTE: Settings generally support python interpolation. This means +# values can contain python format strings which refer to other values +# in the same section, or values in a special DEFAULT section. This +# allows you for example to use common settings for multiple accounts: # # [Repository Gmail1] # trashfolder: %(gmailtrashfolder)s @@ -443,6 +446,10 @@ remoteuser = username # value. I suggest a lambda. This example below will remove "INBOX." from # the leading edge of folders (great for Courier IMAP users) # +# See the user documentation for details and use cases. They are also +# online at: +# http://docs.offlineimap.org/en/latest/nametrans.html +# # WARNING: you MUST construct this such that it NEVER returns # the same value for two folders, UNLESS the second values are # filtered out by folderfilter below. Failure to follow this rule diff --git a/offlineimap.conf.minimal b/offlineimap.conf.minimal index 2979d3d..3ca6e09 100644 --- a/offlineimap.conf.minimal +++ b/offlineimap.conf.minimal @@ -1,5 +1,5 @@ # Sample minimal config file. Copy this to ~/.offlineimaprc and edit to -# suit to get started fast. +# get started fast. [general] accounts = Test From c1f75a6c945609d1f2a9ff51e5db259ff52e50f5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Feb 2012 09:55:26 +0100 Subject: [PATCH 473/817] test: Create all intermediary maildirs Don't fail if root maildir folder does not exist it. Create it recursively. Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 183f67a..742d118 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -108,7 +108,7 @@ class OLITestLib(): maildir = os.path.join(cls.testdir, 'mail', folder) for subdir in ('','tmp','cur','new'): try: - os.mkdir(os.path.join(maildir, subdir)) + os.makedirs(os.path.join(maildir, subdir)) except OSError as e: if e.errno != 17: # 'already exists' is ok. raise From 75ea403278ec080d110589914fb1cb63431836bd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Feb 2012 09:31:14 +0100 Subject: [PATCH 474/817] test: Split configuration generation so we can tweak values Use get_default_config and write_config_file (which can be handed an optional config object), so we can manipulate configuration options easily from within the test function. Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 742d118..3c21257 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -54,23 +54,36 @@ class OLITestLib(): cls.testdir = os.path.abspath( tempfile.mkdtemp(prefix='tmp_%s_'%suffix, dir=os.path.dirname(cls.cred_file))) - cls.create_config_file() + cls.write_config_file() return cls.testdir @classmethod - def create_config_file(cls): - """Creates a OLI configuration file + def get_default_config(cls): + """Creates a default ConfigParser file and returns it - It is created in testdir (so create_test_dir has to be called - earlier) using the credentials information given (so they had to - be set earlier). Failure to do either of them will raise an - AssertionException.""" + The returned config can be manipulated and then saved with + write_config_file()""" assert cls.cred_file != None assert cls.testdir != None config = SafeConfigParser() config.readfp(default_conf) + default_conf.seek(0) # rewind config_file to start config.read(cls.cred_file) config.set("general", "metadata", cls.testdir) + return config + + @classmethod + def write_config_file(cls, config=None): + """Creates a OLI configuration file + + It is created in testdir (so create_test_dir has to be called + earlier) using the credentials information given (so they had + to be set earlier). Failure to do either of them will raise an + AssertionException. If config is None, a default one will be + used via get_default_config, otherwise it needs to be a config + object derived from that.""" + if config is None: + config = cls.get_default_config() localfolders = os.path.join(cls.testdir, 'mail') config.set("Repository Maildir", "localfolders", localfolders) with open(os.path.join(cls.testdir, 'offlineimap.conf'), "wa") as f: From 10dd317026fbbb62d2f2b2edad5ac8a0f20b2da9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Feb 2012 11:03:33 +0100 Subject: [PATCH 475/817] Folder: Implement __eq__ for folders This allows to compare folders directly with strings. It also allows constructs such as "if 'moo' in repo.getfolders()". See the code documentation for the exact behavior (it basically is equal if it's the same instance *or* a string matching the untranslated folder name. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 1c05370..fcea99d 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -479,3 +479,20 @@ class BaseFolder(object): self.ui.error(e, exc_info()[2], "Syncing folder %s [acc: %s]" %\ (self, self.accountname)) raise # raise unknown Exceptions so we can fix them + + def __eq__(self, other): + """Comparisons work either on string comparing folder names or + on the same instance + + MailDirFolder('foo') == 'foo' --> True + a = MailDirFolder('foo'); a == b --> True + MailDirFolder('foo') == 'moo' --> False + MailDirFolder('foo') == IMAPFolder('foo') --> False + MailDirFolder('foo') == MaildirFolder('foo') --> False + """ + if isinstance(other, basestring): + return other == self.name + return id(self) == id(other) + + def __ne__(self, other): + return not self.__eq__(other) From 189d78cc5c5215ad265f23374a13209d49dacec8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Feb 2012 11:12:07 +0100 Subject: [PATCH 476/817] sync_folder_structure: make more readable Rename variable src_name to src_name_t to indicate that it is the transposed name. Also rather than testing the hash thingie, we can simply test for "if source_name_t in dst_folders" now. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index cca5cf3..22fc15f 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -156,20 +156,20 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): dst_hash[folder.name] = folder # Find new folders on src_repo. - for src_name, src_folder in src_hash.iteritems(): + for src_name_t, src_folder in src_hash.iteritems(): # Don't create on dst_repo, if it is readonly if dst_repo.getconfboolean('readonly', False): break - if src_folder.sync_this and not src_name in dst_hash: + if src_folder.sync_this and not src_name_t in dst_folders: try: - dst_repo.makefolder(src_name) + dst_repo.makefolder(src_name_t) dst_haschanged = True # Need to refresh list except OfflineImapError as e: self.ui.error(e, exc_info()[2], "Creating folder %s on repository %s" %\ - (src_name, dst_repo)) + (src_name_t, dst_repo)) raise - status_repo.makefolder(src_name.replace(dst_repo.getsep(), + status_repo.makefolder(src_name_t.replace(dst_repo.getsep(), status_repo.getsep())) # Find new folders on dst_repo. for dst_name, dst_folder in dst_hash.iteritems(): From ac033c68fd693029fc53bc697eb6be892a368145 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Feb 2012 11:45:18 +0100 Subject: [PATCH 477/817] Improve nametrans local->remote folder syncing While improving the test suite, I noticed that we would not create folders on the remote in some cases when we should (yay for test suites!). This is because we were testing the untransposed LOCAL foldername and check if it existed on the remote side when deciding whether we should potentially create a new folder. Simplify the code by transposing the LOCAL folder names in dst_hash, saving us to create another confusing "newsrc" temp variable. Make the code a bit more readable by using dst_name_t to indicate we operate a transposed folder name. This now passes test 03 (using invalid nametrans rules) when test 03 would pass before. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 33 +++++++++++++++++---------------- test/tests/test_01_basic.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 22fc15f..e855210 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -153,7 +153,8 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): src_repo.getsep(), dst_repo.getsep())] = folder dst_hash = {} for folder in dst_folders: - dst_hash[folder.name] = folder + dst_hash[folder.getvisiblename().replace( + dst_repo.getsep(), src_repo.getsep())] = folder # Find new folders on src_repo. for src_name_t, src_folder in src_hash.iteritems(): @@ -172,30 +173,30 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): status_repo.makefolder(src_name_t.replace(dst_repo.getsep(), status_repo.getsep())) # Find new folders on dst_repo. - for dst_name, dst_folder in dst_hash.iteritems(): + for dst_name_t, dst_folder in dst_hash.iteritems(): if self.getconfboolean('readonly', False): # Don't create missing folder on readonly repo. break - if dst_folder.sync_this and not dst_name in src_hash: + if dst_folder.sync_this and not dst_name_t in src_folders: # nametrans sanity check! # Does nametrans back&forth lead to identical names? - #src_name is the unmodified full src_name that would be created - newsrc_name = dst_folder.getvisiblename().replace( - dst_repo.getsep(), - src_repo.getsep()) - folder = self.getfolder(newsrc_name) - # would src repo filter out the new folder name? In this + # 1) would src repo filter out the new folder name? In this # case don't create it on it: - if not self.folderfilter(newsrc_name): + if not self.folderfilter(dst_name_t): self.ui.debug('', "Not creating folder '%s' (repository '%s" "') as it would be filtered out on that repository." % - (newsrc_name, self)) + (dst_name_t, self)) continue + # get IMAPFolder and see if the reverse nametrans + # works fine TODO: getfolder() works only because we + # succeed in getting inexisting folders which I would + # like to change. Take care! + folder = self.getfolder(dst_name_t) # apply reverse nametrans to see if we end up with the same name newdst_name = folder.getvisiblename().replace( src_repo.getsep(), dst_repo.getsep()) - if dst_name != newdst_name: + if dst_folder.name != newdst_name: raise OfflineImapError("INFINITE FOLDER CREATION DETECTED! " "Folder '%s' (repository '%s') would be created as fold" "er '%s' (repository '%s'). The latter becomes '%s' in " @@ -204,18 +205,18 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): "itories so they lead to identical names if applied bac" "k and forth. 2) Use folderfilter settings on a reposit" "ory to prevent some folders from being created on the " - "other side." % (dst_name, dst_repo, newsrc_name, + "other side." % (dst_folder.name, dst_repo, dst_name_t, src_repo, newdst_name), OfflineImapError.ERROR.REPO) # end sanity check, actually create the folder try: - src_repo.makefolder(newsrc_name) + src_repo.makefolder(dst_name_t) src_haschanged = True # Need to refresh list except OfflineImapError as e: self.ui.error(e, exc_info()[2], "Creating folder %s on " - "repository %s" % (newsrc_name, src_repo)) + "repository %s" % (dst_name_t, src_repo)) raise - status_repo.makefolder(newsrc_name.replace( + status_repo.makefolder(dst_name_t.replace( src_repo.getsep(), status_repo.getsep())) # Find deleted folders. # TODO: We don't delete folders right now. diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index a0697ec..926baa0 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -72,3 +72,21 @@ class TestBasicFunctions(unittest.TestCase): self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') logging.warn("%d boxes and %d mails" % (boxes, mails)) + + def test_03_nametransmismatch(self): + """Create mismatching remote and local nametrans rules + + This should raise an error.""" + config = OLITestLib.get_default_config() + config.set('Repository IMAP', 'nametrans', + 'lambda f: f' ) + config.set('Repository Maildir', 'nametrans', + 'lambda f: f + "moo"' ) + OLITestLib.write_config_file(config) + code, res = OLITestLib.run_OLI() + #logging.warn("%s %s "% (code, res)) + # We expect an INFINITE FOLDER CREATION WARNING HERE.... + mismatch = "ERROR: INFINITE FOLDER CREATION DETECTED!" in res + self.assertEqual(mismatch, True, "Mismatching nametrans rules did NOT" + "trigger an 'infinite folder generation' error.") + boxes, mails = OLITestLib.count_maildir_mails('') From b69a74541737d84ed7c1229f7414564a9c7171cf Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Feb 2012 11:54:53 +0100 Subject: [PATCH 478/817] Changelog entry Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 9a046ac..a612ab8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -20,5 +20,10 @@ Changes * Improve user documentation of nametrans/folderfilter +* Fixed some cases where invalid nametrans rules were not caught and + we would not propagate local folders to the remote repository. + (now tested in test03) + + Bug Fixes --------- From 7da50e638d45bcea1d456c6fadbe2a17934660df Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 16 Feb 2012 16:49:06 +0100 Subject: [PATCH 479/817] folder/IMAP: better error when savemessage fails If we cannot identify the new UID after a sendmessage(), log a better error message, including the server response for better debugging. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index d06c740..8b46996 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -577,7 +577,10 @@ class IMAPFolder(BaseFolder): "appending a message.") else: uid = long(resp[-1].split(' ')[1]) - + if uid == 0: + self.ui.warn("savemessage: Server supports UIDPLUS, but" + " we got no usable uid back. APPENDUID reponse was " + "'%s'" % str(resp)) else: # Don't support UIDPLUS # Checkpoint. Let it write out stuff, etc. Eg searches for @@ -593,9 +596,11 @@ class IMAPFolder(BaseFolder): # compare the message size... if uid == 0: self.ui.debug('imap', 'savemessage: attempt to get new UID ' - 'UID failed. Search headers manually.') + 'UID failed. Search headers manually.') uid = self.savemessage_fetchheaders(imapobj, headername, headervalue) + self.ui.warn('imap', "savemessage: Searching mails for new " + "Message-ID failed. Could not determine new UID.") finally: self.imapserver.releaseconnection(imapobj) From bf44d30b46d530b0c6723a2128b4cd5a8ba1ff95 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 08:48:59 +0100 Subject: [PATCH 480/817] UIDMaps: Better error message when not finding a mapping Bail out with a better Exception and error text. The whole mapped UID situation needs to be improved though. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/UIDMaps.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index f306459..99ad2d3 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -82,7 +82,13 @@ class MappedIMAPFolder(IMAPFolder): if dolock: self.maplock.release() def _uidlist(self, mapping, items): - return [mapping[x] for x in items] + try: + return [mapping[x] for x in items] + except KeyError as e: + raise OfflineImapError("Could not find UID for msg '{0}' (f:'{1}'." + " This is usually a bad thing and should be reported on the ma" + "iling list.".format(e.args[0], self), + OfflineImapError.ERROR.MESSAGE) def cachemessagelist(self): self._mb.cachemessagelist() From 79ddb0be71fb83968dc578969cb06405439f913e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 10:12:53 +0100 Subject: [PATCH 481/817] tests: add delete remote folder helper function We need to clean out the remote folders before we invoke the test suite. Implement a helper function that does this, and improve the test output (less verbose) and the setup.py --help-commands (more verbose). Document that it is possible to run a single test only. (although it is not guaranteed that a test does not rely on the output of previous tests). Signed-off-by: Sebastian Spaeth --- setup.py | 5 +++- test/OLItest/TestRunner.py | 49 +++++++++++++++++++++++++++++++++++++ test/tests/test_01_basic.py | 28 +++++++++++++++------ 3 files changed, 73 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 3d04af2..27932eb 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,10 @@ from test.OLItest import TextTestRunner, TestLoader, OLITestLib class TestCommand(Command): """runs the OLI testsuite""" - description = "Runs the test suite" + description = """Runs the test suite. In order to execute only a single + test, you could also issue e.g. 'python -m unittest + test.tests.test_01_basic.TestBasicFunctions.test_01_olistartup' on the + command line.""" user_options = [] def initialize_options(self): diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 3c21257..e6277f2 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -13,9 +13,11 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import imaplib import unittest import logging import os +import re import sys import shutil import subprocess @@ -50,6 +52,7 @@ class OLITestLib(): directory at a time. OLITestLib is not suited for running several tests in parallel. The user is responsible for cleaning that up herself.""" + assert cls.cred_file != None # creating temporary dir for testing in same dir as credentials.conf cls.testdir = os.path.abspath( tempfile.mkdtemp(prefix='tmp_%s_'%suffix, @@ -63,6 +66,7 @@ class OLITestLib(): The returned config can be manipulated and then saved with write_config_file()""" + #TODO, only do first time and cache then for subsequent calls? assert cls.cred_file != None assert cls.testdir != None config = SafeConfigParser() @@ -112,6 +116,51 @@ class OLITestLib(): return (e.returncode, e.output) return (0, output) + @classmethod + def delete_remote_testfolders(cls, reponame=None): + """Delete all INBOX.OLITEST* folders on the remote IMAP repository + + reponame: All on `reponame` or all IMAP-type repositories if None""" + config = cls.get_default_config() + if reponame: + sections = ['Repository {}'.format(reponame)] + else: + sections = [r for r in config.sections() \ + if r.startswith('Repository')] + sections = filter(lambda s: \ + config.get(s, 'Type', None).lower() == 'imap', + sections) + for sec in sections: + # Connect to each IMAP repo and delete all folders + # matching the folderfilter setting. We only allow basic + # settings and no fancy password getting here... + # 1) connect and get dir listing + host = config.get(sec, 'remotehost') + user = config.get(sec, 'remoteuser') + passwd = config.get(sec, 'remotepass') + imapobj = imaplib.IMAP4(host) + imapobj.login(user, passwd) + res_t, data = imapobj.list() + assert res_t == 'OK' + dirs = [] + for d in data: + m = re.search(r''' # Find last quote + "((?: # Non-tripple quoted can contain... + [^"] | # a non-quote + \\" # a backslashded quote + )*)" # closing quote + [^"]*$ # followed by no more quotes + ''', d, flags=re.VERBOSE) + folder = m.group(1) + folder = folder.replace(r'\"', '"') # remove quoting + dirs.append(folder) + # 2) filter out those not starting with INBOX.OLItest and del... + dirs = [d for d in dirs if d.startswith('INBOX.OLItest')] + for folder in dirs: + res_t, data = imapobj.delete(folder) + assert res_t == 'OK' + imapobj.logout() + @classmethod def create_maildir(cls, folder): """Create empty maildir 'folder' in our test maildir diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 926baa0..0967d9d 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -19,6 +19,12 @@ import logging import os, sys from test.OLItest import OLITestLib +# Things need to be setup first, usually setup.py initializes everything. +# but if e.g. called from command line, we take care of default values here: +if not OLITestLib.cred_file: + OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') + + def setUpModule(): logging.info("Set Up test module %s" % __name__) tdir = OLITestLib.create_test_dir(suffix=__name__) @@ -52,13 +58,16 @@ class TestBasicFunctions(unittest.TestCase): def test_01_olistartup(self): """Tests if OLI can be invoked without exceptions - It syncs all "OLItest* (specified in the default config) to our - local Maildir at keeps it there.""" + Cleans existing remote tet folders. Then syncs all "OLItest* + (specified in the default config) to our local Maildir. The + result should be 0 folders and 0 mails.""" + OLITestLib.delete_remote_testfolders() code, res = OLITestLib.run_OLI() self.assertEqual(res, "") - boxes, mails = OLITestLib.count_maildir_mails('') - logging.warn("%d boxes and %d mails" % (boxes, mails)) + self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0" + "mails, but sync led to {} folders and {} mails".format( + boxes, mails)) def test_02_createdir(self): """Create local OLItest 1 & OLItest "1" maildir, sync @@ -71,7 +80,9 @@ class TestBasicFunctions(unittest.TestCase): #logging.warn("%s %s "% (code, res)) self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') - logging.warn("%d boxes and %d mails" % (boxes, mails)) + self.assertTrue((boxes, mails)==(2,0), msg="Expected 2 folders and 0" + "mails, but sync led to {} folders and {} mails".format( + boxes, mails)) def test_03_nametransmismatch(self): """Create mismatching remote and local nametrans rules @@ -87,6 +98,7 @@ class TestBasicFunctions(unittest.TestCase): #logging.warn("%s %s "% (code, res)) # We expect an INFINITE FOLDER CREATION WARNING HERE.... mismatch = "ERROR: INFINITE FOLDER CREATION DETECTED!" in res - self.assertEqual(mismatch, True, "Mismatching nametrans rules did NOT" - "trigger an 'infinite folder generation' error.") - boxes, mails = OLITestLib.count_maildir_mails('') + self.assertEqual(mismatch, True, msg="Mismatching nametrans rules did " + "NOT trigger an 'infinite folder generation' error. Output was:\n" + "{}".format(res)) + From 5979cc8ff9867ec30592e8e97aaad5f3996fa1d8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 10:26:43 +0100 Subject: [PATCH 482/817] tests: Add MappedIMAP test skeleton No working tests yet. Signed-off-by: Sebastian Spaeth --- test/tests/test_02_MappedIMAP.py | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 test/tests/test_02_MappedIMAP.py diff --git a/test/tests/test_02_MappedIMAP.py b/test/tests/test_02_MappedIMAP.py new file mode 100644 index 0000000..05aa394 --- /dev/null +++ b/test/tests/test_02_MappedIMAP.py @@ -0,0 +1,71 @@ +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import random +import unittest +import logging +import os, sys +from test.OLItest import OLITestLib + +# Things need to be setup first, usually setup.py initializes everything. +# but if e.g. called from command line, we take care of default values here: +if not OLITestLib.cred_file: + OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') + + +def setUpModule(): + logging.info("Set Up test module %s" % __name__) + tdir = OLITestLib.create_test_dir(suffix=__name__) + +def tearDownModule(): + logging.info("Tear Down test module") + OLITestLib.delete_test_dir() + +#Stuff that can be used +#self.assertEqual(self.seq, range(10)) +# should raise an exception for an immutable sequence +#self.assertRaises(TypeError, random.shuffle, (1,2,3)) +#self.assertTrue(element in self.seq) +#self.assertFalse(element in self.seq) + +class TestBasicFunctions(unittest.TestCase): + #@classmethod + #def setUpClass(cls): + #This is run before all tests in this class + # cls._connection = createExpensiveConnectionObject() + + #@classmethod + #This is run after all tests in this class + #def tearDownClass(cls): + # cls._connection.destroy() + + # This will be run before each test + #def setUp(self): + # self.seq = range(10) + + def test_01_MappedImap(self): + """Tests if a MappedIMAP sync can be invoked without exceptions + + Cleans existing remote test folders. Then syncs all "OLItest* + (specified in the default config) to our local IMAP (Gmail). The + result should be 0 folders and 0 mails.""" + pass #TODO + #OLITestLib.delete_remote_testfolders() + #code, res = OLITestLib.run_OLI() + #self.assertEqual(res, "") + #boxes, mails = OLITestLib.count_maildir_mails('') + #self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0" + # "mails, but sync led to {} folders and {} mails".format( + # boxes, mails)) From a8c6407f50dfcdebbb3de38713637848bcadc241 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 13:08:04 +0200 Subject: [PATCH 483/817] Implement CustomConfig.set_if_not_exists() A convenience helper function that allows to set a configuration value if the user has not explicitly configured anything ie the option does not exist yet in the configuration. It won't do anything, if the option exists. Signed-off-by: Sebastian Spaeth --- offlineimap/CustomConfig.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index b5869e7..3cd1cef 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -71,6 +71,14 @@ class CustomConfigParser(SafeConfigParser): return [x[len(key):] for x in self.sections() \ if x.startswith(key)] + def set_if_not_exists(self, section, option, value): + """Set a value if it does not exist yet + + This allows to set default if the user has not explicitly + configured anything.""" + if not self.has_option(section, option): + self.set(section, option, value) + def CustomConfigDefault(): """Just a constant that won't occur anywhere else. From b7e0a51751ac95138095969121c612f34047e52d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 13:24:48 +0200 Subject: [PATCH 484/817] Add command line option --dry-run And set the [general]dry-run=True setting if yes. It is not used yet. Also set ui.dryrun to True so we can output what WE WOULD HAVE DONE in dryrun mode. Signed-off-by: Sebastian Spaeth --- offlineimap/init.py | 13 +++++++++++++ offlineimap/ui/UIBase.py | 2 ++ 2 files changed, 15 insertions(+) diff --git a/offlineimap/init.py b/offlineimap/init.py index 51d535d..d381a65 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -52,6 +52,13 @@ class OfflineImap: description="%s.\n\n%s" % (offlineimap.__copyright__, offlineimap.__license__)) + parser.add_option("--dry-run", + action="store_true", dest="dryrun", + default=False, + help="Do not actually modify any store but check and print " + "what synchronization actions would be taken if a sync would be" + " performed.") + parser.add_option("-1", action="store_true", dest="singlethreading", default=False, @@ -201,6 +208,12 @@ class OfflineImap: logging.warning('Using old interface name, consider using one ' 'of %s' % ', '.join(UI_LIST.keys())) if options.diagnostics: ui_type = 'basic' # enforce basic UI for --info + + #dry-run? Set [general]dry-run=True + if options.dryrun: + dryrun = config.set('general','dry-run', "True") + config.set_if_not_exists('general','dry-run','False') + try: # create the ui class self.ui = UI_LIST[ui_type.lower()](config) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index a8aeeaf..d716968 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -48,6 +48,8 @@ def getglobalui(): class UIBase(object): def __init__(self, config, loglevel = logging.INFO): self.config = config + # Is this a 'dryrun'? + self.dryrun = config.getboolean('general', 'dry-run') self.debuglist = [] """list of debugtypes we are supposed to log""" self.debugmessages = {} From 33f55b5362f6edef64d00bb41206d083ff285d73 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 13:56:04 +0200 Subject: [PATCH 485/817] Implement dry-run on Account() level 1) Set attribute self.dryrun depending on whether we are in dry-run mode. 2) Don't actually call hooks in --dry-run (just log what you would invoke 3) Don't write out the mbnames file in --dry-run mode. Repository, and Folder levels still need to be protected in dry-run mode as of now. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 10 ++++++++-- offlineimap/ui/UIBase.py | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index ed08cad..8b39c7e 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -65,9 +65,11 @@ class Account(CustomConfig.ConfigHelperMixin): self.name = name self.metadatadir = config.getmetadatadir() self.localeval = config.getlocaleval() - #Contains the current :mod:`offlineimap.ui`, and can be used for logging etc. + # current :mod:`offlineimap.ui`, can be used for logging: self.ui = getglobalui() self.refreshperiod = self.getconffloat('autorefresh', 0.0) + # should we run in "dry-run" mode? + self.dryrun = self.config.getboolean('general', 'dry-run') self.quicknum = 0 if self.refreshperiod == 0.0: self.refreshperiod = None @@ -312,7 +314,9 @@ class SyncableAccount(Account): # wait for all threads to finish for thr in folderthreads: thr.join() - mbnames.write() + # Write out mailbox names if required and not in dry-run mode + if not self.dryrun: + mbnames.write() localrepos.forgetfolders() remoterepos.forgetfolders() except: @@ -337,6 +341,8 @@ class SyncableAccount(Account): return try: self.ui.callhook("Calling hook: " + cmd) + if self.dryrun: # don't if we are in dry-run mode + return p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index d716968..da77b5a 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -461,7 +461,10 @@ class UIBase(object): ################################################## Hooks def callhook(self, msg): - self.info(msg) + if self.dryrun: + self.info("[DRYRUN] {}".format(msg)) + else: + self.info(msg) ################################################## Other From b6807355b5133dae5c669381aa9965f44c38b7a0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 15 Sep 2011 15:37:52 +0200 Subject: [PATCH 486/817] Implement ui.makefolder and abort repo.makefolder() in dry-run mode IMAP, Maildir, and LocalStatus abort if in dry-run mode. IMAP and Maildir will log that they "would have" created a new folder. This will probably fail later on as we can not cache messagelists on folder that don't exist, so --dry-run is not yet safe when new folders have been created. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 1 + offlineimap/repository/IMAP.py | 6 +++--- offlineimap/repository/LocalStatus.py | 7 ++++--- offlineimap/repository/Maildir.py | 4 +++- offlineimap/ui/UIBase.py | 6 ++++++ 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index e855210..606294e 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -124,6 +124,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): raise NotImplementedError def makefolder(self, foldername): + """Create a new folder""" raise NotImplementedError def deletefolder(self, foldername): diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 1b13d79..28e73b7 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -341,11 +341,11 @@ class IMAPRepository(BaseRepository): foldername = self.getreference() + self.getsep() + foldername if not foldername: # Create top level folder as folder separator foldername = self.getsep() - + self.ui.makefolder(self, foldername) + if self.account.dryrun: + return imapobj = self.imapserver.acquireconnection() try: - self.ui._msg("Creating new IMAP folder '%s' on server %s" %\ - (foldername, self)) result = imapobj.create(foldername) if result[0] != 'OK': raise OfflineImapError("Folder '%s'[%s] could not be created. " diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 30c8560..3e5db2f 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -67,10 +67,11 @@ class LocalStatusRepository(BaseRepository): Empty Folder for plain backend. NoOp for sqlite backend as those are created on demand.""" - # Invalidate the cache. - self._folders = None if self._backend == 'sqlite': - return + return # noop for sqlite which creates on-demand + + if self.account.dryrun: + return # bail out in dry-run mode filename = self.getfolderfilename(foldername) file = open(filename + ".tmp", "wt") diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 7c08d64..f197002 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -81,7 +81,9 @@ class MaildirRepository(BaseRepository): levels will be created if they do not exist yet. 'cur', 'tmp', and 'new' subfolders will be created in the maildir. """ - self.debug("makefolder called with arg '%s'" % (foldername)) + self.ui.makefolder(self, foldername) + if self.account.dryrun: + return full_path = os.path.abspath(os.path.join(self.root, foldername)) # sanity tests diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index da77b5a..4ff98c3 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -298,6 +298,12 @@ class UIBase(object): (src_repo, dst_repo)) ############################## Folder syncing + def makefolder(self, repo, foldername): + """Called when a folder is created""" + prefix = "[DRYRUN] " if self.dryrun else "" + self.info("{}Creating folder {}[{}]".format( + prefix, foldername, repo)) + def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder): """Called when a folder sync operation is started.""" self.logger.info("Syncing %s: %s -> %s" % (srcfolder, From 5ef69e95c02f69f0a13818d418e6a5768bd4be04 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 16 Sep 2011 11:44:50 +0200 Subject: [PATCH 487/817] Prevent modifications on a folder level to occur in dry-run Prevent savemessage(), and savemessageflags() to occur in dryrun mode in all backends. Still need to protect against deletemessage(). Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 37 +++++++++++++++++++++++-- offlineimap/folder/IMAP.py | 12 ++++++-- offlineimap/folder/LocalStatus.py | 5 ++++ offlineimap/folder/LocalStatusSQLite.py | 5 ++++ offlineimap/folder/Maildir.py | 14 ++++++++-- offlineimap/folder/UIDMaps.py | 10 +++++++ offlineimap/ui/UIBase.py | 5 ++++ 7 files changed, 81 insertions(+), 7 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index fcea99d..01a6ab5 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -208,6 +208,10 @@ class BaseFolder(object): If the uid is > 0, the backend should set the uid to this, if it can. If it cannot set the uid to that, it will save it anyway. It will return the uid assigned in any case. + + Note that savemessage() does not check against dryrun settings, + so you need to ensure that savemessage is never called in a + dryrun mode. """ raise NotImplementedException @@ -220,27 +224,48 @@ class BaseFolder(object): raise NotImplementedException def savemessageflags(self, uid, flags): - """Sets the specified message's flags to the given set.""" + """Sets the specified message's flags to the given set. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" raise NotImplementedException def addmessageflags(self, uid, flags): """Adds the specified flags to the message's flag set. If a given flag is already present, it will not be duplicated. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + :param flags: A set() of flags""" newflags = self.getmessageflags(uid) | flags self.savemessageflags(uid, newflags) def addmessagesflags(self, uidlist, flags): + """ + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" for uid in uidlist: self.addmessageflags(uid, flags) def deletemessageflags(self, uid, flags): """Removes each flag given from the message's flag set. If a given - flag is already removed, no action will be taken for that flag.""" + flag is already removed, no action will be taken for that flag. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" newflags = self.getmessageflags(uid) - flags self.savemessageflags(uid, newflags) def deletemessagesflags(self, uidlist, flags): + """ + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" for uid in uidlist: self.deletemessageflags(uid, flags) @@ -345,6 +370,10 @@ class BaseFolder(object): statusfolder.uidexists(uid), self.getmessageuidlist()) num_to_copy = len(copylist) + if num_to_copy and self.repository.account.dryrun: + self.ui.info("[DRYRUN] Copy {} messages from {}[{}] to {}".format( + num_to_copy, self, self.repository, dstfolder.repository)) + return for num, uid in enumerate(copylist): # bail out on CTRL-C or SIGTERM if offlineimap.accounts.Account.abort_NOW_signal.is_set(): @@ -422,11 +451,15 @@ class BaseFolder(object): for flag, uids in addflaglist.items(): self.ui.addingflags(uids, flag, dstfolder) + if self.repository.account.dryrun: + continue #don't actually add in a dryrun dstfolder.addmessagesflags(uids, set(flag)) statusfolder.addmessagesflags(uids, set(flag)) for flag,uids in delflaglist.items(): self.ui.deletingflags(uids, flag, dstfolder) + if self.repository.account.dryrun: + continue #don't actually remove in a dryrun dstfolder.deletemessagesflags(uids, set(flag)) statusfolder.deletemessagesflags(uids, set(flag)) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8b46996..f783d22 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -488,12 +488,16 @@ class IMAPFolder(BaseFolder): This function will update the self.messagelist dict to contain the new message after sucessfully saving it. + See folder/Base for details. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode. + :param rtime: A timestamp to be used as the mail date :returns: the UID of the new message as assigned by the server. If the message is saved, but it's UID can not be found, it will return 0. If the message can't be written (folder is read-only for example) it will return -1.""" - self.ui.debug('imap', 'savemessage: called') + self.ui.savemessage('imap', uid, flags, self) # already have it, just save modified flags if uid > 0 and self.uidexists(uid): @@ -611,7 +615,11 @@ class IMAPFolder(BaseFolder): return uid def savemessageflags(self, uid, flags): - """Change a message's flags to `flags`.""" + """Change a message's flags to `flags`. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" imapobj = self.imapserver.acquireconnection() try: try: diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 7f27768..b3779fd 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -111,6 +111,11 @@ class LocalStatusFolder(BaseFolder): return self.messagelist def savemessage(self, uid, content, flags, rtime): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode.""" if uid < 0: # We cannot assign a uid. return uid diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index dbe3e66..ac67c2f 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -216,6 +216,11 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): # assert False,"getmessageflags() called on non-existing message" def savemessage(self, uid, content, flags, rtime): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode.""" if uid < 0: # We cannot assign a uid. return uid diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 16745ab..24d943c 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -237,13 +237,18 @@ class MaildirFolder(BaseFolder): uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) def savemessage(self, uid, content, flags, rtime): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode.""" # This function only ever saves to tmp/, # but it calls savemessageflags() to actually save to cur/ or new/. - self.ui.debug('maildir', 'savemessage: called to write with flags %s ' - 'and content %s' % (repr(flags), repr(content))) + self.ui.savemessage('maildir', uid, flags, self) if uid < 0: # We cannot assign a new uid. return uid + if uid in self.messagelist: # We already have it, just update flags. self.savemessageflags(uid, flags) @@ -291,8 +296,11 @@ class MaildirFolder(BaseFolder): """Sets the specified message's flags to the given set. This function moves the message to the cur or new subdir, - depending on the 'S'een flag.""" + depending on the 'S'een flag. + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) # If a message has been seen, it goes into 'cur' diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 99ad2d3..f1c11e4 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -184,7 +184,12 @@ class MappedIMAPFolder(IMAPFolder): If the uid is > 0, the backend should set the uid to this, if it can. If it cannot set the uid to that, it will save it anyway. It will return the uid assigned in any case. + + See folder/Base for details. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode. """ + self.ui.savemessage('imap', uid, flags, self) # Mapped UID instances require the source to already have a # positive UID, so simply return here. if uid < 0: @@ -217,6 +222,11 @@ class MappedIMAPFolder(IMAPFolder): return None def savemessageflags(self, uid, flags): + """ + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" self._mb.savemessageflags(self.r2l[uid], flags) def addmessageflags(self, uid, flags): diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 4ff98c3..32511a4 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -408,6 +408,11 @@ class UIBase(object): if conn: #release any existing IMAP connection repository.imapserver.close() + def savemessage(self, debugtype, uid, flags, folder): + """Output a log line stating that we save a msg""" + self.debug(debugtype, "Write mail '%s:%d' with flags %s" % + (folder, uid, repr(flags))) + ################################################## Threads def getThreadDebugLog(self, thread): From 256a26a64911ceebe6ed004600f96e3d03e62f62 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 11:43:41 +0100 Subject: [PATCH 488/817] dry-run mode: Protect us from actually deleting a message in dry-run mode Document which functions honor dry-run mode and which don't. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 23 ++++++++++++++++++++++- offlineimap/ui/UIBase.py | 5 +++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 01a6ab5..77cc8ad 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -279,15 +279,27 @@ class BaseFolder(object): raise NotImplementedException def deletemessage(self, uid): + """ + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" raise NotImplementedException def deletemessages(self, uidlist): + """ + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" for uid in uidlist: self.deletemessage(uid) def copymessageto(self, uid, dstfolder, statusfolder, register = 1): """Copies a message from self to dst if needed, updating the status + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + :param uid: uid of the message to be copied. :param dstfolder: A BaseFolder-derived instance :param statusfolder: A LocalStatusFolder instance @@ -363,6 +375,8 @@ class BaseFolder(object): 2) invoke copymessageto() on those which: - If dstfolder doesn't have it yet, add them to dstfolder. - Update statusfolder + + This function checks and protects us from action in ryrun mode. """ threads = [] @@ -400,12 +414,17 @@ class BaseFolder(object): Get all UIDS in statusfolder but not self. These are messages that were deleted in 'self'. Delete those from dstfolder and - statusfolder.""" + statusfolder. + + This function checks and protects us from action in ryrun mode. + """ deletelist = filter(lambda uid: uid>=0 \ and not self.uidexists(uid), statusfolder.getmessageuidlist()) if len(deletelist): self.ui.deletingmessages(deletelist, [dstfolder]) + if self.repository.account.dryrun: + return #don't delete messages in dry-run mode # delete in statusfolder first to play safe. In case of abort, we # won't lose message, we will just retransmit some unneccessary. for folder in [statusfolder, dstfolder]: @@ -418,6 +437,8 @@ class BaseFolder(object): msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to both dstfolder and statusfolder. + + This function checks and protects us from action in ryrun mode. """ # For each flag, we store a list of uids to which it should be # added. Then, we can call addmessagesflags() to apply them in diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 32511a4..4bf4423 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -345,8 +345,9 @@ class UIBase(object): def deletingmessages(self, uidlist, destlist): ds = self.folderlist(destlist) - self.logger.info("Deleting %d messages (%s) in %s" % ( - len(uidlist), + prefix = "[DRYRUN] " if self.dryrun else "" + self.info("{}Deleting {} messages ({}) in {}".format( + prefix, len(uidlist), offlineimap.imaputil.uid_sequence(uidlist), ds)) def addingflags(self, uidlist, flags, dest): From 3a73d4bf6426a397c8d07f070cefd188ce6e8784 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 11:59:27 +0100 Subject: [PATCH 489/817] Add Changelog entry for dry-run mode Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index a612ab8..cd252b3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,12 @@ others. New Features ------------ +* --dry-run mode protects us from performing any actual action. + It will not precisely give the exact information what will + happen. If e.g. it would need to create a folder, it merely + outputs "Would create folder X", but not how many and which mails + it would transfer. + Changes ------- From aa01fe754a3c79d0a68a27ae6255335f93775b70 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 11:59:37 +0100 Subject: [PATCH 490/817] Improve command line option help text Shuffle around and update command line option descriptions in the docs. Signed-off-by: Sebastian Spaeth --- docs/MANUAL.rst | 97 +++----------------------------- docs/dev-doc-src/offlineimap.rst | 9 +++ offlineimap/init.py | 55 ++++++++++-------- 3 files changed, 46 insertions(+), 115 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 8d7a8b7..9715cb8 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -43,6 +43,10 @@ Most configuration is done via the configuration file. However, any setting can OfflineImap is well suited to be frequently invoked by cron jobs, or can run in daemon mode to periodically check your email (however, it will exit in some error situations). +The documentation is included in the git repository and can be created by +issueing `make dev-doc` in the `doc` folder (python-sphinx required), or it can +be viewed online at `http://docs.offlineimap.org`_. + .. _configuration: Configuration @@ -66,96 +70,9 @@ Check out the `Use Cases`_ section for some example configurations. OPTIONS ======= - --1 Disable most multithreading operations - - Use solely a single-connection sync. This effectively sets the - maxsyncaccounts and all maxconnections configuration file variables to 1. - - --P profiledir - - Sets OfflineIMAP into profile mode. The program will create profiledir (it - must not already exist). As it runs, Python profiling information about each - thread is logged into profiledir. Please note: This option is present for - debugging and optimization only, and should NOT be used unless you have a - specific reason to do so. It will significantly slow program performance, may - reduce reliability, and can generate huge amounts of data. You must use the - -1 option when you use -P. - - --a accountlist - - Overrides the accounts option in the general section of the configuration - file. You might use this to exclude certain accounts, or to sync some - accounts that you normally prefer not to. Separate the accounts by commas, - and use no embedded spaces. - - --c configfile - - Specifies a configuration file to use in lieu of the default, - ``~/.offlineimaprc``. - - --d debugtype[,...] - - Enables debugging for OfflineIMAP. This is useful if you are trying to track - down a malfunction or figure out what is going on under the hood. I suggest - that you use this with -1 to make the results more sensible. - - -d requires one or more debugtypes, separated by commas. These define what - exactly will be debugged, and include three options: imap, maildir, and - thread. The imap option will enable IMAP protocol stream and parsing - debugging. Note that the output may contain passwords, so take care to remove - that from the debugging output before sending it to anyone else. The maildir - option will enable debugging for certain Maildir operations. And thread will - debug the threading model. - - --f foldername[,foldername] - - Only sync the specified folders. The foldernames are the untranslated - foldernames. This command-line option overrides any folderfilter and - folderincludes options in the configuration file. - - --k [section:]option=value - - Override configuration file option. If "section" is omitted, it defaults to - general. Any underscores "_" in the section name are replaced with spaces: - for instance, to override option autorefresh in the "[Account Personal]" - section in the config file one would use "-k Account_Personal:autorefresh=30". - You may give more than one -k on the command line if you wish. - - --l filename - - Enables logging to filename. This will log everything that goes to the screen - to the specified file. Additionally, if any debugging is specified with -d, - then debug messages will not go to the screen, but instead to the logfile - only. - - --o Run only once, - - ignoring all autorefresh settings in the configuration file. - - --q Run only quick synchronizations. - - Ignore any flag updates on IMAP servers. - - --h|--help Show summary of options. - - --u interface - - Specifies an alternative user interface module to use. This overrides the - default specified in the configuration file. The pre-defined options are - listed in the User Interfaces section. The interface name is case insensitive. - +The command line options are described by issueing `offlineimap --help`. +Details on their use can be found either in the sample offlineimap.conf file or +in the user docs at `http://docs.offlineimap.org`_. User Interfaces =============== diff --git a/docs/dev-doc-src/offlineimap.rst b/docs/dev-doc-src/offlineimap.rst index bc98c16..37d78dc 100644 --- a/docs/dev-doc-src/offlineimap.rst +++ b/docs/dev-doc-src/offlineimap.rst @@ -6,6 +6,15 @@ Offlineimap is invoked with the following pattern: `offlineimap [args...]`. Where [args...] are as follows: Options: + --dry-run This mode protects us from performing any actual action. + It will not precisely give the exact information what + will happen. If e.g. it would need to create a folder, + it merely outputs "Would create folder X", but not how + many and which mails it would transfer. + --info Output information on the configured email + repositories. Useful for debugging and bug reporting. + Use in conjunction with the -a option to limit the + output to a single account. --version show program's version number and exit -h, --help show this help message and exit -1 Disable all multithreading operations and use solely a diff --git a/offlineimap/init.py b/offlineimap/init.py index d381a65..8668cc1 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -57,7 +57,19 @@ class OfflineImap: default=False, help="Do not actually modify any store but check and print " "what synchronization actions would be taken if a sync would be" - " performed.") + " performed. It will not precisely give the exact information w" + "hat will happen. If e.g. we need to create a folder, it merely" + " outputs 'Would create folder X', but not how many and which m" + "ails it would transfer.") + + parser.add_option("--info", + action="store_true", dest="diagnostics", + default=False, + help="Output information on the configured email repositories" + ". Useful for debugging and bug reporting. Use in conjunction wit" + "h the -a option to limit the output to a single account. This mo" + "de will prevent any actual sync to occur and exits after it outp" + "ut the debug information.") parser.add_option("-1", action="store_true", dest="singlethreading", @@ -80,11 +92,11 @@ class OfflineImap: "implies the -1 option.") parser.add_option("-a", dest="accounts", metavar="ACCOUNTS", - help="""Overrides the accounts section in the config file. - Lets you specify a particular account or set of - accounts to sync without having to edit the config - file. You might use this to exclude certain accounts, - or to sync some accounts that you normally prefer not to.""") + help="Overrides the accounts section in the config file. " + "Lets you specify a particular account or set of " + "accounts to sync without having to edit the config " + "file. You might use this to exclude certain accounts, " + "or to sync some accounts that you normally prefer not to.") parser.add_option("-c", dest="configfile", metavar="FILE", default="~/.offlineimaprc", @@ -92,18 +104,18 @@ class OfflineImap: "%default.") parser.add_option("-d", dest="debugtype", metavar="type1,[type2...]", - help="""Enables debugging for OfflineIMAP. This is useful - if you are to track down a malfunction or figure out what is - going on under the hood. This option requires one or more - debugtypes, separated by commas. These define what exactly - will be debugged, and so far include two options: imap, thread, - maildir or ALL. The imap option will enable IMAP protocol - stream and parsing debugging. Note that the output may contain - passwords, so take care to remove that from the debugging - output before sending it to anyone else. The maildir option - will enable debugging for certain Maildir operations. - The use of any debug option (unless 'thread' is included), - implies the single-thread option -1.""") + help="Enables debugging for OfflineIMAP. This is useful " + "if you are to track down a malfunction or figure out what is " + "going on under the hood. This option requires one or more " + "debugtypes, separated by commas. These define what exactly " + "will be debugged, and so far include two options: imap, thread, " + "maildir or ALL. The imap option will enable IMAP protocol " + "stream and parsing debugging. Note that the output may contain " + "passwords, so take care to remove that from the debugging " + "output before sending it to anyone else. The maildir option " + "will enable debugging for certain Maildir operations. " + "The use of any debug option (unless 'thread' is included), " + "implies the single-thread option -1.") parser.add_option("-l", dest="logfile", metavar="FILE", help="Log to FILE") @@ -149,13 +161,6 @@ class OfflineImap: "not usable. Possible interface choices are: %s " % ", ".join(UI_LIST.keys())) - parser.add_option("--info", - action="store_true", dest="diagnostics", - default=False, - help="Output information on the configured email repositories" - ". Useful for debugging and bug reporting. Use in conjunction wit" - "h the -a option to limit the output to a single account") - (options, args) = parser.parse_args() #read in configuration file From 40bc1f50e7b0605522feb4ac86daebb9f785eb88 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 13:22:05 +0100 Subject: [PATCH 491/817] tests: Use only 1 IMAP connection by default We don't want to hammmer IMAP servers for the test series too much to avoid being locked out. We will need a few tests to test concurrent connections, but by default one connection should be fine. Signed-off-by: Sebastian Spaeth --- test/OLItest/globals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/OLItest/globals.py b/test/OLItest/globals.py index 5d1f122..88806f2 100644 --- a/test/OLItest/globals.py +++ b/test/OLItest/globals.py @@ -33,5 +33,7 @@ localfolders = [Repository IMAP] type=IMAP +# Don't hammer the server with too many connection attempts: +maxconnections=1 folderfilter= lambda f: f.startswith('INBOX.OLItest') """) From d0723fb8a210d5c9eb412fa44438796ac74605ae Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 14:08:05 +0100 Subject: [PATCH 492/817] python3: Fix import in tests to work with python3 DOH Signed-off-by: Sebastian Spaeth --- test/OLItest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/OLItest/__init__.py b/test/OLItest/__init__.py index 2f85210..6fcd52f 100644 --- a/test/OLItest/__init__.py +++ b/test/OLItest/__init__.py @@ -30,5 +30,5 @@ banner = """%(__productname__)s %(__version__)s import unittest from unittest import TestLoader, TextTestRunner -from globals import default_conf +from .globals import default_conf from TestRunner import OLITestLib From d1e6e1f09e5fe5d492a3db677c9e811af85b2658 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 17 Feb 2012 14:57:11 +0100 Subject: [PATCH 493/817] tests: make tests (nearly) work with python3 Tests work now with python 3 with the exception of the deletion of remote testfolders which fails for some reasons. Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 28 ++++++++++++++++------------ test/OLItest/__init__.py | 2 +- test/OLItest/globals.py | 5 ++++- test/tests/test_01_basic.py | 2 +- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index e6277f2..7a2bb51 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -22,7 +22,10 @@ import sys import shutil import subprocess import tempfile -from ConfigParser import SafeConfigParser +try: + from configparser import SafeConfigParser +except ImportError: # python 2 + from ConfigParser import SafeConfigParser from . import default_conf class OLITestLib(): @@ -90,7 +93,7 @@ class OLITestLib(): config = cls.get_default_config() localfolders = os.path.join(cls.testdir, 'mail') config.set("Repository Maildir", "localfolders", localfolders) - with open(os.path.join(cls.testdir, 'offlineimap.conf'), "wa") as f: + with open(os.path.join(cls.testdir, 'offlineimap.conf'), "wt") as f: config.write(f) @classmethod @@ -105,7 +108,7 @@ class OLITestLib(): def run_OLI(cls): """Runs OfflineImap - :returns: (rescode, stdout) + :returns: (rescode, stdout (as unicode)) """ try: output = subprocess.check_output( @@ -113,8 +116,8 @@ class OLITestLib(): "-c%s" % os.path.join(cls.testdir, 'offlineimap.conf')], shell=False) except subprocess.CalledProcessError as e: - return (e.returncode, e.output) - return (0, output) + return (e.returncode, e.output.decode('utf-8')) + return (0, output.decode('utf-8')) @classmethod def delete_remote_testfolders(cls, reponame=None): @@ -128,7 +131,7 @@ class OLITestLib(): sections = [r for r in config.sections() \ if r.startswith('Repository')] sections = filter(lambda s: \ - config.get(s, 'Type', None).lower() == 'imap', + config.get(s, 'Type').lower() == 'imap', sections) for sec in sections: # Connect to each IMAP repo and delete all folders @@ -144,21 +147,22 @@ class OLITestLib(): assert res_t == 'OK' dirs = [] for d in data: - m = re.search(r''' # Find last quote + m = re.search(br''' # Find last quote "((?: # Non-tripple quoted can contain... [^"] | # a non-quote \\" # a backslashded quote )*)" # closing quote [^"]*$ # followed by no more quotes ''', d, flags=re.VERBOSE) - folder = m.group(1) - folder = folder.replace(r'\"', '"') # remove quoting + folder = bytearray(m.group(1)) + folder = folder.replace(br'\"', b'"') # remove quoting dirs.append(folder) # 2) filter out those not starting with INBOX.OLItest and del... - dirs = [d for d in dirs if d.startswith('INBOX.OLItest')] + dirs = [d for d in dirs if d.startswith(b'INBOX.OLItest')] for folder in dirs: - res_t, data = imapobj.delete(folder) - assert res_t == 'OK' + res_t, data = imapobj.delete(str(folder)) + assert res_t == 'OK', "Folder deletion of {} failed with error"\ + ":\n{} {}".format(folder.decode('utf-8'), res_t, data) imapobj.logout() @classmethod diff --git a/test/OLItest/__init__.py b/test/OLItest/__init__.py index 6fcd52f..ca6ef61 100644 --- a/test/OLItest/__init__.py +++ b/test/OLItest/__init__.py @@ -31,4 +31,4 @@ banner = """%(__productname__)s %(__version__)s import unittest from unittest import TestLoader, TextTestRunner from .globals import default_conf -from TestRunner import OLITestLib +from .TestRunner import OLITestLib diff --git a/test/OLItest/globals.py b/test/OLItest/globals.py index 88806f2..8e69ee8 100644 --- a/test/OLItest/globals.py +++ b/test/OLItest/globals.py @@ -14,7 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: #python3 + from io import StringIO default_conf=StringIO("""[general] #will be set automatically diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 0967d9d..6cda54e 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -65,7 +65,7 @@ class TestBasicFunctions(unittest.TestCase): code, res = OLITestLib.run_OLI() self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') - self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0" + self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0 " "mails, but sync led to {} folders and {} mails".format( boxes, mails)) From bc73c11239124aa668321b45ab5d4b3a94ca8054 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 08:31:31 +0100 Subject: [PATCH 494/817] Revert "Don't CHECK imapserver after each APPEND" This reverts commit 47390e03d6a07b2ab73b65b74b668b7ce0663f57. It is one of two potential candidates for the APPENDUID regression that John Wiegley reported. We need to examine this carefully before reintroducing this patch. Resolved Changelog.draft.rst conflict. --- Changelog.draft.rst | 2 ++ offlineimap/folder/IMAP.py | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index cd252b3..61dbe35 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -30,6 +30,8 @@ Changes we would not propagate local folders to the remote repository. (now tested in test03) +* Revert "* Slight performance enhancement uploading mails to an IMAP + server in the common case." It might have led to instabilities. Bug Fixes --------- diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index f783d22..64770ef 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -568,6 +568,10 @@ class IMAPFolder(BaseFolder): "failed (error). Server reponded: %s\nMessage content was: " "%s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) + # Checkpoint. Let it write out stuff, etc. Eg searches for + # just uploaded messages won't work if we don't do this. + typ, dat = imapobj.check() + assert(typ == 'OK') # get the new UID, default to 0 (=unknown) uid = 0 @@ -587,11 +591,6 @@ class IMAPFolder(BaseFolder): "'%s'" % str(resp)) else: # Don't support UIDPLUS - # Checkpoint. Let it write out stuff, etc. Eg searches for - # just uploaded messages won't work if we don't do this. - typ, dat = imapobj.check() - assert(typ == 'OK') - uid = self.savemessage_searchforheader(imapobj, headername, headervalue) # If everything failed up to here, search the message From 0e6b4ae798e5eb5ef0c5e1b4f64a59a5933f762d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 08:35:59 +0100 Subject: [PATCH 495/817] Revert "Clean up and improve APPENDUID handling" This reverts commit 4d47f7bf3c7e0af56e0bc1f74c3afe6c762beb3e. This is one of two candidates for introducing the instabilities that John Wiegley observed. We need to reintroduce with careful testing only. The original patch has been mostly reverted. --- offlineimap/folder/IMAP.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 64770ef..6bdcc71 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -539,10 +539,9 @@ class IMAPFolder(BaseFolder): self.ui.msgtoreadonly(self, uid, content, flags) return uid - # Clean out existing APPENDUID responses and do APPEND + #Do the APPEND try: - imapobj.response('APPENDUID') # flush APPENDUID responses - typ, dat = imapobj.append(self.getfullname(), + (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) retry_left = 0 # Mark as success @@ -570,33 +569,34 @@ class IMAPFolder(BaseFolder): OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. - typ, dat = imapobj.check() + (typ,dat) = imapobj.check() assert(typ == 'OK') - # get the new UID, default to 0 (=unknown) - uid = 0 - if use_uidplus: + # get the new UID. Test for APPENDUID response even if the + # server claims to not support it, as e.g. Gmail does :-( + if use_uidplus or imapobj._get_untagged_response('APPENDUID', True): # get new UID from the APPENDUID response, it could look # like OK [APPENDUID 38505 3955] APPEND completed with - # 38505 being folder UIDvalidity and 3955 the new UID. - typ, resp = imapobj.response('APPENDUID') - if resp == [None] or resp == None: + # 38505 bein folder UIDvalidity and 3955 the new UID. + # note: we would want to use .response() here but that + # often seems to return [None], even though we have + # data. TODO + resp = imapobj._get_untagged_response('APPENDUID') + if resp == [None]: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " "appending a message.") - else: - uid = long(resp[-1].split(' ')[1]) - if uid == 0: - self.ui.warn("savemessage: Server supports UIDPLUS, but" + return 0 + uid = long(resp[-1].split(' ')[1]) + if uid == 0: + self.ui.warn("savemessage: Server supports UIDPLUS, but" " we got no usable uid back. APPENDUID reponse was " "'%s'" % str(resp)) else: - # Don't support UIDPLUS + # we don't support UIDPLUS uid = self.savemessage_searchforheader(imapobj, headername, headervalue) - # If everything failed up to here, search the message - # manually TODO: rather than inserting and searching for our - # custom header, we should be searching the Message-ID and - # compare the message size... + # See docs for savemessage in Base.py for explanation + # of this and other return values if uid == 0: self.ui.debug('imap', 'savemessage: attempt to get new UID ' 'UID failed. Search headers manually.') From 2800a71a28362cd2ec73fa3c8a2bae56a3ce09d5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 09:39:39 +0100 Subject: [PATCH 496/817] tests: Add "create email test" This is the first test that actually creates a (local) email and syncs. We check the result of the sync operation, to see if the server has actually been assigning a proper UID to the email and bail out if not. This test therefore excercises our ability to properly detect the new UID of an APPENDED email. Obviously we still need some IMAP<->IMAP tests too, but since this is the same codepath being used for APPENDs in that case, it could also help to detect instabilities there. In order to get this test in, the OLITestLib got a few new helper functions: - delete_maildir - create_mail - get_maildir_uids The test passes here. I invoke it via: python -m unittest test.tests.test_01_basic.TestBasicFunctions.test_04_createmail or run python setup.py test, to run the whole suite. Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 52 +++++++++++++++++++++++++++++++++++++ test/tests/test_01_basic.py | 26 ++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 7a2bb51..1ce3b5f 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -22,12 +22,15 @@ import sys import shutil import subprocess import tempfile +import random +random.seed() try: from configparser import SafeConfigParser except ImportError: # python 2 from ConfigParser import SafeConfigParser from . import default_conf + class OLITestLib(): cred_file = None testdir = None @@ -179,6 +182,36 @@ class OLITestLib(): if e.errno != 17: # 'already exists' is ok. raise + @classmethod + def delete_maildir(cls, folder): + """Delete maildir 'folder' in our test maildir + + Does not fail if not existing""" + assert cls.testdir != None + maildir = os.path.join(cls.testdir, 'mail', folder) + shutil.rmtree(maildir, ignore_errors=True) + + @classmethod + def create_mail(cls, folder, mailfile=None, content=None): + """Create a mail in maildir 'folder'/new + + Use default mailfilename if not given. + Use some default content if not given""" + assert cls.testdir != None + while True: # Loop till we found a unique filename + mailfile = '{}:2,'.format(random.randint(0,999999999)) + mailfilepath = os.path.join(cls.testdir, 'mail', + folder, 'new', mailfile) + if not os.path.isfile(mailfilepath): + break + with open(mailfilepath,"wb") as mailf: + mailf.write(b'''From: test +Subject: Boo +Date: 1 Jan 1980 +To: test@offlineimap.org + +Content here.''') + @classmethod def count_maildir_mails(cls, folder): """Returns the number of mails in maildir 'folder' @@ -196,3 +229,22 @@ class OLITestLib(): if dirpath.endswith(('/cur', '/new')): mails += len(files) return boxes, mails + + # find UID in a maildir filename + re_uidmatch = re.compile(',U=(\d+)') + + @classmethod + def get_maildir_uids(cls, folder): + """Returns a list of maildir mail uids, 'None' if no valid uid""" + assert cls.testdir != None + mailfilepath = os.path.join(cls.testdir, 'mail', folder) + assert os.path.isdir(mailfilepath) + ret = [] + for dirpath, dirs, files in os.walk(mailfilepath): + if not dirpath.endswith((os.path.sep + 'new', os.path.sep + 'cur')): + continue # only /new /cur are interesting + for file in files: + m = cls.re_uidmatch.search(file) + uid = m.group(1) if m else None + ret.append(uid) + return ret diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 6cda54e..aadec72 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -31,6 +31,7 @@ def setUpModule(): def tearDownModule(): logging.info("Tear Down test module") + # comment out next line to keep testdir after test runs. TODO: make nicer OLITestLib.delete_test_dir() #Stuff that can be used @@ -80,7 +81,7 @@ class TestBasicFunctions(unittest.TestCase): #logging.warn("%s %s "% (code, res)) self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') - self.assertTrue((boxes, mails)==(2,0), msg="Expected 2 folders and 0" + self.assertTrue((boxes, mails)==(2,0), msg="Expected 2 folders and 0 " "mails, but sync led to {} folders and {} mails".format( boxes, mails)) @@ -101,4 +102,27 @@ class TestBasicFunctions(unittest.TestCase): self.assertEqual(mismatch, True, msg="Mismatching nametrans rules did " "NOT trigger an 'infinite folder generation' error. Output was:\n" "{}".format(res)) + # Write out default config file again + OLITestLib.write_config_file() + def test_04_createmail(self): + """Create mail in OLItest 1, sync, wipe folder sync + + Currently, this will mean the folder will be recreated + locally. At some point when remote folder deletion is + implemented, this behavior will change.""" + OLITestLib.delete_remote_testfolders() + OLITestLib.create_maildir('INBOX.OLItest') + OLITestLib.create_mail('INBOX.OLItest') + code, res = OLITestLib.run_OLI() + #logging.warn("%s %s "% (code, res)) + self.assertEqual(res, "") + boxes, mails = OLITestLib.count_maildir_mails('') + self.assertTrue((boxes, mails)==(1,1), msg="Expected 1 folders and 1 " + "mails, but sync led to {} folders and {} mails".format( + boxes, mails)) + # The local Mail should have been assigned a proper UID now, check! + uids = OLITestLib.get_maildir_uids('INBOX.OLItest') + self.assertFalse (None in uids, msg = "All mails should have been "+ \ + "assigned the IMAP's UID number, but {} messages had no valid ID "\ + .format(len([None for x in uids if x==None]))) From 74b133c500ef1d8954b5d402fd428a2cc062aa42 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 11:13:27 +0100 Subject: [PATCH 497/817] Revamped documentation structure and some doc fixes `make` in the `docs` dir or `make doc` in the root dir will now create the 1) man page and 2) the user documentation using sphinx (requiring python-doctools, and sphinx). The resulting user docs are in `docs/html`. You can also only create the man pages with `make man` in the `docs` dir. Also fixed all .rst conversion errors as requested by Gentoo downstream. Signed-off-by: Sebastian Spaeth --- .gitignore | 3 +- Changelog.draft.rst | 6 + Makefile | 1 - README.rst | 120 +++++------------- docs/HACKING.rst | 3 +- docs/MANUAL.rst | 19 ++- docs/Makefile | 10 +- docs/dev-doc-src/FAQ.rst | 1 - docs/{dev-doc-src => doc-src}/API.rst | 0 docs/{ => doc-src}/FAQ.rst | 15 ++- docs/{dev-doc-src => doc-src}/INSTALL.rst | 0 docs/{dev-doc-src => doc-src}/MANUAL.rst | 0 docs/{dev-doc-src => doc-src}/conf.py | 0 docs/doc-src/features.rst | 66 ++++++++++ docs/{dev-doc-src => doc-src}/index.rst | 2 + docs/{dev-doc-src => doc-src}/nametrans.rst | 12 +- docs/{dev-doc-src => doc-src}/offlineimap.rst | 0 docs/{dev-doc-src => doc-src}/repository.rst | 0 docs/{dev-doc-src => doc-src}/ui.rst | 0 19 files changed, 140 insertions(+), 118 deletions(-) delete mode 120000 docs/dev-doc-src/FAQ.rst rename docs/{dev-doc-src => doc-src}/API.rst (100%) rename docs/{ => doc-src}/FAQ.rst (98%) rename docs/{dev-doc-src => doc-src}/INSTALL.rst (100%) rename docs/{dev-doc-src => doc-src}/MANUAL.rst (100%) rename docs/{dev-doc-src => doc-src}/conf.py (100%) create mode 100644 docs/doc-src/features.rst rename docs/{dev-doc-src => doc-src}/index.rst (97%) rename docs/{dev-doc-src => doc-src}/nametrans.rst (98%) rename docs/{dev-doc-src => doc-src}/offlineimap.rst (100%) rename docs/{dev-doc-src => doc-src}/repository.rst (100%) rename docs/{dev-doc-src => doc-src}/ui.rst (100%) diff --git a/.gitignore b/.gitignore index 9307862..620e7e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ # Generated files -/docs/dev-doc/ +/docs/html/ /build/ *.pyc offlineimap.1 # backups .*.swp .*.swo +*.html *~ diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 61dbe35..f29e325 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -33,5 +33,11 @@ Changes * Revert "* Slight performance enhancement uploading mails to an IMAP server in the common case." It might have led to instabilities. +* Revamped documentation structure. `make` in the `docs` dir or `make + doc` in the root dir will now create the 1) man page and 2) the user + documentation using sphinx (requiring python-doctools, and + sphinx). The resulting user docs are in `docs/html`. You can also + only create the man pages with `make man` in the `docs` dir. + Bug Fixes --------- diff --git a/Makefile b/Makefile index 69532d4..d014388 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,6 @@ man: doc: @$(MAKE) -C docs $(RST2HTML) README.rst readme.html - $(RST2HTML) SubmittingPatches.rst SubmittingPatches.html $(RST2HTML) Changelog.rst Changelog.html targz: ../$(TARGZ) diff --git a/README.rst b/README.rst index b4bdccc..f7cc456 100644 --- a/README.rst +++ b/README.rst @@ -1,26 +1,13 @@ .. -*- coding: utf-8 -*- - .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project ====== README ====== -.. contents:: -.. sectnum:: - - Description =========== -Welcome to the official OfflineIMAP project. - -*NOTICE:* this software was written by John Goerzen, who retired from -maintaining. It is now maintained by Nicolas Sebrecht at -https://github.com/nicolas33/offlineimap. Thanks to John for his great job and -to have share this project with us. - - OfflineIMAP is a tool to simplify your e-mail reading. With OfflineIMAP, you can read the same mailbox from multiple computers. You get a current copy of your messages on each computer, and changes you make one place will be visible on all @@ -29,75 +16,42 @@ it will appear deleted on your work computer as well. OfflineIMAP is also useful if you want to use a mail reader that does not have IMAP support, has poor IMAP support, or does not provide disconnected operation. -OfflineIMAP works on pretty much any POSIX operating system, such as Linux, BSD -operating systems, MacOS X, Solaris, etc. +OfflineIMAP does not require additional python dependencies (although python-sqlite is strongly recommended) OfflineIMAP is a Free Software project licensed under the GNU General Public License. You can download it for free, and you can modify it. In fact, you are encouraged to contribute to OfflineIMAP, and doing so is fast and easy. +Documentation +============= -OfflineIMAP is FAST; it synchronizes my two accounts with over 50 folders in 3 -seconds. Other similar tools might take over a minute, and achieve a -less-reliable result. Some mail readers can take over 10 minutes to do the same -thing, and some don't even support it at all. Unlike other mail tools, -OfflineIMAP features a multi-threaded synchronization algorithm that can -dramatically speed up performance in many situations by synchronizing several -different things simultaneously. +The documentation (in .rst format) is included in the `docs` +directory. Read it directly or generate nicer html (python-sphinx +needed) or the man page (python-docutils needed) via:: -OfflineIMAP is FLEXIBLE; you can customize which folders are synced via regular -expressions, lists, or Python expressions; a versatile and comprehensive -configuration file is used to control behavior; two user interfaces are -built-in; fine-tuning of synchronization performance is possible; internal or -external automation is supported; SSL and PREAUTH tunnels are both supported; -offline (or "unplugged") reading is supported; and esoteric IMAP features are -supported to ensure compatibility with the widest variety of IMAP servers. + 'make doc' (user docs), 'make man' (man page only) or 'make' (both) -OfflineIMAP is SAFE; it uses an algorithm designed to prevent mail loss at all -costs. Because of the design of this algorithm, even programming errors should -not result in loss of mail. I am so confident in the algorithm that I use my -own personal and work accounts for testing of OfflineIMAP pre-release, -development, and beta releases. Of course, legally speaking, OfflineIMAP comes -with no warranty, so I am not responsible if this turns out to be wrong. - - -Method of Operation -=================== - -OfflineIMAP traditionally operates by maintaining a hierarchy of mail folders in -Maildir format locally. Your own mail reader will read mail from this tree, and -need never know that the mail comes from IMAP. OfflineIMAP will detect changes -to the mail folders on your IMAP server and your own computer and -bi-directionally synchronize them, copying, marking, and deleting messages as -necessary. - -With OfflineIMAP 4.0, a powerful new ability has been introduced ― the program -can now synchronize two IMAP servers with each other, with no need to have a -Maildir layer in-between. Many people use this if they use a mail reader on -their local machine that does not support Maildirs. People may install an IMAP -server on their local machine, and point both OfflineIMAP and their mail reader -of choice at it. This is often preferable to the mail reader's own IMAP support -since OfflineIMAP supports many features (offline reading, for one) that most -IMAP-aware readers don't. However, this feature is not as time-tested as -traditional syncing, so my advice is to stick with normal methods of operation -for the time being. + (`make html` will simply create html versions of all *.rst files in /docs) +The resulting user documentation will be in `docs/html`. The full user +docs are also at: ``_. Please see there for +detailed information on how to install and configure OfflineImap. Quick Start =========== -If you have already installed OfflineIMAP system-wide, or your system -administrator has done that for you, your task for setting up OfflineIMAP for -the first time is quite simple. You just need to set up your configuration -file, make your folder directory, and run it! +First, install OfflineIMAP. See docs/INSTALL.rst or read +``_. +(hint: `sudo python setup.py install`) -You can quickly set up your configuration file. The distribution includes a -file offlineimap.conf.minimal (Debian users may find this at -``/usr/share/doc/offlineimap/examples/offlineimap.conf.minimal``) that is a -basic example of setting of OfflineIMAP. You can simply copy this file into -your home directory and name it ``.offlineimaprc`` (note the leading period). A -command such as ``cp offlineimap.conf.minimal ~/.offlineimaprc`` will do it. -Or, if you prefer, you can just copy this text to ``~/.offlineimaprc``:: +Second, set up your configuration file and run it! The distribution +includes offlineimap.conf.minimal (Debian users may find this at +``/usr/share/doc/offlineimap/examples/offlineimap.conf.minimal``) that +provides you with the bare minimum of setting up OfflineIMAP. You can +simply copy this file into your home directory and name it +``.offlineimaprc``. A command such as ``cp offlineimap.conf.minimal +~/.offlineimaprc`` will do it. Or, if you prefer, you can just copy +this text to ``~/.offlineimaprc``:: [general] accounts = Test @@ -121,30 +75,23 @@ to do is specify a directory for your folders to be in (on the localfolders line), the host name of your IMAP server (on the remotehost line), and your login name on the remote (on the remoteuser line). That's it! -To run OfflineIMAP, you just have to say offlineimap ― it will fire up, ask you -for a login password if necessary, synchronize your folders, and exit. See? +To run OfflineIMAP, you just have to say `offlineimap` ― it will fire +up, ask you for a login password if necessary, synchronize your +folders, and exit. See? -You can just throw away the rest of this finely-crafted, perfectly-honed manual! -Of course, if you want to see how you can make OfflineIMAP FIVE TIMES FASTER FOR -JUST $19.95 (err, well, $0), you have to read on! - - -Documentation -============= - -If you are reading this file on github, you can find more documentations in the -`docs` directory. - -Using your git repository, you can generate documentation with:: - - $ make doc +You can just throw away the rest of the finely-crafted, +perfectly-honed user manual! Of course, if you want to see how you can +make OfflineIMAP FIVE TIMES FASTER FOR JUST $19.95 (err, well, $0), +you have to read on our full user documentation and peruse the sample +offlineimap.conf (which includes all available options) for further +tweaks! Mailing list ============ The user discussion, development and all exciting stuff take place in the -`mailing list`_. You're *NOT* supposed to subscribe to send emails. +`mailing list`_. You do *NOT* need to subscribe to send emails. Reporting bugs and contributions @@ -154,9 +101,6 @@ Bugs ---- Bugs, issues and contributions should be reported to the `mailing list`_. -**Please, don't use the github features (messages, pull requests, etc) at all. -It would most likely be discarded or ignored.** - ======== Examples diff --git a/docs/HACKING.rst b/docs/HACKING.rst index 71ec112..bdd9a44 100644 --- a/docs/HACKING.rst +++ b/docs/HACKING.rst @@ -183,6 +183,5 @@ branch. ,-) API === -API is documented in the dev-doc-src directory using the sphinx tools (also used -for python itself). This is a WIP. Contributions in this area would be very +The API is documented in the user documentation in the docs/ directory and browsable at ``_. This is a WIP. Contributions in this area would be very appreciated. diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 9715cb8..96d5d91 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -2,17 +2,14 @@ OfflineIMAP Manual ==================== +.. _OfflineIMAP: http://offlineimap.org + -------------------------------------------------------- Powerful IMAP/Maildir synchronization and reader support -------------------------------------------------------- :Author: John Goerzen & contributors -:Date: 2011-01-15 -:Copyright: GPL v2 -:Manual section: 1 - -.. TODO: :Manual group: - +:Date: 2012-02-23 DESCRIPTION =========== @@ -45,7 +42,7 @@ OfflineImap is well suited to be frequently invoked by cron jobs, or can run in The documentation is included in the git repository and can be created by issueing `make dev-doc` in the `doc` folder (python-sphinx required), or it can -be viewed online at `http://docs.offlineimap.org`_. +be viewed online at http://docs.offlineimap.org. .. _configuration: @@ -72,7 +69,7 @@ OPTIONS The command line options are described by issueing `offlineimap --help`. Details on their use can be found either in the sample offlineimap.conf file or -in the user docs at `http://docs.offlineimap.org`_. +in the user docs at http://docs.offlineimap.org. User Interfaces =============== @@ -306,6 +303,8 @@ as Man-In-The-Middle attacks which cause you to connect to the wrong server and pretend to be your mail server. DO NOT RELY ON STARTTLS AS A SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF YOUR IMAP SERVER! +.. _UNIX signals: + UNIX Signals ============ @@ -445,7 +444,7 @@ and Sent which should keep the same name:: Synchronizing 2 IMAP accounts to local Maildirs that are "next to each other", so that mutt can work on both. Full email setup described by -Thomas Kahle at `http://dev.gentoo.org/~tomka/mail.html`_ +Thomas Kahle at ``_ offlineimap.conf:: @@ -507,7 +506,7 @@ purposes: Fetching passwords from the gnome-keyring and translating folder names on the server to local foldernames. An example implementation of get_username and get_password showing how to query gnome-keyring is contained in -`http://dev.gentoo.org/~tomka/mail-setup.tar.bz2`_ The folderfilter is +``_ The folderfilter is a lambda term that, well, filters which folders to get. The function `oimaptransfolder_acc2` translates remote folders into local folders with a very simple logic. The `INBOX` folder will have the same name diff --git a/docs/Makefile b/docs/Makefile index 8fd880d..cd688f7 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -9,7 +9,7 @@ RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` RST2MAN=`type rst2man >/dev/null 2>&1 && echo rst2man || echo rst2man.py` SPHINXBUILD = sphinx-build -all: html dev-doc +all: man doc html: $(HTML_TARGETS) @@ -22,12 +22,12 @@ offlineimap.1: MANUAL.rst $(RST2MAN) MANUAL.rst offlineimap.1 cp -f offlineimap.1 .. -dev-doc: - $(SPHINXBUILD) -b html -d dev-doc/doctrees dev-doc-src dev-doc/html +doc: + $(SPHINXBUILD) -b html -d html/doctrees doc-src html clean: $(RM) -f $(HTML_TARGETS) $(RM) -f offlineimap.1 ../offlineimap.1 - $(RM) -rf dev-doc/* + $(RM) -rf html/* -.PHONY: dev-doc +.PHONY: clean doc diff --git a/docs/dev-doc-src/FAQ.rst b/docs/dev-doc-src/FAQ.rst deleted file mode 120000 index e3ca8b5..0000000 --- a/docs/dev-doc-src/FAQ.rst +++ /dev/null @@ -1 +0,0 @@ -../FAQ.rst \ No newline at end of file diff --git a/docs/dev-doc-src/API.rst b/docs/doc-src/API.rst similarity index 100% rename from docs/dev-doc-src/API.rst rename to docs/doc-src/API.rst diff --git a/docs/FAQ.rst b/docs/doc-src/FAQ.rst similarity index 98% rename from docs/FAQ.rst rename to docs/doc-src/FAQ.rst index 8381681..1d49c3d 100644 --- a/docs/FAQ.rst +++ b/docs/doc-src/FAQ.rst @@ -203,9 +203,10 @@ How is OfflineIMAP conformance? Can I force OfflineIMAP to sync a folder right now? --------------------------------------------------- -Yes, - 1) if you use the `Blinkenlights` UI. That UI shows the active accounts -as follows:: +Yes: + +1) if you use the `Blinkenlights` UI. That UI shows the active +accounts as follows:: 4: [active] *Control: . 3: [ 4:36] personal: @@ -216,8 +217,9 @@ as follows:: resync that account immediately. This will be ignored if a resync is already in progress for that account. - 2) while in sleep mode, you can also send a SIGUSR1. See the `Signals - on UNIX`_ section in the MANUAL for details. +2) while in sleep mode, you can also send a SIGUSR1. See the :ref:`UNIX + signals` section in the MANUAL for details. + I get a "Mailbox already exists" error -------------------------------------- @@ -291,14 +293,17 @@ certificates chain) in PEM format. (See the documentation of `ssl.wrap_socket`_'s `certfile` parameter for the gory details.) You can use either openssl or gnutls to create a certificate file in the required format. #. via openssl:: + openssl s_client -CApath /etc/ssl/certs -connect ${hostname}:imaps -showcerts \ | perl -ne 'print if /BEGIN/../END/; print STDERR if /return/' > $sslcacertfile ^D + #. via gnutls:: gnutls-cli --print-cert -p imaps ${host} $sslcacertfile + The path `/etc/ssl/certs` is not standardized; your system may store SSL certificates elsewhere. (On some systems it may be in `/usr/local/share/certs/`.) diff --git a/docs/dev-doc-src/INSTALL.rst b/docs/doc-src/INSTALL.rst similarity index 100% rename from docs/dev-doc-src/INSTALL.rst rename to docs/doc-src/INSTALL.rst diff --git a/docs/dev-doc-src/MANUAL.rst b/docs/doc-src/MANUAL.rst similarity index 100% rename from docs/dev-doc-src/MANUAL.rst rename to docs/doc-src/MANUAL.rst diff --git a/docs/dev-doc-src/conf.py b/docs/doc-src/conf.py similarity index 100% rename from docs/dev-doc-src/conf.py rename to docs/doc-src/conf.py diff --git a/docs/doc-src/features.rst b/docs/doc-src/features.rst new file mode 100644 index 0000000..e9272d1 --- /dev/null +++ b/docs/doc-src/features.rst @@ -0,0 +1,66 @@ +Description +=========== + +OfflineIMAP is a tool to simplify your e-mail reading. With OfflineIMAP, you can +read the same mailbox from multiple computers. You get a current copy of your +messages on each computer, and changes you make one place will be visible on all +other systems. For instance, you can delete a message on your home computer, and +it will appear deleted on your work computer as well. OfflineIMAP is also useful +if you want to use a mail reader that does not have IMAP support, has poor IMAP +support, or does not provide disconnected operation. + +OfflineIMAP works on pretty much any POSIX operating system, such as Linux, BSD +operating systems, MacOS X, Solaris, etc. + +OfflineIMAP is a Free Software project licensed under the GNU General Public +License. You can download it for free, and you can modify it. In fact, you are +encouraged to contribute to OfflineIMAP, and doing so is fast and easy. + +OfflineIMAP is FAST; it synchronizes my two accounts with over 50 folders in 3 +seconds. Other similar tools might take over a minute, and achieve a +less-reliable result. Some mail readers can take over 10 minutes to do the same +thing, and some don't even support it at all. Unlike other mail tools, +OfflineIMAP features a multi-threaded synchronization algorithm that can +dramatically speed up performance in many situations by synchronizing several +different things simultaneously. + +OfflineIMAP is FLEXIBLE; you can customize which folders are synced via regular +expressions, lists, or Python expressions; a versatile and comprehensive +configuration file is used to control behavior; two user interfaces are +built-in; fine-tuning of synchronization performance is possible; internal or +external automation is supported; SSL and PREAUTH tunnels are both supported; +offline (or "unplugged") reading is supported; and esoteric IMAP features are +supported to ensure compatibility with the widest variety of IMAP servers. + +OfflineIMAP is SAFE; it uses an algorithm designed to prevent mail loss at all +costs. Because of the design of this algorithm, even programming errors should +not result in loss of mail. I am so confident in the algorithm that I use my +own personal and work accounts for testing of OfflineIMAP pre-release, +development, and beta releases. Of course, legally speaking, OfflineIMAP comes +with no warranty, so I am not responsible if this turns out to be wrong. + +.. note: OfflineImap was written by John Goerzen, who retired from + maintaining. It is now maintained by Nicolas Sebrecht & Sebastian + Spaeth at https://github.com/spaetz/offlineimap. Thanks to John + for his great job and to have share this project with us. + +Method of Operation +=================== + +OfflineIMAP traditionally operates by maintaining a hierarchy of mail folders in +Maildir format locally. Your own mail reader will read mail from this tree, and +need never know that the mail comes from IMAP. OfflineIMAP will detect changes +to the mail folders on your IMAP server and your own computer and +bi-directionally synchronize them, copying, marking, and deleting messages as +necessary. + +With OfflineIMAP 4.0, a powerful new ability has been introduced ― the program +can now synchronize two IMAP servers with each other, with no need to have a +Maildir layer in-between. Many people use this if they use a mail reader on +their local machine that does not support Maildirs. People may install an IMAP +server on their local machine, and point both OfflineIMAP and their mail reader +of choice at it. This is often preferable to the mail reader's own IMAP support +since OfflineIMAP supports many features (offline reading, for one) that most +IMAP-aware readers don't. However, this feature is not as time-tested as +traditional syncing, so my advice is to stick with normal methods of operation +for the time being. diff --git a/docs/dev-doc-src/index.rst b/docs/doc-src/index.rst similarity index 97% rename from docs/dev-doc-src/index.rst rename to docs/doc-src/index.rst index d6aa939..6645948 100644 --- a/docs/dev-doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -15,6 +15,7 @@ If you just want to get started with minimal fuzz, have a look at our `online qu More information on specific topics can be found on the following pages: **User documentation** + * :doc:`Overview and features ` * :doc:`installation/uninstall ` * :doc:`user manual/Configuration ` * :doc:`Folder filtering & name transformation guide ` @@ -28,6 +29,7 @@ More information on specific topics can be found on the following pages: .. toctree:: :hidden: + features INSTALL MANUAL nametrans diff --git a/docs/dev-doc-src/nametrans.rst b/docs/doc-src/nametrans.rst similarity index 98% rename from docs/dev-doc-src/nametrans.rst rename to docs/doc-src/nametrans.rst index fabf7a4..ca25345 100644 --- a/docs/dev-doc-src/nametrans.rst +++ b/docs/doc-src/nametrans.rst @@ -48,7 +48,7 @@ at the end when required by Python syntax) For instance:: Usually it suffices to put a `folderfilter`_ setting in the remote repository section. You might want to put a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. (Even in this case, folder filters on the remote repository will prevent that) folderincludes -^^^^^^^^^^^^^^ +-------------- You can specify `folderincludes`_ to manually include additional folders to be synced, even if they had been filtered out by a folderfilter setting. `folderincludes`_ should return a Python list. @@ -92,8 +92,9 @@ locally? Try this:: this rule will result in undefined behavior. See also *Sharing a maildir with multiple IMAP servers* in the :ref:`pitfalls` section. + Reverse nametrans -^^^^^^^^^^^^^^^^^^ ++++++++++++++++++ Since 6.4.0, OfflineImap supports the creation of folders on the remote repository and that complicates things. Previously, only one nametrans setting on the remote repository was needed and that transformed a remote to a local name. However, nametrans transformations are one-way, and OfflineImap has no way using those rules on the remote repository to back local names to remote names. @@ -161,9 +162,10 @@ What folder separators do I need to use in nametrans rules? Maildir using the default folder separator '.' which do I need to use in nametrans rules?:: - nametrans = lambda f: "INBOX/" + f -or:: - nametrans = lambda f: "INBOX." + f + nametrans = lambda f: "INBOX/" + f + + or:: + nametrans = lambda f: "INBOX." + f **A:** Generally use the folder separator as defined in the repository you write the nametrans rule for. That is, use '/' in the above diff --git a/docs/dev-doc-src/offlineimap.rst b/docs/doc-src/offlineimap.rst similarity index 100% rename from docs/dev-doc-src/offlineimap.rst rename to docs/doc-src/offlineimap.rst diff --git a/docs/dev-doc-src/repository.rst b/docs/doc-src/repository.rst similarity index 100% rename from docs/dev-doc-src/repository.rst rename to docs/doc-src/repository.rst diff --git a/docs/dev-doc-src/ui.rst b/docs/doc-src/ui.rst similarity index 100% rename from docs/dev-doc-src/ui.rst rename to docs/doc-src/ui.rst From b7a72b742d53f546ec7d30818a218e05a6661c2d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 11:17:34 +0100 Subject: [PATCH 498/817] Fix up Changelog .rst->.html compile errors Signed-off-by: Sebastian Spaeth --- Changelog.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index c0fd517..2b41ff1 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -514,9 +514,8 @@ I'd like to thank reporters who involved in this cycle: - Pan Tsu - Vincent Beffara - Will Styler - (my apologies if I forget somebody) - -...and all active developers, of course! + +(my apologies if I forget somebody) ...and all active developers, of course! The imaplib2 migration looks to go the right way to be definetly released but still needs more tests. So, here we go... @@ -595,9 +594,10 @@ OfflineIMAP v6.3.2 (2010-02-21) Notes ----- -First of all I'm really happy to announce our new official `website`_. Most of -the work started from the impulse of Philippe LeCavalier with the help of -Sebastian Spaeth and other contributors. Thanks to everybody. +First of all I'm really happy to announce our new official `website +`_. Most of the work started from the impulse +of Philippe LeCavalier with the help of Sebastian Spaeth and other +contributors. Thanks to everybody. In this release, we are still touched by the "SSL3 write pending" but I think time was long enough to try to fix it. We have our first entry in the "KNOWN From 925538f02d7473429f6c846e3168deed034cefe7 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 11:20:22 +0100 Subject: [PATCH 499/817] docs: Fix docstrings to proper .rst syntax Prevents compile errors when creating the user documentation. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 77cc8ad..6f6f364 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -114,7 +114,7 @@ class BaseFolder(object): concurrent threads. :returns: Boolean indicating the match. Returns True in case it - implicitely saved the UIDVALIDITY.""" + implicitely saved the UIDVALIDITY.""" if self.get_saveduidvalidity() != None: return self.get_saveduidvalidity() == self.get_uidvalidity() @@ -273,6 +273,7 @@ class BaseFolder(object): """Change the message from existing uid to new_uid If the backend supports it (IMAP does not). + :param new_uid: (optional) If given, the old UID will be changed to a new UID. This allows backends efficient renaming of messages if the UID has changed.""" From d4c7a7bf17c7962cba3f5b52ed17b719876df010 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 11:21:11 +0100 Subject: [PATCH 500/817] Delete UPGRADE.rst This is the upgrade instruction from before 4.0 and long obsolete. Delete it. Signed-off-by: Sebastian Spaeth --- docs/UPGRADE.rst | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 docs/UPGRADE.rst diff --git a/docs/UPGRADE.rst b/docs/UPGRADE.rst deleted file mode 100644 index 169ce36..0000000 --- a/docs/UPGRADE.rst +++ /dev/null @@ -1,26 +0,0 @@ - - -Upgrading to 4.0 ----------------- - -If you are upgrading from a version of OfflineIMAP prior to 3.99.12, you will -find that you will get errors when OfflineIMAP starts up (relating to -ConfigParser or AccountHashGenerator) and the configuration file. This is -because the config file format had to change to accommodate new features in 4.0. -Fortunately, it's not difficult to adjust it to suit. - - -First thing you need to do is stop any running OfflineIMAP instance, making sure -first that it's synced all your mail. Then, modify your `~/.offlineimaprc` file. -You'll need to split up each account section (make sure that it now starts with -"Account ") into two Repository sections (one for the local side and another for -the remote side.) See the files offlineimap.conf.minimal and offlineimap.conf -in the distribution if you need more assistance. - - -OfflineIMAP's status directory area has also changed. Therefore, you should -delete everything in `~/.offlineimap` as well as your local mail folders. - - -When you start up OfflineIMAP 4.0, it will re-download all your mail from the -server and then you can continue using it like normal. From ae85b666d404bba08eeee088c1e706eafecc1f52 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 11:30:45 +0100 Subject: [PATCH 501/817] Declutter root dir: COPYRIGHT --> COPYING No need to keep a COPYING (GPL v2 license) AND a file COPYRIGHT in the root. All files have the boilerplate anyway. Add the relevant part on top of the COPYING file and do away with COPYRIGHT. Signed-off-by: Sebastian Spaeth --- COPYING | 10 ++++++++++ COPYRIGHT | 17 ----------------- 2 files changed, 10 insertions(+), 17 deletions(-) delete mode 100644 COPYRIGHT diff --git a/COPYING b/COPYING index e70efd4..5f3f605 100644 --- a/COPYING +++ b/COPYING @@ -1,3 +1,13 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + GNU GENERAL PUBLIC LICENSE Version 2, June 1991 diff --git a/COPYRIGHT b/COPYRIGHT deleted file mode 100644 index 4a11bce..0000000 --- a/COPYRIGHT +++ /dev/null @@ -1,17 +0,0 @@ -offlineimap Mail syncing software -Copyright (C) 2002 - 2009 John Goerzen - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - From 8bbdaa8c66d88c2f3f67bb9431b4e83df605bf89 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 12:03:56 +0100 Subject: [PATCH 502/817] docs: Integrate SubmittingPatches.rst into HACKING.rst Signed-off-by: Sebastian Spaeth --- docs/HACKING.rst | 187 -------------- docs/doc-src/API.rst | 6 +- .../doc-src/HACKING.rst | 240 +++++++++++++++--- docs/doc-src/index.rst | 2 + 4 files changed, 216 insertions(+), 219 deletions(-) delete mode 100644 docs/HACKING.rst rename SubmittingPatches.rst => docs/doc-src/HACKING.rst (77%) diff --git a/docs/HACKING.rst b/docs/HACKING.rst deleted file mode 100644 index bdd9a44..0000000 --- a/docs/HACKING.rst +++ /dev/null @@ -1,187 +0,0 @@ -.. -*- coding: utf-8 -*- - -.. _OfflineIMAP: https://github.com/nicolas33/offlineimap - -=================== -Hacking OfflineIMAP -=================== - -Welcome to the `OfflineIMAP`_ project. You'll find here all the information you -need to start hacking OfflineIMAP. Be aware there are a lot of very usefull tips -in the mailing list. You may want to subscribe if you didn't, yet. This is -where you'll get help. - -.. contents:: -.. sectnum:: - - -================================= -Git: Branching Model And Workflow -================================= - -Introduction -============ - -In order to involve into OfflineIMAP you need some knowledges about Git and our -workflow. Don't be afraid if you don't know much, we would be pleased to help -you. - -You can find the API docs autogenerated on http://docs.offlineimap.org. - -Release cycles -============== - -We use a classical cycle based workflow: - -1. A stable release is out. - -2. Feature topics are sent, discussed and merged. - -3. When enough work was merged, we start the freeze cycle: the first release - candidate is out. - -4. During the freeze cycle, no more features are merged. It's time to test - OfflineIMAP. New candidates version are released. The more we are late in -rc - releases the less patches are merged but bug fixes. - -5. When we think a release is stable enough, we restart from step 1. - - -Branching model -=============== - -The branching model with use in OfflineIMAP is very near from the Git project. -We use a topic oriented workflow. A topic may be one or more patches. - -The branches you'll find in the official repository are: - -* gh-pages -* master -* next -* pu -* maint - -gh-pages --------- - -This comes from a feature offered by Github. We maintain the online home github -page using this branch. - -master ------- - -If you're not sure what branch you should use, this one is for you. This is the -mainline. Simple users should use this branch to follow OfflineIMAP's evolution. - -Usually, patches submitted to the mailing list should start off of this branch. - -next ----- - -Patches recently merged are good candidates for this branch. The content of next -is merged into the mainline (master) at release time for both stable and -rc -releases. - -When patches are sent to the mailing list, contributors discuss about them. Once -done and when patches looks ready for mainline, patches are first merged into -next. Advanced users and testers use this branch to test last merged patches -before they hit the mainline. This helps not introducing strong breackages -directly in master. - -pu --- - -pu stands for "proposed updates". If a topic is not ready for master nor next, -it may be merged into pu. This branch only help developers to work on someone -else topic or an earlier pending topic. - -This branch is **not intended to be checkouted**; never. Even developers don't -do that. Due to the way pu is built you can't expect content there to work in -any way... unless you clearly want to run into troubles. - -Developers can extract a topic from this branch to work on it. See the following -section "Extract a topic from pu" in this documentation. - -maint ------ - -This is the maintenance branch. It gets its own releases starting from an old -stable release. It helps both users having troubles with last stable releases -and users not wanting latest features or so to still benefit from strong bug -fixes and security fixes. - - -Working with Git -================ - -Extract a topic from pu ------------------------ - -pu is built this way:: - - git checkout pu - git reset --keep next - git merge --no-ff -X theirs topic1 - git merge --no-ff -X theirs topic2 - git merge --no-ff -X theirs blue - git merge --no-ff -X theirs orange - ... - -As a consequence: - -1. Each topic merged uses a merge commit. A merge commit is a commit having 2 - ancestors. Actually, Git allows more than 2 parents but we don't use this - feature. It's intended. - -2. Paths in pu may mix up multiple versions if all the topics don't use the same - base commit. This is very often the case as topics aren't rebased: it guarantees - each topic is strictly identical to the last version sent to the mailing list. - No surprise. - - -What you need to extract a particular topic is the sha1 of the tip of that -branch (the last commit of the topic). Assume you want the branch of the topic -called 'blue'. First, look at the log given by this command:: - - git log --reverse --merges --parents origin/next..origin/pu - -With this command you ask for the log: - -* from next to pu -* in reverse order (older first) -* merge commits only -* with the sha1 of the ancestors - -In this list, find the topic you're looking for, basing you search on the lines -like:: - - Merge branch 'topic/name' into pu - -By convention, it has the form /. When you're at -it, pick the topic ancestor sha1. It's always the last sha1 in the line starting -by 'commit'. For you to know: - -* the first is the sha1 of the commit you see: the merge commit -* the following sha1 is the ancestor of the branch checkouted at merge time - (always the previous merged topic or the ancien next in our case) -* last is the branch merged - -Giving:: - - commit sha1_of_merge_commit sha1_of_ancient_pu sha1_of_topic_blue - -Then, you only have to checkout the topic from there:: - - git checkout -b blue sha1_of_topic_blue - -and you're done! You've just created a new branch called "blue" with the blue -content. Be aware this topic is almostly not updated against current next -branch. ,-) - - -=== -API -=== - -The API is documented in the user documentation in the docs/ directory and browsable at ``_. This is a WIP. Contributions in this area would be very -appreciated. diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index 20f3d1f..38df996 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -2,8 +2,10 @@ .. currentmodule:: offlineimap -Welcome to :mod:`offlineimaps`'s documentation -============================================== +.. _API docs: + +:mod:`offlineimap's` API documentation +====================================== Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. Email repositories are represented by a :class:`offlineimap.repository.Base.BaseRepository` or derivatives (see :mod:`offlineimap.repository` for details). A folder within a repository is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative from :mod:`offlineimap.folder`. diff --git a/SubmittingPatches.rst b/docs/doc-src/HACKING.rst similarity index 77% rename from SubmittingPatches.rst rename to docs/doc-src/HACKING.rst index 5161519..9192458 100644 --- a/SubmittingPatches.rst +++ b/docs/doc-src/HACKING.rst @@ -1,14 +1,134 @@ .. -*- coding: utf-8 -*- - -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project +.. _OfflineIMAP: http://offlineimap.org .. _commits mailing list: http://lists.offlineimap.org/listinfo.cgi/commits-offlineimap.org +.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project -================================================= -Checklist (and a short version for the impatient) -================================================= +Hacking OfflineIMAP +=================== -Commits -======= +In this section you'll find all the information you need to start +hacking `OfflineIMAP`_. Be aware there are a lot of very usefull tips +in the mailing list. You may want to subscribe if you didn't, +yet. This is where you will get help. + +.. contents:: :depth: 2 + +API +--- + +:ref:`OfflineImap's API ` documentation is included in the user +documentation (next section) and online browsable at +``_. It is mostly auto-generated from the +source code and is a work in progress. Contributions in this area +would be very appreciated. + +Following new commits +--------------------- + +You can follow upstream commits on + - `CIA.vc `, + - `Ohloh `, + - `GitHub `, + - or on the `commits mailing list`_. + + + +Git: OfflineImap's branching Model And Workflow +=============================================== + +Introduction +------------ + +This optional section provides you with information on how we use git +branches and do releases. You will need to know very little about git +to get started. + +For the impatient, see the :ref:`contribution checklist` below. + +Git Branching model +-------------------- + +OfflineIMAP uses the following branches: + + * master + * next + * maint + * (pu) + * & several topic oriented feature branches. A topic may consist of + one or more patches. + +master +++++++ + +If you're not sure what branch you should use, this one is for you. +This is the mainline. Simple users should use this branch to follow +OfflineIMAP's evolution. + +Usually, patches submitted to the mailing list should start off of +this branch. + +next +++++ + +Patches recently merged are good candidates for this branch. The content of next +is merged into the mainline (master) at release time for both stable and -rc +releases. + +When patches are sent to the mailing list, contributors discuss about them. Once +done and when patches looks ready for mainline, patches are first merged into +next. Advanced users and testers use this branch to test last merged patches +before they hit the mainline. This helps not introducing strong breackages +directly in master. + +pu ++++ + +pu stands for "proposed updates". If a topic is not ready for master nor next, +it may be merged into pu. This branch only help developers to work on someone +else topic or an earlier pending topic. + +This branch is **not intended to be checkouted**; never. Even developers don't +do that. Due to the way pu is built you can't expect content there to work in +any way... unless you clearly want to run into troubles. + +Developers can extract a topic from this branch to work on it. See the following +section "Extract a topic from pu" in this documentation. + +maint ++++++ + +This is the maintenance branch. It gets its own releases starting from an old +stable release. It helps both users having troubles with last stable releases +and users not wanting latest features or so to still benefit from strong bug +fixes and security fixes. + +Release cycles +-------------- + +A typical release cycle works like this: + +1. A stable release is out. + +2. Feature topics are sent, discussed and merged. + +3. When enough work was merged, we start the freeze cycle: the first release + candidate is out. + +4. During the freeze cycle, no more features are merged. It's time to test + OfflineIMAP. New candidates version are released. The more we are late in -rc + releases the less patches are merged but bug fixes. + +5. When we think a release is stable enough, we restart from step 1. + + +.. _contribution checklist: + + +Contribution Checklist (and a short version for the impatient) +=============================================================== + +Create commits +-------------- * make commits of logical units * check for unnecessary whitespace with ``git diff --check`` @@ -28,8 +148,9 @@ Commits * make sure that you have tests for the bug you are fixing * make sure that the test suite passes after your commit -Patch -===== + +Export commits as patches +------------------------- * use ``git format-patch -M`` to create the patch * do not PGP sign your patch @@ -52,9 +173,9 @@ Patch * see below for instructions specific to your mailer -============ + Long version -============ +------------ I started reading over the SubmittingPatches document for Git, primarily because I wanted to have a document similar to it for OfflineIMAP to make sure people @@ -64,8 +185,8 @@ But the patch submission requirements are a lot more relaxed here on the technical/contents front, because the OfflineIMAP is a lot smaller ;-). So here is only the relevant bits. -Decide what to base your work on -================================ +Decide what branch to base your work on ++++++++++++++++++++++++++++++++++++++++ In general, always base your work on the oldest branch that your change is relevant to. @@ -92,13 +213,12 @@ master..pu`` and look for the merge commit. The second parent of this commit is the tip of the topic branch. Make separate commits for logically separate changes -==================================================== +++++++++++++++++++++++++++++++++++++++++++++++++++++ -Unless your patch is really trivial, you should not be sending -out a patch that was generated between your working tree and -your commit head. Instead, always make a commit with complete -commit message and generate a series of patches from your -repository. It is a good discipline. +Unless your patch is really trivial, you should not be sending your +changes in a single patch. Instead, always make a commit with +complete commit message and generate a series of small patches from +your repository. Describe the technical detail of the change(s). @@ -115,7 +235,7 @@ but for code. Generate your patch using git tools out of your commits -------------------------------------------------------- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ git based diff tools (git, Cogito, and StGIT included) generate unidiff which is the preferred format. @@ -133,7 +253,7 @@ that is fine, but please mark it as such. Sending your patches -==================== +++++++++++++++++++++ People on the mailing list need to be able to read and comment on the changes you are submitting. It is important for @@ -205,7 +325,7 @@ necessary. Sign your work -============== +++++++++++++++ To improve tracking of who did what, we've borrowed the "sign-off" procedure from the Linux kernel project on patches @@ -218,7 +338,7 @@ the right to pass it on as a open-source patch**. The rules are pretty simple: if you can certify the below: **Developer's Certificate of Origin 1.1** ------------------------------------------ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By making a contribution to this project, I certify that: @@ -320,13 +440,7 @@ Know the status of your patch after submission of the branch in which your patch has been merged (i.e. it will not tell you if your patch is merged in pu if you rebase on top of master). - -* You can follow upstream commits on -`CIA.vc `, -`Ohloh `, -`GitHub `, -or on the `commits mailing list`_. - + .. * Read the git mailing list, the maintainer regularly posts messages entitled "What's cooking in git.git" and "What's in git.git" giving the status of various proposed changes. @@ -601,3 +715,69 @@ Just make sure to disable line wrapping in the email client (GMail web interface will line wrap no matter what, so you need to use a real IMAP client). +Working with Git +================ + +Extract a topic from pu +----------------------- + +pu is built this way:: + + git checkout pu + git reset --keep next + git merge --no-ff -X theirs topic1 + git merge --no-ff -X theirs topic2 + git merge --no-ff -X theirs blue + git merge --no-ff -X theirs orange + ... + +As a consequence: + +1. Each topic merged uses a merge commit. A merge commit is a commit having 2 + ancestors. Actually, Git allows more than 2 parents but we don't use this + feature. It's intended. + +2. Paths in pu may mix up multiple versions if all the topics don't use the same + base commit. This is very often the case as topics aren't rebased: it guarantees + each topic is strictly identical to the last version sent to the mailing list. + No surprise. + + +What you need to extract a particular topic is the sha1 of the tip of that +branch (the last commit of the topic). Assume you want the branch of the topic +called 'blue'. First, look at the log given by this command:: + + git log --reverse --merges --parents origin/next..origin/pu + +With this command you ask for the log: + +* from next to pu +* in reverse order (older first) +* merge commits only +* with the sha1 of the ancestors + +In this list, find the topic you're looking for, basing you search on the lines +like:: + + Merge branch 'topic/name' into pu + +By convention, it has the form /. When you're at +it, pick the topic ancestor sha1. It's always the last sha1 in the line starting +by 'commit'. For you to know: + +* the first is the sha1 of the commit you see: the merge commit +* the following sha1 is the ancestor of the branch checkouted at merge time + (always the previous merged topic or the ancien next in our case) +* last is the branch merged + +Giving:: + + commit sha1_of_merge_commit sha1_of_ancient_pu sha1_of_topic_blue + +Then, you only have to checkout the topic from there:: + + git checkout -b blue sha1_of_topic_blue + +and you're done! You've just created a new branch called "blue" with the blue +content. Be aware this topic is almostly not updated against current next +branch. ,-) diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index 6645948..771269e 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -23,6 +23,7 @@ More information on specific topics can be found on the following pages: * :doc:`Frequently Asked Questions ` **Developer documentation** + * :doc:`HACKING HowTo & git workflows ` * :doc:`API documentation ` for internal details on the :mod:`offlineimap` module @@ -36,6 +37,7 @@ More information on specific topics can be found on the following pages: offlineimap FAQ + HACKING API repository ui From bfb7a79d6bd2bc9e3d6846814a923de3c4bbeccd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 14:46:14 +0100 Subject: [PATCH 503/817] documentation: Make top-level README a plain text file. It makes direct reading much simpler. Signed-off-by: Sebastian Spaeth --- Makefile | 1 - README.rst => README | 88 +++++++++++++++----------------------------- 2 files changed, 30 insertions(+), 59 deletions(-) rename README.rst => README (73%) diff --git a/Makefile b/Makefile index d014388..c355583 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,6 @@ man: doc: @$(MAKE) -C docs - $(RST2HTML) README.rst readme.html $(RST2HTML) Changelog.rst Changelog.html targz: ../$(TARGZ) diff --git a/README.rst b/README similarity index 73% rename from README.rst rename to README index f7cc456..dc95198 100644 --- a/README.rst +++ b/README @@ -1,12 +1,8 @@ -.. -*- coding: utf-8 -*- -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project - -====== -README -====== +OfflineImap README +================== Description -=========== +----------- OfflineIMAP is a tool to simplify your e-mail reading. With OfflineIMAP, you can read the same mailbox from multiple computers. You get a current copy of your @@ -14,34 +10,35 @@ messages on each computer, and changes you make one place will be visible on all other systems. For instance, you can delete a message on your home computer, and it will appear deleted on your work computer as well. OfflineIMAP is also useful if you want to use a mail reader that does not have IMAP support, has poor IMAP -support, or does not provide disconnected operation. +support, or does not provide disconnected operation. It's homepage at http://offlineimap.org contains more information, source code, and online documentation. -OfflineIMAP does not require additional python dependencies (although python-sqlite is strongly recommended) +OfflineIMAP does not require additional python dependencies beyond python >=2.6 +(although python-sqlite is strongly recommended). OfflineIMAP is a Free Software project licensed under the GNU General Public -License. You can download it for free, and you can modify it. In fact, you are -encouraged to contribute to OfflineIMAP, and doing so is fast and easy. +License version 2 (or later). You can download it for free, and you can modify +it. In fact, you are encouraged to contribute to OfflineIMAP. Documentation -============= +------------- -The documentation (in .rst format) is included in the `docs` -directory. Read it directly or generate nicer html (python-sphinx -needed) or the man page (python-docutils needed) via:: +The documentation is included (in .rst format) in the `docs` directory. +Read it directly or generate nice html docs (python-sphinx needed) and/or +the man page (python-docutils needed) while being in the `docs` dir via:: 'make doc' (user docs), 'make man' (man page only) or 'make' (both) (`make html` will simply create html versions of all *.rst files in /docs) The resulting user documentation will be in `docs/html`. The full user -docs are also at: ``_. Please see there for +docs are also at: http://docs.offlineimap.org. Please see there for detailed information on how to install and configure OfflineImap. Quick Start =========== First, install OfflineIMAP. See docs/INSTALL.rst or read -``_. +http://docs.offlineimap.org/en/latest/INSTALL.html. (hint: `sudo python setup.py install`) Second, set up your configuration file and run it! The distribution @@ -76,42 +73,33 @@ line), the host name of your IMAP server (on the remotehost line), and your login name on the remote (on the remoteuser line). That's it! To run OfflineIMAP, you just have to say `offlineimap` ― it will fire -up, ask you for a login password if necessary, synchronize your -folders, and exit. See? +up, ask you for a login password if necessary, synchronize your folders, +and exit. See? -You can just throw away the rest of the finely-crafted, -perfectly-honed user manual! Of course, if you want to see how you can -make OfflineIMAP FIVE TIMES FASTER FOR JUST $19.95 (err, well, $0), -you have to read on our full user documentation and peruse the sample -offlineimap.conf (which includes all available options) for further -tweaks! +You can just throw away the rest of the finely-crafted, perfectly-honed user +manual! Of course, if you want to see how you can make OfflineIMAP +FIVE TIMES FASTER FOR JUST $19.95 (err, well, $0), you have to read on our +full user documentation and peruse the sample offlineimap.conf (which +includes all available options) for further tweaks! -Mailing list -============ +Mailing list & bug reporting +---------------------------- The user discussion, development and all exciting stuff take place in the -`mailing list`_. You do *NOT* need to subscribe to send emails. +OfflineImap mailing list at http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project. You do not need to subscribe to send emails. +Bugs, issues and contributions should be reported to the mailing list. Bugs can also be reported in the issue tracker at https://github.com/spaetz/offlineimap/issues. -Reporting bugs and contributions -================================ - -Bugs ----- - -Bugs, issues and contributions should be reported to the `mailing list`_. - -======== -Examples -======== +Configuration Examples +====================== Here are some example configurations for various situations. Please e-mail any other examples you have that may be useful to me. Multiple Accounts with Mutt -=========================== +--------------------------- This example shows you how to set up OfflineIMAP to synchronize multiple accounts with the mutt mail reader. @@ -145,7 +133,7 @@ That's it! UW-IMAPD and References -======================= +----------------------- Some users with a UW-IMAPD server need to use OfflineIMAP's "reference" feature to get at their mailboxes, specifying a reference of ``~/Mail`` or ``#mh/`` @@ -183,7 +171,7 @@ folders synced to just three:: pythonfile Configuration File Option -==================================== +------------------------------------- You can have OfflineIMAP load up a Python file before evaluating the configuration file options that are Python expressions. This example is based @@ -222,19 +210,3 @@ Then, the ~/.offlineimap.py file will contain:: This code snippet illustrates how the foldersort option can be customized with a Python function from the pythonfile to always synchronize certain folders first. - - -Signals -======= - -OfflineIMAP writes its current PID into ``~/.offlineimap/pid`` when it is -running. It is not guaranteed that this file will not exist when OfflineIMAP is -not running. - - From 3c481d9ce5ab34f9d539f815a6d3b89e57ab47cb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 24 Feb 2012 14:52:30 +0100 Subject: [PATCH 504/817] -f command line option only works on the untranslated remote names Previously folderfilters had to match both the local AND remote name which caused unwanted behavior in combination with nametrans rules. Make it operate on the untranslated remote names now and clarify in the command line option help text. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ offlineimap/init.py | 20 ++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index f29e325..d9c8aa1 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -39,5 +39,10 @@ Changes sphinx). The resulting user docs are in `docs/html`. You can also only create the man pages with `make man` in the `docs` dir. +* -f command line option only works on the untranslated remote + repository folder names now. Previously folderfilters had to match + both the local AND remote name which caused unwanted behavior in + combination with nametrans rules. Clarify in the help text. + Bug Fixes --------- diff --git a/offlineimap/init.py b/offlineimap/init.py index 8668cc1..24d6c1a 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -121,12 +121,10 @@ class OfflineImap: help="Log to FILE") parser.add_option("-f", dest="folders", metavar="folder1,[folder2...]", - help= - "Only sync the specified folders. The folder names " - "are the *untranslated* foldernames. This " - "command-line option overrides any 'folderfilter' " - "and 'folderincludes' options in the configuration " - "file.") + help="Only sync the specified folders. The folder names " + "are the *untranslated* foldernames of the remote repository. " + "This command-line option overrides any 'folderfilter' " + "and 'folderincludes' options in the configuration file.") parser.add_option("-k", dest="configoverride", action="append", @@ -269,12 +267,10 @@ class OfflineImap: for accountname in accounts.getaccountlist(config): account_section = 'Account ' + accountname remote_repo_section = 'Repository ' + \ - config.get(account_section, 'remoterepository') - local_repo_section = 'Repository ' + \ - config.get(account_section, 'localrepository') - for section in [remote_repo_section, local_repo_section]: - config.set(section, "folderfilter", folderfilter) - config.set(section, "folderincludes", folderincludes) + config.get(account_section, 'remoterepository') + config.set(remote_repo_section, "folderfilter", folderfilter) + config.set(remote_repo_section, "folderincludes", + folderincludes) if options.logfile: sys.stderr = self.ui.logfile From d6da65b18f87725868bcc28cffaba7e3b4876c28 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 27 Feb 2012 16:35:17 +0100 Subject: [PATCH 505/817] tests: Fix test #4 1) Fix test #4 by deleting all local mailfolders remaining from previous tests, the mailfolder count will be off, otherwise. 2) Make folder deletion work in python3, it weirdly enough needs to be quoted like this to work in python3 (I found a python bug about this somewhere). Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 4 ++-- test/tests/test_01_basic.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 1ce3b5f..fc06a87 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -158,12 +158,12 @@ class OLITestLib(): [^"]*$ # followed by no more quotes ''', d, flags=re.VERBOSE) folder = bytearray(m.group(1)) - folder = folder.replace(br'\"', b'"') # remove quoting + #folder = folder.replace(br'\"', b'"') # remove quoting dirs.append(folder) # 2) filter out those not starting with INBOX.OLItest and del... dirs = [d for d in dirs if d.startswith(b'INBOX.OLItest')] for folder in dirs: - res_t, data = imapobj.delete(str(folder)) + res_t, data = imapobj.delete(b'\"'+folder+b'\"') assert res_t == 'OK', "Folder deletion of {} failed with error"\ ":\n{} {}".format(folder.decode('utf-8'), res_t, data) imapobj.logout() diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index aadec72..f5a0ea1 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -112,6 +112,7 @@ class TestBasicFunctions(unittest.TestCase): locally. At some point when remote folder deletion is implemented, this behavior will change.""" OLITestLib.delete_remote_testfolders() + OLITestLib.delete_maildir('') #Delete all local maildir folders OLITestLib.create_maildir('INBOX.OLItest') OLITestLib.create_mail('INBOX.OLItest') code, res = OLITestLib.run_OLI() From 29ba2fc523eede41faa1693157d703e5cf93f949 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 2 Apr 2012 23:26:59 +0200 Subject: [PATCH 506/817] Warn about nonsensical config option 'sep' for IMAP repositories We autodetect the folder separator on IMAP servers and ignore any 'sep' setting in the repository section for IMAP servers. Detect if there is such a setting and warn the user about it. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/repository/IMAP.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index d9c8aa1..5fdacac 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -44,5 +44,7 @@ Changes both the local AND remote name which caused unwanted behavior in combination with nametrans rules. Clarify in the help text. +* Some better output when using nonsensical configuration settings + Bug Fixes --------- diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 28e73b7..5ad787a 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -33,6 +33,9 @@ class IMAPRepository(BaseRepository): self._host = None self.imapserver = imapserver.IMAPServer(self) self.folders = None + if self.getconf('sep', None): + self.ui.info("The 'sep' setting is being ignored for IMAP " + "repository '%s' (it's autodetected)" % self) def startkeepalive(self): keepalivetime = self.getkeepalive() From 2f88b0296a8715263f22f6342a58ec4db3d33fd2 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 19 Feb 2012 20:55:06 +0400 Subject: [PATCH 507/817] Fix Curses interface for Python 2.6 In 2.6 all logging classes are not the new-style ones, so they have no mro() method and, thus, we can't use super() for them. Since CursesLogHanler is singly-inherited method, there will be no problems in usage of the explicit superclass name. Signed-off-by: Eygene Ryabinkin Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/Curses.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5fdacac..c2b7dbc 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -48,3 +48,5 @@ Changes Bug Fixes --------- + +* Improve compatability of the curses UI with python 2.6 diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 365be60..7463a6a 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -281,7 +281,7 @@ class CursesLogHandler(logging.StreamHandler): """self.ui has been set to the UI class before anything is invoked""" def emit(self, record): - log_str = super(CursesLogHandler, self).format(record) + log_str = logging.StreamHandler.format(self, record) color = self.ui.gettf().curses_color # We must acquire both locks. Otherwise, deadlock can result. # This can happen if one thread calls _msg (locking curses, then From d3427a664ece1f3a5a5480adc41e5c96b8f4eb9a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 2 Apr 2012 23:47:32 +0200 Subject: [PATCH 508/817] v6.5.3 release see changelog Signed-off-by: Sebastian Spaeth --- .gitignore | 2 +- Changelog.draft.rst | 52 ----------------------------------------- Changelog.rst | 51 ++++++++++++++++++++++++++++++++++++---- offlineimap/__init__.py | 2 +- 4 files changed, 48 insertions(+), 59 deletions(-) delete mode 100644 Changelog.draft.rst diff --git a/.gitignore b/.gitignore index 620e7e4..c5af353 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # Generated files -/docs/html/ +/docs/dev-doc/ /build/ *.pyc offlineimap.1 diff --git a/Changelog.draft.rst b/Changelog.draft.rst deleted file mode 100644 index c2b7dbc..0000000 --- a/Changelog.draft.rst +++ /dev/null @@ -1,52 +0,0 @@ -========= -ChangeLog -========= - -Users should ignore this content: **it is draft**. - -Contributors should add entries here in the following section, on top of the -others. - -`WIP (coming releases)` -======================= - -New Features ------------- - -* --dry-run mode protects us from performing any actual action. - It will not precisely give the exact information what will - happen. If e.g. it would need to create a folder, it merely - outputs "Would create folder X", but not how many and which mails - it would transfer. - -Changes -------- - -* internal code changes to prepare for Python3 - -* Improve user documentation of nametrans/folderfilter - -* Fixed some cases where invalid nametrans rules were not caught and - we would not propagate local folders to the remote repository. - (now tested in test03) - -* Revert "* Slight performance enhancement uploading mails to an IMAP - server in the common case." It might have led to instabilities. - -* Revamped documentation structure. `make` in the `docs` dir or `make - doc` in the root dir will now create the 1) man page and 2) the user - documentation using sphinx (requiring python-doctools, and - sphinx). The resulting user docs are in `docs/html`. You can also - only create the man pages with `make man` in the `docs` dir. - -* -f command line option only works on the untranslated remote - repository folder names now. Previously folderfilters had to match - both the local AND remote name which caused unwanted behavior in - combination with nametrans rules. Clarify in the help text. - -* Some better output when using nonsensical configuration settings - -Bug Fixes ---------- - -* Improve compatability of the curses UI with python 2.6 diff --git a/Changelog.rst b/Changelog.rst index 2b41ff1..f8bdfaa 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,11 +5,52 @@ ChangeLog :website: http://offlineimap.org -**NOTE FROM THE MAINTAINER:** - Contributors should use the `WIP` section in Changelog.draft.rst in order to - add changes they are working on. I will use it to make the new changelog entry - on releases. And because I'm lazy, it will also be used as a draft for the - releases announces. +WIP (add new stuff for the next release) +======================================== + +New Features +------------ + +Changes +------- + +Bug Fixes +--------- + + +OfflineIMAP v6.5.3 (2012-04-02) +=============================== + +* --dry-run mode protects us from performing any actual action. It will + not precisely give the exact information what will happen. If e.g. it + would need to create a folder, it merely outputs "Would create folder + X", but not how many and which mails it would transfer. + +* internal code changes to prepare for Python3 + +* Improve user documentation of nametrans/folderfilter + +* Fixed some cases where invalid nametrans rules were not caught and + we would not propagate local folders to the remote repository. + (now tested in test03) + +* Revert "* Slight performance enhancement uploading mails to an IMAP + server in the common case." It might have led to instabilities. + +* Revamped documentation structure. `make` in the `docs` dir or `make + doc` in the root dir will now create the 1) man page and 2) the user + documentation using sphinx (requiring python-doctools, and + sphinx). The resulting user docs are in `docs/html`. You can also + only create the man pages with `make man` in the `docs` dir. + +* -f command line option only works on the untranslated remote + repository folder names now. Previously folderfilters had to match + both the local AND remote name which caused unwanted behavior in + combination with nametrans rules. Clarify in the help text. + +* Some better output when using nonsensical configuration settings + +* Improve compatability of the curses UI with python 2.6 OfflineIMAP v6.5.2.1 (2012-04-04) ===================================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 0dc0943..5673a89 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.2.1" +__version__ = "6.5.3" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From c708e36ff7900e93b46af8f25541c6c349c02b97 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 3 Apr 2012 01:13:27 +0200 Subject: [PATCH 509/817] Don't fail if no dry-run setting has been specified Somehow we failed if no dry-run setting had been specified in the config file. This got caught thanks to extending the test suite with a stock configuration. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 3 +++ offlineimap/ui/UIBase.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index f8bdfaa..53a9797 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -17,6 +17,9 @@ Changes Bug Fixes --------- +* Don't fail if no dry-run setting exists in offlineimap.conf + (introduced in 6.5.3) + OfflineIMAP v6.5.3 (2012-04-02) =============================== diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 4bf4423..eea929d 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -49,7 +49,7 @@ class UIBase(object): def __init__(self, config, loglevel = logging.INFO): self.config = config # Is this a 'dryrun'? - self.dryrun = config.getboolean('general', 'dry-run') + self.dryrun = config.getdefaultboolean('general', 'dry-run', False) self.debuglist = [] """list of debugtypes we are supposed to log""" self.debugmessages = {} From e5c4a558fbcbdd5f3e30251cae252bd93af9ede2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 3 Apr 2012 01:13:27 +0200 Subject: [PATCH 510/817] Don't fail if no dry-run setting has been specified Somehow we failed if no dry-run setting had been specified in the config file. This got caught thanks to extending the test suite with a stock configuration. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 3 +++ offlineimap/ui/UIBase.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index f8bdfaa..53a9797 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -17,6 +17,9 @@ Changes Bug Fixes --------- +* Don't fail if no dry-run setting exists in offlineimap.conf + (introduced in 6.5.3) + OfflineIMAP v6.5.3 (2012-04-02) =============================== diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 4bf4423..eea929d 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -49,7 +49,7 @@ class UIBase(object): def __init__(self, config, loglevel = logging.INFO): self.config = config # Is this a 'dryrun'? - self.dryrun = config.getboolean('general', 'dry-run') + self.dryrun = config.getdefaultboolean('general', 'dry-run', False) self.debuglist = [] """list of debugtypes we are supposed to log""" self.debugmessages = {} From f67278b2a82acd72c196c5a9fe08431521555ebd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 3 Apr 2012 01:34:59 +0200 Subject: [PATCH 511/817] tests: Add tests for imaputil.* functions Add one test for each of the functions in offlineimap.imaputil, to make sure they keep working. These functions tests the internal innards of the offlineimap module rather than invoking offlineimap as a program. Signed-off-by: Sebastian Spaeth --- test/OLItest/TestRunner.py | 8 ++- test/tests/__init__.py | 1 + test/tests/test_00_imaputil.py | 96 ++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 test/tests/test_00_imaputil.py diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index fc06a87..3535b61 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -24,10 +24,8 @@ import subprocess import tempfile import random random.seed() -try: - from configparser import SafeConfigParser -except ImportError: # python 2 - from ConfigParser import SafeConfigParser + +from offlineimap.CustomConfig import CustomConfigParser from . import default_conf @@ -75,7 +73,7 @@ class OLITestLib(): #TODO, only do first time and cache then for subsequent calls? assert cls.cred_file != None assert cls.testdir != None - config = SafeConfigParser() + config = CustomConfigParser() config.readfp(default_conf) default_conf.seek(0) # rewind config_file to start config.read(cls.cred_file) diff --git a/test/tests/__init__.py b/test/tests/__init__.py index e69de29..8b13789 100644 --- a/test/tests/__init__.py +++ b/test/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/test/tests/test_00_imaputil.py b/test/tests/test_00_imaputil.py new file mode 100644 index 0000000..c97d5c9 --- /dev/null +++ b/test/tests/test_00_imaputil.py @@ -0,0 +1,96 @@ +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import unittest +import logging + +from offlineimap import imaputil +from offlineimap.ui import UI_LIST, setglobalui +from offlineimap.CustomConfig import CustomConfigParser + +from test.OLItest import OLITestLib + +# Things need to be setup first, usually setup.py initializes everything. +# but if e.g. called from command line, we take care of default values here: +if not OLITestLib.cred_file: + OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') + +def setUpModule(): + logging.info("Set Up test module %s" % __name__) + tdir = OLITestLib.create_test_dir(suffix=__name__) + +def tearDownModule(): + logging.info("Tear Down test module") + # comment out next line to keep testdir after test runs. TODO: make nicer + OLITestLib.delete_test_dir() + +#Stuff that can be used +#self.assertEqual(self.seq, range(10)) +# should raise an exception for an immutable sequence +#self.assertRaises(TypeError, random.shuffle, (1,2,3)) +#self.assertTrue(element in self.seq) +#self.assertFalse(element in self.seq) + +class TestInternalFunctions(unittest.TestCase): + """While the other test files test OfflineImap as a program, these + tests directly invoke internal helper functions to guarantee that + they deliver results as expected""" + + @classmethod + def setUpClass(cls): + #This is run before all tests in this class + config= OLITestLib.get_default_config() + setglobalui(UI_LIST['quiet'](config)) + + def test_01_imapsplit(self): + """Test imaputil.imapsplit()""" + res = imaputil.imapsplit(b'(\\HasNoChildren) "." "INBOX.Sent"') + self.assertEqual(res, [b'(\\HasNoChildren)', b'"."', b'"INBOX.Sent"']) + + def test_02_flagsplit(self): + """Test imaputil.flagsplit()""" + res = imaputil.flagsplit(b'(\\Draft \\Deleted)') + self.assertEqual(res, [b'\\Draft', b'\\Deleted']) + + res = imaputil.flagsplit(b'(FLAGS (\\Seen Old) UID 4807)') + self.assertEqual(res, [b'FLAGS', b'(\\Seen Old)', b'UID', b'4807']) + + def test_03_options2hash(self): + """Test imaputil.options2hash()""" + res = imaputil.options2hash([1,2,3,4,5,6]) + self.assertEqual(res, {1:2, 3:4, 5:6}) + + def test_04_flags2hash(self): + """Test imaputil.flags2hash()""" + res = imaputil.flags2hash(b'(FLAGS (\\Seen Old) UID 4807)') + self.assertEqual(res, {b'FLAGS': b'(\\Seen Old)', b'UID': b'4807'}) + + def test_05_flagsimap2maildir(self): + """Test imaputil.flagsimap2maildir()""" + res = imaputil.flagsimap2maildir(b'(\\Draft \\Deleted)') + self.assertEqual(res, set(b'DT')) + + def test_06_flagsmaildir2imap(self): + """Test imaputil.flagsmaildir2imap()""" + res = imaputil.flagsmaildir2imap(set(b'DR')) + self.assertEqual(res, b'(\\Answered \\Draft)') + # test all possible flags + res = imaputil.flagsmaildir2imap(set(b'SRFTD')) + self.assertEqual(res, b'(\\Answered \\Deleted \\Draft \\Flagged \\Seen)') + + def test_07_uid_sequence(self): + """Test imaputil.uid_sequence()""" + res = imaputil.uid_sequence([1,2,3,4,5,10,12,13]) + self.assertEqual(res, b'1:5,10,12:13') From 679c491c56c981961e18aa43b31955900491d7a3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 3 Apr 2012 01:39:28 +0200 Subject: [PATCH 512/817] bugfix release v6.5.3.1 a single bugfix when no dry-run option was given in the config file. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 3 +++ offlineimap/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 53a9797..4e803b8 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -17,6 +17,9 @@ Changes Bug Fixes --------- +OfflineIMAP v6.5.3.1 (2012-04-03) +================================= + * Don't fail if no dry-run setting exists in offlineimap.conf (introduced in 6.5.3) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 5673a89..81dcbac 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.3" +__version__ = "6.5.3.1" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 8fe8bbe366e3debcffa30e31cf28ef9f7560fe45 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Apr 2012 13:18:44 +0200 Subject: [PATCH 513/817] imaplib2: bump upstream version 2.29 --> 2.33 Signed-off-by: Sebastian Spaeth --- Changelog.rst | 9 +-------- offlineimap/imaplib2.py | 39 ++++++++++++++++++++------------------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 4e803b8..664c7ca 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,14 +8,7 @@ ChangeLog WIP (add new stuff for the next release) ======================================== -New Features ------------- - -Changes -------- - -Bug Fixes ---------- +* bump bundled imaplib2 library 2.29 --> 2.33 OfflineIMAP v6.5.3.1 (2012-04-03) ================================= diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index ffa2676..8138d6c 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.29" +__version__ = "2.33" __release__ = "2" -__revision__ = "29" +__revision__ = "33" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -462,19 +462,16 @@ class IMAP4(object): cert_reqs = ssl.CERT_NONE self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs) ssl_exc = ssl.SSLError + self.read_fd = self.sock.fileno() except ImportError: - # No ssl module, and socket.ssl does not allow certificate verification - if self.ca_certs is not None: - raise socket.sslerror("SSL CA certificates cannot be checked without ssl module") - self.sock = socket.ssl(self.sock, self.keyfile, self.certfile) - ssl_exc = socket.sslerror + # No ssl module, and socket.ssl has no fileno(), and does not allow certificate verification + raise socket.sslerror("imaplib2 SSL mode does not work without ssl module") if self.cert_verify_cb is not None: cert_err = self.cert_verify_cb(self.sock.getpeercert(), self.host) if cert_err: raise ssl_exc(cert_err) - self.read_fd = self.sock.fileno() def start_compressing(self): @@ -496,7 +493,7 @@ class IMAP4(object): if self.decompressor.unconsumed_tail: data = self.decompressor.unconsumed_tail else: - data = self.sock.recv(8192) + data = self.sock.recv(READ_SIZE) return self.decompressor.decompress(data, size) @@ -1233,9 +1230,10 @@ class IMAP4(object): def _choose_nonull_or_dflt(self, dflt, *args): - dflttyp = type(dflt) - if isinstance(dflttyp, basestring): + if isinstance(dflt, basestring): dflttyp = basestring # Allow any string type + else: + dflttyp = type(dflt) for arg in args: if arg is not None: if isinstance(arg, dflttyp): @@ -1591,7 +1589,8 @@ class IMAP4(object): def _simple_command(self, name, *args, **kw): if 'callback' in kw: - self._command(name, *args, callback=self._command_completer, cb_arg=kw, cb_self=True) + # Note: old calling sequence for back-compat with python <2.6 + self._command(name, callback=self._command_completer, cb_arg=kw, cb_self=True, *args) return (None, None) return self._command_complete(self._command(name, *args), kw) @@ -1752,8 +1751,9 @@ class IMAP4(object): if rxzero > 5: raise IOError("Too many read 0") time.sleep(0.1) - else: - rxzero = 0 + continue # Try again + rxzero = 0 + while True: stop = data.find('\n', start) if stop < 0: @@ -1818,8 +1818,9 @@ class IMAP4(object): if rxzero > 5: raise IOError("Too many read 0") time.sleep(0.1) - else: - rxzero = 0 + continue # Try again + rxzero = 0 + while True: stop = data.find('\n', start) if stop < 0: @@ -2020,7 +2021,7 @@ class IMAP4_SSL(IMAP4): if self.decompressor.unconsumed_tail: data = self.decompressor.unconsumed_tail else: - data = self.sock.read(8192) + data = self.sock.read(READ_SIZE) return self.decompressor.decompress(data, size) @@ -2047,7 +2048,7 @@ class IMAP4_SSL(IMAP4): def ssl(self): """ssl = ssl() - Return socket.ssl instance used to communicate with the IMAP4 server.""" + Return ssl instance used to communicate with the IMAP4 server.""" return self.sock @@ -2103,7 +2104,7 @@ class IMAP4_stream(IMAP4): if self.decompressor.unconsumed_tail: data = self.decompressor.unconsumed_tail else: - data = os.read(self.read_fd, 8192) + data = os.read(self.read_fd, READ_SIZE) return self.decompressor.decompress(data, size) From d079e614ea13e2cc4133694023b4eea1c3016d39 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Apr 2012 17:15:56 +0200 Subject: [PATCH 514/817] tests: extend imapsplit test One more test of the internal imapsplit function. Signed-off-by: Sebastian Spaeth --- test/tests/test_00_imaputil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/tests/test_00_imaputil.py b/test/tests/test_00_imaputil.py index c97d5c9..d1972da 100644 --- a/test/tests/test_00_imaputil.py +++ b/test/tests/test_00_imaputil.py @@ -59,6 +59,9 @@ class TestInternalFunctions(unittest.TestCase): res = imaputil.imapsplit(b'(\\HasNoChildren) "." "INBOX.Sent"') self.assertEqual(res, [b'(\\HasNoChildren)', b'"."', b'"INBOX.Sent"']) + res = imaputil.imapsplit(b'"mo\\" o" sdfsdf') + self.assertEqual(res, [b'"mo\\" o"', b'sdfsdf']) + def test_02_flagsplit(self): """Test imaputil.flagsplit()""" res = imaputil.flagsplit(b'(\\Draft \\Deleted)') From b9af72ea11f0c6e1e8f90e3926b5b6be1a83f312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20H=C3=B6ger?= Date: Wed, 22 Feb 2012 21:50:46 +0100 Subject: [PATCH 515/817] Curses UI: Reset the warn method before terminate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warn() method tries to set the color to red. This leads to a garbled tty after endwin() has been called. So lets simply use the UIBase implementation. Signed-off-by: Christoph Höger --- Changelog.rst | 1 + offlineimap/ui/Curses.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index 664c7ca..89ced02 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -9,6 +9,7 @@ WIP (add new stuff for the next release) ======================================== * bump bundled imaplib2 library 2.29 --> 2.33 +* Curses UI, don't use colors after we shut down curses already (C.Höger) OfflineIMAP v6.5.3.1 (2012-04-03) ================================= diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 7463a6a..1c378ab 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -613,6 +613,8 @@ class Blinkenlights(UIBase, CursesUtil): # basic one, so exceptions and stuff are properly displayed self.logger.removeHandler(self._log_con_handler) UIBase.setup_consolehandler(self) + # reset the warning method, we do not have curses anymore + self.warn = super(Blinkenlights, self).warn # finally call parent terminate which prints out exceptions etc super(Blinkenlights, self).terminate(*args, **kwargs) From a4b4f1ffcbcd7e7d80f4ef95534b2e15344a8394 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Apr 2012 17:36:52 +0200 Subject: [PATCH 516/817] docs: maxage/size skeleton Add skeleton for maxage/maxsize documentation. Also remove the old nametrans.rst from the old doc location. Signed-off-by: Sebastian Spaeth --- docs/dev-doc-src/nametrans.rst | 182 ------------------------------- docs/doc-src/advanced_config.rst | 16 +++ docs/doc-src/index.rst | 4 + 3 files changed, 20 insertions(+), 182 deletions(-) delete mode 100644 docs/dev-doc-src/nametrans.rst create mode 100644 docs/doc-src/advanced_config.rst diff --git a/docs/dev-doc-src/nametrans.rst b/docs/dev-doc-src/nametrans.rst deleted file mode 100644 index fabf7a4..0000000 --- a/docs/dev-doc-src/nametrans.rst +++ /dev/null @@ -1,182 +0,0 @@ -.. _folder_filtering_and_name_translation: - -Folder filtering and Name translation -===================================== - -OfflineImap provides advanced and potentially complex possibilities for -filtering and translating folder names. If you don't need any of this, you can -safely skip this section. - -.. warning:: - Starting with v6.4.0, OfflineImap supports the creation of folders on the remote repostory. This change means that people that only had a nametrans option on the remote repository (everyone) will need to have a nametrans setting on the local repository too that will reverse the name transformation. See section `Reverse nametrans`_ for details. - -folderfilter ------------- - -If you do not want to synchronize all your filters, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. - -The only parameter to that function is the folder name. If the filter -function returns True, the folder will be synced, if it returns False, -it. will be skipped. The folderfilter operates on the *UNTRANSLATED* -name (before any `nametrans`_ fudging takes place). Consider the -examples below to get an idea of what they do. - -Example 1: synchronizing only INBOX and Sent:: - - folderfilter = lambda folder: folder in ['INBOX', 'Sent'] - -Example 2: synchronizing everything except Trash:: - - folderfilter = lambda folder: folder not in ['Trash'] - -Example 3: Using a regular expression to exclude Trash and all folders -containing the characters "Del":: - - folderfilter = lambda folder: not re.search('(^Trash$|Del)', folder) - -.. note:: - If folderfilter is not specified, ALL remote folders will be - synchronized. - -You can span multiple lines by indenting the others. (Use backslashes -at the end when required by Python syntax) For instance:: - - folderfilter = lambda foldername: foldername in - ['INBOX', 'Sent Mail', 'Deleted Items', - 'Received'] - -Usually it suffices to put a `folderfilter`_ setting in the remote repository section. You might want to put a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. (Even in this case, folder filters on the remote repository will prevent that) - -folderincludes -^^^^^^^^^^^^^^ - -You can specify `folderincludes`_ to manually include additional folders to be synced, even if they had been filtered out by a folderfilter setting. `folderincludes`_ should return a Python list. - -This can be used to 1) add a folder that was excluded by your -folderfilter rule, 2) to include a folder that your server does not specify -with its LIST option, or 3) to include a folder that is outside your basic -`reference`. The `reference` value will not be prefixed to this folder -name, even if you have specified one. For example:: - - folderincludes = ['debian.user', 'debian.personal'] - -This will add the "debian.user" and "debian.personal" folders even if you -have filtered out everything starting with "debian" in your folderfilter -settings. - - -nametrans ----------- - -Sometimes, folders need to have different names on the remote and the -local repositories. To achieve this you can specify a folder name -translator. This must be a eval-able Python expression that takes a -foldername arg and returns the new value. We suggest a lambda function, -but it could be any python function really. If you use nametrans rules, you will need to set them both on the remote and the local repository, see `Reverse nametrans`_ just below for details. The following examples are thought to be put in the remote repository section. - -The below will remove "INBOX." from the leading edge of folders (great -for Courier IMAP users):: - - nametrans = lambda folder: re.sub('^INBOX\.', '', folder) - -Using Courier remotely and want to duplicate its mailbox naming -locally? Try this:: - - nametrans = lambda folder: re.sub('^INBOX\.*', '.', folder) - -.. warning:: - You MUST construct nametrans rules such that it NEVER returns the - same value for two folders, UNLESS the second values are filtered - out by folderfilter below. That is, two filters on one side may - never point to the same folder on the other side. Failure to follow - this rule will result in undefined behavior. See also *Sharing a - maildir with multiple IMAP servers* in the :ref:`pitfalls` section. - -Reverse nametrans -^^^^^^^^^^^^^^^^^^ - -Since 6.4.0, OfflineImap supports the creation of folders on the remote repository and that complicates things. Previously, only one nametrans setting on the remote repository was needed and that transformed a remote to a local name. However, nametrans transformations are one-way, and OfflineImap has no way using those rules on the remote repository to back local names to remote names. - -Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts of any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. - -Take the above examples. If your remote nametrans setting was:: - - nametrans = lambda folder: re.sub('^INBOX\.', '', folder) - -then you will want to have this in your local repository, prepending "INBOX" to any local folder name:: - - nametrans = lambda folder: 'INBOX' + folder - -Failure to set the local nametrans rule will lead to weird-looking error messages of -for instance- this type:: - - ERROR: Creating folder moo.foo on repository remote - Folder 'moo.foo'[remote] could not be created. Server responded: ('NO', ['Unknown namespace.']) - -(This indicates that you attempted to create a folder "Sent" when all remote folders needed to be under the prefix of "INBOX."). - -OfflineImap will make some sanity checks if it needs to create a new -folder on the remote side and a back-and-forth nametrans-lation does not -yield the original foldername (as that could potentially lead to -infinite folder creation cycles). - -You can probably already see now that creating nametrans rules can be a pretty daunting and complex endeavour. Check out the Use cases in the manual. If you have some interesting use cases that we can present as examples here, please let us know. - -Debugging folderfilter and nametrans ------------------------------------- - -Given the complexity of the functions and regexes involved, it is easy to misconfigure things. One way to test your configuration without danger to corrupt anything or to create unwanted folders is to invoke offlineimap with the `--info` option. - -It will output a list of folders and their transformations on the screen (save them to a file with -l info.log), and will help you to tweak your rules as well as to understand your configuration. It also provides good output for bug reporting. - -FAQ on nametrans ----------------- - -Where to put nametrans rules, on the remote and/or local repository? -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -If you never intend to create new folders on the LOCAL repository that -need to be synced to the REMOTE repository, it is sufficient to create a -nametrans rule on the remote Repository section. This will be used to -determine the names of new folder names on the LOCAL repository, and to -match existing folders that correspond. - -*IF* you create folders on the local repository, that are supposed to be - automatically created on the remote repository, you will need to create - a nametrans rule that provides the reverse name translation. - -(A nametrans rule provides only a one-way translation of names and in -order to know which names folders on the LOCAL side would have on the -REMOTE side, you need to specify the reverse nametrans rule on the local -repository) - -OfflineImap will complain if it needs to create a new folder on the -remote side and a back-and-forth nametrans-lation does not yield the -original foldername (as that could potentially lead to infinite folder -creation cycles). - -What folder separators do I need to use in nametrans rules? -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -**Q:** If I sync from an IMAP server with folder separator '/' to a - Maildir using the default folder separator '.' which do I need to use - in nametrans rules?:: - - nametrans = lambda f: "INBOX/" + f -or:: - nametrans = lambda f: "INBOX." + f - -**A:** Generally use the folder separator as defined in the repository - you write the nametrans rule for. That is, use '/' in the above - case. We will pass in the untranslated name of the IMAP folder as - parameter (here `f`). The translated name will ultimately have all - folder separators be replaced with the destination repositories' - folder separator. - -So if 'f' was "Sent", the first nametrans yields the translated name -"INBOX/Sent" to be used on the other side. As that repository uses the -folder separator '.' rather than '/', the ultimate name to be used will -be "INBOX.Sent". - -(As a final note, the smart will see that both variants of the above -nametrans rule would have worked identically in this case) - diff --git a/docs/doc-src/advanced_config.rst b/docs/doc-src/advanced_config.rst new file mode 100644 index 0000000..56e6f18 --- /dev/null +++ b/docs/doc-src/advanced_config.rst @@ -0,0 +1,16 @@ +Message filtering +================= + +There are two ways to selectively filter messages out of a folder, using `maxsize` and `maxage`. Setting each option will basically ignore all messages that are on the server by pretending they don't exist. + +:todo: explain them and give tipps on how to use and not use them. Use cases! + +maxage +------ + +:todo: ! + +maxsize +------- + +:todo: ! diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index 771269e..e7d639e 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -17,8 +17,11 @@ More information on specific topics can be found on the following pages: **User documentation** * :doc:`Overview and features ` * :doc:`installation/uninstall ` + +**Configuration** * :doc:`user manual/Configuration ` * :doc:`Folder filtering & name transformation guide ` + * :doc:`maxage ` * :doc:`command line options ` * :doc:`Frequently Asked Questions ` @@ -34,6 +37,7 @@ More information on specific topics can be found on the following pages: INSTALL MANUAL nametrans + advanced_config offlineimap FAQ From 895e709bf23eea3b8f546f240317580e34251cf3 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 19 Apr 2012 18:23:12 +0200 Subject: [PATCH 517/817] Fix SSL fingerprint was not always checked As reported by James Cook, we would not check the fingerprint of the SSL server, as we were looking for the 'ssl' module in locals() rather than globals(). Ooops! Rather than using globals() though, I simply remove the by-now superfluous check. We now rely on python2.6 and we unconditionally import the SSL module in any case, so it needs to be there. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 1 + offlineimap/imaplibutil.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 89ced02..6184622 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -9,6 +9,7 @@ WIP (add new stuff for the next release) ======================================== * bump bundled imaplib2 library 2.29 --> 2.33 +* Actually perform the SSL fingerprint check (reported by J. Cook) * Curses UI, don't use colors after we shut down curses already (C.Höger) OfflineIMAP v6.5.3.1 (2012-04-03) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 2aa81d9..aa165f0 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -1,6 +1,6 @@ # imaplib utilities # Copyright (C) 2002-2007 John Goerzen -# 2010 Sebastian Spaeth +# 2012-2012 Sebastian Spaeth # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -143,8 +143,7 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): def open(self, host=None, port=None): super(WrappedIMAP4_SSL, self).open(host, port) - if (self._fingerprint or not self.ca_certs) and\ - 'ssl' in locals(): # <--disable for python 2.5 + if (self._fingerprint or not self.ca_certs): # compare fingerprints fingerprint = sha1(self.sock.getpeercert(True)).hexdigest() if fingerprint != self._fingerprint: From f6b8426e169c65a8f665f8e1b82470aa32ba9dae Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 20 Apr 2012 11:11:44 +0200 Subject: [PATCH 518/817] Document that '%' needs encoding as '%%' in offlineimap.conf We use python's SafeConfigParser, and this is a built in "feature" used for interpolating variables. But it imples that '%' needs encoding as '%%'. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 1 + offlineimap.conf | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 6184622..2564ae5 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,7 @@ WIP (add new stuff for the next release) * bump bundled imaplib2 library 2.29 --> 2.33 * Actually perform the SSL fingerprint check (reported by J. Cook) * Curses UI, don't use colors after we shut down curses already (C.Höger) +* Document that '%' needs encoding as '%%' in *.conf OfflineIMAP v6.5.3.1 (2012-04-03) ================================= diff --git a/offlineimap.conf b/offlineimap.conf index 627b237..0c1880f 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -21,6 +21,7 @@ # # would set the trashfolder setting for your German gmail accounts. +# NOTE2: This implies that any '%' needs to be encoded as '%%' ################################################## # General definitions @@ -341,7 +342,7 @@ remoteuser = username # OfflineIMAP starts when using a UI that supports this. # # 2. The remote password stored in this file with the remotepass -# option. Example: +# option. Any '%' needs to be encoded as '%%'. Example: # remotepass = mypassword # # 3. The remote password stored as a single line in an external From 61e754c65ef08c85aa160b407d21475174c56262 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 21 Apr 2012 13:26:09 +0200 Subject: [PATCH 519/817] Do not try to release IMAP connection twice Reported by sharat87 in https://github.com/spaetz/offlineimap/pull/38, he would often get an unhandled Exception when trying to releaseconnection() a connection that was not in the pool of connections. The reason this could happen is that when folder.IMAP.quickchanged() raises an Exception in select(), we would release the connection in the "except" handling, and than release the same connection in the "finally" clause, which led to the error. The right thing is to only release the connection once, of course. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 1 + offlineimap/folder/IMAP.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 2564ae5..7acccd3 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,7 @@ WIP (add new stuff for the next release) * Actually perform the SSL fingerprint check (reported by J. Cook) * Curses UI, don't use colors after we shut down curses already (C.Höger) * Document that '%' needs encoding as '%%' in *.conf +* Fix crash when IMAP.quickchanged() led to an Error (reported by sharat87) OfflineIMAP v6.5.3.1 (2012-04-03) ================================= diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 6bdcc71..bb1b51f 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -93,14 +93,17 @@ class IMAPFolder(BaseFolder): # Select folder and get number of messages restype, imapdata = imapobj.select(self.getfullname(), True, True) + self.imapserver.releaseconnection(imapobj) except OfflineImapError as e: # retry on dropped connections, raise otherwise self.imapserver.releaseconnection(imapobj, True) if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: retry = True else: raise - finally: - self.imapserver.releaseconnection(imapobj) + except: + # cleanup and raise on all other errors + self.imapserver.releaseconnection(imapobj, True) + raise # 1. Some mail servers do not return an EXISTS response # if the folder is empty. 2. ZIMBRA servers can return # multiple EXISTS replies in the form 500, 1000, 1500, From 644b9f0bb9ce1f0b60cd4814162c8029e452cbc9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 8 May 2012 16:31:19 +0200 Subject: [PATCH 520/817] Implement .readonly property for repositories Set the value once on repository initialization to centralize the default value. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 606294e..61cdc75 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -32,6 +32,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): self.name = reposname self.localeval = account.getlocaleval() self._accountname = self.account.getname() + self._readonly = self.getconfboolean('readonly', False) self.uiddir = os.path.join(self.config.getmetadatadir(), 'Repository-' + self.name) if not os.path.exists(self.uiddir): os.mkdir(self.uiddir, 0o700) @@ -108,6 +109,11 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getconfig(self): return self.config + @property + def readonly(self): + """Is the repository readonly?""" + return self._readonly + def getlocaleval(self): return self.account.getlocaleval() From 0752c123f518a10b0dfe0786248dfe2af1c8f603 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 8 May 2012 16:41:21 +0200 Subject: [PATCH 521/817] Implement the "createfolders" setting for repositories By default OfflineImap propagates new folders in both directions. Sometimes this is not what you want. E.g. you might want new folders on your IMAP server to propagate to your local MailDir, but not the other way around. The 'readonly' setting on a repository will not help here, as it prevents any change from occuring on that repository. This is what the `createfolders` setting is for. By default it is `True`, meaning that new folders can be created on this repository. To prevent folders from ever being created on a repository, set this to `False`. If you set this to False on the REMOTE repository, you will not have to create the `Reverse nametrans`_ rules on the LOCAL repository. Also implement a test for this Signed-off-by: Sebastian Spaeth --- Changelog.rst | 1 + docs/doc-src/nametrans.rst | 16 ++++++++++++++++ offlineimap.conf | 8 ++++++++ offlineimap/repository/Base.py | 15 +++++++++++++-- test/tests/test_01_basic.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 7acccd3..704d3ae 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -13,6 +13,7 @@ WIP (add new stuff for the next release) * Curses UI, don't use colors after we shut down curses already (C.Höger) * Document that '%' needs encoding as '%%' in *.conf * Fix crash when IMAP.quickchanged() led to an Error (reported by sharat87) +* Implement the createfolders setting to disable folder propagation (see docs) OfflineIMAP v6.5.3.1 (2012-04-03) ================================= diff --git a/docs/doc-src/nametrans.rst b/docs/doc-src/nametrans.rst index ca25345..c211072 100644 --- a/docs/doc-src/nametrans.rst +++ b/docs/doc-src/nametrans.rst @@ -65,6 +65,22 @@ have filtered out everything starting with "debian" in your folderfilter settings. +createfolders +------------- + +By default OfflineImap propagates new folders in both +directions. Sometimes this is not what you want. E.g. you might want +new folders on your IMAP server to propagate to your local MailDir, +but not the other way around. The 'readonly' setting on a repository +will not help here, as it prevents any change from occuring on that +repository. This is what the `createfolders` setting is for. By +default it is `True`, meaning that new folders can be created on this +repository. To prevent folders from ever being created on a +repository, set this to `False`. If you set this to False on the +REMOTE repository, you will not have to create the `Reverse +nametrans`_ rules on the LOCAL repository. + + nametrans ---------- diff --git a/offlineimap.conf b/offlineimap.conf index 0c1880f..fccceab 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -504,6 +504,14 @@ remoteuser = username # one. For example: # folderincludes = ['debian.user', 'debian.personal'] + +# If you do not want to have any folders created on this repository, +# set the createfolders variable to False, the default is True. Using +# this feature you can e.g. disable the propagation of new folders to +# the new repository. +#createfolders = True + + # You can specify 'foldersort' to determine how folders are sorted. # This affects order of synchronization and mbnames. The expression # should return -1, 0, or 1, as the default Python cmp() does. The two diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 61cdc75..81050b0 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -129,6 +129,13 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getsep(self): raise NotImplementedError + def get_create_folders(self): + """Is folder creation enabled on this repository? + + It is disabled by either setting the whole repository + 'readonly' or by using the 'createfolders' setting.""" + return self._readonly or self.getconfboolean('createfolders', True) + def makefolder(self, foldername): """Create a new folder""" raise NotImplementedError @@ -147,6 +154,10 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): that forward and backward nametrans actually match up! Configuring nametrans on BOTH repositories therefore could lead to infinite folder creation cycles.""" + if not self.get_create_folders() and not dst_repo.get_create_folders(): + # quick exit if no folder creation is enabled on either side. + return + src_repo = self src_folders = src_repo.getfolders() dst_folders = dst_repo.getfolders() @@ -166,7 +177,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): # Find new folders on src_repo. for src_name_t, src_folder in src_hash.iteritems(): # Don't create on dst_repo, if it is readonly - if dst_repo.getconfboolean('readonly', False): + if not dst_repo.get_create_folders(): break if src_folder.sync_this and not src_name_t in dst_folders: try: @@ -181,7 +192,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): status_repo.getsep())) # Find new folders on dst_repo. for dst_name_t, dst_folder in dst_hash.iteritems(): - if self.getconfboolean('readonly', False): + if not src_repo.get_create_folders(): # Don't create missing folder on readonly repo. break diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index f5a0ea1..8e45a62 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -105,6 +105,7 @@ class TestBasicFunctions(unittest.TestCase): # Write out default config file again OLITestLib.write_config_file() + def test_04_createmail(self): """Create mail in OLItest 1, sync, wipe folder sync @@ -127,3 +128,30 @@ class TestBasicFunctions(unittest.TestCase): self.assertFalse (None in uids, msg = "All mails should have been "+ \ "assigned the IMAP's UID number, but {} messages had no valid ID "\ .format(len([None for x in uids if x==None]))) + + def test_05_createfolders(self): + """Test if createfolders works as expected + + Create a local Maildir, then sync with remote "createfolders" + disabled. Delete local Maildir and sync. We should have no new + local maildir then. TODO: Rewrite this test to directly test + and count the remote folders when the helper functions have + been written""" + config = OLITestLib.get_default_config() + config.set('Repository IMAP', 'createfolders', + 'False' ) + OLITestLib.write_config_file(config) + + # delete all remote and local testfolders + OLITestLib.delete_remote_testfolders() + OLITestLib.delete_maildir('') + OLITestLib.create_maildir('INBOX.OLItest') + code, res = OLITestLib.run_OLI() + #logging.warn("%s %s "% (code, res)) + self.assertEqual(res, "") + OLITestLib.delete_maildir('INBOX.OLItest') + code, res = OLITestLib.run_OLI() + boxes, mails = OLITestLib.count_maildir_mails('') + self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0 " + "mails, but sync led to {} folders and {} mails".format( + boxes, mails)) From 686561b42cbfc0207426d68a7db8823d096bdf7d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 2 Jun 2012 13:41:20 +0200 Subject: [PATCH 522/817] Release v6.5.4 See Changelog for details Signed-off-by: Sebastian Spaeth --- Changelog.rst | 3 +++ offlineimap/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 704d3ae..e4c21ae 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,9 @@ ChangeLog WIP (add new stuff for the next release) ======================================== +OfflineIMAP v6.5.4 (2012-06-02) +================================= + * bump bundled imaplib2 library 2.29 --> 2.33 * Actually perform the SSL fingerprint check (reported by J. Cook) * Curses UI, don't use colors after we shut down curses already (C.Höger) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 81dcbac..77495e4 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.3.1" +__version__ = "6.5.4" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 7e8233131c5d9fe13656db94d5ab3fc994bf9460 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 3 Apr 2012 01:13:27 +0200 Subject: [PATCH 523/817] Don't fail if no dry-run setting has been specified Somehow we failed if no dry-run setting had been specified in the config file. This got caught thanks to extending the test suite with a stock configuration. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index e4c21ae..6996f98 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -24,6 +24,9 @@ OfflineIMAP v6.5.3.1 (2012-04-03) * Don't fail if no dry-run setting exists in offlineimap.conf (introduced in 6.5.3) +* Don't fail if no dry-run setting exists in offlineimap.conf + (introduced in 6.5.3) + OfflineIMAP v6.5.3 (2012-04-02) =============================== From a614f9a735b2dfe82302f728ea09e16bb1d3ee5f Mon Sep 17 00:00:00 2001 From: David Logie Date: Fri, 18 May 2012 10:31:08 +0100 Subject: [PATCH 524/817] Fix str.format() calls for Python 2.6. Python 2.6 requires a slightly different string formatting that >2.7. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 2 ++ offlineimap/folder/Base.py | 2 +- offlineimap/ui/UIBase.py | 6 +++--- test/OLItest/TestRunner.py | 8 ++++---- test/tests/test_01_basic.py | 10 +++++----- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 6996f98..e6f021a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,8 @@ ChangeLog WIP (add new stuff for the next release) ======================================== +* Fix str.format() calls for Python 2.6 (D. Logie) + OfflineIMAP v6.5.4 (2012-06-02) ================================= diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 6f6f364..b1f5a56 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -386,7 +386,7 @@ class BaseFolder(object): self.getmessageuidlist()) num_to_copy = len(copylist) if num_to_copy and self.repository.account.dryrun: - self.ui.info("[DRYRUN] Copy {} messages from {}[{}] to {}".format( + self.ui.info("[DRYRUN] Copy {0} messages from {1}[{2}] to {3}".format( num_to_copy, self, self.repository, dstfolder.repository)) return for num, uid in enumerate(copylist): diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index eea929d..fab3818 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -301,7 +301,7 @@ class UIBase(object): def makefolder(self, repo, foldername): """Called when a folder is created""" prefix = "[DRYRUN] " if self.dryrun else "" - self.info("{}Creating folder {}[{}]".format( + self.info("{0}Creating folder {1}[{2}]".format( prefix, foldername, repo)) def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder): @@ -346,7 +346,7 @@ class UIBase(object): def deletingmessages(self, uidlist, destlist): ds = self.folderlist(destlist) prefix = "[DRYRUN] " if self.dryrun else "" - self.info("{}Deleting {} messages ({}) in {}".format( + self.info("{0}Deleting {1} messages ({2}) in {3}".format( prefix, len(uidlist), offlineimap.imaputil.uid_sequence(uidlist), ds)) @@ -474,7 +474,7 @@ class UIBase(object): def callhook(self, msg): if self.dryrun: - self.info("[DRYRUN] {}".format(msg)) + self.info("[DRYRUN] {0}".format(msg)) else: self.info(msg) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 3535b61..828df59 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -127,7 +127,7 @@ class OLITestLib(): reponame: All on `reponame` or all IMAP-type repositories if None""" config = cls.get_default_config() if reponame: - sections = ['Repository {}'.format(reponame)] + sections = ['Repository {0}'.format(reponame)] else: sections = [r for r in config.sections() \ if r.startswith('Repository')] @@ -162,8 +162,8 @@ class OLITestLib(): dirs = [d for d in dirs if d.startswith(b'INBOX.OLItest')] for folder in dirs: res_t, data = imapobj.delete(b'\"'+folder+b'\"') - assert res_t == 'OK', "Folder deletion of {} failed with error"\ - ":\n{} {}".format(folder.decode('utf-8'), res_t, data) + assert res_t == 'OK', "Folder deletion of {0} failed with error"\ + ":\n{1} {2}".format(folder.decode('utf-8'), res_t, data) imapobj.logout() @classmethod @@ -197,7 +197,7 @@ class OLITestLib(): Use some default content if not given""" assert cls.testdir != None while True: # Loop till we found a unique filename - mailfile = '{}:2,'.format(random.randint(0,999999999)) + mailfile = '{0}:2,'.format(random.randint(0,999999999)) mailfilepath = os.path.join(cls.testdir, 'mail', folder, 'new', mailfile) if not os.path.isfile(mailfilepath): diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 8e45a62..3cbd5c9 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -67,7 +67,7 @@ class TestBasicFunctions(unittest.TestCase): self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0 " - "mails, but sync led to {} folders and {} mails".format( + "mails, but sync led to {0} folders and {1} mails".format( boxes, mails)) def test_02_createdir(self): @@ -82,7 +82,7 @@ class TestBasicFunctions(unittest.TestCase): self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(2,0), msg="Expected 2 folders and 0 " - "mails, but sync led to {} folders and {} mails".format( + "mails, but sync led to {0} folders and {1} mails".format( boxes, mails)) def test_03_nametransmismatch(self): @@ -101,7 +101,7 @@ class TestBasicFunctions(unittest.TestCase): mismatch = "ERROR: INFINITE FOLDER CREATION DETECTED!" in res self.assertEqual(mismatch, True, msg="Mismatching nametrans rules did " "NOT trigger an 'infinite folder generation' error. Output was:\n" - "{}".format(res)) + "{0}".format(res)) # Write out default config file again OLITestLib.write_config_file() @@ -121,12 +121,12 @@ class TestBasicFunctions(unittest.TestCase): self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(1,1), msg="Expected 1 folders and 1 " - "mails, but sync led to {} folders and {} mails".format( + "mails, but sync led to {0} folders and {1} mails".format( boxes, mails)) # The local Mail should have been assigned a proper UID now, check! uids = OLITestLib.get_maildir_uids('INBOX.OLItest') self.assertFalse (None in uids, msg = "All mails should have been "+ \ - "assigned the IMAP's UID number, but {} messages had no valid ID "\ + "assigned the IMAP's UID number, but {0} messages had no valid ID "\ .format(len([None for x in uids if x==None]))) def test_05_createfolders(self): From 36156fa98545792649771d2599ee06a74d35169c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 6 Jun 2012 10:02:42 +0200 Subject: [PATCH 525/817] Remove the APPENDUID hack, previously introduced As Gmail was only announcing the presence of the UIDPLUS extension after we logged in, and we were then only getting server capabilities before, a hack was introduced that checked the existence of an APPENDUID reply, even if the server did not claim to support it. However, John Wiegley reports problems, where the APPENDUID would be None, and we attempt to go this path (it seems that imaplib2 returns [None] if there is no such reply, so our test here for "!=" might fail. Given that this is an undocumented imaplib2 function anyway, and we do fetch gmail capabilities after authentication, this hack should no longer be necessary. We had problems there earlier, where imapobj.response() would return [None] although we had received a APPENDUID response from the server, this might need more debugging and testing. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 2 ++ offlineimap/folder/IMAP.py | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index e6f021a..30b58eb 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -9,6 +9,8 @@ WIP (add new stuff for the next release) ======================================== * Fix str.format() calls for Python 2.6 (D. Logie) +* Remove APPENDUID hack, previously introduced to fix Gmail, no longer + necessary, it might have been breaking things. (J. Wiegley) OfflineIMAP v6.5.4 (2012-06-02) ================================= diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index bb1b51f..75520c2 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -575,9 +575,8 @@ class IMAPFolder(BaseFolder): (typ,dat) = imapobj.check() assert(typ == 'OK') - # get the new UID. Test for APPENDUID response even if the - # server claims to not support it, as e.g. Gmail does :-( - if use_uidplus or imapobj._get_untagged_response('APPENDUID', True): + # get the new UID, do we use UIDPLUS? + if use_uidplus: # get new UID from the APPENDUID response, it could look # like OK [APPENDUID 38505 3955] APPEND completed with # 38505 bein folder UIDvalidity and 3955 the new UID. @@ -585,7 +584,7 @@ class IMAPFolder(BaseFolder): # often seems to return [None], even though we have # data. TODO resp = imapobj._get_untagged_response('APPENDUID') - if resp == [None]: + if resp == [None] or resp is None: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " "appending a message.") return 0 From d45872e59cfed26b779544a1dec35313cf70a2ac Mon Sep 17 00:00:00 2001 From: Dmitrijs Ledkovs Date: Tue, 5 Jun 2012 01:45:10 +0100 Subject: [PATCH 526/817] Add missing OfflineImapError import in folder/UIDMaps.py Bug-Debian: http://bugs.debian.org/671279 Reported-by: Ansgar Burchardt Signed-off-by: Dmitrijs Ledkovs Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/UIDMaps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index f1c11e4..f571772 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import Lock +from offlineimap import OfflineImapError from .IMAP import IMAPFolder import os.path From 799905325ceba72b2038df132298816c1a2a06eb Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 23 Jul 2012 18:05:46 +0200 Subject: [PATCH 527/817] docs: split long lines (up to 80 characters) Not everybody has a wide screen. Signed-off-by: Nicolas Sebrecht --- README | 8 ++++++-- docs/INSTALL.rst | 27 ++++++++++++++++++++------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/README b/README index dc95198..5f70360 100644 --- a/README +++ b/README @@ -10,7 +10,9 @@ messages on each computer, and changes you make one place will be visible on all other systems. For instance, you can delete a message on your home computer, and it will appear deleted on your work computer as well. OfflineIMAP is also useful if you want to use a mail reader that does not have IMAP support, has poor IMAP -support, or does not provide disconnected operation. It's homepage at http://offlineimap.org contains more information, source code, and online documentation. +support, or does not provide disconnected operation. It's homepage at +http://offlineimap.org contains more information, source code, and online +documentation. OfflineIMAP does not require additional python dependencies beyond python >=2.6 (although python-sqlite is strongly recommended). @@ -87,7 +89,9 @@ Mailing list & bug reporting ---------------------------- The user discussion, development and all exciting stuff take place in the -OfflineImap mailing list at http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project. You do not need to subscribe to send emails. +OfflineImap mailing list at +http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project. You do not +need to subscribe to send emails. Bugs, issues and contributions should be reported to the mailing list. Bugs can also be reported in the issue tracker at https://github.com/spaetz/offlineimap/issues. diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index d14c574..f86849d 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -37,29 +37,40 @@ In order to use `OfflineIMAP`_, you need to have these conditions satisfied: Installation ------------ -Installing OfflineImap should usually be quite easy, as you can simply unpack and run OfflineImap in place if you wish to do so. There are a number of options though: +Installing OfflineImap should usually be quite easy, as you can simply unpack +and run OfflineImap in place if you wish to do so. There are a number of options +though: #. system-wide :ref:`installation via your distribution package manager ` #. system-wide or single user :ref:`installation from the source package ` #. system-wide or single user :ref:`installation from a git checkout ` -Having installed OfflineImap, you will need to configure it, to be actually useful. Please check the :ref:`Configuration` section in the :doc:`MANUAL` for more information on the configuration step. +Having installed OfflineImap, you will need to configure it, to be actually +useful. Please check the :ref:`Configuration` section in the :doc:`MANUAL` for +more information on the configuration step. .. _inst_pkg_man: System-Wide Installation via distribution ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The easiest way to install OfflineIMAP is via your distribution's package manager. OfflineImap is available under the name `offlineimap` in most Linux and BSD distributions. +The easiest way to install OfflineIMAP is via your distribution's package +manager. OfflineImap is available under the name `offlineimap` in most Linux and +BSD distributions. .. _inst_src_tar: Installation from source package ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Download the latest source archive from our `download page `_. Simply click the "Download as .zip" or "Download as .tar.gz" buttons to get the latest "stable" code from the master branch. If you prefer command line, you will want to use: - wget https://github.com/spaetz/offlineimap/tarball/master -Unpack and continue with the :ref:`system-wide installation ` or the :ref:`single-user installation ` section. +Download the latest source archive from our `download page +`_. Simply click the "Download +as .zip" or "Download as .tar.gz" buttons to get the latest "stable" code from +the master branch. If you prefer command line, you will want to use: wget +https://github.com/spaetz/offlineimap/tarball/master + +Unpack and continue with the :ref:`system-wide installation ` +or the :ref:`single-user installation ` section. .. _inst_git: @@ -78,7 +89,9 @@ checkout a particular release like this:: cd offlineimap git checkout v6.5.2.1 -You have now a source tree available and proceed with either the :ref:`system-wide installation ` or the :ref:`single-user installation `. +You have now a source tree available and proceed with either the +:ref:`system-wide installation ` or the :ref:`single-user +installation `. .. _system_wide_inst: From 6c85a4518d91c2dda0f73fe529a62feb6072e899 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 5 Aug 2012 22:40:52 +0400 Subject: [PATCH 528/817] IMAPlib mixin class: pass 'readonly' exception to our callers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will allow our callers who are capable of dealing with readonly folders to properly detect this condition and act accordingly. One example is Gmail's "Chats" folder that is read-only, but contains logs of the quick chats. Tested-by: Abdó Roig-Maranges Signed-off-by: Eygene Ryabinkin --- offlineimap/imaplibutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index aa165f0..4290b2b 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -49,6 +49,9 @@ class UsefulIMAPMixIn(object): del self.untagged_responses[:] try: result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) + except self.readonly as e: + # pass self.readonly to our callers + raise except self.abort as e: # self.abort is raised when we are supposed to retry errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ From 131ac5c8276b7359d307073130406e64835b8f49 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 5 Aug 2012 22:40:52 +0400 Subject: [PATCH 529/817] IMAPlib mixin class: pass 'readonly' exception to our callers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will allow our callers who are capable of dealing with readonly folders to properly detect this condition and act accordingly. One example is Gmail's "Chats" folder that is read-only, but contains logs of the quick chats. Minor Changelog improvements. Tested-by: Abdó Roig-Maranges Signed-off-by: Eygene Ryabinkin Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 14 +++++++------- offlineimap/imaplibutil.py | 3 +++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 30b58eb..4b31f39 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,17 +8,20 @@ ChangeLog WIP (add new stuff for the next release) ======================================== +* Learn to deal with readonly folders to properly detect this condition and act + accordingly. One example is Gmail's "Chats" folder that is read-only, + but contains logs of the quick chats. (E. Ryabinkin) * Fix str.format() calls for Python 2.6 (D. Logie) * Remove APPENDUID hack, previously introduced to fix Gmail, no longer necessary, it might have been breaking things. (J. Wiegley) OfflineIMAP v6.5.4 (2012-06-02) -================================= +=============================== * bump bundled imaplib2 library 2.29 --> 2.33 * Actually perform the SSL fingerprint check (reported by J. Cook) * Curses UI, don't use colors after we shut down curses already (C.Höger) -* Document that '%' needs encoding as '%%' in *.conf +* Document that '%' needs encoding as '%%' in configuration files. * Fix crash when IMAP.quickchanged() led to an Error (reported by sharat87) * Implement the createfolders setting to disable folder propagation (see docs) @@ -28,9 +31,6 @@ OfflineIMAP v6.5.3.1 (2012-04-03) * Don't fail if no dry-run setting exists in offlineimap.conf (introduced in 6.5.3) -* Don't fail if no dry-run setting exists in offlineimap.conf - (introduced in 6.5.3) - OfflineIMAP v6.5.3 (2012-04-02) =============================== @@ -67,7 +67,7 @@ OfflineIMAP v6.5.3 (2012-04-02) * Improve compatability of the curses UI with python 2.6 OfflineIMAP v6.5.2.1 (2012-04-04) -===================================== +================================= * Fix python2.6 compatibility with the TTYUI backend (crash) @@ -123,7 +123,7 @@ Smallish bug fixes that deserve to be put out. * Add filter information to the filter list in --info output OfflineIMAP v6.5.1.1 (2012-01-07) - "Das machine control is nicht fur gerfinger-poken und mittengrabben" -================================================================================================================== +======================================================================================================== Blinkenlights UI 6.5.0 regression fixes only. diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index aa165f0..4290b2b 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -49,6 +49,9 @@ class UsefulIMAPMixIn(object): del self.untagged_responses[:] try: result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) + except self.readonly as e: + # pass self.readonly to our callers + raise except self.abort as e: # self.abort is raised when we are supposed to retry errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ From 44d2199a00e731495117e613e29629504eca82e0 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 23 Jul 2012 18:03:43 +0200 Subject: [PATCH 530/817] docs: update links to the newly created github organization Acually, these are the minimal expected changes. More might be done and/or discussed. Signed-off-by: Nicolas Sebrecht --- README | 4 +++- docs/INSTALL.rst | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README b/README index 5f70360..8f80d7a 100644 --- a/README +++ b/README @@ -93,7 +93,9 @@ OfflineImap mailing list at http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project. You do not need to subscribe to send emails. -Bugs, issues and contributions should be reported to the mailing list. Bugs can also be reported in the issue tracker at https://github.com/spaetz/offlineimap/issues. +Bugs, issues and contributions should be reported to the mailing list. Bugs can +also be reported in the issue tracker at +https://github.com/OfflineIMAP/offlineimap/issues. Configuration Examples ====================== diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index f86849d..83608d4 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -1,6 +1,6 @@ .. -*- coding: utf-8 -*- -.. _OfflineIMAP: https://github.com/spaetz/offlineimap -.. _OLI_git_repo: git://github.com/spaetz/offlineimap.git +.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap +.. _OLI_git_repo: git://github.com/OfflineIMAP/offlineimap.git ============ Installation From 6e5fbebc0e5e6388cf03ce0f7267638081294b77 Mon Sep 17 00:00:00 2001 From: Vladimir Nesov Date: Fri, 17 Aug 2012 17:52:20 +0000 Subject: [PATCH 531/817] fix: createfolders setting is ignored if readonly is True If 'readonly' is True, folders shouldn't be created (regardless of 'createfolders' option). With old behavior, instead folders were always created when 'readonly' is True (even if 'createfolders' was also False), which is a serious bug (offlineimap was creating folders in all read-only repositories). 'createfolders' should only play a role if 'readonly' is False, in which case folders should only be created if 'createfolders' is True. Submitted-by: Vladimir Nesov Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 1 + offlineimap/repository/Base.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 4b31f39..aa0d903 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,7 @@ ChangeLog WIP (add new stuff for the next release) ======================================== +* Don't create folders if readonly is enabled. * Learn to deal with readonly folders to properly detect this condition and act accordingly. One example is Gmail's "Chats" folder that is read-only, but contains logs of the quick chats. (E. Ryabinkin) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 81050b0..febaf3f 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -134,7 +134,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): It is disabled by either setting the whole repository 'readonly' or by using the 'createfolders' setting.""" - return self._readonly or self.getconfboolean('createfolders', True) + return (not self._readonly) and self.getconfboolean('createfolders', Tr def makefolder(self, foldername): """Create a new folder""" From a134301ac505678593a42ac8c483cf1b42ad6bad Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 21 Aug 2012 16:48:26 +0200 Subject: [PATCH 532/817] Improve 'find first quotation' regex Reported by http://www.dfranke.com/blog/2012/08/20/offlineimap-error-beim-syncen-mit-lotus-domino/ our 'find the first quote possible containing encoded quotation marks' regex did not seem to have caught all cases. E.g. "\\". Verified the fix as good. Thanks Daniel Franke. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 2 ++ offlineimap/imaputil.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index aa0d903..f126075 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -15,6 +15,8 @@ WIP (add new stuff for the next release) * Fix str.format() calls for Python 2.6 (D. Logie) * Remove APPENDUID hack, previously introduced to fix Gmail, no longer necessary, it might have been breaking things. (J. Wiegley) +* Improve regex that could lead to 'NoneType' object has no attribute 'group' + (D. Franke) OfflineIMAP v6.5.4 (2012-06-02) =============================== diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 557064b..fe69b7a 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -23,7 +23,7 @@ from offlineimap.ui import getglobalui # find the first quote in a string quotere = re.compile( - r"""(?P"(?:\\"|[^"])*") # Quote, possibly containing encoded + r"""(?P"[^\"\\]*(?:\\"|[^"])*") # Quote, possibly containing encoded # quotation mark \s*(?P.*)$ # Whitespace & remainder of string""", re.VERBOSE) From 925a5bcae16b855e1f2cd990c8e947972bdebb6e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 21 Aug 2012 16:59:52 +0200 Subject: [PATCH 533/817] Fix cut-off line Commit 6e5fbebc introduced a cut-off line. Fixing that. All tests ran successfully. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index febaf3f..0c41c2d 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -133,8 +133,9 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): """Is folder creation enabled on this repository? It is disabled by either setting the whole repository - 'readonly' or by using the 'createfolders' setting.""" - return (not self._readonly) and self.getconfboolean('createfolders', Tr + 'readonly' or by using the 'createfolders' setting.""" + return (not self._readonly) and \ + self.getconfboolean('createfolders', True) def makefolder(self, foldername): """Create a new folder""" From 103524c97970c3f4aaeefcf86f73f91dc0924629 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 31 Aug 2012 20:05:01 +0200 Subject: [PATCH 534/817] Throw OfflineImapErrors rather than weird Exceptions When misconfiguring OLI, e.g. by specifying a repository name that was not configured anywhere, we would bomb out with cryptic "NoSectionError". Throw OfflineImapError that explains what has happened. We still need to avoid throwing exceptions with Tracebacks here though. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 1 + offlineimap/repository/__init__.py | 26 +++++++++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index f126075..0a57919 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -17,6 +17,7 @@ WIP (add new stuff for the next release) necessary, it might have been breaking things. (J. Wiegley) * Improve regex that could lead to 'NoneType' object has no attribute 'group' (D. Franke) +* Improved error throwing on repository misconfiguration OfflineIMAP v6.5.4 (2012-06-02) =============================== diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py index 0059bdf..22cd128 100644 --- a/offlineimap/repository/__init__.py +++ b/offlineimap/repository/__init__.py @@ -15,10 +15,17 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +try: + from configparser import NoSectionError +except ImportError: #python2 + from ConfigParser import NoSectionError + from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository from offlineimap.repository.Gmail import GmailRepository from offlineimap.repository.Maildir import MaildirRepository from offlineimap.repository.LocalStatus import LocalStatusRepository +from offlineimap.error import OfflineImapError + class Repository(object): """Abstract class that returns the correct Repository type @@ -47,17 +54,26 @@ class Repository(object): return LocalStatusRepository(name, account) else: - raise ValueError("Request type %s not supported" % reqtype) + errstr = "Repository type %s not supported" % reqtype + raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO) + # Get repository type config = account.getconfig() - repostype = config.get('Repository ' + name, 'type').strip() + try: + repostype = config.get('Repository ' + name, 'type').strip() + except NoSectionError as e: + errstr = ("Could not find section '%s' in configuration. Required " + "for account '%s'." % ('Repository %s' % name, account)) + raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO) + try: repo = typemap[repostype] except KeyError: - raise ValueError("'%s' repository not supported for %s repositories" - "." % (repostype, reqtype)) - return repo(name, account) + errstr = "'%s' repository not supported for '%s' repositories." \ + % (repostype, reqtype) + raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO) + return repo(name, account) def __init__(self, account, reqtype): """Load the correct Repository type and return that. The From 04ffae2f004648cf2b4a0d4bffb029eca3b19b8d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 31 Aug 2012 20:34:54 +0200 Subject: [PATCH 535/817] Exit with nice error message failing to get repositories If we throw an OfflineImapError in case of the Repository() initialization, we display the nice error message and exit rather than bomb out with a traceback. Misconfiguring a repository name in the configuration file is now nicely pointed out to the user. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 8b39c7e..c199861 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -217,13 +217,19 @@ class SyncableAccount(Account): def syncrunner(self): self.ui.registerthread(self) - accountmetadata = self.getaccountmeta() - if not os.path.exists(accountmetadata): - os.mkdir(accountmetadata, 0o700) + try: + accountmetadata = self.getaccountmeta() + if not os.path.exists(accountmetadata): + os.mkdir(accountmetadata, 0o700) - self.remoterepos = Repository(self, 'remote') - self.localrepos = Repository(self, 'local') - self.statusrepos = Repository(self, 'status') + self.remoterepos = Repository(self, 'remote') + self.localrepos = Repository(self, 'local') + self.statusrepos = Repository(self, 'status') + except OfflineImapError as e: + self.ui.error(e, exc_info()[2]) + if e.severity >= OfflineImapError.ERROR.CRITICAL: + raise + return # Loop account sync if needed (bail out after 3 failures) looping = 3 From 3cb2ddccb897cfd1d6bcb4a2b9393040b899bd3e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 31 Aug 2012 22:34:53 +0200 Subject: [PATCH 536/817] Use "with lock" pattern While looking at the code to investigate if an why we sometimes don't seem to honor the write lock, I made it use the more modern "with lock:" pattern. Still have not found out how we could ever be using 2 instances of the LocalStatusFolder for the same folder. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/LocalStatus.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index b3779fd..6f65f83 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -85,8 +85,7 @@ class LocalStatusFolder(BaseFolder): file.close() def save(self): - self.savelock.acquire() - try: + with self.savelock: file = open(self.filename + ".tmp", "wt") file.write(magicline + "\n") for msg in self.messagelist.values(): @@ -104,9 +103,6 @@ class LocalStatusFolder(BaseFolder): os.fsync(fd) os.close(fd) - finally: - self.savelock.release() - def getmessagelist(self): return self.messagelist From a1dc76ae917d93d0c387bbd506faea3487211a6c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 31 Aug 2012 23:11:11 +0200 Subject: [PATCH 537/817] Don't output initial blurb in "quiet" mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When logging to a file using the -l switch, we would still write an initial banner to the file. This was never intended. Quiet should be really quiet unless it experiences an error. Simplify the logging statement, to do nothing if logevel is set to "WARNING" aka quiet. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index fab3818..2c30b94 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -88,7 +88,6 @@ class UIBase(object): def setlogfile(self, logfile): """Create file handler which logs to file""" fh = logging.FileHandler(logfile, 'at') - #fh.setLevel(logging.DEBUG) file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " "%(message)s", '%Y-%m-%d %H:%M:%S') fh.setFormatter(file_formatter) @@ -98,9 +97,7 @@ class UIBase(object): msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ "Args: %s" % (offlineimap.__version__, p_ver, sys.platform, " ".join(sys.argv)) - record = logging.LogRecord('OfflineImap', logging.INFO, __file__, - None, msg, None, None) - fh.emit(record) + self.logger.info(msg) def _msg(self, msg): """Display a message.""" From 1b54b85f2026be053bad4dcbcba9b16bbdd95a3d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 1 Sep 2012 01:02:20 +0200 Subject: [PATCH 538/817] Reuse LocalStatus() folders rather than recreate instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we ask twice for a LocalStatusFolder via getfolder(), we would get a newly created instance each time. This can lead to problems, as e.g. write locks protecting files only work within the same Folder instance. Make it so, that we cache all Folder instances that we have asked for and hand back the existing one if we ask again for it, rather than recreate a new instance. Also, make getfolders() a noop for LocalStatus. We attempted to derive the foldername from the name of the LocalStatusfile. However, this is not really possible, as we do file name mangling (".$" -> "dot", "/" -> ".") and there is no way to get the original folder name from the LocalStatus file name anyway. This commit could potentially solve the "file not found" errors, that people have been seeing with their LocalStatusCache files. If we have 2 instances of a LocalStatusFolder pointing to the same file, our locking system would not work. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 1 - offlineimap/repository/LocalStatus.py | 26 +++++++++++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index c199861..1ce9b85 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -295,7 +295,6 @@ class SyncableAccount(Account): # folder delimiter etc) remoterepos.getfolders() localrepos.getfolders() - statusrepos.getfolders() remoterepos.sync_folder_structure(localrepos, statusrepos) # replicate the folderstructure between REMOTE to LOCAL diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 3e5db2f..73a6a62 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -43,8 +43,8 @@ class LocalStatusRepository(BaseRepository): if not os.path.exists(self.root): os.mkdir(self.root, 0o700) - # self._folders is a list of LocalStatusFolders() - self._folders = None + # self._folders is a dict of name:LocalStatusFolders() + self._folders = {} def getsep(self): return '.' @@ -83,19 +83,23 @@ class LocalStatusRepository(BaseRepository): def getfolder(self, foldername): """Return the Folder() object for a foldername""" - return self.LocalStatusFolderClass(foldername, self) + if foldername in self._folders: + return self._folders[foldername] + + folder = self.LocalStatusFolderClass(foldername, self) + self._folders[foldername] = folder + return folder def getfolders(self): - """Returns a list of all cached folders.""" - if self._folders != None: - return self._folders + """Returns a list of all cached folders. - self._folders = [] - for folder in os.listdir(self.root): - self._folders.append(self.getfolder(folder)) - return self._folders + Does nothing for this backend. We mangle the folder file names + (see getfolderfilename) so we can not derive folder names from + the file names that we have available. TODO: need to store a + list of folder names somehow?""" + pass def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" - self._folders = None + self._folders = {} From e7ca5b25cba0679aa800b490d063b24c40864839 Mon Sep 17 00:00:00 2001 From: Dave Abrahams Date: Sat, 1 Sep 2012 02:16:06 +0200 Subject: [PATCH 539/817] Combine checks for ignored folders in one place - Factor out the code to find a local folder given a remote folder Patch by Dave, split and modified by Sebastian. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 1ce9b85..f6aae6a 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -261,6 +261,12 @@ class SyncableAccount(Account): if looping and self.sleeper() >= 2: looping = 0 + def get_local_folder(self, remotefolder): + """Return the corresponding local folder for a given remotefolder""" + return self.localrepos.getfolder( + remotefolder.getvisiblename(). + replace(self.remoterepos.getsep(), self.localrepos.getsep())) + def sync(self): """Synchronize the account once, then return @@ -305,10 +311,13 @@ class SyncableAccount(Account): for remotefolder in remoterepos.getfolders(): # check for CTRL-C or SIGTERM if Account.abort_NOW_signal.is_set(): break - if not remotefolder.sync_this: - self.ui.debug('', "Not syncing filtered remote folder '%s'" + + localfolder = self.get_local_folder(remotefolder) + if not (remotefolder.sync_this + and localfolder.sync_this): + self.ui.debug('', "Not syncing filtered folder '%s'" "[%s]" % (remotefolder, remoterepos)) - continue # Filtered out remote folder + continue # Ignore filtered folder thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, @@ -372,17 +381,8 @@ def syncfolder(account, remotefolder, quick): ui.registerthread(account) try: # Load local folder. - localfolder = localrepos.\ - getfolder(remotefolder.getvisiblename().\ - replace(remoterepos.getsep(), localrepos.getsep())) + localfolder = account.get_local_folder(remotefolder) - #Filtered folders on the remote side will not invoke this - #function, but we need to NOOP if the local folder is filtered - #out too: - if not localfolder.sync_this: - ui.debug('', "Not syncing filtered local folder '%s'" \ - % localfolder) - return # Write the mailboxes mbnames.add(account.name, localfolder.getname()) @@ -462,15 +462,8 @@ def syncfolder(account, remotefolder, quick): if e.severity > OfflineImapError.ERROR.FOLDER: raise else: - #if the initial localfolder assignement bailed out, the localfolder var will not be available, so we need ui.error(e, exc_info()[2], msg = "Aborting sync, folder '%s' " - "[acc: '%s']" % ( - remotefolder.getvisiblename().\ - replace(remoterepos.getsep(), localrepos.getsep()), - account)) - # we reconstruct foldername above rather than using - # localfolder, as the localfolder var is not - # available if assignment fails. + "[acc: '%s']" % (localfolder, account)) except Exception as e: ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ (account, remotefolder.getvisiblename(), From e94642bb4d591a1ea12b837daeaafb94076a9f0b Mon Sep 17 00:00:00 2001 From: Dave Abrahams Date: Sat, 1 Sep 2012 02:30:46 +0200 Subject: [PATCH 540/817] Determine folder syncing on Folder initialization Rather than later when retrieving them by IMAP, we determine the "sync_this" value of a folder on Folder() initialization. This also fixes a bug where folder structure was not propagated when a folder was filtered out but included in the folderincludes list. Patch by Dave, split and modified slightly by Sebastian Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 12 ++++++++++-- offlineimap/repository/Base.py | 6 +++++- offlineimap/repository/IMAP.py | 5 ----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index b1f5a56..c132713 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -31,9 +31,12 @@ class BaseFolder(object): :para name: Path & name of folder minus root or reference :para repository: Repository() in which the folder is. """ - self.sync_this = True - """Should this folder be included in syncing?""" self.ui = getglobalui() + """Should this folder be included in syncing?""" + self._sync_this = repository.should_sync_folder(name) + if not self._sync_this: + self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter" \ + % (name, repository)) # Top level dir name is always '' self.name = name if not name == self.getsep() else '' self.repository = repository @@ -57,6 +60,11 @@ class BaseFolder(object): """Account name as string""" return self.repository.accountname + @property + def sync_this(self): + """Should this folder be synced or is it e.g. filtered out?""" + return self._sync_this + def suggeststhreads(self): """Returns true if this folder suggests using threads for actions; false otherwise. Probably only IMAP will return true.""" diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 0c41c2d..575c993 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -129,6 +129,10 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getsep(self): raise NotImplementedError + def should_sync_folder(self, fname): + """Should this folder be synced?""" + return fname in self.folderincludes or self.folderfilter(fname) + def get_create_folders(self): """Is folder creation enabled on this repository? @@ -202,7 +206,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): # Does nametrans back&forth lead to identical names? # 1) would src repo filter out the new folder name? In this # case don't create it on it: - if not self.folderfilter(dst_name_t): + if not self.should_sync_folder(dst_name_t): self.ui.debug('', "Not creating folder '%s' (repository '%s" "') as it would be filtered out on that repository." % (dst_name_t, self)) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 5ad787a..be8c858 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -287,11 +287,6 @@ class IMAPRepository(BaseRepository): foldername = imaputil.dequote(name) retval.append(self.getfoldertype()(self.imapserver, foldername, self)) - # filter out the folder? - if not self.folderfilter(foldername): - self.ui.debug('imap', "Filtering out '%s'[%s] due to folderfilt" - "er" % (foldername, self)) - retval[-1].sync_this = False # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() From 7d1d5283f8c058e23200817bd0c47c04982075a4 Mon Sep 17 00:00:00 2001 From: Dave Abrahams Date: Sat, 1 Sep 2012 02:49:00 +0200 Subject: [PATCH 541/817] No need to filter Maildir folders here commit e94642bb4d591a1ea12 centralized folder filtering by using the repository.should_sync_folder() function. Therefore there is no need to check for folderfilter in the Maildir backend separately. Origina patch by Dave, split into 3 by Sebastian Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Maildir.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index f197002..f2d5581 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -181,11 +181,6 @@ class MaildirRepository(BaseRepository): foldername, self.getsep(), self)) - # filter out the folder? - if not self.folderfilter(foldername): - self.debug("Filtering out '%s'[%s] due to folderfilt" - "er" % (foldername, self)) - retval[-1].sync_this = False if self.getsep() == '/' and dirname != '': # Recursively check sub-directories for folders too. From 49c11fc8f0ea8cb72de558758b9870863b20ad85 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 1 Sep 2012 02:58:14 +0200 Subject: [PATCH 542/817] LocalStatus._folders needs to be {} not None the LocalStatus._folders cache was changed to be a dict that can be searched for names. One instance were _folders was set to "None" was accidentally left over. Fix this. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/LocalStatus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 73a6a62..b1b763a 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -79,7 +79,7 @@ class LocalStatusRepository(BaseRepository): file.close() os.rename(filename + ".tmp", filename) # Invalidate the cache. - self._folders = None + self._folders = {} def getfolder(self, foldername): """Return the Folder() object for a foldername""" From 3476e9ab36b861fefde5211cb39a71623738dd71 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 1 Sep 2012 03:12:16 +0200 Subject: [PATCH 543/817] Fix fallout when filtering folders Previous commit e7ca5b25cba0679aa800b490d063b24c40864839 combined checks for filtered folders in one place. However, it turns out there was a reason to have them separate. getfolder() on a non-existent Maildir fails and there might not be an equivalent local Maildir folder for a filtered out IMAP folder. Fix this by first checking if the remote folder should be filtered, and only then retrieving the local folder (which should exist then). This bug was found by our test suite! Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index f6aae6a..dd65842 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -312,12 +312,15 @@ class SyncableAccount(Account): # check for CTRL-C or SIGTERM if Account.abort_NOW_signal.is_set(): break - localfolder = self.get_local_folder(remotefolder) - if not (remotefolder.sync_this - and localfolder.sync_this): + if not remotefolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" "[%s]" % (remotefolder, remoterepos)) continue # Ignore filtered folder + localfolder = self.get_local_folder(remotefolder) + if not localfolder.sync_this: + self.ui.debug('', "Not syncing filtered folder '%s'" + "[%s]" % (localfolder, localfolder.repository)) + continue # Ignore filtered folder thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, From ad66a4648e0bcc253e4c28427641358f7f53687d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 25 Aug 2012 20:37:06 +0200 Subject: [PATCH 544/817] add official maintainers list with minimal informations Signed-off-by: Nicolas Sebrecht --- MAINTAINERS | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 MAINTAINERS diff --git a/MAINTAINERS b/MAINTAINERS new file mode 100644 index 0000000..f237f00 --- /dev/null +++ b/MAINTAINERS @@ -0,0 +1,19 @@ + +Official maintainers +==================== + +Dmitrijs Ledkovs + email: xnox at debian.org + github: xnox + +Eygene Ryabinkin + email: rea at freebsd.org + github: konvpalto + +Nicolas Sebrecht + email: nicolas.s-dev at laposte.net + github: nicolas33 + +Sebastian Spaeth + email: sebastian at sspaeth.de + github: spaetz From 721035eacef450247efd8aa2eb4b964e41407f1b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 5 Sep 2012 17:40:04 +0200 Subject: [PATCH 545/817] Release v6.5.5-rc2 (Same as rc1, but with Changelog updated and version number bumped) Signed-off-by: Sebastian Spaeth --- Changelog.rst | 9 ++++++++- offlineimap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 0a57919..643e63e 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -4,10 +4,17 @@ ChangeLog :website: http://offlineimap.org - WIP (add new stuff for the next release) ======================================== +OfflineIMAP v6.5.5-rc1 (2012-09-05) +=================================== + +* Bump version number + +OfflineIMAP v6.5.5-rc1 (2012-09-05) +=================================== + * Don't create folders if readonly is enabled. * Learn to deal with readonly folders to properly detect this condition and act accordingly. One example is Gmail's "Chats" folder that is read-only, diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 77495e4..a7d043c 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.4" +__version__ = "6.5.5-rc2" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From caef9a72fc3220f342f4d2323fcc1041f6589473 Mon Sep 17 00:00:00 2001 From: Tobias Thierer Date: Mon, 1 Oct 2012 20:30:00 +1000 Subject: [PATCH 546/817] Fix IMAP folder throwing away time zone when parsing email Date headers Fix imapfolder.getmessageinternaldate misparsing the Date: header from emails due to a bug or surprising behaviour by email.utils.parsedate. This is because email.utils.parsedate's return value contains the unadjusted hour value from the string parsed but does not include information about the time zone in which it is specified. For example (Python 2.7.3): $ python -c "import email.utils; print email.utils.parsedate('Mon, 20 Nov 1995 19:12:08 -0500')" (1995, 11, 20, 19, 12, 8, 0, 1, -1) (the -1 is the isdst field); the -0500 time zone is completely ignored, so e.g. the same input with time "19:12:08 +0300" has the same result. When passed to time.struct_time as allowed per the parsedate documentation, this time is interpreted in GMT and thus deviates from the correct value by the timezone offset (in this example, -5 hours). I consider this a bug in email.utils.parsedate: In my opinion, since the return value of the parsetime doesn't include a timezone, it should be expressed in terms of UTC rather than in terms of the time zone from the Date header; the existence of email.utils.parsedate_tz, to which I've switched, indicates that maybe the authors were aware of this problem. Signed-off-by: Sebastian Spaeth --- Changelog.rst | 2 ++ offlineimap/folder/IMAP.py | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 643e63e..5ddcccf 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -7,6 +7,8 @@ ChangeLog WIP (add new stuff for the next release) ======================================== +* Honor the timezone of emails (Tobias Thierer) + OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 75520c2..298f9fd 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -432,16 +432,16 @@ class IMAPFolder(BaseFolder): (which is fine as value for append).""" if rtime is None: message = email.message_from_string(content) - # parsedate returns a 9-tuple that can be passed directly to - # time.mktime(); Will be None if missing or not in a valid - # format. Note that indexes 6, 7, and 8 of the result tuple are - # not usable. - datetuple = email.utils.parsedate(message.get('Date')) + dateheader = message.get('Date') + # parsedate_tz returns a 10-tuple that can be passed to mktime_tz; + # Will be None if missing or not in a valid format. Note that + # indexes 6, 7, and 8 of the result tuple are not usable. + datetuple = email.utils.parsedate_tz(dateheader) if datetuple is None: #could not determine the date, use the local time. return None #make it a real struct_time, so we have named attributes - datetuple = time.struct_time(datetuple) + datetuple = time.localtime(email.utils.mktime_tz(datetuple)) else: #rtime is set, use that instead datetuple = time.localtime(rtime) From 83e8fca2e0cce5daaa792bac982a22b74e8bb341 Mon Sep 17 00:00:00 2001 From: Dmitrijs Ledkovs Date: Tue, 5 Jun 2012 03:47:15 +0100 Subject: [PATCH 547/817] Make SIGHUP singal handler equivalent to SIGTERM and SIGINT. offlineimap has several frontends that encourage running it from a terminal under an X session. When X session closes for a system shutdown, the terminals exit, after sending SIGHUP to their children. Previously SIGHUP was treated to be equivalent to SIGUSR1, i.e. wake up and sync all accounts. This causes delays during shutdown. According to Wikipedia [0], SIGHUP has been repurposed from a historical meaning to one of: * re-read configuration files, or reinitialize (e.g. Apache, sendmail) * controlling pseudo or virtual terminal has been closed I believe second meaning is more appropriate for offlineimap, and hence this patch makes SIGHUP to be handled in the same way SIGTERM and SIGINT are handled. [0] http://en.wikipedia.org/wiki/SIGHUP Debian-Bug: http://bugs.debian.org/670120 Reported-By: Steve Langasek Signed-off-by: Dmitrijs Ledkovs Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ docs/MANUAL.rst | 9 ++++++++- offlineimap/init.py | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 5ddcccf..7d8992a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -7,6 +7,9 @@ ChangeLog WIP (add new stuff for the next release) ======================================== +======= +* SIGHUP is now handled as the termination notification rather than + the signal to reread the configuration (Dmitrijs Ledkovs) * Honor the timezone of emails (Tobias Thierer) OfflineIMAP v6.5.5-rc1 (2012-09-05) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 96d5d91..0e080a1 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -308,7 +308,8 @@ SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF YOUR IMAP SERVER! UNIX Signals ============ -OfflineImap listens to the unix signals SIGUSR1 and SIGUSR2. +OfflineImap listens to the unix signals SIGUSR1, SIGUSR2, SIGTERM, +SIGINT, SIGHUP: If sent a SIGUSR1 it will abort any current (or next future) sleep of all accounts that are configured to "autorefresh". In effect, this will trigger a @@ -319,6 +320,12 @@ accounts will abort any current sleep and will exit after a currently running synchronization has finished. This signal can be used to gracefully exit out of a running offlineimap "daemon". +SIGTERM, SIGINT, SIGHUP are all treated to gracefully terminate as +soon as possible. This means it will finish syncing the current folder +in each account, close keep alive connections, remove locks on the +accounts and exit. It may take up to 10 seconds, if autorefresh option +is used. + Folder filtering and nametrans ============================== diff --git a/offlineimap/init.py b/offlineimap/init.py index 24d6c1a..94a9a18 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -329,14 +329,14 @@ class OfflineImap: syncaccounts.append(account) def sig_handler(sig, frame): - if sig == signal.SIGUSR1 or sig == signal.SIGHUP: + if sig == signal.SIGUSR1: # tell each account to stop sleeping accounts.Account.set_abort_event(self.config, 1) elif sig == signal.SIGUSR2: # tell each account to stop looping getglobalui().warn("Terminating after this sync...") accounts.Account.set_abort_event(self.config, 2) - elif sig == signal.SIGTERM or sig == signal.SIGINT: + elif sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): # tell each account to ABORT ASAP (ctrl-c) getglobalui().warn("Terminating NOW (this may "\ "take a few seconds)...") From de84c3941ce730bec2e97527238a49e527c751ee Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 28 Jan 2013 22:49:29 +0400 Subject: [PATCH 548/817] Implement stack trace dump for all running threads on SIGQUIT This is handy when we're debugging the thread locks: we can try to understand which thread does what and how it was called. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 ++- docs/MANUAL.rst | 5 ++++- offlineimap/init.py | 5 +++++ offlineimap/utils/stacktrace.py | 25 +++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 offlineimap/utils/stacktrace.py diff --git a/Changelog.rst b/Changelog.rst index 7d8992a..0064681 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -7,7 +7,8 @@ ChangeLog WIP (add new stuff for the next release) ======================================== -======= +* Dump stacktrace for all threads on SIGQUIT: ease debugging + of threading and other issues * SIGHUP is now handled as the termination notification rather than the signal to reread the configuration (Dmitrijs Ledkovs) * Honor the timezone of emails (Tobias Thierer) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 0e080a1..15f0625 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -309,7 +309,7 @@ UNIX Signals ============ OfflineImap listens to the unix signals SIGUSR1, SIGUSR2, SIGTERM, -SIGINT, SIGHUP: +SIGINT, SIGHUP, SIGQUIT: If sent a SIGUSR1 it will abort any current (or next future) sleep of all accounts that are configured to "autorefresh". In effect, this will trigger a @@ -326,6 +326,9 @@ in each account, close keep alive connections, remove locks on the accounts and exit. It may take up to 10 seconds, if autorefresh option is used. +SIGQUIT dumps stack traces for all threads and tries to dump process +core. + Folder filtering and nametrans ============================== diff --git a/offlineimap/init.py b/offlineimap/init.py index 94a9a18..e93a382 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -28,6 +28,7 @@ from offlineimap import accounts, threadutil, syncmaster from offlineimap.error import OfflineImapError from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser +from offlineimap.utils import stacktrace class OfflineImap: @@ -341,12 +342,16 @@ class OfflineImap: getglobalui().warn("Terminating NOW (this may "\ "take a few seconds)...") accounts.Account.set_abort_event(self.config, 3) + elif sig == signal.SIGQUIT: + stacktrace.dump (sys.stderr) + os.abort() signal.signal(signal.SIGHUP,sig_handler) signal.signal(signal.SIGUSR1,sig_handler) signal.signal(signal.SIGUSR2,sig_handler) signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) + signal.signal(signal.SIGQUIT, sig_handler) #various initializations that need to be performed: offlineimap.mbnames.init(self.config, syncaccounts) diff --git a/offlineimap/utils/stacktrace.py b/offlineimap/utils/stacktrace.py new file mode 100644 index 0000000..7c885b0 --- /dev/null +++ b/offlineimap/utils/stacktrace.py @@ -0,0 +1,25 @@ +# Copyright 2013 Eygene A. Ryabinkin +# Functions to perform stack tracing (for multithreaded programs +# as well as for single-threaded ones). + +import sys +import threading +import traceback + + +def dump(out): + """ Dumps current stack trace into I/O object 'out' """ + id2name = {} + for th in threading.enumerate(): + id2name[th.ident] = th.name + n = 0 + for i, stack in sys._current_frames().items(): + out.write ("\n# Thread #%d (id=%d), %s\n" % \ + (n, i, id2name[i])) + n = n + 1 + for f, lno, name, line in traceback.extract_stack (stack): + out.write ('File: "%s", line %d, in %s' % \ + (f, lno, name)) + if line: + out.write (" %s" % (line.strip())) + out.write ("\n") From 4dcd676b234ec6c1bb470e4244cfd7010de51185 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 5 Feb 2013 07:38:11 +0400 Subject: [PATCH 549/817] Add offlineimap.utils to the list of our modules Forgot to do it while was adding stacktrace support. Signed-off-by: Eygene Ryabinkin --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 27932eb..73db977 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,8 @@ setup(name = "offlineimap", author_email = offlineimap.__author_email__, url = offlineimap.__homepage__, packages = ['offlineimap', 'offlineimap.folder', - 'offlineimap.repository', 'offlineimap.ui'], + 'offlineimap.repository', 'offlineimap.ui', + 'offlineimap.utils'], scripts = ['bin/offlineimap'], license = offlineimap.__copyright__ + \ ", Licensed under the GPL version 2", From 1c8917b03611ae0f1403726ee9e1983bb53e9dfc Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 5 Feb 2013 07:52:25 +0400 Subject: [PATCH 550/817] And add forgotten __init__.py for offlineimap.utils I should learn to check my commits when rebasing and reordering. Signed-off-by: Eygene Ryabinkin --- offlineimap/utils/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 offlineimap/utils/__init__.py diff --git a/offlineimap/utils/__init__.py b/offlineimap/utils/__init__.py new file mode 100644 index 0000000..e69de29 From 5c842c01bd0df5db1474e24c70ee199a7689fea6 Mon Sep 17 00:00:00 2001 From: X-Ryl669 Date: Thu, 17 Jan 2013 12:02:41 +0100 Subject: [PATCH 551/817] Fix deadlock for IMAP folder synced in singlethreaded mode The problem lies in the fact that offlineimap.folder.Base's method syncmessagesto_copy() uses threaded code everytime it is suggested by the derived class's suggeststhreads() (currently, only IMAP does this suggestion), but offlineimap/init.py will not spawn the exitnotifymonitorloop() from offlineimap.threadutil. The root cause is that ExitNotifyThread-derived threads need offlineimap.threadutil's exitnotifymonitorloop() to be running the cleaner for the exitthreads Queue(), because it fills the queue via the run() method from this class: it wants to put() itself to the Queue on exit, so when no exitnotifymonitorloop() is running, the queue will fill up. And if this thread is an instance of InstanceLimitedThread that hits the limit on the number of threads, then it will hold the instancelimitedsems[] semaphore will prevent other InstanceLimitedThread()s of the same name to pass its start() method. The fix is to avoid using threaded code if we're running single-threaded. Signed-off-by: Eygene Ryabinkin Obtained-from: X-Ryl669 --- Changelog.rst | 2 ++ offlineimap/accounts.py | 20 +++++++++++--------- offlineimap/folder/Base.py | 2 +- offlineimap/init.py | 3 +++ offlineimap/threadutil.py | 2 ++ 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 0064681..5e9be44 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -7,6 +7,8 @@ ChangeLog WIP (add new stuff for the next release) ======================================== +* Avoid lockups for IMAP synchronizations running with the + "-1" command-line switch (X-Ryl669 ) * Dump stacktrace for all threads on SIGQUIT: ease debugging of threading and other issues * SIGHUP is now handled as the termination notification rather than diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index dd65842..88d62d8 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -321,13 +321,16 @@ class SyncableAccount(Account): self.ui.debug('', "Not syncing filtered folder '%s'" "[%s]" % (localfolder, localfolder.repository)) continue # Ignore filtered folder - thread = InstanceLimitedThread(\ - instancename = 'FOLDER_' + self.remoterepos.getname(), - target = syncfolder, - name = "Folder %s [acc: %s]" % (remotefolder, self), - args = (self, remotefolder, quick)) - thread.start() - folderthreads.append(thread) + if self.config.get('general', 'single-thread') == 'False': + thread = InstanceLimitedThread(\ + instancename = 'FOLDER_' + self.remoterepos.getname(), + target = syncfolder, + name = "Folder %s [acc: %s]" % (remotefolder, self), + args = (self, remotefolder, quick)) + thread.start() + folderthreads.append(thread) + else: + syncfolder(self, remotefolder, quick) # wait for all threads to finish for thr in folderthreads: thr.join() @@ -372,8 +375,7 @@ class SyncableAccount(Account): self.ui.error(e, exc_info()[2], msg = "Calling hook") def syncfolder(account, remotefolder, quick): - """This function is called as target for the - InstanceLimitedThread invokation in SyncableAccount. + """Synchronizes given remote folder for the specified account. Filtered folders on the remote side will not invoke this function.""" remoterepos = account.remoterepos diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index c132713..e330086 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -403,7 +403,7 @@ class BaseFolder(object): break self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # exceptions are caught in copymessageto() - if self.suggeststhreads(): + if self.suggeststhreads() and self.config.get('general', 'single-thread') == 'False': self.waitforthread() thread = threadutil.InstanceLimitedThread(\ self.getcopyinstancelimit(), diff --git a/offlineimap/init.py b/offlineimap/init.py index e93a382..d25d7fa 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -251,6 +251,9 @@ class OfflineImap: if type.lower() == 'imap': imaplib.Debug = 5 + # XXX: can we avoid introducing fake configuration item? + config.set_if_not_exists('general', 'single-thread', 'True' if options.singlethreading else 'False') + if options.runonce: # FIXME: maybe need a better for section in accounts.getaccountlist(config): diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index a29e68f..33dbd64 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -39,6 +39,8 @@ def semaphorereset(semaphore, originalstate): semaphore.release() class threadlist: + """Store the list of all threads in the software so it can be used to find out + what's running and what's not.""" def __init__(self): self.lock = Lock() self.list = [] From f4140cbbed13679491d10b7731e638f0701e318c Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 5 Feb 2013 07:49:56 +0400 Subject: [PATCH 552/817] Create global instance of command-line options This eases testing of option values inside the code. This instance is implemented as the read-only copy of the obtained 'options' object, so callers won't be able to modify its contents. Signed-off-by: Eygene Ryabinkin --- docs/doc-src/API.rst | 27 +++++++++++++++++++ offlineimap/accounts.py | 3 ++- offlineimap/folder/Base.py | 3 ++- offlineimap/globals.py | 12 +++++++++ offlineimap/init.py | 5 ++-- offlineimap/utils/const.py | 40 +++++++++++++++++++++++++++ test/tests/test_00_globals.py | 51 +++++++++++++++++++++++++++++++++++ 7 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 offlineimap/globals.py create mode 100644 offlineimap/utils/const.py create mode 100755 test/tests/test_00_globals.py diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index 38df996..d3c80bf 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -60,3 +60,30 @@ An :class:`accounts.Account` connects two email repositories that are to be sync This execption inherits directly from :exc:`Exception` and is raised on errors during the offlineimap execution. It has an attribute `severity` that denotes the severity level of the error. + + +:mod:`offlineimap.globals` -- module with global variables +========================================================== + +.. module:: offlineimap.globals + +Module :mod:`offlineimap.globals` provides the read-only storage +for the global variables. + +All exported module attributes can be set manually, but this practice +is highly discouraged and shouldn't be used. +However, attributes of all stored variables can only be read, write +access to them is denied. + +Currently, we have only :attr:`options` attribute that holds +command-line options as returned by OptionParser. +The value of :attr:`options` must be set by :func:`set_options` +prior to its first use. + +.. automodule:: offlineimap.globals + :members: + + .. data:: options + + You can access the values of stored options using the usual + syntax, offlineimap.globals.options. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 88d62d8..c78d7d8 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -15,6 +15,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap import mbnames, CustomConfig, OfflineImapError +from offlineimap import globals from offlineimap.repository import Repository from offlineimap.ui import getglobalui from offlineimap.threadutil import InstanceLimitedThread @@ -321,7 +322,7 @@ class SyncableAccount(Account): self.ui.debug('', "Not syncing filtered folder '%s'" "[%s]" % (localfolder, localfolder.repository)) continue # Ignore filtered folder - if self.config.get('general', 'single-thread') == 'False': + if not globals.options.singlethreading: thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index e330086..bf9a3a1 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap import threadutil +from offlineimap import globals from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError import offlineimap.accounts @@ -403,7 +404,7 @@ class BaseFolder(object): break self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # exceptions are caught in copymessageto() - if self.suggeststhreads() and self.config.get('general', 'single-thread') == 'False': + if self.suggeststhreads() and not globals.options.singlethreading: self.waitforthread() thread = threadutil.InstanceLimitedThread(\ self.getcopyinstancelimit(), diff --git a/offlineimap/globals.py b/offlineimap/globals.py new file mode 100644 index 0000000..b4253f9 --- /dev/null +++ b/offlineimap/globals.py @@ -0,0 +1,12 @@ +# Copyright 2013 Eygene A. Ryabinkin. +# +# Module that holds various global objects. + +from offlineimap.utils import const + +# Holds command-line options for OfflineIMAP. +options = const.ConstProxy() + +def set_options (source): + """ Sets the source for options variable """ + options.set_source (source) diff --git a/offlineimap/init.py b/offlineimap/init.py index d25d7fa..d52fd3a 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -25,6 +25,7 @@ import logging from optparse import OptionParser import offlineimap from offlineimap import accounts, threadutil, syncmaster +from offlineimap import globals from offlineimap.error import OfflineImapError from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser @@ -161,6 +162,7 @@ class OfflineImap: ", ".join(UI_LIST.keys())) (options, args) = parser.parse_args() + globals.set_options (options) #read in configuration file configfilename = os.path.expanduser(options.configfile) @@ -251,9 +253,6 @@ class OfflineImap: if type.lower() == 'imap': imaplib.Debug = 5 - # XXX: can we avoid introducing fake configuration item? - config.set_if_not_exists('general', 'single-thread', 'True' if options.singlethreading else 'False') - if options.runonce: # FIXME: maybe need a better for section in accounts.getaccountlist(config): diff --git a/offlineimap/utils/const.py b/offlineimap/utils/const.py new file mode 100644 index 0000000..a62b6a6 --- /dev/null +++ b/offlineimap/utils/const.py @@ -0,0 +1,40 @@ +# Copyright 2013 Eygene A. Ryabinkin. +# +# Collection of classes that implement const-like behaviour +# for various objects. + +import copy + +class ConstProxy (object): + """ + Implements read-only access to a given object + that can be attached to each instance only once. + + """ + + def __init__ (self): + self.__dict__['__source'] = None + + + def __getattr__ (self, name): + src = self.__dict__['__source'] + if src == None: + raise ValueError ("using non-initialized ConstProxy() object") + return copy.deepcopy (getattr (src, name)) + + + def __setattr__ (self, name, value): + raise AttributeError ("tried to set '%s' to '%s' for constant object" % \ + (name, value)) + + + def __delattr__ (self, name): + raise RuntimeError ("tried to delete field '%s' from constant object" % \ + (name)) + + + def set_source (self, source): + """ Sets source object for this instance. """ + if (self.__dict__['__source'] != None): + raise ValueError ("source object is already set") + self.__dict__['__source'] = source diff --git a/test/tests/test_00_globals.py b/test/tests/test_00_globals.py new file mode 100755 index 0000000..b4572f9 --- /dev/null +++ b/test/tests/test_00_globals.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# Copyright 2013 Eygene A. Ryabinkin + +from offlineimap import globals +import unittest + +class Opt: + def __init__(self): + self.one = "baz" + self.two = 42 + self.three = True + + +class TestOfflineimapGlobals(unittest.TestCase): + + @classmethod + def setUpClass(klass): + klass.o = Opt() + globals.set_options (klass.o) + + def test_initial_state(self): + for k in self.o.__dict__.keys(): + self.assertTrue(getattr(self.o, k) == + getattr(globals.options, k)) + + def test_object_changes(self): + self.o.one = "one" + self.o.two = 119 + self.o.three = False + return self.test_initial_state() + + def test_modification(self): + with self.assertRaises(AttributeError): + globals.options.two = True + + def test_deletion(self): + with self.assertRaises(RuntimeError): + del globals.options.three + + def test_nonexistent_key(self): + with self.assertRaises(AttributeError): + a = globals.options.nosuchoption + + def test_double_init(self): + with self.assertRaises(ValueError): + globals.set_options (True) + + +if __name__ == "__main__": + suite = unittest.TestLoader().loadTestsFromTestCase(TestOfflineimapGlobals) + unittest.TextTestRunner(verbosity=2).run(suite) From 611f6e89c07b606a2fa327fdf3d1a087bc156b22 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 28 Jan 2013 23:08:20 +0400 Subject: [PATCH 553/817] IMAP class: don't suggest multithreading in single-threaded mode Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/IMAP.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 298f9fd..ec753c7 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -23,6 +23,7 @@ import time from sys import exc_info from .Base import BaseFolder from offlineimap import imaputil, imaplibutil, OfflineImapError +from offlineimap import globals from offlineimap.imaplib2 import MonthNames @@ -53,7 +54,7 @@ class IMAPFolder(BaseFolder): imapobj.select(self.getfullname(), readonly = True, force = force) def suggeststhreads(self): - return 1 + return not globals.options.singlethreading def waitforthread(self): self.imapserver.connectionwait() From aa5202396c3b5fb4228654124a059ce445bea146 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Wed, 27 Mar 2013 12:42:32 +0000 Subject: [PATCH 554/817] fix typos --- offlineimap/folder/Base.py | 2 +- offlineimap/folder/IMAP.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index c132713..6d0e8ff 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -366,7 +366,7 @@ class BaseFolder(object): raise except OfflineImapError as e: if e.severity > OfflineImapError.ERROR.MESSAGE: - raise # buble severe errors up + raise # bubble severe errors up self.ui.error(e, exc_info()[2]) except Exception as e: self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 75520c2..69bac9a 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -555,7 +555,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() if not retry_left: raise OfflineImapError("Saving msg in folder '%s', " - "repository '%s' failed (abort). Server reponded: %s\n" + "repository '%s' failed (abort). Server responded: %s\n" "Message content was: %s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) @@ -567,7 +567,7 @@ class IMAPFolder(BaseFolder): self.imapserver.releaseconnection(imapobj, True) imapobj = None raise OfflineImapError("Saving msg folder '%s', repo '%s'" - "failed (error). Server reponded: %s\nMessage content was: " + "failed (error). Server responded: %s\nMessage content was: " "%s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for From 84857f9430d9b755a76464d89f6f4ee8730d9370 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Fri, 19 Apr 2013 19:28:17 -0300 Subject: [PATCH 555/817] README.md: Renaming file so that github renders it nicely. --- README => README.md | 148 ++++++++++++++++++++++---------------------- 1 file changed, 74 insertions(+), 74 deletions(-) rename README => README.md (70%) diff --git a/README b/README.md similarity index 70% rename from README rename to README.md index 8f80d7a..edafae7 100644 --- a/README +++ b/README.md @@ -1,5 +1,5 @@ -OfflineImap README -================== +OfflineImap +=========== Description ----------- @@ -24,9 +24,9 @@ it. In fact, you are encouraged to contribute to OfflineIMAP. Documentation ------------- -The documentation is included (in .rst format) in the `docs` directory. -Read it directly or generate nice html docs (python-sphinx needed) and/or -the man page (python-docutils needed) while being in the `docs` dir via:: +The documentation is included (in .rst format) in the `docs` directory. +Read it directly or generate nice html docs (python-sphinx needed) and/or +the man page (python-docutils needed) while being in the `docs` dir via: 'make doc' (user docs), 'make man' (man page only) or 'make' (both) @@ -40,7 +40,7 @@ Quick Start =========== First, install OfflineIMAP. See docs/INSTALL.rst or read -http://docs.offlineimap.org/en/latest/INSTALL.html. +http://docs.offlineimap.org/en/latest/INSTALL.html. (hint: `sudo python setup.py install`) Second, set up your configuration file and run it! The distribution @@ -50,23 +50,23 @@ provides you with the bare minimum of setting up OfflineIMAP. You can simply copy this file into your home directory and name it ``.offlineimaprc``. A command such as ``cp offlineimap.conf.minimal ~/.offlineimaprc`` will do it. Or, if you prefer, you can just copy -this text to ``~/.offlineimaprc``:: +this text to ``~/.offlineimaprc``: - [general] - accounts = Test + [general] + accounts = Test - [Account Test] - localrepository = Local - remoterepository = Remote + [Account Test] + localrepository = Local + remoterepository = Remote - [Repository Local] - type = Maildir - localfolders = ~/Test + [Repository Local] + type = Maildir + localfolders = ~/Test - [Repository Remote] - type = IMAP - remotehost = examplehost - remoteuser = jgoerzen + [Repository Remote] + type = IMAP + remotehost = examplehost + remoteuser = jgoerzen Now, edit the ``~/.offlineimaprc`` file with your favorite editor. All you have @@ -79,9 +79,9 @@ up, ask you for a login password if necessary, synchronize your folders, and exit. See? You can just throw away the rest of the finely-crafted, perfectly-honed user -manual! Of course, if you want to see how you can make OfflineIMAP +manual! Of course, if you want to see how you can make OfflineIMAP FIVE TIMES FASTER FOR JUST $19.95 (err, well, $0), you have to read on our -full user documentation and peruse the sample offlineimap.conf (which +full user documentation and peruse the sample offlineimap.conf (which includes all available options) for further tweaks! @@ -111,28 +111,28 @@ This example shows you how to set up OfflineIMAP to synchronize multiple accounts with the mutt mail reader. Start by creating a directory to hold your folders by running ``mkdir ~/Mail``. -Then, in your ``~/.offlineimaprc``, specify:: +Then, in your ``~/.offlineimaprc``, specify: - accounts = Personal, Work + accounts = Personal, Work Make sure that you have both an [Account Personal] and an [Account Work] section. The local repository for each account must have different localfolder path names. Also, make sure to enable [mbnames]. -In each local repository section, write something like this:: +In each local repository section, write something like this: - localfolders = ~/Mail/Personal + localfolders = ~/Mail/Personal -Finally, add these lines to your ``~/.muttrc``:: +Finally, add these lines to your ``~/.muttrc``: - source ~/path-to-mbnames-muttrc-mailboxes - folder-hook Personal set from="youremail@personal.com" - folder-hook Work set from="youremail@work.com" - set mbox_type=Maildir - set folder=$HOME/Mail - spoolfile=+Personal/INBOX + source ~/path-to-mbnames-muttrc-mailboxes + folder-hook Personal set from="youremail@personal.com" + folder-hook Work set from="youremail@work.com" + set mbox_type=Maildir + set folder=$HOME/Mail + spoolfile=+Personal/INBOX That's it! @@ -146,34 +146,34 @@ to get at their mailboxes, specifying a reference of ``~/Mail`` or ``#mh/`` depending on the configuration. The below configuration from (originally from docwhat@gerf.org) shows using a reference of Mail, a nametrans that strips the leading Mail/ off incoming folder names, and a folderfilter that limits the -folders synced to just three:: +folders synced to just three: - [Account Gerf] - localrepository = GerfLocal - remoterepository = GerfRemote + [Account Gerf] + localrepository = GerfLocal + remoterepository = GerfRemote - [Repository GerfLocal] - type = Maildir - localfolders = ~/Mail + [Repository GerfLocal] + type = Maildir + localfolders = ~/Mail - [Repository GerfRemote] - type = IMAP - remotehost = gerf.org - ssl = yes - remoteuser = docwhat - reference = Mail - # Trims off the preceeding Mail on all the folder names. - nametrans = lambda foldername: \ + [Repository GerfRemote] + type = IMAP + remotehost = gerf.org + ssl = yes + remoteuser = docwhat + reference = Mail + # Trims off the preceeding Mail on all the folder names. + nametrans = lambda foldername: \ re.sub('^Mail/', '', foldername) - # Yeah, you have to mention the Mail dir, even though it - # would seem intuitive that reference would trim it. - folderfilter = lambda foldername: foldername in [ + # Yeah, you have to mention the Mail dir, even though it + # would seem intuitive that reference would trim it. + folderfilter = lambda foldername: foldername in [ 'Mail/INBOX', 'Mail/list/zaurus-general', 'Mail/list/zaurus-dev', - ] - maxconnections = 1 - holdconnectionopen = no + ] + maxconnections = 1 + holdconnectionopen = no pythonfile Configuration File Option @@ -184,34 +184,34 @@ configuration file options that are Python expressions. This example is based on one supplied by Tommi Virtanen for this feature. -In ~/.offlineimaprc, he adds these options:: +In ~/.offlineimaprc, he adds these options: - [general] - pythonfile=~/.offlineimap.py - [Repository foo] - foldersort=mycmp + [general] + pythonfile=~/.offlineimap.py + [Repository foo] + foldersort=mycmp -Then, the ~/.offlineimap.py file will contain:: +Then, the ~/.offlineimap.py file will contain: prioritized = ['INBOX', 'personal', 'announce', 'list'] - def mycmp(x, y): - for prefix in prioritized: - xsw = x.startswith(prefix) - ysw = y.startswith(prefix) - if xsw and ysw: + def mycmp(x, y): + for prefix in prioritized: + xsw = x.startswith(prefix) + ysw = y.startswith(prefix) + if xsw and ysw: + return cmp(x, y) + elif xsw: + return -1 + elif ysw: + return +1 return cmp(x, y) - elif xsw: - return -1 - elif ysw: - return +1 - return cmp(x, y) - def test_mycmp(): - import os, os.path - folders=os.listdir(os.path.expanduser('~/data/mail/tv@hq.yok.utu.fi')) - folders.sort(mycmp) - print folders + def test_mycmp(): + import os, os.path + folders=os.listdir(os.path.expanduser('~/data/mail/tv@hq.yok.utu.fi')) + folders.sort(mycmp) + print folders This code snippet illustrates how the foldersort option can be customized with a From b212e60c490d92bc180f04caea2390d27a9f0a2a Mon Sep 17 00:00:00 2001 From: Chris Pick Date: Sun, 19 May 2013 22:07:20 -0400 Subject: [PATCH 556/817] Fix small offlineimap.conf typo Maxage comment 'therefor' -> 'therefore'. --- offlineimap.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap.conf b/offlineimap.conf index fccceab..cf5a69d 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -234,7 +234,7 @@ remoterepository = RemoteExample # Messages older than maxage days will not be synced, their flags will # not be changed, they will not be deleted etc. For offlineimap it will # be like these messages do not exist. This will perform an IMAP search -# in the case of IMAP or Gmail and therefor requires that the server +# in the case of IMAP or Gmail and therefore requires that the server # support server side searching. This will calculate the earliest day # that would be included in the search and include all messages from # that day until today. e.g. maxage = 3 to sync only the last 3 days From dde021ce52990497a08d066d92e56ea019b56cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20Lidstro=CC=88m?= Date: Thu, 20 Jun 2013 11:38:45 +0200 Subject: [PATCH 557/817] Added remoteporteval --- offlineimap/repository/IMAP.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index be8c858..01e78b5 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -148,6 +148,13 @@ class IMAPRepository(BaseRepository): def getport(self): + port = None + + if self.config.has_option(self.getsection(), 'remoteporteval'): + port = self.getconf('remoteporteval') + if port != None: + return self.localeval.eval(port) + return self.getconfint('remoteport', None) def getssl(self): From 2bacdb7fa3cc203cef69da0dee247c21b06e69da Mon Sep 17 00:00:00 2001 From: Ryan Kavanagh Date: Sun, 7 Jul 2013 17:18:59 -0400 Subject: [PATCH 558/817] Allow setting IMAP servers' SSL version We now allow setting the SSL version used when connecting to IMAPS servers, and do so via the `ssl_version` configuration option. We default to the current practice (letting python's "ssl" library automatically detect the correct version). There are however rare cases where one must specify the version to use. Signed-off-by: Ryan Kavanagh --- offlineimap.conf | 7 +++++++ offlineimap/imaplib2.py | 29 +++++++++++++++++++++++------ offlineimap/imapserver.py | 2 ++ offlineimap/repository/IMAP.py | 3 +++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index fccceab..fabb52b 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -325,6 +325,13 @@ ssl = yes #cert_fingerprint = +# SSL version (optional) +# It is best to leave this unset, in which case the correct version will be +# automatically detected. In rare cases, it may be necessary to specify a +# particular version from: tls1, ssl2, ssl3, ssl23 (SSLv2 or SSLv3) + +# sslversion = ssl23 + # Specify the port. If not specified, use a default port. # remoteport = 993 diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 8138d6c..b7e0d22 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -39,7 +39,8 @@ Timeout handling further improved by Ethan Glasser-Camp Dece Time2Internaldate() patch to match RFC2060 specification of English month names from bugs.python.org/issue11024 March 2011. starttls() bug fixed with the help of Sebastian Spaeth April 2011. Threads now set the "daemon" flag (suggested by offlineimap-project) April 2011. -Single quoting introduced with the help of Vladimir Marek August 2011.""" +Single quoting introduced with the help of Vladimir Marek August 2011. +Support for specifying SSL version by Ryan Kavanagh July 2013.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -460,7 +461,20 @@ class IMAP4(object): cert_reqs = ssl.CERT_REQUIRED else: cert_reqs = ssl.CERT_NONE - self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs) + + if self.ssl_version == "tls1": + ssl_version = ssl.PROTOCOL_TLSv1 + elif self.ssl_version == "ssl2": + ssl_version = ssl.PROTOCOL_SSLv2 + elif self.ssl_version == "ssl3": + ssl_version = ssl.PROTOCOL_SSLv3 + elif self.ssl_version == "ssl23" or self.ssl_version is None: + ssl_version = ssl.PROTOCOL_SSLv23 + else: + raise socket.sslerror("Invalid SSL version requested: %s", + self.ssl_version) + + self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version) ssl_exc = ssl.SSLError self.read_fd = self.sock.fileno() except ImportError: @@ -1040,8 +1054,8 @@ class IMAP4(object): return self._simple_command(name, sort_criteria, charset, *search_criteria, **kw) - def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, **kw): - """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None) + def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", **kw): + """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23") Start TLS negotiation as per RFC 2595.""" name = 'STARTTLS' @@ -1076,6 +1090,7 @@ class IMAP4(object): self.certfile = certfile self.ca_certs = ca_certs self.cert_verify_cb = cert_verify_cb + self.ssl_version = ssl_version try: self.ssl_wrap_socket() @@ -1972,7 +1987,7 @@ class IMAP4_SSL(IMAP4): """IMAP4 client class over SSL connection Instantiate with: - IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, debug=None, debug_file=None, identifier=None, timeout=None) + IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None) host - host's name (default: localhost); port - port number (default: standard IMAP4 SSL port); @@ -1980,6 +1995,7 @@ class IMAP4_SSL(IMAP4): certfile - PEM formatted certificate chain file (default: None); ca_certs - PEM formatted certificate chain file used to validate server certificates (default: None); cert_verify_cb - function to verify authenticity of server certificates (default: None); + ssl_version - SSL version to use (default: "ssl23", choose from: "tls1","ssl2","ssl3","ssl23"); debug - debug level (default: 0 - no debug); debug_file - debug stream (default: sys.stderr); identifier - thread identifier prefix (default: host); @@ -1990,11 +2006,12 @@ class IMAP4_SSL(IMAP4): """ - def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): + def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): self.keyfile = keyfile self.certfile = certfile self.ca_certs = ca_certs self.cert_verify_cb = cert_verify_cb + self.ssl_version = ssl_version IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout, debug_buf_lvl) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 22c5c16..c8f70d6 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -65,6 +65,7 @@ class IMAPServer: self.sslclientcert = repos.getsslclientcert() self.sslclientkey = repos.getsslclientkey() self.sslcacertfile = repos.getsslcacertfile() + self.sslversion = repos.getsslversion() if self.sslcacertfile is None: self.verifycert = None # disable cert verification self.delim = None @@ -211,6 +212,7 @@ class IMAPServer: self.sslclientcert, self.sslcacertfile, self.verifycert, + self.sslversion, timeout=socket.getdefaulttimeout(), fingerprint=fingerprint ) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index be8c858..7ccabfa 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -172,6 +172,9 @@ class IMAPRepository(BaseRepository): % (self.name, cacertfile)) return cacertfile + def getsslversion(self): + return self.getconf('ssl_version', None) + def get_ssl_fingerprint(self): return self.getconf('cert_fingerprint', None) From cbceffcb6a24f543c3e83ade171ccaa730b52748 Mon Sep 17 00:00:00 2001 From: Dmitrijs Ledkovs Date: Wed, 10 Jul 2013 01:59:25 +0100 Subject: [PATCH 559/817] Rename README to README.md for github --- README => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.md (100%) diff --git a/README b/README.md similarity index 100% rename from README rename to README.md From 7a21cab6c56b1e5b7090883e2b8772391ba56abb Mon Sep 17 00:00:00 2001 From: Dmitrijs Ledkovs Date: Wed, 10 Jul 2013 02:17:01 +0100 Subject: [PATCH 560/817] Correct whitespace & add docs about remoteporteval. --- offlineimap.conf | 1 + offlineimap/repository/IMAP.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index fccceab..7b1755e 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -292,6 +292,7 @@ type = IMAP # "getcredentials" that parses a file "filename" and returns the account # details for "hostname". # remotehosteval = getcredentials("filename", "hostname", "hostname") +# remoteporteval = getcredentials("filename", "hostname", "port") # remoteusereval = getcredentials("filename", "hostname", "user") # remotepasseval = getcredentials("filename", "hostname", "passwd") diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 01e78b5..f18090f 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -148,12 +148,12 @@ class IMAPRepository(BaseRepository): def getport(self): - port = None + port = None - if self.config.has_option(self.getsection(), 'remoteporteval'): - port = self.getconf('remoteporteval') - if port != None: - return self.localeval.eval(port) + if self.config.has_option(self.getsection(), 'remoteporteval'): + port = self.getconf('remoteporteval') + if port != None: + return self.localeval.eval(port) return self.getconfint('remoteport', None) From 41cb0f577f6921a644d0c4c1ac23dd391270fee7 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 21 Jul 2013 23:00:23 +0400 Subject: [PATCH 561/817] Prune trailing whitespaces from code and documentation They are redundant in all pruned cases and sometimes even create some problems, e.g., when one tries to jump through paragraphs in vi. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 10 +++++----- docs/MANUAL.rst | 20 ++++++++++---------- docs/doc-src/API.rst | 4 ++-- docs/doc-src/FAQ.rst | 14 +++++++------- docs/doc-src/HACKING.rst | 2 +- offlineimap.conf | 2 +- offlineimap/CustomConfig.py | 4 ++-- offlineimap/__init__.py | 2 +- offlineimap/folder/Base.py | 6 +++--- offlineimap/folder/IMAP.py | 10 +++++----- offlineimap/folder/LocalStatusSQLite.py | 4 ++-- offlineimap/folder/Maildir.py | 10 +++++----- offlineimap/folder/UIDMaps.py | 2 +- offlineimap/imaplib2.py | 4 ++-- offlineimap/imapserver.py | 6 +++--- offlineimap/init.py | 2 +- offlineimap/mbnames.py | 2 -- offlineimap/repository/Base.py | 2 +- offlineimap/repository/Gmail.py | 2 +- offlineimap/repository/IMAP.py | 2 +- offlineimap/repository/LocalStatus.py | 2 +- offlineimap/repository/Maildir.py | 2 +- offlineimap/ui/Machine.py | 2 +- offlineimap/ui/__init__.py | 2 +- offlineimap/ui/debuglock.py | 4 ++-- 25 files changed, 60 insertions(+), 62 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 5e9be44..2b8aef8 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -122,7 +122,7 @@ OfflineIMAP v6.5.2 (2012-01-17) * Some sanity checks and improved error messages. -* Revert 6.5.1.1 change to use public imaplib2 function, it was reported to +* Revert 6.5.1.1 change to use public imaplib2 function, it was reported to not always work. * Don't fail when ~/netrc is not readable by us. @@ -280,7 +280,7 @@ Changes * Refresh server capabilities after login, so we know that Gmail supports UIDPLUS (it only announces that after login, not before). This prevents us from adding custom headers to Gmail uploads. - + Bug Fixes --------- @@ -338,7 +338,7 @@ New Features * When a message upload/download fails, we do not abort the whole folder synchronization, but only skip that message, informing the user at the end of the sync run. - + * If you connect via ssl and 'cert_fingerprint' is configured, we check that the server certificate is actually known and identical by comparing the stored sha1 fingerprint with the current one. @@ -437,7 +437,7 @@ Notes ----- This was a very active rc1 and we could expect a lot of new fixes for the next -release. +release. The most important fix is about a bug that could lead to data loss. Find more information about his bug here: @@ -588,7 +588,7 @@ I'd like to thank reporters who involved in this cycle: - Pan Tsu - Vincent Beffara - Will Styler - + (my apologies if I forget somebody) ...and all active developers, of course! The imaplib2 migration looks to go the right way to be definetly released but diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 15f0625..936abc3 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -40,7 +40,7 @@ Most configuration is done via the configuration file. However, any setting can OfflineImap is well suited to be frequently invoked by cron jobs, or can run in daemon mode to periodically check your email (however, it will exit in some error situations). -The documentation is included in the git repository and can be created by +The documentation is included in the git repository and can be created by issueing `make dev-doc` in the `doc` folder (python-sphinx required), or it can be viewed online at http://docs.offlineimap.org. @@ -420,7 +420,7 @@ This is an example of a setup where "TheOtherImap" requires all folders to be un # The below will put all GMAIL folders as sub-folders of the 'local' INBOX, # assuming that your path separator on 'local' is a dot. nametrans = lambda x: 'INBOX.' + x - + [Repository TheOtherImap] #This is the 'local' repository type = IMAP @@ -437,7 +437,7 @@ Add this to the remote gmail repository section to only sync mails which are in To only get the All Mail folder from a Gmail account, you would e.g. do:: - folderfilter = lambda folder: folder.startswith('[Gmail]/All Mail') + folderfilter = lambda folder: folder.startswith('[Gmail]/All Mail') Another nametrans transpose example @@ -464,25 +464,25 @@ offlineimap.conf:: ui = ttyui pythonfile=~/bin/offlineimap-helpers.py socktimeout = 90 - + [Account acc1] localrepository = acc1local remoterepository = acc1remote autorefresh = 2 - + [Account acc2] localrepository = acc2local remoterepository = acc2remote autorefresh = 4 - + [Repository acc1local] type = Maildir localfolders = ~/Mail/acc1 - + [Repository acc2local] type = Maildir localfolders = ~/Mail/acc2 - + [Repository acc1remote] type = IMAP remotehost = imap.acc1.com @@ -494,7 +494,7 @@ offlineimap.conf:: # Folders to get: folderfilter = lambda foldername: foldername in [ 'INBOX', 'Drafts', 'Sent', 'archiv'] - + [Repository acc2remote] type = IMAP remotehost = imap.acc2.net @@ -532,7 +532,7 @@ Offlineimap handles the renaming correctly in both directions:: retval = "acc1." + foldername retval = re.sub("/", ".", retval) return retval - + def oimaptransfolder_acc2(foldername): if(foldername == "INBOX"): retval = "acc2" diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index d3c80bf..c456435 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -9,7 +9,7 @@ Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. Email repositories are represented by a :class:`offlineimap.repository.Base.BaseRepository` or derivatives (see :mod:`offlineimap.repository` for details). A folder within a repository is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative from :mod:`offlineimap.folder`. -This page contains the main API overview of OfflineImap |release|. +This page contains the main API overview of OfflineImap |release|. OfflineImap can be imported as:: @@ -22,7 +22,7 @@ be merged into the main documentation. :mod:`offlineimap` -- The OfflineImap module ============================================= - + .. module:: offlineimap .. autoclass:: offlineimap.OfflineImap(cmdline_opts = None) diff --git a/docs/doc-src/FAQ.rst b/docs/doc-src/FAQ.rst index 1d49c3d..b8c2b56 100644 --- a/docs/doc-src/FAQ.rst +++ b/docs/doc-src/FAQ.rst @@ -67,13 +67,13 @@ based in instructions submitted by Chris Walker:: First, you must run OfflineIMAP in the Cygwin environment. The Windows filesystem is not powerful enough to accomodate Maildir by itself. - + Next, you’ll need to mount your Maildir directory in a special way. There is information for doing that at http://barnson.org/node/295. That site gives this example:: - + mount -f -s -b -o managed "d:/tmp/mail" "/home/of/mail" - + That URL also has more details on making OfflineIMAP work with Windows. @@ -383,9 +383,9 @@ you’ll list this:: [mbnames] enabled = yes filename = ~/Mutt/muttrc.mailboxes - header = "mailboxes " - peritem = "+%(accountname)s/%(foldername)s" - sep = " " + header = "mailboxes " + peritem = "+%(accountname)s/%(foldername)s" + sep = " " footer = "\n" Then in your ``.muttrc``:: @@ -448,7 +448,7 @@ written in Korn, so you’ll need ksh, pdksh, or mksh to run it:: # remove any old instances of this shell script or offlineimap for pid in $(pgrep offlineimap) do - if $pid -ne $$ + if $pid -ne $$ then kill $pid fi diff --git a/docs/doc-src/HACKING.rst b/docs/doc-src/HACKING.rst index 9192458..54a89ef 100644 --- a/docs/doc-src/HACKING.rst +++ b/docs/doc-src/HACKING.rst @@ -440,7 +440,7 @@ Know the status of your patch after submission of the branch in which your patch has been merged (i.e. it will not tell you if your patch is merged in pu if you rebase on top of master). - + .. * Read the git mailing list, the maintainer regularly posts messages entitled "What's cooking in git.git" and "What's in git.git" giving the status of various proposed changes. diff --git a/offlineimap.conf b/offlineimap.conf index 71eac11..3228c21 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -190,7 +190,7 @@ remoterepository = RemoteExample # In this case a call to imapfilter to filter mail before the sync process # starts and a custom shell script after the sync completes. # The pre sync script has to complete before a sync to the account will -# start. +# start. # presynchook = imapfilter # postsynchook = notifysync.sh diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 3cd1cef..5e95ae6 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -29,7 +29,7 @@ class CustomConfigParser(SafeConfigParser): return self.get(*(section, option) + args, **kwargs) else: return default - + def getdefaultint(self, section, option, default, *args, **kwargs): if self.has_option(section, option): return self.getint (*(section, option) + args, **kwargs) @@ -120,7 +120,7 @@ class ConfigHelperMixin: return self._confighelper_runner(option, default, self.getconfig().getdefaultint, self.getconfig().getint) - + def getconffloat(self, option, default = CustomConfigDefault): return self._confighelper_runner(option, default, self.getconfig().getdefaultfloat, diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index a7d043c..8152284 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -14,6 +14,6 @@ __homepage__ = "http://offlineimap.org" banner = __bigcopyright__ from offlineimap.error import OfflineImapError -# put this last, so we don't run into circular dependencies using +# put this last, so we don't run into circular dependencies using # e.g. offlineimap.__version__. from offlineimap.init import OfflineImap diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 81a81eb..27c9a2b 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -210,7 +210,7 @@ class BaseFolder(object): If the backend CAN assign a new uid, but cannot find out what this UID is (as is the case with some IMAP servers), it returns 0 but DOES save the message. - + IMAP backend should be the only one that can assign a new uid. @@ -493,7 +493,7 @@ class BaseFolder(object): continue #don't actually remove in a dryrun dstfolder.deletemessagesflags(uids, set(flag)) statusfolder.deletemessagesflags(uids, set(flag)) - + def syncmessagesto(self, dstfolder, statusfolder): """Syncs messages in this folder to the destination dstfolder. @@ -514,7 +514,7 @@ class BaseFolder(object): uids present (except for potential negative uids that couldn't be placed anywhere). - Pass3: Synchronize flag changes + Pass3: Synchronize flag changes Compare flag mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to both dstfolder and diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 4218886..db23813 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -44,7 +44,7 @@ class IMAPFolder(BaseFolder): Prefer SELECT to EXAMINE if we can, since some servers (Courier) do not stabilize UID validity until the folder is - selected. + selected. .. todo: Still valid? Needs verification :param: Enforce new SELECT even if we are on that folder already. :returns: raises :exc:`OfflineImapError` severity FOLDER on error""" @@ -116,7 +116,7 @@ class IMAPFolder(BaseFolder): maxmsgid = max(long(msgid), maxmsgid) # Different number of messages than last time? if maxmsgid != statusfolder.getmessagecount(): - return True + return True return False def cachemessagelist(self): @@ -526,7 +526,7 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage: header is: %s: %s' %\ (headername, headervalue)) content = self.savemessage_addheader(content, headername, - headervalue) + headervalue) if len(content)>200: dbg_output = "%s...%s" % (content[:150], content[-50:]) else: @@ -710,11 +710,11 @@ class IMAPFolder(BaseFolder): def change_message_uid(self, uid, new_uid): """Change the message from existing uid to new_uid - If the backend supports it. IMAP does not and will throw errors.""" + If the backend supports it. IMAP does not and will throw errors.""" raise OfflineImapError('IMAP backend cannot change a messages UID from ' '%d to %d' % (uid, new_uid), OfflineImapError.ERROR.MESSAGE) - + def deletemessage(self, uid): self.deletemessages_noconvert([uid]) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index ac67c2f..b624134 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -35,7 +35,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): #though. According to sqlite docs, you need to commit() before #the connection is closed or your changes will be lost!""" #get db connection which autocommits - #connection = sqlite.connect(self.filename, isolation_level=None) + #connection = sqlite.connect(self.filename, isolation_level=None) #cursor = connection.cursor() #return connection, cursor @@ -43,7 +43,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): cur_version = 1 def __init__(self, name, repository): - super(LocalStatusSQLiteFolder, self).__init__(name, repository) + super(LocalStatusSQLiteFolder, self).__init__(name, repository) # dblock protects against concurrent writes in same connection self._dblock = Lock() #Try to establish connection, no need for threadsafety in __init__ diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 24d943c..23f48b4 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -196,7 +196,7 @@ class MaildirFolder(BaseFolder): if sorted(self.getmessageuidlist()) != \ sorted(statusfolder.getmessageuidlist()): return True - # Also check for flag changes, it's quick on a Maildir + # Also check for flag changes, it's quick on a Maildir for (uid, message) in self.getmessagelist().iteritems(): if message['flags'] != statusfolder.getmessageflags(uid): return True @@ -235,7 +235,7 @@ class MaildirFolder(BaseFolder): return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s' % \ (timeval, timeseq, os.getpid(), socket.gethostname(), uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) - + def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. @@ -263,7 +263,7 @@ class MaildirFolder(BaseFolder): fd = os.open(os.path.join(tmpdir, messagename), os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666) except OSError as e: - if e.errno == 17: + if e.errno == 17: #FILE EXISTS ALREADY severity = OfflineImapError.ERROR.MESSAGE raise OfflineImapError("Unique filename %s already existing." %\ @@ -348,7 +348,7 @@ class MaildirFolder(BaseFolder): os.path.join(self.getfullname(), dir_prefix, filename)) self.messagelist[new_uid] = self.messagelist[uid] del self.messagelist[uid] - + def deletemessage(self, uid): """Unlinks a message file from the Maildir. @@ -373,4 +373,4 @@ class MaildirFolder(BaseFolder): os.unlink(filepath) # Yep -- return. del(self.messagelist[uid]) - + diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index f571772..7dc43f3 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -42,7 +42,7 @@ class MappedIMAPFolder(IMAPFolder): def _getmapfilename(self): return os.path.join(self.repository.getmapdir(), self.getfolderbasename()) - + def _loadmaps(self): self.maplock.acquire() try: diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index b7e0d22..dfd6900 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -1330,7 +1330,7 @@ class IMAP4(object): self.ouq.put(rqb) return rqb - # Must setup continuation expectancy *before* ouq.put + # Must setup continuation expectancy *before* ouq.put crqb = self._request_push(tag='continuation') self.ouq.put(rqb) @@ -2442,7 +2442,7 @@ if __name__ == '__main__': run('id', ()) run('id', ('("name", "imaplib2")',)) run('id', ("version", __version__, "os", os.uname()[0])) - + for cmd,args in test_seq2: if (cmd,args) != ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')): run(cmd, args) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index c8f70d6..ef05cb7 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -137,7 +137,7 @@ class IMAPServer: try: if self.gss_step == self.GSS_STATE_STEP: if not self.gss_vc: - rc, self.gss_vc = kerberos.authGSSClientInit('imap@' + + rc, self.gss_vc = kerberos.authGSSClientInit('imap@' + self.hostname) response = kerberos.authGSSClientResponse(self.gss_vc) rc = kerberos.authGSSClientStep(self.gss_vc, data) @@ -188,7 +188,7 @@ class IMAPServer: self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj - + self.connectionlock.release() # Release until need to modify data """ Must be careful here that if we fail we should bail out gracefully @@ -433,7 +433,7 @@ class IMAPServer: certnames = [] # cert expired? - notafter = cert.get('notAfter') + notafter = cert.get('notAfter') if notafter: if time.time() >= cert_time_to_seconds(notafter): return '%s certificate expired %s' % (errstr, notafter) diff --git a/offlineimap/init.py b/offlineimap/init.py index d52fd3a..ce24e48 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -277,7 +277,7 @@ class OfflineImap: if options.logfile: sys.stderr = self.ui.logfile - + socktimeout = config.getdefaultint("general", "socktimeout", 0) if socktimeout > 0: socket.setdefaulttimeout(socktimeout) diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index fb8af74..facbfa7 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -70,5 +70,3 @@ def genmbnames(): file.close() finally: mblock.release() - - diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 575c993..db80ecd 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -116,7 +116,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getlocaleval(self): return self.account.getlocaleval() - + def getfolders(self): """Returns a list of ALL folders on this server.""" return [] diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index f4260c0..5f86ed3 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -28,7 +28,7 @@ class GmailRepository(IMAPRepository): HOSTNAME = "imap.gmail.com" # Gmail IMAP server port PORT = 993 - + def __init__(self, reposname, account): """Initialize a GmailRepository object.""" # Enforce SSL usage diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 67ae2c0..3aec2fa 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -362,7 +362,7 @@ class IMAPRepository(BaseRepository): OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) - + class MappedIMAPRepository(IMAPRepository): def getfoldertype(self): return MappedIMAPFolder diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index b1b763a..bb9ada4 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -91,7 +91,7 @@ class LocalStatusRepository(BaseRepository): return folder def getfolders(self): - """Returns a list of all cached folders. + """Returns a list of all cached folders. Does nothing for this backend. We mangle the folder file names (see getfolderfilename) so we can not derive folder names from diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index f2d5581..1e23bea 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -85,7 +85,7 @@ class MaildirRepository(BaseRepository): if self.account.dryrun: return full_path = os.path.abspath(os.path.join(self.root, foldername)) - + # sanity tests if self.getsep() == '/': for component in foldername.split('/'): diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 069bb35..cee2849 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -79,7 +79,7 @@ class MachineUI(UIBase): s._printData('connecting', "%s\n%s" % (hostname, str(port))) def syncfolders(s, srcrepos, destrepos): - s._printData('syncfolders', "%s\n%s" % (s.getnicename(srcrepos), + s._printData('syncfolders', "%s\n%s" % (s.getnicename(srcrepos), s.getnicename(destrepos))) def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): diff --git a/offlineimap/ui/__init__.py b/offlineimap/ui/__init__.py index 6ddbaf6..3da42b9 100644 --- a/offlineimap/ui/__init__.py +++ b/offlineimap/ui/__init__.py @@ -20,7 +20,7 @@ from offlineimap.ui import TTY, Noninteractive, Machine UI_LIST = {'ttyui': TTY.TTYUI, 'basic': Noninteractive.Basic, - 'quiet': Noninteractive.Quiet, + 'quiet': Noninteractive.Quiet, 'machineui': Machine.MachineUI} #add Blinkenlights UI if it imports correctly (curses installed) diff --git a/offlineimap/ui/debuglock.py b/offlineimap/ui/debuglock.py index 4f2a4f9..4756b08 100644 --- a/offlineimap/ui/debuglock.py +++ b/offlineimap/ui/debuglock.py @@ -25,7 +25,7 @@ class DebuggingLock: def __init__(self, name): self.lock = Lock() self.name = name - + def acquire(self, blocking = 1): self.print_tb("Acquire lock") self.lock.acquire(blocking) @@ -45,5 +45,5 @@ class DebuggingLock: self.logmsg(".... %s: Thread %s attempting to %s\n" % \ (self.name, currentThread().getName(), msg) + \ "\n".join(traceback.format_list(traceback.extract_stack()))) - + From 67d68c2fc87a7c183186f74f7563c181d4ea206a Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 22 Jul 2013 00:08:39 +0400 Subject: [PATCH 562/817] IMAP: simplify locking and fix potential lock loss Run the locked code under 'with': this guarantees that lock will be released in any case. This modification also avoids the case when our thread wasn't running locked when exception was caught, another thread got the lock, our code checked it via self.connectionlock.locked() and errorneously released the lock thinking that is was running locked. Signed-off-by: Eygene Ryabinkin --- offlineimap/imapserver.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index ef05cb7..75203e0 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -297,10 +297,9 @@ class IMAPServer: self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) - self.connectionlock.acquire() - self.assignedconnections.append(imapobj) - self.lastowner[imapobj] = curThread.ident - self.connectionlock.release() + with self.connectionlock: + self.assignedconnections.append(imapobj) + self.lastowner[imapobj] = curThread.ident return imapobj except Exception as e: """If we are here then we did not succeed in getting a @@ -308,9 +307,6 @@ class IMAPServer: error...""" self.semaphore.release() - if(self.connectionlock.locked()): - self.connectionlock.release() - severity = OfflineImapError.ERROR.REPO if type(e) == gaierror: #DNS related errors. Abort Repo sync From 0d992ee7d370a1caa9b2c9be712fd09385185d8f Mon Sep 17 00:00:00 2001 From: mxgr7 Date: Thu, 4 Jul 2013 05:48:12 +0200 Subject: [PATCH 563/817] Execute pre/post hooks for IDLE-toggled syncs Make IDLE syncs be equal to the regular synchronisations in respect to pre-sync and post-sync hooks. From: mxgr7 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 ++- offlineimap/imapserver.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 2b8aef8..5f81af9 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -18,7 +18,8 @@ WIP (add new stuff for the next release) OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== -* Bump version number +* Execute pre/post-sync hooks during synchronizations + toggled by IMAP IDLE message processing. (maxgerer@gmail.com) OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 75203e0..89cced2 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -507,7 +507,13 @@ class IdleThread(object): remoterepos = account.remoterepos statusrepos = account.statusrepos remotefolder = remoterepos.getfolder(self.folder) + + hook = account.getconf('presynchook', '') + account.callhook(hook) offlineimap.accounts.syncfolder(account, remotefolder, quick=False) + hook = account.getconf('postsynchook', '') + account.callhook(hook) + ui = getglobalui() ui.unregisterthread(currentThread()) #syncfolder registered the thread From 56b0c5dbac2c7f39c8830ce09d0142bbd0424387 Mon Sep 17 00:00:00 2001 From: Johan Herland Date: Sun, 19 May 2013 12:13:57 +0200 Subject: [PATCH 564/817] Allow custom sorting of mailboxes for mbnames mutt-sidebar and, probably, other MUA show mailboxes in the order they are listed in the file written by mbnames. Therefore, to allow customization of the order with which mailboxes are listed, introduce the new 'sort_keyfunc' directive in the [mbnames] section. 'sort_keyfunc' must be a function that will be called once for each mailbox. It must accept the only argument -- a dict with 2 items, 'accountname' and 'foldername', and should return an object that will be used as the sorting key for each mailbox. Default key function returns (d['accountname'], d['foldername']), thus sorting by account name and then by the folder name. Signed-off-by: Johan Herland Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ offlineimap.conf | 14 ++++++++++++++ offlineimap/mbnames.py | 10 ++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 5f81af9..3cf2381 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -14,6 +14,8 @@ WIP (add new stuff for the next release) * SIGHUP is now handled as the termination notification rather than the signal to reread the configuration (Dmitrijs Ledkovs) * Honor the timezone of emails (Tobias Thierer) +* Allow mbnames output to be sorted by a custom sort key by specifying + a 'sort_keyfunc' function in the [mbnames] section of the config. OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap.conf b/offlineimap.conf index 3228c21..1114883 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -145,6 +145,20 @@ footer = "\n" # # Note that this filter can be used only to further restrict mbnames # to a subset of folders that pass the account's folderfilter. +# +# +# You can customize the order in which mailbox names are listed in the +# generated file by specifying a sort_keyfunc, which takes a single +# dict argument containing keys 'accountname' and 'foldername'. This +# function will be called once for each mailbox, and should return a +# suitable sort key that defines this mailbox' position in the custom +# ordering. +# +# This is useful with e.g. Mutt-sidebar, which uses the mailbox order +# from the generated file when listing mailboxes in the sidebar. +# +# Default setting is +# sort_keyfunc = lambda d: (d['accountname'], d['foldername']) ################################################## diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index facbfa7..34eccf9 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -58,13 +58,19 @@ def genmbnames(): if config.has_option("mbnames", "folderfilter"): folderfilter = localeval.eval(config.get("mbnames", "folderfilter"), {'re': re}) + mb_sort_keyfunc = lambda d: (d['accountname'], d['foldername']) + if config.has_option("mbnames", "sort_keyfunc"): + mb_sort_keyfunc = localeval.eval(config.get("mbnames", "sort_keyfunc"), + {'re': re}) itemlist = [] for accountname in boxes.keys(): for foldername in boxes[accountname]: if folderfilter(accountname, foldername): - itemlist.append(config.get("mbnames", "peritem", raw=1) % \ - {'accountname': accountname, + itemlist.append({'accountname': accountname, 'foldername': foldername}) + itemlist.sort(key = mb_sort_keyfunc) + format_string = config.get("mbnames", "peritem", raw=1) + itemlist = [format_string % d for d in itemlist] file.write(localeval.eval(config.get("mbnames", "sep")).join(itemlist)) file.write(localeval.eval(config.get("mbnames", "footer"))) file.close() From 3a580049ad20aca4a3e59b2ec32956eac1451973 Mon Sep 17 00:00:00 2001 From: Bart Kerkvliet Date: Thu, 11 Apr 2013 20:09:08 +0200 Subject: [PATCH 565/817] Typos fixed in config file Fixed some minor typos in the commented config file. I think these days Google also uses Gmail in Germany, so also changed German Google Mail to German Gmail. --- offlineimap.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 1114883..382032c 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -111,7 +111,7 @@ accounts = Test # write cycles. Therefore, you can disable OfflineIMAP's use of fsync(). # Doing so will come at the expense of greater risk of message duplication # in the event of a system crash or power loss. Default is fsync = true. -# Set fsync = false ot disable fsync. +# Set fsync = false to disable fsync. # # fsync = true @@ -454,7 +454,7 @@ remoteuser = username # You can disable that if you wish. This means that OfflineIMAP will # mark them deleted on the server, but not actually delete them. # You must use some other IMAP client to delete them if you use this -# setting; otherwise, the messgaes will just pile up there forever. +# setting; otherwise, the messages will just pile up there forever. # Therefore, this setting is definitely NOT recommended. # #expunge = no @@ -571,7 +571,7 @@ type = Gmail remoteuser = username@gmail.com # The trash folder name may be different from [Gmail]/Trash -# for example on german googlemail, this setting should be +# for example on German Gmail, this setting should be # # trashfolder = [Google Mail]/Papierkorb # From d39a1f864f3549d5e518440f11b46a997194af56 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Wed, 27 Mar 2013 12:43:39 +0000 Subject: [PATCH 566/817] make savemessage() handle NO response to APPEND correctly IMAP servers can return `NO` responses to the `APPEND` command, e.g. here's an example response from Groupwise's IMAP server: NO APPEND The 1500 MB storage limit has been exceeded. In this case, savemessage() should abort the repository sync rather than returning UID 0 which would cause the local copy of the message being saved to get irreversibly deleted. Signed-off-by: Adam Spiers Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ offlineimap/folder/IMAP.py | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 3cf2381..0aab40b 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -22,6 +22,9 @@ OfflineIMAP v6.5.5-rc1 (2012-09-05) * Execute pre/post-sync hooks during synchronizations toggled by IMAP IDLE message processing. (maxgerer@gmail.com) +* Catch unsuccessful local mail uploads when IMAP server + responds with "NO" status; that resulted in a loss of such + local messages. (Adam Spiers) OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index db23813..deba1db 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -548,7 +548,21 @@ class IMAPFolder(BaseFolder): (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) - retry_left = 0 # Mark as success + # This should only catch 'NO' responses since append() + # will raise an exception for 'BAD' responses: + if typ != 'OK': + # For example, Groupwise IMAP server can return something like: + # + # NO APPEND The 1500 MB storage limit has been exceeded. + # + # In this case, we should immediately abort the repository sync + # and continue with the next account. + msg = \ + "Saving msg in folder '%s', repository '%s' failed (abort). " \ + "Server responded: %s %s\n" % \ + (self, self.getrepository(), typ, dat) + raise OfflineImapError(msg, OfflineImapError.ERROR.REPO) + retry_left = 0 # Mark as success except imapobj.abort as e: # connection has been reset, release connection and retry. retry_left -= 1 From 1184b6c1a39d256c0c8cb9e1237cc74dd2de1545 Mon Sep 17 00:00:00 2001 From: Bart Kerkvliet Date: Thu, 11 Apr 2013 20:15:14 +0200 Subject: [PATCH 567/817] Fix country-specific GMail folder names * Germany (and Austria) now use GMail as the base folder names. * Suggests users to check their spam and trash folder names and set them appropriately. From: Bart Kerkvliet Signed-off-by: Eygene Ryabinkin --- offlineimap.conf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 382032c..2794f32 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -17,9 +17,9 @@ # trashfolder: %(gmailtrashfolder)s # # [DEFAULT] -# gmailtrashfolder = [Google Mail]/Papierkorb +# gmailtrashfolder = [Gmail]/Papierkorb # -# would set the trashfolder setting for your German gmail accounts. +# would set the trashfolder setting for your German Gmail accounts. # NOTE2: This implies that any '%' needs to be encoded as '%%' @@ -232,7 +232,7 @@ remoterepository = RemoteExample # messages (e.g. those with large attachments etc). If you do this it # will appear to offlineimap that these messages do not exist at all. They # will not be copied, have flags changed etc. For this to work on an IMAP -# server the server must have server side search enabled. This works with gmail +# server the server must have server side search enabled. This works with Gmail # and most imap servers (e.g. cyrus etc) # The maximum size should be specified in bytes - e.g. 2000000 for approx 2MB @@ -573,11 +573,10 @@ remoteuser = username@gmail.com # The trash folder name may be different from [Gmail]/Trash # for example on German Gmail, this setting should be # -# trashfolder = [Google Mail]/Papierkorb +# trashfolder = [Gmail]/Papierkorb # -# The same is valid for the spam folder -# -# spamfolder = [Google Mail]/Spam +# You should look for the localized names of the spam folder too: +# "spamfolder" tunable will help you to override the standard name. # Enable 1-way synchronization. See above for explanation. # From 7d313f49dc60c73816cda4f713f5deb6cbf6a744 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 6 Aug 2013 01:10:10 +0400 Subject: [PATCH 568/817] Refactored authentication handling - created helper routine that will do authentication; - routine tries each method in turn, first successful one terminates it: makes things easier to read and handle; - renamed plainauth() inside offlineimap/imapserver.py to loginauth(): the function does IMAP LOGIN authentication and there is PLAIN SASL method, so previous name was a bit misleading; - slightly improved error reporting: all exceptions during authentication will be reported at the end of the run; - now loginauth() is never called if LOGINDISABLED is advertized by the server; it used to be invoked unconditionally when CRAM-MD5 fails, but we should respect server's opinion on how to handle its users. Signed-off-by: Eygene Ryabinkin --- offlineimap/imapserver.py | 157 ++++++++++++++++++++++++++------------ 1 file changed, 108 insertions(+), 49 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 89cced2..896c073 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -128,8 +128,9 @@ class IMAPServer: self.ui.debug('imap', 'md5handler: returning %s' % retval) return retval - def plainauth(self, imapobj): - self.ui.debug('imap', 'Attempting plain authentication') + def loginauth(self, imapobj): + """ Basic authentication via LOGIN command """ + self.ui.debug('imap', 'Attempting IMAP LOGIN authentication') imapobj.login(self.username, self.getpassword()) def gssauth(self, response): @@ -159,6 +160,107 @@ class IMAPServer: response = '' return base64.b64decode(response) + + def _authn_helper(self, imapobj): + """ + Authentication machinery for self.acquireconnection(). + + Raises OfflineImapError() of type ERROR.REPO when + there are either fatal problems or no authentications + succeeded. + + If any authentication method succeeds, routine should exit: + warnings for failed methods are to be produced in the + respective except blocks. + + """ + + # Stack stores pairs of (method name, exception) + exc_stack = [] + tried_to_authn = False + + # Try GSSAPI and continue if it fails + if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss: + self.connectionlock.acquire() + self.ui.debug('imap', 'Attempting GSSAPI authentication') + tried_to_authn = True + try: + imapobj.authenticate('GSSAPI', self.gssauth) + except imapobj.error as e: + self.gssapi = False + self.ui.warn('GSSAPI authentication failed: %s' % e) + exc_stack.append(('GSSAPI', e)) + else: + self.gssapi = True + kerberos.authGSSClientClean(self.gss_vc) + self.gss_vc = None + self.gss_step = self.GSS_STATE_STEP + #if we do self.password = None then the next attempt cannot try... + #self.password = None + return + finally: + self.connectionlock.release() + + # Fire up TLS if we can and asked to: gonna to authenticate + # via plaintext or hashed schemes, so it is best to have + # channel that is protected from eavesdropping. + if 'STARTTLS' in imapobj.capabilities and not self.usessl: + self.ui.debug('imap', 'Using STARTTLS connection') + try: + imapobj.starttls() + except imapobj.error as e: + raise OfflineImapError("Failed to start " + "TLS connection: %s" % str(e), + OfflineImapError.ERROR.REPO) + + if 'AUTH=CRAM-MD5' in imapobj.capabilities: + tried_to_authn = True + self.ui.debug('imap', 'Attempting ' + 'CRAM-MD5 authentication') + try: + imapobj.authenticate('CRAM-MD5', self.md5handler) + return + except imapobj.error as e: + self.ui.warn('CRAM-MD5 authentication failed: %s' % e) + exc_stack.append(('CRAM-MD5', e)) + + # Last resort: use LOGIN command, + # unless LOGINDISABLED is advertized (RFC 2595) + if 'LOGINDISABLED' in imapobj.capabilities: + e = OfflineImapError("IMAP LOGIN is " + "disabled by server. Need to use SSL?", + OfflineImapError.ERROR.REPO) + exc_stack.append(('IMAP LOGIN', e)) + else: + tried_to_authn = True + self.ui.debug('imap', 'Attempting ' + 'IMAP LOGIN authentication') + try: + self.loginauth(imapobj) + return + except imapobj.error as e: + self.ui.warn('IMAP LOGIN authentication failed: %s' % e) + exc_stack.append(('IMAP LOGIN', e)) + + if len(exc_stack): + msg = "\n\t".join(map( + lambda x: ": ".join((x[0], str(x[1]))), + exc_stack + )) + raise OfflineImapError("All authentication types " + "failed:\n\t%s" % msg, OfflineImapError.ERROR.REPO) + + if not tried_to_authn: + methods = ", ".join(map( + lambda x: x[5:], filter(lambda x: x[0:5] == "AUTH=", + imapobj.capabilities) + )) + raise OfflineImapError("No supported " + "authentication mechanisms found; " + "server advertises %s" % methods, + OfflineImapError.ERROR.REPO) + + def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. @@ -223,54 +325,11 @@ class IMAPServer: if not self.tunnel: try: - # Try GSSAPI and continue if it fails - if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss: - self.connectionlock.acquire() - self.ui.debug('imap', - 'Attempting GSSAPI authentication') - try: - imapobj.authenticate('GSSAPI', self.gssauth) - except imapobj.error as val: - self.gssapi = False - self.ui.debug('imap', - 'GSSAPI Authentication failed') - else: - self.gssapi = True - kerberos.authGSSClientClean(self.gss_vc) - self.gss_vc = None - self.gss_step = self.GSS_STATE_STEP - #if we do self.password = None then the next attempt cannot try... - #self.password = None - self.connectionlock.release() - - if not self.gssapi: - if 'STARTTLS' in imapobj.capabilities and not\ - self.usessl: - self.ui.debug('imap', - 'Using STARTTLS connection') - imapobj.starttls() - - if 'AUTH=CRAM-MD5' in imapobj.capabilities: - self.ui.debug('imap', - 'Attempting CRAM-MD5 authentication') - try: - imapobj.authenticate('CRAM-MD5', - self.md5handler) - except imapobj.error as val: - self.plainauth(imapobj) - else: - # Use plaintext login, unless - # LOGINDISABLED (RFC2595) - if 'LOGINDISABLED' in imapobj.capabilities: - raise OfflineImapError("Plaintext login " - "disabled by server. Need to use SSL?", - OfflineImapError.ERROR.REPO) - self.plainauth(imapobj) - # Would bail by here if there was a failure. - success = 1 + self._authn_helper(imapobj) self.goodpassword = self.password - except imapobj.error as val: - self.passworderror = str(val) + success = 1 + except OfflineImapError as e: + self.passworderror = str(e) raise # update capabilities after login, e.g. gmail serves different ones From acaa96291d2fe6e03d46c48140a64832bedc312a Mon Sep 17 00:00:00 2001 From: Andreas Mack Date: Sat, 3 Aug 2013 14:06:44 +0200 Subject: [PATCH 569/817] Add SASL PLAIN authentication method - this method isn't as deprecated as IMAP LOGIN; - it allows to keep hashed passwords on the server side; - it has the ability to specify that the remote identity is different from authenticating username, so it even can be useful in some cases (e.g., migrated mailboxes); configuration variable "remote_identity" was introduced to leverage this functionality. From: Andreas Mack Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 1 + offlineimap.conf | 12 ++++++++++++ offlineimap/imapserver.py | 33 +++++++++++++++++++++++++++++++++ offlineimap/repository/IMAP.py | 12 ++++++++++++ 4 files changed, 58 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index 0aab40b..5a17b90 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -16,6 +16,7 @@ WIP (add new stuff for the next release) * Honor the timezone of emails (Tobias Thierer) * Allow mbnames output to be sorted by a custom sort key by specifying a 'sort_keyfunc' function in the [mbnames] section of the config. +* Support SASL PLAIN authentication method. (Andreas Mack) OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap.conf b/offlineimap.conf index 2794f32..9aba014 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -353,6 +353,18 @@ ssl = yes # Specify the remote user name. remoteuser = username +# Specify the user to be authorized as. Sometimes we want to +# authenticate with our login/password, but tell the server that we +# really want to be treated as some other user; perhaps server will +# allow us to do that (or, may be, not). Some IMAP servers migrate +# account names using this functionality: your credentials remain +# intact, but remote identity changes. +# +# Currently this variable is used only for SASL PLAIN authentication +# mechanism. +# +# remote_identity = authzuser + # There are six ways to specify the password for the IMAP server: # # 1. No password at all specified in the config file. diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 896c073..f6d7c27 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -55,6 +55,7 @@ class IMAPServer: self.tunnel = repos.getpreauthtunnel() self.usessl = repos.getssl() self.username = None if self.tunnel else repos.getuser() + self.user_identity = repos.get_remote_identity() self.password = None self.passworderror = None self.goodpassword = None @@ -133,6 +134,24 @@ class IMAPServer: self.ui.debug('imap', 'Attempting IMAP LOGIN authentication') imapobj.login(self.username, self.getpassword()) + + def plainhandler(self, response): + """ + Implements SASL PLAIN authentication, RFC 4616, + http://tools.ietf.org/html/rfc4616 + + """ + authc = self.username + passwd = self.getpassword() + authz = '' + if self.user_identity != None: + authz = self.user_identity + NULL = u'\x00' + retval = NULL.join((authz, authc, passwd)).encode('utf-8') + self.ui.debug('imap', 'plainhandler: returning %s' % retval) + return retval + + def gssauth(self, response): data = base64.b64encode(response) try: @@ -213,6 +232,8 @@ class IMAPServer: "TLS connection: %s" % str(e), OfflineImapError.ERROR.REPO) + # Hashed authenticators come first: they don't reveal + # passwords. if 'AUTH=CRAM-MD5' in imapobj.capabilities: tried_to_authn = True self.ui.debug('imap', 'Attempting ' @@ -224,6 +245,18 @@ class IMAPServer: self.ui.warn('CRAM-MD5 authentication failed: %s' % e) exc_stack.append(('CRAM-MD5', e)) + # Try plaintext authenticators. + if 'AUTH=PLAIN' in imapobj.capabilities: + tried_to_authn = True + self.ui.debug('imap', 'Attempting ' + 'PLAIN authentication') + try: + imapobj.authenticate('PLAIN', self.plainhandler) + return + except imapobj.error as e: + self.ui.warn('PLAIN authentication failed: %s' % e) + exc_stack.append(('PLAIN', e)) + # Last resort: use LOGIN command, # unless LOGINDISABLED is advertized (RFC 2595) if 'LOGINDISABLED' in imapobj.capabilities: diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 3aec2fa..ab466b6 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -115,6 +115,18 @@ class IMAPRepository(BaseRepository): "'%s' specified." % self, OfflineImapError.ERROR.REPO) + + def get_remote_identity(self): + """ + Remote identity is used for certain SASL mechanisms + (currently -- PLAIN) to inform server about the ID + we want to authorize as instead of our login name. + + """ + + return self.getconf('remote_identity', default=None) + + def getuser(self): user = None localeval = self.localeval From 968d5520dab5ac5188cc0070cd3acc492bfd023f Mon Sep 17 00:00:00 2001 From: Steve Purcell Date: Fri, 3 May 2013 14:56:20 +0100 Subject: [PATCH 570/817] Allow transport-type tunnels to the IMAP servers It's nice to set up an ssh tunnel command which forwards an IMAP tcp port inside an encrypted session, e.g. with ssh's "-W" flag. In this case the tunnelled connection still requires authentication inside IMAP session, because this is transport-only tunnel that substitutes normal TCP/SSL connection. New directive, 'transporttunnel' was added: it specifies the command that will create the tunnel. Only one type of tunnel must be specified for a single repository: we can't have both preauthenticated and transport-type tunnels, they won't chain together. From: Steve Purcell Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ offlineimap.conf | 23 +++++++++++++++++++++++ offlineimap/imapserver.py | 25 ++++++++++++++++++++----- offlineimap/repository/IMAP.py | 3 +++ 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 5a17b90..8fe9e04 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -17,6 +17,8 @@ WIP (add new stuff for the next release) * Allow mbnames output to be sorted by a custom sort key by specifying a 'sort_keyfunc' function in the [mbnames] section of the config. * Support SASL PLAIN authentication method. (Andreas Mack) +* Support transport-only tunnels that requre full IMAP authentication. + (Steve Purcell) OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap.conf b/offlineimap.conf index 9aba014..b72d84a 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -365,6 +365,8 @@ remoteuser = username # # remote_identity = authzuser +########## Passwords + # There are six ways to specify the password for the IMAP server: # # 1. No password at all specified in the config file. @@ -408,6 +410,27 @@ remoteuser = username ########## Advanced settings +# Tunnels. There are two types: +# +# - preauth: they teleport your connection to the remote system +# and you don't need to authenticate yourself there; the sole +# fact that you succeeded to get the tunnel running is enough. +# This tunnel type was explained above in the 'Passwords' section. +# +# - transport: the just provide the transport (probably encrypted) +# to the IMAP server, but you still need to authenticate at the +# IMAP server. +# +# Tunnels are currently working only with IMAP servers and their +# derivatives (currently, GMail). Additionally, for GMail accounts +# preauth tunnel settings are ignored: we don't believe that there +# are ways to preauthenticate at Google mail system IMAP servers. +# +# You must choose at most one tunnel type, be wise M'Lord. +# +# preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' +# transporttunnel = openssl s_client -host myimap -port 993 -quiet + # Some IMAP servers need a "reference" which often refers to the "folder # root". This is most commonly needed with UW IMAP, where you might # need to specify the directory in which your mail is stored. The diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index f6d7c27..2d9a1b1 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -52,14 +52,28 @@ class IMAPServer: self.ui = getglobalui() self.repos = repos self.config = repos.getconfig() - self.tunnel = repos.getpreauthtunnel() - self.usessl = repos.getssl() - self.username = None if self.tunnel else repos.getuser() + + self.preauth_tunnel = repos.getpreauthtunnel() + self.transport_tunnel = repos.gettransporttunnel() + if self.preauth_tunnel and self.transport_tunnel: + raise OfflineImapError('%s: ' % repos + \ + 'you must enable precisely one ' + 'type of tunnel (preauth or transport), ' + 'not both', OfflineImapError.ERROR.REPO) + self.tunnel = \ + self.preauth_tunnel if self.preauth_tunnel \ + else self.transport_tunnel + + self.username = \ + None if self.preauth_tunnel else repos.getuser() self.user_identity = repos.get_remote_identity() self.password = None self.passworderror = None self.goodpassword = None - self.hostname = None if self.tunnel else repos.gethost() + + self.usessl = repos.getssl() + self.hostname = \ + None if self.preauth_tunnel else repos.gethost() self.port = repos.getport() if self.port == None: self.port = 993 if self.usessl else 143 @@ -69,6 +83,7 @@ class IMAPServer: self.sslversion = repos.getsslversion() if self.sslcacertfile is None: self.verifycert = None # disable cert verification + self.delim = None self.root = None self.maxconnections = repos.getmaxconnections() @@ -356,7 +371,7 @@ class IMAPServer: imapobj = imaplibutil.WrappedIMAP4(self.hostname, self.port, timeout=socket.getdefaulttimeout()) - if not self.tunnel: + if not self.preauth_tunnel: try: self._authn_helper(imapobj) self.goodpassword = self.password diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index ab466b6..35190c0 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -200,6 +200,9 @@ class IMAPRepository(BaseRepository): def getpreauthtunnel(self): return self.getconf('preauthtunnel', None) + def gettransporttunnel(self): + return self.getconf('transporttunnel', None) + def getreference(self): return self.getconf('reference', '') From e26827c1cb12db7d44a84061bab8782656fbf430 Mon Sep 17 00:00:00 2001 From: Andreas Mack Date: Wed, 7 Aug 2013 13:43:51 +0200 Subject: [PATCH 571/817] Make authentication mechanisms configurable Added configuration option "auth_mechanisms" to the config file: it is a list of mechanisms that will be tried in the specified order. Author: Andreas Mack Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 + offlineimap.conf | 16 ++- offlineimap/CustomConfig.py | 38 ++++++- offlineimap/imapserver.py | 187 ++++++++++++++++++++------------- offlineimap/repository/IMAP.py | 20 ++++ 5 files changed, 183 insertions(+), 80 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 8fe9e04..19bc664 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -19,6 +19,8 @@ WIP (add new stuff for the next release) * Support SASL PLAIN authentication method. (Andreas Mack) * Support transport-only tunnels that requre full IMAP authentication. (Steve Purcell) +* Make the list of authentication mechanisms to be configurable. + (Andreas Mack) OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap.conf b/offlineimap.conf index b72d84a..039e210 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -361,10 +361,24 @@ remoteuser = username # intact, but remote identity changes. # # Currently this variable is used only for SASL PLAIN authentication -# mechanism. +# mechanism, so consider using auth_mechanisms to prioritize PLAIN +# or even make it the only mechanism to be tried. # # remote_identity = authzuser +# Specify which authentication/authorization mechanisms we should try +# and the order in which OfflineIMAP will try them. NOTE: any given +# mechanism will be tried only if it is supported by the remote IMAP +# server. +# +# Due to the technical limitations, if you're specifying GSSAPI +# as the mechanism to try, it will be tried first, no matter where +# it was specified in the list. +# +# Default value is +# auth_mechanisms = GSSAPI, CRAM-MD5, PLAIN, LOGIN +# ranged is from strongest to more weak ones. + ########## Passwords # There are six ways to specify the password for the IMAP server: diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 5e95ae6..447ac19 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -15,11 +15,12 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA try: - from ConfigParser import SafeConfigParser + from ConfigParser import SafeConfigParser, Error, NoOptionError except ImportError: #python3 - from configparser import SafeConfigParser + from configparser import SafeConfigParser, Error, NoOptionError from offlineimap.localeval import LocalEval import os +import re class CustomConfigParser(SafeConfigParser): def getdefault(self, section, option, default, *args, **kwargs): @@ -48,6 +49,25 @@ class CustomConfigParser(SafeConfigParser): else: return default + def getlist(self, section, option, separator_re): + """ + Parses option as the list of values separated + by the given regexp. + + """ + try: + val = self.get(section, option).strip() + return re.split(separator_re, val) + except re.error as e: + raise Error("Bad split regexp '%s': %s" % \ + (separator_re, e)) + + def getdefaultlist(self, section, option, default, separator_re): + if self.has_option(section, option): + return self.getlist(*(section, option, separator_re)) + else: + return default + def getmetadatadir(self): metadatadir = os.path.expanduser(self.getdefault("general", "metadata", "~/.offlineimap")) if not os.path.exists(metadatadir): @@ -97,12 +117,14 @@ class ConfigHelperMixin: will then return the configuration values for the ConfigParser object in the specific section.""" - def _confighelper_runner(self, option, default, defaultfunc, mainfunc): + def _confighelper_runner(self, option, default, defaultfunc, mainfunc, *args): """Return config value for getsection()""" + lst = [self.getsection(), option] if default == CustomConfigDefault: - return mainfunc(*[self.getsection(), option]) + return mainfunc(*(lst + list(args))) else: - return defaultfunc(*[self.getsection(), option, default]) + lst.append(default) + return defaultfunc(*(lst + list(args))) def getconf(self, option, @@ -125,3 +147,9 @@ class ConfigHelperMixin: return self._confighelper_runner(option, default, self.getconfig().getdefaultfloat, self.getconfig().getfloat) + + def getconflist(self, option, separator_re, + default = CustomConfigDefault): + return self._confighelper_runner(option, default, + self.getconfig().getdefaultlist, + self.getconfig().getlist, separator_re) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 2d9a1b1..d384bd2 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -67,6 +67,7 @@ class IMAPServer: self.username = \ None if self.preauth_tunnel else repos.getuser() self.user_identity = repos.get_remote_identity() + self.authmechs = repos.get_auth_mechanisms() self.password = None self.passworderror = None self.goodpassword = None @@ -195,6 +196,73 @@ class IMAPServer: return base64.b64decode(response) + def _start_tls(self, imapobj): + if 'STARTTLS' in imapobj.capabilities and not self.usessl: + self.ui.debug('imap', 'Using STARTTLS connection') + try: + imapobj.starttls() + except imapobj.error as e: + raise OfflineImapError("Failed to start " + "TLS connection: %s" % str(e), + OfflineImapError.ERROR.REPO) + + + ## All _authn_* procedures are helpers that do authentication. + ## They are class methods that take one parameter, IMAP object. + ## + ## Each function should return True if authentication was + ## successful and False if authentication wasn't even tried + ## for some reason (but not when IMAP has no such authentication + ## capability, calling code checks that). + ## + ## Functions can also raise exceptions; two types are special + ## and will be handled by the calling code: + ## + ## - imapobj.error means that there was some error that + ## comes from imaplib2; + ## + ## - OfflineImapError means that function detected some + ## problem by itself. + + def _authn_gssapi(self, imapobj): + if not have_gss: + return False + + self.connectionlock.acquire() + try: + imapobj.authenticate('GSSAPI', self.gssauth) + return True + except imapobj.error as e: + self.gssapi = False + raise + else: + self.gssapi = True + kerberos.authGSSClientClean(self.gss_vc) + self.gss_vc = None + self.gss_step = self.GSS_STATE_STEP + finally: + self.connectionlock.release() + + def _authn_cram_md5(self, imapobj): + imapobj.authenticate('CRAM-MD5', self.md5handler) + return True + + def _authn_plain(self, imapobj): + imapobj.authenticate('PLAIN', self.plainhandler) + return True + + def _authn_login(self, imapobj): + # Use LOGIN command, unless LOGINDISABLED is advertized + # (per RFC 2595) + if 'LOGINDISABLED' in imapobj.capabilities: + raise OfflineImapError("IMAP LOGIN is " + "disabled by server. Need to use SSL?", + OfflineImapError.ERROR.REPO) + else: + self.loginauth(imapobj) + return True + + def _authn_helper(self, imapobj): """ Authentication machinery for self.acquireconnection(). @@ -209,86 +277,56 @@ class IMAPServer: """ + # Authentication routines, hash keyed by method name + # with value that is a tuple with + # - authentication function, + # - tryTLS flag, + # - check IMAP capability flag. + auth_methods = { + "GSSAPI": (self._authn_gssapi, False, True), + "CRAM-MD5": (self._authn_cram_md5, True, True), + "PLAIN": (self._authn_plain, True, True), + "LOGIN": (self._authn_login, True, False), + } # Stack stores pairs of (method name, exception) exc_stack = [] tried_to_authn = False + tried_tls = False + mechs = self.authmechs - # Try GSSAPI and continue if it fails - if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss: - self.connectionlock.acquire() - self.ui.debug('imap', 'Attempting GSSAPI authentication') - tried_to_authn = True - try: - imapobj.authenticate('GSSAPI', self.gssauth) - except imapobj.error as e: - self.gssapi = False - self.ui.warn('GSSAPI authentication failed: %s' % e) - exc_stack.append(('GSSAPI', e)) - else: - self.gssapi = True - kerberos.authGSSClientClean(self.gss_vc) - self.gss_vc = None - self.gss_step = self.GSS_STATE_STEP - #if we do self.password = None then the next attempt cannot try... - #self.password = None - return - finally: - self.connectionlock.release() + # GSSAPI must be tried first: we will probably go TLS after it + # and GSSAPI mustn't be tunneled over TLS. + if "GSSAPI" in mechs: + mechs.remove("GSSAPI") + mechs.insert(0, "GSSAPI") - # Fire up TLS if we can and asked to: gonna to authenticate - # via plaintext or hashed schemes, so it is best to have - # channel that is protected from eavesdropping. - if 'STARTTLS' in imapobj.capabilities and not self.usessl: - self.ui.debug('imap', 'Using STARTTLS connection') - try: - imapobj.starttls() - except imapobj.error as e: - raise OfflineImapError("Failed to start " - "TLS connection: %s" % str(e), - OfflineImapError.ERROR.REPO) + for m in mechs: + if m not in auth_methods: + raise Exception("Bad authentication method %s, " + "please, file OfflineIMAP bug" % m) + + func, tryTLS, check_cap = auth_methods[m] + + # TLS must be initiated before checking capabilities: + # they could have been changed after STARTTLS. + if tryTLS and not tried_tls: + tried_tls = True + self._start_tls(imapobj) + + if check_cap: + cap = "AUTH=" + m + if cap not in imapobj.capabilities: + continue - # Hashed authenticators come first: they don't reveal - # passwords. - if 'AUTH=CRAM-MD5' in imapobj.capabilities: tried_to_authn = True self.ui.debug('imap', 'Attempting ' - 'CRAM-MD5 authentication') + '%s authentication' % m) try: - imapobj.authenticate('CRAM-MD5', self.md5handler) - return - except imapobj.error as e: - self.ui.warn('CRAM-MD5 authentication failed: %s' % e) - exc_stack.append(('CRAM-MD5', e)) - - # Try plaintext authenticators. - if 'AUTH=PLAIN' in imapobj.capabilities: - tried_to_authn = True - self.ui.debug('imap', 'Attempting ' - 'PLAIN authentication') - try: - imapobj.authenticate('PLAIN', self.plainhandler) - return - except imapobj.error as e: - self.ui.warn('PLAIN authentication failed: %s' % e) - exc_stack.append(('PLAIN', e)) - - # Last resort: use LOGIN command, - # unless LOGINDISABLED is advertized (RFC 2595) - if 'LOGINDISABLED' in imapobj.capabilities: - e = OfflineImapError("IMAP LOGIN is " - "disabled by server. Need to use SSL?", - OfflineImapError.ERROR.REPO) - exc_stack.append(('IMAP LOGIN', e)) - else: - tried_to_authn = True - self.ui.debug('imap', 'Attempting ' - 'IMAP LOGIN authentication') - try: - self.loginauth(imapobj) - return - except imapobj.error as e: - self.ui.warn('IMAP LOGIN authentication failed: %s' % e) - exc_stack.append(('IMAP LOGIN', e)) + if func(imapobj): + return + except (imapobj.error, OfflineImapError) as e: + self.ui.warn('%s authentication failed: %s' % (m, e)) + exc_stack.append((m, e)) if len(exc_stack): msg = "\n\t".join(map( @@ -303,9 +341,10 @@ class IMAPServer: lambda x: x[5:], filter(lambda x: x[0:5] == "AUTH=", imapobj.capabilities) )) - raise OfflineImapError("No supported " - "authentication mechanisms found; " - "server advertises %s" % methods, + raise OfflineImapError("Repository %s: no supported " + "authentication mechanisms found; configured %s, " + "server advertises %s" % (self.repos, + ", ".join(self.authmechs), methods), OfflineImapError.ERROR.REPO) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 35190c0..2b65de0 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -126,6 +126,26 @@ class IMAPRepository(BaseRepository): return self.getconf('remote_identity', default=None) + def get_auth_mechanisms(self): + supported = ["GSSAPI", "CRAM-MD5", "PLAIN", "LOGIN"] + # Mechanisms are ranged from the strongest to the + # weakest ones. + # TODO: we need DIGEST-MD5, it must come before CRAM-MD5 + # TODO: due to the chosen-plaintext resistance. + default = ["GSSAPI", "CRAM-MD5", "PLAIN", "LOGIN"] + + mechs = self.getconflist('auth_mechanisms', r',\s*', + default) + + for m in mechs: + if m not in supported: + raise OfflineImapError("Repository %s: " % self + \ + "unknown authentication mechanism '%s'" % m, + OfflineImapError.ERROR.REPO) + + self.ui.debug('imap', "Using authentication mechanisms %s" % mechs) + return mechs + def getuser(self): user = None From 3bc66c0858ecc2e69ff5e06b27cdf03a69128441 Mon Sep 17 00:00:00 2001 From: X-Ryl669 Date: Thu, 17 Jan 2013 14:21:21 +0100 Subject: [PATCH 572/817] Add support for alternative message date synchronisation Global or per-repository option utime_from_message tells OfflineIMAP to set file modification time of messages pushed from one repository to another basing on the message's "Date" header. This is useful if you are doing some processing/finding on your Maildir (for example, finding messages older than 3 months), without parsing each file/message content. From: Cyril RUSSO Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ offlineimap/emailutil.py | 38 ++++++++++++++++++++++++++++++++++++++ offlineimap/folder/Base.py | 16 +++++++++++++++- offlineimap/folder/IMAP.py | 20 +++++--------------- 4 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 offlineimap/emailutil.py diff --git a/Changelog.rst b/Changelog.rst index 19bc664..db31674 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -21,6 +21,8 @@ WIP (add new stuff for the next release) (Steve Purcell) * Make the list of authentication mechanisms to be configurable. (Andreas Mack) +* Allow to set message access and modification timestamps based + on the "Date" header of the message itself. (Cyril Russo) OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/emailutil.py b/offlineimap/emailutil.py new file mode 100644 index 0000000..28463f7 --- /dev/null +++ b/offlineimap/emailutil.py @@ -0,0 +1,38 @@ +# Some useful functions to extract data out of emails +# Copyright (C) 2002-2012 John Goerzen & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +import email +from email.Parser import Parser as MailParser +import time + +def get_message_date(content, header='Date'): + """ + Parses mail and returns resulting timestamp. + + :param header: the header to extract date from; + :returns: timestamp or `None` in the case of failure. + + """ + message = MailParser().parsestr(content, True) + dateheader = message.get(header) + # parsedate_tz returns a 10-tuple that can be passed to mktime_tz + # Will be None if missing or not in a valid format. Note that + # indexes 6, 7, and 8 of the result tuple are not usable. + datetuple = email.utils.parsedate_tz(dateheader) + if datetuple is None: + return None + return email.utils.mktime_tz(datetuple) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 27c9a2b..755a31a 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -15,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import threadutil +from offlineimap import threadutil, emailutil from offlineimap import globals from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError @@ -48,6 +48,13 @@ class BaseFolder(object): if self.visiblename == self.getsep(): self.visiblename = '' self.config = repository.getconfig() + utime_from_message_global = \ + self.config.getdefaultboolean("general", + "utime_from_message", False) + repo = "Repository " + repository.name + self._utime_from_message = \ + self.config.getdefaultboolean(repo, + "utime_from_message", utime_from_message_global) def getname(self): """Returns name""" @@ -66,6 +73,10 @@ class BaseFolder(object): """Should this folder be synced or is it e.g. filtered out?""" return self._sync_this + @property + def utime_from_message(self): + return self._utime_from_message + def suggeststhreads(self): """Returns true if this folder suggests using threads for actions; false otherwise. Probably only IMAP will return true.""" @@ -327,6 +338,9 @@ class BaseFolder(object): message = None flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) + if dstfolder.utime_from_message: + content = self.getmessage(uid) + rtime = emailutil.get_message_date(content, 'Date') if uid > 0 and dstfolder.uidexists(uid): # dst has message with that UID already, only update status diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index deba1db..cec9eda 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -15,14 +15,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import email import random import binascii import re import time from sys import exc_info from .Base import BaseFolder -from offlineimap import imaputil, imaplibutil, OfflineImapError +from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError from offlineimap import globals from offlineimap.imaplib2 import MonthNames @@ -431,21 +430,12 @@ class IMAPFolder(BaseFolder): :returns: string in the form of "DD-Mmm-YYYY HH:MM:SS +HHMM" (including double quotes) or `None` in case of failure (which is fine as value for append).""" + if rtime is None: - message = email.message_from_string(content) - dateheader = message.get('Date') - # parsedate_tz returns a 10-tuple that can be passed to mktime_tz; - # Will be None if missing or not in a valid format. Note that - # indexes 6, 7, and 8 of the result tuple are not usable. - datetuple = email.utils.parsedate_tz(dateheader) - if datetuple is None: - #could not determine the date, use the local time. + rtime = emailutil.get_message_date(content) + if rtime == None: return None - #make it a real struct_time, so we have named attributes - datetuple = time.localtime(email.utils.mktime_tz(datetuple)) - else: - #rtime is set, use that instead - datetuple = time.localtime(rtime) + datetuple = time.localtime(rtime) try: # Check for invalid dates From 7ec77a1c62740a07f21d6bb31ce2837796afc617 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 27 Aug 2013 15:55:47 +0400 Subject: [PATCH 573/817] Specify proper name for SSL version knob in configuration file Github issue: https://github.com/OfflineIMAP/offlineimap/issues/48 Spotted-by: gitprojs@GitHub Signed-off-by: Eygene Ryabinkin --- offlineimap.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap.conf b/offlineimap.conf index 039e210..ec69bbb 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -345,7 +345,7 @@ ssl = yes # automatically detected. In rare cases, it may be necessary to specify a # particular version from: tls1, ssl2, ssl3, ssl23 (SSLv2 or SSLv3) -# sslversion = ssl23 +# ssl_version = ssl23 # Specify the port. If not specified, use a default port. # remoteport = 993 From 8b6f10e2e78669125995e9445491a487179500ba Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 27 Aug 2013 16:23:12 +0400 Subject: [PATCH 574/817] Pass folder names for the foldersort function Bring the description in the template offlineimap.conf in sync to the actual implementation: pass folder names to the sorting function, not the offlineimap.folder.IMAP.IMAPFolder objects themselves. GitHub issue: https://github.com/OfflineIMAP/offlineimap/issues/27 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ offlineimap/repository/IMAP.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index db31674..2e621f5 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -23,6 +23,8 @@ WIP (add new stuff for the next release) (Andreas Mack) * Allow to set message access and modification timestamps based on the "Date" header of the message itself. (Cyril Russo) +* [regression] pass folder names to the foldersort function, + revert the documented behaviour OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 2b65de0..19db50f 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -365,7 +365,7 @@ class IMAPRepository(BaseRepository): def __init__(self, obj, *args): self.obj = obj def __cmp__(self, other): - return mycmp(self.obj, other.obj) + return mycmp(self.obj.getvisiblename(), other.obj.getvisiblename()) return K retval.sort(key=cmp2key(self.foldersort)) From 95aea5e489bb8656e813baa461e2d51ee8e4d2ae Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 27 Aug 2013 18:11:57 +0400 Subject: [PATCH 575/817] Add new expansion key for mbnames.peritem config variable It is called localfolders and holds expanded name for the same variable for the local repository of the account that is being processed. GitHub issue: https://github.com/OfflineIMAP/offlineimap/issues/21 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ offlineimap.conf | 6 ++++++ offlineimap/accounts.py | 3 ++- offlineimap/mbnames.py | 8 ++++++-- offlineimap/repository/Base.py | 5 +++++ 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 2e621f5..2d1b221 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -23,6 +23,9 @@ WIP (add new stuff for the next release) (Andreas Mack) * Allow to set message access and modification timestamps based on the "Date" header of the message itself. (Cyril Russo) +* "peritem" format string for [mbnames] got new expansion key + "localfolders" that corresponds to the same parameter of the + local repository for the account being processed. * [regression] pass folder names to the foldersort function, revert the documented behaviour diff --git a/offlineimap.conf b/offlineimap.conf index ec69bbb..836d075 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -129,6 +129,12 @@ accounts = Test # # The header, peritem, sep, and footer are all Python expressions passed # through eval, so you can (and must) use Python quoting. +# +# The following hash key are available to the expansion for 'peritem': +# - accountname: the name of the corresponding account; +# - foldername: the name of the folder; +# - localfolders: path to the local directory hosting all Maildir +# folders for the account. enabled = no filename = ~/Mutt/muttrc.mailboxes diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index c78d7d8..113d716 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -390,7 +390,8 @@ def syncfolder(account, remotefolder, quick): localfolder = account.get_local_folder(remotefolder) # Write the mailboxes - mbnames.add(account.name, localfolder.getname()) + mbnames.add(account.name, localfolder.getname(), + localrepos.getlocalroot()) # Load status folder. statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 34eccf9..39dcf03 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -21,6 +21,7 @@ import re # for folderfilter from threading import Lock boxes = {} +localroots = {} config = None accounts = None mblock = Lock() @@ -30,9 +31,10 @@ def init(conf, accts): config = conf accounts = accts -def add(accountname, foldername): +def add(accountname, foldername, localfolders): if not accountname in boxes: boxes[accountname] = [] + localroots[accountname] = localfolders if not foldername in boxes[accountname]: boxes[accountname].append(foldername) @@ -64,10 +66,12 @@ def genmbnames(): {'re': re}) itemlist = [] for accountname in boxes.keys(): + localroot = localroots[accountname] for foldername in boxes[accountname]: if folderfilter(accountname, foldername): itemlist.append({'accountname': accountname, - 'foldername': foldername}) + 'foldername': foldername, + 'localfolders': localroot}) itemlist.sort(key = mb_sort_keyfunc) format_string = config.get("mbnames", "peritem", raw=1) itemlist = [format_string % d for d in itemlist] diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index db80ecd..1050721 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -259,3 +259,8 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): for the threads to terminate.""" pass + def getlocalroot(self): + """ Local root folder for storing messages. + Will not be set for remote repositories.""" + return None + From d55e4ef15e9c0b82068a10944b59e08e0e94c858 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 27 Aug 2013 18:56:43 +0400 Subject: [PATCH 576/817] imaplib2: fix handling of zero-sized IMAP objects self._expecting_data was used both as the expected data length and the flag that we expect some data. This obviously fails when advertized data length is zero, so self._expecting_data_len was introduced to hold the length of the expected data and self._expecting_data was left as the flag that we expect the data to come. GitHub issue: https://github.com/OfflineIMAP/offlineimap/issues/15 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 1 + offlineimap/imaplib2.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 2d1b221..f4e18ed 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -28,6 +28,7 @@ WIP (add new stuff for the next release) local repository for the account being processed. * [regression] pass folder names to the foldersort function, revert the documented behaviour +* Fix handling of zero-sized IMAP data items (GitHub#15). OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index dfd6900..8a78864 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -299,7 +299,8 @@ class IMAP4(object): self.idle_rqb = None # Server IDLE Request - see _IdleCont self.idle_timeout = None # Must prod server occasionally - self._expecting_data = 0 # Expecting message data + self._expecting_data = False # Expecting message data + self._expecting_data_len = 0 # How many characters we expect self._accumulated_data = [] # Message data accumulated so far self._literal_expected = None # Message data descriptor @@ -1463,10 +1464,11 @@ class IMAP4(object): def _put_response(self, resp): - if self._expecting_data > 0: + if self._expecting_data: rlen = len(resp) - dlen = min(self._expecting_data, rlen) - self._expecting_data -= dlen + dlen = min(self._expecting_data_len, rlen) + self._expecting_data_len -= dlen + self._expecting_data = (self._expecting_data_len != 0) if rlen <= dlen: self._accumulated_data.append(resp) return @@ -1490,8 +1492,9 @@ class IMAP4(object): dat = resp if self._match(self.literal_cre, dat): self._literal_expected[1] = dat - self._expecting_data = int(self.mo.group('size')) - if __debug__: self._log(4, 'expecting literal size %s' % self._expecting_data) + self._expecting_data = True + self._expecting_data_len = int(self.mo.group('size')) + if __debug__: self._log(4, 'expecting literal size %s' % self._expecting_data_len) return typ = self._literal_expected[0] self._literal_expected = None @@ -1537,8 +1540,9 @@ class IMAP4(object): # Is there a literal to come? if self._match(self.literal_cre, dat): - self._expecting_data = int(self.mo.group('size')) - if __debug__: self._log(4, 'read literal size %s' % self._expecting_data) + self._expecting_data = True + self._expecting_data_len = int(self.mo.group('size')) + if __debug__: self._log(4, 'read literal size %s' % self._expecting_data_len) self._literal_expected = [typ, dat] return From f2c858330f0aa6e70f62af2a53dc7ec805a7b5fa Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 27 Aug 2013 09:41:18 +0400 Subject: [PATCH 577/817] Always show full tracebacks at the end of the run This greatly simplifies developer's life and will, possibly, allow users familiar with Python to debug and fix the problems by themselves. We, possibly, should not give tracebacks for the problems like "can't open connection", but this is up to the caller of this routine not to provide traceback in this case. Signed-off-by: Eygene Ryabinkin --- offlineimap/ui/UIBase.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 2c30b94..8558421 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -124,6 +124,11 @@ class UIBase(object): of the sync run when offlineiamp exits. It is recommended to always pass in exceptions if possible, so we can give the user the best debugging info. + + We are always pushing tracebacks to the exception queue to + make them to be output at the end of the run to allow users + pass sensible diagnostics to the developers or to solve + problems by themselves. One example of such a call might be: @@ -135,13 +140,14 @@ class UIBase(object): else: self._msg("ERROR: %s" % (exc)) + instant_traceback = exc_traceback if not self.debuglist: # only output tracebacks in debug mode - exc_traceback = None + instant_traceback = None # push exc on the queue for later output self.exc_queue.put((msg, exc, exc_traceback)) - if exc_traceback: - self._msg(traceback.format_tb(exc_traceback)) + if instant_traceback: + self._msg(traceback.format_tb(instant_traceback)) def registerthread(self, account): """Register current thread as being associated with an account name""" From 69765a3ef03725bad7900c86b5a086a213deda12 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 30 Aug 2013 21:44:46 +0400 Subject: [PATCH 578/817] Properly call error() function from ui.UIBase Second argument is exception traceback, not the message; without this tracebacks like mentioned in http://permalink.gmane.org/gmane.mail.imap.offlineimap.general/5712 were happening when this exception handling block was hit. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 755a31a..f8b79b4 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -384,9 +384,9 @@ class BaseFolder(object): raise # bubble severe errors up self.ui.error(e, exc_info()[2]) except Exception as e: - self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ - (uid, self.accountname, - exc_info()[2])) + self.ui.error(e, exc_info()[2], + msg="Copying message %s [acc: %s]" %\ + (uid, self.accountname)) raise #raise on unknown errors, so we can fix those def syncmessagesto_copy(self, dstfolder, statusfolder): From 1ef506655cd923e5a8f5db0ad81b73cf79c29f9b Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 27 Aug 2013 15:57:55 +0400 Subject: [PATCH 579/817] Catch two instances of untested regexp matches They could possibly lead to the problems mentioned in https://github.com/OfflineIMAP/offlineimap/issues/6 though there are no sound evidences for this. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Maildir.py | 2 ++ offlineimap/imaputil.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 23f48b4..3d6fd62 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -104,6 +104,8 @@ class MaildirFolder(BaseFolder): oldest_time_utc -= oldest_time_today_seconds timestampmatch = re_timestampmatch.search(messagename) + if not timestampmatch: + return True timestampstr = timestampmatch.group() timestamplong = long(timestampstr) if(timestamplong < oldest_time_utc): diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index fe69b7a..231fb14 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -145,6 +145,10 @@ def imapsplit(imapstring): elif workstr[0] == '"': # quoted fragments '"...\"..."' m = quotere.match(workstr) + if not m: + raise ValueError ("failed to parse " + "quoted component %s " % str(workstr) + \ + "while working with %s" % str(imapstring)) retval.append(m.group('quote')) workstr = m.group('rest') else: From 2df5b716b55ab2f6c0f8186aa961ca5a6eb7fa40 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 2 Sep 2013 09:08:50 +0400 Subject: [PATCH 580/817] Updated bundled imaplib2 to 2.35: - Fix for Gmail sending a BYE response after reading >100 messages in a session. - Includes fix for GitHub#15: patch was accepted upstream. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 4 ++++ offlineimap/imaplib2.py | 44 +++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index f4e18ed..aa40d59 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -29,6 +29,10 @@ WIP (add new stuff for the next release) * [regression] pass folder names to the foldersort function, revert the documented behaviour * Fix handling of zero-sized IMAP data items (GitHub#15). +* Updated bundled imaplib2 to 2.35: + - fix for Gmail sending a BYE response after reading >100 messages + in a session; + - includes fix for GitHub#15: patch was accepted upstream. OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 8a78864..0958166 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.33" +__version__ = "2.35" __release__ = "2" -__revision__ = "33" +__revision__ = "35" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -40,7 +40,9 @@ Time2Internaldate() patch to match RFC2060 specification of English month names starttls() bug fixed with the help of Sebastian Spaeth April 2011. Threads now set the "daemon" flag (suggested by offlineimap-project) April 2011. Single quoting introduced with the help of Vladimir Marek August 2011. -Support for specifying SSL version by Ryan Kavanagh July 2013.""" +Support for specifying SSL version by Ryan Kavanagh July 2013. +Fix for gmail "read 0" error provided by Jim Greenleaf August 2013. +Fix for offlineimap "indexerror: string index out of range" bug provided by Eygene Ryabinkin August 2013.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -1331,7 +1333,7 @@ class IMAP4(object): self.ouq.put(rqb) return rqb - # Must setup continuation expectancy *before* ouq.put + # Must setup continuation expectancy *before* ouq.put crqb = self._request_push(tag='continuation') self.ouq.put(rqb) @@ -1373,8 +1375,8 @@ class IMAP4(object): # Called for non-callback commands - typ, dat = rqb.get_response('command: %s => %%s' % rqb.name) self._check_bye() + typ, dat = rqb.get_response('command: %s => %%s' % rqb.name) if typ == 'BAD': if __debug__: self._print_log() raise self.error('%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) @@ -1903,11 +1905,12 @@ class IMAP4(object): if __debug__: def _init_debug(self, debug=None, debug_file=None, debug_buf_lvl=None): + self.debug_lock = threading.Lock() + self.debug = self._choose_nonull_or_dflt(0, debug, Debug) self.debug_file = self._choose_nonull_or_dflt(sys.stderr, debug_file) self.debug_buf_lvl = self._choose_nonull_or_dflt(DFLT_DEBUG_BUF_LVL, debug_buf_lvl) - self.debug_lock = threading.Lock() self._cmd_log_len = 20 self._cmd_log_idx = 0 self._cmd_log = {} # Last `_cmd_log_len' interactions @@ -2111,6 +2114,7 @@ class IMAP4_stream(IMAP4): from subprocess import Popen, PIPE + if __debug__: self._log(0, 'opening stream from command "%s"' % self.command) self._P = Popen(self.command, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True) self.writefile, self.readfile = self._P.stdin, self._P.stdout self.read_fd = self.readfile.fileno() @@ -2313,19 +2317,22 @@ if __name__ == '__main__': # To test: invoke either as 'python imaplib2.py [IMAP4_server_hostname]', # or as 'python imaplib2.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' # or as 'python imaplib2.py -l "keyfile[:certfile]" [IMAP4_SSL_server_hostname]' + # Option "-i" tests that IDLE is interruptible import getopt, getpass try: - optlist, args = getopt.getopt(sys.argv[1:], 'd:l:s:p:') + optlist, args = getopt.getopt(sys.argv[1:], 'd:il:s:p:') except getopt.error, val: optlist, args = (), () - debug, debug_buf_lvl, port, stream_command, keyfile, certfile = (None,)*6 + debug, debug_buf_lvl, port, stream_command, keyfile, certfile, idle_intr = (None,)*7 for opt,val in optlist: if opt == '-d': debug = int(val) debug_buf_lvl = debug - 1 + elif opt == '-i': + idle_intr = 1 elif opt == '-l': try: keyfile,certfile = val.split(':') @@ -2446,7 +2453,7 @@ if __name__ == '__main__': run('id', ()) run('id', ('("name", "imaplib2")',)) run('id', ("version", __version__, "os", os.uname()[0])) - + for cmd,args in test_seq2: if (cmd,args) != ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')): run(cmd, args) @@ -2476,6 +2483,16 @@ if __name__ == '__main__': M._mesg('fetch %s => %s' % (num, `dat`)) run('uid', ('STORE', num, 'FLAGS', '(\Deleted)')) run('expunge', ()) + if idle_intr: + M._mesg('HIT CTRL-C to interrupt IDLE') + try: + run('idle', (99,), cb=False) # Synchronous, to test interruption of 'idle' by INTR + except KeyboardInterrupt: + M._mesg('Thanks!') + M._mesg('') + raise + elif idle_intr: + M._mesg('chosen server does not report IDLE capability') run('logout', (), cb=False) @@ -2489,12 +2506,13 @@ if __name__ == '__main__': print 'All tests OK.' except: - print 'Tests failed.' + if not idle_intr or not 'IDLE' in M.capabilities: + print 'Tests failed.' - if not debug: - print ''' + if not debug: + print ''' If you would like to see debugging output, try: %s -d5 ''' % sys.argv[0] - raise + raise From be1c72ea5f7c26c14aa994df636c7ec2fdf7a5a9 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Thu, 19 Sep 2013 17:19:37 +0400 Subject: [PATCH 581/817] Updated bundled imaplib2 to 2.36 2.36 it includes support for SSL version override that was integrated into our code before, no other changes. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ offlineimap/imaplib2.py | 9 ++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index aa40d59..7ca2f1c 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -33,6 +33,9 @@ WIP (add new stuff for the next release) - fix for Gmail sending a BYE response after reading >100 messages in a session; - includes fix for GitHub#15: patch was accepted upstream. +* Updated bundled imaplib2 to 2.36: it includes support for SSL + version override that was integrated into our code before, + no other changes. OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 0958166..14a802b 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.35" +__version__ = "2.36" __release__ = "2" -__revision__ = "35" +__revision__ = "36" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -474,8 +474,7 @@ class IMAP4(object): elif self.ssl_version == "ssl23" or self.ssl_version is None: ssl_version = ssl.PROTOCOL_SSLv23 else: - raise socket.sslerror("Invalid SSL version requested: %s", - self.ssl_version) + raise socket.sslerror("Invalid SSL version requested: %s", self.ssl_version) self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version) ssl_exc = ssl.SSLError @@ -1994,7 +1993,7 @@ class IMAP4_SSL(IMAP4): """IMAP4 client class over SSL connection Instantiate with: - IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None) + IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None) host - host's name (default: localhost); port - port number (default: standard IMAP4 SSL port); From 57adfc23a5a1f0494da647ca4881d1616003eb35 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Thu, 19 Sep 2013 22:56:55 +0400 Subject: [PATCH 582/817] Fix parsing of quoted strings When imaputil was parsing quoted strings, it treated "abcd\\" as incomplete quoted string having escaped quote, rather than properly-quoted string having escaped backslash. GitHub issue: https://github.com/OfflineIMAP/offlineimap/issues/53 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ offlineimap/imaputil.py | 56 ++++++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 7ca2f1c..55fe78b 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -36,6 +36,9 @@ WIP (add new stuff for the next release) * Updated bundled imaplib2 to 2.36: it includes support for SSL version override that was integrated into our code before, no other changes. +* Fixed parsing of quoted strings in IMAP responses: strings like "\\" + were treated as having \" as the escaped quote, rather than treating + it as the quoted escaped backslash (GitHub#53). OfflineIMAP v6.5.5-rc1 (2012-09-05) =================================== diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 231fb14..c6e660c 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -21,13 +21,6 @@ import string from offlineimap.ui import getglobalui -# find the first quote in a string -quotere = re.compile( - r"""(?P"[^\"\\]*(?:\\"|[^"])*") # Quote, possibly containing encoded - # quotation mark - \s*(?P.*)$ # Whitespace & remainder of string""", - re.VERBOSE) - def debug(*args): msg = [] for arg in args: @@ -144,13 +137,9 @@ def imapsplit(imapstring): retval.append(parenlist) elif workstr[0] == '"': # quoted fragments '"...\"..."' - m = quotere.match(workstr) - if not m: - raise ValueError ("failed to parse " - "quoted component %s " % str(workstr) + \ - "while working with %s" % str(imapstring)) - retval.append(m.group('quote')) - workstr = m.group('rest') + (quoted, rest) = _split_quoted(workstr) + retval.append(quoted) + workstr = rest else: splits = string.split(workstr, maxsplit = 1) splitslen = len(splits) @@ -222,3 +211,42 @@ def uid_sequence(uidlist): retval.append(getrange(start, end)) # Add final range/item return ",".join(retval) + + +def _split_quoted(string): + """ + Looks for the ending quote character in the string that starts + with quote character, splitting out quoted component and the + rest of the string (without possible space between these two + parts. + + First character of the string is taken to be quote character. + + Examples: + - "this is \" a test" (\\None) => ("this is \" a test", (\\None)) + - "\\" => ("\\", ) + + """ + + if len(string) == 0: + return ('', '') + + q = quoted = string[0] + rest = string[1:] + while True: + next_q = rest.find(q) + if next_q == -1: + raise ValueError("can't find ending quote '%s' in '%s'" % (q, string)) + # If quote is preceeded by even number of backslashes, + # then it is the ending quote, otherwise the quote + # character is escaped by backslash, so we should + # continue our search. + is_escaped = False + i = next_q - 1 + while i >= 0 and rest[i] == '\\': + i -= 1 + is_escaped = not is_escaped + quoted += rest[0:next_q + 1] + rest = rest[next_q + 1:] + if not is_escaped: + return (quoted, rest.lstrip()) From 254e84814064299669f7d295456a3572afcdb7e3 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 20 Sep 2013 14:34:14 +0400 Subject: [PATCH 583/817] Rolling in release candidate #3 for 6.5.5 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 6 +++--- offlineimap/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 55fe78b..f74d073 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -4,8 +4,8 @@ ChangeLog :website: http://offlineimap.org -WIP (add new stuff for the next release) -======================================== +OfflineIMAP v6.5.5-rc3 (2013-09-20) +=================================== * Avoid lockups for IMAP synchronizations running with the "-1" command-line switch (X-Ryl669 ) @@ -40,7 +40,7 @@ WIP (add new stuff for the next release) were treated as having \" as the escaped quote, rather than treating it as the quoted escaped backslash (GitHub#53). -OfflineIMAP v6.5.5-rc1 (2012-09-05) +OfflineIMAP v6.5.5-rc2 (2012-09-05) =================================== * Execute pre/post-sync hooks during synchronizations diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 8152284..1814f7c 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,8 +1,8 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.5-rc2" -__copyright__ = "Copyright 2002-2012 John Goerzen & contributors" +__version__ = "6.5.5-rc3" +__copyright__ = "Copyright 2002-2013 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" From 8bc2f35bf683f24448132b2f9738aaf809aff818 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 4 Oct 2013 17:03:06 +0400 Subject: [PATCH 584/817] OfflineIMAP 6.5.5 is out Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 23 ++++++++--------------- offlineimap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index f74d073..db4eb24 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -4,8 +4,8 @@ ChangeLog :website: http://offlineimap.org -OfflineIMAP v6.5.5-rc3 (2013-09-20) -=================================== +OfflineIMAP v6.5.5 (2013-10-07) +=============================== * Avoid lockups for IMAP synchronizations running with the "-1" command-line switch (X-Ryl669 ) @@ -39,28 +39,21 @@ OfflineIMAP v6.5.5-rc3 (2013-09-20) * Fixed parsing of quoted strings in IMAP responses: strings like "\\" were treated as having \" as the escaped quote, rather than treating it as the quoted escaped backslash (GitHub#53). - -OfflineIMAP v6.5.5-rc2 (2012-09-05) -=================================== - * Execute pre/post-sync hooks during synchronizations toggled by IMAP IDLE message processing. (maxgerer@gmail.com) * Catch unsuccessful local mail uploads when IMAP server responds with "NO" status; that resulted in a loss of such local messages. (Adam Spiers) - -OfflineIMAP v6.5.5-rc1 (2012-09-05) -=================================== - * Don't create folders if readonly is enabled. -* Learn to deal with readonly folders to properly detect this condition and act - accordingly. One example is Gmail's "Chats" folder that is read-only, - but contains logs of the quick chats. (E. Ryabinkin) +* Learn to deal with readonly folders to properly detect this + condition and act accordingly. One example is Gmail's "Chats" + folder that is read-only, but contains logs of the quick chats. (E. + Ryabinkin) * Fix str.format() calls for Python 2.6 (D. Logie) * Remove APPENDUID hack, previously introduced to fix Gmail, no longer necessary, it might have been breaking things. (J. Wiegley) -* Improve regex that could lead to 'NoneType' object has no attribute 'group' - (D. Franke) +* Improve regex that could lead to 'NoneType' object has no attribute + 'group' (D. Franke) * Improved error throwing on repository misconfiguration OfflineIMAP v6.5.4 (2012-06-02) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 1814f7c..60c79a3 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.5-rc3" +__version__ = "6.5.5" __copyright__ = "Copyright 2002-2013 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From aeeea56cb16fa38953d2567173e8b5ca9caf7338 Mon Sep 17 00:00:00 2001 From: Dan Milon Date: Tue, 10 Dec 2013 20:02:26 +0200 Subject: [PATCH 585/817] docs: fix typos --- docs/doc-src/nametrans.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/doc-src/nametrans.rst b/docs/doc-src/nametrans.rst index c211072..f6b9924 100644 --- a/docs/doc-src/nametrans.rst +++ b/docs/doc-src/nametrans.rst @@ -13,7 +13,7 @@ safely skip this section. folderfilter ------------ -If you do not want to synchronize all your filters, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. +If you do not want to synchronize all your folders, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. The only parameter to that function is the folder name. If the filter function returns True, the folder will be synced, if it returns False, @@ -114,7 +114,7 @@ Reverse nametrans Since 6.4.0, OfflineImap supports the creation of folders on the remote repository and that complicates things. Previously, only one nametrans setting on the remote repository was needed and that transformed a remote to a local name. However, nametrans transformations are one-way, and OfflineImap has no way using those rules on the remote repository to back local names to remote names. -Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts of any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. +Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts off any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. Take the above examples. If your remote nametrans setting was:: @@ -122,7 +122,7 @@ Take the above examples. If your remote nametrans setting was:: then you will want to have this in your local repository, prepending "INBOX" to any local folder name:: - nametrans = lambda folder: 'INBOX' + folder + nametrans = lambda folder: 'INBOX.' + folder Failure to set the local nametrans rule will lead to weird-looking error messages of -for instance- this type:: From e3fbe0232b302d77de621159e0cef9fb39636790 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 14 Dec 2013 14:46:41 +0100 Subject: [PATCH 586/817] docs: add "development state" section and update some links Signed-off-by: Nicolas Sebrecht --- docs/INSTALL.rst | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index 83608d4..d4670c5 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -9,6 +9,22 @@ Installation .. contents:: .. .. sectnum:: +----------------- +Development state +----------------- + +Since several months, the official maintainers Nicolas Sebrecht and Sebatian +Spaeth are too much busy to actively contribute to this project. + +In order to preserve contributions, a team of official maintainers have been +promoted to have write access to the official repository at `OfflineIMAP`_. +OfflineIMAP is now maintained by occasional contributors and the official +maintainers. All the documentation links might not be up-to-date to reflect +this change. + +The best place to get the latest news about the development state is at the +mailing list. + ------------- Prerequisites ------------- @@ -80,7 +96,7 @@ Installation from git checkout Get your own copy of the `official git repository `_ at `OfflineIMAP`_:: - git clone git://github.com/spaetz/offlineimap.git + git clone https://github.com/OfflineIMAP/offlineimap.git This will download the source with history. By default, git sets up the `master` branch up, which is most likely what you want. If not, you can @@ -108,7 +124,7 @@ Finally, install the program (as root):: python setup.py install -Next, proceed to below. You tofflineimap to invoke the program. +Next, proceed to below. Type `offlineimap` to invoke the program. .. _single_user_inst: From b01274ce3838404436caab077ad619648ecc7f0a Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 26 Feb 2014 17:51:48 +0400 Subject: [PATCH 587/817] Fix double release of IMAP connection object This commit fixes the case when we're invoking releaseconnection() for a given imapobj twice. This bug manifests itself as {{{ ValueError: list.remove(x): x not in list File "[...]/offlineimap/folder/IMAP.py", line 615, in savemessage self.imapserver.releaseconnection(imapobj) File "[...]/offlineimap/imapserver.py", line 130, in releaseconnection self.assignedconnections.remove(connection) }}} Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/IMAP.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index cec9eda..1f3803c 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -500,6 +500,11 @@ class IMAPFolder(BaseFolder): retry_left = 2 # succeeded in APPENDING? imapobj = self.imapserver.acquireconnection() + # NB: in the finally clause for this try we will release + # NB: the acquired imapobj, so don't do that twice unless + # NB: you will put another connection to imapobj. If you + # NB: really do need to release connection manually, set + # NB: imapobj to None. try: while retry_left: # UIDPLUS extension provides us with an APPENDUID response. @@ -612,7 +617,7 @@ class IMAPFolder(BaseFolder): self.ui.warn('imap', "savemessage: Searching mails for new " "Message-ID failed. Could not determine new UID.") finally: - self.imapserver.releaseconnection(imapobj) + if imapobj: self.imapserver.releaseconnection(imapobj) if uid: # avoid UID FETCH 0 crash happening later on self.messagelist[uid] = {'uid': uid, 'flags': flags} From 829c9cf4be168bf6c8ab0ba303adac23c0bc310e Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 2 Mar 2014 16:37:41 +0400 Subject: [PATCH 588/817] Introduce dynamic filtering for repository folders Allow people who want folder filtering to depend on the external conditions or to make it dynamic for other reasons, to do what they want. New repository configuration knob 'dynamic_folderfilter' was introduced; it defaults to 'False' that matches historical behaviour. GitHub: #73 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 7 +++++++ offlineimap.conf | 6 ++++++ offlineimap/folder/Base.py | 26 ++++++++++++++++++++------ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index db4eb24..e541358 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -4,6 +4,13 @@ ChangeLog :website: http://offlineimap.org + +OfflineIMAP v6.5.6 (YYYY-MM-DD) +=============================== + +* Add knob to invoke folderfilter dynamically on each sync (GitHub#73) + + OfflineIMAP v6.5.5 (2013-10-07) =============================== diff --git a/offlineimap.conf b/offlineimap.conf index 836d075..0a8d44c 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -540,6 +540,12 @@ remoteuser = username # # nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) +# Determines if folderfilter will be invoked on each run +# (dynamic folder filtering) or filtering status will be determined +# at startup (default behaviour). +# +# dynamic_folderfilter = False + # You can specify which folders to sync using the folderfilter # setting. You can provide any python function (e.g. a lambda function) # which will be invoked for each foldername. If the filter function diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index f8b79b4..b26e169 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -33,11 +33,8 @@ class BaseFolder(object): :para repository: Repository() in which the folder is. """ self.ui = getglobalui() - """Should this folder be included in syncing?""" - self._sync_this = repository.should_sync_folder(name) - if not self._sync_this: - self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter" \ - % (name, repository)) + # Save original name for folderfilter operations + self.ffilter_name = name # Top level dir name is always '' self.name = name if not name == self.getsep() else '' self.repository = repository @@ -47,6 +44,7 @@ class BaseFolder(object): # return for the top-level dir. if self.visiblename == self.getsep(): self.visiblename = '' + self.config = repository.getconfig() utime_from_message_global = \ self.config.getdefaultboolean("general", @@ -56,6 +54,19 @@ class BaseFolder(object): self.config.getdefaultboolean(repo, "utime_from_message", utime_from_message_global) + # Determine if we're running static or dynamic folder filtering + # and check filtering status + self._dynamic_folderfilter = \ + self.config.getdefaultboolean(repo, "dynamic_folderfilter", False) + self._sync_this = repository.should_sync_folder(self.ffilter_name) + if self._dynamic_folderfilter: + self.ui.debug('', "Running dynamic folder filtering on '%s'[%s]" \ + % (self.ffilter_name, repository)) + elif not self._sync_this: + self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter" \ + % (self.ffilter_name, repository)) + + def getname(self): """Returns name""" return self.name @@ -71,7 +82,10 @@ class BaseFolder(object): @property def sync_this(self): """Should this folder be synced or is it e.g. filtered out?""" - return self._sync_this + if not self._dynamic_folderfilter: + return self._sync_this + else: + return this.repository.should_sync_folder(self.ffilter_name) @property def utime_from_message(self): From af2d6dc5e11c658dc2fbd60f88cac509e5636c19 Mon Sep 17 00:00:00 2001 From: Stefan Huber Date: Wed, 5 Mar 2014 19:32:36 +0100 Subject: [PATCH 589/817] Fix: Undefined variable 'this' should be 'self' Pointyhat-to: Eygene Ryabinkin Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index b26e169..e9d4a9c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -85,7 +85,7 @@ class BaseFolder(object): if not self._dynamic_folderfilter: return self._sync_this else: - return this.repository.should_sync_folder(self.ffilter_name) + return self.repository.should_sync_folder(self.ffilter_name) @property def utime_from_message(self): From 6cbd2498ae483ba332ccebef547defb9bedb9509 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 16 Mar 2014 16:27:35 +0400 Subject: [PATCH 590/817] Refactoring: make functions to be private if we can Make external API of class/module to be smaller, explicitely mark all internal functions. Also annotate methods that are implemented as the part of the parent class interface. Signed-off-by: Eygene Ryabinkin --- docs/doc-src/API.rst | 2 +- offlineimap/accounts.py | 16 ++--- offlineimap/folder/Base.py | 38 +++++++--- offlineimap/folder/IMAP.py | 92 +++++++++++++++---------- offlineimap/folder/LocalStatus.py | 13 ++++ offlineimap/folder/LocalStatusSQLite.py | 30 +++++--- offlineimap/folder/Maildir.py | 23 +++++-- offlineimap/folder/UIDMaps.py | 17 +++++ offlineimap/imaplibutil.py | 5 +- offlineimap/imapserver.py | 73 ++++++++++---------- offlineimap/imaputil.py | 20 +++--- offlineimap/init.py | 16 ++--- offlineimap/mbnames.py | 4 +- test/tests/test_00_imaputil.py | 5 -- 14 files changed, 218 insertions(+), 136 deletions(-) diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index c456435..1eb9adb 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -30,7 +30,7 @@ be merged into the main documentation. .. automethod:: run - .. automethod:: parse_cmd_options + .. automethod:: __parse_cmd_options .. .. autoattribute:: ui diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 113d716..2c9551d 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -134,7 +134,7 @@ class Account(CustomConfig.ConfigHelperMixin): return skipsleep or Account.abort_soon_signal.is_set() or \ Account.abort_NOW_signal.is_set() - def sleeper(self): + def _sleeper(self): """Sleep if the account is set to autorefresh :returns: 0:timeout expired, 1: canceled the timer, @@ -192,7 +192,7 @@ class SyncableAccount(Account): self._lockfilepath = os.path.join(self.config.getmetadatadir(), "%s.lock" % self) - def lock(self): + def __lock(self): """Lock the account, throwing an exception if it is locked already""" self._lockfd = open(self._lockfilepath, 'w') try: @@ -206,7 +206,7 @@ class SyncableAccount(Account): "instance using this account?" % self, OfflineImapError.ERROR.REPO) - def unlock(self): + def _unlock(self): """Unlock the account, deleting the lock file""" #If we own the lock file, delete it if self._lockfd and not self._lockfd.closed: @@ -237,8 +237,8 @@ class SyncableAccount(Account): while looping: self.ui.acct(self) try: - self.lock() - self.sync() + self.__lock() + self.__sync() except (KeyboardInterrupt, SystemExit): raise except OfflineImapError as e: @@ -258,8 +258,8 @@ class SyncableAccount(Account): looping = 3 finally: self.ui.acctdone(self) - self.unlock() - if looping and self.sleeper() >= 2: + self._unlock() + if looping and self._sleeper() >= 2: looping = 0 def get_local_folder(self, remotefolder): @@ -268,7 +268,7 @@ class SyncableAccount(Account): remotefolder.getvisiblename(). replace(self.remoterepos.getsep(), self.localrepos.getsep())) - def sync(self): + def __sync(self): """Synchronize the account once, then return Assumes that `self.remoterepos`, `self.localrepos`, and diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index e9d4a9c..0062dc2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -96,6 +96,22 @@ class BaseFolder(object): false otherwise. Probably only IMAP will return true.""" return 0 + def waitforthread(self): + """Implements method that waits for thread to be usable. + Should be implemented only for folders that suggest threads.""" + raise NotImplementedException + + # XXX: we may need someting like supports_quickstatus() to check + # XXX: if user specifies 'quick' flag for folder that doesn't + # XXX: support quick status queries, so one believes that quick + # XXX: status checks will be done, but it won't really be so. + def quickchanged(self, statusfolder): + """ Runs quick check for folder changes and returns changed + status: True -- changed, False -- not changed. + :param statusfolder: keeps track of the last known folder state. + """ + return True + def getcopyinstancelimit(self): """For threading folders, returns the instancelimitname for InstanceLimitedThreads.""" @@ -328,7 +344,7 @@ class BaseFolder(object): for uid in uidlist: self.deletemessage(uid) - def copymessageto(self, uid, dstfolder, statusfolder, register = 1): + def __copymessageto(self, uid, dstfolder, statusfolder, register = 1): """Copies a message from self to dst if needed, updating the status Note that this function does not check against dryrun settings, @@ -403,14 +419,14 @@ class BaseFolder(object): (uid, self.accountname)) raise #raise on unknown errors, so we can fix those - def syncmessagesto_copy(self, dstfolder, statusfolder): + def __syncmessagesto_copy(self, dstfolder, statusfolder): """Pass1: Copy locally existing messages not on the other side This will copy messages to dstfolder that exist locally but are not in the statusfolder yet. The strategy is: 1) Look for messages present in self but not in statusfolder. - 2) invoke copymessageto() on those which: + 2) invoke __copymessageto() on those which: - If dstfolder doesn't have it yet, add them to dstfolder. - Update statusfolder @@ -431,23 +447,23 @@ class BaseFolder(object): if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) - # exceptions are caught in copymessageto() + # exceptions are caught in __copymessageto() if self.suggeststhreads() and not globals.options.singlethreading: self.waitforthread() thread = threadutil.InstanceLimitedThread(\ self.getcopyinstancelimit(), - target = self.copymessageto, + target = self.__copymessageto, name = "Copy message from %s:%s" % (self.repository, self), args = (uid, dstfolder, statusfolder)) thread.start() threads.append(thread) else: - self.copymessageto(uid, dstfolder, statusfolder, + self.__copymessageto(uid, dstfolder, statusfolder, register = 0) for thread in threads: thread.join() - def syncmessagesto_delete(self, dstfolder, statusfolder): + def __syncmessagesto_delete(self, dstfolder, statusfolder): """Pass 2: Remove locally deleted messages on dst Get all UIDS in statusfolder but not self. These are messages @@ -468,7 +484,7 @@ class BaseFolder(object): for folder in [statusfolder, dstfolder]: folder.deletemessages(deletelist) - def syncmessagesto_flags(self, dstfolder, statusfolder): + def __syncmessagesto_flags(self, dstfolder, statusfolder): """Pass 3: Flag synchronization Compare flag mismatches in self with those in statusfolder. If @@ -551,9 +567,9 @@ class BaseFolder(object): :param dstfolder: Folderinstance to sync the msgs to. :param statusfolder: LocalStatus instance to sync against. """ - passes = [('copying messages' , self.syncmessagesto_copy), - ('deleting messages' , self.syncmessagesto_delete), - ('syncing flags' , self.syncmessagesto_flags)] + passes = [('copying messages' , self.__syncmessagesto_copy), + ('deleting messages' , self.__syncmessagesto_delete), + ('syncing flags' , self.__syncmessagesto_flags)] for (passdesc, action) in passes: # bail out on CTRL-C or SIGTERM diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 1f3803c..d727b31 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -38,7 +38,7 @@ class IMAPFolder(BaseFolder): self.randomgenerator = random.Random() #self.ui is set in BaseFolder - def selectro(self, imapobj, force = False): + def __selectro(self, imapobj, force = False): """Select this folder when we do not need write access. Prefer SELECT to EXAMINE if we can, since some servers @@ -52,15 +52,19 @@ class IMAPFolder(BaseFolder): except imapobj.readonly: imapobj.select(self.getfullname(), readonly = True, force = force) + # Interface from BaseFolder def suggeststhreads(self): return not globals.options.singlethreading + # Interface from BaseFolder def waitforthread(self): self.imapserver.connectionwait() + # Interface from BaseFolder def getcopyinstancelimit(self): return 'MSGCOPY_' + self.repository.getname() + # Interface from BaseFolder def get_uidvalidity(self): """Retrieve the current connections UIDVALIDITY value @@ -72,7 +76,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: # SELECT (if not already done) and get current UIDVALIDITY - self.selectro(imapobj) + self.__selectro(imapobj) typ, uidval = imapobj.response('UIDVALIDITY') assert uidval != [None] and uidval != None, \ "response('UIDVALIDITY') returned [None]!" @@ -81,6 +85,7 @@ class IMAPFolder(BaseFolder): finally: self.imapserver.releaseconnection(imapobj) + # Interface from BaseFolder def quickchanged(self, statusfolder): # An IMAP folder has definitely changed if the number of # messages or the UID of the last message have changed. Otherwise @@ -118,6 +123,7 @@ class IMAPFolder(BaseFolder): return True return False + # Interface from BaseFolder def cachemessagelist(self): maxage = self.config.getdefaultint("Account %s" % self.accountname, "maxage", -1) @@ -199,9 +205,11 @@ class IMAPFolder(BaseFolder): rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + # Interface from BaseFolder def getmessagelist(self): return self.messagelist + # Interface from BaseFolder def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body) @@ -253,13 +261,15 @@ class IMAPFolder(BaseFolder): self.imapserver.releaseconnection(imapobj) return data + # Interface from BaseFolder def getmessagetime(self, uid): return self.messagelist[uid]['time'] + # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] - def generate_randomheader(self, content): + def __generate_randomheader(self, content): """Returns a unique X-OfflineIMAP header Generate an 'X-OfflineIMAP' mail header which contains a random @@ -286,28 +296,28 @@ class IMAPFolder(BaseFolder): return (headername, headervalue) - def savemessage_addheader(self, content, headername, headervalue): + def __savemessage_addheader(self, content, headername, headervalue): self.ui.debug('imap', - 'savemessage_addheader: called to add %s: %s' % (headername, + '__savemessage_addheader: called to add %s: %s' % (headername, headervalue)) insertionpoint = content.find("\r\n\r\n") - self.ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint) + self.ui.debug('imap', '__savemessage_addheader: insertionpoint = %d' % insertionpoint) leader = content[0:insertionpoint] - self.ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader)) + self.ui.debug('imap', '__savemessage_addheader: leader = %s' % repr(leader)) if insertionpoint == 0 or insertionpoint == -1: newline = '' insertionpoint = 0 else: newline = "\r\n" newline += "%s: %s" % (headername, headervalue) - self.ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline)) + self.ui.debug('imap', '__savemessage_addheader: newline = ' + repr(newline)) trailer = content[insertionpoint:] - self.ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer)) + self.ui.debug('imap', '__savemessage_addheader: trailer = ' + repr(trailer)) return leader + newline + trailer - def savemessage_searchforheader(self, imapobj, headername, headervalue): - self.ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \ + def __savemessage_searchforheader(self, imapobj, headername, headervalue): + self.ui.debug('imap', '__savemessage_searchforheader called for %s: %s' % \ (headername, headervalue)) # Now find the UID it got. headervalue = imapobj._quote(headervalue) @@ -315,16 +325,16 @@ class IMAPFolder(BaseFolder): matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0] except imapobj.error as err: # IMAP server doesn't implement search or had a problem. - self.ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername)) + self.ui.debug('imap', "__savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername)) return 0 - self.ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids)) + self.ui.debug('imap', '__savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids)) if matchinguids == '': - self.ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername) + self.ui.debug('imap', "__savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername) return 0 matchinguids = matchinguids.split(' ') - self.ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \ + self.ui.debug('imap', '__savemessage_searchforheader: matchinguids now ' + \ repr(matchinguids)) if len(matchinguids) != 1 or matchinguids[0] == None: raise ValueError("While attempting to find UID for message with " @@ -332,7 +342,7 @@ class IMAPFolder(BaseFolder): (headername, str(matchinguids))) return long(matchinguids[0]) - def savemessage_fetchheaders(self, imapobj, headername, headervalue): + def __savemessage_fetchheaders(self, imapobj, headername, headervalue): """ We fetch all new mail headers and search for the right X-OfflineImap line by hand. The response from the server has form: ( @@ -355,7 +365,7 @@ class IMAPFolder(BaseFolder): Returns UID when found, 0 when not found. """ - self.ui.debug('imap', 'savemessage_fetchheaders called for %s: %s' % \ + self.ui.debug('imap', '__savemessage_fetchheaders called for %s: %s' % \ (headername, headervalue)) # run "fetch X:* rfc822.header" @@ -401,7 +411,7 @@ class IMAPFolder(BaseFolder): return 0 - def getmessageinternaldate(self, content, rtime=None): + def __getmessageinternaldate(self, content, rtime=None): """Parses mail and returns an INTERNALDATE string It will use information in the following order, falling back as an attempt fails: @@ -474,6 +484,7 @@ class IMAPFolder(BaseFolder): return internaldate + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Save the message on the Server @@ -511,16 +522,16 @@ class IMAPFolder(BaseFolder): use_uidplus = 'UIDPLUS' in imapobj.capabilities # get the date of the message, so we can pass it to the server. - date = self.getmessageinternaldate(content, rtime) + date = self.__getmessageinternaldate(content, rtime) content = re.sub("(?200: dbg_output = "%s...%s" % (content[:150], content[-50:]) @@ -605,14 +616,14 @@ class IMAPFolder(BaseFolder): "'%s'" % str(resp)) else: # we don't support UIDPLUS - uid = self.savemessage_searchforheader(imapobj, headername, + uid = self.__savemessage_searchforheader(imapobj, headername, headervalue) # See docs for savemessage in Base.py for explanation # of this and other return values if uid == 0: self.ui.debug('imap', 'savemessage: attempt to get new UID ' 'UID failed. Search headers manually.') - uid = self.savemessage_fetchheaders(imapobj, headername, + uid = self.__savemessage_fetchheaders(imapobj, headername, headervalue) self.ui.warn('imap', "savemessage: Searching mails for new " "Message-ID failed. Could not determine new UID.") @@ -625,6 +636,7 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage: returning new UID %d' % uid) return uid + # Interface from BaseFolder def savemessageflags(self, uid, flags): """Change a message's flags to `flags`. @@ -650,29 +662,34 @@ class IMAPFolder(BaseFolder): flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS'] self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags) + # Interface from BaseFolder def addmessageflags(self, uid, flags): self.addmessagesflags([uid], flags) - def addmessagesflags_noconvert(self, uidlist, flags): - self.processmessagesflags('+', uidlist, flags) + def __addmessagesflags_noconvert(self, uidlist, flags): + self.__processmessagesflags('+', uidlist, flags) + # Interface from BaseFolder def addmessagesflags(self, uidlist, flags): """This is here for the sake of UIDMaps.py -- deletemessages must add flags and get a converted UID, and if we don't have noconvert, then UIDMaps will try to convert it twice.""" - self.addmessagesflags_noconvert(uidlist, flags) + self.__addmessagesflags_noconvert(uidlist, flags) + # Interface from BaseFolder def deletemessageflags(self, uid, flags): self.deletemessagesflags([uid], flags) + # Interface from BaseFolder def deletemessagesflags(self, uidlist, flags): - self.processmessagesflags('-', uidlist, flags) + self.__processmessagesflags('-', uidlist, flags) - def processmessagesflags(self, operation, uidlist, flags): + def __processmessagesflags(self, operation, uidlist, flags): + # XXX: should really iterate over batches of 100 UIDs if len(uidlist) > 101: - # Hack for those IMAP ervers with a limited line length - self.processmessagesflags(operation, uidlist[:100], flags) - self.processmessagesflags(operation, uidlist[100:], flags) + # Hack for those IMAP servers with a limited line length + self.__processmessagesflags(operation, uidlist[:100], flags) + self.__processmessagesflags(operation, uidlist[100:], flags) return imapobj = self.imapserver.acquireconnection() @@ -716,6 +733,7 @@ class IMAPFolder(BaseFolder): elif operation == '-': self.messagelist[uid]['flags'] -= flags + # Interface from BaseFolder def change_message_uid(self, uid, new_uid): """Change the message from existing uid to new_uid @@ -724,19 +742,21 @@ class IMAPFolder(BaseFolder): '%d to %d' % (uid, new_uid), OfflineImapError.ERROR.MESSAGE) + # Interface from BaseFolder def deletemessage(self, uid): - self.deletemessages_noconvert([uid]) + self.__deletemessages_noconvert([uid]) + # Interface from BaseFolder def deletemessages(self, uidlist): - self.deletemessages_noconvert(uidlist) + self.__deletemessages_noconvert(uidlist) - def deletemessages_noconvert(self, uidlist): + def __deletemessages_noconvert(self, uidlist): # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if self.uidexists(uid)] if not len(uidlist): return - self.addmessagesflags_noconvert(uidlist, set('T')) + self.__addmessagesflags_noconvert(uidlist, set('T')) imapobj = self.imapserver.acquireconnection() try: try: @@ -750,5 +770,3 @@ class IMAPFolder(BaseFolder): self.imapserver.releaseconnection(imapobj) for uid in uidlist: del self.messagelist[uid] - - diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 6f65f83..c3f24f6 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -33,21 +33,26 @@ class LocalStatusFolder(BaseFolder): False) """Should we perform fsyncs as often as possible?""" + # Interface from BaseFolder def storesmessages(self): return 0 def isnewfolder(self): return not os.path.exists(self.filename) + # Interface from BaseFolder def getname(self): return self.name + # Interface from BaseFolder def getroot(self): return self.repository.root + # Interface from BaseFolder def getsep(self): return self.sep + # Interface from BaseFolder def getfullname(self): return self.filename @@ -55,6 +60,7 @@ class LocalStatusFolder(BaseFolder): if not self.isnewfolder(): os.unlink(self.filename) + # Interface from BaseFolder def cachemessagelist(self): if self.isnewfolder(): self.messagelist = {} @@ -103,9 +109,11 @@ class LocalStatusFolder(BaseFolder): os.fsync(fd) os.close(fd) + # Interface from BaseFolder def getmessagelist(self): return self.messagelist + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. @@ -124,19 +132,24 @@ class LocalStatusFolder(BaseFolder): self.save() return uid + # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] + # Interface from BaseFolder def getmessagetime(self, uid): return self.messagelist[uid]['time'] + # Interface from BaseFolder def savemessageflags(self, uid, flags): self.messagelist[uid]['flags'] = flags self.save() + # Interface from BaseFolder def deletemessage(self, uid): self.deletemessages([uid]) + # Interface from BaseFolder def deletemessages(self, uidlist): # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if uid in self.messagelist] diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index b624134..a22c68b 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -62,14 +62,14 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): cursor = self.connection.execute("SELECT value from metadata WHERE key='db_version'") except sqlite.DatabaseError: #db file missing or corrupt, recreate it. - self.upgrade_db(0) + self.__upgrade_db(0) else: # fetch db version and upgrade if needed version = int(cursor.fetchone()[0]) if version < LocalStatusSQLiteFolder.cur_version: - self.upgrade_db(version) + self.__upgrade_db(version) - def sql_write(self, sql, vars=None, executemany=False): + def __sql_write(self, sql, vars=None, executemany=False): """Execute some SQL, retrying if the db was locked. :param sql: the SQL string passed to execute() @@ -106,7 +106,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): self._dblock.release() return cursor - def upgrade_db(self, from_ver): + def __upgrade_db(self, from_ver): """Upgrade the sqlite format from version 'from_ver' to current""" if hasattr(self, 'connection'): @@ -116,7 +116,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): if from_ver == 0: # from_ver==0: no db existent: plain text migration? - self.create_db() + self.__create_db() # below was derived from repository.getfolderfilename() logic plaintextfilename = os.path.join( self.repository.account.getaccountmeta(), @@ -144,7 +144,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): # if from_ver <= 1: ... #upgrade from 1 to 2 # if from_ver <= 2: ... #upgrade from 2 to 3 - def create_db(self): + def __create_db(self): """Create a new db file""" self.ui._msg('Creating new Local Status db for %s:%s' \ % (self.repository, self)) @@ -158,16 +158,19 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): """) self.connection.commit() + # Interface from LocalStatusFolder def isnewfolder(self): # testing the existence of the db file won't work. It is created # as soon as this class instance was intitiated. So say it is a # new folder when there are no messages at all recorded in it. return self.getmessagecount() > 0 + # Interface from LocalStatusFolder def deletemessagelist(self): """delete all messages in the db""" - self.sql_write('DELETE FROM status') + self.__sql_write('DELETE FROM status') + # Interface from BaseFolder def cachemessagelist(self): self.messagelist = {} cursor = self.connection.execute('SELECT id,flags from status') @@ -175,6 +178,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): flags = set(row[1]) self.messagelist[row[0]] = {'uid': row[0], 'flags': flags} + # Interface from LocalStatusFolder def save(self): #Noop in this backend pass @@ -215,6 +219,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): # return flags # assert False,"getmessageflags() called on non-existing message" + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. @@ -231,21 +236,24 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} flags = ''.join(sorted(flags)) - self.sql_write('INSERT INTO status (id,flags) VALUES (?,?)', + self.__sql_write('INSERT INTO status (id,flags) VALUES (?,?)', (uid,flags)) return uid + # Interface from BaseFolder def savemessageflags(self, uid, flags): self.messagelist[uid] = {'uid': uid, 'flags': flags} flags = ''.join(sorted(flags)) - self.sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) + self.__sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) + # Interface from BaseFolder def deletemessage(self, uid): if not uid in self.messagelist: return - self.sql_write('DELETE FROM status WHERE id=?', (uid, )) + self.__sql_write('DELETE FROM status WHERE id=?', (uid, )) del(self.messagelist[uid]) + # Interface from BaseFolder def deletemessages(self, uidlist): """Delete list of UIDs from status cache @@ -257,6 +265,6 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): if not len(uidlist): return # arg2 needs to be an iterable of 1-tuples [(1,),(2,),...] - self.sql_write('DELETE FROM status WHERE id=?', zip(uidlist, ), True) + self.__sql_write('DELETE FROM status WHERE id=?', zip(uidlist, ), True) for uid in uidlist: del(self.messagelist[uid]) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 3d6fd62..24d9ced 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -43,7 +43,7 @@ timeseq = 0 lasttime = 0 timelock = Lock() -def gettimeseq(): +def _gettimeseq(): global lasttime, timeseq, timelock timelock.acquire() try: @@ -79,10 +79,12 @@ class MaildirFolder(BaseFolder): # Cache the full folder path, as we use getfullname() very often self._fullname = os.path.join(self.getroot(), self.getname()) + # Interface from BaseFolder def getfullname(self): """Return the absolute file path to the Maildir folder (sans cur|new)""" return self._fullname + # Interface from BaseFolder def get_uidvalidity(self): """Retrieve the current connections UIDVALIDITY value @@ -191,6 +193,7 @@ class MaildirFolder(BaseFolder): retval[uid] = {'flags': flags, 'filename': filepath} return retval + # Interface from BaseFolder def quickchanged(self, statusfolder): """Returns True if the Maildir has changed""" self.cachemessagelist() @@ -204,13 +207,16 @@ class MaildirFolder(BaseFolder): return True return False #Nope, nothing changed + # Interface from BaseFolder def cachemessagelist(self): if self.messagelist is None: self.messagelist = self._scanfolder() + # Interface from BaseFolder def getmessagelist(self): return self.messagelist + # Interface from BaseFolder def getmessage(self, uid): """Return the content of the message""" filename = self.messagelist[uid]['filename'] @@ -222,22 +228,24 @@ class MaildirFolder(BaseFolder): # read it as text? return retval.replace("\r\n", "\n") + # Interface from BaseFolder def getmessagetime(self, uid): filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) - def new_message_filename(self, uid, flags=set()): + def __new_message_filename(self, uid, flags=set()): """Creates a new unique Maildir filename :param uid: The UID`None`, or a set of maildir flags :param flags: A set of maildir flags :returns: String containing unique message filename""" - timeval, timeseq = gettimeseq() + timeval, timeseq = _gettimeseq() return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s' % \ (timeval, timeseq, os.getpid(), socket.gethostname(), uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. @@ -259,7 +267,7 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - messagename = self.new_message_filename(uid, flags) + messagename = self.__new_message_filename(uid, flags) # open file and write it out try: fd = os.open(os.path.join(tmpdir, messagename), @@ -291,9 +299,11 @@ class MaildirFolder(BaseFolder): self.ui.debug('maildir', 'savemessage: returning uid %d' % uid) return uid + # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] + # Interface from BaseFolder def savemessageflags(self, uid, flags): """Sets the specified message's flags to the given set. @@ -331,6 +341,7 @@ class MaildirFolder(BaseFolder): self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename + # Interface from BaseFolder def change_message_uid(self, uid, new_uid): """Change the message from existing uid to new_uid @@ -345,12 +356,13 @@ class MaildirFolder(BaseFolder): oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) flags = self.getmessageflags(uid) - filename = self.new_message_filename(new_uid, flags) + filename = self.__new_message_filename(new_uid, flags) os.rename(os.path.join(self.getfullname(), oldfilename), os.path.join(self.getfullname(), dir_prefix, filename)) self.messagelist[new_uid] = self.messagelist[uid] del self.messagelist[uid] + # Interface from BaseFolder def deletemessage(self, uid): """Unlinks a message file from the Maildir. @@ -375,4 +387,3 @@ class MaildirFolder(BaseFolder): os.unlink(filepath) # Yep -- return. del(self.messagelist[uid]) - diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 7dc43f3..10173f7 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -91,6 +91,7 @@ class MappedIMAPFolder(IMAPFolder): "iling list.".format(e.args[0], self), OfflineImapError.ERROR.MESSAGE) + # Interface from BaseFolder def cachemessagelist(self): self._mb.cachemessagelist() reallist = self._mb.getmessagelist() @@ -122,12 +123,14 @@ class MappedIMAPFolder(IMAPFolder): finally: self.maplock.release() + # Interface from BaseFolder def uidexists(self, ruid): """Checks if the (remote) UID exists in this Folder""" # This implementation overrides the one in BaseFolder, as it is # much more efficient for the mapped case. return ruid in self.r2l + # Interface from BaseFolder def getmessageuidlist(self): """Gets a list of (remote) UIDs. You may have to call cachemessagelist() before calling this function!""" @@ -135,6 +138,7 @@ class MappedIMAPFolder(IMAPFolder): # much more efficient for the mapped case. return self.r2l.keys() + # Interface from BaseFolder def getmessagecount(self): """Gets the number of messages in this folder. You may have to call cachemessagelist() before calling this function!""" @@ -142,6 +146,7 @@ class MappedIMAPFolder(IMAPFolder): # much more efficient for the mapped case. return len(self.r2l) + # Interface from BaseFolder def getmessagelist(self): """Gets the current message list. This function's implementation is quite expensive for the mapped UID case. You must call @@ -167,10 +172,12 @@ class MappedIMAPFolder(IMAPFolder): finally: self.maplock.release() + # Interface from BaseFolder def getmessage(self, uid): """Returns the content of the specified message.""" return self._mb.getmessage(self.r2l[uid]) + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. @@ -216,12 +223,15 @@ class MappedIMAPFolder(IMAPFolder): self.maplock.release() return uid + # Interface from BaseFolder def getmessageflags(self, uid): return self._mb.getmessageflags(self.r2l[uid]) + # Interface from BaseFolder def getmessagetime(self, uid): return None + # Interface from BaseFolder def savemessageflags(self, uid, flags): """ @@ -230,13 +240,16 @@ class MappedIMAPFolder(IMAPFolder): dryrun mode.""" self._mb.savemessageflags(self.r2l[uid], flags) + # Interface from BaseFolder def addmessageflags(self, uid, flags): self._mb.addmessageflags(self.r2l[uid], flags) + # Interface from BaseFolder def addmessagesflags(self, uidlist, flags): self._mb.addmessagesflags(self._uidlist(self.r2l, uidlist), flags) + # Interface from BaseFolder def change_message_uid(self, ruid, new_ruid): """Change the message from existing ruid to new_ruid @@ -279,17 +292,21 @@ class MappedIMAPFolder(IMAPFolder): finally: self.maplock.release() + # Interface from BaseFolder def deletemessageflags(self, uid, flags): self._mb.deletemessageflags(self.r2l[uid], flags) + # Interface from BaseFolder def deletemessagesflags(self, uidlist, flags): self._mb.deletemessagesflags(self._uidlist(self.r2l, uidlist), flags) + # Interface from BaseFolder def deletemessage(self, uid): self._mb.deletemessage(self.r2l[uid]) self._mapped_delete([uid]) + # Interface from BaseFolder def deletemessages(self, uidlist): self._mb.deletemessages(self._uidlist(self.r2l, uidlist)) self._mapped_delete(uidlist) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 4290b2b..f8806dd 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -31,7 +31,7 @@ from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, IMAP4_PORT, InternalDat class UsefulIMAPMixIn(object): - def getselectedfolder(self): + def __getselectedfolder(self): if self.state == 'SELECTED': return self.mailbox return None @@ -41,7 +41,7 @@ class UsefulIMAPMixIn(object): :returns: 'OK' on success, nothing if the folder was already selected or raises an :exc:`OfflineImapError`""" - if self.getselectedfolder() == mailbox and self.is_readonly == readonly \ + if self.__getselectedfolder() == mailbox and self.is_readonly == readonly \ and not force: # No change; return. return @@ -66,6 +66,7 @@ class UsefulIMAPMixIn(object): raise OfflineImapError(errstr, severity) return result + # Overrides private function from IMAP4 (@imaplib2) def _mesg(self, s, tn=None, secs=None): new_mesg(self, s, tn, secs) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index d384bd2..ecd7602 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -83,7 +83,7 @@ class IMAPServer: self.sslcacertfile = repos.getsslcacertfile() self.sslversion = repos.getsslversion() if self.sslcacertfile is None: - self.verifycert = None # disable cert verification + self.__verifycert = None # disable cert verification self.delim = None self.root = None @@ -99,7 +99,7 @@ class IMAPServer: self.gss_vc = None self.gssapi = False - def getpassword(self): + def __getpassword(self): """Returns the server password or None""" if self.goodpassword != None: # use cached good one first return self.goodpassword @@ -114,6 +114,7 @@ class IMAPServer: self.passworderror = None return self.password + # XXX: is this function used anywhere? def getroot(self): """Returns this server's folder root. Can only be called after one or more calls to acquireconnection.""" @@ -136,39 +137,40 @@ class IMAPServer: self.connectionlock.release() self.semaphore.release() - def md5handler(self, response): + def __md5handler(self, response): challenge = response.strip() - self.ui.debug('imap', 'md5handler: got challenge %s' % challenge) + self.ui.debug('imap', '__md5handler: got challenge %s' % challenge) - passwd = self.getpassword() + passwd = self.__getpassword() retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest() - self.ui.debug('imap', 'md5handler: returning %s' % retval) + self.ui.debug('imap', '__md5handler: returning %s' % retval) return retval - def loginauth(self, imapobj): + def __loginauth(self, imapobj): """ Basic authentication via LOGIN command """ self.ui.debug('imap', 'Attempting IMAP LOGIN authentication') - imapobj.login(self.username, self.getpassword()) + imapobj.login(self.username, self.__getpassword()) - def plainhandler(self, response): + def __plainhandler(self, response): """ Implements SASL PLAIN authentication, RFC 4616, http://tools.ietf.org/html/rfc4616 """ authc = self.username - passwd = self.getpassword() + passwd = self.__getpassword() authz = '' if self.user_identity != None: authz = self.user_identity NULL = u'\x00' retval = NULL.join((authz, authc, passwd)).encode('utf-8') - self.ui.debug('imap', 'plainhandler: returning %s' % retval) + self.ui.debug('imap', '__plainhandler: returning %s' % retval) return retval - def gssauth(self, response): + # XXX: describe function + def __gssauth(self, response): data = base64.b64encode(response) try: if self.gss_step == self.GSS_STATE_STEP: @@ -196,7 +198,7 @@ class IMAPServer: return base64.b64decode(response) - def _start_tls(self, imapobj): + def __start_tls(self, imapobj): if 'STARTTLS' in imapobj.capabilities and not self.usessl: self.ui.debug('imap', 'Using STARTTLS connection') try: @@ -207,7 +209,7 @@ class IMAPServer: OfflineImapError.ERROR.REPO) - ## All _authn_* procedures are helpers that do authentication. + ## All __authn_* procedures are helpers that do authentication. ## They are class methods that take one parameter, IMAP object. ## ## Each function should return True if authentication was @@ -224,13 +226,13 @@ class IMAPServer: ## - OfflineImapError means that function detected some ## problem by itself. - def _authn_gssapi(self, imapobj): + def __authn_gssapi(self, imapobj): if not have_gss: return False self.connectionlock.acquire() try: - imapobj.authenticate('GSSAPI', self.gssauth) + imapobj.authenticate('GSSAPI', self.__gssauth) return True except imapobj.error as e: self.gssapi = False @@ -243,15 +245,15 @@ class IMAPServer: finally: self.connectionlock.release() - def _authn_cram_md5(self, imapobj): - imapobj.authenticate('CRAM-MD5', self.md5handler) + def __authn_cram_md5(self, imapobj): + imapobj.authenticate('CRAM-MD5', self.__md5handler) return True - def _authn_plain(self, imapobj): - imapobj.authenticate('PLAIN', self.plainhandler) + def __authn_plain(self, imapobj): + imapobj.authenticate('PLAIN', self.__plainhandler) return True - def _authn_login(self, imapobj): + def __authn_login(self, imapobj): # Use LOGIN command, unless LOGINDISABLED is advertized # (per RFC 2595) if 'LOGINDISABLED' in imapobj.capabilities: @@ -259,11 +261,11 @@ class IMAPServer: "disabled by server. Need to use SSL?", OfflineImapError.ERROR.REPO) else: - self.loginauth(imapobj) + self.__loginauth(imapobj) return True - def _authn_helper(self, imapobj): + def __authn_helper(self, imapobj): """ Authentication machinery for self.acquireconnection(). @@ -283,10 +285,10 @@ class IMAPServer: # - tryTLS flag, # - check IMAP capability flag. auth_methods = { - "GSSAPI": (self._authn_gssapi, False, True), - "CRAM-MD5": (self._authn_cram_md5, True, True), - "PLAIN": (self._authn_plain, True, True), - "LOGIN": (self._authn_login, True, False), + "GSSAPI": (self.__authn_gssapi, False, True), + "CRAM-MD5": (self.__authn_cram_md5, True, True), + "PLAIN": (self.__authn_plain, True, True), + "LOGIN": (self.__authn_login, True, False), } # Stack stores pairs of (method name, exception) exc_stack = [] @@ -311,7 +313,7 @@ class IMAPServer: # they could have been changed after STARTTLS. if tryTLS and not tried_tls: tried_tls = True - self._start_tls(imapobj) + self.__start_tls(imapobj) if check_cap: cap = "AUTH=" + m @@ -348,6 +350,7 @@ class IMAPServer: OfflineImapError.ERROR.REPO) + # XXX: move above, closer to releaseconnection() def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. @@ -400,7 +403,7 @@ class IMAPServer: self.sslclientkey, self.sslclientcert, self.sslcacertfile, - self.verifycert, + self.__verifycert, self.sslversion, timeout=socket.getdefaulttimeout(), fingerprint=fingerprint @@ -412,7 +415,7 @@ class IMAPServer: if not self.preauth_tunnel: try: - self._authn_helper(imapobj) + self.__authn_helper(imapobj) self.goodpassword = self.password success = 1 except OfflineImapError as e: @@ -562,7 +565,7 @@ class IMAPServer: self.ui.debug('imap', 'keepalive: event is set; exiting') return - def verifycert(self, cert, hostname): + def __verifycert(self, cert, hostname): '''Verify that cert (in socket.getpeercert() format) matches hostname. CRLs are not handled. @@ -646,7 +649,7 @@ class IdleThread(object): self.parent.releaseconnection(imapobj) self.stop_sig.wait() # wait until we are supposed to quit - def dosync(self): + def __dosync(self): remoterepos = self.parent.repos account = remoterepos.account localrepos = account.localrepos @@ -663,7 +666,7 @@ class IdleThread(object): ui = getglobalui() ui.unregisterthread(currentThread()) #syncfolder registered the thread - def idle(self): + def __idle(self): """Invoke IDLE mode until timeout or self.stop() is invoked""" def callback(args): """IDLE callback function invoked by imaplib2 @@ -696,7 +699,7 @@ class IdleThread(object): else: success = True if "IDLE" in imapobj.capabilities: - imapobj.idle(callback=callback) + imapobj.__idle(callback=callback) else: self.ui.warn("IMAP IDLE not supported on server '%s'." "Sleep until next refresh cycle." % imapobj.identifier) @@ -716,4 +719,4 @@ class IdleThread(object): # here not via self.stop, but because IDLE responded. Do # another round and invoke actual syncing. self.stop_sig.clear() - self.dosync() + self.__dosync() diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index c6e660c..5d69f59 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -21,7 +21,7 @@ import string from offlineimap.ui import getglobalui -def debug(*args): +def __debug(*args): msg = [] for arg in args: msg.append(str(arg)) @@ -50,7 +50,7 @@ def flagsplit(string): raise ValueError("Passed string '%s' is not a flag list" % string) return imapsplit(string[1:-1]) -def options2hash(list): +def __options2hash(list): """convert list [1,2,3,4,5,6] to {1:2, 3:4, 5:6}""" # effectively this does dict(zip(l[::2],l[1::2])), however # measurements seemed to have indicated that the manual variant is @@ -60,7 +60,7 @@ def options2hash(list): while (counter < len(list)): retval[list[counter]] = list[counter + 1] counter += 2 - debug("options2hash returning:", retval) + __debug("__options2hash returning:", retval) return retval def flags2hash(flags): @@ -68,7 +68,7 @@ def flags2hash(flags): E.g. '(FLAGS (\\Seen Old) UID 4807)' leads to {'FLAGS': '(\\Seen Old)', 'UID': '4807'}""" - return options2hash(flagsplit(flags)) + return __options2hash(flagsplit(flags)) def imapsplit(imapstring): """Takes a string from an IMAP conversation and returns a list containing @@ -81,7 +81,7 @@ def imapsplit(imapstring): ['(\\HasNoChildren)', '"."', '"INBOX.Sent"']""" if not isinstance(imapstring, basestring): - debug("imapsplit() got a non-string input; working around.") + __debug("imapsplit() got a non-string input; working around.") # Sometimes, imaplib will throw us a tuple if the input # contains a literal. See Python bug # #619732 at https://sourceforge.net/tracker/index.php?func=detail&aid=619732&group_id=5470&atid=105470 @@ -103,7 +103,7 @@ def imapsplit(imapstring): arg = arg.replace('\\', '\\\\') arg = arg.replace('"', '\\"') arg = '"%s"' % arg - debug("imapsplit() non-string [%d]: Appending %s" %\ + __debug("imapsplit() non-string [%d]: Appending %s" %\ (i, arg)) retval.append(arg) else: @@ -113,10 +113,10 @@ def imapsplit(imapstring): # Recursion to the rescue. arg = imapstring[i] arg = re.sub('\{\d+\}$', '', arg) - debug("imapsplit() non-string [%d]: Feeding %s to recursion" %\ + __debug("imapsplit() non-string [%d]: Feeding %s to recursion" %\ (i, arg)) retval.extend(imapsplit(arg)) - debug("imapsplit() non-string: returning %s" % str(retval)) + __debug("imapsplit() non-string: returning %s" % str(retval)) return retval workstr = imapstring.strip() @@ -137,7 +137,7 @@ def imapsplit(imapstring): retval.append(parenlist) elif workstr[0] == '"': # quoted fragments '"...\"..."' - (quoted, rest) = _split_quoted(workstr) + (quoted, rest) = __split_quoted(workstr) retval.append(quoted) workstr = rest else: @@ -213,7 +213,7 @@ def uid_sequence(uidlist): return ",".join(retval) -def _split_quoted(string): +def __split_quoted(string): """ Looks for the ending quote character in the string that starts with quote character, splitting out quoted component and the diff --git a/offlineimap/init.py b/offlineimap/init.py index ce24e48..d4c9a6d 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -43,13 +43,13 @@ class OfflineImap: def run(self): """Parse the commandline and invoke everything""" # next line also sets self.config and self.ui - options, args = self.parse_cmd_options() + options, args = self.__parse_cmd_options() if options.diagnostics: - self.serverdiagnostics(options) + self.__serverdiagnostics(options) else: - self.sync(options) + self.__sync(options) - def parse_cmd_options(self): + def __parse_cmd_options(self): parser = OptionParser(version=offlineimap.__version__, description="%s.\n\n%s" % (offlineimap.__copyright__, @@ -297,7 +297,7 @@ class OfflineImap: self.config = config return (options, args) - def sync(self, options): + def __sync(self, options): """Invoke the correct single/multithread syncing self.config is supposed to have been correctly initialized @@ -360,7 +360,7 @@ class OfflineImap: if options.singlethreading: #singlethreaded - self.sync_singlethreaded(syncaccounts) + self.__sync_singlethreaded(syncaccounts) else: # multithreaded t = threadutil.ExitNotifyThread(target=syncmaster.syncitall, @@ -376,7 +376,7 @@ class OfflineImap: self.ui.error(e) self.ui.terminate() - def sync_singlethreaded(self, accs): + def __sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread :param accs: A list of accounts that should be synced @@ -387,7 +387,7 @@ class OfflineImap: threading.currentThread().name = "Account sync %s" % accountname account.syncrunner() - def serverdiagnostics(self, options): + def __serverdiagnostics(self, options): activeaccounts = self.config.get("general", "accounts") if options.accounts: activeaccounts = options.accounts diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 39dcf03..2d9ab5e 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -44,9 +44,9 @@ def write(): if account not in boxes: return - genmbnames() + __genmbnames() -def genmbnames(): +def __genmbnames(): """Takes a configparser object and a boxlist, which is a list of hashes containing 'accountname' and 'foldername' keys.""" mblock.acquire() diff --git a/test/tests/test_00_imaputil.py b/test/tests/test_00_imaputil.py index d1972da..4e9d142 100644 --- a/test/tests/test_00_imaputil.py +++ b/test/tests/test_00_imaputil.py @@ -70,11 +70,6 @@ class TestInternalFunctions(unittest.TestCase): res = imaputil.flagsplit(b'(FLAGS (\\Seen Old) UID 4807)') self.assertEqual(res, [b'FLAGS', b'(\\Seen Old)', b'UID', b'4807']) - def test_03_options2hash(self): - """Test imaputil.options2hash()""" - res = imaputil.options2hash([1,2,3,4,5,6]) - self.assertEqual(res, {1:2, 3:4, 5:6}) - def test_04_flags2hash(self): """Test imaputil.flags2hash()""" res = imaputil.flags2hash(b'(FLAGS (\\Seen Old) UID 4807)') From 697ca8a229299dd284cc81f5e441148d9d0893dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abd=C3=B3=20Roig-Maranges?= Date: Thu, 18 Oct 2012 19:23:34 +0200 Subject: [PATCH 591/817] Changed NotImplementedException to NotImplementedError It seems NotImplementedException does not exist. It must be a relic from old Python... Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 0062dc2..fe8951d 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -115,7 +115,7 @@ class BaseFolder(object): def getcopyinstancelimit(self): """For threading folders, returns the instancelimitname for InstanceLimitedThreads.""" - raise NotImplementedException + raise NotImplementedError def storesmessages(self): """Should be true for any backend that actually saves message bodies. @@ -211,18 +211,18 @@ class BaseFolder(object): This function needs to be implemented by each Backend :returns: UIDVALIDITY as a (long) number""" - raise NotImplementedException + raise NotImplementedError def cachemessagelist(self): """Reads the message list from disk or network and stores it in memory for later use. This list will not be re-read from disk or memory unless this function is called again.""" - raise NotImplementedException + raise NotImplementedError def getmessagelist(self): """Gets the current message list. You must call cachemessagelist() before calling this function!""" - raise NotImplementedException + raise NotImplementedError def uidexists(self, uid): """Returns True if uid exists""" @@ -239,7 +239,7 @@ class BaseFolder(object): def getmessage(self, uid): """Returns the content of the specified message.""" - raise NotImplementedException + raise NotImplementedError def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. @@ -263,15 +263,15 @@ class BaseFolder(object): so you need to ensure that savemessage is never called in a dryrun mode. """ - raise NotImplementedException + raise NotImplementedError def getmessagetime(self, uid): """Return the received time for the specified message.""" - raise NotImplementedException + raise NotImplementedError def getmessageflags(self, uid): """Returns the flags for the specified message.""" - raise NotImplementedException + raise NotImplementedError def savemessageflags(self, uid, flags): """Sets the specified message's flags to the given set. @@ -279,7 +279,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" - raise NotImplementedException + raise NotImplementedError def addmessageflags(self, uid, flags): """Adds the specified flags to the message's flag set. If a given @@ -327,14 +327,14 @@ class BaseFolder(object): :param new_uid: (optional) If given, the old UID will be changed to a new UID. This allows backends efficient renaming of messages if the UID has changed.""" - raise NotImplementedException + raise NotImplementedError def deletemessage(self, uid): """ Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" - raise NotImplementedException + raise NotImplementedError def deletemessages(self, uidlist): """ From 0c4fe6bada65d7101821031b808f887d8ec37851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abd=C3=B3=20Roig-Maranges?= Date: Tue, 23 Oct 2012 20:05:59 +0200 Subject: [PATCH 592/817] Bugfix: make change_message_uid to update messagelist's filename This broke code that relied on the filename being up to date in memory after messages are copied. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Maildir.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 24d9ced..ba55e74 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -356,10 +356,12 @@ class MaildirFolder(BaseFolder): oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) flags = self.getmessageflags(uid) - filename = self.__new_message_filename(new_uid, flags) + newfilename = os.path.join(dir_prefix, + self.__new_message_filename(new_uid, flags)) os.rename(os.path.join(self.getfullname(), oldfilename), - os.path.join(self.getfullname(), dir_prefix, filename)) + os.path.join(self.getfullname(), newfilename)) self.messagelist[new_uid] = self.messagelist[uid] + self.messagelist[new_uid]['filename'] = newfilename del self.messagelist[uid] # Interface from BaseFolder From 0903428fdaf102a1dd258b5d19f6a9c26fb57415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abd=C3=B3=20Roig-Maranges?= Date: Thu, 25 Oct 2012 14:25:19 +0200 Subject: [PATCH 593/817] Enable compressed connections to the IMAP server Added the configuration setting usecompression for the IMAP repositories. When enabled, the data from and to the IMAP server is compressed. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 1 + offlineimap.conf | 5 +++++ offlineimap/imapserver.py | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index e541358..cb60d5e 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -9,6 +9,7 @@ OfflineIMAP v6.5.6 (YYYY-MM-DD) =============================== * Add knob to invoke folderfilter dynamically on each sync (GitHub#73) +* Add knob to apply compression to IMAP connections (Abdó Roig-Maranges) OfflineIMAP v6.5.5 (2013-10-07) diff --git a/offlineimap.conf b/offlineimap.conf index 0a8d44c..f3cd479 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -479,6 +479,11 @@ remoteuser = username # idlefolders = ['INBOX', 'INBOX.Alerts'] # +# OfflineIMAP can use a compressed connection to the IMAP server. +# This can result in faster downloads for some cases. +# +#usecompression = yes + # OfflineIMAP can use multiple connections to the server in order # to perform multiple synchronization actions simultaneously. # This may place a higher burden on the server. In most cases, diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index ecd7602..b00fd08 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -422,6 +422,10 @@ class IMAPServer: self.passworderror = str(e) raise + # Enable compression + if self.repos.getconfboolean('usecompression', 0): + imapobj.enable_compression() + # update capabilities after login, e.g. gmail serves different ones typ, dat = imapobj.capability() if dat != [None]: From 844ca6b08c0fd48035b58bfd8b88050c2688307b Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 2 May 2014 15:14:22 +0400 Subject: [PATCH 594/817] Don't embed CRLF multiple times when saving a message Since we just do multiple passes for saving the message without actually modifying its content (apart from header insertion that is CRLF-clean), we can change line ends to the proper CRLF just once. And we can also get message's date only once too. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/IMAP.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index d727b31..b7c0369 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -509,6 +509,12 @@ class IMAPFolder(BaseFolder): self.savemessageflags(uid, flags) return uid + # Use proper CRLF all over the message + content = re.sub("(? Date: Tue, 16 Oct 2012 20:53:54 +0200 Subject: [PATCH 595/817] Add ability to trim some local mail headers When filterheaders is set to a comma-separated list of headers, OfflineIMAP removes those headers from messages before uploading them to the server. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ offlineimap.conf | 11 ++++++++ offlineimap/folder/IMAP.py | 55 +++++++++++++++++++++++++++++++++++--- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index cb60d5e..0d3dca9 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -10,6 +10,8 @@ OfflineIMAP v6.5.6 (YYYY-MM-DD) * Add knob to invoke folderfilter dynamically on each sync (GitHub#73) * Add knob to apply compression to IMAP connections (Abdó Roig-Maranges) +* Add knob to filter some headers before uploading message + to IMAP server (Abdó Roig-Maranges) OfflineIMAP v6.5.5 (2013-10-07) diff --git a/offlineimap.conf b/offlineimap.conf index f3cd479..333e52e 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -272,6 +272,17 @@ remoterepository = RemoteExample #maildir-windows-compatible = no +# OfflineIMAP can strip off some headers when your messages are propagated +# back to the IMAP server. This option carries the comma-separated list +# of headers to trim off. Header name matching is case-sensitive. +# +# This knob is respected only by IMAP-based accounts. Value of labelsheader +# for GMail-based accounts is automatically added to this list, you don't +# need to specify it explicitely. +# +#filterheaders = X-Some-Weird-Header + + [Repository LocalExample] # Each repository requires a "type" declaration. The types supported for diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index b7c0369..bab80f4 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -26,6 +26,10 @@ from offlineimap import globals from offlineimap.imaplib2 import MonthNames +# Globals +CRLF = '\r\n' + + class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, repository): name = imaputil.dequote(name) @@ -38,6 +42,10 @@ class IMAPFolder(BaseFolder): self.randomgenerator = random.Random() #self.ui is set in BaseFolder + fh_conf = self.repository.account.getconf('filterheaders', '') + self.filterheaders = [h for h in re.split(r'\s*,\s*', fh_conf) if h] + + def __selectro(self, imapobj, force = False): """Select this folder when we do not need write access. @@ -248,7 +256,7 @@ class IMAPFolder(BaseFolder): # data looks now e.g. [('320 (UID 17061 BODY[] # {2565}','msgbody....')] we only asked for one message, # and that msg is in data[0]. msbody is in [0][1] - data = data[0][1].replace("\r\n", "\n") + data = data[0][1].replace(CRLF, "\n") if len(data)>200: dbg_output = "%s...%s" % (str(data)[:150], @@ -300,7 +308,7 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', '__savemessage_addheader: called to add %s: %s' % (headername, headervalue)) - insertionpoint = content.find("\r\n\r\n") + insertionpoint = content.find(CRLF + CRLF) self.ui.debug('imap', '__savemessage_addheader: insertionpoint = %d' % insertionpoint) leader = content[0:insertionpoint] self.ui.debug('imap', '__savemessage_addheader: leader = %s' % repr(leader)) @@ -308,7 +316,7 @@ class IMAPFolder(BaseFolder): newline = '' insertionpoint = 0 else: - newline = "\r\n" + newline = CRLF newline += "%s: %s" % (headername, headervalue) self.ui.debug('imap', '__savemessage_addheader: newline = ' + repr(newline)) trailer = content[insertionpoint:] @@ -316,6 +324,43 @@ class IMAPFolder(BaseFolder): return leader + newline + trailer + def __savemessage_delheaders(self, content, header_list): + """ + Deletes headers in the given list from the message content. + + Arguments: + - content: message itself + - header_list: list of headers to be deleted or just the header name + + We expect our message to have proper CRLF as line endings. + + """ + if type(header_list) != type([]): + header_list = [header_list] + self.ui.debug('imap', + '__savemessage_delheaders: called to delete %s' % (header_list)) + + if not len(header_list): return content + + eoh = content.find(CRLF + CRLF) + if eoh == -1: + eoh = len(content) + self.ui.debug('imap', '__savemessage_delheaders: end of headers = %d' % eoh) + headers = content[0:eoh] + rest = content[eoh:] + self.ui.debug('imap', '__savemessage_delheaders: headers = %s' % repr(headers)) + new_headers = [] + for h in headers.split(CRLF): + keep_it = True + for trim_h in self.filterheaders: + if len(h) > len(trim_h) and h[0:len(trim_h)+1] == (trim_h + ":"): + keep_it = False + break + if keep_it: new_headers.append(h) + + return (CRLF.join(new_headers) + rest) + + def __savemessage_searchforheader(self, imapobj, headername, headervalue): self.ui.debug('imap', '__savemessage_searchforheader called for %s: %s' % \ (headername, headervalue)) @@ -510,11 +555,13 @@ class IMAPFolder(BaseFolder): return uid # Use proper CRLF all over the message - content = re.sub("(? Date: Wed, 17 Oct 2012 21:45:19 +0200 Subject: [PATCH 596/817] Restructured folder/IMAP code In preparation for GMail label sync, we had split our some functionality that will be needed further into their own functions. This also permitted the code to look more compact and concise. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 113 ++++++++++++- offlineimap/folder/IMAP.py | 317 +++++++++++++++++++------------------ 2 files changed, 273 insertions(+), 157 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index fe8951d..13d43ae 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -66,6 +66,10 @@ class BaseFolder(object): self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter" \ % (self.ffilter_name, repository)) + # Passes for syncmessagesto + self.syncmessagesto_passes = [('copying messages' , self.__syncmessagesto_copy), + ('deleting messages' , self.__syncmessagesto_delete), + ('syncing flags' , self.__syncmessagesto_flags)] def getname(self): """Returns name""" @@ -319,6 +323,103 @@ class BaseFolder(object): for uid in uidlist: self.deletemessageflags(uid, flags) + + def addmessageheader(self, content, headername, headervalue): + self.ui.debug('', + 'addmessageheader: called to add %s: %s' % (headername, + headervalue)) + insertionpoint = content.find('\n\n') + self.ui.debug('', 'addmessageheader: insertionpoint = %d' % insertionpoint) + leader = content[0:insertionpoint] + self.ui.debug('', 'addmessageheader: leader = %s' % repr(leader)) + if insertionpoint == 0 or insertionpoint == -1: + newline = '' + insertionpoint = 0 + else: + newline = '\n' + newline += "%s: %s" % (headername, headervalue) + self.ui.debug('', 'addmessageheader: newline = ' + repr(newline)) + trailer = content[insertionpoint:] + self.ui.debug('', 'addmessageheader: trailer = ' + repr(trailer)) + return leader + newline + trailer + + + def __find_eoh(self, content): + """ + Searches for the point where mail headers end. + Either double '\n', or end of string. + + Arguments: + - content: contents of the message to search in + Returns: position of the first non-header byte. + + """ + eoh_cr = content.find('\n\n') + if eoh_cr == -1: + eoh_cr = len(content) + + return eoh_cr + + + def getmessageheader(self, content, name): + """ + Searches for the given header and returns its value. + Arguments: + - contents: message itself + - name: name of the header to be searched + + Returns: header value or None if no such header was found + + """ + self.ui.debug('', 'getmessageheader: called to get %s' % name) + eoh = self.__find_eoh(content) + self.ui.debug('', 'getmessageheader: eoh = %d' % eoh) + headers = content[0:eoh] + self.ui.debug('', 'getmessageheader: headers = %s' % repr(headers)) + + m = re.search('^%s:(.*)$' % name, headers, flags = re.MULTILINE) + if m: + return m.group(1).strip() + else: + return None + + + def deletemessageheaders(self, content, header_list): + """ + Deletes headers in the given list from the message content. + + Arguments: + - content: message itself + - header_list: list of headers to be deleted or just the header name + + We expect our message to have '\n' as line endings. + + """ + if type(header_list) != type([]): + header_list = [header_list] + self.ui.debug('', 'deletemessageheaders: called to delete %s' % (header_list)) + + if not len(header_list): return content + + eoh = self.__find_eoh(content) + self.ui.debug('', 'deletemessageheaders: end of headers = %d' % eoh) + headers = content[0:eoh] + rest = content[eoh:] + self.ui.debug('', 'deletemessageheaders: headers = %s' % repr(headers)) + new_headers = [] + for h in headers.split('\n'): + keep_it = True + for trim_h in self.filterheaders: + if len(h) > len(trim_h) and h[0:len(trim_h)+1] == (trim_h + ":"): + keep_it = False + break + if keep_it: new_headers.append(h) + + return ('\n'.join(new_headers) + rest) + + + + def change_message_uid(self, uid, new_uid): """Change the message from existing uid to new_uid @@ -564,14 +665,16 @@ class BaseFolder(object): deleted there), sync the flag change to both dstfolder and statusfolder. + Pass4: Synchronize label changes (Gmail only) + Compares label mismatches in self with those in statusfolder. + If msg has a valid UID and exists on dstfolder, syncs the labels + to both dstfolder and statusfolder. + :param dstfolder: Folderinstance to sync the msgs to. :param statusfolder: LocalStatus instance to sync against. - """ - passes = [('copying messages' , self.__syncmessagesto_copy), - ('deleting messages' , self.__syncmessagesto_delete), - ('syncing flags' , self.__syncmessagesto_flags)] - for (passdesc, action) in passes: + """ + for (passdesc, action) in self.syncmessagesto_passes: # bail out on CTRL-C or SIGTERM if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index bab80f4..dd87165 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -30,6 +30,13 @@ from offlineimap.imaplib2 import MonthNames CRLF = '\r\n' +# NB: message returned from getmessage() will have '\n' all over the place, +# NB: there will be no CRLFs. Just before the sending stage of savemessage() +# NB: '\n' will be transformed back to CRLF. So, for the most parts of the +# NB: code the stored content will be clean of CRLF and one can rely that +# NB: line endings will be pure '\n'. + + class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, repository): name = imaputil.dequote(name) @@ -41,6 +48,7 @@ class IMAPFolder(BaseFolder): self.messagelist = None self.randomgenerator = random.Random() #self.ui is set in BaseFolder + self.imap_query = ['BODY.PEEK[]'] fh_conf = self.repository.account.getconf('filterheaders', '') self.filterheaders = [h for h in re.split(r'\s*,\s*', fh_conf) if h] @@ -131,57 +139,76 @@ class IMAPFolder(BaseFolder): return True return False - # Interface from BaseFolder - def cachemessagelist(self): + + def _msgs_to_fetch(self, imapobj): + """ + Determines UIDS of messages to be fetched + + Arguments: + - imapobj: instance of IMAPlib + + Returns: UID ranges for messages or None if no messages + are to be fetched. + + """ + res_type, imapdata = imapobj.select(self.getfullname(), True, True) + if imapdata == [None] or imapdata[0] == '0': + # Empty folder, no need to populate message list + return None + + # By default examine all UIDs in this folder + msgsToFetch = '1:*' + maxage = self.config.getdefaultint("Account %s" % self.accountname, "maxage", -1) maxsize = self.config.getdefaultint("Account %s" % self.accountname, "maxsize", -1) + + # Build search condition + if (maxage != -1) | (maxsize != -1): + search_cond = "("; + + if(maxage != -1): + #find out what the oldest message is that we should look at + oldest_struct = time.gmtime(time.time() - (60*60*24*maxage)) + if oldest_struct[0] < 1900: + raise OfflineImapError("maxage setting led to year %d. " + "Abort syncing." % oldest_struct[0], + OfflineImapError.ERROR.REPO) + search_cond += "SINCE %02d-%s-%d" % ( + oldest_struct[2], + MonthNames[oldest_struct[1]], + oldest_struct[0]) + + if(maxsize != -1): + if(maxage != -1): # There are two conditions, add space + search_cond += " " + search_cond += "SMALLER %d" % maxsize + + search_cond += ")" + + res_type, res_data = imapobj.search(None, search_cond) + if res_type != 'OK': + raise OfflineImapError("SEARCH in folder [%s]%s failed. " + "Search string was '%s'. Server responded '[%s] %s'" % ( + self.getrepository(), self, search_cond, res_type, res_data), + OfflineImapError.ERROR.FOLDER) + + # Result UIDs are seperated by space, coalesce into ranges + msgsToFetch = imaputil.uid_sequence(res_data[0].split()) + + return msgsToFetch + + + # Interface from BaseFolder + def cachemessagelist(self): self.messagelist = {} imapobj = self.imapserver.acquireconnection() try: - res_type, imapdata = imapobj.select(self.getfullname(), True, True) - if imapdata == [None] or imapdata[0] == '0': - # Empty folder, no need to populate message list - return - # By default examine all UIDs in this folder - msgsToFetch = '1:*' - - if (maxage != -1) | (maxsize != -1): - search_cond = "("; - - if(maxage != -1): - #find out what the oldest message is that we should look at - oldest_struct = time.gmtime(time.time() - (60*60*24*maxage)) - if oldest_struct[0] < 1900: - raise OfflineImapError("maxage setting led to year %d. " - "Abort syncing." % oldest_struct[0], - OfflineImapError.ERROR.REPO) - search_cond += "SINCE %02d-%s-%d" % ( - oldest_struct[2], - MonthNames[oldest_struct[1]], - oldest_struct[0]) - - if(maxsize != -1): - if(maxage != -1): # There are two conditions, add space - search_cond += " " - search_cond += "SMALLER %d" % maxsize - - search_cond += ")" - - res_type, res_data = imapobj.search(None, search_cond) - if res_type != 'OK': - raise OfflineImapError("SEARCH in folder [%s]%s failed. " - "Search string was '%s'. Server responded '[%s] %s'" % ( - self.getrepository(), self, - search_cond, res_type, res_data), - OfflineImapError.ERROR.FOLDER) - - # Result UIDs are seperated by space, coalesce into ranges - msgsToFetch = imaputil.uid_sequence(res_data[0].split()) - if not msgsToFetch: - return # No messages to sync + msgsToFetch = self._msgs_to_fetch(imapobj) + if not msgsToFetch: + return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. @@ -213,60 +240,42 @@ class IMAPFolder(BaseFolder): rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + # Interface from BaseFolder def getmessagelist(self): return self.messagelist # Interface from BaseFolder def getmessage(self, uid): - """Retrieve message with UID from the IMAP server (incl body) + """ + Retrieve message with UID from the IMAP server (incl body) + + After this function all CRLFs will be transformed to '\n'. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with this UID could be found. + """ imapobj = self.imapserver.acquireconnection() try: - fails_left = 2 # retry on dropped connection - while fails_left: - try: - imapobj.select(self.getfullname(), readonly = True) - res_type, data = imapobj.uid('fetch', str(uid), - '(BODY.PEEK[])') - fails_left = 0 - except imapobj.abort as e: - # Release dropped connection, and get a new one - self.imapserver.releaseconnection(imapobj, True) - imapobj = self.imapserver.acquireconnection() - self.ui.error(e, exc_info()[2]) - fails_left -= 1 - if not fails_left: - raise e - if data == [None] or res_type != 'OK': - #IMAP server says bad request or UID does not exist - severity = OfflineImapError.ERROR.MESSAGE - reason = "IMAP server '%s' failed to fetch message UID '%d'."\ - "Server responded: %s %s" % (self.getrepository(), uid, - res_type, data) - if data == [None]: - #IMAP server did not find a message with this UID - reason = "IMAP server '%s' does not have a message "\ - "with UID '%s'" % (self.getrepository(), uid) - raise OfflineImapError(reason, severity) - # data looks now e.g. [('320 (UID 17061 BODY[] - # {2565}','msgbody....')] we only asked for one message, - # and that msg is in data[0]. msbody is in [0][1] - data = data[0][1].replace(CRLF, "\n") - - if len(data)>200: - dbg_output = "%s...%s" % (str(data)[:150], - str(data)[-50:]) - else: - dbg_output = data - self.ui.debug('imap', "Returned object from fetching %d: '%s'" % - (uid, dbg_output)) + data = self._fetch_from_imap(imapobj, str(uid), 2) finally: self.imapserver.releaseconnection(imapobj) + + # data looks now e.g. [('320 (UID 17061 BODY[] + # {2565}','msgbody....')] we only asked for one message, + # and that msg is in data[0]. msbody is in [0][1] + data = data[0][1].replace(CRLF, "\n") + + if len(data)>200: + dbg_output = "%s...%s" % (str(data)[:150], str(data)[-50:]) + else: + dbg_output = data + + self.ui.debug('imap', "Returned object from fetching %d: '%s'" % + (uid, dbg_output)) + return data # Interface from BaseFolder @@ -304,63 +313,6 @@ class IMAPFolder(BaseFolder): return (headername, headervalue) - def __savemessage_addheader(self, content, headername, headervalue): - self.ui.debug('imap', - '__savemessage_addheader: called to add %s: %s' % (headername, - headervalue)) - insertionpoint = content.find(CRLF + CRLF) - self.ui.debug('imap', '__savemessage_addheader: insertionpoint = %d' % insertionpoint) - leader = content[0:insertionpoint] - self.ui.debug('imap', '__savemessage_addheader: leader = %s' % repr(leader)) - if insertionpoint == 0 or insertionpoint == -1: - newline = '' - insertionpoint = 0 - else: - newline = CRLF - newline += "%s: %s" % (headername, headervalue) - self.ui.debug('imap', '__savemessage_addheader: newline = ' + repr(newline)) - trailer = content[insertionpoint:] - self.ui.debug('imap', '__savemessage_addheader: trailer = ' + repr(trailer)) - return leader + newline + trailer - - - def __savemessage_delheaders(self, content, header_list): - """ - Deletes headers in the given list from the message content. - - Arguments: - - content: message itself - - header_list: list of headers to be deleted or just the header name - - We expect our message to have proper CRLF as line endings. - - """ - if type(header_list) != type([]): - header_list = [header_list] - self.ui.debug('imap', - '__savemessage_delheaders: called to delete %s' % (header_list)) - - if not len(header_list): return content - - eoh = content.find(CRLF + CRLF) - if eoh == -1: - eoh = len(content) - self.ui.debug('imap', '__savemessage_delheaders: end of headers = %d' % eoh) - headers = content[0:eoh] - rest = content[eoh:] - self.ui.debug('imap', '__savemessage_delheaders: headers = %s' % repr(headers)) - new_headers = [] - for h in headers.split(CRLF): - keep_it = True - for trim_h in self.filterheaders: - if len(h) > len(trim_h) and h[0:len(trim_h)+1] == (trim_h + ":"): - keep_it = False - break - if keep_it: new_headers.append(h) - - return (CRLF.join(new_headers) + rest) - - def __savemessage_searchforheader(self, imapobj, headername, headervalue): self.ui.debug('imap', '__savemessage_searchforheader called for %s: %s' % \ (headername, headervalue)) @@ -554,14 +506,14 @@ class IMAPFolder(BaseFolder): self.savemessageflags(uid, flags) return uid + content = self.deletemessageheaders(content, self.filterheaders) + # Use proper CRLF all over the message content = re.sub("(?200: dbg_output = "%s...%s" % (content[:150], content[-50:]) else: @@ -685,6 +637,69 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage: returning new UID %d' % uid) return uid + + def _fetch_from_imap(self, imapobj, uids, retry_num=1): + """ + Fetches data from IMAP server. + + Arguments: + - imapobj: IMAPlib object + - uids: message UIDS + - retry_num: number of retries to make + + Returns: data obtained by this query. + """ + query = "(%s)" % (" ".join(self.imap_query)) + fails_left = retry_num # retry on dropped connection + while fails_left: + try: + imapobj.select(self.getfullname(), readonly = True) + res_type, data = imapobj.uid('fetch', uids, query) + fails_left = 0 + except imapobj.abort as e: + # Release dropped connection, and get a new one + self.imapserver.releaseconnection(imapobj, True) + imapobj = self.imapserver.acquireconnection() + self.ui.error(e, exc_info()[2]) + fails_left -= 1 + if not fails_left: + raise e + if data == [None] or res_type != 'OK': + #IMAP server says bad request or UID does not exist + severity = OfflineImapError.ERROR.MESSAGE + reason = "IMAP server '%s' failed to fetch messages UID '%s'."\ + "Server responded: %s %s" % (self.getrepository(), uids, + res_type, data) + if data == [None]: + #IMAP server did not find a message with this UID + reason = "IMAP server '%s' does not have a message "\ + "with UID '%s'" % (self.getrepository(), uids) + raise OfflineImapError(reason, severity) + + return data + + + def _store_to_imap(self, imapobj, uid, field, data): + """ + Stores data to IMAP server + + Arguments: + - imapobj: instance of IMAPlib to use + - uid: message UID + - field: field name to be stored/updated + - data: field contents + + """ + imapobj.select(self.getfullname()) + res_type, retdata = imapobj.uid('store', uid, field, data) + if res_type != 'OK': + severity = OfflineImapError.ERROR.MESSAGE + reason = "IMAP server '%s' failed to store %s for message UID '%d'."\ + "Server responded: %s %s" % (self.getrepository(), field, uid, + res_type, retdata) + raise OfflineImapError(reason, severity) + return retdata[0] + # Interface from BaseFolder def savemessageflags(self, uid, flags): """Change a message's flags to `flags`. @@ -694,17 +709,15 @@ class IMAPFolder(BaseFolder): dryrun mode.""" imapobj = self.imapserver.acquireconnection() try: - try: - imapobj.select(self.getfullname()) - except imapobj.readonly: - self.ui.flagstoreadonly(self, [uid], flags) - return - result = imapobj.uid('store', '%d' % uid, 'FLAGS', - imaputil.flagsmaildir2imap(flags)) - assert result[0] == 'OK', 'Error with store: ' + '. '.join(result[1]) + result = self._store_to_imap(imapobj, str(uid), 'FLAGS', imaputil.flagsmaildir2imap(flags)) + + except imapobj.readonly: + self.ui.flagstoreadonly(self, [uid], data) + return + finally: self.imapserver.releaseconnection(imapobj) - result = result[1][0] + if not result: self.messagelist[uid]['flags'] = flags else: From 0e4afa913253c43409e6a32a6b6e11e8b03ed3d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abd=C3=B3=20Roig-Maranges?= Date: Tue, 16 Oct 2012 20:20:35 +0200 Subject: [PATCH 597/817] Make GmailFolder sync GMail labels When synclabels config flag is set to "yes" for the GMail repo, offlineimap fetches the message labels along with the messages, and embeds them into the body under the header X-Keywords (or whatever 'labelsheader' was set to), as a comma separated list. It also adds an extra pass to savemessageto, that performs label synchronization on existing messages from GMail to local, the same way it is done with flags. We also introduce GmailMaildir repository that adds functionality to change message labels. It keeps track of messages modification time, so one can quickly detect when the labels may have changed. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 + docs/MANUAL.rst | 35 +++ offlineimap.conf | 40 ++- offlineimap/folder/Base.py | 125 ++++++++-- offlineimap/folder/Gmail.py | 319 ++++++++++++++++++++++++ offlineimap/folder/GmailMaildir.py | 315 +++++++++++++++++++++++ offlineimap/folder/IMAP.py | 11 +- offlineimap/folder/LocalStatusSQLite.py | 115 +++++++-- offlineimap/folder/Maildir.py | 80 ++++-- offlineimap/imaputil.py | 10 + offlineimap/repository/Gmail.py | 8 +- offlineimap/repository/GmailMaildir.py | 37 +++ offlineimap/repository/Maildir.py | 10 +- offlineimap/repository/__init__.py | 5 +- offlineimap/ui/UIBase.py | 28 +++ 15 files changed, 1071 insertions(+), 69 deletions(-) create mode 100644 offlineimap/folder/GmailMaildir.py create mode 100644 offlineimap/repository/GmailMaildir.py diff --git a/Changelog.rst b/Changelog.rst index 0d3dca9..d2148e6 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,8 @@ OfflineIMAP v6.5.6 (YYYY-MM-DD) * Add knob to apply compression to IMAP connections (Abdó Roig-Maranges) * Add knob to filter some headers before uploading message to IMAP server (Abdó Roig-Maranges) +* Allow to sync GMail labels and implement GmailMaildir repository that + adds mechanics to change message labels (Abdó Roig-Maranges) OfflineIMAP v6.5.5 (2013-10-07) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 936abc3..1c0ce0f 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -429,6 +429,41 @@ This is an example of a setup where "TheOtherImap" requires all folders to be un remoteuser = XXX #Do not use nametrans here. + +Sync from Gmail to a local Maildir with labels +---------------------------------------------- + +This is an example of a setup where GMail gets synced with a local Maildir. +It also keeps track of GMail labels, that get embedded into the messages +under the header X-Keywords (or whatever labelsheader is set to), and syncs +them back and forth the same way as flags. + +The first time it runs on a large repository may take some time as the labels +are read / embedded on every message. Afterwards local label changes are detected +using modification times (much faster):: + + [Account Gmail-mine] + localrepository = Gmaillocal-mine + remoterepository = Gmailserver-mine + # Need this to be able to sync labels + status_backend = sqlite + synclabels = yes + # This header is where labels go. Usually you will be fine + # with default value, but in case you want it different, + # here we go: + labelsheader = X-GMail-Keywords + + [Repository Gmailserver-mine] + #This is the remote repository + type = Gmail + remotepass = XXX + remoteuser = XXX + + [Repository Gmaillocal-mine] + #This is the 'local' repository + type = GmailMaildir + + Selecting only a few folders to sync ------------------------------------ Add this to the remote gmail repository section to only sync mails which are in a certain folder:: diff --git a/offlineimap.conf b/offlineimap.conf index 333e52e..3b219d4 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -271,6 +271,22 @@ remoterepository = RemoteExample # #maildir-windows-compatible = no +# Specifies if we want to sync GMail lables with the local repository. +# Effective only for GMail IMAP repositories. You should use SQlite +# backend for this to work (see status_backend). +# +#synclabels = no + +# Name of the header to use for label storage. +# +#labelsheader = X-Keywords + +# Set of labels to be ignored. Comma-separated list. GMail-specific +# labels all start with backslash ('\'). +# +#ignorelabels = \Inbox, \Starred, \Sent, \Draft, \Spam, \Trash, \Important + + # OfflineIMAP can strip off some headers when your messages are propagated # back to the IMAP server. This option carries the comma-separated list @@ -283,10 +299,11 @@ remoterepository = RemoteExample #filterheaders = X-Some-Weird-Header + [Repository LocalExample] # Each repository requires a "type" declaration. The types supported for -# local repositories are Maildir and IMAP. +# local repositories are Maildir, GmailMaildir and IMAP. type = Maildir @@ -313,6 +330,24 @@ localfolders = ~/Test #restoreatime = no + +[Repository GmailLocalExample] + +# This type of repository enables syncing of Gmail. All Maildir +# configuration settings are also valid here. +# +# This is a separate Repository type from Maildir because it involves +# some extra overhead which sometimes may be significant. We look for +# modified tags in local messages by looking only to the files +# modified since last run. This is usually rather fast, but the first +# time OfflineIMAP runs with synclabels enabled, it will have to check +# the contents of all individual messages for labels and this may take +# a while. + +type = GmailMaildir + + + [Repository RemoteExample] # And this is the remote repository. We only support IMAP or Gmail here. @@ -658,3 +693,6 @@ remoteuser = username@gmail.com # Enable 1-way synchronization. See above for explanation. # #readonly = False +# +# To enable GMail labels synchronisation, set the option synclabels +# in the corresponding "Account" section. diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 13d43ae..67e678d 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -273,6 +273,10 @@ class BaseFolder(object): """Return the received time for the specified message.""" raise NotImplementedError + def getmessagemtime(self, uid): + """Returns the message modification time of the specified message.""" + raise NotImplementedError + def getmessageflags(self, uid): """Returns the flags for the specified message.""" raise NotImplementedError @@ -324,24 +328,116 @@ class BaseFolder(object): self.deletemessageflags(uid, flags) + def getmessagelabels(self, uid): + """Returns the labels for the specified message.""" + raise NotImplementedError + + def savemessagelabels(self, uid, labels, ignorelabels=set(), mtime=0): + """Sets the specified message's labels to the given set. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + raise NotImplementedError + + def addmessagelabels(self, uid, labels): + """Adds the specified labels to the message's labels set. If a given + label is already present, it will not be duplicated. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + + :param labels: A set() of labels""" + newlabels = self.getmessagelabels(uid) | labels + self.savemessagelabels(uid, newlabels) + + def addmessageslabels(self, uidlist, labels): + """ + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + for uid in uidlist: + self.addmessagelabels(uid, labels) + + def deletemessagelabels(self, uid, labels): + """Removes each label given from the message's label set. If a given + label is already removed, no action will be taken for that label. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + newlabels = self.getmessagelabels(uid) - labels + self.savemessagelabels(uid, newlabels) + + def deletemessageslabels(self, uidlist, labels): + """ + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + for uid in uidlist: + self.deletemessagelabels(uid, labels) + + + """ + Illustration of all cases for addmessageheader(). + '+' means the added contents. + +Case 1: No '\n\n', leading '\n' ++X-Flying-Pig-Header: i am here\n +\n +This is the body\n +next line\n + +Case 2: '\n\n' at position 0 ++X-Flying-Pig-Header: i am here\n +\n +\n +This is the body\n +next line\n + +Case 3: No '\n\n', no leading '\n' ++X-Flying-Pig-Header: i am here\n ++\n +This is the body\n +next line\n + +Case 4: '\n\n' at non-zero position +Subject: Something wrong with OI\n +From: some@person.at+\n +X-Flying-Pig-Header: i am here\n <-- orig '\n' +\n +This is the body\n +next line\n + + """ + def addmessageheader(self, content, headername, headervalue): self.ui.debug('', 'addmessageheader: called to add %s: %s' % (headername, headervalue)) + prefix = '\n' + suffix = '' insertionpoint = content.find('\n\n') - self.ui.debug('', 'addmessageheader: insertionpoint = %d' % insertionpoint) - leader = content[0:insertionpoint] - self.ui.debug('', 'addmessageheader: leader = %s' % repr(leader)) if insertionpoint == 0 or insertionpoint == -1: - newline = '' + prefix = '' + suffix = '\n' + if insertionpoint == -1: insertionpoint = 0 - else: - newline = '\n' - newline += "%s: %s" % (headername, headervalue) - self.ui.debug('', 'addmessageheader: newline = ' + repr(newline)) - trailer = content[insertionpoint:] - self.ui.debug('', 'addmessageheader: trailer = ' + repr(trailer)) - return leader + newline + trailer + # When body starts immediately, without preceding '\n' + # (this shouldn't happen with proper mail messages, but + # we seen many broken ones), we should add '\n' to make + # new (and the only header, in this case) to be properly + # separated from the message body. + if content[0] != '\n': + suffix = suffix + '\n' + + self.ui.debug('', 'addmessageheader: insertionpoint = %d' % insertionpoint) + headers = content[0:insertionpoint] + self.ui.debug('', 'addmessageheader: headers = %s' % repr(headers)) + new_header = prefix + ("%s: %s" % (headername, headervalue)) + suffix + self.ui.debug('', 'addmessageheader: new_header = ' + repr(new_header)) + return headers + new_header + content[insertionpoint:] def __find_eoh(self, content): @@ -607,9 +703,10 @@ class BaseFolder(object): continue selfflags = self.getmessageflags(uid) - statusflags = statusfolder.getmessageflags(uid) - #if we could not get message flags from LocalStatus, assume empty. - if statusflags is None: + + if statusfolder.uidexists(uid): + statusflags = statusfolder.getmessageflags(uid) + else: statusflags = set() addflags = selfflags - statusflags diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index e3433c0..b735c29 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -16,6 +16,12 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import re + +from offlineimap import imaputil +from offlineimap import imaplibutil +import offlineimap.accounts + """Folder implementation to support features of the Gmail IMAP server. """ from .IMAP import IMAPFolder @@ -33,6 +39,7 @@ class GmailFolder(IMAPFolder): For more information on the Gmail IMAP server: http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 + https://developers.google.com/google-apps/gmail/imap_extensions """ def __init__(self, imapserver, name, repository): @@ -40,3 +47,315 @@ class GmailFolder(IMAPFolder): self.trash_folder = repository.gettrashfolder(name) # Gmail will really delete messages upon EXPUNGE in these folders self.real_delete_folders = [ self.trash_folder, repository.getspamfolder() ] + + # The header under which labels are stored + self.labelsheader = self.repository.account.getconf('labelsheader', 'X-Keywords') + + # enables / disables label sync + self.synclabels = self.repository.account.getconfboolean('synclabels', False) + + # if synclabels is enabled, add a 4th pass to sync labels + if self.synclabels: + self.imap_query.insert(0, 'X-GM-LABELS') + self.syncmessagesto_passes.append(('syncing labels', self.syncmessagesto_labels)) + + # Labels to be left alone + ignorelabels = self.repository.account.getconf('ignorelabels', '') + self.ignorelabels = set([l for l in re.split(r'\s*,\s*', ignorelabels) if len(l)]) + + + def getmessage(self, uid): + """Retrieve message with UID from the IMAP server (incl body). Also + gets Gmail labels and embeds them into the message. + + :returns: the message body or throws and OfflineImapError + (probably severity MESSAGE) if e.g. no message with + this UID could be found. + """ + imapobj = self.imapserver.acquireconnection() + try: + data = self._fetch_from_imap(imapobj, str(uid), 2) + finally: + self.imapserver.releaseconnection(imapobj) + + # data looks now e.g. + #[('320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....')] + # we only asked for one message, and that msg is in data[0]. + # msbody is in [0][1]. + body = data[0][1].replace("\r\n", "\n") + + # Embed the labels into the message headers + if self.synclabels: + m = re.search('X-GM-LABELS\s*\(([^\)]*)\)', data[0][0]) + if m: + labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) + else: + labels = set() + labels = labels - self.ignorelabels + labels = ', '.join(sorted(labels)) + body = self.addmessageheader(body, self.labelsheader, labels) + + if len(body)>200: + dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:]) + else: + dbg_output = body + + self.ui.debug('imap', "Returned object from fetching %d: '%s'" % + (uid, dbg_output)) + return body + + def getmessagelabels(self, uid): + if 'labels' in self.messagelist[uid]: + return self.messagelist[uid]['labels'] + else: + return set() + + # TODO: merge this code with the parent's cachemessagelist: + # TODO: they have too much common logics. + def cachemessagelist(self): + if not self.synclabels: + return super(GmailFolder, self).cachemessagelist() + + self.messagelist = {} + + self.ui.collectingdata(None, self) + imapobj = self.imapserver.acquireconnection() + try: + msgsToFetch = self._msgs_to_fetch(imapobj) + if not msgsToFetch: + return # No messages to sync + + # Get the flags and UIDs for these. single-quotes prevent + # imaplib2 from quoting the sequence. + # + # NB: msgsToFetch are sequential numbers, not UID's + res_type, response = imapobj.fetch("'%s'" % msgsToFetch, + '(FLAGS X-GM-LABELS UID)') + if res_type != 'OK': + raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " % \ + (self.getrepository(), self) + \ + "Server responded '[%s] %s'" % \ + (res_type, response), OfflineImapError.ERROR.FOLDER) + finally: + self.imapserver.releaseconnection(imapobj) + + for messagestr in response: + # looks like: '1 (FLAGS (\\Seen Old) X-GM-LABELS (\\Inbox \\Favorites) UID 4807)' or None if no msg + # Discard initial message number. + if messagestr == None: + continue + messagestr = messagestr.split(' ', 1)[1] + options = imaputil.flags2hash(messagestr) + if not 'UID' in options: + self.ui.warn('No UID in message with options %s' %\ + str(options), + minor = 1) + else: + uid = long(options['UID']) + flags = imaputil.flagsimap2maildir(options['FLAGS']) + m = re.search('\(([^\)]*)\)', options['X-GM-LABELS']) + if m: + labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) + else: + labels = set() + labels = labels - self.ignorelabels + rtime = imaplibutil.Internaldate2epoch(messagestr) + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime} + + def savemessage(self, uid, content, flags, rtime): + """Save the message on the Server + + This backend always assigns a new uid, so the uid arg is ignored. + + This function will update the self.messagelist dict to contain + the new message after sucessfully saving it, including labels. + + See folder/Base for details. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode. + + :param rtime: A timestamp to be used as the mail date + :returns: the UID of the new message as assigned by the server. If the + message is saved, but it's UID can not be found, it will + return 0. If the message can't be written (folder is + read-only for example) it will return -1.""" + + if not self.synclabels: + return super(GmailFolder, self).savemessage(uid, content, flags, rtime) + + labels = self.getmessageheader(content, self.labelsheader) + if labels: + labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) + else: + labels = set() + + ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) + self.savemessagelabels(ret, labels) + return ret + + def _messagelabels_aux(self, arg, uidlist, labels): + """Common code to savemessagelabels and addmessagelabels""" + labels = labels - self.ignorelabels + uidlist = [uid for uid in uidlist if uid > 0] + if len(uidlist) > 0: + imapobj = self.imapserver.acquireconnection() + try: + labels_str = '(' + ' '.join([imaputil.quote(lb) for lb in labels]) + ')' + # Coalesce uid's into ranges + uid_str = imaputil.uid_sequence(uidlist) + result = self._store_to_imap(imapobj, uid_str, arg, labels_str) + + except imapobj.readonly: + self.ui.labelstoreadonly(self, uidlist, data) + return None + + finally: + self.imapserver.releaseconnection(imapobj) + + if result: + retlabels = imaputil.flags2hash(imaputil.imapsplit(result)[1])['X-GM-LABELS'] + retlabels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(retlabels)]) + return retlabels + return None + + def savemessagelabels(self, uid, labels): + """Change a message's labels to `labels`. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a dryrun mode.""" + if uid in self.messagelist and 'labels' in self.messagelist[uid]: + oldlabels = self.messagelist[uid]['labels'] + else: + oldlabels = set() + labels = labels - self.ignorelabels + newlabels = labels | (oldlabels & self.ignorelabels) + if oldlabels != newlabels: + result = self._messagelabels_aux('X-GM-LABELS', [uid], newlabels) + if result: + self.messagelist[uid]['labels'] = newlabels + else: + self.messagelist[uid]['labels'] = oldlabels + + def addmessageslabels(self, uidlist, labels): + """Add `labels` to all messages in uidlist. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a dryrun mode.""" + + labels = labels - self.ignorelabels + result = self._messagelabels_aux('+X-GM-LABELS', uidlist, labels) + if result: + for uid in uidlist: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels + + def deletemessageslabels(self, uidlist, labels): + """Delete `labels` from all messages in uidlist. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a dryrun mode.""" + + labels = labels - self.ignorelabels + result = self._messagelabels_aux('-X-GM-LABELS', uidlist, labels) + if result: + for uid in uidlist: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels + + def copymessageto(self, uid, dstfolder, statusfolder, register = 1): + """Copies a message from self to dst if needed, updating the status + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + + :param uid: uid of the message to be copied. + :param dstfolder: A BaseFolder-derived instance + :param statusfolder: A LocalStatusFolder instance + :param register: whether we should register a new thread." + :returns: Nothing on success, or raises an Exception.""" + + # Check if we are really copying + realcopy = uid > 0 and not dstfolder.uidexists(uid) + + # first copy the message + super(GmailFolder, self).copymessageto(uid, dstfolder, statusfolder, register) + + # sync labels and mtime now when the message is new (the embedded labels are up to date) + # otherwise we may be spending time for nothing, as they will get updated on a later pass. + if realcopy and self.synclabels: + try: + mtime = dstfolder.getmessagemtime(uid) + labels = dstfolder.getmessagelabels(uid) + statusfolder.savemessagelabels(uid, labels, mtime=mtime) + + # either statusfolder is not sqlite or dstfolder is not GmailMaildir. + except NotImplementedError: + return + + def syncmessagesto_labels(self, dstfolder, statusfolder): + """Pass 4: Label Synchronization (Gmail only) + + Compare label mismatches in self with those in statusfolder. If + msg has a valid UID and exists on dstfolder (has not e.g. been + deleted there), sync the labels change to both dstfolder and + statusfolder. + + This function checks and protects us from action in dryrun mode. + """ + # This applies the labels message by message, as this makes more sense for a + # Maildir target. If applied with an other Gmail IMAP target it would not be + # the fastest thing in the world though... + uidlist = [] + + # filter the uids (fast) + try: + for uid in self.getmessageuidlist(): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + # Ignore messages with negative UIDs missed by pass 1 and + # don't do anything if the message has been deleted remotely + if uid < 0 or not dstfolder.uidexists(uid): + continue + + selflabels = self.getmessagelabels(uid) - self.ignorelabels + + if statusfolder.uidexists(uid): + statuslabels = statusfolder.getmessagelabels(uid) - self.ignorelabels + else: + statuslabels = set() + + if selflabels != statuslabels: + uidlist.append(uid) + + # now sync labels (slow) + mtimes = {} + labels = {} + for i, uid in enumerate(uidlist): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + selflabels = self.getmessagelabels(uid) - self.ignorelabels + + if statusfolder.uidexists(uid): + statuslabels = statusfolder.getmessagelabels(uid) - self.ignorelabels + else: + statuslabels = set() + + if selflabels != statuslabels: + self.ui.settinglabels(uid, i+1, len(uidlist), sorted(selflabels), dstfolder) + if self.repository.account.dryrun: + continue #don't actually add in a dryrun + dstfolder.savemessagelabels(uid, selflabels, ignorelabels = self.ignorelabels) + mtime = dstfolder.getmessagemtime(uid) + mtimes[uid] = mtime + labels[uid] = selflabels + + # Update statusfolder in a single DB transaction. It is safe, as if something fails, + # statusfolder will be updated on the next run. + statusfolder.savemessageslabelsbulk(labels) + statusfolder.savemessagesmtimebulk(mtimes) + + except NotImplementedError: + self.ui.warn("Can't sync labels. You need to configure a local repository of type GmailMaildir") diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py new file mode 100644 index 0000000..7e903e6 --- /dev/null +++ b/offlineimap/folder/GmailMaildir.py @@ -0,0 +1,315 @@ +# Maildir folder support with labels +# Copyright (C) 2002 - 2011 John Goerzen & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +import os +from .Maildir import MaildirFolder +from offlineimap import OfflineImapError +import offlineimap.accounts + +class GmailMaildirFolder(MaildirFolder): + """Folder implementation to support adding labels to messages in a Maildir. + """ + def __init__(self, root, name, sep, repository): + super(GmailMaildirFolder, self).__init__(root, name, sep, repository) + + # The header under which labels are stored + self.labelsheader = self.repository.account.getconf('labelsheader', 'X-Keywords') + + # enables / disables label sync + self.synclabels = self.repository.account.getconfboolean('synclabels', 0) + + # if synclabels is enabled, add a 4th pass to sync labels + if self.synclabels: + self.syncmessagesto_passes.append(('syncing labels', self.syncmessagesto_labels)) + + def quickchanged(self, statusfolder): + """Returns True if the Maildir has changed. Checks uids, flags and mtimes""" + + self.cachemessagelist() + # Folder has different uids than statusfolder => TRUE + if sorted(self.getmessageuidlist()) != \ + sorted(statusfolder.getmessageuidlist()): + return True + # check for flag changes, it's quick on a Maildir + for (uid, message) in self.getmessagelist().iteritems(): + if message['flags'] != statusfolder.getmessageflags(uid): + return True + # check for newer mtimes. it is also fast + for (uid, message) in self.getmessagelist().iteritems(): + if message['mtime'] > statusfolder.getmessagemtime(uid): + return True + return False #Nope, nothing changed + + def cachemessagelist(self): + if self.messagelist is None: + self.messagelist = self._scanfolder() + + # Get mtimes + if self.synclabels: + for uid, msg in self.messagelist.items(): + filepath = os.path.join(self.getfullname(), msg['filename']) + msg['mtime'] = long(os.stat(filepath).st_mtime) + + def getmessagelabels(self, uid): + # Labels are not cached in cachemessagelist because it is too slow. + if not 'labels' in self.messagelist[uid]: + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + + if not os.path.exists(filepath): + return set() + + file = open(filepath, 'rt') + content = file.read() + file.close() + + labels = self.getmessageheader(content, self.labelsheader) + if labels: + labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) + else: + labels = set() + self.messagelist[uid]['labels'] = labels + + return self.messagelist[uid]['labels'] + + def getmessagemtime(self, uid): + if not 'mtime' in self.messagelist[uid]: + return 0 + else: + return self.messagelist[uid]['mtime'] + + def savemessage(self, uid, content, flags, rtime): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode.""" + + if not self.synclabels: + return super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) + + labels = self.getmessageheader(content, self.labelsheader) + if labels: + labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) + else: + labels = set() + ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) + + # Update the mtime and labels + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + self.messagelist[uid]['mtime'] = long(os.stat(filepath).st_mtime) + self.messagelist[uid]['labels'] = labels + return ret + + def savemessagelabels(self, uid, labels, ignorelabels=set()): + """Change a message's labels to `labels`. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a dryrun mode.""" + + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + + file = open(filepath, 'rt') + content = file.read() + file.close() + + oldlabels = self.getmessageheader(content, self.labelsheader) + + if oldlabels: + oldlabels = set([lb.strip() for lb in oldlabels.split(',') if len(lb.strip()) > 0]) + else: + oldlabels = set() + + labels = labels - ignorelabels + ignoredlabels = oldlabels & ignorelabels + oldlabels = oldlabels - ignorelabels + + # Nothing to change + if labels == oldlabels: + return + + # Change labels into content + labels_str = ', '.join(sorted(labels | ignoredlabels)) + content = self.addmessageheader(content, self.labelsheader, labels_str) + rtime = self.messagelist[uid].get('rtime', None) + + # write file with new labels to a unique file in tmp + messagename = self.new_message_filename(uid, set()) + tmpname = self.save_tmp_file(messagename, content) + tmppath = os.path.join(self.getfullname(), tmpname) + + # move to actual location + try: + os.rename(tmppath, filepath) + except OSError as e: + raise OfflineImapError("Can't rename file '%s' to '%s': %s" % \ + (tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER) + + if rtime != None: + os.utime(filepath, (rtime, rtime)) + + # save the new mtime and labels + self.messagelist[uid]['mtime'] = long(os.stat(filepath).st_mtime) + self.messagelist[uid]['labels'] = labels + + def copymessageto(self, uid, dstfolder, statusfolder, register = 1): + """Copies a message from self to dst if needed, updating the status + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + + :param uid: uid of the message to be copied. + :param dstfolder: A BaseFolder-derived instance + :param statusfolder: A LocalStatusFolder instance + :param register: whether we should register a new thread." + :returns: Nothing on success, or raises an Exception.""" + + # Check if we are really copying + realcopy = uid > 0 and not dstfolder.uidexists(uid) + + # first copy the message + super(GmailMaildirFolder, self).copymessageto(uid, dstfolder, statusfolder, register) + + # sync labels and mtime now when the message is new (the embedded labels are up to date, + # and have already propagated to the remote server. + # for message which already existed on the remote, this is useless, as later the labels may + # get updated. + if realcopy and self.synclabels: + try: + labels = dstfolder.getmessagelabels(uid) + statusfolder.savemessagelabels(uid, labels, mtime=self.getmessagemtime(uid)) + + # either statusfolder is not sqlite or dstfolder is not GmailMaildir. + except NotImplementedError: + return + + def syncmessagesto_labels(self, dstfolder, statusfolder): + """Pass 4: Label Synchronization (Gmail only) + + Compare label mismatches in self with those in statusfolder. If + msg has a valid UID and exists on dstfolder (has not e.g. been + deleted there), sync the labels change to both dstfolder and + statusfolder. + + Also skips messages whose mtime remains the same as statusfolder, as the + contents have not changed. + + This function checks and protects us from action in ryrun mode. + """ + # For each label, we store a list of uids to which it should be + # added. Then, we can call addmessageslabels() to apply them in + # bulk, rather than one call per message. + addlabellist = {} + dellabellist = {} + uidlist = [] + + try: + # filter uids (fast) + for uid in self.getmessageuidlist(): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + # Ignore messages with negative UIDs missed by pass 1 and + # don't do anything if the message has been deleted remotely + if uid < 0 or not dstfolder.uidexists(uid): + continue + + selfmtime = self.getmessagemtime(uid) + + if statusfolder.uidexists(uid): + statusmtime = statusfolder.getmessagemtime(uid) + else: + statusmtime = 0 + + if selfmtime > statusmtime: + uidlist.append(uid) + + + self.ui.collectingdata(uidlist, self) + # This can be slow if there is a lot of modified files + for uid in uidlist: + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + selflabels = self.getmessagelabels(uid) + + if statusfolder.uidexists(uid): + statuslabels = statusfolder.getmessagelabels(uid) + else: + statuslabels = set() + + addlabels = selflabels - statuslabels + dellabels = statuslabels - selflabels + + for lb in addlabels: + if not lb in addlabellist: + addlabellist[lb] = [] + addlabellist[lb].append(uid) + + for lb in dellabels: + if not lb in dellabellist: + dellabellist[lb] = [] + dellabellist[lb].append(uid) + + for lb, uids in addlabellist.items(): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + self.ui.addinglabels(uids, lb, dstfolder) + if self.repository.account.dryrun: + continue #don't actually add in a dryrun + dstfolder.addmessageslabels(uids, set([lb])) + statusfolder.addmessageslabels(uids, set([lb])) + + for lb, uids in dellabellist.items(): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + self.ui.deletinglabels(uids, lb, dstfolder) + if self.repository.account.dryrun: + continue #don't actually remove in a dryrun + dstfolder.deletemessageslabels(uids, set([lb])) + statusfolder.deletemessageslabels(uids, set([lb])) + + # Update mtimes on StatusFolder. It is done last to be safe. If something els fails + # and the mtime is not updated, the labels will still be synced next time. + mtimes = {} + for uid in uidlist: + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + if self.repository.account.dryrun: + continue #don't actually update statusfolder + + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + mtimes[uid] = long(os.stat(filepath).st_mtime) + + # finally update statusfolder in a single DB transaction + statusfolder.savemessagesmtimebulk(mtimes) + + except NotImplementedError: + self.ui.warn("Can't sync labels. You need to configure a remote repository of type Gmail.") diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index dd87165..3e8d94c 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -142,12 +142,15 @@ class IMAPFolder(BaseFolder): def _msgs_to_fetch(self, imapobj): """ - Determines UIDS of messages to be fetched + Determines sequence numbers of messages to be fetched. + + Message sequence numbers (MSNs) are more easily compacted + into ranges which makes transactions slightly faster. Arguments: - imapobj: instance of IMAPlib - Returns: UID ranges for messages or None if no messages + Returns: range(s) for messages or None if no messages are to be fetched. """ @@ -156,7 +159,7 @@ class IMAPFolder(BaseFolder): # Empty folder, no need to populate message list return None - # By default examine all UIDs in this folder + # By default examine all messages in this folder msgsToFetch = '1:*' maxage = self.config.getdefaultint("Account %s" % self.accountname, @@ -194,7 +197,7 @@ class IMAPFolder(BaseFolder): self.getrepository(), self, search_cond, res_type, res_data), OfflineImapError.ERROR.FOLDER) - # Result UIDs are seperated by space, coalesce into ranges + # Resulting MSN are separated by space, coalesce into ranges msgsToFetch = imaputil.uid_sequence(res_data[0].split()) return msgsToFetch diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index a22c68b..17a1d23 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -40,7 +40,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): #return connection, cursor #current version of our db format - cur_version = 1 + cur_version = 2 def __init__(self, name, repository): super(LocalStatusSQLiteFolder, self).__init__(name, repository) @@ -140,21 +140,36 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): self.connection.commit() file.close() os.rename(plaintextfilename, plaintextfilename + ".old") + + # Upgrade from database version 1 to version 2 + # This change adds labels and mtime columns, to be used by Gmail IMAP and Maildir folders. + if from_ver <= 1: + self.ui._msg('Upgrading LocalStatus cache from version 1 to version 2 for %s:%s' %\ + (self.repository, self)) + self.connection.executescript("""ALTER TABLE status ADD mtime INTEGER DEFAULT 0; + ALTER TABLE status ADD labels VARCHAR(256) DEFAULT ''; + UPDATE metadata SET value='2' WHERE key='db_version'; + """) + self.connection.commit() + # Future version upgrades come here... - # if from_ver <= 1: ... #upgrade from 1 to 2 # if from_ver <= 2: ... #upgrade from 2 to 3 + # if from_ver <= 3: ... #upgrade from 3 to 4 + def __create_db(self): - """Create a new db file""" + """ + Create a new db file. + + self.connection must point to the opened and valid SQlite + database connection. + """ self.ui._msg('Creating new Local Status db for %s:%s' \ % (self.repository, self)) - if hasattr(self, 'connection'): - self.connection.close() #close old connections first - self.connection = sqlite.connect(self.filename, check_same_thread = False) self.connection.executescript(""" CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128)); INSERT INTO metadata VALUES('db_version', '1'); - CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50)); + CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50), mtime INTEGER, labels VARCHAR(256)); """) self.connection.commit() @@ -173,10 +188,11 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): # Interface from BaseFolder def cachemessagelist(self): self.messagelist = {} - cursor = self.connection.execute('SELECT id,flags from status') + cursor = self.connection.execute('SELECT id,flags,mtime,labels from status') for row in cursor: - flags = set(row[1]) - self.messagelist[row[0]] = {'uid': row[0], 'flags': flags} + flags = set(row[1]) + labels = set([lb.strip() for lb in row[3].split(',') if len(lb.strip()) > 0]) + self.messagelist[row[0]] = {'uid': row[0], 'flags': flags, 'mtime': row[2], 'labels': labels} # Interface from LocalStatusFolder def save(self): @@ -220,12 +236,15 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): # assert False,"getmessageflags() called on non-existing message" # Interface from BaseFolder - def savemessage(self, uid, content, flags, rtime): - """Writes a new message, with the specified uid. + def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): + """ + Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not check against dryrun settings, so you need to ensure that - savemessage is never called in a dryrun mode.""" + savemessage is never called in a dryrun mode. + + """ if uid < 0: # We cannot assign a uid. return uid @@ -234,10 +253,11 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): self.savemessageflags(uid, flags) return uid - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels} flags = ''.join(sorted(flags)) - self.__sql_write('INSERT INTO status (id,flags) VALUES (?,?)', - (uid,flags)) + labels = ', '.join(sorted(labels)) + self.__sql_write('INSERT INTO status (id,flags,mtime,labels) VALUES (?,?,?,?)', + (uid,flags,mtime,labels)) return uid # Interface from BaseFolder @@ -246,6 +266,69 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): flags = ''.join(sorted(flags)) self.__sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) + + def getmessageflags(self, uid): + return self.messagelist[uid]['flags'] + + + def savemessagelabels(self, uid, labels, mtime=None): + self.messagelist[uid]['labels'] = labels + if mtime: self.messagelist[uid]['mtime'] = mtime + + labels = ', '.join(sorted(labels)) + if mtime: + self.__sql_write('UPDATE status SET labels=?, mtime=? WHERE id=?',(labels,mtime,uid)) + else: + self.__sql_write('UPDATE status SET labels=? WHERE id=?',(labels,uid)) + + + def savemessageslabelsbulk(self, labels): + """ + Saves labels from a dictionary in a single database operation. + + """ + data = [(', '.join(sorted(l)), uid) for uid, l in labels.items()] + self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) + for uid, l in labels.items(): + self.messagelist[uid]['labels'] = l + + + def addmessageslabels(self, uids, labels): + data = [] + for uid in uids: + newlabels = self.messagelist[uid]['labels'] | labels + data.append((', '.join(sorted(newlabels)), uid)) + self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels + + + def deletemessageslabels(self, uids, labels): + data = [] + for uid in uids: + newlabels = self.messagelist[uid]['labels'] - labels + data.append((', '.join(sorted(newlabels)), uid)) + self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels + + + def getmessagelabels(self, uid): + return self.messagelist[uid]['labels'] + + + def savemessagesmtimebulk(self, mtimes): + """Saves mtimes from the mtimes dictionary in a single database operation.""" + data = [(mt, uid) for uid, mt in mtimes.items()] + self.__sql_write('UPDATE status SET mtime=? WHERE id=?', data, executemany=True) + for uid, mt in mtimes.items(): + self.messagelist[uid]['mtime'] = mt + + + def getmessagemtime(self, uid): + return self.messagelist[uid]['mtime'] + + # Interface from BaseFolder def deletemessage(self, uid): if not uid in self.messagelist: diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index ba55e74..0314f62 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -19,6 +19,7 @@ import socket import time import re import os +import tempfile from .Base import BaseFolder from threading import Lock @@ -234,7 +235,7 @@ class MaildirFolder(BaseFolder): filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) - def __new_message_filename(self, uid, flags=set()): + def new_message_filename(self, uid, flags=set()): """Creates a new unique Maildir filename :param uid: The UID`None`, or a set of maildir flags @@ -245,6 +246,52 @@ class MaildirFolder(BaseFolder): (timeval, timeseq, os.getpid(), socket.gethostname(), uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) + + def save_to_tmp_file(self, filename, content): + """ + Saves given content to the named temporary file in the + 'tmp' subdirectory of $CWD. + + Arguments: + - filename: name of the temporary file; + - content: data to be saved. + + Returns: relative path to the temporary file + that was created. + + """ + + tmpname = os.path.join('tmp', filename) + # open file and write it out + tries = 7 + while tries: + tries = tries - 1 + try: + fd = os.open(os.path.join(self.getfullname(), tmpname), + os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666) + break + except OSError as e: + if e.errno == e.EEXIST: + if tries: + time.slep(0.23) + continue + severity = OfflineImapError.ERROR.MESSAGE + raise OfflineImapError("Unique filename %s already exists." % \ + filename, severity) + else: + raise + + fd = os.fdopen(fd, 'wt') + fd.write(content) + # Make sure the data hits the disk + fd.flush() + if self.dofsync: + os.fsync(fd) + fd.close() + + return tmpname + + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. @@ -267,33 +314,12 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - messagename = self.__new_message_filename(uid, flags) - # open file and write it out - try: - fd = os.open(os.path.join(tmpdir, messagename), - os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666) - except OSError as e: - if e.errno == 17: - #FILE EXISTS ALREADY - severity = OfflineImapError.ERROR.MESSAGE - raise OfflineImapError("Unique filename %s already existing." %\ - messagename, severity) - else: - raise - - file = os.fdopen(fd, 'wt') - file.write(content) - # Make sure the data hits the disk - file.flush() - if self.dofsync: - os.fsync(fd) - file.close() - + messagename = self.new_message_filename(uid, flags) + tmpname = self.save_to_tmp_file(messagename, content) if rtime != None: - os.utime(os.path.join(tmpdir, messagename), (rtime, rtime)) + os.utime(os.path.join(self.getfullname(), tmpname), (rtime, rtime)) - self.messagelist[uid] = {'flags': flags, - 'filename': os.path.join('tmp', messagename)} + self.messagelist[uid] = {'flags': flags, 'filename': tmpname} # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) self.ui.debug('maildir', 'savemessage: returning uid %d' % uid) @@ -357,7 +383,7 @@ class MaildirFolder(BaseFolder): dir_prefix, filename = os.path.split(oldfilename) flags = self.getmessageflags(uid) newfilename = os.path.join(dir_prefix, - self.__new_message_filename(new_uid, flags)) + self.new_message_filename(new_uid, flags)) os.rename(os.path.join(self.getfullname(), oldfilename), os.path.join(self.getfullname(), newfilename)) self.messagelist[new_uid] = self.messagelist[uid] diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 5d69f59..cdf3c4e 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -39,6 +39,16 @@ def dequote(string): string = string.replace('\\\\', '\\') return string +def quote(string): + """Takes an unquoted string and quotes it. + + It only adds double quotes. This function does NOT consider + parenthised lists to be quoted. + """ + string = string.replace('"', '\\"') + string = string.replace('\\', '\\\\') + return '"%s"' % string + def flagsplit(string): """Converts a string of IMAP flags to a list diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 5f86ed3..61d4486 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -36,6 +36,13 @@ class GmailRepository(IMAPRepository): 'ssl', 'yes') IMAPRepository.__init__(self, reposname, account) + if self.account.getconfboolean('synclabels', 0) and \ + self.account.getconf('status_backend', 'plain') != 'sqlite': + raise OfflineImapError("The Gmail repository needs the sqlite backend to sync labels.\n" + "To enable it add 'status_backend = sqlite' in the account section", + OfflineImapError.ERROR.REPO) + + def gethost(self): """Return the server name to connect to. @@ -71,4 +78,3 @@ class GmailRepository(IMAPRepository): def getspamfolder(self): #: Gmail also deletes messages upon EXPUNGE in the Spam folder return self.getconf('spamfolder','[Gmail]/Spam') - diff --git a/offlineimap/repository/GmailMaildir.py b/offlineimap/repository/GmailMaildir.py new file mode 100644 index 0000000..62f9f83 --- /dev/null +++ b/offlineimap/repository/GmailMaildir.py @@ -0,0 +1,37 @@ +# Maildir repository support +# Copyright (C) 2002 John Goerzen +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +from offlineimap.repository.Maildir import MaildirRepository +from offlineimap.folder.GmailMaildir import GmailMaildirFolder +from offlineimap.error import OfflineImapError + +class GmailMaildirRepository(MaildirRepository): + def __init__(self, reposname, account): + """Initialize a MaildirRepository object. Takes a path name + to the directory holding all the Maildir directories.""" + super(GmailMaildirRepository, self).__init__(reposname, account) + if self.account.getconfboolean('synclabels', 0) and \ + self.account.getconf('status_backend', 'plain') != 'sqlite': + raise OfflineImapError("The GmailMaildir repository needs the sqlite backend to sync labels.\n" + "To enable it add 'status_backend = sqlite' in the account section", + OfflineImapError.ERROR.REPO) + + + + def getfoldertype(self): + return GmailMaildirFolder diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 1e23bea..0e5d65e 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -177,10 +177,9 @@ class MaildirRepository(BaseRepository): self.debug(" This is maildir folder '%s'." % foldername) if self.getconfboolean('restoreatime', False): self._append_folder_atimes(foldername) - retval.append(folder.Maildir.MaildirFolder(self.root, - foldername, - self.getsep(), - self)) + fd = self.getfoldertype()(self.root, foldername, + self.getsep(), self) + retval.append(fd) if self.getsep() == '/' and dirname != '': # Recursively check sub-directories for folders too. @@ -194,6 +193,9 @@ class MaildirRepository(BaseRepository): self.folders = self._getfolders_scandir(self.root) return self.folders + def getfoldertype(self): + return folder.Maildir.MaildirFolder + def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py index 22cd128..93861c2 100644 --- a/offlineimap/repository/__init__.py +++ b/offlineimap/repository/__init__.py @@ -23,6 +23,7 @@ except ImportError: #python2 from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository from offlineimap.repository.Gmail import GmailRepository from offlineimap.repository.Maildir import MaildirRepository +from offlineimap.repository.GmailMaildir import GmailMaildirRepository from offlineimap.repository.LocalStatus import LocalStatusRepository from offlineimap.error import OfflineImapError @@ -46,7 +47,8 @@ class Repository(object): elif reqtype == 'local': name = account.getconf('localrepository') typemap = {'IMAP': MappedIMAPRepository, - 'Maildir': MaildirRepository} + 'Maildir': MaildirRepository, + 'GmailMaildir': GmailMaildirRepository} elif reqtype == 'status': # create and return a LocalStatusRepository @@ -84,4 +86,3 @@ class Repository(object): :param regtype: 'remote', 'local', or 'status' """ pass - diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 8558421..4dfd092 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -254,6 +254,15 @@ class UIBase(object): "for that message." % ( str(uidlist), self.getnicename(destfolder), destfolder)) + def labelstoreadonly(self, destfolder, uidlist, labels): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to modify labels for messages %s in folder %s[%s], " + "but that folder is read-only. No labels have been modified " + "for that message." % ( + str(uidlist), self.getnicename(destfolder), destfolder)) + def deletereadonly(self, destfolder, uidlist): if self.config.has_option('general', 'ignore-readonly') and \ self.config.getboolean('general', 'ignore-readonly'): @@ -361,6 +370,25 @@ class UIBase(object): self.logger.info("Deleting flag %s from %d messages on %s" % ( ", ".join(flags), len(uidlist), dest)) + def addinglabels(self, uidlist, label, dest): + self.logger.info("Adding label %s to %d messages on %s" % ( + label, len(uidlist), dest)) + + def deletinglabels(self, uidlist, label, dest): + self.logger.info("Deleting label %s from %d messages on %s" % ( + label, len(uidlist), dest)) + + def settinglabels(self, uid, num, num_to_set, labels, dest): + self.logger.info("Setting labels to message %d on %s (%d of %d): %s" % ( + uid, dest, num, num_to_set, ", ".join(labels))) + + def collectingdata(self, uidlist, source): + if uidlist: + self.logger.info("Collecting data from %d messages on %s" % ( + len(uidlist), source)) + else: + self.logger.info("Collecting data from messages on %s" % source) + def serverdiagnostics(self, repository, type): """Connect to repository and output useful information for debugging""" conn = None From 789e047734c778669f785a1c6410bbdb7b2cb893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abd=C3=B3=20Roig-Maranges?= Date: Wed, 28 Nov 2012 18:29:23 +0100 Subject: [PATCH 598/817] Extend handling of GMail labels header Format headers X-Label and Keywords as a space separated list and all other ones as comma-separated entities. This makes OfflineIMAP label handling to be compatible with some user agents that recognise these headers. Signed-off-by: Eygene Ryabinkin --- offlineimap.conf | 9 +++- offlineimap/folder/Gmail.py | 11 ++--- offlineimap/folder/GmailMaildir.py | 30 +++++-------- offlineimap/imaputil.py | 72 ++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 26 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 3b219d4..7d74df4 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -277,7 +277,14 @@ remoterepository = RemoteExample # #synclabels = no -# Name of the header to use for label storage. +# Name of the header to use for label storage. Format for the header +# value differs for different headers, because there are some de-facto +# standards set by popular clients: +# - X-Label or Keywords keep values separated with spaces; for these +# you, obviously, should not have label values that contain spaces; +# - X-Keywords use comma (',') as the separator. +# To be consistent with the usual To-like headers, for the rest of header +# types we use comma as the separator. # #labelsheader = X-Keywords diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index b735c29..08185a1 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -92,8 +92,8 @@ class GmailFolder(IMAPFolder): else: labels = set() labels = labels - self.ignorelabels - labels = ', '.join(sorted(labels)) - body = self.addmessageheader(body, self.labelsheader, labels) + labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) + body = self.addmessageheader(body, self.labelsheader, labels_str) if len(body)>200: dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:]) @@ -183,11 +183,8 @@ class GmailFolder(IMAPFolder): if not self.synclabels: return super(GmailFolder, self).savemessage(uid, content, flags, rtime) - labels = self.getmessageheader(content, self.labelsheader) - if labels: - labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) - else: - labels = set() + labels = imaputil.labels_from_header(self.labelsheader, + self.getmessageheader(content, self.labelsheader)) ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) self.savemessagelabels(ret, labels) diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 7e903e6..e94dffe 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -20,6 +20,7 @@ import os from .Maildir import MaildirFolder from offlineimap import OfflineImapError import offlineimap.accounts +from offlineimap import imaputil class GmailMaildirFolder(MaildirFolder): """Folder implementation to support adding labels to messages in a Maildir. @@ -78,12 +79,10 @@ class GmailMaildirFolder(MaildirFolder): content = file.read() file.close() - labels = self.getmessageheader(content, self.labelsheader) - if labels: - labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) - else: - labels = set() - self.messagelist[uid]['labels'] = labels + self.messagelist[uid]['labels'] = \ + imaputil.labels_from_header(self.labelsheader, + self.getmessageheader(content, self.labelsheader)) + return self.messagelist[uid]['labels'] @@ -103,11 +102,8 @@ class GmailMaildirFolder(MaildirFolder): if not self.synclabels: return super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) - labels = self.getmessageheader(content, self.labelsheader) - if labels: - labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) - else: - labels = set() + labels = imaputil.labels_from_header(self.labelsheader, + self.getmessageheader(content, self.labelsheader)) ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) # Update the mtime and labels @@ -130,12 +126,9 @@ class GmailMaildirFolder(MaildirFolder): content = file.read() file.close() - oldlabels = self.getmessageheader(content, self.labelsheader) + oldlabels = imaputil.labels_from_header(self.labelsheader, + self.getmessageheader(content, self.labelsheader)) - if oldlabels: - oldlabels = set([lb.strip() for lb in oldlabels.split(',') if len(lb.strip()) > 0]) - else: - oldlabels = set() labels = labels - ignorelabels ignoredlabels = oldlabels & ignorelabels @@ -146,13 +139,14 @@ class GmailMaildirFolder(MaildirFolder): return # Change labels into content - labels_str = ', '.join(sorted(labels | ignoredlabels)) + labels_str = imaputil.format_labels_string(self.labelsheader, + sorted(labels | ignoredlabels)) content = self.addmessageheader(content, self.labelsheader, labels_str) rtime = self.messagelist[uid].get('rtime', None) # write file with new labels to a unique file in tmp messagename = self.new_message_filename(uid, set()) - tmpname = self.save_tmp_file(messagename, content) + tmpname = self.save_to_tmp_file(messagename, content) tmppath = os.path.join(self.getfullname(), tmpname) # move to actual location diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index cdf3c4e..c8e3f76 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -21,6 +21,12 @@ import string from offlineimap.ui import getglobalui +## Globals + +# Message headers that use space as the separator (for label storage) +SPACE_SEPARATED_LABEL_HEADERS = ('X-Label', 'Keywords') + + def __debug(*args): msg = [] for arg in args: @@ -260,3 +266,69 @@ def __split_quoted(string): rest = rest[next_q + 1:] if not is_escaped: return (quoted, rest.lstrip()) + + +def format_labels_string(header, labels): + """ + Formats labels for embedding into a message, + with format according to header name. + + Headers from SPACE_SEPARATED_LABEL_HEADERS keep space-separated list + of labels, the rest uses comma (',') as the separator. + + Also see parse_labels_string() and modify it accordingly + if logics here gets changed. + + """ + if header in SPACE_SEPARATED_LABEL_HEADERS: + sep = ' ' + else: + sep = ',' + + return sep.join(labels) + + +def parse_labels_string(header, labels_str): + """ + Parses a string into a set of labels, with a format according to + the name of the header. + + See __format_labels_string() for explanation on header handling + and keep these two functions synced with each other. + + TODO: add test to ensure that + format_labels_string * parse_labels_string is unity + and + parse_labels_string * format_labels_string is unity + + """ + + if header in SPACE_SEPARATED_LABEL_HEADERS: + sep = ' ' + else: + sep = ',' + + labels = labels_str.strip().split(sep) + + return set([l.strip() for l in labels if l.strip()]) + + +def labels_from_header(header_name, header_value): + """ + Helper that builds label set from the corresponding header value. + + Arguments: + - header_name: name of the header that keeps labels; + - header_value: value of the said header, can be None + + Returns: set of labels parsed from the header (or empty set). + + """ + + if header_value: + labels = parse_labels_string(header_name, header_value) + else: + labels = set() + + return labels + From 09556d645ea81a53c15ce21383a5b754936cee0a Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Sat, 27 Jul 2013 23:25:13 +0200 Subject: [PATCH 599/817] Adapt plain status folder to gmail labels stuff * Implements Status Folder format v2, with a mechanism to upgrade an old statusfolder. * Do not warn about Gmail and GmailMaildir needing sqlite backend anymore. * Clean repository.LocalStatus reusing some code from folder.LocalStatus. * Change field separator in the plaintext file from ':' to '|'. Now the local status stores gmail labels. If they contain field separator character (formerly ':'), they get messed up. The new character '|' is less likely to appear in a label. Signed-off-by: Eygene Ryabinkin --- offlineimap.conf | 3 +- offlineimap/folder/Gmail.py | 2 +- offlineimap/folder/GmailMaildir.py | 2 +- offlineimap/folder/LocalStatus.py | 144 +++++++++++++++++++++++-- offlineimap/repository/Gmail.py | 6 -- offlineimap/repository/GmailMaildir.py | 6 -- offlineimap/repository/LocalStatus.py | 24 +---- 7 files changed, 141 insertions(+), 46 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 7d74df4..3fc20f7 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -272,8 +272,7 @@ remoterepository = RemoteExample #maildir-windows-compatible = no # Specifies if we want to sync GMail lables with the local repository. -# Effective only for GMail IMAP repositories. You should use SQlite -# backend for this to work (see status_backend). +# Effective only for GMail IMAP repositories. # #synclabels = no diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 08185a1..1e315a7 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -284,7 +284,7 @@ class GmailFolder(IMAPFolder): labels = dstfolder.getmessagelabels(uid) statusfolder.savemessagelabels(uid, labels, mtime=mtime) - # either statusfolder is not sqlite or dstfolder is not GmailMaildir. + # dstfolder is not GmailMaildir. except NotImplementedError: return diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index e94dffe..3f37b02 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -191,7 +191,7 @@ class GmailMaildirFolder(MaildirFolder): labels = dstfolder.getmessagelabels(uid) statusfolder.savemessagelabels(uid, labels, mtime=self.getmessagemtime(uid)) - # either statusfolder is not sqlite or dstfolder is not GmailMaildir. + # dstfolder is not GmailMaildir. except NotImplementedError: return diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index c3f24f6..1be80e0 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -19,10 +19,13 @@ from .Base import BaseFolder import os import threading -magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" - class LocalStatusFolder(BaseFolder): + """LocalStatus backend implemented as a plain text file""" + + cur_version = 2 + magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT %d" + def __init__(self, name, repository): self.sep = '.' #needs to be set before super.__init__() super(LocalStatusFolder, self).__init__(name, repository) @@ -76,7 +79,17 @@ class LocalStatusFolder(BaseFolder): file.close() return assert(line == magicline) - for line in file.xreadlines(): + + + def readstatus_v1(self, fp): + """ + Read status folder in format version 1. + + Arguments: + - fp: I/O object that points to the opened database file. + + """ + for line in fp.xreadlines(): line = line.strip() try: uid, flags = line.split(':') @@ -87,17 +100,91 @@ class LocalStatusFolder(BaseFolder): (line, self.filename) self.ui.warn(errstr) raise ValueError(errstr) - self.messagelist[uid] = {'uid': uid, 'flags': flags} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': 0, 'labels': set()} + + + def readstatus(self, fp): + """ + Read status file in the current format. + + Arguments: + - fp: I/O object that points to the opened database file. + + """ + for line in fp.xreadlines(): + line = line.strip() + try: + uid, flags, mtime, labels = line.split('|') + uid = long(uid) + flags = set(flags) + mtime = long(mtime) + labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) + except ValueError as e: + errstr = "Corrupt line '%s' in cache file '%s'" % \ + (line, self.filename) + self.ui.warn(errstr) + raise ValueError(errstr) + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': mtime, 'labels': labels} + + + def cachemessagelist(self): + if self.isnewfolder(): + self.messagelist = {} + return + + # loop as many times as version, and update format + for i in range(1, self.cur_version+1): + file = open(self.filename, "rt") + self.messagelist = {} + line = file.readline().strip() + + # convert from format v1 + if line == (self.magicline % 1): + self.ui._msg('Upgrading LocalStatus cache from version 1 to version 2 for %s:%s' %\ + (self.repository, self)) + self.readstatus_v1(file) + file.close() + self.save() + + # NOTE: Add other format transitions here in the future. + # elif line == (self.magicline % 2): + # self.ui._msg('Upgrading LocalStatus cache from version 2 to version 3 for %s:%s' %\ + # (self.repository, self)) + # self.readstatus_v2(file) + # file.close() + # file.save() + + # format is up to date. break + elif line == (self.magicline % self.cur_version): + break + + # something is wrong + else: + errstr = "Unrecognized cache magicline in '%s'" % self.filename + self.ui.warn(errstr) + raise ValueError(errstr) + + if not line: + # The status file is empty - should not have happened, + # but somehow did. + errstr = "Cache file '%s' is empty. Closing..." % self.filename + self.ui.warn(errstr) + file.close() + return + + assert(line == (self.magicline % self.cur_version)) + self.readstatus(file) file.close() + def save(self): with self.savelock: file = open(self.filename + ".tmp", "wt") - file.write(magicline + "\n") + file.write((self.magicline % self.cur_version) + "\n") for msg in self.messagelist.values(): - flags = msg['flags'] - flags = ''.join(sorted(flags)) - file.write("%s:%s\n" % (msg['uid'], flags)) + flags = ''.join(sorted(msg['flags'])) + labels = ', '.join(sorted(msg['labels'])) + file.write("%s|%s|%d|%s\n" % (msg['uid'], flags, msg['mtime'], labels)) file.flush() if self.doautosave: os.fsync(file.fileno()) @@ -114,7 +201,7 @@ class LocalStatusFolder(BaseFolder): return self.messagelist # Interface from BaseFolder - def savemessage(self, uid, content, flags, rtime): + def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): """Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not @@ -124,11 +211,11 @@ class LocalStatusFolder(BaseFolder): # We cannot assign a uid. return uid - if uid in self.messagelist: # already have it + if self.uidexists(uid): # already have it self.savemessageflags(uid, flags) return uid - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels} self.save() return uid @@ -145,6 +232,41 @@ class LocalStatusFolder(BaseFolder): self.messagelist[uid]['flags'] = flags self.save() + + def savemessagelabels(self, uid, labels, mtime=None): + self.messagelist[uid]['labels'] = labels + if mtime: self.messagelist[uid]['mtime'] = mtime + self.save() + + def savemessageslabelsbulk(self, labels): + """Saves labels from a dictionary in a single database operation.""" + for uid, lb in labels.items(): + self.messagelist[uid]['labels'] = lb + self.save() + + def addmessageslabels(self, uids, labels): + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels + self.save() + + def deletemessageslabels(self, uids, labels): + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels + self.save() + + def getmessagelabels(self, uid): + return self.messagelist[uid]['labels'] + + def savemessagesmtimebulk(self, mtimes): + """Saves mtimes from the mtimes dictionary in a single database operation.""" + for uid, mt in mtimes.items(): + self.messagelist[uid]['mtime'] = mt + self.save() + + def getmessagemtime(self, uid): + return self.messagelist[uid]['mtime'] + + # Interface from BaseFolder def deletemessage(self, uid): self.deletemessages([uid]) diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 61d4486..2e23e62 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -36,12 +36,6 @@ class GmailRepository(IMAPRepository): 'ssl', 'yes') IMAPRepository.__init__(self, reposname, account) - if self.account.getconfboolean('synclabels', 0) and \ - self.account.getconf('status_backend', 'plain') != 'sqlite': - raise OfflineImapError("The Gmail repository needs the sqlite backend to sync labels.\n" - "To enable it add 'status_backend = sqlite' in the account section", - OfflineImapError.ERROR.REPO) - def gethost(self): """Return the server name to connect to. diff --git a/offlineimap/repository/GmailMaildir.py b/offlineimap/repository/GmailMaildir.py index 62f9f83..9072b7c 100644 --- a/offlineimap/repository/GmailMaildir.py +++ b/offlineimap/repository/GmailMaildir.py @@ -25,12 +25,6 @@ class GmailMaildirRepository(MaildirRepository): """Initialize a MaildirRepository object. Takes a path name to the directory holding all the Maildir directories.""" super(GmailMaildirRepository, self).__init__(reposname, account) - if self.account.getconfboolean('synclabels', 0) and \ - self.account.getconf('status_backend', 'plain') != 'sqlite': - raise OfflineImapError("The GmailMaildir repository needs the sqlite backend to sync labels.\n" - "To enable it add 'status_backend = sqlite' in the account section", - OfflineImapError.ERROR.REPO) - def getfoldertype(self): diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index bb9ada4..b75d44a 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap.folder.LocalStatus import LocalStatusFolder, magicline +from offlineimap.folder.LocalStatus import LocalStatusFolder from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder from offlineimap.repository.Base import BaseRepository import os @@ -49,19 +49,6 @@ class LocalStatusRepository(BaseRepository): def getsep(self): return '.' - def getfolderfilename(self, foldername): - """Return the full path of the status file - - This mimics the path that Folder().getfolderbasename() would return""" - if not foldername: - basename = '.' - else: #avoid directory hierarchies and file names such as '/' - basename = foldername.replace('/', '.') - # replace with literal 'dot' if final path name is '.' as '.' is - # an invalid file name. - basename = re.sub('(^|\/)\.$','\\1dot', basename) - return os.path.join(self.root, basename) - def makefolder(self, foldername): """Create a LocalStatus Folder @@ -73,11 +60,10 @@ class LocalStatusRepository(BaseRepository): if self.account.dryrun: return # bail out in dry-run mode - filename = self.getfolderfilename(foldername) - file = open(filename + ".tmp", "wt") - file.write(magicline + '\n') - file.close() - os.rename(filename + ".tmp", filename) + # Create an empty StatusFolder + folder = self.LocalStatusFolderClass(foldername, self) + folder.save() + # Invalidate the cache. self._folders = {} From 214137eb7be2bc48dc6f02cd764477301cf918c7 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Sun, 28 Jul 2013 13:58:30 +0200 Subject: [PATCH 600/817] Improvements to the SQlite-based local status folder * Do not inherit LocalStatusSQLiteFolder class from the plaintext one. * Use some functions already in BaseFolder in both, plaintext and sqlite classes. * Add a saveall method. The idea is that saveall dumps the entire messagelist to disk, while save only commits the uncommited changes. Right now, save is noop for sqlite, and equivalent to saveall for plaintext, but it enables to be more clever on when we commit to disk in the future. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 1 + offlineimap/folder/LocalStatus.py | 14 +++-- offlineimap/folder/LocalStatusSQLite.py | 72 +++++++++++++++++++------ 3 files changed, 63 insertions(+), 24 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 67e678d..8584cf8 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -36,6 +36,7 @@ class BaseFolder(object): # Save original name for folderfilter operations self.ffilter_name = name # Top level dir name is always '' + self.root = None self.name = name if not name == self.getsep() else '' self.repository = repository self.visiblename = repository.nametrans(name) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 1be80e0..9abb20a 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -29,6 +29,7 @@ class LocalStatusFolder(BaseFolder): def __init__(self, name, repository): self.sep = '.' #needs to be set before super.__init__() super(LocalStatusFolder, self).__init__(name, repository) + self.root = repository.root self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self.messagelist = {} self.savelock = threading.Lock() @@ -47,14 +48,6 @@ class LocalStatusFolder(BaseFolder): def getname(self): return self.name - # Interface from BaseFolder - def getroot(self): - return self.repository.root - - # Interface from BaseFolder - def getsep(self): - return self.sep - # Interface from BaseFolder def getfullname(self): return self.filename @@ -178,6 +171,11 @@ class LocalStatusFolder(BaseFolder): def save(self): + """Save changed data to disk. For this backend it is the same as saveall""" + self.saveall() + + def saveall(self): + """Saves the entire messagelist to disk""" with self.savelock: file = open(self.filename + ".tmp", "wt") file.write((self.magicline % self.cur_version) + "\n") diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 17a1d23..010e81b 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -17,14 +17,14 @@ import os.path import re from threading import Lock -from .LocalStatus import LocalStatusFolder +from .Base import BaseFolder try: import sqlite3 as sqlite except: pass #fail only if needed later on, not on import -class LocalStatusSQLiteFolder(LocalStatusFolder): +class LocalStatusSQLiteFolder(BaseFolder): """LocalStatus backend implemented with an SQLite database As python-sqlite currently does not allow to access the same sqlite @@ -43,9 +43,17 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): cur_version = 2 def __init__(self, name, repository): + self.sep = '.' #needs to be set before super.__init__() super(LocalStatusSQLiteFolder, self).__init__(name, repository) + self.root = repository.root + self.filename = os.path.join(self.getroot(), self.getfolderbasename()) + self.messagelist = {} + + self._newfolder = False # flag if the folder is new + # dblock protects against concurrent writes in same connection self._dblock = Lock() + #Try to establish connection, no need for threadsafety in __init__ try: self.connection = sqlite.connect(self.filename, check_same_thread = False) @@ -62,13 +70,35 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): cursor = self.connection.execute("SELECT value from metadata WHERE key='db_version'") except sqlite.DatabaseError: #db file missing or corrupt, recreate it. - self.__upgrade_db(0) + self.__create_db() else: # fetch db version and upgrade if needed version = int(cursor.fetchone()[0]) if version < LocalStatusSQLiteFolder.cur_version: self.__upgrade_db(version) + + def storesmessages(self): + return False + + def getname(self): + return self.name + + def getfullname(self): + return self.filename + + + # Interface from LocalStatusFolder + def isnewfolder(self): + return self._newfolder + + + # Interface from LocalStatusFolder + def deletemessagelist(self): + """delete all messages in the db""" + self.__sql_write('DELETE FROM status') + + def __sql_write(self, sql, vars=None, executemany=False): """Execute some SQL, retrying if the db was locked. @@ -123,6 +153,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): 'LocalStatus', self.getfolderbasename()) # MIGRATE from plaintext if needed + # TODO: adopt for plain-text v2 if os.path.exists(plaintextfilename): self.ui._msg('Migrating LocalStatus cache from plain text ' 'to sqlite database for %s:%s' %\ @@ -168,22 +199,12 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): % (self.repository, self)) self.connection.executescript(""" CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128)); - INSERT INTO metadata VALUES('db_version', '1'); + INSERT INTO metadata VALUES('db_version', '2'); CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50), mtime INTEGER, labels VARCHAR(256)); """) self.connection.commit() + self._newfolder = True - # Interface from LocalStatusFolder - def isnewfolder(self): - # testing the existence of the db file won't work. It is created - # as soon as this class instance was intitiated. So say it is a - # new folder when there are no messages at all recorded in it. - return self.getmessagecount() > 0 - - # Interface from LocalStatusFolder - def deletemessagelist(self): - """delete all messages in the db""" - self.__sql_write('DELETE FROM status') # Interface from BaseFolder def cachemessagelist(self): @@ -196,8 +217,21 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): # Interface from LocalStatusFolder def save(self): - #Noop in this backend pass + # Noop. every transaction commits to database! + + def saveall(self): + """Saves the entire messagelist to the database.""" + data = [] + for uid, msg in self.messagelist.items(): + mtime = msg['mtime'] + flags = ''.join(sorted(msg['flags'])) + labels = ', '.join(sorted(msg['labels'])) + data.append((uid, flags, mtime, labels)) + + self.__sql_write('INSERT OR REPLACE INTO status (id,flags,mtime,labels) VALUES (?,?,?,?)', + data, executemany=True) + # Following some pure SQLite functions, where we chose to use # BaseFolder() methods instead. Doing those on the in-memory list is @@ -235,6 +269,12 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): # return flags # assert False,"getmessageflags() called on non-existing message" + + # Interface from BaseFolder + def getmessagelist(self): + return self.messagelist + + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): """ From 1b954c3b4c8c623c5878a8387e125a2aef0873ed Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Sun, 28 Jul 2013 14:03:46 +0200 Subject: [PATCH 601/817] Add ability to migrate status data across backends If when we request a LocalStatus folder, the folder has to be created, we look whether the other backend has data, and if it does we migrate it to the new backend. The old backend data is left untouched, so that if you change back say from sqlite to plaintext, the older data is still there. That should not lead to data loss, only a slower sync while the status folder gets updated. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 + offlineimap/folder/LocalStatusSQLite.py | 28 ---------- offlineimap/repository/LocalStatus.py | 73 ++++++++++++++++++------- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index d2148e6..4fd26a1 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -14,6 +14,8 @@ OfflineIMAP v6.5.6 (YYYY-MM-DD) to IMAP server (Abdó Roig-Maranges) * Allow to sync GMail labels and implement GmailMaildir repository that adds mechanics to change message labels (Abdó Roig-Maranges) +* Allow to migrate status data across differend backends + (Abdó Roig-Maranges) OfflineIMAP v6.5.5 (2013-10-07) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 010e81b..f8921f1 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -144,34 +144,6 @@ class LocalStatusSQLiteFolder(BaseFolder): self.connection = sqlite.connect(self.filename, check_same_thread = False) - if from_ver == 0: - # from_ver==0: no db existent: plain text migration? - self.__create_db() - # below was derived from repository.getfolderfilename() logic - plaintextfilename = os.path.join( - self.repository.account.getaccountmeta(), - 'LocalStatus', - self.getfolderbasename()) - # MIGRATE from plaintext if needed - # TODO: adopt for plain-text v2 - if os.path.exists(plaintextfilename): - self.ui._msg('Migrating LocalStatus cache from plain text ' - 'to sqlite database for %s:%s' %\ - (self.repository, self)) - file = open(plaintextfilename, "rt") - line = file.readline().strip() - data = [] - for line in file.xreadlines(): - uid, flags = line.strip().split(':') - uid = long(uid) - flags = ''.join(sorted(flags)) - data.append((uid,flags)) - self.connection.executemany('INSERT INTO status (id,flags) VALUES (?,?)', - data) - self.connection.commit() - file.close() - os.rename(plaintextfilename, plaintextfilename + ".old") - # Upgrade from database version 1 to version 2 # This change adds labels and mtime columns, to be used by Gmail IMAP and Maildir folders. if from_ver <= 1: diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index b75d44a..3bd97ce 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -25,20 +25,21 @@ import re class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): BaseRepository.__init__(self, reposname, account) - # Root directory in which the LocalStatus folders reside - self.root = os.path.join(account.getaccountmeta(), 'LocalStatus') - # statusbackend can be 'plain' or 'sqlite' - backend = self.account.getconf('status_backend', 'plain') - if backend == 'sqlite': - self._backend = 'sqlite' - self.LocalStatusFolderClass = LocalStatusSQLiteFolder - self.root += '-sqlite' - elif backend == 'plain': - self._backend = 'plain' - self.LocalStatusFolderClass = LocalStatusFolder - else: - raise SyntaxWarning("Unknown status_backend '%s' for account '%s'" \ - % (backend, account.name)) + + # class and root for all backends + self.backends = {} + self.backends['sqlite'] = { + 'class': LocalStatusSQLiteFolder, + 'root': os.path.join(account.getaccountmeta(), 'LocalStatus-sqlite') + } + + self.backends['plain'] = { + 'class': LocalStatusFolder, + 'root': os.path.join(account.getaccountmeta(), 'LocalStatus') + } + + # Set class and root for the configured backend + self.setup_backend(self.account.getconf('status_backend', 'plain')) if not os.path.exists(self.root): os.mkdir(self.root, 0o700) @@ -46,16 +47,41 @@ class LocalStatusRepository(BaseRepository): # self._folders is a dict of name:LocalStatusFolders() self._folders = {} + def setup_backend(self, backend): + if backend in self.backends.keys(): + self._backend = backend + self.root = self.backends[backend]['root'] + self.LocalStatusFolderClass = self.backends[backend]['class'] + + else: + raise SyntaxWarning("Unknown status_backend '%s' for account '%s'" \ + % (backend, account.name)) + + def import_other_backend(self, folder): + for bk, dic in self.backends.items(): + # skip folder's own type + if dic['class'] == type(folder): + continue + + repobk = LocalStatusRepository(self.name, self.account) + repobk.setup_backend(bk) # fake the backend + folderbk = dic['class'](folder.name, repobk) + + # if backend contains data, import it to folder. + if not folderbk.isnewfolder(): + self.ui._msg('Migrating LocalStatus cache from %s to %s ' % (bk, self._backend) + \ + 'status folder for %s:%s' % (self.name, folder.name)) + + folderbk.cachemessagelist() + folder.messagelist = folderbk.messagelist + folder.saveall() + break + def getsep(self): return '.' def makefolder(self, foldername): - """Create a LocalStatus Folder - - Empty Folder for plain backend. NoOp for sqlite backend as those - are created on demand.""" - if self._backend == 'sqlite': - return # noop for sqlite which creates on-demand + """Create a LocalStatus Folder""" if self.account.dryrun: return # bail out in dry-run mode @@ -65,7 +91,7 @@ class LocalStatusRepository(BaseRepository): folder.save() # Invalidate the cache. - self._folders = {} + self.forgetfolders() def getfolder(self, foldername): """Return the Folder() object for a foldername""" @@ -73,6 +99,11 @@ class LocalStatusRepository(BaseRepository): return self._folders[foldername] folder = self.LocalStatusFolderClass(foldername, self) + + # if folder is empty, try to import data from an other backend + if folder.isnewfolder(): + self.import_other_backend(folder) + self._folders[foldername] = folder return folder From 7c271d5131b0fb81363d58ca8393d6dbf1f96c2f Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 2 May 2014 20:04:09 -0400 Subject: [PATCH 602/817] Imapserver: add missing space to the log message Signed-off-by: Eygene Ryabinkin --- offlineimap/imapserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index b00fd08..84b1289 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -479,7 +479,7 @@ class IMAPServer: " to the correct port." % (self.hostname, self.port) else: reason = "Unknown SSL protocol connecting to host '%s' for"\ - "repository '%s'. OpenSSL responded:\n%s"\ + " repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) raise OfflineImapError(reason, severity) From 9eba0e5eb7b32d50c576467e1e532f65196dd97e Mon Sep 17 00:00:00 2001 From: Nathan Guerin Date: Thu, 1 May 2014 10:15:08 +0200 Subject: [PATCH 603/817] User manual: fix typo, "rebuild" -> "rebuilt" Signed-off-by: Eygene Ryabinkin --- docs/MANUAL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 1c0ce0f..8a0c5a6 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -219,7 +219,7 @@ achieve this. order to use this. This will save you plenty of disk activity. Do note that the sqlite backend is still considered experimental as it has only been included recently (although a loss of your status - cache should not be a tragedy as that file can be rebuild + cache should not be a tragedy as that file can be rebuilt automatically) 4) Use quick sync. A regular sync will request all flags and all UIDs From d96af192ed3baa1e47ccdec2699e270ea2a7429b Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 7 May 2014 00:07:56 +0400 Subject: [PATCH 604/817] Move space to the end of string ... to be aligned with the rest of the code. Signed-off-by: Eygene Ryabinkin --- offlineimap/imapserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 84b1289..be6fdb9 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -478,8 +478,8 @@ class IMAPServer: "tandard ssl port %d configured. Make sure you connect"\ " to the correct port." % (self.hostname, self.port) else: - reason = "Unknown SSL protocol connecting to host '%s' for"\ - " repository '%s'. OpenSSL responded:\n%s"\ + reason = "Unknown SSL protocol connecting to host '%s' for "\ + "repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) raise OfflineImapError(reason, severity) From 5150de5514d17761340db9b8385a44f734cd2f28 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 7 May 2014 00:40:59 +0400 Subject: [PATCH 605/817] Add support for XDG Base Directory Specification $XDG_CONFIG_HOME/offlineimap/config will now be tried before the canonical ~/.offlineimaprc. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 4 ++++ README.md | 7 +++++++ docs/MANUAL.rst | 3 +++ docs/doc-src/FAQ.rst | 9 +++++++++ offlineimap/init.py | 19 +++++++++++++++---- 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 4fd26a1..129709a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -16,6 +16,10 @@ OfflineIMAP v6.5.6 (YYYY-MM-DD) adds mechanics to change message labels (Abdó Roig-Maranges) * Allow to migrate status data across differend backends (Abdó Roig-Maranges) +* Support XDG Base Directory Specification + (if $XDG_CONFIG_HOME/offlineimap/config exists, use it as the + default configuration path; ~/.offlineimaprc is still tried after + XDG location) (GitHub#32) OfflineIMAP v6.5.5 (2013-10-07) diff --git a/README.md b/README.md index edafae7..bc839af 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,13 @@ to do is specify a directory for your folders to be in (on the localfolders line), the host name of your IMAP server (on the remotehost line), and your login name on the remote (on the remoteuser line). That's it! +If you prefer to be XDG-compatible, + http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html +then substitute the above ``~/.offlineimaprc'' with +``$XDG\_CONFIG\_HOME/offlineimap/config'' and don't forget to set +XDG\_CONFIG\_HOME properly if you want it to be different from +the default ``$HOME/.config'' for any reason. + To run OfflineIMAP, you just have to say `offlineimap` ― it will fire up, ask you for a login password if necessary, synchronize your folders, and exit. See? diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 8a0c5a6..a5c61bd 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -63,6 +63,9 @@ set, and you can read about other features later with `offlineimap.conf`. Check out the `Use Cases`_ section for some example configurations. +If you want to be XDG-compatible, you can put your configuration file into +`$XDG_CONFIG_HOME/offlineimap/config`. + OPTIONS ======= diff --git a/docs/doc-src/FAQ.rst b/docs/doc-src/FAQ.rst index b8c2b56..29fa928 100644 --- a/docs/doc-src/FAQ.rst +++ b/docs/doc-src/FAQ.rst @@ -77,6 +77,15 @@ based in instructions submitted by Chris Walker:: That URL also has more details on making OfflineIMAP work with Windows. +Does OfflineIMAP supports XDG Base Directory specification? +----------------------------------------------------------- + +Yes. We are trying to use `$XDG_CONFIG_HOME/offlineimap/config` +as the primary configuration file, falling back to `~/.offlineimaprc` +if configuration file location was not explicitely specified at the +command line. + + Does OfflineIMAP support mbox, mh, or anything else other than Maildir? ----------------------------------------------------------------------- diff --git a/offlineimap/init.py b/offlineimap/init.py index d4c9a6d..9193bd8 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -101,9 +101,8 @@ class OfflineImap: "or to sync some accounts that you normally prefer not to.") parser.add_option("-c", dest="configfile", metavar="FILE", - default="~/.offlineimaprc", - help="Specifies a configuration file to use in lieu of " - "%default.") + default=None, + help="Specifies a configuration file to use") parser.add_option("-d", dest="debugtype", metavar="type1,[type2...]", help="Enables debugging for OfflineIMAP. This is useful " @@ -165,7 +164,19 @@ class OfflineImap: globals.set_options (options) #read in configuration file - configfilename = os.path.expanduser(options.configfile) + if not options.configfile: + # Try XDG location, then fall back to ~/.offlineimaprc + xdg_var = 'XDG_CONFIG_HOME' + if not xdg_var in os.environ or not os.environ[xdg_var]: + xdg_home = os.path.expanduser('~/.config') + else: + xdg_home = os.environ[xdg_var] + options.configfile = os.path.join(xdg_home, "offlineimap", "config") + if not os.path.exists(options.configfile): + options.configfile = os.path.expanduser('~/.offlineimaprc') + configfilename = options.configfile + else: + configfilename = os.path.expanduser(options.configfile) config = CustomConfigParser() if not os.path.exists(configfilename): From 1300f2289b2a8a32fb2b73375e2ff580340021d2 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Wed, 7 May 2014 00:05:55 +0200 Subject: [PATCH 606/817] deletemessageheaders(): use passed list of headers to remove ... and not self.filterheaders. With the current code this change is no-op (since self.filterheaders is always passed as header_list), but it is a bug in general. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 8584cf8..cf9379a 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -506,7 +506,7 @@ next line\n new_headers = [] for h in headers.split('\n'): keep_it = True - for trim_h in self.filterheaders: + for trim_h in header_list: if len(h) > len(trim_h) and h[0:len(trim_h)+1] == (trim_h + ":"): keep_it = False break From 96c9cca83a5ea5c49d3caed1d9a766128fcf64cd Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Tue, 6 May 2014 23:17:52 +0200 Subject: [PATCH 607/817] Class IMAPServer: idle() is private now Signed-off-by: Eygene Ryabinkin --- offlineimap/imapserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index be6fdb9..2518058 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -621,7 +621,7 @@ class IdleThread(object): if folder is None: self.thread = Thread(target=self.noop) else: - self.thread = Thread(target=self.idle) + self.thread = Thread(target=self.__idle) self.thread.setDaemon(1) def start(self): From b982549e4d31b84dadcbe026a30801100c7ba07b Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Tue, 6 May 2014 23:27:30 +0200 Subject: [PATCH 608/817] Class LocalStatusRepository: add missing self qualifier Signed-off-by: Eygene Ryabinkin --- offlineimap/repository/LocalStatus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 3bd97ce..0375d0d 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -55,7 +55,7 @@ class LocalStatusRepository(BaseRepository): else: raise SyntaxWarning("Unknown status_backend '%s' for account '%s'" \ - % (backend, account.name)) + % (backend, self.account.name)) def import_other_backend(self, folder): for bk, dic in self.backends.items(): From 8b4f86be6780fb6921bfe9dcea00f314ea796a4d Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Tue, 6 May 2014 23:47:05 +0200 Subject: [PATCH 609/817] Class LocalStatusFolder: remove duplicate cachemessagelist() Remained from GMail label sync merge. Pointyhat-to: Eygene Ryabinkin Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/LocalStatus.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 9abb20a..7e5928e 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -56,23 +56,6 @@ class LocalStatusFolder(BaseFolder): if not self.isnewfolder(): os.unlink(self.filename) - # Interface from BaseFolder - def cachemessagelist(self): - if self.isnewfolder(): - self.messagelist = {} - return - file = open(self.filename, "rt") - self.messagelist = {} - line = file.readline().strip() - if not line: - # The status file is empty - should not have happened, - # but somehow did. - errstr = "Cache file '%s' is empty. Closing..." % self.filename - self.ui.warn(errstr) - file.close() - return - assert(line == magicline) - def readstatus_v1(self, fp): """ @@ -102,7 +85,7 @@ class LocalStatusFolder(BaseFolder): Arguments: - fp: I/O object that points to the opened database file. - + """ for line in fp.xreadlines(): line = line.strip() @@ -120,6 +103,7 @@ class LocalStatusFolder(BaseFolder): self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': mtime, 'labels': labels} + # Interface from BaseFolder def cachemessagelist(self): if self.isnewfolder(): self.messagelist = {} From 8dd6f7893ccbb950d20012ce12b4820d824f0ffe Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Tue, 6 May 2014 23:56:25 +0200 Subject: [PATCH 610/817] Add import for OfflineImapError Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Gmail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 1e315a7..5e07aeb 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -18,7 +18,7 @@ import re -from offlineimap import imaputil +from offlineimap import imaputil, OfflineImapError from offlineimap import imaplibutil import offlineimap.accounts From 8c43e52173d80eb5ff31f7adfe3bb37c78de1d86 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Tue, 6 May 2014 23:12:50 +0200 Subject: [PATCH 611/817] Fix multiple typos in var, function and exception names Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 2 +- offlineimap/folder/Gmail.py | 2 +- offlineimap/folder/IMAP.py | 2 +- offlineimap/folder/Maildir.py | 2 +- offlineimap/imapserver.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index cf9379a..551a03a 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -104,7 +104,7 @@ class BaseFolder(object): def waitforthread(self): """Implements method that waits for thread to be usable. Should be implemented only for folders that suggest threads.""" - raise NotImplementedException + raise NotImplementedError # XXX: we may need someting like supports_quickstatus() to check # XXX: if user specifies 'quick' flag for folder that doesn't diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 5e07aeb..f051938 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -203,7 +203,7 @@ class GmailFolder(IMAPFolder): result = self._store_to_imap(imapobj, uid_str, arg, labels_str) except imapobj.readonly: - self.ui.labelstoreadonly(self, uidlist, data) + self.ui.labelstoreadonly(self, uidlist, labels) return None finally: diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 3e8d94c..4801eef 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -715,7 +715,7 @@ class IMAPFolder(BaseFolder): result = self._store_to_imap(imapobj, str(uid), 'FLAGS', imaputil.flagsmaildir2imap(flags)) except imapobj.readonly: - self.ui.flagstoreadonly(self, [uid], data) + self.ui.flagstoreadonly(self, [uid], flags) return finally: diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 0314f62..d067194 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -273,7 +273,7 @@ class MaildirFolder(BaseFolder): except OSError as e: if e.errno == e.EEXIST: if tries: - time.slep(0.23) + time.sleep(0.23) continue severity = OfflineImapError.ERROR.MESSAGE raise OfflineImapError("Unique filename %s already exists." % \ diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 2518058..676dc31 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -276,7 +276,7 @@ class IMAPServer: If any authentication method succeeds, routine should exit: warnings for failed methods are to be produced in the respective except blocks. - + """ # Authentication routines, hash keyed by method name @@ -489,7 +489,7 @@ class IMAPServer: reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ - "network." % (self.hostname, self.port, self.reposname) + "network." % (self.hostname, self.port, self.repos) raise OfflineImapError(reason, severity) # Could not acquire connection to the remote; # socket.error(last_error) raised From 6453ab0db7496e7689cd8310236d5951f6974763 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Sat, 10 May 2014 21:53:18 +0200 Subject: [PATCH 612/817] Bring GMail labels section on the manual up to date - there is no need to set SQlite backend anymore; - explain a bit more that some headers are recognized by email clients. Signed-off-by: Eygene Ryabinkin --- docs/MANUAL.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index a5c61bd..acf7395 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -438,18 +438,20 @@ Sync from Gmail to a local Maildir with labels This is an example of a setup where GMail gets synced with a local Maildir. It also keeps track of GMail labels, that get embedded into the messages -under the header X-Keywords (or whatever labelsheader is set to), and syncs -them back and forth the same way as flags. +under the header configured in labelsheader, and syncs them back and forth +the same way as flags. +The header used for the labels may need to be set according to the email +client used. +Some choices that may be recognized by email clients are `X-Keywords` or `X-Labels`. -The first time it runs on a large repository may take some time as the labels -are read / embedded on every message. Afterwards local label changes are detected -using modification times (much faster):: +The first time OfflineIMAP runs with synclabels enabled on a large repository it +may take some time as the labels are read / embedded on every message. +Afterwards local label changes are detected using modification times, which is +much faster:: [Account Gmail-mine] localrepository = Gmaillocal-mine remoterepository = Gmailserver-mine - # Need this to be able to sync labels - status_backend = sqlite synclabels = yes # This header is where labels go. Usually you will be fine # with default value, but in case you want it different, From 1690e5f74eb3327397aac4e7522d081510ac0d2a Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Sun, 11 May 2014 12:32:05 +0200 Subject: [PATCH 613/817] Copymessageto should not be private since we override it private methods prevent them from being overriden on derived classes. In GmailFolder we need to override copymessageto, so it can't be private. Before this commit, copymessageto was made private in Base but not in GmailFolder. The end result was that labels were not set when copying the message content, and always needed to be set on the label copying pass. Pointyhat-to: Eygene Ryabinkin Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 551a03a..43e79ff 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -542,7 +542,7 @@ next line\n for uid in uidlist: self.deletemessage(uid) - def __copymessageto(self, uid, dstfolder, statusfolder, register = 1): + def copymessageto(self, uid, dstfolder, statusfolder, register = 1): """Copies a message from self to dst if needed, updating the status Note that this function does not check against dryrun settings, @@ -624,7 +624,7 @@ next line\n not in the statusfolder yet. The strategy is: 1) Look for messages present in self but not in statusfolder. - 2) invoke __copymessageto() on those which: + 2) invoke copymessageto() on those which: - If dstfolder doesn't have it yet, add them to dstfolder. - Update statusfolder @@ -645,18 +645,18 @@ next line\n if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) - # exceptions are caught in __copymessageto() + # exceptions are caught in copymessageto() if self.suggeststhreads() and not globals.options.singlethreading: self.waitforthread() thread = threadutil.InstanceLimitedThread(\ self.getcopyinstancelimit(), - target = self.__copymessageto, + target = self.copymessageto, name = "Copy message from %s:%s" % (self.repository, self), args = (uid, dstfolder, statusfolder)) thread.start() threads.append(thread) else: - self.__copymessageto(uid, dstfolder, statusfolder, + self.copymessageto(uid, dstfolder, statusfolder, register = 0) for thread in threads: thread.join() From d2ec2a4e9e0b574aba90b3e0cffe9b73456f66ba Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 7 May 2014 01:22:29 +0400 Subject: [PATCH 614/817] Extend handling of cert_fingerprint Add ability to specify multiple fingerprints. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ offlineimap.conf | 7 ++++++- offlineimap/imaplibutil.py | 21 ++++++++++++++------- offlineimap/repository/IMAP.py | 11 ++++++++++- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 129709a..b45d430 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -20,6 +20,8 @@ OfflineIMAP v6.5.6 (YYYY-MM-DD) (if $XDG_CONFIG_HOME/offlineimap/config exists, use it as the default configuration path; ~/.offlineimaprc is still tried after XDG location) (GitHub#32) +* Allow multiple certificate fingerprints to be specified inside + 'cert_fingerprint' OfflineIMAP v6.5.5 (2013-10-07) diff --git a/offlineimap.conf b/offlineimap.conf index 3fc20f7..c7cda28 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -395,8 +395,13 @@ ssl = yes # has not changed on each connect and refuse to connect otherwise. # You can also configure this in addition to CA certificate validation # above and it will check both ways. +# +# Multiple fingerprints can be specified, separated by commas. +# +# Fingerprints must be in hexadecimal form without leading '0x': +# 40 hex digits like bbfe29cf97acb204591edbafe0aa8c8f914287c9. -#cert_fingerprint = +#cert_fingerprint = [, ] # SSL version (optional) # It is best to leave this unset, in which case the correct version will be diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index f8806dd..2869623 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -141,21 +141,28 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): """Improved version of imaplib.IMAP4_SSL overriding select()""" def __init__(self, *args, **kwargs): self._fingerprint = kwargs.get('fingerprint', None) + if type(self._fingerprint) != type([]): + self._fingerprint = [self._fingerprint] if 'fingerprint' in kwargs: del kwargs['fingerprint'] super(WrappedIMAP4_SSL, self).__init__(*args, **kwargs) def open(self, host=None, port=None): + if not self.ca_certs and not self._fingerprint: + raise OfflineImapError("No CA certificates " + \ + "and no server fingerprints configured. " + \ + "You must configure at least something, otherwise " + \ + "having SSL helps nothing.", OfflineImapError.ERROR.REPO) super(WrappedIMAP4_SSL, self).open(host, port) - if (self._fingerprint or not self.ca_certs): + if self._fingerprint: # compare fingerprints fingerprint = sha1(self.sock.getpeercert(True)).hexdigest() - if fingerprint != self._fingerprint: - raise OfflineImapError("Server SSL fingerprint '%s' for hostnam" - "e '%s' does not match configured fingerprint. Please ver" - "ify and set 'cert_fingerprint' accordingly if not set ye" - "t." % (fingerprint, host), - OfflineImapError.ERROR.REPO) + if fingerprint not in self._fingerprint: + raise OfflineImapError("Server SSL fingerprint '%s' " % fingerprint + \ + "for hostname '%s' " % host + \ + "does not match configured fingerprint(s) %s. " % self._fingerprint + \ + "Please verify and set 'cert_fingerprint' accordingly " + \ + "if not set yet.", OfflineImapError.ERROR.REPO) class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 19db50f..ff1d5e2 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -215,7 +215,16 @@ class IMAPRepository(BaseRepository): return self.getconf('ssl_version', None) def get_ssl_fingerprint(self): - return self.getconf('cert_fingerprint', None) + """ + Return array of possible certificate fingerprints. + + Configuration item cert_fingerprint can contain multiple + comma-separated fingerprints in hex form. + + """ + + value = self.getconf('cert_fingerprint', "") + return [f.strip().lower() for f in value.split(',') if f] def getpreauthtunnel(self): return self.getconf('preauthtunnel', None) From fb41ce84f9abdc40db814ea7d2da35385daee9fc Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 14 May 2014 14:06:03 +0400 Subject: [PATCH 615/817] Prepared v6.5.6-RC1 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index b45d430..1026491 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,8 +5,8 @@ ChangeLog :website: http://offlineimap.org -OfflineIMAP v6.5.6 (YYYY-MM-DD) -=============================== +OfflineIMAP v6.5.6-RC1 (2014-05-14) +=================================== * Add knob to invoke folderfilter dynamically on each sync (GitHub#73) * Add knob to apply compression to IMAP connections (Abdó Roig-Maranges) From ee89610f3ad9a489bcd165273cb1d32b53946b96 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 14 May 2014 18:02:53 +0400 Subject: [PATCH 616/817] Hacking manual: documented how to create tags Signed-off-by: Eygene Ryabinkin --- docs/doc-src/HACKING.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/doc-src/HACKING.rst b/docs/doc-src/HACKING.rst index 54a89ef..d4b3001 100644 --- a/docs/doc-src/HACKING.rst +++ b/docs/doc-src/HACKING.rst @@ -120,6 +120,15 @@ A typical release cycle works like this: 5. When we think a release is stable enough, we restart from step 1. +Tagging release or RC +--------------------- + +It is done via Git's ``tag`` command, but you must do ``git tag -a`` +to create annotated tag. + +Release tags are named ``vX.Y.Z`` and release candidate tags are named +``vX.Y.Z-rcN``. + .. _contribution checklist: From 0005bcb2f013082ae13f3a466a8b3a9d75e7dea7 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 23 May 2014 10:24:55 +0400 Subject: [PATCH 617/817] Make IDLE mode to work again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactoring in commit 6cbd2498 touched wrong class's "idle" call. Found-by: Tomasz Żok Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 8 ++++++++ offlineimap/imapserver.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index b45d430..a05dec2 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,14 @@ ChangeLog OfflineIMAP v6.5.6 (YYYY-MM-DD) =============================== +* Fix IDLE mode regression (it didn't worked) introduced + after v6.5.5 (pointy hat goes to Eygene Ryabinkin, kudos -- + to Tomasz Żok) + + +OfflineIMAP v6.5.6-RC1 (2014-05-14) +=================================== + * Add knob to invoke folderfilter dynamically on each sync (GitHub#73) * Add knob to apply compression to IMAP connections (Abdó Roig-Maranges) * Add knob to filter some headers before uploading message diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 676dc31..844012e 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -703,7 +703,7 @@ class IdleThread(object): else: success = True if "IDLE" in imapobj.capabilities: - imapobj.__idle(callback=callback) + imapobj.idle(callback=callback) else: self.ui.warn("IMAP IDLE not supported on server '%s'." "Sleep until next refresh cycle." % imapobj.identifier) From efeca9eca7190e47df2bb905cec967f1b09eeeb3 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 14 May 2014 18:02:53 +0400 Subject: [PATCH 618/817] Hacking manual: documented how to create tags Signed-off-by: Eygene Ryabinkin --- docs/doc-src/HACKING.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/doc-src/HACKING.rst b/docs/doc-src/HACKING.rst index 54a89ef..d4b3001 100644 --- a/docs/doc-src/HACKING.rst +++ b/docs/doc-src/HACKING.rst @@ -120,6 +120,15 @@ A typical release cycle works like this: 5. When we think a release is stable enough, we restart from step 1. +Tagging release or RC +--------------------- + +It is done via Git's ``tag`` command, but you must do ``git tag -a`` +to create annotated tag. + +Release tags are named ``vX.Y.Z`` and release candidate tags are named +``vX.Y.Z-rcN``. + .. _contribution checklist: From 1746676af846a344777df709f5d72f60a101bd62 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 23 May 2014 10:24:55 +0400 Subject: [PATCH 619/817] Make IDLE mode to work again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactoring in commit 6cbd2498 touched wrong class's "idle" call. Found-by: Tomasz Żok Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 8 ++++++++ offlineimap/imapserver.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 1026491..d7c2645 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,6 +5,14 @@ ChangeLog :website: http://offlineimap.org +OfflineIMAP v6.5.6 (2014-05-14) +=============================== + +* Fix IDLE mode regression (it didn't worked) introduced + after v6.5.5 (pointy hat goes to Eygene Ryabinkin, kudos -- + to Tomasz Żok) + + OfflineIMAP v6.5.6-RC1 (2014-05-14) =================================== diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 676dc31..844012e 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -703,7 +703,7 @@ class IdleThread(object): else: success = True if "IDLE" in imapobj.capabilities: - imapobj.__idle(callback=callback) + imapobj.idle(callback=callback) else: self.ui.warn("IMAP IDLE not supported on server '%s'." "Sleep until next refresh cycle." % imapobj.identifier) From 7770b5ff73737d1269eb1ba7554b8d3486c7f5ec Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 23 May 2014 21:50:46 +0400 Subject: [PATCH 620/817] Add friendly message about Python 3.x being unsupported A bit better than a traceback. Also introduce environment variable DEVELOPING_OFFLINEIMAP_PYTHON3_SUPPORT that will allow developers to grok the code under Python 3.x. Signed-off-by: Eygene Ryabinkin --- offlineimap.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/offlineimap.py b/offlineimap.py index ce1c70a..05da925 100755 --- a/offlineimap.py +++ b/offlineimap.py @@ -17,6 +17,19 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import os +import sys + +if not 'DEVELOPING_OFFLINEIMAP_PYTHON3_SUPPORT' in os.environ: + if sys.version_info[0] > 2: + sys.stderr.write("""IIMAPS! + +Sorry, OfflineIMAP currently doesn't support Python higher than 2.x. +We're doing our best to bring in support for 3.x really soon. You can +also join us at https://github.com/OfflineIMAP/offlineimap/ and help. +""") + sys.exit(1) + from offlineimap import OfflineImap oi = OfflineImap() From e8db1217d43010173adbf937a3a46e956f5abcfd Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 1 Jun 2014 22:09:44 +0400 Subject: [PATCH 621/817] Fix improper header separator for X-OfflineIMAP header For servers without UIDPLUS we are inserting additional header just after transformation '\n' -> CRLF was done. addmessageheaders() was written to work with just '\n' as the separator, so X-OfflineIMAP header wasn't preceeded by the CRLF, but just by '\n'. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 10 +++++++++- offlineimap/folder/Base.py | 22 ++++++++++++++++------ offlineimap/folder/Gmail.py | 2 +- offlineimap/folder/GmailMaildir.py | 2 +- offlineimap/folder/IMAP.py | 3 ++- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index a05dec2..4a4db93 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,7 +5,15 @@ ChangeLog :website: http://offlineimap.org -OfflineIMAP v6.5.6 (YYYY-MM-DD) +OfflineIMAP v6.5.6.1 (YYYY-MM-DD) +================================= + +* Fix mangled message headers for servers without UIDPLUS: + X-OfflineIMAP was added with preceeding '\n' instead of + '\r\n' just before message was uploaded to the IMAP server. + + +OfflineIMAP v6.5.6 (2014-05-14) =============================== * Fix IDLE mode regression (it didn't worked) introduced diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 43e79ff..bca52a2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -413,16 +413,26 @@ next line\n """ - def addmessageheader(self, content, headername, headervalue): + def addmessageheader(self, content, crlf, headername, headervalue): + """ + Adds new header to the provided message. + + Arguments: + - content: message content, headers and body as a single string + - crlf: string that carries line ending + - headername: name of the header to add + - headervalue: value of the header to add + + """ self.ui.debug('', 'addmessageheader: called to add %s: %s' % (headername, headervalue)) - prefix = '\n' + prefix = crlf suffix = '' - insertionpoint = content.find('\n\n') + insertionpoint = content.find(crlf + crlf) if insertionpoint == 0 or insertionpoint == -1: prefix = '' - suffix = '\n' + suffix = crlf if insertionpoint == -1: insertionpoint = 0 # When body starts immediately, without preceding '\n' @@ -430,8 +440,8 @@ next line\n # we seen many broken ones), we should add '\n' to make # new (and the only header, in this case) to be properly # separated from the message body. - if content[0] != '\n': - suffix = suffix + '\n' + if content[0:len(crlf)] != crlf: + suffix = suffix + crlf self.ui.debug('', 'addmessageheader: insertionpoint = %d' % insertionpoint) headers = content[0:insertionpoint] diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index f051938..2a598de 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -93,7 +93,7 @@ class GmailFolder(IMAPFolder): labels = set() labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) - body = self.addmessageheader(body, self.labelsheader, labels_str) + body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) if len(body)>200: dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:]) diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 3f37b02..be0d20d 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -141,7 +141,7 @@ class GmailMaildirFolder(MaildirFolder): # Change labels into content labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels | ignoredlabels)) - content = self.addmessageheader(content, self.labelsheader, labels_str) + content = self.addmessageheader(content, '\n', self.labelsheader, labels_str) rtime = self.messagelist[uid].get('rtime', None) # write file with new labels to a unique file in tmp diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 4801eef..3b506e5 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -526,6 +526,7 @@ class IMAPFolder(BaseFolder): # NB: imapobj to None. try: while retry_left: + # XXX: we can mangle message only once, out of the loop # UIDPLUS extension provides us with an APPENDUID response. use_uidplus = 'UIDPLUS' in imapobj.capabilities @@ -535,7 +536,7 @@ class IMAPFolder(BaseFolder): content) self.ui.debug('imap', 'savemessage: header is: %s: %s' %\ (headername, headervalue)) - content = self.addmessageheader(content, headername, headervalue) + content = self.addmessageheader(content, CRLF, headername, headervalue) if len(content)>200: dbg_output = "%s...%s" % (content[:150], content[-50:]) From c92c4e56a092f271d71daedf796461945e682dee Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 1 Jun 2014 22:32:57 +0400 Subject: [PATCH 622/817] Add version qualifier to differentiate releases and development ones It is always good to see which version we're talking about, so I had added explicit marker for -devel, -release, -rcX and other states of the OfflineIMAP. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ offlineimap/__init__.py | 6 ++++-- offlineimap/init.py | 2 +- offlineimap/ui/Curses.py | 2 +- offlineimap/ui/UIBase.py | 4 ++-- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 4a4db93..27a31a5 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,6 +12,9 @@ OfflineIMAP v6.5.6.1 (YYYY-MM-DD) X-OfflineIMAP was added with preceeding '\n' instead of '\r\n' just before message was uploaded to the IMAP server. +* Add missing version bump for 6.5.6 (it was released with + 6.5.5 in setup.py and other places). + OfflineIMAP v6.5.6 (2014-05-14) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 60c79a3..1df5d2c 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,13 +1,15 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.5" +__version__ = "6.5.6.1" +__revision__ = "-devel" +__bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2013 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" -__bigcopyright__ = """%(__productname__)s %(__version__)s +__bigcopyright__ = """%(__productname__)s %(__bigversion__)s %(__license__)s""" % locals() __homepage__ = "http://offlineimap.org" diff --git a/offlineimap/init.py b/offlineimap/init.py index 9193bd8..5bb0438 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -50,7 +50,7 @@ class OfflineImap: self.__sync(options) def __parse_cmd_options(self): - parser = OptionParser(version=offlineimap.__version__, + parser = OptionParser(version=offlineimap.__bigversion__, description="%s.\n\n%s" % (offlineimap.__copyright__, offlineimap.__license__)) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 1c378ab..4150066 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -576,7 +576,7 @@ class Blinkenlights(UIBase, CursesUtil): self.bannerwin.clear() # Delete old content (eg before resizes) self.bannerwin.bkgd(' ', color) # Fill background with that color string = "%s %s" % (offlineimap.__productname__, - offlineimap.__version__) + offlineimap.__bigversion__) self.bannerwin.addstr(0, 0, string, color) self.bannerwin.addstr(0, self.width -len(offlineimap.__copyright__) -1, offlineimap.__copyright__, color) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 4dfd092..dace655 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -95,7 +95,7 @@ class UIBase(object): # write out more verbose initial info blurb on the log file p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ - "Args: %s" % (offlineimap.__version__, p_ver, sys.platform, + "Args: %s" % (offlineimap.__bigversion__, p_ver, sys.platform, " ".join(sys.argv)) self.logger.info(msg) @@ -409,7 +409,7 @@ class UIBase(object): #TODO: Debug and make below working, it hangs Gmail #res_type, response = conn.id(( # 'name', offlineimap.__productname__, - # 'version', offlineimap.__version__)) + # 'version', offlineimap.__bigversion__)) #self._msg("Server ID: %s %s" % (res_type, response[0])) self._msg("Server welcome string: %s" % str(conn.welcome)) self._msg("Server capabilities: %s\n" % str(conn.capabilities)) From 968ffc20bd0f1b3e9bbc0a0df15eab6f2acf21f8 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 24 Jun 2014 18:48:58 +0400 Subject: [PATCH 623/817] More clearly show results of folder name translation For cases like http://article.gmane.org/gmane.mail.imap.offlineimap.general/6468 it is beneficial to see that folder name was translated and the result of this translation on a single line: having log like {{{ Folder Boring/Wreck [acc: tmarble@info9.net]: Syncing Boring/Breck: Gmail -> Maildir }}} with translated name on the "Folder" line and original one on the "Syncing" line isn't very intuitive. Signed-off-by: Eygene Ryabinkin --- offlineimap/accounts.py | 2 +- offlineimap/folder/Base.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 2c9551d..f8a2281 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -326,7 +326,7 @@ class SyncableAccount(Account): thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, - name = "Folder %s [acc: %s]" % (remotefolder, self), + name = "Folder %s [acc: %s]" % (remotefolder.getexplainedname(), self), args = (self, remotefolder, quick)) thread.start() folderthreads.append(thread) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index bca52a2..a80a6e0 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -132,6 +132,13 @@ class BaseFolder(object): """The nametrans-transposed name of the folder's name""" return self.visiblename + def getexplainedname(self): + """ Name that shows both real and nametrans-mangled values""" + if self.name == self.visiblename: + return self.name + else: + return "%s [remote name %s]" % (self.visiblename, self.name) + def getrepository(self): """Returns the repository object that this folder is within.""" return self.repository From f282f0b0e0404fd156589e2a83e03bb67102cfb2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 24 Jun 2014 16:32:49 +0200 Subject: [PATCH 624/817] Merge pull request #67 from danmilon/fix-docs-folderfilter-typo docs: fix folderfilter typo --- docs/doc-src/nametrans.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/doc-src/nametrans.rst b/docs/doc-src/nametrans.rst index c211072..f6b9924 100644 --- a/docs/doc-src/nametrans.rst +++ b/docs/doc-src/nametrans.rst @@ -13,7 +13,7 @@ safely skip this section. folderfilter ------------ -If you do not want to synchronize all your filters, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. +If you do not want to synchronize all your folders, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. The only parameter to that function is the folder name. If the filter function returns True, the folder will be synced, if it returns False, @@ -114,7 +114,7 @@ Reverse nametrans Since 6.4.0, OfflineImap supports the creation of folders on the remote repository and that complicates things. Previously, only one nametrans setting on the remote repository was needed and that transformed a remote to a local name. However, nametrans transformations are one-way, and OfflineImap has no way using those rules on the remote repository to back local names to remote names. -Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts of any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. +Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts off any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. Take the above examples. If your remote nametrans setting was:: @@ -122,7 +122,7 @@ Take the above examples. If your remote nametrans setting was:: then you will want to have this in your local repository, prepending "INBOX" to any local folder name:: - nametrans = lambda folder: 'INBOX' + folder + nametrans = lambda folder: 'INBOX.' + folder Failure to set the local nametrans rule will lead to weird-looking error messages of -for instance- this type:: From e791901b975a60c6e06b9f1fdefc36ef58c9021a Mon Sep 17 00:00:00 2001 From: Gioele Date: Mon, 23 Jun 2014 08:33:47 +0200 Subject: [PATCH 625/817] Fix Markdown formatting Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ README.md | 66 +++++++++++++++++++++++++-------------------------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 27a31a5..781e252 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -15,6 +15,8 @@ OfflineIMAP v6.5.6.1 (YYYY-MM-DD) * Add missing version bump for 6.5.6 (it was released with 6.5.5 in setup.py and other places). +* Various fixes in documentation. + OfflineIMAP v6.5.6 (2014-05-14) =============================== diff --git a/README.md b/README.md index bc839af..81f78de 100644 --- a/README.md +++ b/README.md @@ -39,18 +39,18 @@ detailed information on how to install and configure OfflineImap. Quick Start =========== -First, install OfflineIMAP. See docs/INSTALL.rst or read -http://docs.offlineimap.org/en/latest/INSTALL.html. -(hint: `sudo python setup.py install`) +First, install OfflineIMAP. See `docs/INSTALL.rst` or read + +(hint: `sudo python setup.py install`). Second, set up your configuration file and run it! The distribution includes offlineimap.conf.minimal (Debian users may find this at -``/usr/share/doc/offlineimap/examples/offlineimap.conf.minimal``) that +`/usr/share/doc/offlineimap/examples/offlineimap.conf.minimal`) that provides you with the bare minimum of setting up OfflineIMAP. You can simply copy this file into your home directory and name it -``.offlineimaprc``. A command such as ``cp offlineimap.conf.minimal -~/.offlineimaprc`` will do it. Or, if you prefer, you can just copy -this text to ``~/.offlineimaprc``: +`.offlineimaprc`. A command such as `cp offlineimap.conf.minimal +~/.offlineimaprc` will do it. Or, if you prefer, you can just copy +this text to `~/.offlineimaprc`: [general] accounts = Test @@ -69,17 +69,17 @@ this text to ``~/.offlineimaprc``: remoteuser = jgoerzen -Now, edit the ``~/.offlineimaprc`` file with your favorite editor. All you have -to do is specify a directory for your folders to be in (on the localfolders -line), the host name of your IMAP server (on the remotehost line), and your -login name on the remote (on the remoteuser line). That's it! +Now, edit the `~/.offlineimaprc` file with your favorite editor. All you have +to do is specify a directory for your folders to be in (on the `localfolders` +line), the host name of your IMAP server (on the `remotehost` line), and your +login name on the remote (on the `remoteuser` line). That's it! -If you prefer to be XDG-compatible, - http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html -then substitute the above ``~/.offlineimaprc'' with -``$XDG\_CONFIG\_HOME/offlineimap/config'' and don't forget to set -XDG\_CONFIG\_HOME properly if you want it to be different from -the default ``$HOME/.config'' for any reason. +If you prefer to be compatible with the [XDG Base Directory +spec](http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html), +then substitute the above `~/.offlineimaprc` with +`$XDG_CONFIG_HOME/offlineimap/config` and don't forget to set +`XDG_CONFIG_HOME` properly if you want it to be different from +the default `$HOME/.config` for any reason. To run OfflineIMAP, you just have to say `offlineimap` ― it will fire up, ask you for a login password if necessary, synchronize your folders, @@ -97,12 +97,12 @@ Mailing list & bug reporting The user discussion, development and all exciting stuff take place in the OfflineImap mailing list at -http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project. You do not +. You do not need to subscribe to send emails. Bugs, issues and contributions should be reported to the mailing list. Bugs can also be reported in the issue tracker at -https://github.com/OfflineIMAP/offlineimap/issues. +. Configuration Examples ====================== @@ -117,22 +117,22 @@ Multiple Accounts with Mutt This example shows you how to set up OfflineIMAP to synchronize multiple accounts with the mutt mail reader. -Start by creating a directory to hold your folders by running ``mkdir ~/Mail``. -Then, in your ``~/.offlineimaprc``, specify: +Start by creating a directory to hold your folders by running `mkdir ~/Mail`. +Then, in your `~/.offlineimaprc`, specify: accounts = Personal, Work -Make sure that you have both an [Account Personal] and an [Account Work] -section. The local repository for each account must have different localfolder -path names. Also, make sure to enable [mbnames]. +Make sure that you have both an `[Account Personal]` and an `[Account Work]` +section. The local repository for each account must have different `localfolder` +path names. Also, make sure to enable `[mbnames]`. In each local repository section, write something like this: localfolders = ~/Mail/Personal -Finally, add these lines to your ``~/.muttrc``: +Finally, add these lines to your `~/.muttrc`: source ~/path-to-mbnames-muttrc-mailboxes folder-hook Personal set from="youremail@personal.com" @@ -149,10 +149,10 @@ UW-IMAPD and References ----------------------- Some users with a UW-IMAPD server need to use OfflineIMAP's "reference" feature -to get at their mailboxes, specifying a reference of ``~/Mail`` or ``#mh/`` +to get at their mailboxes, specifying a reference of `~/Mail` or `#mh/` depending on the configuration. The below configuration from (originally from -docwhat@gerf.org) shows using a reference of Mail, a nametrans that strips the -leading Mail/ off incoming folder names, and a folderfilter that limits the +docwhat@gerf.org) shows using a reference of Mail, a `nametrans` that strips the +leading `Mail/` off incoming folder names, and a `folderfilter` that limits the folders synced to just three: [Account Gerf] @@ -191,16 +191,16 @@ configuration file options that are Python expressions. This example is based on one supplied by Tommi Virtanen for this feature. -In ~/.offlineimaprc, he adds these options: +In `~/.offlineimaprc`, he adds these options: [general] pythonfile=~/.offlineimap.py [Repository foo] foldersort=mycmp -Then, the ~/.offlineimap.py file will contain: +Then, the `~/.offlineimap.py` file will contain: - prioritized = ['INBOX', 'personal', 'announce', 'list'] + prioritized = ['INBOX', 'personal', 'announce', 'list'] def mycmp(x, y): for prefix in prioritized: @@ -221,5 +221,5 @@ Then, the ~/.offlineimap.py file will contain: print folders -This code snippet illustrates how the foldersort option can be customized with a -Python function from the pythonfile to always synchronize certain folders first. +This code snippet illustrates how the `foldersort` option can be customized with a +Python function from the `pythonfile` to always synchronize certain folders first. From afead6c48e33dc8078d768bbf24011da914a7f72 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Tue, 24 Jun 2014 18:05:21 +0100 Subject: [PATCH 626/817] Rename addmessageheader()'s crlf parameter to linebreak The parameter's value is a string representing the linebreak, and can sometimes contain just '\n', in which case naming it crlf is slightly misleading. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index a80a6e0..63dbada 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -420,13 +420,13 @@ next line\n """ - def addmessageheader(self, content, crlf, headername, headervalue): + def addmessageheader(self, content, linebreak, headername, headervalue): """ Adds new header to the provided message. Arguments: - content: message content, headers and body as a single string - - crlf: string that carries line ending + - linebreak: string that carries line ending - headername: name of the header to add - headervalue: value of the header to add @@ -434,12 +434,12 @@ next line\n self.ui.debug('', 'addmessageheader: called to add %s: %s' % (headername, headervalue)) - prefix = crlf + prefix = linebreak suffix = '' - insertionpoint = content.find(crlf + crlf) + insertionpoint = content.find(linebreak * 2) if insertionpoint == 0 or insertionpoint == -1: prefix = '' - suffix = crlf + suffix = linebreak if insertionpoint == -1: insertionpoint = 0 # When body starts immediately, without preceding '\n' @@ -447,8 +447,8 @@ next line\n # we seen many broken ones), we should add '\n' to make # new (and the only header, in this case) to be properly # separated from the message body. - if content[0:len(crlf)] != crlf: - suffix = suffix + crlf + if content[0:len(linebreak)] != linebreak: + suffix = suffix + linebreak self.ui.debug('', 'addmessageheader: insertionpoint = %d' % insertionpoint) headers = content[0:insertionpoint] From 37f74d859adee09ebdf245c5eb819bb98160b083 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Tue, 24 Jun 2014 18:06:34 +0100 Subject: [PATCH 627/817] addmessageheader: fix case #2 and flesh out docstring The example illustrations were slightly cryptic; modify them to be more obvious. Fix case #2 (message starts with two line breaks) where additional line break was shown and coded. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 86 ++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 63dbada..ff36d5a 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -386,40 +386,6 @@ class BaseFolder(object): for uid in uidlist: self.deletemessagelabels(uid, labels) - - """ - Illustration of all cases for addmessageheader(). - '+' means the added contents. - -Case 1: No '\n\n', leading '\n' -+X-Flying-Pig-Header: i am here\n -\n -This is the body\n -next line\n - -Case 2: '\n\n' at position 0 -+X-Flying-Pig-Header: i am here\n -\n -\n -This is the body\n -next line\n - -Case 3: No '\n\n', no leading '\n' -+X-Flying-Pig-Header: i am here\n -+\n -This is the body\n -next line\n - -Case 4: '\n\n' at non-zero position -Subject: Something wrong with OI\n -From: some@person.at+\n -X-Flying-Pig-Header: i am here\n <-- orig '\n' -\n -This is the body\n -next line\n - - """ - def addmessageheader(self, content, linebreak, headername, headervalue): """ Adds new header to the provided message. @@ -430,19 +396,65 @@ next line\n - headername: name of the header to add - headervalue: value of the header to add + This has to deal with strange corner cases where the header is + missing or empty. Here are illustrations for all the cases, + showing where the header gets inserted and what the end result + is. In each illustration, '+' means the added contents. Note + that these examples assume LF for linebreak, not CRLF, so '\n' + denotes a linebreak and '\n\n' corresponds to the transition + between header and body. However if the linebreak parameter + is set to '\r\n' then you would have to substitute '\r\n' for + '\n' in the below examples. + + * Case 1: No '\n\n', leading '\n' + + +X-Flying-Pig-Header: i am here\n + \n + This is the body\n + next line\n + + * Case 2: '\n\n' at position 0 + + +X-Flying-Pig-Header: i am here + \n + \n + This is the body\n + next line\n + + * Case 3: No '\n\n', no leading '\n' + + +X-Flying-Pig-Header: i am here\n + +\n + This is the body\n + next line\n + + * Case 4: '\n\n' at non-zero position + + Subject: Something wrong with OI\n + From: some@person.at + +\nX-Flying-Pig-Header: i am here + \n + \n + This is the body\n + next line\n """ self.ui.debug('', 'addmessageheader: called to add %s: %s' % (headername, headervalue)) + # Hoping for case #4 prefix = linebreak suffix = '' insertionpoint = content.find(linebreak * 2) - if insertionpoint == 0 or insertionpoint == -1: + # Case #2 + if insertionpoint == 0: + prefix = '' + suffix = '' + # Either case #1 or #3 + elif insertionpoint == -1: prefix = '' suffix = linebreak - if insertionpoint == -1: insertionpoint = 0 - # When body starts immediately, without preceding '\n' + # Case #3: when body starts immediately, without preceding '\n' # (this shouldn't happen with proper mail messages, but # we seen many broken ones), we should add '\n' to make # new (and the only header, in this case) to be properly From 807f3da880820f87b8db06e8756deda737ea5347 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Tue, 24 Jun 2014 18:07:17 +0100 Subject: [PATCH 628/817] addmessageheader(): add debug for header insertion Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index ff36d5a..e6c21bd 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -441,10 +441,22 @@ class BaseFolder(object): self.ui.debug('', 'addmessageheader: called to add %s: %s' % (headername, headervalue)) + + insertionpoint = content.find(linebreak * 2) + if insertionpoint == -1: + self.ui.debug('', 'addmessageheader: headers were missing') + else: + self.ui.debug('', 'addmessageheader: headers end at position %d' % insertionpoint) + mark = '==>EOH<==' + contextstart = max(0, insertionpoint - 100) + contextend = min(len(content), insertionpoint + 100) + self.ui.debug('', 'addmessageheader: header/body transition context (marked by %s): %s' % + (mark, repr(content[contextstart:insertionpoint]) + \ + mark + repr(content[insertionpoint:contextend]))) + # Hoping for case #4 prefix = linebreak suffix = '' - insertionpoint = content.find(linebreak * 2) # Case #2 if insertionpoint == 0: prefix = '' From 863113efa383c23c2b7b02835796ead159697c6b Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 30 Jun 2014 16:41:17 +0400 Subject: [PATCH 629/817] Match header names case-insensitively http://tools.ietf.org/html/rfc5234#section-2.3 says that ABNF strings are case-insensitive. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index e6c21bd..41fd627 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -502,6 +502,8 @@ class BaseFolder(object): def getmessageheader(self, content, name): """ Searches for the given header and returns its value. + Header name is case-insensitive. + Arguments: - contents: message itself - name: name of the header to be searched @@ -515,7 +517,7 @@ class BaseFolder(object): headers = content[0:eoh] self.ui.debug('', 'getmessageheader: headers = %s' % repr(headers)) - m = re.search('^%s:(.*)$' % name, headers, flags = re.MULTILINE) + m = re.search('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE) if m: return m.group(1).strip() else: From ffd1b1d691d3206758e2549fda4ac0107ceab871 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 1 Jul 2014 07:44:18 +0400 Subject: [PATCH 630/817] IMAP: provide message-id in error messages This is handy for debug purposes when one tries to locate exact message that was e.g. rejected by server. Feature-request: http://comments.gmane.org/gmane.mail.imap.offlineimap.general/6491 Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/IMAP.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 3b506e5..1029cf7 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -517,6 +517,11 @@ class IMAPFolder(BaseFolder): # get the date of the message, so we can pass it to the server. date = self.__getmessageinternaldate(content, rtime) + # Message-ID is handy for debugging messages + msg_id = self.getmessageheader(content, "message-id") + if not msg_id: + msg_id = '[unknown message-id]' + retry_left = 2 # succeeded in APPENDING? imapobj = self.imapserver.acquireconnection() # NB: in the finally clause for this try we will release @@ -569,9 +574,9 @@ class IMAPFolder(BaseFolder): # In this case, we should immediately abort the repository sync # and continue with the next account. msg = \ - "Saving msg in folder '%s', repository '%s' failed (abort). " \ + "Saving msg (%s) in folder '%s', repository '%s' failed (abort). " \ "Server responded: %s %s\n" % \ - (self, self.getrepository(), typ, dat) + (msg_id, self, self.getrepository(), typ, dat) raise OfflineImapError(msg, OfflineImapError.ERROR.REPO) retry_left = 0 # Mark as success except imapobj.abort as e: @@ -580,10 +585,10 @@ class IMAPFolder(BaseFolder): self.imapserver.releaseconnection(imapobj, True) imapobj = self.imapserver.acquireconnection() if not retry_left: - raise OfflineImapError("Saving msg in folder '%s', " + raise OfflineImapError("Saving msg (%s) in folder '%s', " "repository '%s' failed (abort). Server responded: %s\n" "Message content was: %s" % - (self, self.getrepository(), str(e), dbg_output), + (msg_id, self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) self.ui.error(e, exc_info()[2]) except imapobj.error as e: # APPEND failed @@ -592,9 +597,9 @@ class IMAPFolder(BaseFolder): # drop conn, it might be bad. self.imapserver.releaseconnection(imapobj, True) imapobj = None - raise OfflineImapError("Saving msg folder '%s', repo '%s'" + raise OfflineImapError("Saving msg (%s) folder '%s', repo '%s'" "failed (error). Server responded: %s\nMessage content was: " - "%s" % (self, self.getrepository(), str(e), dbg_output), + "%s" % (msg_id, self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. From 1f2e8af8aaf08db8df03e45962aa67e47297f3f5 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Tue, 8 Jul 2014 12:23:05 +0400 Subject: [PATCH 631/817] Trade recursion by plain old cycle We can do a simpler and more stack-friendly hack for IMAP servers with limited line lenghts. Reported-by: Josh Berry, https://github.com/josh-berry GH: https://github.com/OfflineIMAP/offlineimap/pull/100 Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/IMAP.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 1029cf7..05e0a36 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -755,14 +755,7 @@ class IMAPFolder(BaseFolder): def deletemessagesflags(self, uidlist, flags): self.__processmessagesflags('-', uidlist, flags) - def __processmessagesflags(self, operation, uidlist, flags): - # XXX: should really iterate over batches of 100 UIDs - if len(uidlist) > 101: - # Hack for those IMAP servers with a limited line length - self.__processmessagesflags(operation, uidlist[:100], flags) - self.__processmessagesflags(operation, uidlist[100:], flags) - return - + def __processmessagesflags_real(self, operation, uidlist, flags): imapobj = self.imapserver.acquireconnection() try: try: @@ -804,6 +797,16 @@ class IMAPFolder(BaseFolder): elif operation == '-': self.messagelist[uid]['flags'] -= flags + + def __processmessagesflags(self, operation, uidlist, flags): + # Hack for those IMAP servers with a limited line length + batch_size = 100 + while len(uidlist): + self.__processmessagesflags_real(operation, uidlist[:batch_size], flags) + uidlist = uidlist[batch_size:] + return + + # Interface from BaseFolder def change_message_uid(self, uid, new_uid): """Change the message from existing uid to new_uid From aa55b38e26e0ccb904638b4b99f563a45ab3ec0e Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Thu, 10 Jul 2014 10:24:11 +0400 Subject: [PATCH 632/817] Avoid copying array every time, just slice it Suggested-by: Josh Berry Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ offlineimap/folder/IMAP.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 781e252..0b485ee 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -17,6 +17,8 @@ OfflineIMAP v6.5.6.1 (YYYY-MM-DD) * Various fixes in documentation. +* Fix unbounded recursion during flag update (Josh Berry). + OfflineIMAP v6.5.6 (2014-05-14) =============================== diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 05e0a36..e317baf 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -801,9 +801,9 @@ class IMAPFolder(BaseFolder): def __processmessagesflags(self, operation, uidlist, flags): # Hack for those IMAP servers with a limited line length batch_size = 100 - while len(uidlist): - self.__processmessagesflags_real(operation, uidlist[:batch_size], flags) - uidlist = uidlist[batch_size:] + for i in xrange(0, len(uidlist), batch_size): + self.__processmessagesflags_real(operation, + uidlist[i:i + batch_size], flags) return From 73e2c95acd829282b537df4144a3145b9452c0e7 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sat, 2 Aug 2014 22:23:31 +0400 Subject: [PATCH 633/817] Create SQLite DB directory if it doesn't exist yet Also check if DB path doesn't point to a directory. By: Nick Farrell GitHub: https://github.com/OfflineIMAP/offlineimap/pull/102 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 4 ++++ offlineimap/folder/LocalStatusSQLite.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 0b485ee..082950f 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,10 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Create SQLite database directory if it doesn't exist + yet; warn if path is not a directory (Nick Farrell, + GutHub pull #102) + * Fix mangled message headers for servers without UIDPLUS: X-OfflineIMAP was added with preceeding '\n' instead of '\r\n' just before message was uploaded to the IMAP server. diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index f8921f1..e48545d 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import os.path +import os import re from threading import Lock from .Base import BaseFolder @@ -51,6 +51,12 @@ class LocalStatusSQLiteFolder(BaseFolder): self._newfolder = False # flag if the folder is new + dirname = os.path.dirname(self.filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + if not os.path.isdir(dirname): + raise UserWarning("SQLite database path '%s' is not a directory." % dirname) + # dblock protects against concurrent writes in same connection self._dblock = Lock() From 7df765cfdb4096a33f35bf709db6820a6c2c79fd Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 3 Aug 2014 16:47:26 +0400 Subject: [PATCH 634/817] Properly manipulate contents of messagelist for folder Create initializer function that puts default values to all fields of message list item. Fix all code that directly assigns some hash to the elements of messagelist: for direct assignments only initializer is now permitted, all other modification are done in-place. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/Base.py | 11 +++++++++++ offlineimap/folder/Gmail.py | 6 ++++++ offlineimap/folder/GmailMaildir.py | 6 ++++++ offlineimap/folder/IMAP.py | 8 +++++++- offlineimap/folder/LocalStatus.py | 19 ++++++++++++++++--- offlineimap/folder/LocalStatusSQLite.py | 16 ++++++++++++++-- offlineimap/folder/Maildir.py | 17 +++++++++++++++-- 7 files changed, 75 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 41fd627..b56cd1b 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -236,6 +236,17 @@ class BaseFolder(object): You must call cachemessagelist() before calling this function!""" raise NotImplementedError + def msglist_item_initializer(self, uid): + """ + Returns value for empty messagelist element with given UID. + + This function must initialize all fields of messagelist item + and must be called every time when one creates new messagelist + entry to ensure that all fields that must be present are present. + + """ + raise NotImplementedError + def uidexists(self, uid): """Returns True if uid exists""" return uid in self.getmessagelist() diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 2a598de..e3ef92a 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -110,6 +110,11 @@ class GmailFolder(IMAPFolder): else: return set() + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0} + + # TODO: merge this code with the parent's cachemessagelist: # TODO: they have too much common logics. def cachemessagelist(self): @@ -152,6 +157,7 @@ class GmailFolder(IMAPFolder): minor = 1) else: uid = long(options['UID']) + self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) m = re.search('\(([^\)]*)\)', options['X-GM-LABELS']) if m: diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index be0d20d..b62b701 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -56,6 +56,12 @@ class GmailMaildirFolder(MaildirFolder): return True return False #Nope, nothing changed + + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'flags': set(), 'labels': set(), 'filename': '/no-dir/no-such-file/', 'mtime': 0} + + def cachemessagelist(self): if self.messagelist is None: self.messagelist = self._scanfolder() diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index e317baf..a698888 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -202,6 +202,10 @@ class IMAPFolder(BaseFolder): return msgsToFetch + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'uid': uid, 'flags': set(), 'time': 0} + # Interface from BaseFolder def cachemessagelist(self): @@ -239,6 +243,7 @@ class IMAPFolder(BaseFolder): minor = 1) else: uid = long(options['UID']) + self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} @@ -641,7 +646,8 @@ class IMAPFolder(BaseFolder): if imapobj: self.imapserver.releaseconnection(imapobj) if uid: # avoid UID FETCH 0 crash happening later on - self.messagelist[uid] = {'uid': uid, 'flags': flags} + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags self.ui.debug('imap', 'savemessage: returning new UID %d' % uid) return uid diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 7e5928e..1dccf90 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -57,6 +57,11 @@ class LocalStatusFolder(BaseFolder): os.unlink(self.filename) + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0} + + def readstatus_v1(self, fp): """ Read status folder in format version 1. @@ -76,7 +81,8 @@ class LocalStatusFolder(BaseFolder): (line, self.filename) self.ui.warn(errstr) raise ValueError(errstr) - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': 0, 'labels': set()} + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags def readstatus(self, fp): @@ -100,7 +106,10 @@ class LocalStatusFolder(BaseFolder): (line, self.filename) self.ui.warn(errstr) raise ValueError(errstr) - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': mtime, 'labels': labels} + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags + self.messagelist[uid]['mtime'] = mtime + self.messagelist[uid]['labels'] = labels # Interface from BaseFolder @@ -197,7 +206,11 @@ class LocalStatusFolder(BaseFolder): self.savemessageflags(uid, flags) return uid - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels} + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags + self.messagelist[uid]['time'] = rtime + self.messagelist[uid]['mtime'] = mtime + self.messagelist[uid]['labels'] = labels self.save() return uid diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index e48545d..a9ef6c2 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -184,14 +184,23 @@ class LocalStatusSQLiteFolder(BaseFolder): self._newfolder = True + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0} + + # Interface from BaseFolder def cachemessagelist(self): self.messagelist = {} cursor = self.connection.execute('SELECT id,flags,mtime,labels from status') for row in cursor: + uid = row[0] + self.messagelist[uid] = self.msglist_item_initializer(uid) flags = set(row[1]) labels = set([lb.strip() for lb in row[3].split(',') if len(lb.strip()) > 0]) - self.messagelist[row[0]] = {'uid': row[0], 'flags': flags, 'mtime': row[2], 'labels': labels} + self.messagelist[uid]['flags'] = flags + self.messagelist[uid]['labels'] = labels + self.messagelist[uid]['mtime'] = row[2] # Interface from LocalStatusFolder def save(self): @@ -271,6 +280,7 @@ class LocalStatusSQLiteFolder(BaseFolder): self.savemessageflags(uid, flags) return uid + self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels} flags = ''.join(sorted(flags)) labels = ', '.join(sorted(labels)) @@ -278,9 +288,11 @@ class LocalStatusSQLiteFolder(BaseFolder): (uid,flags,mtime,labels)) return uid + # Interface from BaseFolder def savemessageflags(self, uid, flags): - self.messagelist[uid] = {'uid': uid, 'flags': flags} + assert self.uidexists(uid) + self.messagelist[uid]['flags'] = flags flags = ''.join(sorted(flags)) self.__sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index d067194..156b39e 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -191,7 +191,9 @@ class MaildirFolder(BaseFolder): else: uid = long(uidmatch.group(1)) # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S - retval[uid] = {'flags': flags, 'filename': filepath} + retval[uid] = self.msglist_item_initializer(uid) + retval[uid]['flags'] = flags + retval[uid]['filename'] = filepath return retval # Interface from BaseFolder @@ -208,6 +210,12 @@ class MaildirFolder(BaseFolder): return True return False #Nope, nothing changed + + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'flags': set(), 'filename': '/no-dir/no-such-file/'} + + # Interface from BaseFolder def cachemessagelist(self): if self.messagelist is None: @@ -319,7 +327,9 @@ class MaildirFolder(BaseFolder): if rtime != None: os.utime(os.path.join(self.getfullname(), tmpname), (rtime, rtime)) - self.messagelist[uid] = {'flags': flags, 'filename': tmpname} + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags + self.messagelist[uid]['filename'] = tmpname # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) self.ui.debug('maildir', 'savemessage: returning uid %d' % uid) @@ -339,6 +349,9 @@ class MaildirFolder(BaseFolder): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + + assert uid in self.messagelist + oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) # If a message has been seen, it goes into 'cur' From aef88cc1f85ca021935b1ac09fd12652500a9ddf Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 22 Aug 2014 17:43:36 +0400 Subject: [PATCH 635/817] Fix label processing in GmailMaildir Commit 7df765cfdb4096a33f35bf709db6820a6c2c79fd introduced regression: GmailMaildir caches labels in its own function and it was testing the presence of the 'labels' key in message descriptor. But 7df765cf changed descriptor initialization and this key is always present. So now we have 'labels_cached' flag that tells us if labels were already cached or not. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/GmailMaildir.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index b62b701..2cab213 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -59,7 +59,8 @@ class GmailMaildirFolder(MaildirFolder): # Interface from BaseFolder def msglist_item_initializer(self, uid): - return {'flags': set(), 'labels': set(), 'filename': '/no-dir/no-such-file/', 'mtime': 0} + return {'flags': set(), 'labels': set(), 'labels_cached': False, + 'filename': '/no-dir/no-such-file/', 'mtime': 0} def cachemessagelist(self): @@ -72,9 +73,10 @@ class GmailMaildirFolder(MaildirFolder): filepath = os.path.join(self.getfullname(), msg['filename']) msg['mtime'] = long(os.stat(filepath).st_mtime) + def getmessagelabels(self, uid): # Labels are not cached in cachemessagelist because it is too slow. - if not 'labels' in self.messagelist[uid]: + if not self.messagelist[uid]['labels_cached']: filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) @@ -88,10 +90,11 @@ class GmailMaildirFolder(MaildirFolder): self.messagelist[uid]['labels'] = \ imaputil.labels_from_header(self.labelsheader, self.getmessageheader(content, self.labelsheader)) - + self.messagelist[uid]['labels_cached'] = True return self.messagelist[uid]['labels'] + def getmessagemtime(self, uid): if not 'mtime' in self.messagelist[uid]: return 0 From 818486283ed2ccf55e9a0d6be8a1acd8727fb69e Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 21 Sep 2014 11:15:38 +0400 Subject: [PATCH 636/817] Add support for OS-specific CA bundle locations GitHub pull: #19 Suggested-by: Michael Vogt Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 ++ offlineimap/repository/IMAP.py | 3 +- offlineimap/utils/distro.py | 74 ++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 offlineimap/utils/distro.py diff --git a/Changelog.rst b/Changelog.rst index 082950f..13eba38 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,9 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Support default CA bundle locations for a couple of + known Unix systems (Michael Vogt, GutHub pull #19) + * Create SQLite database directory if it doesn't exist yet; warn if path is not a directory (Nick Farrell, GutHub pull #102) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index ff1d5e2..c98419d 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -19,6 +19,7 @@ from offlineimap.repository.Base import BaseRepository from offlineimap import folder, imaputil, imapserver, OfflineImapError from offlineimap.folder.UIDMaps import MappedIMAPFolder from offlineimap.threadutil import ExitNotifyThread +from offlineimap.utils.distro import get_os_sslcertfile from threading import Event import os from sys import exc_info @@ -200,7 +201,7 @@ class IMAPRepository(BaseRepository): def getsslcacertfile(self): """Return the absolute path of the CA certfile to use, if any""" - cacertfile = self.getconf('sslcacertfile', None) + cacertfile = self.getconf('sslcacertfile', get_os_sslcertfile()) if cacertfile is None: return None cacertfile = os.path.expanduser(cacertfile) diff --git a/offlineimap/utils/distro.py b/offlineimap/utils/distro.py new file mode 100644 index 0000000..96aab1b --- /dev/null +++ b/offlineimap/utils/distro.py @@ -0,0 +1,74 @@ +# Copyright 2014 Eygene A. Ryabinkin. +# +# Module that supports distribution-specific functions. + +import platform +import os + + +# Each dictionary value is either string or some iterable. +# +# For the former we will just return the value, for an iterable +# we will walk through the values and will return the first +# one that corresponds to the existing file. +__DEF_OS_LOCATIONS = { + 'freebsd': '/usr/local/share/certs/ca-root-nss.crt', + 'openbsd': None, + 'netbsd': None, + 'dragonfly': None, + 'darwin': [ + # MacPorts, port curl-ca-bundle + '/opt/local/share/curl/curl-ca-bundle.crt', + ], + 'linux-ubuntu': '/etc/ssl/certs/ca-certificates.crt', + 'linux-debian': '/etc/ssl/certs/ca-certificates.crt', + 'linux-fedora': '/etc/pki/tls/certs/ca-bundle.crt', + 'linux-redhat': '/etc/pki/tls/certs/ca-bundle.crt', + 'linux-suse': '/etc/ssl/ca-bundle.pem', +} + + +def get_os_name(): + """ + Finds out OS name. For non-Linux system it will be just a plain + OS name (like FreeBSD), for Linux it will be "linux-", + where is the name of the distribution, as returned by + the first component of platform.linux_distribution. + + Return value will be all-lowercase to avoid confusion about + proper name capitalisation. + + """ + OS = platform.system().lower() + + if OS.startswith('linux'): + DISTRO = platform.linux_distribution()[0] + if DISTRO: + OS = OS + "-%s" % DISTRO.lower() + + return OS + + +def get_os_sslcertfile(): + """ + Finds out the location for the distribution-specific + CA certificate file bundle. + + Returns the location of the file or None if there is + no known CA certificate file or all known locations + correspond to non-existing filesystem objects. + + """ + OS = get_os_name() + + if OS in __DEF_OS_LOCATIONS: + l = __DEF_OS_LOCATIONS[OS] + if not hasattr(l, '__iter__'): + l = (l, ) + for f in l: + assert (type(f) == type("")) + if os.path.exists(f) and \ + (os.path.isfile(f) or os.path.islink(f)): + return f + + return None From 5cacda68ac76f1c706223c61ddfd024c95002ac5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 24 Sep 2014 10:24:49 +0200 Subject: [PATCH 637/817] Add the OpenSSL exception The GNU GPL and the OpenSSL license are incompatible. Some distributions take a hardline stance and do not consider OpenSSL to be a systems library (which would permit the usage/distribution of OpenSSL) when distributing apps such as OfflineImap from the same repository. In order to solve these distributions dilemma, we add the OpenSSL exception to our GNU GPL v2+ license. This allows for unambiguous use/distribution of our GNU GPL'ed application with a python linking to openssl. Consent of all contributors has been requested via email by Sebastian@SSpaeth.de. With very few exceptions of minor contributions (which might or might not by copyright-worthy) all past contributors have consented to adding the OpenSSL exception. None of the replying authors has disagreed with adding the exception. The corresponding issues at question: https://github.com/OfflineIMAP/offlineimap/issues/104 Debian bug #747033 We are still missing consent from: 1 Asheesh Laroia 2 Bart Kerkvliet 4 Daniel Burrows 5 David Favro 1 David Logie 1 Eric Dorland 1 Ethan Schoonover 49 Eygene Ryabinkin 1 Loui Chang 1 Luca Capello 1 Michael Witten 2 Mike Dawson 1 Peter Colberg 1 Scott Henson 1 Tom Lawton 1 W. Trevor King 2 X-Ryl669 1 buergi 2 dtk 5 mj Signed-off-by: Sebastian Spaeth --- COPYING | 14 ++++++++++++++ README.md | 7 ++++--- offlineimap/__init__.py | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/COPYING b/COPYING index 5f3f605..6526be4 100644 --- a/COPYING +++ b/COPYING @@ -348,3 +348,17 @@ proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. + +---------------------------------------------------------------- +In addition, as a special exception, the copyright holders give +permission to link the code of portions of this program with the OpenSSL +library under certain conditions as described in each individual source +file, and distribute linked combinations including the two. + +You must obey the GNU General Public License in all respects for all of +the code used other than OpenSSL. If you modify file(s) with this +exception, you may extend this exception to your version of the file(s), +but you are not obligated to do so. If you do not wish to do so, delete +this exception statement from your version. If you delete this exception +statement from all source files in the program, then also delete it +here. diff --git a/README.md b/README.md index 81f78de..7748f62 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,10 @@ documentation. OfflineIMAP does not require additional python dependencies beyond python >=2.6 (although python-sqlite is strongly recommended). -OfflineIMAP is a Free Software project licensed under the GNU General Public -License version 2 (or later). You can download it for free, and you can modify -it. In fact, you are encouraged to contribute to OfflineIMAP. +OfflineIMAP is a Free Software project licensed under the GNU General +Public License version 2 (or later) with a special exception that allows +the OpenSSL library to be used. You can download it for free, and you +can modify it. In fact, you are encouraged to contribute to OfflineIMAP. Documentation ------------- diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 1df5d2c..9c62089 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -8,7 +8,7 @@ __copyright__ = "Copyright 2002-2013 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" -__license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" +__license__ = "Licensed under the GNU GPL v2 or any later version (with an openssl exception)" __bigcopyright__ = """%(__productname__)s %(__bigversion__)s %(__license__)s""" % locals() __homepage__ = "http://offlineimap.org" From 31d06d1ccb5ab8a0113e537151d2f18aa2ff5263 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 5 Oct 2014 11:33:22 +0400 Subject: [PATCH 638/817] Properly capitalize OpenSSL And I am also not against this change in licensing. Signed-off-by: Eygene Ryabinkin --- offlineimap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 9c62089..6a4e0e2 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -8,7 +8,7 @@ __copyright__ = "Copyright 2002-2013 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" -__license__ = "Licensed under the GNU GPL v2 or any later version (with an openssl exception)" +__license__ = "Licensed under the GNU GPL v2 or any later version (with an OpenSSL exception)" __bigcopyright__ = """%(__productname__)s %(__bigversion__)s %(__license__)s""" % locals() __homepage__ = "http://offlineimap.org" From 0aa44c5df41e761367e055f39811713e89ea8c5d Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 1 Oct 2014 08:14:43 +0400 Subject: [PATCH 639/817] Fix warning processing for MachineUI Old code were trying to pass the message to be output as-is, but it should really be passed as the 'mesg' key in order to be properly processed by MAchineLogFormatter.format(). GitHub pull: https://github.com/OfflineIMAP/offlineimap/pull/64 GitHub pull: https://github.com/OfflineIMAP/offlineimap/pull/118 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 + offlineimap/ui/Machine.py | 138 ++++++++++++++++++++++---------------- 2 files changed, 85 insertions(+), 56 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 13eba38..47e28ae 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,9 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Fix warning-level message processing by MachineUI + (GitHub pull #64, GitHub pull #118). + * Support default CA bundle locations for a couple of known Unix systems (Michael Vogt, GutHub pull #19) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index cee2849..01abd6f 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -28,82 +28,107 @@ protocol = '7.0.0' class MachineLogFormatter(logging.Formatter): """urlencodes any outputted line, to avoid multi-line output""" - def format(self, record): - # urlencode the "mesg" attribute and append to regular line... - line = super(MachineLogFormatter, self).format(record) - return line + urlencode([('', record.mesg)])[1:] + def format(s, record): + # Mapping of log levels to historic tag names + severity_map = { + 'info': 'msg', + 'warning': 'warn', + } + line = super(MachineLogFormatter, s).format(record) + severity = record.levelname.lower() + if severity in severity_map: + severity = severity_map[severity] + if hasattr(record, "machineui"): + command = record.machineui["command"] + whoami = record.machineui["id"] + else: + command = "" + whoami = currentThread().getName() + + prefix = "%s:%s" % (command, urlencode([('', whoami)])[1:]) + return "%s:%s:%s" % (severity, prefix, urlencode([('', line)])[1:]) + class MachineUI(UIBase): - def __init__(self, config, loglevel = logging.INFO): - super(MachineUI, self).__init__(config, loglevel) - self._log_con_handler.createLock() + def __init__(s, config, loglevel = logging.INFO): + super(MachineUI, s).__init__(config, loglevel) + s._log_con_handler.createLock() """lock needed to block on password input""" # Set up the formatter that urlencodes the strings... - self._log_con_handler.setFormatter(MachineLogFormatter()) + s._log_con_handler.setFormatter(MachineLogFormatter()) - def _printData(self, command, msg): - self.logger.info("%s:%s:%s" % ( - 'msg', command, currentThread().getName()), extra={'mesg': msg}) + # Arguments: + # - handler: must be method from s.logger that reflects + # the severity of the passed message + # - command: command that produced this message + # - msg: the message itself + def _printData(s, handler, command, msg): + handler(msg, + extra = { + 'machineui': { + 'command': command, + 'id': currentThread().getName(), + } + }) def _msg(s, msg): - s._printData('_display', msg) + s._printData(s.logger.info, '_display', msg) - def warn(self, msg, minor = 0): + def warn(s, msg, minor = 0): # TODO, remove and cleanup the unused minor stuff - self.logger.warning("%s:%s:%s:%s" % ( - 'warn', '', currentThread().getName(), msg)) + s._printData(s.logger.warning, '', msg) - def registerthread(self, account): - super(MachineUI, self).registerthread(account) - self._printData('registerthread', account) + def registerthread(s, account): + super(MachineUI, s).registerthread(account) + s._printData(s.logger.info, 'registerthread', account) def unregisterthread(s, thread): UIBase.unregisterthread(s, thread) - s._printData('unregisterthread', thread.getName()) + s._printData(s.logger.info, 'unregisterthread', thread.getName()) def debugging(s, debugtype): - s._printData('debugging', debugtype) + s._printData(s.logger.debug, 'debugging', debugtype) def acct(s, accountname): - s._printData('acct', accountname) + s._printData(s.logger.info, 'acct', accountname) def acctdone(s, accountname): - s._printData('acctdone', accountname) + s._printData(s.logger.info, 'acctdone', accountname) def validityproblem(s, folder): - s._printData('validityproblem', "%s\n%s\n%s\n%s" % \ + s._printData(s.logger.warning, 'validityproblem', "%s\n%s\n%s\n%s" % \ (folder.getname(), folder.getrepository().getname(), folder.get_saveduidvalidity(), folder.get_uidvalidity())) def connecting(s, hostname, port): - s._printData('connecting', "%s\n%s" % (hostname, str(port))) + s._printData(s.logger.info, 'connecting', "%s\n%s" % (hostname, str(port))) def syncfolders(s, srcrepos, destrepos): - s._printData('syncfolders', "%s\n%s" % (s.getnicename(srcrepos), + s._printData(s.logger.info, 'syncfolders', "%s\n%s" % (s.getnicename(srcrepos), s.getnicename(destrepos))) def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): - s._printData('syncingfolder', "%s\n%s\n%s\n%s\n" % \ + s._printData(s.logger.info, 'syncingfolder', "%s\n%s\n%s\n%s\n" % \ (s.getnicename(srcrepos), srcfolder.getname(), s.getnicename(destrepos), destfolder.getname())) def loadmessagelist(s, repos, folder): - s._printData('loadmessagelist', "%s\n%s" % (s.getnicename(repos), + s._printData(s.logger.info, 'loadmessagelist', "%s\n%s" % (s.getnicename(repos), folder.getvisiblename())) def messagelistloaded(s, repos, folder, count): - s._printData('messagelistloaded', "%s\n%s\n%d" % \ + s._printData(s.logger.info, 'messagelistloaded', "%s\n%s\n%d" % \ (s.getnicename(repos), folder.getname(), count)) def syncingmessages(s, sr, sf, dr, df): - s._printData('syncingmessages', "%s\n%s\n%s\n%s\n" % \ + s._printData(s.logger.info, 'syncingmessages', "%s\n%s\n%s\n%s\n" % \ (s.getnicename(sr), sf.getname(), s.getnicename(dr), df.getname())) - def copyingmessage(self, uid, num, num_to_copy, srcfolder, destfolder): - self._printData('copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ - (uid, self.getnicename(srcfolder), srcfolder.getname(), - self.getnicename(destfolder), destfolder)) + def copyingmessage(s, uid, num, num_to_copy, srcfolder, destfolder): + s._printData(s.logger.info, 'copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ + (uid, s.getnicename(srcfolder), srcfolder.getname(), + s.getnicename(destfolder), destfolder)) def folderlist(s, list): return ("\f".join(["%s\t%s" % (s.getnicename(x), x.getname()) for x in list])) @@ -113,57 +138,58 @@ class MachineUI(UIBase): def deletingmessages(s, uidlist, destlist): ds = s.folderlist(destlist) - s._printData('deletingmessages', "%s\n%s" % (s.uidlist(uidlist), ds)) + s._printData(s.logger.info, 'deletingmessages', "%s\n%s" % (s.uidlist(uidlist), ds)) def addingflags(s, uidlist, flags, dest): - s._printData("addingflags", "%s\n%s\n%s" % (s.uidlist(uidlist), + s._printData(s.logger.info, "addingflags", "%s\n%s\n%s" % (s.uidlist(uidlist), "\f".join(flags), dest)) def deletingflags(s, uidlist, flags, dest): - s._printData('deletingflags', "%s\n%s\n%s" % (s.uidlist(uidlist), + s._printData(s.logger.info, 'deletingflags', "%s\n%s\n%s" % (s.uidlist(uidlist), "\f".join(flags), dest)) - def threadException(self, thread): - self._printData('threadException', "%s\n%s" % \ - (thread.getName(), self.getThreadExceptionString(thread))) - self.delThreadDebugLog(thread) - self.terminate(100) + def threadException(s, thread): + s._printData(s.logger.warning, 'threadException', "%s\n%s" % \ + (thread.getName(), s.getThreadExceptionString(thread))) + s.delThreadDebugLog(thread) + s.terminate(100) def terminate(s, exitstatus = 0, errortitle = '', errormsg = ''): - s._printData('terminate', "%d\n%s\n%s" % (exitstatus, errortitle, errormsg)) + s._printData(s.logger.info, 'terminate', "%d\n%s\n%s" % (exitstatus, errortitle, errormsg)) sys.exit(exitstatus) def mainException(s): - s._printData('mainException', s.getMainExceptionString()) + s._printData(s.logger.warning, 'mainException', s.getMainExceptionString()) def threadExited(s, thread): - s._printData('threadExited', thread.getName()) + s._printData(s.logger.info, 'threadExited', thread.getName()) UIBase.threadExited(s, thread) def sleeping(s, sleepsecs, remainingsecs): - s._printData('sleeping', "%d\n%d" % (sleepsecs, remainingsecs)) + s._printData(s.logger.info, 'sleeping', "%d\n%d" % (sleepsecs, remainingsecs)) if sleepsecs > 0: time.sleep(sleepsecs) return 0 - def getpass(self, accountname, config, errmsg = None): + def getpass(s, accountname, config, errmsg = None): if errmsg: - self._printData('getpasserror', "%s\n%s" % (accountname, errmsg), - False) + s._printData(s.logger.warning, + 'getpasserror', "%s\n%s" % (accountname, errmsg), + False) - self._log_con_handler.acquire() # lock the console output + s._log_con_handler.acquire() # lock the console output try: - self._printData('getpass', accountname, False) + s._printData(s.logger.info, 'getpass', accountname, False) return (sys.stdin.readline()[:-1]) finally: - self._log_con_handler.release() + s._log_con_handler.release() - def init_banner(self): - self._printData('protocol', protocol) - self._printData('initbanner', offlineimap.banner) + def init_banner(s): + s._printData(s.logger.info, 'protocol', protocol) + s._printData(s.logger.info, 'initbanner', offlineimap.banner) - def callhook(self, msg): - self._printData('callhook', msg) + def callhook(s, msg): + s._printData(s.logger.info, 'callhook', msg) From a76bb0d2a3cb584ebd200abf18f4c425f5bce5d1 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 5 Oct 2014 11:42:07 +0400 Subject: [PATCH 640/817] Announce addition of OpenSSL license exception in ChangeLog Someone is probably picky about licensing, so allow them to discover the recent change (provided that they read change logs). Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index 47e28ae..ea7b082 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,12 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Added OpenSSL exception clause to our main GPL to allow + people to link with OpenSSL in run-time. It is needed + at least for Debian, see + https://lists.debian.org/debian-legal/2002/10/msg00113.html + for details. + * Fix warning-level message processing by MachineUI (GitHub pull #64, GitHub pull #118). From 716a6f47188e51a077645608e640ebfba52ed449 Mon Sep 17 00:00:00 2001 From: nopjmp Date: Tue, 7 Oct 2014 21:39:47 -0500 Subject: [PATCH 641/817] Add OpenBSD default CA certificates file location GitHub pull: https://github.com/OfflineIMAP/offlineimap/pull/120 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ offlineimap/utils/distro.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index ea7b082..95398ef 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,9 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Added default CA bundle location for OpenBSD + (GitHub pull #120). + * Added OpenSSL exception clause to our main GPL to allow people to link with OpenSSL in run-time. It is needed at least for Debian, see diff --git a/offlineimap/utils/distro.py b/offlineimap/utils/distro.py index 96aab1b..babefe7 100644 --- a/offlineimap/utils/distro.py +++ b/offlineimap/utils/distro.py @@ -13,7 +13,7 @@ import os # one that corresponds to the existing file. __DEF_OS_LOCATIONS = { 'freebsd': '/usr/local/share/certs/ca-root-nss.crt', - 'openbsd': None, + 'openbsd': '/etc/ssl/cert.pem', 'netbsd': None, 'dragonfly': None, 'darwin': [ From 0a569bea3d1d96b349861cc240e834f680ceccd7 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 8 Oct 2014 10:39:07 +0400 Subject: [PATCH 642/817] Add default CA bundle location for DragonFlyBSD Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 +- offlineimap/utils/distro.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 95398ef..66f5c05 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -9,7 +9,7 @@ OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= * Added default CA bundle location for OpenBSD - (GitHub pull #120). + (GitHub pull #120) and DragonFlyBSD. * Added OpenSSL exception clause to our main GPL to allow people to link with OpenSSL in run-time. It is needed diff --git a/offlineimap/utils/distro.py b/offlineimap/utils/distro.py index babefe7..7c944b9 100644 --- a/offlineimap/utils/distro.py +++ b/offlineimap/utils/distro.py @@ -15,7 +15,7 @@ __DEF_OS_LOCATIONS = { 'freebsd': '/usr/local/share/certs/ca-root-nss.crt', 'openbsd': '/etc/ssl/cert.pem', 'netbsd': None, - 'dragonfly': None, + 'dragonfly': '/etc/ssl/cert.pem', 'darwin': [ # MacPorts, port curl-ca-bundle '/opt/local/share/curl/curl-ca-bundle.crt', From a1bf8db5175d028f4733313e6e22b6b9887d6d2b Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 5 Oct 2014 11:49:08 +0400 Subject: [PATCH 643/817] Brought CustomConfig.py into more proper shape - Multi-line documentation for functions and methods now has ending triple-double-quotes on an own line, as per PEP 257. - Added documentation and comments to almost all functions and methods. - Added stub implementations for getconfig() and getsection() inside CustomConfig.ConfigHelperMixin to provide sane run-time diagnostics for classes that doesn't implement them. Signed-off-by: Eygene Ryabinkin --- offlineimap/CustomConfig.py | 128 +++++++++++++++++++++++++++------ offlineimap/accounts.py | 2 + offlineimap/repository/Base.py | 2 + 3 files changed, 110 insertions(+), 22 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 447ac19..9fef69a 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -24,26 +24,46 @@ import re class CustomConfigParser(SafeConfigParser): def getdefault(self, section, option, default, *args, **kwargs): - """Same as config.get, but returns the "default" option if there - is no such option specified.""" + """ + Same as config.get, but returns the value of `default` + if there is no such option specified. + + """ if self.has_option(section, option): return self.get(*(section, option) + args, **kwargs) else: return default + def getdefaultint(self, section, option, default, *args, **kwargs): + """ + Same as config.getint, but returns the value of `default` + if there is no such option specified. + + """ if self.has_option(section, option): return self.getint (*(section, option) + args, **kwargs) else: return default + def getdefaultfloat(self, section, option, default, *args, **kwargs): + """ + Same as config.getfloat, but returns the value of `default` + if there is no such option specified. + + """ if self.has_option(section, option): return self.getfloat(*(section, option) + args, **kwargs) else: return default def getdefaultboolean(self, section, option, default, *args, **kwargs): + """ + Same as config.getboolean, but returns the value of `default` + if there is no such option specified. + + """ if self.has_option(section, option): return self.getboolean(*(section, option) + args, **kwargs) else: @@ -63,6 +83,11 @@ class CustomConfigParser(SafeConfigParser): (separator_re, e)) def getdefaultlist(self, section, option, default, separator_re): + """ + Same as getlist, but returns the value of `default` + if there is no such option specified. + + """ if self.has_option(section, option): return self.getlist(*(section, option, separator_re)) else: @@ -82,43 +107,78 @@ class CustomConfigParser(SafeConfigParser): return LocalEval(path) def getsectionlist(self, key): - """Returns a list of sections that start with key + " ". That is, - if key is "Account", returns all section names that start with - "Account ", but strips off the "Account ". For instance, for - "Account Test", returns "Test".""" + """ + Returns a list of sections that start with key + " ". + + That is, if key is "Account", returns all section names that + start with "Account ", but strips off the "Account ". + + For instance, for "Account Test", returns "Test". + """ key = key + ' ' return [x[len(key):] for x in self.sections() \ if x.startswith(key)] def set_if_not_exists(self, section, option, value): - """Set a value if it does not exist yet + """ + Set a value if it does not exist yet This allows to set default if the user has not explicitly - configured anything.""" + configured anything. + + """ if not self.has_option(section, option): self.set(section, option, value) + + def CustomConfigDefault(): - """Just a constant that won't occur anywhere else. + """ + Just a constant that won't occur anywhere else. This allows us to differentiate if the user has passed in any default value to the getconf* functions in ConfigHelperMixin - derived classes.""" + derived classes. + + """ pass -class ConfigHelperMixin: - """Allow comfortable retrieving of config values pertaining to a section. - If a class inherits from this cls:`ConfigHelperMixin`, it needs - to provide 2 functions: meth:`getconfig` (returning a - ConfigParser object) and meth:`getsection` (returning a string - which represents the section to look up). All calls to getconf* - will then return the configuration values for the ConfigParser - object in the specific section.""" + +class ConfigHelperMixin: + """ + Allow comfortable retrieving of config values pertaining + to a section. + + If a class inherits from cls:`ConfigHelperMixin`, it needs + to provide 2 functions: + - meth:`getconfig` (returning a CustomConfigParser object) + - and meth:`getsection` (returning a string which represents + the section to look up). + All calls to getconf* will then return the configuration values + for the CustomConfigParser object in the specific section. + + """ def _confighelper_runner(self, option, default, defaultfunc, mainfunc, *args): - """Return config value for getsection()""" + """ + Return configuration or default value for option + that contains in section identified by getsection(). + + Arguments: + - option: name of the option to retrieve; + - default: governs which function we will call. + * When CustomConfigDefault is passed, we will call + the mainfunc. + * When any other value is passed, we will call + the defaultfunc and the value of `default` will + be passed as the third argument to this function. + - defaultfunc and mainfunc: processing helpers. + - args: additional trailing arguments that will be passed + to all processing helpers. + + """ lst = [self.getsection(), option] if default == CustomConfigDefault: return mainfunc(*(lst + list(args))) @@ -127,8 +187,32 @@ class ConfigHelperMixin: return defaultfunc(*(lst + list(args))) - def getconf(self, option, - default = CustomConfigDefault): + def getconfig(self): + """ + Returns CustomConfigParser object that we will use + for all our actions. + + Must be overriden in all classes that use this mix-in. + + """ + raise NotImplementedError("ConfigHelperMixin.getconfig() " + "is to be overriden") + + + + def getsection(self): + """ + Returns name of configuration section in which our + class keeps its configuration. + + Must be overriden in all classes that use this mix-in. + + """ + raise NotImplementedError("ConfigHelperMixin.getsection() " + "is to be overriden") + + + def getconf(self, option, default = CustomConfigDefault): return self._confighelper_runner(option, default, self.getconfig().getdefault, self.getconfig().get) @@ -149,7 +233,7 @@ class ConfigHelperMixin: self.getconfig().getfloat) def getconflist(self, option, separator_re, - default = CustomConfigDefault): + default = CustomConfigDefault): return self._confighelper_runner(option, default, self.getconfig().getdefaultlist, self.getconfig().getlist, separator_re) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index f8a2281..eabbd23 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -78,6 +78,7 @@ class Account(CustomConfig.ConfigHelperMixin): def getlocaleval(self): return self.localeval + # Interface from CustomConfig.ConfigHelperMixin def getconfig(self): return self.config @@ -90,6 +91,7 @@ class Account(CustomConfig.ConfigHelperMixin): def getaccountmeta(self): return os.path.join(self.metadatadir, 'Account-' + self.name) + # Interface from CustomConfig.ConfigHelperMixin def getsection(self): return 'Account ' + self.getname() diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 1050721..cc6abdd 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -103,9 +103,11 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getmapdir(self): return self.mapdir + # Interface from CustomConfig.ConfigHelperMixin def getsection(self): return 'Repository ' + self.name + # Interface from CustomConfig.ConfigHelperMixin def getconfig(self): return self.config From 3d88a76959c6c305a5c81e6cc4a9b9780c89ec86 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 10 Oct 2014 16:23:07 +0400 Subject: [PATCH 644/817] Update imaplib2 to 2.37 The only fix is - add missing idle_lock in _handler(). Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ offlineimap/imaplib2.py | 29 +++++++++++++++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 66f5c05..cdd871f 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,9 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Updated bundled imaplib2 to 2.37: + - add missing idle_lock in _handler() + * Added default CA bundle location for OpenBSD (GitHub pull #120) and DragonFlyBSD. diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 14a802b..8921c9d 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.36" +__version__ = "2.37" __release__ = "2" -__revision__ = "36" +__revision__ = "37" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -42,7 +42,8 @@ Threads now set the "daemon" flag (suggested by offlineimap-project) April 2011. Single quoting introduced with the help of Vladimir Marek August 2011. Support for specifying SSL version by Ryan Kavanagh July 2013. Fix for gmail "read 0" error provided by Jim Greenleaf August 2013. -Fix for offlineimap "indexerror: string index out of range" bug provided by Eygene Ryabinkin August 2013.""" +Fix for offlineimap "indexerror: string index out of range" bug provided by Eygene Ryabinkin August 2013. +Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -1663,16 +1664,20 @@ class IMAP4(object): typ, val = self.abort, 'connection terminated' while not self.Terminate: + + self.idle_lock.acquire() + if self.idle_timeout is not None: + timeout = self.idle_timeout - time.time() + if timeout <= 0: + timeout = 1 + if __debug__: + if self.idle_rqb is not None: + self._log(5, 'server IDLING, timeout=%.2f' % timeout) + else: + timeout = resp_timeout + self.idle_lock.release() + try: - if self.idle_timeout is not None: - timeout = self.idle_timeout - time.time() - if timeout <= 0: - timeout = 1 - if __debug__: - if self.idle_rqb is not None: - self._log(5, 'server IDLING, timeout=%.2f' % timeout) - else: - timeout = resp_timeout line = self.inq.get(True, timeout) except Queue.Empty: if self.idle_rqb is None: From 080317ab4fb23e5413c7d7d1e7cf6d54a7e9159a Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 10 Oct 2014 16:13:16 +0400 Subject: [PATCH 645/817] Imaplib2: trade backticks to repr() Backticks are gone in Python 3.x. GitHub issue: https://github.com/OfflineIMAP/offlineimap/issues/121 Signed-off-by: Eygene Ryabinkin --- offlineimap/imaplib2.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 8921c9d..8975145 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -378,7 +378,7 @@ class IMAP4(object): elif self._get_untagged_response('OK'): if __debug__: self._log(1, 'state => NONAUTH') else: - raise self.error('unrecognised server welcome message: %s' % `self.welcome`) + raise self.error('unrecognised server welcome message: %s' % repr(self.welcome)) typ, dat = self.capability() if dat == [None]: @@ -1603,7 +1603,7 @@ class IMAP4(object): tag = rqb.tag self.tagged_commands[tag] = rqb self.commands_lock.release() - if __debug__: self._log(4, '_request_push(%s, %s, %s) = %s' % (tag, name, `kw`, rqb.tag)) + if __debug__: self._log(4, '_request_push(%s, %s, %s) = %s' % (tag, name, repr(kw), rqb.tag)) return rqb @@ -1707,7 +1707,7 @@ class IMAP4(object): self.Terminate = True - if __debug__: self._log(1, 'terminating: %s' % `val`) + if __debug__: self._log(1, 'terminating: %s' % repr(val)) while not self.ouq.empty(): try: @@ -1760,7 +1760,7 @@ class IMAP4(object): timeout = read_poll_timeout try: r = poll.poll(timeout) - if __debug__: self._log(5, 'poll => %s' % `r`) + if __debug__: self._log(5, 'poll => %s' % repr(r)) if not r: continue # Timeout @@ -2480,11 +2480,11 @@ if __name__ == '__main__': run('append', (None, None, None, test_mesg), cb=False) num = run('search', (None, 'ALL'), cb=False)[0].split()[0] dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) - M._mesg('fetch %s => %s' % (num, `dat`)) + M._mesg('fetch %s => %s' % (num, repr(dat))) run('idle', (2,)) run('store', (num, '-FLAGS', '(\Seen)'), cb=False), dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) - M._mesg('fetch %s => %s' % (num, `dat`)) + M._mesg('fetch %s => %s' % (num, repr(dat))) run('uid', ('STORE', num, 'FLAGS', '(\Deleted)')) run('expunge', ()) if idle_intr: From af464c1b567af8e0150034634cfba8ac541fe94d Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 10 Oct 2014 16:23:07 +0400 Subject: [PATCH 646/817] Update imaplib2 to 2.37 The only fix is - add missing idle_lock in _handler(). Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 3 +++ offlineimap/imaplib2.py | 29 +++++++++++++++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 66f5c05..cdd871f 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,9 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Updated bundled imaplib2 to 2.37: + - add missing idle_lock in _handler() + * Added default CA bundle location for OpenBSD (GitHub pull #120) and DragonFlyBSD. diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 14a802b..8921c9d 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.36" +__version__ = "2.37" __release__ = "2" -__revision__ = "36" +__revision__ = "37" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -42,7 +42,8 @@ Threads now set the "daemon" flag (suggested by offlineimap-project) April 2011. Single quoting introduced with the help of Vladimir Marek August 2011. Support for specifying SSL version by Ryan Kavanagh July 2013. Fix for gmail "read 0" error provided by Jim Greenleaf August 2013. -Fix for offlineimap "indexerror: string index out of range" bug provided by Eygene Ryabinkin August 2013.""" +Fix for offlineimap "indexerror: string index out of range" bug provided by Eygene Ryabinkin August 2013. +Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -1663,16 +1664,20 @@ class IMAP4(object): typ, val = self.abort, 'connection terminated' while not self.Terminate: + + self.idle_lock.acquire() + if self.idle_timeout is not None: + timeout = self.idle_timeout - time.time() + if timeout <= 0: + timeout = 1 + if __debug__: + if self.idle_rqb is not None: + self._log(5, 'server IDLING, timeout=%.2f' % timeout) + else: + timeout = resp_timeout + self.idle_lock.release() + try: - if self.idle_timeout is not None: - timeout = self.idle_timeout - time.time() - if timeout <= 0: - timeout = 1 - if __debug__: - if self.idle_rqb is not None: - self._log(5, 'server IDLING, timeout=%.2f' % timeout) - else: - timeout = resp_timeout line = self.inq.get(True, timeout) except Queue.Empty: if self.idle_rqb is None: From dbf683ed388df3211828a2c37e2bf3f400f98eff Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Fri, 10 Oct 2014 16:13:16 +0400 Subject: [PATCH 647/817] Imaplib2: trade backticks to repr() Backticks are gone in Python 3.x. GitHub issue: https://github.com/OfflineIMAP/offlineimap/issues/121 Signed-off-by: Eygene Ryabinkin --- offlineimap/imaplib2.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 8921c9d..8975145 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -378,7 +378,7 @@ class IMAP4(object): elif self._get_untagged_response('OK'): if __debug__: self._log(1, 'state => NONAUTH') else: - raise self.error('unrecognised server welcome message: %s' % `self.welcome`) + raise self.error('unrecognised server welcome message: %s' % repr(self.welcome)) typ, dat = self.capability() if dat == [None]: @@ -1603,7 +1603,7 @@ class IMAP4(object): tag = rqb.tag self.tagged_commands[tag] = rqb self.commands_lock.release() - if __debug__: self._log(4, '_request_push(%s, %s, %s) = %s' % (tag, name, `kw`, rqb.tag)) + if __debug__: self._log(4, '_request_push(%s, %s, %s) = %s' % (tag, name, repr(kw), rqb.tag)) return rqb @@ -1707,7 +1707,7 @@ class IMAP4(object): self.Terminate = True - if __debug__: self._log(1, 'terminating: %s' % `val`) + if __debug__: self._log(1, 'terminating: %s' % repr(val)) while not self.ouq.empty(): try: @@ -1760,7 +1760,7 @@ class IMAP4(object): timeout = read_poll_timeout try: r = poll.poll(timeout) - if __debug__: self._log(5, 'poll => %s' % `r`) + if __debug__: self._log(5, 'poll => %s' % repr(r)) if not r: continue # Timeout @@ -2480,11 +2480,11 @@ if __name__ == '__main__': run('append', (None, None, None, test_mesg), cb=False) num = run('search', (None, 'ALL'), cb=False)[0].split()[0] dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) - M._mesg('fetch %s => %s' % (num, `dat`)) + M._mesg('fetch %s => %s' % (num, repr(dat))) run('idle', (2,)) run('store', (num, '-FLAGS', '(\Seen)'), cb=False), dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) - M._mesg('fetch %s => %s' % (num, `dat`)) + M._mesg('fetch %s => %s' % (num, repr(dat))) run('uid', ('STORE', num, 'FLAGS', '(\Deleted)')) run('expunge', ()) if idle_intr: From 8f3c22e22798bf35abb99225cf2b2f93c1396874 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 2 Nov 2014 11:14:42 +0300 Subject: [PATCH 648/817] Introduce CustomConfig method that applies set of transforms It is a bit cleaner than making chains of calls like {{{ value = os.path.expanduser(value) value = os.path.abspath(value) }}} since we do see all transformations to be applied in a single iterable and have no repeated code like in the above example. Signed-off-by: Eygene Ryabinkin --- offlineimap/CustomConfig.py | 99 ++++++++++++++++++++++++++++++- offlineimap/repository/IMAP.py | 6 +- offlineimap/repository/Maildir.py | 3 +- 3 files changed, 101 insertions(+), 7 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 9fef69a..47e82f3 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -94,14 +94,17 @@ class CustomConfigParser(SafeConfigParser): return default def getmetadatadir(self): - metadatadir = os.path.expanduser(self.getdefault("general", "metadata", "~/.offlineimap")) + xforms = [os.path.expanduser] + d = self.getdefault("general", "metadata", "~/.offlineimap") + metadatadir = self.apply_xforms(d, xforms) if not os.path.exists(metadatadir): os.mkdir(metadatadir, 0o700) return metadatadir def getlocaleval(self): + xforms = [os.path.expanduser] if self.has_option("general", "pythonfile"): - path = os.path.expanduser(self.get("general", "pythonfile")) + path = self.apply_xforms(self.get("general", "pythonfile"), xforms) else: path = None return LocalEval(path) @@ -132,6 +135,26 @@ class CustomConfigParser(SafeConfigParser): self.set(section, option, value) + def apply_xforms(self, string, transforms): + """ + Applies set of transformations to a string. + + Arguments: + - string: source string; if None, then no processing will + take place. + - transforms: iterable that returns transformation function + on each turn. + + Returns transformed string. + + """ + if string == None: + return None + for f in transforms: + string = f(string) + return string + + def CustomConfigDefault(): """ @@ -161,9 +184,10 @@ class ConfigHelperMixin: """ + def _confighelper_runner(self, option, default, defaultfunc, mainfunc, *args): """ - Return configuration or default value for option + Returns configuration or default value for option that contains in section identified by getsection(). Arguments: @@ -213,27 +237,96 @@ class ConfigHelperMixin: def getconf(self, option, default = CustomConfigDefault): + """ + Retrieves string from the configuration. + + Arguments: + - option: option name whose value is to be retrieved; + - default: default return value if no such option + exists. + + """ return self._confighelper_runner(option, default, self.getconfig().getdefault, self.getconfig().get) + + def getconf_xform(self, option, xforms, default = CustomConfigDefault): + """ + Retrieves string from the configuration transforming the result. + + Arguments: + - option: option name whose value is to be retrieved; + - xforms: iterable that returns transform functions + to be applied to the value of the option, + both retrieved and default one; + - default: default value for string if no such option + exists. + + """ + value = self.getconf(option, default) + return self.getconfig().apply_xforms(value, xforms) + + def getconfboolean(self, option, default = CustomConfigDefault): + """ + Retrieves boolean value from the configuration. + + Arguments: + - option: option name whose value is to be retrieved; + - default: default return value if no such option + exists. + + """ return self._confighelper_runner(option, default, self.getconfig().getdefaultboolean, self.getconfig().getboolean) + def getconfint(self, option, default = CustomConfigDefault): + """ + Retrieves integer value from the configuration. + + Arguments: + - option: option name whose value is to be retrieved; + - default: default return value if no such option + exists. + + """ return self._confighelper_runner(option, default, self.getconfig().getdefaultint, self.getconfig().getint) + def getconffloat(self, option, default = CustomConfigDefault): + """ + Retrieves floating-point value from the configuration. + + Arguments: + - option: option name whose value is to be retrieved; + - default: default return value if no such option + exists. + + """ return self._confighelper_runner(option, default, self.getconfig().getdefaultfloat, self.getconfig().getfloat) + def getconflist(self, option, separator_re, default = CustomConfigDefault): + """ + Retrieves strings from the configuration and splits it + into the list of strings. + + Arguments: + - option: option name whose value is to be retrieved; + - separator_re: regular expression for separator + to be used for split operation; + - default: default return value if no such option + exists. + + """ return self._confighelper_runner(option, default, self.getconfig().getdefaultlist, self.getconfig().getlist, separator_re) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index c98419d..2560b7d 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -201,11 +201,11 @@ class IMAPRepository(BaseRepository): def getsslcacertfile(self): """Return the absolute path of the CA certfile to use, if any""" - cacertfile = self.getconf('sslcacertfile', get_os_sslcertfile()) + xforms = [os.path.expanduser, os.path.abspath] + cacertfile = self.getconf_xform('sslcacertfile', xforms, + get_os_sslcertfile()) if cacertfile is None: return None - cacertfile = os.path.expanduser(cacertfile) - cacertfile = os.path.abspath(cacertfile) if not os.path.isfile(cacertfile): raise SyntaxWarning("CA certfile for repository '%s' could " "not be found. No such file: '%s'" \ diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 0e5d65e..dae811d 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -61,7 +61,8 @@ class MaildirRepository(BaseRepository): os.utime(cur_dir, (cur_atime, os.path.getmtime(cur_dir))) def getlocalroot(self): - return os.path.expanduser(self.getconf('localfolders')) + xforms = [os.path.expanduser] + return self.getconf_xform('localfolders', xforms) def debug(self, msg): self.ui.debug('maildir', msg) From e51ed80ecc1830ca3184761ccbfa63cf22cc2d42 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 2 Nov 2014 11:49:26 +0300 Subject: [PATCH 649/817] Enable tilde and environment variable expansion on some items Expand environment variables in the following configuration items: - general.pythonfile; - general.metadata; - mbnames.filename; - Repository.localfolders. - Repository.sslcacertfile. Make tilde and environment variable expansion in the following configuration items: - Repository.sslclientcert; - Repository.sslclientkey. GitHub pull request: https://github.com/OfflineIMAP/offlineimap/pull/113 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 12 ++++++++++++ offlineimap.conf | 22 ++++++++++++++++++++++ offlineimap/CustomConfig.py | 4 ++-- offlineimap/mbnames.py | 4 +++- offlineimap/repository/IMAP.py | 8 +++++--- 5 files changed, 44 insertions(+), 6 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index cdd871f..321bf87 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,18 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Expand environment variables in the following + configuration items: + - general.pythonfile; + - general.metadata; + - mbnames.filename; + - Repository.localfolders. + - Repository.sslcacertfile. + Make tilde and environment variable expansion in the following + configuration items: + - Repository.sslclientcert; + - Repository.sslclientkey. + * Updated bundled imaplib2 to 2.37: - add missing idle_lock in _handler() diff --git a/offlineimap.conf b/offlineimap.conf index c7cda28..94094a6 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -23,6 +23,12 @@ # NOTE2: This implies that any '%' needs to be encoded as '%%' +# NOTE3: Any variables that are subject to the environment variables +# ($NAME) and tilde (~username/~) expansions will receive tilde +# expansion first and only after this environment variables will be +# expanded in the resulting string. This behaviour is intentional +# as it coincides with typical shell expansion strategy. + ################################################## # General definitions ################################################## @@ -31,6 +37,8 @@ # This specifies where offlineimap is to store its metadata. # This directory will be created if it does not already exist. +# +# Tilde and environment variable expansions will be performed. #metadata = ~/.offlineimap @@ -89,6 +97,8 @@ accounts = Test # source file and call them from this config file. You can find # an example of this in the manual. # +# Tilde and environment variable expansions will be performed. +# # pythonfile = ~/.offlineimap.py # @@ -135,6 +145,9 @@ accounts = Test # - foldername: the name of the folder; # - localfolders: path to the local directory hosting all Maildir # folders for the account. +# +# Tilde and environment variable expansions will be performed +# for "filename" knob. enabled = no filename = ~/Mutt/muttrc.mailboxes @@ -375,9 +388,15 @@ remotehost = examplehost ssl = yes # SSL Client certificate (optional) +# +# Tilde and environment variable expansions will be performed. + # sslclientcert = /path/to/file.crt # SSL Client key (optional) +# +# Tilde and environment variable expansions will be performed. + # sslclientkey = /path/to/file.key # SSL CA Cert(s) to verify the server cert against (optional). @@ -385,6 +404,9 @@ ssl = yes # specified, the CA Cert(s) need to verify the Server cert AND # match the hostname (* wildcard allowed on the left hand side) # The certificate should be in PEM format. +# +# Tilde and environment variable expansions will be performed. + # sslcacertfile = /path/to/cacertfile.crt # If you connect via SSL/TLS (ssl=true) and you have no CA certificate diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 47e82f3..1f4bcd0 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -94,7 +94,7 @@ class CustomConfigParser(SafeConfigParser): return default def getmetadatadir(self): - xforms = [os.path.expanduser] + xforms = [os.path.expanduser, os.path.expandvars] d = self.getdefault("general", "metadata", "~/.offlineimap") metadatadir = self.apply_xforms(d, xforms) if not os.path.exists(metadatadir): @@ -102,7 +102,7 @@ class CustomConfigParser(SafeConfigParser): return metadatadir def getlocaleval(self): - xforms = [os.path.expanduser] + xforms = [os.path.expanduser, os.path.expandvars] if self.has_option("general", "pythonfile"): path = self.apply_xforms(self.get("general", "pythonfile"), xforms) else: diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 2d9ab5e..436a7e6 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -49,12 +49,14 @@ def write(): def __genmbnames(): """Takes a configparser object and a boxlist, which is a list of hashes containing 'accountname' and 'foldername' keys.""" + xforms = [os.path.expanduser, os.path.expandvars] mblock.acquire() try: localeval = config.getlocaleval() if not config.getdefaultboolean("mbnames", "enabled", 0): return - file = open(os.path.expanduser(config.get("mbnames", "filename")), "wt") + path = config.apply_xform(config.get("mbnames", "filename"), xforms) + file = open(path, "wt") file.write(localeval.eval(config.get("mbnames", "header"))) folderfilter = lambda accountname, foldername: 1 if config.has_option("mbnames", "folderfilter"): diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 2560b7d..32cf3ac 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -194,14 +194,16 @@ class IMAPRepository(BaseRepository): return self.getconfboolean('ssl', 0) def getsslclientcert(self): - return self.getconf('sslclientcert', None) + xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] + return self.getconf_xform('sslclientcert', xforms, None) def getsslclientkey(self): - return self.getconf('sslclientkey', None) + xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] + return self.getconf_xform('sslclientkey', xforms, None) def getsslcacertfile(self): """Return the absolute path of the CA certfile to use, if any""" - xforms = [os.path.expanduser, os.path.abspath] + xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] cacertfile = self.getconf_xform('sslcacertfile', xforms, get_os_sslcertfile()) if cacertfile is None: From fc4c7549d6d03415edc2537b29c83fd6baea1bc6 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:03:15 +0100 Subject: [PATCH 650/817] Do not ignore gmail labels if header appears multiple times There should be just one header storing gmail labels, but due to a bug, multiple X-Keywords (or equivalent) headers may be found on the local messages. Now we, when extracting the labels from a message, we read all label headers, instead of just the first one. This has the consequence that some old labels stored locally in a second X-Keywords (or third...) header, which effectively was rendered invisible to offlineimap until now, may pop back up again and be pushed to gmail. No labels will be removed by the changes in this commit, though. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Base.py | 25 +++++++++++++++++++++++-- offlineimap/folder/Gmail.py | 5 +++-- offlineimap/folder/GmailMaildir.py | 19 +++++++++++-------- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index b56cd1b..4b30d8c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -512,8 +512,8 @@ class BaseFolder(object): def getmessageheader(self, content, name): """ - Searches for the given header and returns its value. - Header name is case-insensitive. + Searches for the first occurence of the given header and returns + its value. Header name is case-insensitive. Arguments: - contents: message itself @@ -535,6 +535,27 @@ class BaseFolder(object): return None + def getmessageheaderlist(self, content, name): + """ + Searches for the given header and returns a list of values for + that header. + + Arguments: + - contents: message itself + - name: name of the header to be searched + + Returns: list of header values or emptylist if no such header was found + + """ + self.ui.debug('', 'getmessageheaderlist: called to get %s' % name) + eoh = self.__find_eoh(content) + self.ui.debug('', 'getmessageheaderlist: eoh = %d' % eoh) + headers = content[0:eoh] + self.ui.debug('', 'getmessageheaderlist: headers = %s' % repr(headers)) + + return re.findall('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE) + + def deletemessageheaders(self, content, header_list): """ Deletes headers in the given list from the message content. diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index e3ef92a..2a8afd2 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -189,8 +189,9 @@ class GmailFolder(IMAPFolder): if not self.synclabels: return super(GmailFolder, self).savemessage(uid, content, flags, rtime) - labels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + labels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) self.savemessagelabels(ret, labels) diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 2cab213..7a67c87 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -87,9 +87,10 @@ class GmailMaildirFolder(MaildirFolder): content = file.read() file.close() - self.messagelist[uid]['labels'] = \ - imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + self.messagelist[uid]['labels'] = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + self.messagelist[uid]['labels'].update( + imaputil.labels_from_header(self.labelsheader, hstr)) self.messagelist[uid]['labels_cached'] = True return self.messagelist[uid]['labels'] @@ -111,8 +112,10 @@ class GmailMaildirFolder(MaildirFolder): if not self.synclabels: return super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) - labels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + labels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) + ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) # Update the mtime and labels @@ -135,9 +138,9 @@ class GmailMaildirFolder(MaildirFolder): content = file.read() file.close() - oldlabels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) - + oldlabels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + oldlabels.update(imaputil.labels_from_header(self.labelsheader, hstr)) labels = labels - ignorelabels ignoredlabels = oldlabels & ignorelabels From 2a5ef8c2ef2d6d64ef9ab8c6216ef68d6f07937c Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:16:48 +0100 Subject: [PATCH 651/817] Delete gmail labels header before adding a new one This fixes a bug in which a message ended up with multiple gmail labels header (X-Keywords or so). Fix fix it by removing all labels headers before adding the updated one. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Gmail.py | 4 ++++ offlineimap/folder/GmailMaildir.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 2a8afd2..69f0075 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -93,6 +93,10 @@ class GmailFolder(IMAPFolder): labels = set() labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) + + # First remove old label headers that may be in the message content retrieved + # from gmail Then add a labels header with current gmail labels. + body = self.deletemessageheaders(body, self.labelsheader) body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) if len(body)>200: diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 7a67c87..3b127d2 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -153,7 +153,11 @@ class GmailMaildirFolder(MaildirFolder): # Change labels into content labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels | ignoredlabels)) + + # First remove old labels header, and then add the new one + content = self.deletemessageheaders(content, self.labelsheader) content = self.addmessageheader(content, '\n', self.labelsheader, labels_str) + rtime = self.messagelist[uid].get('rtime', None) # write file with new labels to a unique file in tmp From e70607e3e33701cc3b7fcc4cc952f57097a0ac12 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:18:35 +0100 Subject: [PATCH 652/817] Warn about a tricky piece of code in addmessageheader As requested by Nicholas Sebrecht. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 4b30d8c..106c390 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -401,6 +401,9 @@ class BaseFolder(object): """ Adds new header to the provided message. + WARNING: This function is a bit tricky, and modifying it in the wrong way, + may easily lead to data-loss. + Arguments: - content: message content, headers and body as a single string - linebreak: string that carries line ending From 15e8e089133741a7e2e74c2cb223fa137eb63781 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 26 Nov 2014 23:17:43 +0300 Subject: [PATCH 653/817] Properly generate package via "sdist" Include tests, configuration examples and other stuff we usually include to the tarballs. GitHub issue: https://github.com/OfflineIMAP/offlineimap/issues/137 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ MANIFEST.in | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 MANIFEST.in diff --git a/Changelog.rst b/Changelog.rst index 321bf87..9a98d29 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,8 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Properly generate tarball from "sdist" command (GitHub #137) + * Expand environment variables in the following configuration items: - general.pythonfile; diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..3a403a3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,14 @@ +global-exclude .gitignore .git *.bak *.orig *.rej +include setup.py +include COPYING +include Changelog* +include MAINTAINERS +include MANIFEST.in +include Makefile +include README.md +include offlineimap.conf* +include offlineimap.py +recursive-include offlineimap *.py +recursive-include bin * +recursive-include docs * +recursive-include test * From 0521aa270653cb0e42d27c01d7f8723ec24f336c Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 27 Nov 2014 00:56:38 -0800 Subject: [PATCH 654/817] Fix typo in apply_xforms invocation Fixes a bug introduced in e51ed80ecc1830ca3184761ccbfa63cf22cc2d42 since apply_xform (without the 's') doesn't exist. Signed-off-by: Eygene Ryabinkin --- offlineimap/mbnames.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 436a7e6..176a760 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -55,7 +55,7 @@ def __genmbnames(): localeval = config.getlocaleval() if not config.getdefaultboolean("mbnames", "enabled", 0): return - path = config.apply_xform(config.get("mbnames", "filename"), xforms) + path = config.apply_xforms(config.get("mbnames", "filename"), xforms) file = open(path, "wt") file.write(localeval.eval(config.get("mbnames", "header"))) folderfilter = lambda accountname, foldername: 1 From 54cff7f7860914b2c2d7cb42129acb3e67cf3449 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:23:35 +0100 Subject: [PATCH 655/817] imaplibutil.py: remove unused imports Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 2869623..05aaf7d 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -17,9 +17,6 @@ import os import fcntl -import re -import socket -import ssl import time import subprocess import threading @@ -27,7 +24,7 @@ from hashlib import sha1 from offlineimap.ui import getglobalui from offlineimap import OfflineImapError -from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, IMAP4_PORT, InternalDate, Mon2num +from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, InternalDate, Mon2num class UsefulIMAPMixIn(object): From de5f22a23af72273107a7ea2ada8003eb891cb7d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:18:45 +0100 Subject: [PATCH 656/817] CustomConfig.py: remove unused imports Signed-off-by: Nicolas Sebrecht --- offlineimap/CustomConfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 1f4bcd0..61bf639 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -15,9 +15,9 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA try: - from ConfigParser import SafeConfigParser, Error, NoOptionError + from ConfigParser import SafeConfigParser, Error except ImportError: #python3 - from configparser import SafeConfigParser, Error, NoOptionError + from configparser import SafeConfigParser, Error from offlineimap.localeval import LocalEval import os import re From 2785e779e2820edb4deae570e9f713d0545c89ea Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:19:16 +0100 Subject: [PATCH 657/817] init.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 5bb0438..d9425d5 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -26,7 +26,6 @@ from optparse import OptionParser import offlineimap from offlineimap import accounts, threadutil, syncmaster from offlineimap import globals -from offlineimap.error import OfflineImapError from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser from offlineimap.utils import stacktrace From 62afde18250db2f956d8f37c7818f6a9d5a2ef4c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:19:39 +0100 Subject: [PATCH 658/817] repository/Base.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index cc6abdd..d7a6866 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -17,7 +17,6 @@ import re import os.path -import traceback from sys import exc_info from offlineimap import CustomConfig from offlineimap.ui import getglobalui From 7f1419a40ac7cead70b71fecc5ab855f785f1190 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:20:13 +0100 Subject: [PATCH 659/817] repository/GmailMaildir.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/GmailMaildir.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/repository/GmailMaildir.py b/offlineimap/repository/GmailMaildir.py index 9072b7c..b790c73 100644 --- a/offlineimap/repository/GmailMaildir.py +++ b/offlineimap/repository/GmailMaildir.py @@ -18,7 +18,6 @@ from offlineimap.repository.Maildir import MaildirRepository from offlineimap.folder.GmailMaildir import GmailMaildirFolder -from offlineimap.error import OfflineImapError class GmailMaildirRepository(MaildirRepository): def __init__(self, reposname, account): From 24a4ab3e167ec2315b0c918f746c7ff64459eb34 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:20:43 +0100 Subject: [PATCH 660/817] repository/LocalStatus.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/LocalStatus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 0375d0d..ba29d37 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -16,11 +16,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os + from offlineimap.folder.LocalStatus import LocalStatusFolder from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder from offlineimap.repository.Base import BaseRepository -import os -import re class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): From 7b453efcce7be684d687b094744ce8cc910ba9fd Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:21:13 +0100 Subject: [PATCH 661/817] ui/Curses.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Curses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 4150066..6cc1855 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -20,7 +20,6 @@ from collections import deque import time import sys import os -import signal import curses import logging from offlineimap.ui.UIBase import UIBase From e613f6992dfefa0881136a91ef289da4fc9e48a3 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:21:44 +0100 Subject: [PATCH 662/817] ui/UIBase.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index dace655..1013a98 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -19,7 +19,6 @@ import logging import re import time import sys -import os import traceback import threading try: From 4589cfeff2fad8db016f54ba05e57ef2fe0aa51f Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 23 Dec 2014 10:12:23 +0100 Subject: [PATCH 663/817] localeval: comment on security issues Minor syntax fixes. Signed-off-by: Nicolas Sebrecht --- offlineimap/localeval.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/offlineimap/localeval.py b/offlineimap/localeval.py index 22014e6..e7d656f 100644 --- a/offlineimap/localeval.py +++ b/offlineimap/localeval.py @@ -1,7 +1,6 @@ """Eval python code with global namespace of a python source file.""" -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2014 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,18 +23,24 @@ except: pass class LocalEval: + """Here is a powerfull but very dangerous option, of course. + + Assume source file to be ASCII encoded.""" + def __init__(self, path=None): - self.namespace={} + self.namespace = {} if path is not None: - file=open(path, 'r') - module=imp.load_module( + # FIXME: limit opening files owned by current user with rights set + # to fixed mode 644. + file = open(path, 'r') + module = imp.load_module( '', file, path, ('', 'r', imp.PY_SOURCE)) for attr in dir(module): - self.namespace[attr]=getattr(module, attr) + self.namespace[attr] = getattr(module, attr) def eval(self, text, namespace=None): names = {} From 532278b4dd01fc487ff3e5f549ead1b5a3de3fac Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 30 Dec 2014 01:04:00 +0100 Subject: [PATCH 664/817] docs: remove obsolete comment about SubmittingPatches.rst Signed-off-by: Nicolas Sebrecht --- docs/doc-src/API.rst | 7 ++-- docs/doc-src/advanced_config.rst | 6 ++- docs/doc-src/nametrans.rst | 65 ++++++++++++++++++++++++-------- docs/doc-src/ui.rst | 5 ++- 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index 1eb9adb..d47fcf4 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -15,10 +15,9 @@ OfflineImap can be imported as:: from offlineimap import OfflineImap -The file ``SubmittingPatches.rst`` in the source distribution documents a -number of resources and conventions you may find useful. It will eventually -be merged into the main documentation. -.. TODO: merge SubmittingPatches.rst to the main documentation +The file ``HACKING.rst`` in the source distribution documents a +number of resources and conventions you may find useful. + :mod:`offlineimap` -- The OfflineImap module ============================================= diff --git a/docs/doc-src/advanced_config.rst b/docs/doc-src/advanced_config.rst index 56e6f18..49f2ea4 100644 --- a/docs/doc-src/advanced_config.rst +++ b/docs/doc-src/advanced_config.rst @@ -1,9 +1,11 @@ Message filtering ================= -There are two ways to selectively filter messages out of a folder, using `maxsize` and `maxage`. Setting each option will basically ignore all messages that are on the server by pretending they don't exist. +There are two ways to selectively filter messages out of a folder, using +`maxsize` and `maxage`. Setting each option will basically ignore all messages +that are on the server by pretending they don't exist. -:todo: explain them and give tipps on how to use and not use them. Use cases! +:todo: explain them and give tips on how to use and not use them. Use cases! maxage ------ diff --git a/docs/doc-src/nametrans.rst b/docs/doc-src/nametrans.rst index f6b9924..030ead2 100644 --- a/docs/doc-src/nametrans.rst +++ b/docs/doc-src/nametrans.rst @@ -13,7 +13,10 @@ safely skip this section. folderfilter ------------ -If you do not want to synchronize all your folders, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. +If you do not want to synchronize all your folders, you can specify a +`folderfilter`_ function that determines which folders to include in a sync and +which to exclude. Typically, you would set a folderfilter option on the remote +repository only, and it would be a lambda or any other python function. The only parameter to that function is the folder name. If the filter function returns True, the folder will be synced, if it returns False, @@ -45,12 +48,18 @@ at the end when required by Python syntax) For instance:: ['INBOX', 'Sent Mail', 'Deleted Items', 'Received'] -Usually it suffices to put a `folderfilter`_ setting in the remote repository section. You might want to put a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. (Even in this case, folder filters on the remote repository will prevent that) +Usually it suffices to put a `folderfilter`_ setting in the remote repository +section. You might want to put a folderfilter option on the local repository if +you want to prevent some folders on the local repository to be created on the +remote one. (Even in this case, folder filters on the remote repository will +prevent that) folderincludes -------------- -You can specify `folderincludes`_ to manually include additional folders to be synced, even if they had been filtered out by a folderfilter setting. `folderincludes`_ should return a Python list. +You can specify `folderincludes`_ to manually include additional folders to be +synced, even if they had been filtered out by a folderfilter setting. +`folderincludes`_ should return a Python list. This can be used to 1) add a folder that was excluded by your folderfilter rule, 2) to include a folder that your server does not specify @@ -84,11 +93,14 @@ nametrans`_ rules on the LOCAL repository. nametrans ---------- -Sometimes, folders need to have different names on the remote and the -local repositories. To achieve this you can specify a folder name -translator. This must be a eval-able Python expression that takes a -foldername arg and returns the new value. We suggest a lambda function, -but it could be any python function really. If you use nametrans rules, you will need to set them both on the remote and the local repository, see `Reverse nametrans`_ just below for details. The following examples are thought to be put in the remote repository section. +Sometimes, folders need to have different names on the remote and the local +repositories. To achieve this you can specify a folder name translator. This +must be a eval-able Python expression that takes a foldername arg and returns +the new value. We suggest a lambda function, but it could be any python +function really. If you use nametrans rules, you will need to set them both on +the remote and the local repository, see `Reverse nametrans`_ just below for +details. The following examples are thought to be put in the remote repository +section. The below will remove "INBOX." from the leading edge of folders (great for Courier IMAP users):: @@ -112,38 +124,59 @@ locally? Try this:: Reverse nametrans +++++++++++++++++ -Since 6.4.0, OfflineImap supports the creation of folders on the remote repository and that complicates things. Previously, only one nametrans setting on the remote repository was needed and that transformed a remote to a local name. However, nametrans transformations are one-way, and OfflineImap has no way using those rules on the remote repository to back local names to remote names. +Since 6.4.0, OfflineImap supports the creation of folders on the remote +repository and that complicates things. Previously, only one nametrans setting +on the remote repository was needed and that transformed a remote to a local +name. However, nametrans transformations are one-way, and OfflineImap has no way +using those rules on the remote repository to back local names to remote names. -Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts off any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. +Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts off +any existing INBOX prefix. Now, if we parse a list of local folders, finding +e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We +have no way of knowing. This is why **every nametrans setting on a remote +repository requires an equivalent nametrans rule on the local repository that +reverses the transformation**. Take the above examples. If your remote nametrans setting was:: nametrans = lambda folder: re.sub('^INBOX\.', '', folder) -then you will want to have this in your local repository, prepending "INBOX" to any local folder name:: +then you will want to have this in your local repository, prepending "INBOX" to +any local folder name:: nametrans = lambda folder: 'INBOX.' + folder -Failure to set the local nametrans rule will lead to weird-looking error messages of -for instance- this type:: +Failure to set the local nametrans rule will lead to weird-looking error +messages of -for instance- this type:: ERROR: Creating folder moo.foo on repository remote Folder 'moo.foo'[remote] could not be created. Server responded: ('NO', ['Unknown namespace.']) -(This indicates that you attempted to create a folder "Sent" when all remote folders needed to be under the prefix of "INBOX."). +(This indicates that you attempted to create a folder "Sent" when all remote +folders needed to be under the prefix of "INBOX."). OfflineImap will make some sanity checks if it needs to create a new folder on the remote side and a back-and-forth nametrans-lation does not yield the original foldername (as that could potentially lead to infinite folder creation cycles). -You can probably already see now that creating nametrans rules can be a pretty daunting and complex endeavour. Check out the Use cases in the manual. If you have some interesting use cases that we can present as examples here, please let us know. +You can probably already see now that creating nametrans rules can be a pretty +daunting and complex endeavour. Check out the Use cases in the manual. If you +have some interesting use cases that we can present as examples here, please let +us know. Debugging folderfilter and nametrans ------------------------------------ -Given the complexity of the functions and regexes involved, it is easy to misconfigure things. One way to test your configuration without danger to corrupt anything or to create unwanted folders is to invoke offlineimap with the `--info` option. +Given the complexity of the functions and regexes involved, it is easy to +misconfigure things. One way to test your configuration without danger to +corrupt anything or to create unwanted folders is to invoke offlineimap with the +`--info` option. -It will output a list of folders and their transformations on the screen (save them to a file with -l info.log), and will help you to tweak your rules as well as to understand your configuration. It also provides good output for bug reporting. +It will output a list of folders and their transformations on the screen (save +them to a file with -l info.log), and will help you to tweak your rules as well +as to understand your configuration. It also provides good output for bug +reporting. FAQ on nametrans ---------------- diff --git a/docs/doc-src/ui.rst b/docs/doc-src/ui.rst index 27be221..2931010 100644 --- a/docs/doc-src/ui.rst +++ b/docs/doc-src/ui.rst @@ -3,7 +3,10 @@ .. currentmodule:: offlineimap.ui -OfflineImap has various ui systems, that can be selected. They offer various functionalities. They must implement all functions that the :class:`offlineimap.ui.UIBase` offers. Early on, the ui must be set using :meth:`getglobalui` +OfflineImap has various ui systems, that can be selected. They offer various +functionalities. They must implement all functions that the +:class:`offlineimap.ui.UIBase` offers. Early on, the ui must be set using +:meth:`getglobalui` .. automethod:: offlineimap.ui.setglobalui .. automethod:: offlineimap.ui.getglobalui From a35c432671143cff2a6d5d1006aa4d351fe6c200 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 30 Dec 2014 01:05:20 +0100 Subject: [PATCH 665/817] utils/const.py: fix ident Signed-off-by: Nicolas Sebrecht --- offlineimap/utils/const.py | 47 ++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/offlineimap/utils/const.py b/offlineimap/utils/const.py index a62b6a6..f4584bc 100644 --- a/offlineimap/utils/const.py +++ b/offlineimap/utils/const.py @@ -1,40 +1,37 @@ -# Copyright 2013 Eygene A. Ryabinkin. +# Copyright (C) 2013-2014 Eygene A. Ryabinkin and contributors # # Collection of classes that implement const-like behaviour # for various objects. import copy -class ConstProxy (object): - """ - Implements read-only access to a given object - that can be attached to each instance only once. +class ConstProxy(object): + """Implements read-only access to a given object + that can be attached to each instance only once.""" - """ - - def __init__ (self): - self.__dict__['__source'] = None + def __init__(self): + self.__dict__['__source'] = None - def __getattr__ (self, name): - src = self.__dict__['__source'] - if src == None: - raise ValueError ("using non-initialized ConstProxy() object") - return copy.deepcopy (getattr (src, name)) + def __getattr__(self, name): + src = self.__dict__['__source'] + if src == None: + raise ValueError("using non-initialized ConstProxy() object") + return copy.deepcopy(getattr(src, name)) - def __setattr__ (self, name, value): - raise AttributeError ("tried to set '%s' to '%s' for constant object" % \ - (name, value)) + def __setattr__(self, name, value): + raise AttributeError("tried to set '%s' to '%s' for constant object"% \ + (name, value)) - def __delattr__ (self, name): - raise RuntimeError ("tried to delete field '%s' from constant object" % \ - (name)) + def __delattr__(self, name): + raise RuntimeError("tried to delete field '%s' from constant object"% \ + (name)) - def set_source (self, source): - """ Sets source object for this instance. """ - if (self.__dict__['__source'] != None): - raise ValueError ("source object is already set") - self.__dict__['__source'] = source + def set_source(self, source): + """ Sets source object for this instance. """ + if (self.__dict__['__source'] != None): + raise ValueError("source object is already set") + self.__dict__['__source'] = source From 11a28fb0cb2bddf6e8f6d322f2d6f73b543f4933 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 21:55:24 +0100 Subject: [PATCH 666/817] ui/UIBase: folderlist(): avoid built-in list() redefinition Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 1013a98..0edfa18 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -230,9 +230,9 @@ class UIBase(object): raise NotImplementedError("Prompting for a password is not supported"\ " in this UI backend.") - def folderlist(self, list): - return ', '.join(["%s[%s]" % \ - (self.getnicename(x), x.getname()) for x in list]) + def folderlist(self, folder_list): + return ', '.join(["%s[%s]"% \ + (self.getnicename(x), x.getname()) for x in folder_list]) ################################################## WARNINGS def msgtoreadonly(self, destfolder, uid, content, flags): From 61021260cb6676447979b5f6247295409b023418 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 21:41:11 +0100 Subject: [PATCH 667/817] more consistent style Signed-off-by: Nicolas Sebrecht --- offlineimap/CustomConfig.py | 147 +++++++++++--------------- offlineimap/accounts.py | 16 +-- offlineimap/error.py | 2 + offlineimap/folder/Base.py | 14 ++- offlineimap/folder/LocalStatus.py | 29 ++--- offlineimap/folder/Maildir.py | 25 +++-- offlineimap/folder/UIDMaps.py | 8 +- offlineimap/imaplibutil.py | 42 ++++---- offlineimap/imaputil.py | 45 ++++---- offlineimap/init.py | 24 ++--- offlineimap/mbnames.py | 5 +- offlineimap/repository/Base.py | 9 +- offlineimap/repository/IMAP.py | 35 +++--- offlineimap/repository/LocalStatus.py | 7 +- offlineimap/repository/Maildir.py | 10 +- offlineimap/ui/Curses.py | 8 +- offlineimap/ui/Machine.py | 2 +- offlineimap/ui/TTY.py | 10 +- offlineimap/ui/UIBase.py | 81 ++++++++------ offlineimap/ui/debuglock.py | 3 +- 20 files changed, 277 insertions(+), 245 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 61bf639..ae27d41 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -1,4 +1,4 @@ -# Copyright (C) 2003-2012 John Goerzen & contributors +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,21 +14,20 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os +import re + try: from ConfigParser import SafeConfigParser, Error except ImportError: #python3 from configparser import SafeConfigParser, Error from offlineimap.localeval import LocalEval -import os -import re class CustomConfigParser(SafeConfigParser): def getdefault(self, section, option, default, *args, **kwargs): - """ - Same as config.get, but returns the value of `default` - if there is no such option specified. - - """ + """Same as config.get, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): return self.get(*(section, option) + args, **kwargs) else: @@ -36,45 +35,37 @@ class CustomConfigParser(SafeConfigParser): def getdefaultint(self, section, option, default, *args, **kwargs): - """ - Same as config.getint, but returns the value of `default` - if there is no such option specified. - - """ + """Same as config.getint, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): - return self.getint (*(section, option) + args, **kwargs) + return self.getint(*(section, option) + args, **kwargs) else: return default def getdefaultfloat(self, section, option, default, *args, **kwargs): - """ - Same as config.getfloat, but returns the value of `default` - if there is no such option specified. - - """ + """Same as config.getfloat, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): return self.getfloat(*(section, option) + args, **kwargs) else: return default def getdefaultboolean(self, section, option, default, *args, **kwargs): - """ - Same as config.getboolean, but returns the value of `default` - if there is no such option specified. - - """ + """Same as config.getboolean, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): return self.getboolean(*(section, option) + args, **kwargs) else: return default def getlist(self, section, option, separator_re): - """ - Parses option as the list of values separated - by the given regexp. + """Parses option as the list of values separated + by the given regexp.""" - """ try: val = self.get(section, option).strip() return re.split(separator_re, val) @@ -83,11 +74,9 @@ class CustomConfigParser(SafeConfigParser): (separator_re, e)) def getdefaultlist(self, section, option, default, separator_re): - """ - Same as getlist, but returns the value of `default` - if there is no such option specified. - - """ + """Same as getlist, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): return self.getlist(*(section, option, separator_re)) else: @@ -104,40 +93,48 @@ class CustomConfigParser(SafeConfigParser): def getlocaleval(self): xforms = [os.path.expanduser, os.path.expandvars] if self.has_option("general", "pythonfile"): - path = self.apply_xforms(self.get("general", "pythonfile"), xforms) + if globals.options.use_unicode: + path = uni.fsEncode(self.get("general", "pythonfile"), + exception_msg="cannot convert character for pythonfile") + else: + path = self.get("general", "pythonfile") + path = self.apply_xforms(path, xforms) else: path = None return LocalEval(path) def getsectionlist(self, key): - """ - Returns a list of sections that start with key + " ". + """Returns a list of sections that start with (str) key + " ". That is, if key is "Account", returns all section names that start with "Account ", but strips off the "Account ". - For instance, for "Account Test", returns "Test". + For instance, for "Account Test", returns "Test".""" - """ key = key + ' ' - return [x[len(key):] for x in self.sections() \ + if globals.options.use_unicode: + sections = [] + for section in self.sections(): + sections.append(uni.uni2str(section, exception_msg= + "non ASCII character in section %s"% section)) + return [x[len(key):] for x in sections \ + if x.startswith(key)] + else: + return [x[len(key):] for x in self.sections() \ if x.startswith(key)] def set_if_not_exists(self, section, option, value): - """ - Set a value if it does not exist yet + """Set a value if it does not exist yet. This allows to set default if the user has not explicitly - configured anything. - - """ + configured anything.""" + if not self.has_option(section, option): self.set(section, option, value) def apply_xforms(self, string, transforms): - """ - Applies set of transformations to a string. + """Applies set of transformations to a string. Arguments: - string: source string; if None, then no processing will @@ -145,9 +142,8 @@ class CustomConfigParser(SafeConfigParser): - transforms: iterable that returns transformation function on each turn. - Returns transformed string. + Returns transformed string.""" - """ if string == None: return None for f in transforms: @@ -157,21 +153,18 @@ class CustomConfigParser(SafeConfigParser): def CustomConfigDefault(): - """ - Just a constant that won't occur anywhere else. + """Just a constant that won't occur anywhere else. This allows us to differentiate if the user has passed in any default value to the getconf* functions in ConfigHelperMixin - derived classes. + derived classes.""" - """ pass class ConfigHelperMixin: - """ - Allow comfortable retrieving of config values pertaining + """Allow comfortable retrieving of config values pertaining to a section. If a class inherits from cls:`ConfigHelperMixin`, it needs @@ -181,13 +174,10 @@ class ConfigHelperMixin: the section to look up). All calls to getconf* will then return the configuration values for the CustomConfigParser object in the specific section. - """ - def _confighelper_runner(self, option, default, defaultfunc, mainfunc, *args): - """ - Returns configuration or default value for option + """Returns configuration or default value for option that contains in section identified by getsection(). Arguments: @@ -201,8 +191,8 @@ class ConfigHelperMixin: - defaultfunc and mainfunc: processing helpers. - args: additional trailing arguments that will be passed to all processing helpers. - """ + lst = [self.getsection(), option] if default == CustomConfigDefault: return mainfunc(*(lst + list(args))) @@ -210,50 +200,43 @@ class ConfigHelperMixin: lst.append(default) return defaultfunc(*(lst + list(args))) - def getconfig(self): - """ - Returns CustomConfigParser object that we will use + """Returns CustomConfigParser object that we will use for all our actions. - Must be overriden in all classes that use this mix-in. + Must be overriden in all classes that use this mix-in.""" - """ raise NotImplementedError("ConfigHelperMixin.getconfig() " "is to be overriden") def getsection(self): - """ - Returns name of configuration section in which our + """Returns name of configuration section in which our class keeps its configuration. - Must be overriden in all classes that use this mix-in. + Must be overriden in all classes that use this mix-in.""" - """ raise NotImplementedError("ConfigHelperMixin.getsection() " "is to be overriden") def getconf(self, option, default = CustomConfigDefault): - """ - Retrieves string from the configuration. + """Retrieves string from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. - """ + return self._confighelper_runner(option, default, self.getconfig().getdefault, self.getconfig().get) def getconf_xform(self, option, xforms, default = CustomConfigDefault): - """ - Retrieves string from the configuration transforming the result. + """Retrieves string from the configuration transforming the result. Arguments: - option: option name whose value is to be retrieved; @@ -262,22 +245,21 @@ class ConfigHelperMixin: both retrieved and default one; - default: default value for string if no such option exists. - """ + value = self.getconf(option, default) return self.getconfig().apply_xforms(value, xforms) def getconfboolean(self, option, default = CustomConfigDefault): - """ - Retrieves boolean value from the configuration. + """Retrieves boolean value from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. - """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultboolean, self.getconfig().getboolean) @@ -293,21 +275,21 @@ class ConfigHelperMixin: exists. """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultint, self.getconfig().getint) def getconffloat(self, option, default = CustomConfigDefault): - """ - Retrieves floating-point value from the configuration. + """Retrieves floating-point value from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. - """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultfloat, self.getconfig().getfloat) @@ -315,8 +297,7 @@ class ConfigHelperMixin: def getconflist(self, option, separator_re, default = CustomConfigDefault): - """ - Retrieves strings from the configuration and splits it + """Retrieves strings from the configuration and splits it into the list of strings. Arguments: @@ -325,8 +306,8 @@ class ConfigHelperMixin: to be used for split operation; - default: default return value if no such option exists. - """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultlist, self.getconfig().getlist, separator_re) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index eabbd23..190f6de 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2003-2011 John Goerzen & contributors +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,29 +14,33 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import mbnames, CustomConfig, OfflineImapError -from offlineimap import globals -from offlineimap.repository import Repository -from offlineimap.ui import getglobalui -from offlineimap.threadutil import InstanceLimitedThread from subprocess import Popen, PIPE from threading import Event import os from sys import exc_info import traceback +from offlineimap import mbnames, CustomConfig, OfflineImapError +from offlineimap import globals +from offlineimap.repository import Repository +from offlineimap.ui import getglobalui +from offlineimap.threadutil import InstanceLimitedThread + try: import fcntl except: pass # ok if this fails, we can do without +# FIXME: spaghetti code alert! def getaccountlist(customconfig): return customconfig.getsectionlist('Account') +# FIXME: spaghetti code alert! def AccountListGenerator(customconfig): return [Account(customconfig, accountname) for accountname in getaccountlist(customconfig)] +# FIXME: spaghetti code alert! def AccountHashGenerator(customconfig): retval = {} for item in AccountListGenerator(customconfig): diff --git a/offlineimap/error.py b/offlineimap/error.py index aa7f535..1be64ac 100644 --- a/offlineimap/error.py +++ b/offlineimap/error.py @@ -10,6 +10,7 @@ class OfflineImapError(Exception): * **REPO**: Abort repository sync, continue with next account * **CRITICAL**: Immediately exit offlineimap """ + MESSAGE, FOLDER_RETRY, FOLDER, REPO, CRITICAL = 0, 10, 15, 20, 30 def __init__(self, reason, severity, errcode=None): @@ -26,6 +27,7 @@ class OfflineImapError(Exception): value). So far, no errcodes have been defined yet. :type severity: OfflineImapError.ERROR value""" + self.errcode = errcode self.severity = severity diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 106c390..2e1b95f 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -1,5 +1,5 @@ # Base folder support -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,7 +23,6 @@ import offlineimap.accounts import os.path import re from sys import exc_info -import traceback class BaseFolder(object): @@ -113,6 +112,7 @@ class BaseFolder(object): def quickchanged(self, statusfolder): """ Runs quick check for folder changes and returns changed status: True -- changed, False -- not changed. + :param statusfolder: keeps track of the last known folder state. """ return True @@ -129,11 +129,13 @@ class BaseFolder(object): return 1 def getvisiblename(self): - """The nametrans-transposed name of the folder's name""" + """The nametrans-transposed name of the folder's name.""" + return self.visiblename def getexplainedname(self): - """ Name that shows both real and nametrans-mangled values""" + """Name that shows both real and nametrans-mangled values.""" + if self.name == self.visiblename: return self.name else: @@ -603,6 +605,7 @@ class BaseFolder(object): :param new_uid: (optional) If given, the old UID will be changed to a new UID. This allows backends efficient renaming of messages if the UID has changed.""" + raise NotImplementedError def deletemessage(self, uid): @@ -610,6 +613,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + raise NotImplementedError def deletemessages(self, uidlist): @@ -617,6 +621,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + for uid in uidlist: self.deletemessage(uid) @@ -632,6 +637,7 @@ class BaseFolder(object): :param statusfolder: A LocalStatusFolder instance :param register: whether we should register a new thread." :returns: Nothing on success, or raises an Exception.""" + # Sometimes, it could be the case that if a sync takes awhile, # a message might be deleted from the maildir before it can be # synced to the status cache. This is only a problem with diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 1dccf90..5f3d32d 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -1,5 +1,5 @@ # Local status cache virtual folder -# Copyright (C) 2002 - 2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,10 +15,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from .Base import BaseFolder import os import threading +from .Base import BaseFolder class LocalStatusFolder(BaseFolder): """LocalStatus backend implemented as a plain text file""" @@ -33,9 +33,9 @@ class LocalStatusFolder(BaseFolder): self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self.messagelist = {} self.savelock = threading.Lock() - self.doautosave = self.config.getdefaultboolean("general", "fsync", - False) - """Should we perform fsyncs as often as possible?""" + # Should we perform fsyncs as often as possible? + self.doautosave = self.config.getdefaultboolean( + "general", "fsync", False) # Interface from BaseFolder def storesmessages(self): @@ -63,13 +63,12 @@ class LocalStatusFolder(BaseFolder): def readstatus_v1(self, fp): - """ - Read status folder in format version 1. + """Read status folder in format version 1. Arguments: - fp: I/O object that points to the opened database file. - """ + for line in fp.xreadlines(): line = line.strip() try: @@ -86,13 +85,12 @@ class LocalStatusFolder(BaseFolder): def readstatus(self, fp): - """ - Read status file in the current format. + """Read status file in the current format. Arguments: - fp: I/O object that points to the opened database file. - """ + for line in fp.xreadlines(): line = line.strip() try: @@ -164,11 +162,13 @@ class LocalStatusFolder(BaseFolder): def save(self): - """Save changed data to disk. For this backend it is the same as saveall""" + """Save changed data to disk. For this backend it is the same as saveall.""" + self.saveall() def saveall(self): - """Saves the entire messagelist to disk""" + """Saves the entire messagelist to disk.""" + with self.savelock: file = open(self.filename + ".tmp", "wt") file.write((self.magicline % self.cur_version) + "\n") @@ -198,6 +198,7 @@ class LocalStatusFolder(BaseFolder): See folder/Base for detail. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode.""" + if uid < 0: # We cannot assign a uid. return uid @@ -235,6 +236,7 @@ class LocalStatusFolder(BaseFolder): def savemessageslabelsbulk(self, labels): """Saves labels from a dictionary in a single database operation.""" + for uid, lb in labels.items(): self.messagelist[uid]['labels'] = lb self.save() @@ -254,6 +256,7 @@ class LocalStatusFolder(BaseFolder): def savemessagesmtimebulk(self, mtimes): """Saves mtimes from the mtimes dictionary in a single database operation.""" + for uid, mt in mtimes.items(): self.messagelist[uid]['mtime'] = mt self.save() diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 156b39e..88ece8f 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -1,5 +1,5 @@ # Maildir folder support -# Copyright (C) 2002 - 2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,15 +19,12 @@ import socket import time import re import os -import tempfile from .Base import BaseFolder from threading import Lock - try: from hashlib import md5 except ImportError: from md5 import md5 - try: # python 2.6 has set() built in set except NameError: @@ -131,6 +128,7 @@ class MaildirFolder(BaseFolder): :returns: (prefix, UID, FMD5, flags). UID is a numeric "long" type. flags is a set() of Maildir flags""" + prefix, uid, fmd5, flags = None, None, None, set() prefixmatch = self.re_prefixmatch.match(filename) if prefixmatch: @@ -227,7 +225,8 @@ class MaildirFolder(BaseFolder): # Interface from BaseFolder def getmessage(self, uid): - """Return the content of the message""" + """Return the content of the message.""" + filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) file = open(filepath, 'rt') @@ -249,6 +248,7 @@ class MaildirFolder(BaseFolder): :param uid: The UID`None`, or a set of maildir flags :param flags: A set of maildir flags :returns: String containing unique message filename""" + timeval, timeseq = _gettimeseq() return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s' % \ (timeval, timeseq, os.getpid(), socket.gethostname(), @@ -256,8 +256,7 @@ class MaildirFolder(BaseFolder): def save_to_tmp_file(self, filename, content): - """ - Saves given content to the named temporary file in the + """Saves given content to the named temporary file in the 'tmp' subdirectory of $CWD. Arguments: @@ -265,9 +264,7 @@ class MaildirFolder(BaseFolder): - content: data to be saved. Returns: relative path to the temporary file - that was created. - - """ + that was created.""" tmpname = os.path.join('tmp', filename) # open file and write it out @@ -364,7 +361,7 @@ class MaildirFolder(BaseFolder): infomatch = self.re_flagmatch.search(filename) if infomatch: filename = filename[:-len(infomatch.group())] #strip off - infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) + infostr = '%s2,%s'% (self.infosep, ''.join(sorted(flags))) filename += infostr newfilename = os.path.join(dir_prefix, filename) @@ -386,8 +383,10 @@ class MaildirFolder(BaseFolder): This will not update the statusfolder UID, you need to do that yourself. :param new_uid: (optional) If given, the old UID will be changed - to a new UID. The Maildir backend can implement this as an efficient - rename.""" + to a new UID. The Maildir backend can implement this as + an efficient rename. + """ + if not uid in self.messagelist: raise OfflineImapError("Cannot change unknown Maildir UID %s" % uid) if uid == new_uid: return diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 10173f7..7815793 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -1,5 +1,5 @@ # Base folder support -# Copyright (C) 2002-2012 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -59,8 +59,8 @@ class MappedIMAPFolder(IMAPFolder): try: line = line.strip() except ValueError: - raise Exception("Corrupt line '%s' in UID mapping file '%s'" \ - %(line, mapfilename)) + raise Exception("Corrupt line '%s' in UID mapping file '%s'"% + (line, mapfilename)) (str1, str2) = line.split(':') loc = long(str1) rem = long(str2) @@ -89,7 +89,7 @@ class MappedIMAPFolder(IMAPFolder): raise OfflineImapError("Could not find UID for msg '{0}' (f:'{1}'." " This is usually a bad thing and should be reported on the ma" "iling list.".format(e.args[0], self), - OfflineImapError.ERROR.MESSAGE) + OfflineImapError.ERROR.MESSAGE) # Interface from BaseFolder def cachemessagelist(self): diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 05aaf7d..e012a01 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -1,6 +1,5 @@ # imaplib utilities -# Copyright (C) 2002-2007 John Goerzen -# 2012-2012 Sebastian Spaeth +# Copyright (C) 2002-2015 John Goerzen & contributors # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -33,11 +32,12 @@ class UsefulIMAPMixIn(object): return self.mailbox return None - def select(self, mailbox='INBOX', readonly=False, force = False): + def select(self, mailbox='INBOX', readonly=False, force=False): """Selects a mailbox on the IMAP server :returns: 'OK' on success, nothing if the folder was already - selected or raises an :exc:`OfflineImapError`""" + selected or raises an :exc:`OfflineImapError`.""" + if self.__getselectedfolder() == mailbox and self.is_readonly == readonly \ and not force: # No change; return. @@ -67,6 +67,7 @@ class UsefulIMAPMixIn(object): def _mesg(self, s, tn=None, secs=None): new_mesg(self, s, tn, secs) + class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): """IMAP4 client class over a tunnel @@ -80,6 +81,7 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): def open(self, host, port): """The tunnelcmd comes in on host!""" + self.host = host self.process = subprocess.Popen(host, shell=True, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) @@ -90,7 +92,8 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): self.set_nonblocking(self.read_fd) def set_nonblocking(self, fd): - "Mark fd as nonblocking" + """Mark fd as nonblocking""" + # get the file's current flag settings fl = fcntl.fcntl(fd, fcntl.F_GETFL) # clear non-blocking mode from flags @@ -115,10 +118,8 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): if self.compressor is not None: data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) - self.outfd.write(data) - def shutdown(self): self.infd.close() self.outfd.close() @@ -135,7 +136,8 @@ def new_mesg(self, s, tn=None, secs=None): class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): - """Improved version of imaplib.IMAP4_SSL overriding select()""" + """Improved version of imaplib.IMAP4_SSL overriding select().""" + def __init__(self, *args, **kwargs): self._fingerprint = kwargs.get('fingerprint', None) if type(self._fingerprint) != type([]): @@ -146,32 +148,34 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): def open(self, host=None, port=None): if not self.ca_certs and not self._fingerprint: - raise OfflineImapError("No CA certificates " + \ - "and no server fingerprints configured. " + \ - "You must configure at least something, otherwise " + \ + raise OfflineImapError("No CA certificates " + "and no server fingerprints configured. " + "You must configure at least something, otherwise " "having SSL helps nothing.", OfflineImapError.ERROR.REPO) super(WrappedIMAP4_SSL, self).open(host, port) if self._fingerprint: # compare fingerprints fingerprint = sha1(self.sock.getpeercert(True)).hexdigest() if fingerprint not in self._fingerprint: - raise OfflineImapError("Server SSL fingerprint '%s' " % fingerprint + \ - "for hostname '%s' " % host + \ - "does not match configured fingerprint(s) %s. " % self._fingerprint + \ - "Please verify and set 'cert_fingerprint' accordingly " + \ - "if not set yet.", OfflineImapError.ERROR.REPO) + raise OfflineImapError("Server SSL fingerprint '%s' " + "for hostname '%s' " + "does not match configured fingerprint(s) %s. " + "Please verify and set 'cert_fingerprint' accordingly " + "if not set yet."% + (fingerprint, host, self._fingerprint), + OfflineImapError.ERROR.REPO) class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): - """Improved version of imaplib.IMAP4 overriding select()""" + """Improved version of imaplib.IMAP4 overriding select().""" + pass def Internaldate2epoch(resp): """Convert IMAP4 INTERNALDATE to UT. - Returns seconds since the epoch. - """ + Returns seconds since the epoch.""" mo = InternalDate.match(resp) if not mo: diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index c8e3f76..24fcfea 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -1,6 +1,5 @@ # IMAP utility module -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,8 +36,8 @@ def dequote(string): """Takes string which may or may not be quoted and unquotes it. It only considers double quotes. This function does NOT consider - parenthised lists to be quoted. - """ + parenthised lists to be quoted.""" + if string and string.startswith('"') and string.endswith('"'): string = string[1:-1] # Strip off the surrounding quotes. string = string.replace('\\"', '"') @@ -49,8 +48,8 @@ def quote(string): """Takes an unquoted string and quotes it. It only adds double quotes. This function does NOT consider - parenthised lists to be quoted. - """ + parenthised lists to be quoted.""" + string = string.replace('"', '\\"') string = string.replace('\\', '\\\\') return '"%s"' % string @@ -62,12 +61,14 @@ def flagsplit(string): (FLAGS (\\Seen Old) UID 4807) returns ['FLAGS,'(\\Seen Old)','UID', '4807'] """ + if string[0] != '(' or string[-1] != ')': raise ValueError("Passed string '%s' is not a flag list" % string) return imapsplit(string[1:-1]) def __options2hash(list): """convert list [1,2,3,4,5,6] to {1:2, 3:4, 5:6}""" + # effectively this does dict(zip(l[::2],l[1::2])), however # measurements seemed to have indicated that the manual variant is # faster for mosly small lists. @@ -84,6 +85,7 @@ def flags2hash(flags): E.g. '(FLAGS (\\Seen Old) UID 4807)' leads to {'FLAGS': '(\\Seen Old)', 'UID': '4807'}""" + return __options2hash(flagsplit(flags)) def imapsplit(imapstring): @@ -182,7 +184,8 @@ flagmap = [('\\Seen', 'S'), ('\\Draft', 'D')] def flagsimap2maildir(flagstring): - """Convert string '(\\Draft \\Deleted)' into a flags set(DR)""" + """Convert string '(\\Draft \\Deleted)' into a flags set(DR).""" + retval = set() imapflaglist = flagstring[1:-1].split() for imapflag, maildirflag in flagmap: @@ -191,7 +194,8 @@ def flagsimap2maildir(flagstring): return retval def flagsmaildir2imap(maildirflaglist): - """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'""" + """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'.""" + retval = [] for imapflag, maildirflag in flagmap: if maildirflag in maildirflaglist: @@ -203,7 +207,8 @@ def uid_sequence(uidlist): [1,2,3,4,5,10,12,13] will return "1:5,10,12:13". This function sorts the list, and only collapses if subsequent entries form a range. - :returns: The collapsed UID list as string""" + :returns: The collapsed UID list as string.""" + def getrange(start, end): if start == end: return(str(start)) @@ -230,8 +235,7 @@ def uid_sequence(uidlist): def __split_quoted(string): - """ - Looks for the ending quote character in the string that starts + """Looks for the ending quote character in the string that starts with quote character, splitting out quoted component and the rest of the string (without possible space between these two parts. @@ -241,7 +245,6 @@ def __split_quoted(string): Examples: - "this is \" a test" (\\None) => ("this is \" a test", (\\None)) - "\\" => ("\\", ) - """ if len(string) == 0: @@ -269,17 +272,15 @@ def __split_quoted(string): def format_labels_string(header, labels): - """ - Formats labels for embedding into a message, + """Formats labels for embedding into a message, with format according to header name. Headers from SPACE_SEPARATED_LABEL_HEADERS keep space-separated list of labels, the rest uses comma (',') as the separator. Also see parse_labels_string() and modify it accordingly - if logics here gets changed. + if logics here gets changed.""" - """ if header in SPACE_SEPARATED_LABEL_HEADERS: sep = ' ' else: @@ -289,18 +290,16 @@ def format_labels_string(header, labels): def parse_labels_string(header, labels_str): - """ - Parses a string into a set of labels, with a format according to + """Parses a string into a set of labels, with a format according to the name of the header. See __format_labels_string() for explanation on header handling and keep these two functions synced with each other. TODO: add test to ensure that - format_labels_string * parse_labels_string is unity + - format_labels_string * parse_labels_string is unity and - parse_labels_string * format_labels_string is unity - + - parse_labels_string * format_labels_string is unity """ if header in SPACE_SEPARATED_LABEL_HEADERS: @@ -314,15 +313,13 @@ def parse_labels_string(header, labels_str): def labels_from_header(header_name, header_value): - """ - Helper that builds label set from the corresponding header value. + """Helper that builds label set from the corresponding header value. Arguments: - header_name: name of the header that keeps labels; - header_value: value of the said header, can be None Returns: set of labels parsed from the header (or empty set). - """ if header_value: diff --git a/offlineimap/init.py b/offlineimap/init.py index d9425d5..143287b 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -1,5 +1,5 @@ # OfflineIMAP initialization code -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -214,7 +214,7 @@ class OfflineImap: config.set(section, key, value) #which ui to use? cmd line option overrides config file - ui_type = config.getdefault('general','ui', 'ttyui') + ui_type = config.getdefault('general', 'ui', 'ttyui') if options.interface != None: ui_type = options.interface if '.' in ui_type: @@ -222,13 +222,13 @@ class OfflineImap: ui_type = ui_type.split('.')[-1] # TODO, make use of chosen ui for logging logging.warning('Using old interface name, consider using one ' - 'of %s' % ', '.join(UI_LIST.keys())) + 'of %s'% ', '.join(UI_LIST.keys())) if options.diagnostics: ui_type = 'basic' # enforce basic UI for --info #dry-run? Set [general]dry-run=True if options.dryrun: - dryrun = config.set('general','dry-run', "True") - config.set_if_not_exists('general','dry-run','False') + dryrun = config.set('general', 'dry-run', 'True') + config.set_if_not_exists('general', 'dry-run', 'False') try: # create the ui class @@ -264,7 +264,7 @@ class OfflineImap: imaplib.Debug = 5 if options.runonce: - # FIXME: maybe need a better + # FIXME: spaghetti code alert! for section in accounts.getaccountlist(config): config.remove_option('Account ' + section, "autorefresh") @@ -275,7 +275,7 @@ class OfflineImap: #custom folder list specified? if options.folders: foldernames = options.folders.split(",") - folderfilter = "lambda f: f in %s" % foldernames + folderfilter = "lambda f: f in %s"% foldernames folderincludes = "[]" for accountname in accounts.getaccountlist(config): account_section = 'Account ' + accountname @@ -355,12 +355,12 @@ class OfflineImap: "take a few seconds)...") accounts.Account.set_abort_event(self.config, 3) elif sig == signal.SIGQUIT: - stacktrace.dump (sys.stderr) + stacktrace.dump(sys.stderr) os.abort() - signal.signal(signal.SIGHUP,sig_handler) - signal.signal(signal.SIGUSR1,sig_handler) - signal.signal(signal.SIGUSR2,sig_handler) + signal.signal(signal.SIGHUP, sig_handler) + signal.signal(signal.SIGUSR1, sig_handler) + signal.signal(signal.SIGUSR2, sig_handler) signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) signal.signal(signal.SIGQUIT, sig_handler) @@ -394,7 +394,7 @@ class OfflineImap: for accountname in accs: account = offlineimap.accounts.SyncableAccount(self.config, accountname) - threading.currentThread().name = "Account sync %s" % accountname + threading.currentThread().name = "Account sync %s"% accountname account.syncrunner() def __serverdiagnostics(self, options): diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 176a760..936a110 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -1,6 +1,6 @@ # Mailbox name generator -# Copyright (C) 2002 John Goerzen -# +# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -49,6 +49,7 @@ def write(): def __genmbnames(): """Takes a configparser object and a boxlist, which is a list of hashes containing 'accountname' and 'foldername' keys.""" + xforms = [os.path.expanduser, os.path.expandvars] mblock.acquire() try: diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index d7a6866..4edd6c5 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -1,5 +1,5 @@ # Base repository support -# Copyright (C) 2002-2012 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ import re import os.path from sys import exc_info + from offlineimap import CustomConfig from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError @@ -113,6 +114,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): @property def readonly(self): """Is the repository readonly?""" + return self._readonly def getlocaleval(self): @@ -120,11 +122,13 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getfolders(self): """Returns a list of ALL folders on this server.""" + return [] def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" + pass def getsep(self): @@ -132,6 +136,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def should_sync_folder(self, fname): """Should this folder be synced?""" + return fname in self.folderincludes or self.folderfilter(fname) def get_create_folders(self): @@ -139,11 +144,13 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): It is disabled by either setting the whole repository 'readonly' or by using the 'createfolders' setting.""" + return (not self._readonly) and \ self.getconfboolean('createfolders', True) def makefolder(self, foldername): """Create a new folder""" + raise NotImplementedError def deletefolder(self, foldername): diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 32cf3ac..d664259 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -1,5 +1,5 @@ # IMAP repository support -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,17 +15,19 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap.repository.Base import BaseRepository -from offlineimap import folder, imaputil, imapserver, OfflineImapError -from offlineimap.folder.UIDMaps import MappedIMAPFolder -from offlineimap.threadutil import ExitNotifyThread -from offlineimap.utils.distro import get_os_sslcertfile from threading import Event import os from sys import exc_info import netrc import errno +from offlineimap.repository.Base import BaseRepository +from offlineimap import folder, imaputil, imapserver, OfflineImapError +from offlineimap.folder.UIDMaps import MappedIMAPFolder +from offlineimap.threadutil import ExitNotifyThread +from offlineimap.utils.distro import get_os_sslcertfile + + class IMAPRepository(BaseRepository): def __init__(self, reposname, account): """Initialize an IMAPRepository object.""" @@ -116,14 +118,10 @@ class IMAPRepository(BaseRepository): "'%s' specified." % self, OfflineImapError.ERROR.REPO) - def get_remote_identity(self): - """ - Remote identity is used for certain SASL mechanisms + """Remote identity is used for certain SASL mechanisms (currently -- PLAIN) to inform server about the ID - we want to authorize as instead of our login name. - - """ + we want to authorize as instead of our login name.""" return self.getconf('remote_identity', default=None) @@ -218,13 +216,10 @@ class IMAPRepository(BaseRepository): return self.getconf('ssl_version', None) def get_ssl_fingerprint(self): - """ - Return array of possible certificate fingerprints. + """Return array of possible certificate fingerprints. Configuration item cert_fingerprint can contain multiple - comma-separated fingerprints in hex form. - - """ + comma-separated fingerprints in hex form.""" value = self.getconf('cert_fingerprint', "") return [f.strip().lower() for f in value.split(',') if f] @@ -262,8 +257,8 @@ class IMAPRepository(BaseRepository): 5. read password from /etc/netrc On success we return the password. - If all strategies fail we return None. - """ + If all strategies fail we return None.""" + # 1. evaluate Repository 'remotepasseval' passwd = self.getconf('remotepasseval', None) if passwd != None: @@ -304,7 +299,6 @@ class IMAPRepository(BaseRepository): # no strategy yielded a password! return None - def getfolder(self, foldername): return self.getfoldertype()(self.imapserver, foldername, self) @@ -392,6 +386,7 @@ class IMAPRepository(BaseRepository): when you are done creating folders yourself. :param foldername: Full path of the folder to be created.""" + if self.getreference(): foldername = self.getreference() + self.getsep() + foldername if not foldername: # Create top level folder as folder separator diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index ba29d37..52ba714 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -1,6 +1,5 @@ # Local status cache repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -81,7 +80,7 @@ class LocalStatusRepository(BaseRepository): return '.' def makefolder(self, foldername): - """Create a LocalStatus Folder""" + """Create a LocalStatus Folder.""" if self.account.dryrun: return # bail out in dry-run mode @@ -114,9 +113,11 @@ class LocalStatusRepository(BaseRepository): (see getfolderfilename) so we can not derive folder names from the file names that we have available. TODO: need to store a list of folder names somehow?""" + pass def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" + self._folders = {} diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index dae811d..f0495fa 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -1,6 +1,5 @@ # Maildir repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,6 +26,7 @@ class MaildirRepository(BaseRepository): def __init__(self, reposname, account): """Initialize a MaildirRepository object. Takes a path name to the directory holding all the Maildir directories.""" + BaseRepository.__init__(self, reposname, account) self.root = self.getlocalroot() @@ -41,6 +41,7 @@ class MaildirRepository(BaseRepository): def _append_folder_atimes(self, foldername): """Store the atimes of a folder's new|cur in self.folder_atimes""" + p = os.path.join(self.root, foldername) new = os.path.join(p, 'new') cur = os.path.join(p, 'cur') @@ -51,6 +52,7 @@ class MaildirRepository(BaseRepository): """Sets folders' atime back to their values after a sync Controlled by the 'restoreatime' config parameter.""" + if not self.getconfboolean('restoreatime', False): return # not configured to restore @@ -82,6 +84,7 @@ class MaildirRepository(BaseRepository): levels will be created if they do not exist yet. 'cur', 'tmp', and 'new' subfolders will be created in the maildir. """ + self.ui.makefolder(self, foldername) if self.account.dryrun: return @@ -134,7 +137,7 @@ class MaildirRepository(BaseRepository): "folder '%s'." % foldername, OfflineImapError.ERROR.FOLDER) - def _getfolders_scandir(self, root, extension = None): + def _getfolders_scandir(self, root, extension=None): """Recursively scan folder 'root'; return a list of MailDirFolder :param root: (absolute) path to Maildir root @@ -200,4 +203,5 @@ class MaildirRepository(BaseRepository): def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" + self.folders = None diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 6cc1855..fb3da80 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -1,5 +1,5 @@ # Curses-based interfaces -# Copyright (C) 2003-2011 John Goerzen & contributors +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,12 +22,13 @@ import sys import os import curses import logging + from offlineimap.ui.UIBase import UIBase from offlineimap.threadutil import ExitNotifyThread import offlineimap -class CursesUtil: +class CursesUtil: def __init__(self, *args, **kwargs): # iolock protects access to the self.iolock = RLock() @@ -322,6 +323,7 @@ class Blinkenlights(UIBase, CursesUtil): Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" + # create console handler with a higher log level ch = CursesLogHandler() #ch.setLevel(logging.DEBUG) @@ -336,6 +338,7 @@ class Blinkenlights(UIBase, CursesUtil): def isusable(s): """Returns true if the backend is usable ie Curses works""" + # Not a terminal? Can't use curses. if not sys.stdout.isatty() and sys.stdin.isatty(): return False @@ -391,6 +394,7 @@ class Blinkenlights(UIBase, CursesUtil): def acct(self, *args): """Output that we start syncing an account (and start counting)""" + self.gettf().setcolor('purple') super(Blinkenlights, self).acct(*args) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 01abd6f..5cf7449 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -1,4 +1,4 @@ -# Copyright (C) 2007-2011 John Goerzen & contributors +# Copyright (C) 2007-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index efde74f..5fa4dde 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -1,5 +1,5 @@ # TTY UI -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +24,7 @@ from offlineimap.ui.UIBase import UIBase class TTYFormatter(logging.Formatter): """Specific Formatter that adds thread information to the log output""" + def __init__(self, *args, **kwargs): #super() doesn't work in py2.6 as 'logging' uses old-style class logging.Formatter.__init__(self, *args, **kwargs) @@ -46,12 +47,14 @@ class TTYFormatter(logging.Formatter): log_str = " %s" % log_str return log_str + class TTYUI(UIBase): def setup_consolehandler(self): """Backend specific console handler Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" + # create console handler with a higher log level ch = logging.StreamHandler() #ch.setLevel(logging.DEBUG) @@ -67,10 +70,12 @@ class TTYUI(UIBase): def isusable(self): """TTYUI is reported as usable when invoked on a terminal""" + return sys.stdout.isatty() and sys.stdin.isatty() - def getpass(self, accountname, config, errmsg = None): + def getpass(self, accountname, config, errmsg=None): """TTYUI backend is capable of querying the password""" + if errmsg: self.warn("%s: %s" % (accountname, errmsg)) self._log_con_handler.acquire() # lock the console output @@ -97,6 +102,7 @@ class TTYUI(UIBase): implementations return 0 for successful sleep and 1 for an 'abort', ie a request to sync immediately. """ + if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: self.logger.info("Next refresh in %.1f minutes" % ( diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 0edfa18..f6007da 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -1,5 +1,5 @@ # UI base class -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -45,22 +45,22 @@ def getglobalui(): return globalui class UIBase(object): - def __init__(self, config, loglevel = logging.INFO): + def __init__(self, config, loglevel=logging.INFO): self.config = config # Is this a 'dryrun'? self.dryrun = config.getdefaultboolean('general', 'dry-run', False) self.debuglist = [] - """list of debugtypes we are supposed to log""" + # list of debugtypes we are supposed to log self.debugmessages = {} - """debugmessages in a deque(v) per thread(k)""" + # debugmessages in a deque(v) per thread(k) self.debugmsglen = 15 self.threadaccounts = {} - """dict linking active threads (k) to account names (v)""" + # dict linking active threads (k) to account names (v) self.acct_startimes = {} - """linking active accounts with the time.time() when sync started""" + # linking active accounts with the time.time() when sync started self.logfile = None self.exc_queue = Queue() - """saves all occuring exceptions, so we can output them at the end""" + # saves all occuring exceptions, so we can output them at the end # create logger with 'OfflineImap' app self.logger = logging.getLogger('OfflineImap') self.logger.setLevel(loglevel) @@ -73,6 +73,7 @@ class UIBase(object): Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" + # create console handler with a higher log level ch = logging.StreamHandler(sys.stdout) #ch.setLevel(logging.DEBUG) @@ -94,12 +95,13 @@ class UIBase(object): # write out more verbose initial info blurb on the log file p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ - "Args: %s" % (offlineimap.__bigversion__, p_ver, sys.platform, + "Args: %s"% (offlineimap.__bigversion__, p_ver, sys.platform, " ".join(sys.argv)) self.logger.info(msg) def _msg(self, msg): """Display a message.""" + # TODO: legacy function, rip out. self.info(msg) @@ -149,7 +151,8 @@ class UIBase(object): self._msg(traceback.format_tb(instant_traceback)) def registerthread(self, account): - """Register current thread as being associated with an account name""" + """Register current thread as being associated with an account name.""" + cur_thread = threading.currentThread() if cur_thread in self.threadaccounts: # was already associated with an old account, update info @@ -162,15 +165,17 @@ class UIBase(object): self.threadaccounts[cur_thread] = account def unregisterthread(self, thr): - """Unregister a thread as being associated with an account name""" + """Unregister a thread as being associated with an account name.""" + if thr in self.threadaccounts: del self.threadaccounts[thr] self.debug('thread', "Unregister thread '%s'" % thr.getName()) - def getthreadaccount(self, thr = None): + def getthreadaccount(self, thr=None): """Get Account() for a thread (current if None) - If no account has been registered with this thread, return 'None'""" + If no account has been registered with this thread, return 'None'.""" + if thr == None: thr = threading.currentThread() if thr in self.threadaccounts: @@ -214,6 +219,7 @@ class UIBase(object): """Return the type of a repository or Folder as string (IMAP, Gmail, Maildir, etc...)""" + prelimname = object.__class__.__name__.split('.')[-1] # Strip off extra stuff. return re.sub('(Folder|Repository)', '', prelimname) @@ -222,6 +228,7 @@ class UIBase(object): """Returns true if this UI object is usable in the current environment. For instance, an X GUI would return true if it's being run in X with a valid DISPLAY setting, and false otherwise.""" + return True ################################################## INPUT @@ -281,7 +288,8 @@ class UIBase(object): pass def connecting(self, hostname, port): - """Log 'Establishing connection to'""" + """Log 'Establishing connection to'.""" + if not self.logger.isEnabledFor(logging.INFO): return displaystr = '' hostname = hostname if hostname else '' @@ -291,19 +299,22 @@ class UIBase(object): self.logger.info("Establishing connection%s" % displaystr) def acct(self, account): - """Output that we start syncing an account (and start counting)""" + """Output that we start syncing an account (and start counting).""" + self.acct_startimes[account] = time.time() self.logger.info("*** Processing account %s" % account) def acctdone(self, account): - """Output that we finished syncing an account (in which time)""" + """Output that we finished syncing an account (in which time).""" + sec = time.time() - self.acct_startimes[account] del self.acct_startimes[account] self.logger.info("*** Finished account '%s' in %d:%02d" % (account, sec // 60, sec % 60)) def syncfolders(self, src_repo, dst_repo): - """Log 'Copying folder structure...'""" + """Log 'Copying folder structure...'.""" + if self.logger.isEnabledFor(logging.DEBUG): self.debug('', "Copying folder structure from %s to %s" %\ (src_repo, dst_repo)) @@ -328,12 +339,12 @@ class UIBase(object): def validityproblem(self, folder): self.logger.warning("UID validity problem for folder %s (repo %s) " "(saved %d; got %d); skipping it. Please see FAQ " - "and manual on how to handle this." % \ + "and manual on how to handle this."% \ (folder, folder.getrepository(), folder.get_saveduidvalidity(), folder.get_uidvalidity())) def loadmessagelist(self, repos, folder): - self.logger.debug("Loading message list for %s[%s]" % ( + self.logger.debug(u"Loading message list for %s[%s]"% ( self.getnicename(repos), folder)) @@ -389,7 +400,8 @@ class UIBase(object): self.logger.info("Collecting data from messages on %s" % source) def serverdiagnostics(self, repository, type): - """Connect to repository and output useful information for debugging""" + """Connect to repository and output useful information for debugging.""" + conn = None self._msg("%s repository '%s': type '%s'" % (type, repository.name, self.getnicename(repository))) @@ -440,8 +452,9 @@ class UIBase(object): repository.imapserver.close() def savemessage(self, debugtype, uid, flags, folder): - """Output a log line stating that we save a msg""" - self.debug(debugtype, "Write mail '%s:%d' with flags %s" % + """Output a log line stating that we save a msg.""" + + self.debug(debugtype, u"Write mail '%s:%d' with flags %s"% (folder, uid, repr(flags))) ################################################## Threads @@ -461,42 +474,46 @@ class UIBase(object): del self.debugmessages[thread] def getThreadExceptionString(self, thread): - message = "Thread '%s' terminated with exception:\n%s" % \ + message = u"Thread '%s' terminated with exception:\n%s"% \ (thread.getName(), thread.exit_stacktrace) - message += "\n" + self.getThreadDebugLog(thread) + message += u"\n" + self.getThreadDebugLog(thread) return message def threadException(self, thread): """Called when a thread has terminated with an exception. The argument is the ExitNotifyThread that has so terminated.""" + self.warn(self.getThreadExceptionString(thread)) self.delThreadDebugLog(thread) self.terminate(100) def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" + #print any exceptions that have occurred over the run if not self.exc_queue.empty(): - self.warn("ERROR: Exceptions occurred during the run!") + self.warn(u"ERROR: Exceptions occurred during the run!") while not self.exc_queue.empty(): msg, exc, exc_traceback = self.exc_queue.get() if msg: - self.warn("ERROR: %s\n %s" % (msg, exc)) + self.warn(u"ERROR: %s\n %s"% (msg, exc)) else: - self.warn("ERROR: %s" % (exc)) + self.warn(u"ERROR: %s"% (exc)) if exc_traceback: - self.warn("\nTraceback:\n%s" %"".join( + self.warn(u"\nTraceback:\n%s"% "".join( traceback.format_tb(exc_traceback))) if errormsg and errortitle: - self.warn('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) + self.warn(u'ERROR: %s\n\n%s\n'% (errortitle, errormsg)) elif errormsg: - self.warn('%s\n' % errormsg) + self.warn(u'%s\n' % errormsg) sys.exit(exitstatus) def threadExited(self, thread): - """Called when a thread has exited normally. Many UIs will - just ignore this.""" + """Called when a thread has exited normally. + + Many UIs will just ignore this.""" + self.delThreadDebugLog(thread) self.unregisterthread(thread) @@ -518,6 +535,7 @@ class UIBase(object): :returns: 0/False if timeout expired, 1/2/True if there is a request to cancel the timer. """ + abortsleep = False while sleepsecs > 0 and not abortsleep: if account.get_abort_event(): @@ -538,6 +556,7 @@ class UIBase(object): implementations return 0 for successful sleep and 1 for an 'abort', ie a request to sync immediately. """ + if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: self.logger.debug("Next refresh in %.1f minutes" % ( diff --git a/offlineimap/ui/debuglock.py b/offlineimap/ui/debuglock.py index 4756b08..ef6e825 100644 --- a/offlineimap/ui/debuglock.py +++ b/offlineimap/ui/debuglock.py @@ -1,6 +1,5 @@ # Locking debugging code -- temporary -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From dbb632275e75a9d0ab9319b92c589c1cce943191 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 7 Jan 2015 22:20:14 +0100 Subject: [PATCH 668/817] v6.5.7-rc1 Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 111 +++++++++++++++++++++++++++++----------- offlineimap/__init__.py | 6 +-- 2 files changed, 85 insertions(+), 32 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 9a98d29..5580e69 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,10 +5,26 @@ ChangeLog :website: http://offlineimap.org -OfflineIMAP v6.5.6.1 (YYYY-MM-DD) -================================= +OfflineIMAP v6.5.7-rc1 (2015-01-07) +=================================== -* Properly generate tarball from "sdist" command (GitHub #137) +Notes +----- + +I think it's time for a new release candidate. Our release cycle are long +enough and users are asked to use the current TIP of the next branch to test +our recent patches. + +The current version makes better support for environment variable expansion and +improves OS portability. Gmail should be better supported: we are still +expecting feedbacks. Embedded library imaplib2 is updated to v2.37. +Debugging messages are added and polished. + +There's some code cleanups and refactoring, also. + + +Features +-------- * Expand environment variables in the following configuration items: @@ -21,39 +37,73 @@ OfflineIMAP v6.5.6.1 (YYYY-MM-DD) configuration items: - Repository.sslclientcert; - Repository.sslclientkey. - -* Updated bundled imaplib2 to 2.37: - - add missing idle_lock in _handler() - +* Support default CA bundle locations for a couple of + known Unix systems (Michael Vogt, GutHub pull #19) * Added default CA bundle location for OpenBSD (GitHub pull #120) and DragonFlyBSD. +Fixes +----- + +* Fix unbounded recursion during flag update (Josh Berry). +* Do not ignore gmail labels if header appears multiple times +* Delete gmail labels header before adding a new one +* Fix improper header separator for X-OfflineIMAP header +* Match header names case-insensitively +* Create SQLite database directory if it doesn't exist + yet; warn if path is not a directory (Nick Farrell, + GutHub pull #102) +* Properly manipulate contents of messagelist for folder +* Fix label processing in GmailMaildir +* Properly capitalize OpenSSL +* Fix warning-level message processing by MachineUI + (GitHub pull #64, GitHub pull #118). +* Properly generate tarball from "sdist" command (GitHub #137) +* Fix Markdown formatting +* Fix typo in apply_xforms invocation +* Merge pull request #136 from aroig/gh/label-fix +* Fix mangled message headers for servers without UIDPLUS: + X-OfflineIMAP was added with preceeding '\n' instead of + '\r\n' just before message was uploaded to the IMAP server. +* Add missing version bump for 6.5.6 (it was released with + 6.5.5 in setup.py and other places). + +Changes +------- + +* Warn about a tricky piece of code in addmessageheader +* Rename addmessageheader()'s crlf parameter to linebreak +* addmessageheader: fix case #2 and flesh out docstring +* addmessageheader(): add debug for header insertion +* Add version qualifier to differentiate releases and development ones +* More clearly show results of folder name translation +* IMAP: provide message-id in error messages +* Trade recursion by plain old cycle +* Avoid copying array every time, just slice it * Added OpenSSL exception clause to our main GPL to allow people to link with OpenSSL in run-time. It is needed at least for Debian, see https://lists.debian.org/debian-legal/2002/10/msg00113.html for details. +* Brought CustomConfig.py into more proper shape +* Updated bundled imaplib2 to 2.37: + - add missing idle_lock in _handler() +* Imaplib2: trade backticks to repr() +* Introduce CustomConfig method that applies set of transforms +* imaplibutil.py: remove unused imports +* CustomConfig.py: remove unused imports +* init.py: remove unused import +* repository/Base.py: remove unused import +* repository/GmailMaildir.py: remove unused import +* repository/LocalStatus.py: remove unused import +* ui/Curses.py: remove unused import +* ui/UIBase.py: remove unused import +* localeval: comment on security issues +* docs: remove obsolete comment about SubmittingPatches.rst +* utils/const.py: fix ident +* ui/UIBase: folderlist(): avoid built-in list() redefinition +* more consistent style -* Fix warning-level message processing by MachineUI - (GitHub pull #64, GitHub pull #118). - -* Support default CA bundle locations for a couple of - known Unix systems (Michael Vogt, GutHub pull #19) - -* Create SQLite database directory if it doesn't exist - yet; warn if path is not a directory (Nick Farrell, - GutHub pull #102) - -* Fix mangled message headers for servers without UIDPLUS: - X-OfflineIMAP was added with preceeding '\n' instead of - '\r\n' just before message was uploaded to the IMAP server. - -* Add missing version bump for 6.5.6 (it was released with - 6.5.5 in setup.py and other places). - -* Various fixes in documentation. - -* Fix unbounded recursion during flag update (Josh Berry). OfflineIMAP v6.5.6 (2014-05-14) @@ -272,7 +322,9 @@ OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" OfflineIMAP v6.5.0 (2012-01-06) =============================== -This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. Please upgrade to avoid potential data loss! The version has been bumped to 6.5.0, please let everyone know that the 6.4.x series is problematic. +This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. +Please upgrade to avoid potential data loss! The version has been bumped to +6.5.0, please let everyone know that the 6.4.x series is problematic. * Uploading multiple emails to an IMAP server would lead to wrong UIDs being returned (ie the same for all), which confused offlineimap and @@ -368,7 +420,8 @@ Bug Fixes OfflineIMAP v6.4.0 (2011-09-29) =============================== -This is the first stable release to support the forward-compatible per-account locks and remote folder creation that has been introduced in the 6.3.5 series. +This is the first stable release to support the forward-compatible per-account +locks and remote folder creation that has been introduced in the 6.3.5 series. * Various regression and bug fixes from the last couple of RCs diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 6a4e0e2..ff6acd9 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,10 +1,10 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.6.1" -__revision__ = "-devel" +__version__ = "6.5.7" +__revision__ = "-rc1" __bigversion__ = __version__ + __revision__ -__copyright__ = "Copyright 2002-2013 John Goerzen & contributors" +__copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" From eab3e1861395a3997454380703ca12d28ea431ed Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 8 Jan 2015 12:07:29 +0100 Subject: [PATCH 669/817] remove garbage about unicode Signed-off-by: Nicolas Sebrecht --- offlineimap/CustomConfig.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index ae27d41..14564c2 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -93,11 +93,7 @@ class CustomConfigParser(SafeConfigParser): def getlocaleval(self): xforms = [os.path.expanduser, os.path.expandvars] if self.has_option("general", "pythonfile"): - if globals.options.use_unicode: - path = uni.fsEncode(self.get("general", "pythonfile"), - exception_msg="cannot convert character for pythonfile") - else: - path = self.get("general", "pythonfile") + path = self.get("general", "pythonfile") path = self.apply_xforms(path, xforms) else: path = None @@ -112,16 +108,8 @@ class CustomConfigParser(SafeConfigParser): For instance, for "Account Test", returns "Test".""" key = key + ' ' - if globals.options.use_unicode: - sections = [] - for section in self.sections(): - sections.append(uni.uni2str(section, exception_msg= - "non ASCII character in section %s"% section)) - return [x[len(key):] for x in sections \ - if x.startswith(key)] - else: - return [x[len(key):] for x in self.sections() \ - if x.startswith(key)] + return [x[len(key):] for x in self.sections() \ + if x.startswith(key)] def set_if_not_exists(self, section, option, value): """Set a value if it does not exist yet. From 3a60c853739f2b38c96c7449daef0a90a5265cec Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 8 Jan 2015 16:00:11 +0100 Subject: [PATCH 670/817] offlineimap.conf: fix minor typo in comments Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap.conf b/offlineimap.conf index 94094a6..63c1b97 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -284,7 +284,7 @@ remoterepository = RemoteExample # #maildir-windows-compatible = no -# Specifies if we want to sync GMail lables with the local repository. +# Specifies if we want to sync GMail labels with the local repository. # Effective only for GMail IMAP repositories. # #synclabels = no From 1339cc891359ba4a9b9a7214018a9ef2efcfea9d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 8 Jan 2015 17:19:01 +0100 Subject: [PATCH 671/817] emailutil.py: remove unused import time Signed-off-by: Nicolas Sebrecht --- offlineimap/emailutil.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offlineimap/emailutil.py b/offlineimap/emailutil.py index 28463f7..4aa34a8 100644 --- a/offlineimap/emailutil.py +++ b/offlineimap/emailutil.py @@ -1,5 +1,5 @@ # Some useful functions to extract data out of emails -# Copyright (C) 2002-2012 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +17,6 @@ import email from email.Parser import Parser as MailParser -import time def get_message_date(content, header='Date'): """ From 11619faf7c763b7abfa3ac378a44f0289d837874 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 8 Jan 2015 22:08:11 +0100 Subject: [PATCH 672/817] imaputil.py: avoid to redefine "string" Signed-off-by: Nicolas Sebrecht --- offlineimap/imaputil.py | 90 ++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 24fcfea..9623589 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -32,29 +32,29 @@ def __debug(*args): msg.append(str(arg)) getglobalui().debug('imap', " ".join(msg)) -def dequote(string): +def dequote(s): """Takes string which may or may not be quoted and unquotes it. It only considers double quotes. This function does NOT consider parenthised lists to be quoted.""" - if string and string.startswith('"') and string.endswith('"'): - string = string[1:-1] # Strip off the surrounding quotes. - string = string.replace('\\"', '"') - string = string.replace('\\\\', '\\') - return string + if s and s.startswith('"') and s.endswith('"'): + s = s[1:-1] # Strip off the surrounding quotes. + s = s.replace('\\"', '"') + s = s.replace('\\\\', '\\') + return s -def quote(string): +def quote(s): """Takes an unquoted string and quotes it. It only adds double quotes. This function does NOT consider parenthised lists to be quoted.""" - string = string.replace('"', '\\"') - string = string.replace('\\', '\\\\') - return '"%s"' % string + s = s.replace('"', '\\"') + s = s.replace('\\', '\\\\') + return '"%s"'% s -def flagsplit(string): +def flagsplit(s): """Converts a string of IMAP flags to a list :returns: E.g. '(\\Draft \\Deleted)' returns ['\\Draft','\\Deleted']. @@ -62,9 +62,9 @@ def flagsplit(string): ['FLAGS,'(\\Seen Old)','UID', '4807'] """ - if string[0] != '(' or string[-1] != ')': - raise ValueError("Passed string '%s' is not a flag list" % string) - return imapsplit(string[1:-1]) + if s[0] != '(' or s[-1] != ')': + raise ValueError("Passed s '%s' is not a flag list"% s) + return imapsplit(s[1:-1]) def __options2hash(list): """convert list [1,2,3,4,5,6] to {1:2, 3:4, 5:6}""" @@ -234,41 +234,41 @@ def uid_sequence(uidlist): return ",".join(retval) -def __split_quoted(string): - """Looks for the ending quote character in the string that starts - with quote character, splitting out quoted component and the - rest of the string (without possible space between these two - parts. +def __split_quoted(s): + """Looks for the ending quote character in the string that starts + with quote character, splitting out quoted component and the + rest of the string (without possible space between these two + parts. - First character of the string is taken to be quote character. + First character of the string is taken to be quote character. - Examples: - - "this is \" a test" (\\None) => ("this is \" a test", (\\None)) - - "\\" => ("\\", ) - """ + Examples: + - "this is \" a test" (\\None) => ("this is \" a test", (\\None)) + - "\\" => ("\\", ) + """ - if len(string) == 0: - return ('', '') + if len(s) == 0: + return ('', '') - q = quoted = string[0] - rest = string[1:] - while True: - next_q = rest.find(q) - if next_q == -1: - raise ValueError("can't find ending quote '%s' in '%s'" % (q, string)) - # If quote is preceeded by even number of backslashes, - # then it is the ending quote, otherwise the quote - # character is escaped by backslash, so we should - # continue our search. - is_escaped = False - i = next_q - 1 - while i >= 0 and rest[i] == '\\': - i -= 1 - is_escaped = not is_escaped - quoted += rest[0:next_q + 1] - rest = rest[next_q + 1:] - if not is_escaped: - return (quoted, rest.lstrip()) + q = quoted = s[0] + rest = s[1:] + while True: + next_q = rest.find(q) + if next_q == -1: + raise ValueError("can't find ending quote '%s' in '%s'"% (q, s)) + # If quote is preceeded by even number of backslashes, + # then it is the ending quote, otherwise the quote + # character is escaped by backslash, so we should + # continue our search. + is_escaped = False + i = next_q - 1 + while i >= 0 and rest[i] == '\\': + i -= 1 + is_escaped = not is_escaped + quoted += rest[0:next_q + 1] + rest = rest[next_q + 1:] + if not is_escaped: + return (quoted, rest.lstrip()) def format_labels_string(header, labels): From 0dc45e421bf84bf4359cea89ecc324c446fe7139 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 8 Jan 2015 22:14:43 +0100 Subject: [PATCH 673/817] init.py: avoid redefining "type" builtin Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 143287b..6ceb860 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -257,10 +257,10 @@ class OfflineImap: options.singlethreading = True debugtypes = options.debugtype.split(',') + [''] - for type in debugtypes: - type = type.strip() - self.ui.add_debug(type) - if type.lower() == 'imap': + for dtype in debugtypes: + dtype = dtype.strip() + self.ui.add_debug(dtype) + if dtype.lower() == u'imap': imaplib.Debug = 5 if options.runonce: From 594a286888bf0ed8de7fe4a40f757a4cac56878b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 8 Jan 2015 22:28:45 +0100 Subject: [PATCH 674/817] repository/Maildir.py: use f variable instead of folder Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Maildir.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index f0495fa..9c244bf 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -130,11 +130,11 @@ class MaildirRepository(BaseRepository): name will return the same object.""" # getfolders() will scan and cache the values *if* necessary folders = self.getfolders() - for folder in folders: - if foldername == folder.name: - return folder + for f in folders: + if foldername == f.name: + return f raise OfflineImapError("getfolder() asked for a nonexisting " - "folder '%s'." % foldername, + "folder '%s'."% foldername, OfflineImapError.ERROR.FOLDER) def _getfolders_scandir(self, root, extension=None): From 0f40ca47998c7f56bf7cd1ce095c66fe93e21d03 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 8 Jan 2015 17:13:33 +0100 Subject: [PATCH 675/817] more style consistency Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 7 +- offlineimap/emailutil.py | 5 +- offlineimap/folder/Base.py | 134 +++++++++++++++----------- offlineimap/folder/IMAP.py | 128 ++++++++++++------------ offlineimap/imapserver.py | 121 +++++++++++------------ offlineimap/imaputil.py | 9 +- offlineimap/init.py | 23 ++--- offlineimap/localeval.py | 4 +- offlineimap/repository/Base.py | 1 + offlineimap/repository/LocalStatus.py | 9 +- offlineimap/repository/Maildir.py | 24 ++--- offlineimap/ui/TTY.py | 20 ++-- offlineimap/ui/UIBase.py | 8 +- offlineimap/ui/debuglock.py | 5 +- 14 files changed, 261 insertions(+), 237 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 190f6de..d9c9b88 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -321,18 +321,18 @@ class SyncableAccount(Account): if not remotefolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" - "[%s]" % (remotefolder, remoterepos)) + "[%s]"% (remotefolder, remoterepos)) continue # Ignore filtered folder localfolder = self.get_local_folder(remotefolder) if not localfolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" - "[%s]" % (localfolder, localfolder.repository)) + "[%s]"% (localfolder, localfolder.repository)) continue # Ignore filtered folder if not globals.options.singlethreading: thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, - name = "Folder %s [acc: %s]" % (remotefolder.getexplainedname(), self), + name = "Folder %s [acc: %s]"% (remotefolder.getexplainedname(), self), args = (self, remotefolder, quick)) thread.start() folderthreads.append(thread) @@ -385,6 +385,7 @@ def syncfolder(account, remotefolder, quick): """Synchronizes given remote folder for the specified account. Filtered folders on the remote side will not invoke this function.""" + remoterepos = account.remoterepos localrepos = account.localrepos statusrepos = account.statusrepos diff --git a/offlineimap/emailutil.py b/offlineimap/emailutil.py index 4aa34a8..0c732e2 100644 --- a/offlineimap/emailutil.py +++ b/offlineimap/emailutil.py @@ -19,13 +19,12 @@ import email from email.Parser import Parser as MailParser def get_message_date(content, header='Date'): - """ - Parses mail and returns resulting timestamp. + """Parses mail and returns resulting timestamp. :param header: the header to extract date from; :returns: timestamp or `None` in the case of failure. - """ + message = MailParser().parsestr(content, True) dateheader = message.get(header) # parsedate_tz returns a 10-tuple that can be passed to mktime_tz diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 2e1b95f..14262e3 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -15,14 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os.path +import re +from sys import exc_info + from offlineimap import threadutil, emailutil from offlineimap import globals from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError import offlineimap.accounts -import os.path -import re -from sys import exc_info class BaseFolder(object): @@ -31,6 +32,7 @@ class BaseFolder(object): :para name: Path & name of folder minus root or reference :para repository: Repository() in which the folder is. """ + self.ui = getglobalui() # Save original name for folderfilter operations self.ffilter_name = name @@ -56,15 +58,15 @@ class BaseFolder(object): # Determine if we're running static or dynamic folder filtering # and check filtering status - self._dynamic_folderfilter = \ - self.config.getdefaultboolean(repo, "dynamic_folderfilter", False) + self._dynamic_folderfilter = self.config.getdefaultboolean( + repo, "dynamic_folderfilter", False) self._sync_this = repository.should_sync_folder(self.ffilter_name) if self._dynamic_folderfilter: - self.ui.debug('', "Running dynamic folder filtering on '%s'[%s]" \ - % (self.ffilter_name, repository)) + self.ui.debug('', "Running dynamic folder filtering on '%s'[%s]"% + (self.ffilter_name, repository)) elif not self._sync_this: - self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter" \ - % (self.ffilter_name, repository)) + self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter"% + (self.ffilter_name, repository)) # Passes for syncmessagesto self.syncmessagesto_passes = [('copying messages' , self.__syncmessagesto_copy), @@ -115,17 +117,20 @@ class BaseFolder(object): :param statusfolder: keeps track of the last known folder state. """ + return True def getcopyinstancelimit(self): """For threading folders, returns the instancelimitname for InstanceLimitedThreads.""" + raise NotImplementedError def storesmessages(self): """Should be true for any backend that actually saves message bodies. (Almost all of them). False for the LocalStatus backend. Saves us from having to slurp up messages just for localstatus purposes.""" + return 1 def getvisiblename(self): @@ -143,14 +148,17 @@ class BaseFolder(object): def getrepository(self): """Returns the repository object that this folder is within.""" + return self.repository def getroot(self): """Returns the root of the folder, in a folder-specific fashion.""" + return self.root def getsep(self): """Returns the separator for this folder type.""" + return self.sep def getfullname(self): @@ -160,7 +168,8 @@ class BaseFolder(object): return self.getname() def getfolderbasename(self): - """Return base file name of file to store Status/UID info in""" + """Return base file name of file to store Status/UID info in.""" + if not self.name: basename = '.' else: #avoid directory hierarchies and file names such as '/' @@ -188,6 +197,7 @@ class BaseFolder(object): def _getuidfilename(self): """provides UIDVALIDITY cache filename for class internal purposes""" + return os.path.join(self.repository.getuiddir(), self.getfolderbasename()) @@ -196,6 +206,7 @@ class BaseFolder(object): :returns: UIDVALIDITY as (long) number or None, if None had been saved yet.""" + if hasattr(self, '_base_saved_uidvalidity'): return self._base_saved_uidvalidity uidfilename = self._getuidfilename() @@ -212,6 +223,7 @@ class BaseFolder(object): This function is not threadsafe, so don't attempt to call it from concurrent threads.""" + newval = self.get_uidvalidity() uidfilename = self._getuidfilename() @@ -225,45 +237,50 @@ class BaseFolder(object): This function needs to be implemented by each Backend :returns: UIDVALIDITY as a (long) number""" + raise NotImplementedError def cachemessagelist(self): """Reads the message list from disk or network and stores it in memory for later use. This list will not be re-read from disk or memory unless this function is called again.""" + raise NotImplementedError def getmessagelist(self): """Gets the current message list. You must call cachemessagelist() before calling this function!""" + raise NotImplementedError def msglist_item_initializer(self, uid): - """ - Returns value for empty messagelist element with given UID. + """Returns value for empty messagelist element with given UID. This function must initialize all fields of messagelist item and must be called every time when one creates new messagelist - entry to ensure that all fields that must be present are present. + entry to ensure that all fields that must be present are present.""" - """ raise NotImplementedError def uidexists(self, uid): """Returns True if uid exists""" + return uid in self.getmessagelist() def getmessageuidlist(self): """Gets a list of UIDs. You may have to call cachemessagelist() before calling this function!""" + return self.getmessagelist().keys() def getmessagecount(self): """Gets the number of messages.""" + return len(self.getmessagelist()) def getmessage(self, uid): """Returns the content of the specified message.""" + raise NotImplementedError def savemessage(self, uid, content, flags, rtime): @@ -286,20 +303,23 @@ class BaseFolder(object): Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a - dryrun mode. - """ + dryrun mode.""" + raise NotImplementedError def getmessagetime(self, uid): """Return the received time for the specified message.""" + raise NotImplementedError def getmessagemtime(self, uid): """Returns the message modification time of the specified message.""" + raise NotImplementedError def getmessageflags(self, uid): """Returns the flags for the specified message.""" + raise NotImplementedError def savemessageflags(self, uid, flags): @@ -308,6 +328,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + raise NotImplementedError def addmessageflags(self, uid, flags): @@ -319,14 +340,15 @@ class BaseFolder(object): dryrun mode. :param flags: A set() of flags""" + newflags = self.getmessageflags(uid) | flags self.savemessageflags(uid, newflags) def addmessagesflags(self, uidlist, flags): - """ - Note that this function does not check against dryrun settings, + """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + for uid in uidlist: self.addmessageflags(uid, flags) @@ -337,6 +359,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + newflags = self.getmessageflags(uid) - flags self.savemessageflags(uid, newflags) @@ -345,10 +368,10 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + for uid in uidlist: self.deletemessageflags(uid, flags) - def getmessagelabels(self, uid): """Returns the labels for the specified message.""" raise NotImplementedError @@ -359,6 +382,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + raise NotImplementedError def addmessagelabels(self, uid, labels): @@ -370,14 +394,15 @@ class BaseFolder(object): dryrun mode. :param labels: A set() of labels""" + newlabels = self.getmessagelabels(uid) | labels self.savemessagelabels(uid, newlabels) def addmessageslabels(self, uidlist, labels): - """ - Note that this function does not check against dryrun settings, + """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + for uid in uidlist: self.addmessagelabels(uid, labels) @@ -388,6 +413,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + newlabels = self.getmessagelabels(uid) - labels self.savemessagelabels(uid, newlabels) @@ -396,12 +422,12 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + for uid in uidlist: self.deletemessagelabels(uid, labels) def addmessageheader(self, content, linebreak, headername, headervalue): - """ - Adds new header to the provided message. + """Adds new header to the provided message. WARNING: This function is a bit tricky, and modifying it in the wrong way, may easily lead to data-loss. @@ -454,9 +480,9 @@ class BaseFolder(object): This is the body\n next line\n """ - self.ui.debug('', - 'addmessageheader: called to add %s: %s' % (headername, - headervalue)) + + self.ui.debug('', 'addmessageheader: called to add %s: %s'% + (headername, headervalue)) insertionpoint = content.find(linebreak * 2) if insertionpoint == -1: @@ -490,24 +516,23 @@ class BaseFolder(object): if content[0:len(linebreak)] != linebreak: suffix = suffix + linebreak - self.ui.debug('', 'addmessageheader: insertionpoint = %d' % insertionpoint) + self.ui.debug('', 'addmessageheader: insertionpoint = %d'% insertionpoint) headers = content[0:insertionpoint] - self.ui.debug('', 'addmessageheader: headers = %s' % repr(headers)) + self.ui.debug('', 'addmessageheader: headers = %s'% repr(headers)) new_header = prefix + ("%s: %s" % (headername, headervalue)) + suffix self.ui.debug('', 'addmessageheader: new_header = ' + repr(new_header)) return headers + new_header + content[insertionpoint:] def __find_eoh(self, content): - """ - Searches for the point where mail headers end. + """ Searches for the point where mail headers end. Either double '\n', or end of string. Arguments: - content: contents of the message to search in Returns: position of the first non-header byte. - """ + eoh_cr = content.find('\n\n') if eoh_cr == -1: eoh_cr = len(content) @@ -516,8 +541,7 @@ class BaseFolder(object): def getmessageheader(self, content, name): - """ - Searches for the first occurence of the given header and returns + """Searches for the first occurence of the given header and returns its value. Header name is case-insensitive. Arguments: @@ -525,13 +549,13 @@ class BaseFolder(object): - name: name of the header to be searched Returns: header value or None if no such header was found - """ - self.ui.debug('', 'getmessageheader: called to get %s' % name) + + self.ui.debug('', 'getmessageheader: called to get %s'% name) eoh = self.__find_eoh(content) - self.ui.debug('', 'getmessageheader: eoh = %d' % eoh) + self.ui.debug('', 'getmessageheader: eoh = %d'% eoh) headers = content[0:eoh] - self.ui.debug('', 'getmessageheader: headers = %s' % repr(headers)) + self.ui.debug('', 'getmessageheader: headers = %s'% repr(headers)) m = re.search('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE) if m: @@ -541,8 +565,7 @@ class BaseFolder(object): def getmessageheaderlist(self, content, name): - """ - Searches for the given header and returns a list of values for + """Searches for the given header and returns a list of values for that header. Arguments: @@ -550,8 +573,8 @@ class BaseFolder(object): - name: name of the header to be searched Returns: list of header values or emptylist if no such header was found - """ + self.ui.debug('', 'getmessageheaderlist: called to get %s' % name) eoh = self.__find_eoh(content) self.ui.debug('', 'getmessageheaderlist: eoh = %d' % eoh) @@ -562,27 +585,26 @@ class BaseFolder(object): def deletemessageheaders(self, content, header_list): - """ - Deletes headers in the given list from the message content. + """Deletes headers in the given list from the message content. Arguments: - content: message itself - header_list: list of headers to be deleted or just the header name We expect our message to have '\n' as line endings. - """ + if type(header_list) != type([]): header_list = [header_list] - self.ui.debug('', 'deletemessageheaders: called to delete %s' % (header_list)) + self.ui.debug('', 'deletemessageheaders: called to delete %s'% (header_list)) if not len(header_list): return content eoh = self.__find_eoh(content) - self.ui.debug('', 'deletemessageheaders: end of headers = %d' % eoh) + self.ui.debug('', 'deletemessageheaders: end of headers = %d'% eoh) headers = content[0:eoh] rest = content[eoh:] - self.ui.debug('', 'deletemessageheaders: headers = %s' % repr(headers)) + self.ui.debug('', 'deletemessageheaders: headers = %s'% repr(headers)) new_headers = [] for h in headers.split('\n'): keep_it = True @@ -609,16 +631,14 @@ class BaseFolder(object): raise NotImplementedError def deletemessage(self, uid): - """ - Note that this function does not check against dryrun settings, + """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" raise NotImplementedError def deletemessages(self, uidlist): - """ - Note that this function does not check against dryrun settings, + """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" @@ -686,9 +706,8 @@ class BaseFolder(object): self.deletemessage(uid) else: raise OfflineImapError("Trying to save msg (uid %d) on folder " - "%s returned invalid uid %d" % (uid, - dstfolder.getvisiblename(), new_uid), - OfflineImapError.ERROR.MESSAGE) + "%s returned invalid uid %d"% (uid, dstfolder.getvisiblename(), + new_uid), OfflineImapError.ERROR.MESSAGE) except (KeyboardInterrupt): # bubble up CTRL-C raise except OfflineImapError as e: @@ -697,8 +716,7 @@ class BaseFolder(object): self.ui.error(e, exc_info()[2]) except Exception as e: self.ui.error(e, exc_info()[2], - msg="Copying message %s [acc: %s]" %\ - (uid, self.accountname)) + msg = "Copying message %s [acc: %s]"% (uid, self.accountname)) raise #raise on unknown errors, so we can fix those def __syncmessagesto_copy(self, dstfolder, statusfolder): @@ -714,6 +732,7 @@ class BaseFolder(object): This function checks and protects us from action in ryrun mode. """ + threads = [] copylist = filter(lambda uid: not \ @@ -754,6 +773,7 @@ class BaseFolder(object): This function checks and protects us from action in ryrun mode. """ + deletelist = filter(lambda uid: uid>=0 \ and not self.uidexists(uid), statusfolder.getmessageuidlist()) @@ -776,6 +796,7 @@ class BaseFolder(object): This function checks and protects us from action in ryrun mode. """ + # For each flag, we store a list of uids to which it should be # added. Then, we can call addmessagesflags() to apply them in # bulk, rather than one call per message. @@ -854,8 +875,8 @@ class BaseFolder(object): :param dstfolder: Folderinstance to sync the msgs to. :param statusfolder: LocalStatus instance to sync against. - """ + for (passdesc, action) in self.syncmessagesto_passes: # bail out on CTRL-C or SIGTERM if offlineimap.accounts.Account.abort_NOW_signal.is_set(): @@ -883,6 +904,7 @@ class BaseFolder(object): MailDirFolder('foo') == IMAPFolder('foo') --> False MailDirFolder('foo') == MaildirFolder('foo') --> False """ + if isinstance(other, basestring): return other == self.name return id(self) == id(other) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index a698888..52493fc 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -20,6 +20,7 @@ import binascii import re import time from sys import exc_info + from .Base import BaseFolder from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError from offlineimap import globals @@ -54,7 +55,7 @@ class IMAPFolder(BaseFolder): self.filterheaders = [h for h in re.split(r'\s*,\s*', fh_conf) if h] - def __selectro(self, imapobj, force = False): + def __selectro(self, imapobj, force=False): """Select this folder when we do not need write access. Prefer SELECT to EXAMINE if we can, since some servers @@ -86,6 +87,7 @@ class IMAPFolder(BaseFolder): UIDVALIDITY value will be cached on the first call. :returns: The UIDVALIDITY as (long) number.""" + if hasattr(self, '_uidvalidity'): # use cached value if existing return self._uidvalidity @@ -141,8 +143,7 @@ class IMAPFolder(BaseFolder): def _msgs_to_fetch(self, imapobj): - """ - Determines sequence numbers of messages to be fetched. + """Determines sequence numbers of messages to be fetched. Message sequence numbers (MSNs) are more easily compacted into ranges which makes transactions slightly faster. @@ -151,9 +152,8 @@ class IMAPFolder(BaseFolder): - imapobj: instance of IMAPlib Returns: range(s) for messages or None if no messages - are to be fetched. + are to be fetched.""" - """ res_type, imapdata = imapobj.select(self.getfullname(), True, True) if imapdata == [None] or imapdata[0] == '0': # Empty folder, no need to populate message list @@ -162,9 +162,9 @@ class IMAPFolder(BaseFolder): # By default examine all messages in this folder msgsToFetch = '1:*' - maxage = self.config.getdefaultint("Account %s" % self.accountname, + maxage = self.config.getdefaultint("Account %s"% self.accountname, "maxage", -1) - maxsize = self.config.getdefaultint("Account %s" % self.accountname, + maxsize = self.config.getdefaultint("Account %s"% self.accountname, "maxsize", -1) # Build search condition @@ -193,7 +193,7 @@ class IMAPFolder(BaseFolder): res_type, res_data = imapobj.search(None, search_cond) if res_type != 'OK': raise OfflineImapError("SEARCH in folder [%s]%s failed. " - "Search string was '%s'. Server responded '[%s] %s'" % ( + "Search string was '%s'. Server responded '[%s] %s'"% ( self.getrepository(), self, search_cond, res_type, res_data), OfflineImapError.ERROR.FOLDER) @@ -219,11 +219,11 @@ class IMAPFolder(BaseFolder): # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. - res_type, response = imapobj.fetch("'%s'" % msgsToFetch, - '(FLAGS UID)') + res_type, response = imapobj.fetch("'%s'"% + msgsToFetch, '(FLAGS UID)') if res_type != 'OK': raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " - "Server responded '[%s] %s'" % ( + "Server responded '[%s] %s'"% ( self.getrepository(), self, res_type, response), OfflineImapError.ERROR.FOLDER) @@ -238,7 +238,7 @@ class IMAPFolder(BaseFolder): messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not 'UID' in options: - self.ui.warn('No UID in message with options %s' %\ + self.ui.warn('No UID in message with options %s'% \ str(options), minor = 1) else: @@ -255,16 +255,15 @@ class IMAPFolder(BaseFolder): # Interface from BaseFolder def getmessage(self, uid): - """ - Retrieve message with UID from the IMAP server (incl body) + """Retrieve message with UID from the IMAP server (incl body) After this function all CRLFs will be transformed to '\n'. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with this UID could be found. - """ + imapobj = self.imapserver.acquireconnection() try: data = self._fetch_from_imap(imapobj, str(uid), 2) @@ -281,7 +280,7 @@ class IMAPFolder(BaseFolder): else: dbg_output = data - self.ui.debug('imap', "Returned object from fetching %d: '%s'" % + self.ui.debug('imap', "Returned object from fetching %d: '%s'"% (uid, dbg_output)) return data @@ -307,6 +306,7 @@ class IMAPFolder(BaseFolder): headername == 'X-OfflineIMAP' and headervalue will be a random string """ + headername = 'X-OfflineIMAP' # We need a random component too. If we ever upload the same # mail twice (e.g. in different folders), we would still need to @@ -322,20 +322,20 @@ class IMAPFolder(BaseFolder): def __savemessage_searchforheader(self, imapobj, headername, headervalue): - self.ui.debug('imap', '__savemessage_searchforheader called for %s: %s' % \ - (headername, headervalue)) + self.ui.debug('imap', '__savemessage_searchforheader called for %s: %s'% \ + (headername, headervalue)) # Now find the UID it got. headervalue = imapobj._quote(headervalue) try: matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0] except imapobj.error as err: # IMAP server doesn't implement search or had a problem. - self.ui.debug('imap', "__savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername)) + self.ui.debug('imap', "__savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s"% (err, headername)) return 0 self.ui.debug('imap', '__savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids)) if matchinguids == '': - self.ui.debug('imap', "__savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername) + self.ui.debug('imap', "__savemessage_searchforheader: UID SEARCH for message with header %s yielded no results"% headername) return 0 matchinguids = matchinguids.split(' ') @@ -343,7 +343,7 @@ class IMAPFolder(BaseFolder): repr(matchinguids)) if len(matchinguids) != 1 or matchinguids[0] == None: raise ValueError("While attempting to find UID for message with " - "header %s, got wrong-sized matchinguids of %s" %\ + "header %s, got wrong-sized matchinguids of %s"%\ (headername, str(matchinguids))) return long(matchinguids[0]) @@ -368,9 +368,9 @@ class IMAPFolder(BaseFolder): We need to locate the UID just after mail headers containing our X-OfflineIMAP line. - Returns UID when found, 0 when not found. - """ - self.ui.debug('imap', '__savemessage_fetchheaders called for %s: %s' % \ + Returns UID when found, 0 when not found.""" + + self.ui.debug('imap', '__savemessage_fetchheaders called for %s: %s'% \ (headername, headervalue)) # run "fetch X:* rfc822.header" @@ -381,7 +381,7 @@ class IMAPFolder(BaseFolder): # ascending. if self.getmessagelist(): - start = 1+max(self.getmessagelist().keys()) + start = 1 + max(self.getmessagelist().keys()) else: # Folder was empty - start from 1 start = 1 @@ -390,7 +390,7 @@ class IMAPFolder(BaseFolder): # with the range X:*. So we use bytearray to stop imaplib from getting # in our way - result = imapobj.uid('FETCH', bytearray('%d:*' % start), 'rfc822.header') + result = imapobj.uid('FETCH', bytearray('%d:*'% start), 'rfc822.header') if result[0] != 'OK': raise OfflineImapError('Error fetching mail headers: ' + '. '.join(result[1]), OfflineImapError.ERROR.MESSAGE) @@ -401,7 +401,7 @@ class IMAPFolder(BaseFolder): for item in result: if found == 0 and type(item) == type( () ): # Walk just tuples - if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)" % (headername, headervalue), + if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)"% (headername, headervalue), item[1], flags=re.IGNORECASE): found = 1 elif found == 1: @@ -467,8 +467,8 @@ class IMAPFolder(BaseFolder): # or something. Argh. It seems that Time2Internaldate # will rause a ValueError if the year is 0102 but not 1902, # but some IMAP servers nonetheless choke on 1902. - self.ui.debug('imap', "Message with invalid date %s. Server will use local time." \ - % datetuple) + self.ui.debug('imap', "Message with invalid date %s. " + "Server will use local time."% datetuple) return None #produce a string representation of datetuple that works as @@ -507,6 +507,7 @@ class IMAPFolder(BaseFolder): message is saved, but it's UID can not be found, it will return 0. If the message can't be written (folder is read-only for example) it will return -1.""" + self.ui.savemessage('imap', uid, flags, self) # already have it, just save modified flags @@ -543,17 +544,17 @@ class IMAPFolder(BaseFolder): if not use_uidplus: # insert a random unique header that we can fetch later (headername, headervalue) = self.__generate_randomheader( - content) - self.ui.debug('imap', 'savemessage: header is: %s: %s' %\ - (headername, headervalue)) + content) + self.ui.debug('imap', 'savemessage: header is: %s: %s'% + (headername, headervalue)) content = self.addmessageheader(content, CRLF, headername, headervalue) if len(content)>200: dbg_output = "%s...%s" % (content[:150], content[-50:]) else: dbg_output = content - self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % - (date, dbg_output)) + self.ui.debug('imap', "savemessage: date: %s, content: '%s'"% + (date, dbg_output)) try: # Select folder for append and make the box READ-WRITE @@ -566,9 +567,8 @@ class IMAPFolder(BaseFolder): #Do the APPEND try: - (typ, dat) = imapobj.append(self.getfullname(), - imaputil.flagsmaildir2imap(flags), - date, content) + (typ, dat) = imapobj.append(fullname, + imaputil.flagsmaildir2imap(flags), date, content) # This should only catch 'NO' responses since append() # will raise an exception for 'BAD' responses: if typ != 'OK': @@ -580,7 +580,7 @@ class IMAPFolder(BaseFolder): # and continue with the next account. msg = \ "Saving msg (%s) in folder '%s', repository '%s' failed (abort). " \ - "Server responded: %s %s\n" % \ + "Server responded: %s %s\n"% \ (msg_id, self, self.getrepository(), typ, dat) raise OfflineImapError(msg, OfflineImapError.ERROR.REPO) retry_left = 0 # Mark as success @@ -592,7 +592,7 @@ class IMAPFolder(BaseFolder): if not retry_left: raise OfflineImapError("Saving msg (%s) in folder '%s', " "repository '%s' failed (abort). Server responded: %s\n" - "Message content was: %s" % + "Message content was: %s"% (msg_id, self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) self.ui.error(e, exc_info()[2]) @@ -604,8 +604,8 @@ class IMAPFolder(BaseFolder): imapobj = None raise OfflineImapError("Saving msg (%s) folder '%s', repo '%s'" "failed (error). Server responded: %s\nMessage content was: " - "%s" % (msg_id, self, self.getrepository(), str(e), dbg_output), - OfflineImapError.ERROR.MESSAGE) + "%s"% (msg_id, self, self.getrepository(), str(e), dbg_output), + OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. (typ,dat) = imapobj.check() @@ -622,24 +622,24 @@ class IMAPFolder(BaseFolder): resp = imapobj._get_untagged_response('APPENDUID') if resp == [None] or resp is None: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " - "appending a message.") + "appending a message.") return 0 uid = long(resp[-1].split(' ')[1]) if uid == 0: self.ui.warn("savemessage: Server supports UIDPLUS, but" - " we got no usable uid back. APPENDUID reponse was " - "'%s'" % str(resp)) + " we got no usable uid back. APPENDUID reponse was " + "'%s'"% str(resp)) else: # we don't support UIDPLUS uid = self.__savemessage_searchforheader(imapobj, headername, - headervalue) + headervalue) # See docs for savemessage in Base.py for explanation # of this and other return values if uid == 0: self.ui.debug('imap', 'savemessage: attempt to get new UID ' 'UID failed. Search headers manually.') uid = self.__savemessage_fetchheaders(imapobj, headername, - headervalue) + headervalue) self.ui.warn('imap', "savemessage: Searching mails for new " "Message-ID failed. Could not determine new UID.") finally: @@ -649,22 +649,21 @@ class IMAPFolder(BaseFolder): self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags - self.ui.debug('imap', 'savemessage: returning new UID %d' % uid) + self.ui.debug('imap', 'savemessage: returning new UID %d'% uid) return uid def _fetch_from_imap(self, imapobj, uids, retry_num=1): - """ - Fetches data from IMAP server. + """Fetches data from IMAP server. Arguments: - imapobj: IMAPlib object - uids: message UIDS - retry_num: number of retries to make - Returns: data obtained by this query. - """ - query = "(%s)" % (" ".join(self.imap_query)) + Returns: data obtained by this query.""" + + query = "(%s)"% (" ".join(self.imap_query)) fails_left = retry_num # retry on dropped connection while fails_left: try: @@ -683,7 +682,7 @@ class IMAPFolder(BaseFolder): #IMAP server says bad request or UID does not exist severity = OfflineImapError.ERROR.MESSAGE reason = "IMAP server '%s' failed to fetch messages UID '%s'."\ - "Server responded: %s %s" % (self.getrepository(), uids, + "Server responded: %s %s"% (self.getrepository(), uids, res_type, data) if data == [None]: #IMAP server did not find a message with this UID @@ -695,23 +694,21 @@ class IMAPFolder(BaseFolder): def _store_to_imap(self, imapobj, uid, field, data): - """ - Stores data to IMAP server + """Stores data to IMAP server Arguments: - imapobj: instance of IMAPlib to use - uid: message UID - field: field name to be stored/updated - data: field contents - """ imapobj.select(self.getfullname()) res_type, retdata = imapobj.uid('store', uid, field, data) if res_type != 'OK': severity = OfflineImapError.ERROR.MESSAGE reason = "IMAP server '%s' failed to store %s for message UID '%d'."\ - "Server responded: %s %s" % (self.getrepository(), field, uid, - res_type, retdata) + "Server responded: %s %s"% ( + self.getrepository(), field, uid, res_type, retdata) raise OfflineImapError(reason, severity) return retdata[0] @@ -724,12 +721,11 @@ class IMAPFolder(BaseFolder): dryrun mode.""" imapobj = self.imapserver.acquireconnection() try: - result = self._store_to_imap(imapobj, str(uid), 'FLAGS', imaputil.flagsmaildir2imap(flags)) - + result = self._store_to_imap(imapobj, str(uid), 'FLAGS', + imaputil.flagsmaildir2imap(flags)) except imapobj.readonly: self.ui.flagstoreadonly(self, [uid], flags) return - finally: self.imapserver.releaseconnection(imapobj) @@ -751,6 +747,7 @@ class IMAPFolder(BaseFolder): """This is here for the sake of UIDMaps.py -- deletemessages must add flags and get a converted UID, and if we don't have noconvert, then UIDMaps will try to convert it twice.""" + self.__addmessagesflags_noconvert(uidlist, flags) # Interface from BaseFolder @@ -770,9 +767,8 @@ class IMAPFolder(BaseFolder): self.ui.flagstoreadonly(self, uidlist, flags) return r = imapobj.uid('store', - imaputil.uid_sequence(uidlist), - operation + 'FLAGS', - imaputil.flagsmaildir2imap(flags)) + imaputil.uid_sequence(uidlist), operation + 'FLAGS', + imaputil.flagsmaildir2imap(flags)) assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) r = r[1] finally: @@ -818,9 +814,9 @@ class IMAPFolder(BaseFolder): """Change the message from existing uid to new_uid If the backend supports it. IMAP does not and will throw errors.""" + raise OfflineImapError('IMAP backend cannot change a messages UID from ' - '%d to %d' % (uid, new_uid), - OfflineImapError.ERROR.MESSAGE) + '%d to %d'% (uid, new_uid), OfflineImapError.ERROR.MESSAGE) # Interface from BaseFolder def deletemessage(self, uid): diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 844012e..5601033 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -56,7 +56,7 @@ class IMAPServer: self.preauth_tunnel = repos.getpreauthtunnel() self.transport_tunnel = repos.gettransporttunnel() if self.preauth_tunnel and self.transport_tunnel: - raise OfflineImapError('%s: ' % repos + \ + raise OfflineImapError('%s: '% repos + \ 'you must enable precisely one ' 'type of tunnel (preauth or transport), ' 'not both', OfflineImapError.ERROR.REPO) @@ -116,8 +116,9 @@ class IMAPServer: # XXX: is this function used anywhere? def getroot(self): - """Returns this server's folder root. Can only be called after one + """Returns this server's folder root. Can only be called after one or more calls to acquireconnection.""" + return self.root @@ -126,6 +127,7 @@ class IMAPServer: :param drop_conn: If True, the connection will be released and not be reused. This can be used to indicate broken connections.""" + if connection is None: return #noop on bad connection self.connectionlock.acquire() self.assignedconnections.remove(connection) @@ -139,25 +141,24 @@ class IMAPServer: def __md5handler(self, response): challenge = response.strip() - self.ui.debug('imap', '__md5handler: got challenge %s' % challenge) + self.ui.debug('imap', '__md5handler: got challenge %s'% challenge) passwd = self.__getpassword() retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest() - self.ui.debug('imap', '__md5handler: returning %s' % retval) + self.ui.debug('imap', '__md5handler: returning %s'% retval) return retval def __loginauth(self, imapobj): - """ Basic authentication via LOGIN command """ + """ Basic authentication via LOGIN command.""" + self.ui.debug('imap', 'Attempting IMAP LOGIN authentication') imapobj.login(self.username, self.__getpassword()) def __plainhandler(self, response): - """ - Implements SASL PLAIN authentication, RFC 4616, - http://tools.ietf.org/html/rfc4616 + """Implements SASL PLAIN authentication, RFC 4616, + http://tools.ietf.org/html/rfc4616""" - """ authc = self.username passwd = self.__getpassword() authz = '' @@ -175,8 +176,8 @@ class IMAPServer: try: if self.gss_step == self.GSS_STATE_STEP: if not self.gss_vc: - rc, self.gss_vc = kerberos.authGSSClientInit('imap@' + - self.hostname) + rc, self.gss_vc = kerberos.authGSSClientInit( + 'imap@' + self.hostname) response = kerberos.authGSSClientResponse(self.gss_vc) rc = kerberos.authGSSClientStep(self.gss_vc, data) if rc != kerberos.AUTH_GSS_CONTINUE: @@ -184,13 +185,13 @@ class IMAPServer: elif self.gss_step == self.GSS_STATE_WRAP: rc = kerberos.authGSSClientUnwrap(self.gss_vc, data) response = kerberos.authGSSClientResponse(self.gss_vc) - rc = kerberos.authGSSClientWrap(self.gss_vc, response, - self.username) + rc = kerberos.authGSSClientWrap( + self.gss_vc, response, self.username) response = kerberos.authGSSClientResponse(self.gss_vc) except kerberos.GSSError as err: # Kerberos errored out on us, respond with None to cancel the # authentication - self.ui.debug('imap', '%s: %s' % (err[0][0], err[1][0])) + self.ui.debug('imap', '%s: %s'% (err[0][0], err[1][0])) return None if not response: @@ -205,7 +206,7 @@ class IMAPServer: imapobj.starttls() except imapobj.error as e: raise OfflineImapError("Failed to start " - "TLS connection: %s" % str(e), + "TLS connection: %s"% str(e), OfflineImapError.ERROR.REPO) @@ -266,8 +267,7 @@ class IMAPServer: def __authn_helper(self, imapobj): - """ - Authentication machinery for self.acquireconnection(). + """Authentication machinery for self.acquireconnection(). Raises OfflineImapError() of type ERROR.REPO when there are either fatal problems or no authentications @@ -275,9 +275,7 @@ class IMAPServer: If any authentication method succeeds, routine should exit: warnings for failed methods are to be produced in the - respective except blocks. - - """ + respective except blocks.""" # Authentication routines, hash keyed by method name # with value that is a tuple with @@ -321,13 +319,13 @@ class IMAPServer: continue tried_to_authn = True - self.ui.debug('imap', 'Attempting ' - '%s authentication' % m) + self.ui.debug('imap', u'Attempting ' + '%s authentication'% m) try: if func(imapobj): return except (imapobj.error, OfflineImapError) as e: - self.ui.warn('%s authentication failed: %s' % (m, e)) + self.ui.warn('%s authentication failed: %s'% (m, e)) exc_stack.append((m, e)) if len(exc_stack): @@ -343,9 +341,9 @@ class IMAPServer: lambda x: x[5:], filter(lambda x: x[0:5] == "AUTH=", imapobj.capabilities) )) - raise OfflineImapError("Repository %s: no supported " + raise OfflineImapError(u"Repository %s: no supported " "authentication mechanisms found; configured %s, " - "server advertises %s" % (self.repos, + "server advertises %s"% (self.repos, ", ".join(self.authmechs), methods), OfflineImapError.ERROR.REPO) @@ -383,9 +381,8 @@ class IMAPServer: self.connectionlock.release() # Release until need to modify data - """ Must be careful here that if we fail we should bail out gracefully - and release locks / threads so that the next attempt can try... - """ + # Must be careful here that if we fail we should bail out gracefully + # and release locks / threads so that the next attempt can try... success = 0 try: while not success: @@ -441,7 +438,7 @@ class IMAPServer: # No Folders were returned. This occurs, e.g. if the # 'reference' prefix does not exist on the mail # server. Raise exception. - err = "Server '%s' returned no folders in '%s'" % \ + err = "Server '%s' returned no folders in '%s'"% \ (self.repos.getname(), self.reference) self.ui.warn(err) raise Exception(err) @@ -458,6 +455,7 @@ class IMAPServer: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" + self.semaphore.release() severity = OfflineImapError.ERROR.REPO @@ -489,7 +487,7 @@ class IMAPServer: reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ - "network." % (self.hostname, self.port, self.repos) + "network."% (self.hostname, self.port, self.repos) raise OfflineImapError(reason, severity) # Could not acquire connection to the remote; # socket.error(last_error) raised @@ -503,13 +501,15 @@ class IMAPServer: raise def connectionwait(self): - """Waits until there is a connection available. Note that between - the time that a connection becomes available and the time it is - requested, another thread may have grabbed it. This function is - mainly present as a way to avoid spawning thousands of threads - to copy messages, then have them all wait for 3 available connections. - It's OK if we have maxconnections + 1 or 2 threads, which is what - this will help us do.""" + """Waits until there is a connection available. + + Note that between the time that a connection becomes available and the + time it is requested, another thread may have grabbed it. This function + is mainly present as a way to avoid spawning thousands of threads to + copy messages, then have them all wait for 3 available connections. + It's OK if we have maxconnections + 1 or 2 threads, which is what this + will help us do.""" + self.semaphore.acquire() self.semaphore.release() @@ -533,11 +533,13 @@ class IMAPServer: self.gssapi = False def keepalive(self, timeout, event): - """Sends a NOOP to each connection recorded. It will wait a maximum - of timeout seconds between doing this, and will continue to do so - until the Event object as passed is true. This method is expected - to be invoked in a separate thread, which should be join()'d after - the event is set.""" + """Sends a NOOP to each connection recorded. + + It will wait a maximum of timeout seconds between doing this, and will + continue to do so until the Event object as passed is true. This method + is expected to be invoked in a separate thread, which should be join()'d + after the event is set.""" + self.ui.debug('imap', 'keepalive thread started') while not event.isSet(): self.connectionlock.acquire() @@ -547,7 +549,7 @@ class IMAPServer: threads = [] for i in range(numconnections): - self.ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections)) + self.ui.debug('imap', 'keepalive: processing connection %d of %d'% (i, numconnections)) if len(self.idlefolders) > i: # IDLE thread idler = IdleThread(self, self.idlefolders[i]) @@ -570,14 +572,14 @@ class IMAPServer: return def __verifycert(self, cert, hostname): - '''Verify that cert (in socket.getpeercert() format) matches hostname. - CRLs are not handled. + """Verify that cert (in socket.getpeercert() format) matches hostname. + + CRLs are not handled. + Returns error message if any problems are found and None on success.""" - Returns error message if any problems are found and None on success. - ''' errstr = "CA Cert verifying failed: " if not cert: - return ('%s no certificate received' % errstr) + return ('%s no certificate received'% errstr) dnsname = hostname.lower() certnames = [] @@ -585,7 +587,7 @@ class IMAPServer: notafter = cert.get('notAfter') if notafter: if time.time() >= cert_time_to_seconds(notafter): - return '%s certificate expired %s' % (errstr, notafter) + return '%s certificate expired %s'% (errstr, notafter) # First read commonName for s in cert.get('subject', []): @@ -593,7 +595,7 @@ class IMAPServer: if key == 'commonName': certnames.append(value.lower()) if len(certnames) == 0: - return ('%s no commonName found in certificate' % errstr) + return ('%s no commonName found in certificate'% errstr) # Then read subjectAltName for key, value in cert.get('subjectAltName', []): @@ -606,7 +608,7 @@ class IMAPServer: '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): return None - return ('%s no matching domain name found in certificate' % errstr) + return ('%s no matching domain name found in certificate'% errstr) class IdleThread(object): @@ -614,6 +616,7 @@ class IdleThread(object): """If invoked without 'folder', perform a NOOP and wait for self.stop() to be called. If invoked with folder, switch to IDLE mode and synchronize once we have a new message""" + self.parent = parent self.folder = folder self.stop_sig = Event() @@ -634,18 +637,18 @@ class IdleThread(object): self.thread.join() def noop(self): - #TODO: AFAIK this is not optimal, we will send a NOOP on one - #random connection (ie not enough to keep all connections - #open). In case we do the noop multiple times, we can well use - #the same connection every time, as we get a random one. This - #function should IMHO send a noop on ALL available connections - #to the server. + # TODO: AFAIK this is not optimal, we will send a NOOP on one + # random connection (ie not enough to keep all connections + # open). In case we do the noop multiple times, we can well use + # the same connection every time, as we get a random one. This + # function should IMHO send a noop on ALL available connections + # to the server. imapobj = self.parent.acquireconnection() try: imapobj.noop() except imapobj.abort: - self.ui.warn('Attempting NOOP on dropped connection %s' % \ - imapobj.identifier) + self.ui.warn('Attempting NOOP on dropped connection %s'% + imapobj.identifier) self.parent.releaseconnection(imapobj, True) imapobj = None finally: diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 9623589..e5eb541 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -121,8 +121,7 @@ def imapsplit(imapstring): arg = arg.replace('\\', '\\\\') arg = arg.replace('"', '\\"') arg = '"%s"' % arg - __debug("imapsplit() non-string [%d]: Appending %s" %\ - (i, arg)) + __debug("imapsplit() non-string [%d]: Appending %s"% (i, arg)) retval.append(arg) else: # Even -- we have a string that ends with a literal @@ -131,8 +130,8 @@ def imapsplit(imapstring): # Recursion to the rescue. arg = imapstring[i] arg = re.sub('\{\d+\}$', '', arg) - __debug("imapsplit() non-string [%d]: Feeding %s to recursion" %\ - (i, arg)) + __debug("imapsplit() non-string [%d]: Feeding %s to recursion"%\ + (i, arg)) retval.extend(imapsplit(arg)) __debug("imapsplit() non-string: returning %s" % str(retval)) return retval @@ -274,7 +273,7 @@ def __split_quoted(s): def format_labels_string(header, labels): """Formats labels for embedding into a message, with format according to header name. - + Headers from SPACE_SEPARATED_LABEL_HEADERS keep space-separated list of labels, the rest uses comma (',') as the separator. diff --git a/offlineimap/init.py b/offlineimap/init.py index 6ceb860..878f5cf 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -23,6 +23,7 @@ import signal import socket import logging from optparse import OptionParser + import offlineimap from offlineimap import accounts, threadutil, syncmaster from offlineimap import globals @@ -180,7 +181,7 @@ class OfflineImap: config = CustomConfigParser() if not os.path.exists(configfilename): # TODO, initialize and make use of chosen ui for logging - logging.error(" *** Config file '%s' does not exist; aborting!" % + logging.error(" *** Config file '%s' does not exist; aborting!"% configfilename) sys.exit(1) config.read(configfilename) @@ -193,14 +194,14 @@ class OfflineImap: options.singlethreading = True if os.path.exists(options.profiledir): # TODO, make use of chosen ui for logging - logging.warn("Profile mode: Directory '%s' already exists!" % + logging.warn("Profile mode: Directory '%s' already exists!"% options.profiledir) else: os.mkdir(options.profiledir) threadutil.ExitNotifyThread.set_profiledir(options.profiledir) # TODO, make use of chosen ui for logging logging.warn("Profile mode: Potentially large data will be " - "created in '%s'" % options.profiledir) + "created in '%s'"% options.profiledir) #override a config value if options.configoverride: @@ -234,8 +235,8 @@ class OfflineImap: # create the ui class self.ui = UI_LIST[ui_type.lower()](config) except KeyError: - logging.error("UI '%s' does not exist, choose one of: %s" % \ - (ui_type,', '.join(UI_LIST.keys()))) + logging.error("UI '%s' does not exist, choose one of: %s"% \ + (ui_type, ', '.join(UI_LIST.keys()))) sys.exit(1) setglobalui(self.ui) @@ -331,13 +332,13 @@ class OfflineImap: for account in activeaccounts: if account not in allaccounts: if len(allaccounts) == 0: - errormsg = "The account '%s' does not exist because no"\ - " accounts are defined!" % account + errormsg = "The account '%s' does not exist because no" \ + " accounts are defined!"% account else: - errormsg = "The account '%s' does not exist. Valid ac"\ - "counts are: " % account - errormsg += ", ".join(allaccounts.keys()) - self.ui.terminate(1, errormsg = errormsg) + errormsg = "The account '%s' does not exist. Valid ac" \ + "counts are: %s"% \ + (account, ", ".join(allaccounts.keys())) + self.ui.terminate(1, errormsg=errormsg) if account not in syncaccounts: syncaccounts.append(account) diff --git a/offlineimap/localeval.py b/offlineimap/localeval.py index e7d656f..06adc57 100644 --- a/offlineimap/localeval.py +++ b/offlineimap/localeval.py @@ -23,9 +23,7 @@ except: pass class LocalEval: - """Here is a powerfull but very dangerous option, of course. - - Assume source file to be ASCII encoded.""" + """Here is a powerfull but very dangerous option, of course.""" def __init__(self, path=None): self.namespace = {} diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 4edd6c5..cb9d06a 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -167,6 +167,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): that forward and backward nametrans actually match up! Configuring nametrans on BOTH repositories therefore could lead to infinite folder creation cycles.""" + if not self.get_create_folders() and not dst_repo.get_create_folders(): # quick exit if no folder creation is enabled on either side. return diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 52ba714..1390347 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -53,8 +53,8 @@ class LocalStatusRepository(BaseRepository): self.LocalStatusFolderClass = self.backends[backend]['class'] else: - raise SyntaxWarning("Unknown status_backend '%s' for account '%s'" \ - % (backend, self.account.name)) + raise SyntaxWarning("Unknown status_backend '%s' for account '%s'"% + (backend, self.account.name)) def import_other_backend(self, folder): for bk, dic in self.backends.items(): @@ -68,8 +68,9 @@ class LocalStatusRepository(BaseRepository): # if backend contains data, import it to folder. if not folderbk.isnewfolder(): - self.ui._msg('Migrating LocalStatus cache from %s to %s ' % (bk, self._backend) + \ - 'status folder for %s:%s' % (self.name, folder.name)) + self.ui._msg('Migrating LocalStatus cache from %s to %s " \ + "status folder for %s:%s'% + (bk, self._backend, self.name, folder.name)) folderbk.cachemessagelist() folder.messagelist = folderbk.messagelist diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 9c244bf..be86b34 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -32,7 +32,7 @@ class MaildirRepository(BaseRepository): self.root = self.getlocalroot() self.folders = None self.ui = getglobalui() - self.debug("MaildirRepository initialized, sep is " + repr(self.getsep())) + self.debug("MaildirRepository initialized, sep is %s"% repr(self.getsep())) self.folder_atimes = [] # Create the top-level folder if it doesn't exist @@ -101,12 +101,12 @@ class MaildirRepository(BaseRepository): # If we're using hierarchical folders, it's possible that # sub-folders may be created before higher-up ones. - self.debug("makefolder: calling makedirs '%s'" % full_path) + self.debug("makefolder: calling makedirs '%s'"% full_path) try: os.makedirs(full_path, 0o700) except OSError as e: if e.errno == 17 and os.path.isdir(full_path): - self.debug("makefolder: '%s' already a directory" % foldername) + self.debug("makefolder: '%s' already a directory"% foldername) else: raise for subdir in ['cur', 'new', 'tmp']: @@ -114,13 +114,13 @@ class MaildirRepository(BaseRepository): os.mkdir(os.path.join(full_path, subdir), 0o700) except OSError as e: if e.errno == 17 and os.path.isdir(full_path): - self.debug("makefolder: '%s' already has subdir %s" % + self.debug("makefolder: '%s' already has subdir %s"% (foldername, subdir)) else: raise def deletefolder(self, foldername): - self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername) + self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s"% foldername) def getfolder(self, foldername): """Return a Folder instance of this Maildir @@ -128,6 +128,7 @@ class MaildirRepository(BaseRepository): If necessary, scan and cache all foldernames to make sure that we only return existing folders and that 2 calls with the same name will return the same object.""" + # getfolders() will scan and cache the values *if* necessary folders = self.getfolders() for f in folders: @@ -142,8 +143,9 @@ class MaildirRepository(BaseRepository): :param root: (absolute) path to Maildir root :param extension: (relative) subfolder to examine within root""" - self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s" \ - % (root, extension)) + + self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s"% + (root, extension)) retval = [] # Configure the full path to this repository -- "toppath" @@ -151,11 +153,11 @@ class MaildirRepository(BaseRepository): toppath = os.path.join(root, extension) else: toppath = root - self.debug(" toppath = %s" % toppath) + self.debug(" toppath = %s"% toppath) # Iterate over directories in top & top itself. for dirname in os.listdir(toppath) + ['']: - self.debug(" dirname = %s" % dirname) + self.debug(" dirname = %s"% dirname) if dirname == '' and extension is not None: self.debug(' skip this entry (already scanned)') continue @@ -178,7 +180,7 @@ class MaildirRepository(BaseRepository): os.path.isdir(os.path.join(fullname, 'new')) and os.path.isdir(os.path.join(fullname, 'tmp'))): # This directory has maildir stuff -- process - self.debug(" This is maildir folder '%s'." % foldername) + self.debug(" This is maildir folder '%s'."% foldername) if self.getconfboolean('restoreatime', False): self._append_folder_atimes(foldername) fd = self.getfoldertype()(self.root, foldername, @@ -188,7 +190,7 @@ class MaildirRepository(BaseRepository): if self.getsep() == '/' and dirname != '': # Recursively check sub-directories for folders too. retval.extend(self._getfolders_scandir(root, foldername)) - self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ + self.debug("_GETFOLDERS_SCANDIR RETURNING %s"% \ repr([x.getname() for x in retval])) return retval diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 5fa4dde..0b5aa6a 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -23,7 +23,7 @@ from offlineimap import banner from offlineimap.ui.UIBase import UIBase class TTYFormatter(logging.Formatter): - """Specific Formatter that adds thread information to the log output""" + """Specific Formatter that adds thread information to the log output.""" def __init__(self, *args, **kwargs): #super() doesn't work in py2.6 as 'logging' uses old-style class @@ -31,7 +31,8 @@ class TTYFormatter(logging.Formatter): self._last_log_thread = None def format(self, record): - """Override format to add thread information""" + """Override format to add thread information.""" + #super() doesn't work in py2.6 as 'logging' uses old-style class log_str = logging.Formatter.format(self, record) # If msg comes from a different thread than our last, prepend @@ -44,7 +45,7 @@ class TTYFormatter(logging.Formatter): self._last_log_thread = t_name log_str = "%s:\n %s" % (t_name, log_str) else: - log_str = " %s" % log_str + log_str = " %s"% log_str return log_str @@ -69,15 +70,15 @@ class TTYUI(UIBase): return ch def isusable(self): - """TTYUI is reported as usable when invoked on a terminal""" + """TTYUI is reported as usable when invoked on a terminal.""" return sys.stdout.isatty() and sys.stdin.isatty() def getpass(self, accountname, config, errmsg=None): - """TTYUI backend is capable of querying the password""" + """TTYUI backend is capable of querying the password.""" if errmsg: - self.warn("%s: %s" % (accountname, errmsg)) + self.warn("%s: %s"% (accountname, errmsg)) self._log_con_handler.acquire() # lock the console output try: return getpass("Enter password for account '%s': " % accountname) @@ -87,7 +88,7 @@ class TTYUI(UIBase): def mainException(self): if isinstance(sys.exc_info()[1], KeyboardInterrupt): self.logger.warn("Timer interrupted at user request; program " - "terminating.\n") + "terminating.\n") self.terminate() else: UIBase.mainException(self) @@ -100,12 +101,11 @@ class TTYUI(UIBase): This implementation in UIBase does not support this, but some implementations return 0 for successful sleep and 1 for an - 'abort', ie a request to sync immediately. - """ + 'abort', ie a request to sync immediately.""" if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: self.logger.info("Next refresh in %.1f minutes" % ( - remainingsecs/60.0)) + remainingsecs/60.0)) time.sleep(sleepsecs) return 0 diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index f6007da..0090291 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -393,11 +393,11 @@ class UIBase(object): uid, dest, num, num_to_set, ", ".join(labels))) def collectingdata(self, uidlist, source): - if uidlist: - self.logger.info("Collecting data from %d messages on %s" % ( + if uidlist: + self.logger.info("Collecting data from %d messages on %s"% ( len(uidlist), source)) - else: - self.logger.info("Collecting data from messages on %s" % source) + else: + self.logger.info("Collecting data from messages on %s"% source) def serverdiagnostics(self, repository, type): """Connect to repository and output useful information for debugging.""" diff --git a/offlineimap/ui/debuglock.py b/offlineimap/ui/debuglock.py index ef6e825..673efb0 100644 --- a/offlineimap/ui/debuglock.py +++ b/offlineimap/ui/debuglock.py @@ -28,7 +28,8 @@ class DebuggingLock: def acquire(self, blocking = 1): self.print_tb("Acquire lock") self.lock.acquire(blocking) - self.logmsg("===== %s: Thread %s acquired lock\n" % (self.name, currentThread().getName())) + self.logmsg("===== %s: Thread %s acquired lock\n"% + (self.name, currentThread().getName())) def release(self): self.print_tb("Release lock") @@ -41,7 +42,7 @@ class DebuggingLock: loglock.release() def print_tb(self, msg): - self.logmsg(".... %s: Thread %s attempting to %s\n" % \ + self.logmsg(".... %s: Thread %s attempting to %s\n"% \ (self.name, currentThread().getName(), msg) + \ "\n".join(traceback.format_list(traceback.extract_stack()))) From 40226705aff32b5b75b7b18870c62fb3fe5dda4a Mon Sep 17 00:00:00 2001 From: Wieland Hoffmann Date: Mon, 27 Oct 2014 12:30:27 +0100 Subject: [PATCH 676/817] MANUAL: dev-doc is no more The folder has been removed in 74b133c500ef1d8954b5d402fd428a2cc062aa42 Signed-off-by: Wieland Hoffmann Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index acf7395..8d87deb 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -41,7 +41,7 @@ Most configuration is done via the configuration file. However, any setting can OfflineImap is well suited to be frequently invoked by cron jobs, or can run in daemon mode to periodically check your email (however, it will exit in some error situations). The documentation is included in the git repository and can be created by -issueing `make dev-doc` in the `doc` folder (python-sphinx required), or it can +issueing `make doc` in the `doc` folder (python-sphinx required), or it can be viewed online at http://docs.offlineimap.org. .. _configuration: @@ -335,7 +335,7 @@ core. Folder filtering and nametrans ============================== -OfflineImap offers flexible (and complex) ways of filtering and transforming folder names. Please see the docs/dev-docs-src/folderfilters.rst document about details how to use folder filters and name transformations. The documentation will be autogenerated by a "make dev-doc" in the docs directory. It is also viewable at :ref:`folder_filtering_and_name_translation`. +OfflineImap offers flexible (and complex) ways of filtering and transforming folder names. Please see the docs/doc-src/nametrans.rst document about details how to use folder filters and name transformations. The documentation will be autogenerated by a "make doc" in the docs directory. It is also viewable at :ref:`folder_filtering_and_name_translation`. KNOWN BUGS ========== From 2f9d8d22ee72ec1e802c2f91d5336a1734ccd3ea Mon Sep 17 00:00:00 2001 From: Wieland Hoffmann Date: Mon, 27 Oct 2014 12:34:05 +0100 Subject: [PATCH 677/817] folder/Base.py: fix comment: para -> param Signed-off-by: Wieland Hoffmann Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 14262e3..87e165c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -29,8 +29,8 @@ import offlineimap.accounts class BaseFolder(object): def __init__(self, name, repository): """ - :para name: Path & name of folder minus root or reference - :para repository: Repository() in which the folder is. + :param name: Path & name of folder minus root or reference + :param repository: Repository() in which the folder is. """ self.ui = getglobalui() From 4273c9b3050f5bddd04c390689ac29911a4c8300 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 9 Jan 2015 01:16:06 +0100 Subject: [PATCH 678/817] doc-src: nametrans.rst: fix minor error - Typo in code was corrected by aeeea5 - Update prose to match code Submitted-by: sumbach (Github) Signed-off-by: Nicolas Sebrecht --- docs/doc-src/nametrans.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/doc-src/nametrans.rst b/docs/doc-src/nametrans.rst index 030ead2..f802c05 100644 --- a/docs/doc-src/nametrans.rst +++ b/docs/doc-src/nametrans.rst @@ -141,7 +141,7 @@ Take the above examples. If your remote nametrans setting was:: nametrans = lambda folder: re.sub('^INBOX\.', '', folder) -then you will want to have this in your local repository, prepending "INBOX" to +then you will want to have this in your local repository, prepending "INBOX." to any local folder name:: nametrans = lambda folder: 'INBOX.' + folder From 4bc766035cb721da0b12d25e132b477a38748f96 Mon Sep 17 00:00:00 2001 From: Wieland Hoffmann Date: Sun, 28 Dec 2014 00:55:08 +0100 Subject: [PATCH 679/817] error: Log the messages with level ERROR Otherwise, messages logged through UIBase.error would only be passed to UIBase._msg, which only logs at INFO. This causes error to not get logged at all for the quit UI. Signed-off-by: Wieland Hoffmann Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 0090291..3c6e499 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -137,9 +137,9 @@ class UIBase(object): "repo %s") """ if msg: - self._msg("ERROR: %s\n %s" % (msg, exc)) + self.logger.error("ERROR: %s\n %s" % (msg, exc)) else: - self._msg("ERROR: %s" % (exc)) + self.logger.error("ERROR: %s" % (exc)) instant_traceback = exc_traceback if not self.debuglist: @@ -148,7 +148,7 @@ class UIBase(object): # push exc on the queue for later output self.exc_queue.put((msg, exc, exc_traceback)) if instant_traceback: - self._msg(traceback.format_tb(instant_traceback)) + self.logger.error(traceback.format_tb(instant_traceback)) def registerthread(self, account): """Register current thread as being associated with an account name.""" From 561a3d4329d1e1f42849bd9c291ff43c25f431dc Mon Sep 17 00:00:00 2001 From: Stefan Huber Date: Wed, 24 Sep 2014 12:29:04 +0200 Subject: [PATCH 680/817] Do not keep reloading pyhtonfile, make it stateful CustomConfigParser.getlocaleval() loads "pythonfile" at each call. Besides unnecessary IO, in case that dynamic_folderfilter is true, the code in "pythonfile" would behave stateless, since it is re-initialized at each call of getlocaleval(), i.e., at every sync. Fix that by keeping a singleton copy of localeval in CustomConfigParser after the first call of getlocaleval(). Signed-off-by: Stefan Huber Signed-off-by: Nicolas Sebrecht --- offlineimap/CustomConfig.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 14564c2..8b070c5 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -24,6 +24,10 @@ except ImportError: #python3 from offlineimap.localeval import LocalEval class CustomConfigParser(SafeConfigParser): + def __init__(self): + SafeConfigParser.__init__(self) + self.localeval = None + def getdefault(self, section, option, default, *args, **kwargs): """Same as config.get, but returns the value of `default` if there is no such option specified.""" @@ -91,13 +95,19 @@ class CustomConfigParser(SafeConfigParser): return metadatadir def getlocaleval(self): + # We already loaded pythonfile, so return this copy. + if self.localeval is not None: + return self.localeval + xforms = [os.path.expanduser, os.path.expandvars] if self.has_option("general", "pythonfile"): path = self.get("general", "pythonfile") path = self.apply_xforms(path, xforms) else: path = None - return LocalEval(path) + + self.localeval = LocalEval(path) + return self.localeval def getsectionlist(self, key): """Returns a list of sections that start with (str) key + " ". From 18bf7b7dc9eac64f97276c6d02a5862fcde60501 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 10 Jan 2015 01:16:56 +0100 Subject: [PATCH 681/817] MANUAL: split long lines Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 8d87deb..e5fa9af 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -36,9 +36,12 @@ password. Finally, you can use IMAPs IDLE infrastructure to always keep a connection to your IMAP server open and immediately be notified (and synchronized) when a new mail arrives (aka Push mail). -Most configuration is done via the configuration file. However, any setting can also be overriden by command line options handed to OfflineIMAP. +Most configuration is done via the configuration file. However, any setting can +also be overriden by command line options handed to OfflineIMAP. -OfflineImap is well suited to be frequently invoked by cron jobs, or can run in daemon mode to periodically check your email (however, it will exit in some error situations). +OfflineImap is well suited to be frequently invoked by cron jobs, or can run in +daemon mode to periodically check your email (however, it will exit in some +error situations). The documentation is included in the git repository and can be created by issueing `make doc` in the `doc` folder (python-sphinx required), or it can @@ -171,11 +174,12 @@ status messages and is generally friendly to use on a console or xterm. Basic ------ -Basic is designed for situations in which OfflineIMAP will be run -non-attended and the status of its execution will be logged. This user -interface is not capable of reading a password from the keyboard; -account passwords must be specified using one of the configuration file -options. For example, it will not print periodic sleep announcements and tends to be a tad less verbose, in general. +Basic is designed for situations in which OfflineIMAP will be run non-attended +and the status of its execution will be logged. This user interface is not +capable of reading a password from the keyboard; account passwords must be +specified using one of the configuration file options. For example, it will not +print periodic sleep announcements and tends to be a tad less verbose, in +general. Quiet @@ -243,7 +247,11 @@ achieve this. Upgrading from plain text cache to SQLITE based cache ===================================================== -OfflineImap uses a cache to store the last know status of mails (flags etc). Historically that has meant plain text files, but recently we introduced sqlite-based cache, which helps with performance and CPU usage on large folders. Here is how to upgrade existing plain text cache installations to sqlite based one: +OfflineImap uses a cache to store the last know status of mails (flags etc). +Historically that has meant plain text files, but recently we introduced +sqlite-based cache, which helps with performance and CPU usage on large folders. +Here is how to upgrade existing plain text cache installations to sqlite based +one: 1) Sync to make sure things are reasonably similar @@ -335,7 +343,11 @@ core. Folder filtering and nametrans ============================== -OfflineImap offers flexible (and complex) ways of filtering and transforming folder names. Please see the docs/doc-src/nametrans.rst document about details how to use folder filters and name transformations. The documentation will be autogenerated by a "make doc" in the docs directory. It is also viewable at :ref:`folder_filtering_and_name_translation`. +OfflineImap offers flexible (and complex) ways of filtering and transforming +folder names. Please see the docs/doc-src/nametrans.rst document about details +how to use folder filters and name transformations. The documentation will be +autogenerated by a "make doc" in the docs directory. It is also viewable at +:ref:`folder_filtering_and_name_translation`. KNOWN BUGS ========== From 16baabdeaa88824e6e9a0bd7f33c02b1f76dc3de Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 10 Jan 2015 01:17:33 +0100 Subject: [PATCH 682/817] minor: offlineimap.conf: give the specific entry to the MANUAL about IDLE Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 63c1b97..d953f0e 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -550,8 +550,8 @@ remoteuser = username # holdconnectionopen - to be true # keepalive - to be 29 minutes unless you specify otherwise # -# This feature isn't complete and may well have problems. See the manual -# for more details. +# This feature isn't complete and may well have problems. See the "Known Issues" +# entry in the manual for more details. # # This option should return a Python list. For example # From bd0461a986743e12abd97e317c32751ed1150acb Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 10 Jan 2015 01:51:10 +0100 Subject: [PATCH 683/817] MANUAL: add minor sample on how to retrieve a password with a helper python file Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index e5fa9af..16af514 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -562,6 +562,16 @@ pythonfile with:: pythonfile=~/bin/offlineimap-helpers.py +Here is a basic content sample:: + + import commands + + def get_password(account_name): + cmd = "security find-internet-password -w -a '%s'"% account_name + (status, output) = commands.getstatusoutput(cmd) + return output + +From this sample, replace the cmd line with whatever can retrieve your password. Your pythonfile needs to contain implementations for the functions that you want to use in offflineimaprc. The example uses it for two purposes: Fetching passwords from the gnome-keyring and translating From f2010cdfb0494ad3831574b815f755012fd2b446 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 10 Jan 2015 02:21:07 +0100 Subject: [PATCH 684/817] Machine.py; more consistent style Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Machine.py | 44 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 5cf7449..2e6019e 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -45,12 +45,12 @@ class MachineLogFormatter(logging.Formatter): command = "" whoami = currentThread().getName() - prefix = "%s:%s" % (command, urlencode([('', whoami)])[1:]) - return "%s:%s:%s" % (severity, prefix, urlencode([('', line)])[1:]) + prefix = "%s:%s"% (command, urlencode([('', whoami)])[1:]) + return "%s:%s:%s"% (severity, prefix, urlencode([('', line)])[1:]) class MachineUI(UIBase): - def __init__(s, config, loglevel = logging.INFO): + def __init__(s, config, loglevel=logging.INFO): super(MachineUI, s).__init__(config, loglevel) s._log_con_handler.createLock() """lock needed to block on password input""" @@ -74,7 +74,7 @@ class MachineUI(UIBase): def _msg(s, msg): s._printData(s.logger.info, '_display', msg) - def warn(s, msg, minor = 0): + def warn(s, msg, minor=0): # TODO, remove and cleanup the unused minor stuff s._printData(s.logger.warning, '', msg) @@ -96,68 +96,68 @@ class MachineUI(UIBase): s._printData(s.logger.info, 'acctdone', accountname) def validityproblem(s, folder): - s._printData(s.logger.warning, 'validityproblem', "%s\n%s\n%s\n%s" % \ + s._printData(s.logger.warning, 'validityproblem', "%s\n%s\n%s\n%s"% (folder.getname(), folder.getrepository().getname(), folder.get_saveduidvalidity(), folder.get_uidvalidity())) def connecting(s, hostname, port): - s._printData(s.logger.info, 'connecting', "%s\n%s" % (hostname, str(port))) + s._printData(s.logger.info, 'connecting', "%s\n%s"% (hostname, str(port))) def syncfolders(s, srcrepos, destrepos): - s._printData(s.logger.info, 'syncfolders', "%s\n%s" % (s.getnicename(srcrepos), + s._printData(s.logger.info, 'syncfolders', "%s\n%s"% (s.getnicename(srcrepos), s.getnicename(destrepos))) def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): - s._printData(s.logger.info, 'syncingfolder', "%s\n%s\n%s\n%s\n" % \ + s._printData(s.logger.info, 'syncingfolder', "%s\n%s\n%s\n%s\n"% (s.getnicename(srcrepos), srcfolder.getname(), s.getnicename(destrepos), destfolder.getname())) def loadmessagelist(s, repos, folder): - s._printData(s.logger.info, 'loadmessagelist', "%s\n%s" % (s.getnicename(repos), + s._printData(s.logger.info, 'loadmessagelist', "%s\n%s"% (s.getnicename(repos), folder.getvisiblename())) def messagelistloaded(s, repos, folder, count): - s._printData(s.logger.info, 'messagelistloaded', "%s\n%s\n%d" % \ + s._printData(s.logger.info, 'messagelistloaded', "%s\n%s\n%d"% (s.getnicename(repos), folder.getname(), count)) def syncingmessages(s, sr, sf, dr, df): - s._printData(s.logger.info, 'syncingmessages', "%s\n%s\n%s\n%s\n" % \ + s._printData(s.logger.info, 'syncingmessages', "%s\n%s\n%s\n%s\n"% (s.getnicename(sr), sf.getname(), s.getnicename(dr), df.getname())) def copyingmessage(s, uid, num, num_to_copy, srcfolder, destfolder): - s._printData(s.logger.info, 'copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ + s._printData(s.logger.info, 'copyingmessage', "%d\n%s\n%s\n%s[%s]"% (uid, s.getnicename(srcfolder), srcfolder.getname(), s.getnicename(destfolder), destfolder)) def folderlist(s, list): - return ("\f".join(["%s\t%s" % (s.getnicename(x), x.getname()) for x in list])) + return ("\f".join(["%s\t%s"% (s.getnicename(x), x.getname()) for x in list])) def uidlist(s, list): return ("\f".join([str(u) for u in list])) def deletingmessages(s, uidlist, destlist): ds = s.folderlist(destlist) - s._printData(s.logger.info, 'deletingmessages', "%s\n%s" % (s.uidlist(uidlist), ds)) + s._printData(s.logger.info, 'deletingmessages', "%s\n%s"% (s.uidlist(uidlist), ds)) def addingflags(s, uidlist, flags, dest): - s._printData(s.logger.info, "addingflags", "%s\n%s\n%s" % (s.uidlist(uidlist), + s._printData(s.logger.info, "addingflags", "%s\n%s\n%s"% (s.uidlist(uidlist), "\f".join(flags), dest)) def deletingflags(s, uidlist, flags, dest): - s._printData(s.logger.info, 'deletingflags', "%s\n%s\n%s" % (s.uidlist(uidlist), + s._printData(s.logger.info, 'deletingflags', "%s\n%s\n%s"% (s.uidlist(uidlist), "\f".join(flags), dest)) def threadException(s, thread): - s._printData(s.logger.warning, 'threadException', "%s\n%s" % \ + s._printData(s.logger.warning, 'threadException', "%s\n%s"% (thread.getName(), s.getThreadExceptionString(thread))) s.delThreadDebugLog(thread) s.terminate(100) - def terminate(s, exitstatus = 0, errortitle = '', errormsg = ''): - s._printData(s.logger.info, 'terminate', "%d\n%s\n%s" % (exitstatus, errortitle, errormsg)) + def terminate(s, exitstatus=0, errortitle='', errormsg=''): + s._printData(s.logger.info, 'terminate', "%d\n%s\n%s"% (exitstatus, errortitle, errormsg)) sys.exit(exitstatus) def mainException(s): @@ -168,16 +168,16 @@ class MachineUI(UIBase): UIBase.threadExited(s, thread) def sleeping(s, sleepsecs, remainingsecs): - s._printData(s.logger.info, 'sleeping', "%d\n%d" % (sleepsecs, remainingsecs)) + s._printData(s.logger.info, 'sleeping', "%d\n%d"% (sleepsecs, remainingsecs)) if sleepsecs > 0: time.sleep(sleepsecs) return 0 - def getpass(s, accountname, config, errmsg = None): + def getpass(s, accountname, config, errmsg=None): if errmsg: s._printData(s.logger.warning, - 'getpasserror', "%s\n%s" % (accountname, errmsg), + 'getpasserror', "%s\n%s"% (accountname, errmsg), False) s._log_con_handler.acquire() # lock the console output From 8c6abc413ea76d26d8396eaa4c7812a6f018719d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 10 Jan 2015 02:21:33 +0100 Subject: [PATCH 685/817] ui: Machine: remove offending param for a _printData() call Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 2e6019e..dc650c3 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -182,7 +182,7 @@ class MachineUI(UIBase): s._log_con_handler.acquire() # lock the console output try: - s._printData(s.logger.info, 'getpass', accountname, False) + s._printData(s.logger.info, 'getpass', accountname) return (sys.stdin.readline()[:-1]) finally: s._log_con_handler.release() From e7fabf9e57b2c10e1262d167814f6ba0985bfe40 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 11 Jan 2015 23:44:24 +0300 Subject: [PATCH 686/817] Properly re-raise exception to save original tracebacks We usually mutate some exceptions to OfflineImapError() and it is a whole lot better if such exception will show up with the original traceback, so all valid occurrences of such mutations were transformed to the 3-tuple form of "raise". Had also added coding guidelines document where this re-raise strategy is documented. Signed-off-by: Eygene Ryabinkin --- docs/CodingGuidelines.rst | 102 ++++++++++++++++++++++++ docs/doc-src/CodingGuidelines.rst | 1 + offlineimap/CustomConfig.py | 3 +- offlineimap/accounts.py | 3 +- offlineimap/folder/Gmail.py | 4 +- offlineimap/folder/GmailMaildir.py | 4 +- offlineimap/folder/IMAP.py | 9 ++- offlineimap/folder/LocalStatus.py | 5 +- offlineimap/folder/LocalStatusSQLite.py | 3 +- offlineimap/folder/Maildir.py | 6 +- offlineimap/folder/UIDMaps.py | 6 +- offlineimap/imaplibutil.py | 3 +- offlineimap/imapserver.py | 16 ++-- offlineimap/repository/IMAP.py | 3 +- offlineimap/repository/__init__.py | 8 +- 15 files changed, 150 insertions(+), 26 deletions(-) create mode 100644 docs/CodingGuidelines.rst create mode 120000 docs/doc-src/CodingGuidelines.rst diff --git a/docs/CodingGuidelines.rst b/docs/CodingGuidelines.rst new file mode 100644 index 0000000..d6d29a2 --- /dev/null +++ b/docs/CodingGuidelines.rst @@ -0,0 +1,102 @@ +.. -*- coding: utf-8 -*- +.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap +.. _OLI_git_repo: git://github.com/OfflineIMAP/offlineimap.git + +================================= +Coding guidelines for OfflineIMAP +================================= + +.. contents:: +.. .. sectnum:: + +This document contains assorted guidelines for programmers that want +to hack OfflineIMAP. + + +------------------ +Exception handling +------------------ + +OfflineIMAP on many occasions re-raises various exceptions and often +changes exception type to `OfflineImapError`. This is not a problem +per se, but you must always remember that we need to preserve original +tracebacks. This is not hard if you follow these simple rules. + +For re-raising original exceptions, just use:: + + raise + +from inside your exception handling code. + +If you need to change exception type, or its argument, or whatever, +use this three-argument form:: + + raise YourExceptionClass(argum, ents), None, sys.exc_info()[2] + +In this form, you're creating an instance of new exception, so ``raise`` +will deduce its ``type`` and ``value`` parameters from the first argument, +thus the second expression passed to ``raise`` is always ``None``. +And the third one is the traceback object obtained from the thread-safe +``exc_info()`` function. + +In fact, if you hadn't already imported the whole ``sys`` module, it will +be better to import just ``exc_info()``:: + + from sys import exc_info + +and raise like this:: + + raise YourExceptionClass(argum, ents), None, exc_info()[2] + +since this is the historically-preferred style in the OfflineIMAP code. +.. -*- coding: utf-8 -*- +.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap +.. _OLI_git_repo: git://github.com/OfflineIMAP/offlineimap.git + +================================= +Coding guidelines for OfflineIMAP +================================= + +.. contents:: +.. .. sectnum:: + +This document contains assorted guidelines for programmers that want +to hack OfflineIMAP. + + +------------------ +Exception handling +------------------ + +OfflineIMAP on many occasions re-raises various exceptions and often +changes exception type to `OfflineImapError`. This is not a problem +per se, but you must always remember that we need to preserve original +tracebacks. This is not hard if you follow these simple rules. + +For re-raising original exceptions, just use:: + + raise + +from inside your exception handling code. + +If you need to change exception type, or its argument, or whatever, +use this three-argument form:: + + raise YourExceptionClass(argum, ents), None, sys.exc_info()[2] + +In this form, you're creating an instance of new exception, so ``raise`` +will deduce its ``type`` and ``value`` parameters from the first argument, +thus the second expression passed to ``raise`` is always ``None``. +And the third one is the traceback object obtained from the thread-safe +``exc_info()`` function. + +In fact, if you hadn't already imported the whole ``sys`` module, it will +be better to import just ``exc_info()``:: + + from sys import exc_info + +and raise like this:: + + raise YourExceptionClass(argum, ents), None, exc_info()[2] + +since this is the historically-preferred style in the OfflineIMAP code. diff --git a/docs/doc-src/CodingGuidelines.rst b/docs/doc-src/CodingGuidelines.rst new file mode 120000 index 0000000..7117956 --- /dev/null +++ b/docs/doc-src/CodingGuidelines.rst @@ -0,0 +1 @@ +../CodingGuidelines.rst \ No newline at end of file diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 8b070c5..44cfcab 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -16,6 +16,7 @@ import os import re +from sys import exc_info try: from ConfigParser import SafeConfigParser, Error @@ -75,7 +76,7 @@ class CustomConfigParser(SafeConfigParser): return re.split(separator_re, val) except re.error as e: raise Error("Bad split regexp '%s': %s" % \ - (separator_re, e)) + (separator_re, e)), None, exc_info()[2] def getdefaultlist(self, section, option, default, separator_re): """Same as getlist, but returns the value of `default` diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index d9c9b88..91314f9 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -210,7 +210,8 @@ class SyncableAccount(Account): self._lockfd.close() raise OfflineImapError("Could not lock account %s. Is another " "instance using this account?" % self, - OfflineImapError.ERROR.REPO) + OfflineImapError.ERROR.REPO), \ + None, exc_info()[2] def _unlock(self): """Unlock the account, deleting the lock file""" diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 69f0075..c12c0ff 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -17,6 +17,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import re +from sys import exc_info from offlineimap import imaputil, OfflineImapError from offlineimap import imaplibutil @@ -144,7 +145,8 @@ class GmailFolder(IMAPFolder): raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " % \ (self.getrepository(), self) + \ "Server responded '[%s] %s'" % \ - (res_type, response), OfflineImapError.ERROR.FOLDER) + (res_type, response), OfflineImapError.ERROR.FOLDER), \ + None, exc_info()[2] finally: self.imapserver.releaseconnection(imapobj) diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 3b127d2..5ca0e1f 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -17,6 +17,7 @@ import os +from sys import exc_info from .Maildir import MaildirFolder from offlineimap import OfflineImapError import offlineimap.accounts @@ -170,7 +171,8 @@ class GmailMaildirFolder(MaildirFolder): os.rename(tmppath, filepath) except OSError as e: raise OfflineImapError("Can't rename file '%s' to '%s': %s" % \ - (tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER) + (tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER), \ + None, exc_info()[2] if rtime != None: os.utime(filepath, (rtime, rtime)) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 52493fc..e67cb06 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -594,7 +594,9 @@ class IMAPFolder(BaseFolder): "repository '%s' failed (abort). Server responded: %s\n" "Message content was: %s"% (msg_id, self, self.getrepository(), str(e), dbg_output), - OfflineImapError.ERROR.MESSAGE) + OfflineImapError.ERROR.MESSAGE), \ + None, exc_info()[2] + # XXX: is this still needed? self.ui.error(e, exc_info()[2]) except imapobj.error as e: # APPEND failed # If the server responds with 'BAD', append() @@ -604,8 +606,8 @@ class IMAPFolder(BaseFolder): imapobj = None raise OfflineImapError("Saving msg (%s) folder '%s', repo '%s'" "failed (error). Server responded: %s\nMessage content was: " - "%s"% (msg_id, self, self.getrepository(), str(e), dbg_output), - OfflineImapError.ERROR.MESSAGE) + "%s" % (msg_id, self, self.getrepository(), str(e), dbg_output), + OfflineImapError.ERROR.MESSAGE), None, exc_info()[2] # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. (typ,dat) = imapobj.check() @@ -676,6 +678,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() self.ui.error(e, exc_info()[2]) fails_left -= 1 + # self.ui.error() will show the original traceback if not fails_left: raise e if data == [None] or res_type != 'OK': diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 5f3d32d..ec34965 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +from sys import exc_info import os import threading @@ -79,7 +80,7 @@ class LocalStatusFolder(BaseFolder): errstr = "Corrupt line '%s' in cache file '%s'" % \ (line, self.filename) self.ui.warn(errstr) - raise ValueError(errstr) + raise ValueError(errstr), None, exc_info()[2] self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags @@ -103,7 +104,7 @@ class LocalStatusFolder(BaseFolder): errstr = "Corrupt line '%s' in cache file '%s'" % \ (line, self.filename) self.ui.warn(errstr) - raise ValueError(errstr) + raise ValueError(errstr), None, exc_info()[2] self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags self.messagelist[uid]['mtime'] = mtime diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index a9ef6c2..1b2adb0 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import re +from sys import exc_info from threading import Lock from .Base import BaseFolder try: @@ -66,7 +67,7 @@ class LocalStatusSQLiteFolder(BaseFolder): except NameError: # sqlite import had failed raise UserWarning('SQLite backend chosen, but no sqlite python ' - 'bindings available. Please install.') + 'bindings available. Please install.'), None, exc_info()[2] #Make sure sqlite is in multithreading SERIALIZE mode assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 88ece8f..af1e5ff 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -19,6 +19,7 @@ import socket import time import re import os +from sys import exc_info from .Base import BaseFolder from threading import Lock try: @@ -282,7 +283,7 @@ class MaildirFolder(BaseFolder): continue severity = OfflineImapError.ERROR.MESSAGE raise OfflineImapError("Unique filename %s already exists." % \ - filename, severity) + filename, severity), None, exc_info()[2] else: raise @@ -372,7 +373,8 @@ class MaildirFolder(BaseFolder): except OSError as e: raise OfflineImapError("Can't rename file '%s' to '%s': %s" % ( oldfilename, newfilename, e[1]), - OfflineImapError.ERROR.FOLDER) + OfflineImapError.ERROR.FOLDER), \ + None, exc_info()[2] self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 7815793..e1a6f5f 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +from sys import exc_info from threading import Lock from offlineimap import OfflineImapError from .IMAP import IMAPFolder @@ -60,7 +62,7 @@ class MappedIMAPFolder(IMAPFolder): line = line.strip() except ValueError: raise Exception("Corrupt line '%s' in UID mapping file '%s'"% - (line, mapfilename)) + (line, mapfilename)), None, exc_info()[2] (str1, str2) = line.split(':') loc = long(str1) rem = long(str2) @@ -89,7 +91,7 @@ class MappedIMAPFolder(IMAPFolder): raise OfflineImapError("Could not find UID for msg '{0}' (f:'{1}'." " This is usually a bad thing and should be reported on the ma" "iling list.".format(e.args[0], self), - OfflineImapError.ERROR.MESSAGE) + OfflineImapError.ERROR.MESSAGE), None, exc_info()[2] # Interface from BaseFolder def cachemessagelist(self): diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index e012a01..917c726 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -18,6 +18,7 @@ import os import fcntl import time import subprocess +from sys import exc_info import threading from hashlib import sha1 @@ -54,7 +55,7 @@ class UsefulIMAPMixIn(object): errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ "ver said: %s" % (self.host, mailbox, e.args[0]) severity = OfflineImapError.ERROR.FOLDER_RETRY - raise OfflineImapError(errstr, severity) + raise OfflineImapError(errstr, severity), None, exc_info()[2] if result[0] != 'OK': #in case of error, bail out with OfflineImapError errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\ diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 5601033..bac681e 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -206,8 +206,8 @@ class IMAPServer: imapobj.starttls() except imapobj.error as e: raise OfflineImapError("Failed to start " - "TLS connection: %s"% str(e), - OfflineImapError.ERROR.REPO) + "TLS connection: %s" % str(e), + OfflineImapError.ERROR.REPO, None, exc_info()[2]) ## All __authn_* procedures are helpers that do authentication. @@ -466,7 +466,7 @@ class IMAPServer: "'%s'. Make sure you have configured the ser"\ "ver name correctly and that you are online."%\ (self.hostname, self.repos) - raise OfflineImapError(reason, severity) + raise OfflineImapError(reason, severity), None, exc_info()[2] elif isinstance(e, SSLError) and e.errno == 1: # SSL unknown protocol error @@ -479,7 +479,7 @@ class IMAPServer: reason = "Unknown SSL protocol connecting to host '%s' for "\ "repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) - raise OfflineImapError(reason, severity) + raise OfflineImapError(reason, severity), None, exc_info()[2] elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: # "Connection refused", can be a non-existing port, or an unauthorized @@ -487,15 +487,15 @@ class IMAPServer: reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ - "network."% (self.hostname, self.port, self.repos) - raise OfflineImapError(reason, severity) + "network." % (self.hostname, self.port, self.repos) + raise OfflineImapError(reason, severity), None, exc_info()[2] # Could not acquire connection to the remote; # socket.error(last_error) raised if str(e)[:24] == "can't open socket; error": raise OfflineImapError("Could not connect to remote server '%s' "\ "for repository '%s'. Remote does not answer." % (self.hostname, self.repos), - OfflineImapError.ERROR.REPO) + OfflineImapError.ERROR.REPO), None, exc_info()[2] else: # re-raise all other errors raise @@ -702,7 +702,7 @@ class IdleThread(object): self.ui.error(e, exc_info()[2]) self.parent.releaseconnection(imapobj, True) else: - raise e + raise else: success = True if "IDLE" in imapobj.capabilities: diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index d664259..2baa046 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -103,7 +103,8 @@ class IMAPRepository(BaseRepository): except Exception as e: raise OfflineImapError("remotehosteval option for repository "\ "'%s' failed:\n%s" % (self, e), - OfflineImapError.ERROR.REPO) + OfflineImapError.ERROR.REPO), \ + None, exc_info()[2] if host: self._host = host return self._host diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py index 93861c2..076255e 100644 --- a/offlineimap/repository/__init__.py +++ b/offlineimap/repository/__init__.py @@ -15,6 +15,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +from sys import exc_info + try: from configparser import NoSectionError except ImportError: #python2 @@ -66,14 +68,16 @@ class Repository(object): except NoSectionError as e: errstr = ("Could not find section '%s' in configuration. Required " "for account '%s'." % ('Repository %s' % name, account)) - raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO) + raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \ + None, exc_info()[2] try: repo = typemap[repostype] except KeyError: errstr = "'%s' repository not supported for '%s' repositories." \ % (repostype, reqtype) - raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO) + raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \ + None, exc_info()[2] return repo(name, account) From 50b01cd7282dbb60c81137ddc06fc4aae30f3e90 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 12 Jan 2015 00:07:14 +0300 Subject: [PATCH 687/817] Fix API documentation syntax - Drop unneeded definition for offlineimap.globals module. - Avoid hyperlinking the section from itself. Signed-off-by: Eygene Ryabinkin --- docs/doc-src/API.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index d47fcf4..80c0460 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -64,9 +64,7 @@ An :class:`accounts.Account` connects two email repositories that are to be sync :mod:`offlineimap.globals` -- module with global variables ========================================================== -.. module:: offlineimap.globals - -Module :mod:`offlineimap.globals` provides the read-only storage +Module `offlineimap.globals` provides the read-only storage for the global variables. All exported module attributes can be set manually, but this practice From 854d0019d9a653ec72cda7507b53dbe2467bf517 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 12 Jan 2015 00:18:19 +0300 Subject: [PATCH 688/817] API documentation: properly auto-document main class Signed-off-by: Eygene Ryabinkin --- docs/doc-src/API.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index 80c0460..a74b602 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -25,16 +25,12 @@ number of resources and conventions you may find useful. .. module:: offlineimap .. autoclass:: offlineimap.OfflineImap(cmdline_opts = None) + :members: + :inherited-members: + :undoc-members: + :private-members: - .. automethod:: run - - .. automethod:: __parse_cmd_options - -.. .. autoattribute:: ui - - :todo: Document - :class:`offlineimap.account` ============================ From 2789ad26bf05b1c3d592717a12c50e3e67c0da35 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 12 Jan 2015 00:21:03 +0300 Subject: [PATCH 689/817] API documentation: fix typo Signed-off-by: Eygene Ryabinkin --- docs/doc-src/API.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index a74b602..774f47b 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -52,7 +52,7 @@ An :class:`accounts.Account` connects two email repositories that are to be sync .. autoexception:: offlineimap.error.OfflineImapError :members: - This execption inherits directly from :exc:`Exception` and is raised + This exception inherits directly from :exc:`Exception` and is raised on errors during the offlineimap execution. It has an attribute `severity` that denotes the severity level of the error. From 14141c07d7cae583353a06e9d7cbb269fd0bfa86 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 12 Jan 2015 19:15:06 +0300 Subject: [PATCH 690/817] Fix regression introduced in 0f40ca47998c7f56bf7cd1ce095c66fe93e21d03 We have no variable "fullname", it must have been slipped in unintentionally. Signed-off-by: Eygene Ryabinkin --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index e67cb06..42ad448 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -567,7 +567,7 @@ class IMAPFolder(BaseFolder): #Do the APPEND try: - (typ, dat) = imapobj.append(fullname, + (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) # This should only catch 'NO' responses since append() # will raise an exception for 'BAD' responses: From 14de280585261abfc9446a5ef37d48f17c5e7686 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 11 Jan 2015 00:17:08 +0100 Subject: [PATCH 691/817] repository: Base: add comment about lying variable name self.uiddir Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index cb9d06a..d30d8a3 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -39,6 +39,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): self.mapdir = os.path.join(self.uiddir, 'UIDMapping') if not os.path.exists(self.mapdir): os.mkdir(self.mapdir, 0o700) + # FIXME: self.uiddir variable name is lying about itself. self.uiddir = os.path.join(self.uiddir, 'FolderValidity') if not os.path.exists(self.uiddir): os.mkdir(self.uiddir, 0o700) From c2b8a99fa23003b0c50a2fc886a417fa34eca129 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 11 Jan 2015 10:00:37 +0100 Subject: [PATCH 692/817] repository: IMAP.py: do not redefine string Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/IMAP.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 2baa046..0334237 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -326,13 +326,13 @@ class IMAPRepository(BaseRepository): listresult = listfunction(directory = self.imapserver.reference)[1] finally: self.imapserver.releaseconnection(imapobj) - for string in listresult: - if string == None or \ - (isinstance(string, basestring) and string == ''): + for s in listresult: + if s == None or \ + (isinstance(s, basestring) and s == ''): # Bug in imaplib: empty strings in results from # literals. TODO: still relevant? continue - flags, delim, name = imaputil.imapsplit(string) + flags, delim, name = imaputil.imapsplit(s) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue From a44718130d18c30615762ca0dcccdb30e7ff45c6 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 11 Jan 2015 13:54:53 +0100 Subject: [PATCH 693/817] minor: add comments Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 6 ++++++ offlineimap/folder/IMAP.py | 2 ++ offlineimap/init.py | 2 ++ offlineimap/repository/IMAP.py | 4 ++++ 4 files changed, 14 insertions(+) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 91314f9..95b3e9f 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -33,15 +33,21 @@ except: # FIXME: spaghetti code alert! def getaccountlist(customconfig): + # Account names in a list. return customconfig.getsectionlist('Account') # FIXME: spaghetti code alert! def AccountListGenerator(customconfig): + """Returns a list of instanciated Account class, one per account name.""" + return [Account(customconfig, accountname) for accountname in getaccountlist(customconfig)] # FIXME: spaghetti code alert! def AccountHashGenerator(customconfig): + """Returns a dict of instanciated Account class with the account name as + key.""" + retval = {} for item in AccountListGenerator(customconfig): retval[item.getname()] = item diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index e67cb06..00afce0 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -40,6 +40,8 @@ CRLF = '\r\n' class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, repository): + # FIXME: decide if unquoted name is from the responsability of the + # caller or not, but not both. name = imaputil.dequote(name) self.sep = imapserver.delim super(IMAPFolder, self).__init__(name, repository) diff --git a/offlineimap/init.py b/offlineimap/init.py index 878f5cf..f53c9a5 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -321,6 +321,8 @@ class OfflineImap: pass try: + # Honor CLI --account option, only. + # Accounts to sync are put into syncaccounts variable. activeaccounts = self.config.get("general", "accounts") if options.accounts: activeaccounts = options.accounts diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 0334237..5b70966 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -301,6 +301,8 @@ class IMAPRepository(BaseRepository): return None def getfolder(self, foldername): + """Return instance of OfflineIMAP representative folder.""" + return self.getfoldertype()(self.imapserver, foldername, self) def getfoldertype(self): @@ -314,6 +316,8 @@ class IMAPRepository(BaseRepository): self.folders = None def getfolders(self): + """Return a list of instances of OfflineIMAP representative folder.""" + if self.folders != None: return self.folders retval = [] From 63e499c6f2f6481891697b6b16a91b23c88e81ba Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 11 Jan 2015 10:54:19 +0100 Subject: [PATCH 694/817] more consistent style Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 6 ++++-- offlineimap/repository/GmailMaildir.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 95b3e9f..18a36fc 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -60,9 +60,10 @@ class Account(CustomConfig.ConfigHelperMixin): Most of the time you will actually want to use the derived :class:`accounts.SyncableAccount` which contains all functions used for syncing an account.""" - #signal gets set when we should stop looping + + # Signal gets set when we should stop looping. abort_soon_signal = Event() - #signal gets set on CTRL-C/SIGTERM + # Signal gets set on CTRL-C/SIGTERM. abort_NOW_signal = Event() def __init__(self, config, name): @@ -72,6 +73,7 @@ class Account(CustomConfig.ConfigHelperMixin): :param name: A string denoting the name of the Account as configured""" + self.config = config self.name = name self.metadatadir = config.getmetadatadir() diff --git a/offlineimap/repository/GmailMaildir.py b/offlineimap/repository/GmailMaildir.py index b790c73..61352f8 100644 --- a/offlineimap/repository/GmailMaildir.py +++ b/offlineimap/repository/GmailMaildir.py @@ -23,8 +23,8 @@ class GmailMaildirRepository(MaildirRepository): def __init__(self, reposname, account): """Initialize a MaildirRepository object. Takes a path name to the directory holding all the Maildir directories.""" - super(GmailMaildirRepository, self).__init__(reposname, account) + super(GmailMaildirRepository, self).__init__(reposname, account) def getfoldertype(self): return GmailMaildirFolder From 29e9b7ab39c0f1589814d451204a105067f490ea Mon Sep 17 00:00:00 2001 From: Giovanni Mascellani Date: Sun, 5 Jan 2014 16:07:03 +0100 Subject: [PATCH 695/817] Drop caches after having processed folders. This enhances a lot memory consumption when you have many folders to process. Signed-off-by: Giovanni Mascellani --- offlineimap/accounts.py | 4 ++++ offlineimap/folder/Base.py | 3 +++ offlineimap/folder/IMAP.py | 2 ++ offlineimap/folder/LocalStatus.py | 2 ++ offlineimap/folder/LocalStatusSQLite.py | 3 +++ offlineimap/folder/Maildir.py | 3 +++ offlineimap/folder/UIDMaps.py | 3 +++ 7 files changed, 20 insertions(+) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 18a36fc..7adc3a4 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -491,3 +491,7 @@ def syncfolder(account, remotefolder, quick): ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ (account, remotefolder.getvisiblename(), traceback.format_exc())) + finally: + for folder in ["statusfolder", "localfolder", "remotefolder"]: + if folder in locals(): + locals()[folder].dropmessagelistcache() diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 87e165c..7121231 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -247,6 +247,9 @@ class BaseFolder(object): raise NotImplementedError + def dropmessagelistcache(self): + raise NotImplementedException + def getmessagelist(self): """Gets the current message list. You must call cachemessagelist() before calling this function!""" diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 00afce0..c405431 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -250,6 +250,8 @@ class IMAPFolder(BaseFolder): rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + def dropmessagelistcache(self): + self.messagelist = None # Interface from BaseFolder def getmessagelist(self): diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index ec34965..d3b6cf0 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -161,6 +161,8 @@ class LocalStatusFolder(BaseFolder): self.readstatus(file) file.close() + def dropmessagelistcache(self): + self.messagelist = None def save(self): """Save changed data to disk. For this backend it is the same as saveall.""" diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 1b2adb0..9d7e559 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -203,6 +203,9 @@ class LocalStatusSQLiteFolder(BaseFolder): self.messagelist[uid]['labels'] = labels self.messagelist[uid]['mtime'] = row[2] + def dropmessagelistcache(self): + self.messagelist = None + # Interface from LocalStatusFolder def save(self): pass diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index af1e5ff..3a19cc3 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -220,6 +220,9 @@ class MaildirFolder(BaseFolder): if self.messagelist is None: self.messagelist = self._scanfolder() + def dropmessagelistcache(self): + self.messagelist = None + # Interface from BaseFolder def getmessagelist(self): return self.messagelist diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index e1a6f5f..eb4fbae 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -125,6 +125,9 @@ class MappedIMAPFolder(IMAPFolder): finally: self.maplock.release() + def dropmessagelistcache(self): + self._mb.dropmessagelistcache() + # Interface from BaseFolder def uidexists(self, ruid): """Checks if the (remote) UID exists in this Folder""" From d08f6d15c21a1517ee27d894a051f3199cda4d8c Mon Sep 17 00:00:00 2001 From: Wieland Hoffmann Date: Sun, 11 Jan 2015 18:15:16 +0100 Subject: [PATCH 696/817] addmessageheader: Add a note about the incorrect rendering of the docstring The note tells people to look at the source of the method, which spinx.ext.viewcode conveniently links right next to the methods signature. Signed-off-by: Wieland Hoffmann --- docs/doc-src/conf.py | 3 ++- offlineimap/folder/Base.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/doc-src/conf.py b/docs/doc-src/conf.py index e961ab2..3f4bb6c 100644 --- a/docs/doc-src/conf.py +++ b/docs/doc-src/conf.py @@ -23,7 +23,8 @@ from offlineimap import __version__,__author__ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] autoclass_content = "both" # Add any paths that contain templates here, relative to this directory. diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 7121231..55b2bfc 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -441,6 +441,11 @@ class BaseFolder(object): - headername: name of the header to add - headervalue: value of the header to add + .. note:: + + The following documentation will not get displayed correctly after being + processed by Sphinx. View the source of this method to read it. + This has to deal with strange corner cases where the header is missing or empty. Here are illustrations for all the cases, showing where the header gets inserted and what the end result From 6fc9c3601452c902f5c815ba811560c2f9cae966 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 12 Jan 2015 19:15:06 +0300 Subject: [PATCH 697/817] Fix regression introduced in 0f40ca47998c7f56bf7cd1ce095c66fe93e21d03 We have no variable "fullname", it must have been slipped in unintentionally. Blame commit: 0f40ca47998c7f more style consistency Signed-off-by: Eygene Ryabinkin Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index c405431..e0c0cc7 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -571,7 +571,7 @@ class IMAPFolder(BaseFolder): #Do the APPEND try: - (typ, dat) = imapobj.append(fullname, + (typ, dat) = imapobj.append(self.getfullname(), imaputil.flagsmaildir2imap(flags), date, content) # This should only catch 'NO' responses since append() # will raise an exception for 'BAD' responses: From 41fa3ae4fceb92619e1ee23423b3115494d25c7b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 13 Jan 2015 13:59:52 +0100 Subject: [PATCH 698/817] more style consistency Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 4 ++-- offlineimap/init.py | 1 + offlineimap/repository/IMAP.py | 5 ++--- offlineimap/repository/LocalStatus.py | 3 ++- offlineimap/repository/Maildir.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 7adc3a4..73b8ff1 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -265,8 +265,8 @@ class SyncableAccount(Account): raise self.ui.error(e, exc_info()[2]) except Exception as e: - self.ui.error(e, exc_info()[2], msg = "While attempting to sync" - " account '%s'" % self) + self.ui.error(e, exc_info()[2], msg="While attempting to sync" + " account '%s'"% self) else: # after success sync, reset the looping counter to 3 if self.refreshperiod: diff --git a/offlineimap/init.py b/offlineimap/init.py index f53c9a5..8b20394 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -40,6 +40,7 @@ class OfflineImap: oi = OfflineImap() oi.run() """ + def run(self): """Parse the commandline and invoke everything""" # next line also sets self.config and self.ui diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 5b70966..c59b3a8 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -357,9 +357,8 @@ class IMAPRepository(BaseRepository): self.ui.error(e, exc_info()[2], 'Invalid folderinclude:') continue - retval.append(self.getfoldertype()(self.imapserver, - foldername, - self)) + retval.append(self.getfoldertype()( + self.imapserver, foldername, self)) finally: self.imapserver.releaseconnection(imapobj) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 1390347..170145b 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -94,7 +94,8 @@ class LocalStatusRepository(BaseRepository): self.forgetfolders() def getfolder(self, foldername): - """Return the Folder() object for a foldername""" + """Return the Folder() object for a foldername.""" + if foldername in self._folders: return self._folders[foldername] diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index be86b34..58bf664 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -170,8 +170,8 @@ class MaildirRepository(BaseRepository): self.debug(" skip this entry (not a directory)") # Not a directory -- not a folder. continue + # extension can be None. if extension: - # extension can be None which fails. foldername = os.path.join(extension, dirname) else: foldername = dirname From 5c56d4f8ab8864749c448490b91108f87ea59633 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 13 Jan 2015 18:18:03 +0100 Subject: [PATCH 699/817] docs: CodingGuidelines: remove duplicate content Signed-off-by: Nicolas Sebrecht --- docs/CodingGuidelines.rst | 51 --------------------------------------- 1 file changed, 51 deletions(-) diff --git a/docs/CodingGuidelines.rst b/docs/CodingGuidelines.rst index d6d29a2..92cc3be 100644 --- a/docs/CodingGuidelines.rst +++ b/docs/CodingGuidelines.rst @@ -13,57 +13,6 @@ This document contains assorted guidelines for programmers that want to hack OfflineIMAP. ------------------- -Exception handling ------------------- - -OfflineIMAP on many occasions re-raises various exceptions and often -changes exception type to `OfflineImapError`. This is not a problem -per se, but you must always remember that we need to preserve original -tracebacks. This is not hard if you follow these simple rules. - -For re-raising original exceptions, just use:: - - raise - -from inside your exception handling code. - -If you need to change exception type, or its argument, or whatever, -use this three-argument form:: - - raise YourExceptionClass(argum, ents), None, sys.exc_info()[2] - -In this form, you're creating an instance of new exception, so ``raise`` -will deduce its ``type`` and ``value`` parameters from the first argument, -thus the second expression passed to ``raise`` is always ``None``. -And the third one is the traceback object obtained from the thread-safe -``exc_info()`` function. - -In fact, if you hadn't already imported the whole ``sys`` module, it will -be better to import just ``exc_info()``:: - - from sys import exc_info - -and raise like this:: - - raise YourExceptionClass(argum, ents), None, exc_info()[2] - -since this is the historically-preferred style in the OfflineIMAP code. -.. -*- coding: utf-8 -*- -.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap -.. _OLI_git_repo: git://github.com/OfflineIMAP/offlineimap.git - -================================= -Coding guidelines for OfflineIMAP -================================= - -.. contents:: -.. .. sectnum:: - -This document contains assorted guidelines for programmers that want -to hack OfflineIMAP. - - ------------------ Exception handling ------------------ From ab3423d039393c3ac3e325012774a06b3a1964a6 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 13 Jan 2015 22:48:28 +0100 Subject: [PATCH 700/817] localeval: avoid redefining 'file' keyword Signed-off-by: Nicolas Sebrecht --- offlineimap/localeval.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/localeval.py b/offlineimap/localeval.py index 06adc57..a9494fb 100644 --- a/offlineimap/localeval.py +++ b/offlineimap/localeval.py @@ -31,10 +31,10 @@ class LocalEval: if path is not None: # FIXME: limit opening files owned by current user with rights set # to fixed mode 644. - file = open(path, 'r') + foo = open(path, 'r') module = imp.load_module( '', - file, + foo, path, ('', 'r', imp.PY_SOURCE)) for attr in dir(module): From 03eafcc00f36f9a840eb9d99c5f351d79219dd4b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 14 Jan 2015 17:12:08 +0100 Subject: [PATCH 701/817] offlineimap.conf: normalize style and format - improve style and reading for the eyes - improve wording - add minor comments - remove on duplicate option - give location of each +1 section Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 701 +++++++++++++++++++++++++++++------------------ 1 file changed, 433 insertions(+), 268 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index d953f0e..e0ef4f4 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -5,7 +5,8 @@ # More details can be found in the included user documention, which is # also available at: http://docs.offlineimap.org/en/latest/ -# NOTE: Settings generally support python interpolation. This means + +# NOTE 1: Settings generally support python interpolation. This means # values can contain python format strings which refer to other values # in the same section, or values in a special DEFAULT section. This # allows you for example to use common settings for multiple accounts: @@ -21,49 +22,58 @@ # # would set the trashfolder setting for your German Gmail accounts. -# NOTE2: This implies that any '%' needs to be encoded as '%%' +# NOTE 2: Above feature implies that any '%' needs to be encoded as '%%' -# NOTE3: Any variables that are subject to the environment variables +# NOTE 3: Any variable that is subject to the environment variables # ($NAME) and tilde (~username/~) expansions will receive tilde -# expansion first and only after this environment variables will be -# expanded in the resulting string. This behaviour is intentional +# expansion first and only after the environment variable will be +# expanded in the resulting string. This behaviour is intentional # as it coincides with typical shell expansion strategy. + ################################################## # General definitions ################################################## [general] -# This specifies where offlineimap is to store its metadata. +# This specifies where OfflineIMAP is to store its metadata. # This directory will be created if it does not already exist. # # Tilde and environment variable expansions will be performed. - +# #metadata = ~/.offlineimap -# This variable specifies which accounts are defined. Separate them -# with commas. Account names should be alphanumeric only. -# You will need to specify one section per account below. You may -# not use "general" for an account name. +# This option stands in the [general] section. +# +# This variable specifies which accounts are defined. Separate them with commas. +# Account names should be alphanumeric only. You will need to specify one +# section per account below. You may not use "general" for an account name. +# +# Always use ASCII characters only. +# accounts = Test -# Offlineimap can synchronize more than one account at a time. If you -# want to enable this feature, set the below value to something -# greater than 1. To force it to synchronize only one account at a -# time, set it to 1. -# -# Note: if you are using autorefresh and have more than one account, -# you must set this number to be >= to the number of accounts you have; -# since any given sync run never "finishes" due to a timer, you will never -# sync your additional accounts if this is 1. +# This option stands in the [general] section. +# +# Offlineimap can synchronize more than one account at a time. If you want to +# enable this feature, set the below value to something greater than 1. To +# force it to synchronize only one account at a time, set it to 1. +# +# NOTE: if you are using autorefresh and have more than one account, you must +# set this number to be >= to the number of accounts you have; since any given +# sync run never "finishes" due to a timer, you will never sync your additional +# accounts if this is 1. +# #maxsyncaccounts = 1 -# You can specify one or more user interface modules for OfflineIMAP -# to use. OfflineIMAP will try the first in the list, and if it -# fails, the second, and so forth. + +# This option stands in the [general] section. +# +# You can specify one or more user interface. OfflineIMAP will try the first in +# the list, and if it fails, the second, and so forth. # # The pre-defined options are: # Blinkenlights -- A fancy (terminal) interface @@ -75,22 +85,28 @@ accounts = Test # parsing. # # You can override this with a command-line option -u. - +# #ui = basic + +# This option stands in the [general] section. +# # If you try to synchronize messages to a folder which the IMAP server # considers read-only, OfflineIMAP will generate a warning. If you want # to suppress these warnings, set ignore-readonly to yes. Read-only # IMAP folders allow reading but not modification, so if you try to # change messages in the local copy of such a folder, the IMAP server # will prevent OfflineIMAP from propagating those changes to the IMAP -# server. Note that ignore-readonly is unrelated to the "readonly" +# server. Note that ignore-readonly is UNRELATED to the "readonly" # setting which prevents a repository from being modified at all. - +# #ignore-readonly = no + ########## Advanced settings +# This option stands in the [general] section. +# # You can give a Python source filename here and all config file # python snippets will be evaluated in the context of that file. # This allows you to e.g. define helper functions in the Python @@ -99,31 +115,36 @@ accounts = Test # # Tilde and environment variable expansions will be performed. # -# pythonfile = ~/.offlineimap.py -# +#pythonfile = ~/.offlineimap.py -# By default, OfflineIMAP will not exit due to a network error until -# the operating system returns an error code. Operating systems can sometimes -# take forever to notice this. Here you can activate a timeout on the -# socket. This timeout applies to individual socket reads and writes, -# not to an overall sync operation. You could perfectly well have a 30s -# timeout here and your sync still take minutes. + +# This option is in the [general] section. +# +# By default, OfflineIMAP will not exit due to a network error until the +# operating system returns an error code. Operating systems can sometimes take +# forever to notice this. Here you can activate a timeout on the socket. This +# timeout applies to individual socket reads and writes, not to an overall sync +# operation. You could perfectly well have a 30s timeout here and your sync +# still take minutes. # # Values in the 30-120 second range are reasonable. # # The default is to have no timeout beyond the OS. Times are given in seconds. # -# socktimeout = 60 +#socktimeout = 60 -# By default, OfflineIMAP will use fsync() to force data out to disk at -# opportune times to ensure consistency. This can, however, reduce -# performance. Users where /home is on SSD (Flash) may also wish to reduce -# write cycles. Therefore, you can disable OfflineIMAP's use of fsync(). -# Doing so will come at the expense of greater risk of message duplication -# in the event of a system crash or power loss. Default is fsync = true. -# Set fsync = false to disable fsync. + +# This option stands in the [general] section. # -# fsync = true +# By default, OfflineIMAP will use fsync() to force data out to disk at +# opportune times to ensure consistency. This can, however, reduce performance. +# Users where /home is on SSD (Flash) may also wish to reduce write cycles. +# Therefore, you can disable OfflineIMAP's use of fsync(). Doing so will come +# at the expense of greater risk of message duplication in the event of a system +# crash or power loss. Default is true. Set it to false to disable fsync. +# +#fsync = true + ################################################## # Mailbox name recorder @@ -131,7 +152,7 @@ accounts = Test [mbnames] -# offlineimap can record your mailbox names in a format you specify. +# OfflineIMAP can record your mailbox names in a format you specify. # You can define the header, each mailbox item, the separator, # and the footer. Here is an example for Mutt. # If enabled is yes, all six setting must be specified, even if they @@ -148,47 +169,46 @@ accounts = Test # # Tilde and environment variable expansions will be performed # for "filename" knob. +# +#enabled = no +#filename = ~/Mutt/muttrc.mailboxes +#header = "mailboxes " +#peritem = "+%(accountname)s/%(foldername)s" +#sep = " " +#footer = "\n" -enabled = no -filename = ~/Mutt/muttrc.mailboxes -header = "mailboxes " -peritem = "+%(accountname)s/%(foldername)s" -sep = " " -footer = "\n" -# You can also specify a folderfilter. It will apply to the -# *translated* folder name here, and it takes TWO arguments: -# accountname and foldername. In all other ways, it will -# behave identically to the folderfilter for accounts. Please see -# that section for more information and examples. +# This option stands in the [mbnames] section. # -# Note that this filter can be used only to further restrict mbnames -# to a subset of folders that pass the account's folderfilter. +# You can also specify a folderfilter. It will apply to the *translated* folder +# name here, and it takes TWO arguments: accountname and foldername. In all +# other ways, it will behave identically to the folderfilter for accounts. +# Please see the folderfilter option for more information and examples. # +# This filter can be used only to further restrict mbnames to a subset of +# folders that pass the account's folderfilter. # -# You can customize the order in which mailbox names are listed in the -# generated file by specifying a sort_keyfunc, which takes a single -# dict argument containing keys 'accountname' and 'foldername'. This -# function will be called once for each mailbox, and should return a -# suitable sort key that defines this mailbox' position in the custom -# ordering. +# You can customize the order in which mailbox names are listed in the generated +# file by specifying a sort_keyfunc, which takes a single dict argument +# containing keys 'accountname' and 'foldername'. This function will be called +# once for each mailbox, and should return a suitable sort key that defines this +# mailbox' position in the custom ordering. # -# This is useful with e.g. Mutt-sidebar, which uses the mailbox order -# from the generated file when listing mailboxes in the sidebar. +# This is useful with e.g. Mutt-sidebar, which uses the mailbox order from the +# generated file when listing mailboxes in the sidebar. # -# Default setting is -# sort_keyfunc = lambda d: (d['accountname'], d['foldername']) +# Default setting is: +#sort_keyfunc = lambda d: (d['accountname'], d['foldername']) ################################################## # Accounts ################################################## -# This is an account definition clause. You'll have one of these -# for each account listed in general/accounts above. +# This is an account definition clause. You'll have one of these for each +# account listed in the "accounts" option in [general] section (above). [Account Test] -########## Basic settings # These settings specify the two folders that you will be syncing. # You'll need to have a "Repository ..." section for each one. @@ -196,44 +216,58 @@ footer = "\n" localrepository = LocalExample remoterepository = RemoteExample + ########## Advanced settings -# You can have offlineimap continue running indefinitely, automatically -# syncing your mail periodically. If you want that, specify how -# frequently to do that (in minutes) here. You can also specify -# fractional minutes (ie, 3.25). +# This option stands in the [Account Test] section. +# +# You can have OfflineIMAP continue running indefinitely, automatically syncing +# your mail periodically. If you want that, specify how frequently to do that +# (in minutes) here. Fractional minutes (ie, 3.25) is allowed. +# +#autorefresh = 5 -# autorefresh = 5 -# OfflineImap can replace a number of full updates by quick -# synchronizations. It only synchronizes a folder if 1) a Maildir -# folder has changed, or 2) if an IMAP folder has received new messages -# or had messages deleted, ie it does not update if only IMAP flags have -# changed. Full updates need to fetch ALL flags for all messages, so -# this makes quite a performance difference (especially if syncing -# between two IMAP servers). -# Specify 0 for never, -1 for always (works even in non-autorefresh -# mode), or a positive integer to do quick updates before doing -# another full synchronization (requires autorefresh). Updates are -# always performed after minutes, be they quick or full. +# This option stands in the [Account Test] section. +# +# OfflineImap can replace a number of full updates by quick synchronizations. +# It only synchronizes a folder if +# +# 1) a Maildir folder has changed +# +# or +# +# 2) if an IMAP folder has received new messages or had messages deleted, ie +# it does not update if only IMAP flags have changed. +# +# Full updates need to fetch ALL flags for all messages, so this makes quite a +# performance difference (especially if syncing between two IMAP servers). +# +# Specify 0 for never, -1 for always (works even in non-autorefresh mode) +# +# A positive integer to do quick updates before doing another full +# synchronization (requires autorefresh). Updates are always performed after +# minutes, be they quick or full. +# +#quick = 10 -# quick = 10 -# You can specify a pre and post sync hook to execute a external command. -# In this case a call to imapfilter to filter mail before the sync process -# starts and a custom shell script after the sync completes. -# The pre sync script has to complete before a sync to the account will -# start. +# This option stands in the [Account Test] section. +# +# You can specify a pre and post sync hook to execute a external command. In +# this case a call to imapfilter to filter mail before the sync process starts +# and a custom shell script after the sync completes. +# +# The pre sync script has to complete before a sync to the account will start. +# +#presynchook = imapfilter -c someotherconfig.lua +#postsynchook = notifysync.sh -# presynchook = imapfilter -# postsynchook = notifysync.sh - -# You can also specify parameters to the commands -# presynchook = imapfilter -c someotherconfig.lua +# This option stands in the [Account Test] section. +# # OfflineImap caches the state of the synchronisation to e.g. be able to -# determine if a mail has been deleted on one side or added on the -# other. +# determine if a mail has been added or deleted on either side. # # The default and historical backend is 'plain' which writes out the # state in plain text files. On Repositories with large numbers of @@ -247,66 +281,92 @@ remoterepository = RemoteExample # #status_backend = plain + +# This option stands in the [Account Test] section. +# # If you have a limited amount of bandwidth available you can exclude larger -# messages (e.g. those with large attachments etc). If you do this it -# will appear to offlineimap that these messages do not exist at all. They -# will not be copied, have flags changed etc. For this to work on an IMAP -# server the server must have server side search enabled. This works with Gmail -# and most imap servers (e.g. cyrus etc) +# messages (e.g. those with large attachments etc). If you do this it will +# appear to OfflineIMAP that these messages do not exist at all. They will not +# be copied, have flags changed etc. For this to work on an IMAP server the +# server must have server side search enabled. This works with Gmail and most +# imap servers (e.g. cyrus etc) +# # The maximum size should be specified in bytes - e.g. 2000000 for approx 2MB - -# maxsize = 2000000 +# +#maxsize = 2000000 +# This option stands in the [Account Test] section. +# # When you are starting to sync an already existing account you can tell -# offlineimap to sync messages from only the last x days. When you do -# this messages older than x days will be completely ignored. This can -# be useful for importing existing accounts when you do not want to -# download large amounts of archive email. +# OfflineIMAP to sync messages from only the last x days. When you do this, +# messages older than x days will be completely ignored. This can be useful for +# importing existing accounts when you do not want to download large amounts of +# archive email. # -# Messages older than maxage days will not be synced, their flags will -# not be changed, they will not be deleted etc. For offlineimap it will -# be like these messages do not exist. This will perform an IMAP search -# in the case of IMAP or Gmail and therefore requires that the server -# support server side searching. This will calculate the earliest day -# that would be included in the search and include all messages from -# that day until today. e.g. maxage = 3 to sync only the last 3 days -# mail +# Messages older than maxage days will not be synced, their flags will not be +# changed, they will not be deleted, etc. For OfflineIMAP it will be like these +# messages do not exist. This will perform an IMAP search in the case of IMAP +# or Gmail and therefore requires that the server support server side searching. +# This will calculate the earliest day that would be included in the search and +# include all messages from that day until today. The maxage option expects an +# integer (for the number of days). # -# maxage = +#maxage = 3 +# This option stands in the [Account Test] section. +# # Maildir file format uses colon (:) separator between uniq name and info. # Unfortunatelly colon is not allowed character in windows file name. If you -# enable maildir-windows-compatible option, offlineimap will be able to store +# enable maildir-windows-compatible option, OfflineIMAP will be able to store # messages on windows drive, but you will probably loose compatibility with -# other programs working with the maildir +# other programs working with the maildir. # #maildir-windows-compatible = no + +# This option stands in the [Account Test] section. +# # Specifies if we want to sync GMail labels with the local repository. # Effective only for GMail IMAP repositories. # +# Non-ASCII characters in labels are bad handled or won't work at all. +# #synclabels = no + +# This option stands in the [Account Test] section. +# # Name of the header to use for label storage. Format for the header # value differs for different headers, because there are some de-facto -# standards set by popular clients: +# "standards" set by popular clients: +# # - X-Label or Keywords keep values separated with spaces; for these # you, obviously, should not have label values that contain spaces; +# # - X-Keywords use comma (',') as the separator. +# # To be consistent with the usual To-like headers, for the rest of header # types we use comma as the separator. # +# Use ASCII characters only. +# #labelsheader = X-Keywords + +# This option stands in the [Account Test] section. +# # Set of labels to be ignored. Comma-separated list. GMail-specific # labels all start with backslash ('\'). # +# Use ASCII characters only. +# #ignorelabels = \Inbox, \Starred, \Sent, \Draft, \Spam, \Trash, \Important - +# This option stands in the [Account Test] section. +# # OfflineIMAP can strip off some headers when your messages are propagated # back to the IMAP server. This option carries the comma-separated list # of headers to trim off. Header name matching is case-sensitive. @@ -315,23 +375,30 @@ remoterepository = RemoteExample # for GMail-based accounts is automatically added to this list, you don't # need to specify it explicitely. # +# Use ASCII characters only. +# #filterheaders = X-Some-Weird-Header - [Repository LocalExample] # Each repository requires a "type" declaration. The types supported for # local repositories are Maildir, GmailMaildir and IMAP. - +# type = Maildir + +# This option stands in the [Repository LocalExample] section. +# # Specify local repository. Your IMAP folders will be synchronized # to maildirs created under this path. OfflineIMAP will create the # maildirs for you as needed. - +# localfolders = ~/Test + +# This option stands in the [Repository LocalExample] section. +# # You can specify the "folder separator character" used for your Maildir # folders. It is inserted in-between the components of the tree. If you # want your folders to be nested directories, set it to "/". 'sep' is @@ -339,6 +406,9 @@ localfolders = ~/Test # #sep = . + +# This option stands in the [Repository LocalExample] section. +# # Some users may not want the atime (last access time) of folders to be # modified by OfflineIMAP. If 'restoreatime' is set to yes, OfflineIMAP # will restore the atime of the "new" and "cur" folders in each maildir @@ -349,7 +419,6 @@ localfolders = ~/Test #restoreatime = no - [Repository GmailLocalExample] # This type of repository enables syncing of Gmail. All Maildir @@ -362,43 +431,64 @@ localfolders = ~/Test # time OfflineIMAP runs with synclabels enabled, it will have to check # the contents of all individual messages for labels and this may take # a while. - +# type = GmailMaildir - [Repository RemoteExample] -# And this is the remote repository. We only support IMAP or Gmail here. +# The remote repository. We only support IMAP or Gmail here. +# type = IMAP + +# These options stands in the [Repository RemoteExample] section. +# # The following can fetch the account credentials via a python expression that # is parsed from the pythonfile parameter. For example, a function called # "getcredentials" that parses a file "filename" and returns the account # details for "hostname". -# remotehosteval = getcredentials("filename", "hostname", "hostname") -# remoteporteval = getcredentials("filename", "hostname", "port") -# remoteusereval = getcredentials("filename", "hostname", "user") -# remotepasseval = getcredentials("filename", "hostname", "passwd") +# +#remotehosteval = getcredentials("filename", "hostname", "hostname") +#remoteporteval = getcredentials("filename", "hostname", "port") +#remoteusereval = getcredentials("filename", "hostname", "user") +#remotepasseval = getcredentials("filename", "hostname", "passwd") + +# This option stands in the [Repository RemoteExample] section. +# # Specify the remote hostname. +# remotehost = examplehost + +# This option stands in the [Repository RemoteExample] section. +# # Whether or not to use SSL. -ssl = yes +# +#ssl = yes -# SSL Client certificate (optional) + +# This option stands in the [Repository RemoteExample] section. +# +# SSL Client certificate (optional). # # Tilde and environment variable expansions will be performed. +# +#sslclientcert = /path/to/file.crt -# sslclientcert = /path/to/file.crt -# SSL Client key (optional) +# This option stands in the [Repository RemoteExample] section. +# +# SSL Client key (optional). # # Tilde and environment variable expansions will be performed. +# +#sslclientkey = /path/to/file.key -# sslclientkey = /path/to/file.key +# This option stands in the [Repository RemoteExample] section. +# # SSL CA Cert(s) to verify the server cert against (optional). # No SSL verification is done without this option. If it is # specified, the CA Cert(s) need to verify the Server cert AND @@ -406,15 +496,18 @@ ssl = yes # The certificate should be in PEM format. # # Tilde and environment variable expansions will be performed. +# +#sslcacertfile = /path/to/cacertfile.crt -# sslcacertfile = /path/to/cacertfile.crt -# If you connect via SSL/TLS (ssl=true) and you have no CA certificate -# specified, offlineimap will refuse to sync as it connects to a server +# This option stands in the [Repository RemoteExample] section. +# +# If you connect via SSL/TLS (ssl = yes) and you have no CA certificate +# specified, OfflineIMAP will refuse to sync as it connects to a server # with an unknown "fingerprint". If you are sure you connect to the # correct server, you can then configure the presented server # fingerprint here. OfflineImap will verify that the server fingerprint -# has not changed on each connect and refuse to connect otherwise. +# has not changed on each connection and refuse to connect otherwise. # You can also configure this in addition to CA certificate validation # above and it will check both ways. # @@ -422,26 +515,41 @@ ssl = yes # # Fingerprints must be in hexadecimal form without leading '0x': # 40 hex digits like bbfe29cf97acb204591edbafe0aa8c8f914287c9. - +# #cert_fingerprint = [, ] -# SSL version (optional) + +# This option stands in the [Repository RemoteExample] section. +# +# SSL version (optional). +# # It is best to leave this unset, in which case the correct version will be # automatically detected. In rare cases, it may be necessary to specify a # particular version from: tls1, ssl2, ssl3, ssl23 (SSLv2 or SSLv3) +# +#ssl_version = ssl23 -# ssl_version = ssl23 +# This option stands in the [Repository RemoteExample] section. +# # Specify the port. If not specified, use a default port. -# remoteport = 993 +# +#remoteport = 993 + +# This option stands in the [Repository RemoteExample] section. +# # Specify the remote user name. +# remoteuser = username + +# This option stands in the [Repository RemoteExample] section. +# # Specify the user to be authorized as. Sometimes we want to # authenticate with our login/password, but tell the server that we # really want to be treated as some other user; perhaps server will -# allow us to do that (or, may be, not). Some IMAP servers migrate +# allow us to do that (or maybe not). Some IMAP servers migrate # account names using this functionality: your credentials remain # intact, but remote identity changes. # @@ -449,20 +557,23 @@ remoteuser = username # mechanism, so consider using auth_mechanisms to prioritize PLAIN # or even make it the only mechanism to be tried. # -# remote_identity = authzuser +#remote_identity = authzuser -# Specify which authentication/authorization mechanisms we should try -# and the order in which OfflineIMAP will try them. NOTE: any given -# mechanism will be tried only if it is supported by the remote IMAP -# server. + +# This option stands in the [Repository RemoteExample] section. # -# Due to the technical limitations, if you're specifying GSSAPI -# as the mechanism to try, it will be tried first, no matter where -# it was specified in the list. +# Specify which authentication/authorization mechanisms we should try and the +# order in which OfflineIMAP will try them. # -# Default value is -# auth_mechanisms = GSSAPI, CRAM-MD5, PLAIN, LOGIN -# ranged is from strongest to more weak ones. +# NOTE: any given mechanism will be tried ONLY if it is supported by the remote +# IMAP server. +# +# Default value is ranged is from strongest to more weak ones. Due to technical +# limitations, if GSSAPI is set, it will be tried first, no matter where it was +# specified in the list. +# +#auth_mechanisms = GSSAPI, CRAM-MD5, PLAIN, LOGIN + ########## Passwords @@ -507,9 +618,13 @@ remoteuser = username # This method can be used to design more elaborate setups, e.g. by # querying the gnome-keyring via its python bindings. + ########## Advanced settings -# Tunnels. There are two types: + +# These options stands in the [Repository RemoteExample] section. +# +# Tunnels. There are two types: # # - preauth: they teleport your connection to the remote system # and you don't need to authenticate yourself there; the sole @@ -521,57 +636,73 @@ remoteuser = username # IMAP server. # # Tunnels are currently working only with IMAP servers and their -# derivatives (currently, GMail). Additionally, for GMail accounts +# derivatives (GMail currently). Additionally, for GMail accounts # preauth tunnel settings are ignored: we don't believe that there # are ways to preauthenticate at Google mail system IMAP servers. # -# You must choose at most one tunnel type, be wise M'Lord. +# You must choose at most one tunnel type, be wise M'Lord! # -# preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' -# transporttunnel = openssl s_client -host myimap -port 993 -quiet +#preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' +#transporttunnel = openssl s_client -host myimap -port 993 -quiet -# Some IMAP servers need a "reference" which often refers to the "folder -# root". This is most commonly needed with UW IMAP, where you might -# need to specify the directory in which your mail is stored. The -# 'reference' value will be prefixed to all folder paths refering to -# that repository. E.g. accessing folder 'INBOX' with reference = Mail -# will try to access Mail/INBOX. Note that the nametrans and -# folderfilter functions will still apply the full path including the -# reference prefix. Most users will not need this. + +# This option stands in the [Repository RemoteExample] section. # -# reference = Mail +# Some IMAP servers need a "reference" which often refers to the "folder root". +# +# This is most commonly needed with UW IMAP, where you might need to specify the +# directory in which your mail is stored. The 'reference' value will be prefixed +# to all folder paths refering to that repository. E.g. accessing folder 'INBOX' +# with "reference = Mail" will try to access Mail/INBOX. +# +# The nametrans and folderfilter functions will apply to the full path, +# including the reference prefix. Most users will not need this. +# +#reference = Mail + +# This option stands in the [Repository RemoteExample] section. +# # In between synchronisations, OfflineIMAP can monitor mailboxes for new -# messages using the IDLE command. If you want to enable this, specify here -# the folders you wish to monitor. Note that the IMAP protocol requires a -# separate connection for each folder monitored in this way, so setting -# this option will force settings for: -# maxconnections - to be at least the number of folders you give -# holdconnectionopen - to be true -# keepalive - to be 29 minutes unless you specify otherwise +# messages using the IDLE command. If you want to enable this, specify here the +# folders you wish to monitor. IMAP protocol requires a separate connection for +# each folder monitored in this way, so setting this option will force settings +# for: +# +# - maxconnections: to be at least the number of folders you give +# - holdconnectionopen: to be true +# - keepalive: to be 29 minutes unless you specify otherwise # # This feature isn't complete and may well have problems. See the "Known Issues" # entry in the manual for more details. # # This option should return a Python list. For example # -# idlefolders = ['INBOX', 'INBOX.Alerts'] -# +#idlefolders = ['INBOX', 'INBOX.Alerts'] + +# This option stands in the [Repository RemoteExample] section. +# # OfflineIMAP can use a compressed connection to the IMAP server. # This can result in faster downloads for some cases. # #usecompression = yes + +# This option stands in the [Repository RemoteExample] section. +# # OfflineIMAP can use multiple connections to the server in order # to perform multiple synchronization actions simultaneously. # This may place a higher burden on the server. In most cases, # setting this value to 2 or 3 will speed up the sync, but in some # cases, it may slow things down. The safe answer is 1. You should # probably never set it to a value more than 5. - +# #maxconnections = 2 + +# This option stands in the [Repository RemoteExample] section. +# # OfflineIMAP normally closes IMAP server connections between refreshes if # the global option autorefresh is specified. If you wish it to keep the # connection open, set this to true. If not specified, the default is @@ -581,151 +712,185 @@ remoteuser = username # #holdconnectionopen = no -# If you want to have "keepalives" sent while waiting between syncs, -# specify the amount of time IN SECONDS between keepalives here. Note that -# sometimes more than this amount of time might pass, so don't make it -# tight. This setting has no effect if autorefresh and holdconnectionopen -# are not both set. -# -# keepalive = 60 -# Normally, OfflineIMAP will expunge deleted messages from the server. -# You can disable that if you wish. This means that OfflineIMAP will -# mark them deleted on the server, but not actually delete them. -# You must use some other IMAP client to delete them if you use this -# setting; otherwise, the messages will just pile up there forever. -# Therefore, this setting is definitely NOT recommended. +# This option stands in the [Repository RemoteExample] section. +# +# If you want to have "keepalives" sent while waiting between syncs, specify the +# amount of time IN SECONDS between keepalives here. Note that sometimes more +# than this amount of time might pass, so don't make it tight. This setting has +# no effect if autorefresh and holdconnectionopen are not both set. +# +#keepalive = 60 + + +# This option stands in the [Repository RemoteExample] section. +# +# Normally, OfflineIMAP will expunge deleted messages from the server. You can +# disable that if you wish. This means that OfflineIMAP will mark them deleted +# on the server, but not actually delete them. You must use some other IMAP +# client to delete them if you use this setting; otherwise, the messages will +# just pile up there forever. Therefore, this setting is definitely NOT +# recommended for a long term. # #expunge = no + +# This option stands in the [Repository RemoteExample] section. +# # Specify whether to process all mail folders on the server, or only # those listed as "subscribed". # #subscribedonly = no -# You can specify a folder translator. This must be a eval-able -# Python expression that takes a foldername arg and returns the new -# value. I suggest a lambda. This example below will remove "INBOX." from -# the leading edge of folders (great for Courier IMAP users) + +# This option stands in the [Repository RemoteExample] section. # -# See the user documentation for details and use cases. They are also -# online at: +# You can specify a folder translator. This must be a eval-able. +# +# Python expression that takes a foldername arg and returns the new value. A +# lambda function is suggested. +# +# WARNING: you MUST construct it so that it NEVER returns the same value for two +# folders, UNLESS the second values are filtered out by folderfilter below. +# Failure to follow this rule will result in undefined behavior. +# +# See the user documentation for details and use cases. They are also online at: # http://docs.offlineimap.org/en/latest/nametrans.html # -# WARNING: you MUST construct this such that it NEVER returns -# the same value for two folders, UNLESS the second values are -# filtered out by folderfilter below. Failure to follow this rule -# will result in undefined behavior +# This example below will remove "INBOX." from the leading edge of folders +# (great for Courier IMAP users). +# +#nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) # -# nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) - # Using Courier remotely and want to duplicate its mailbox naming # locally? Try this: # -# nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) +#nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) -# Determines if folderfilter will be invoked on each run -# (dynamic folder filtering) or filtering status will be determined -# at startup (default behaviour). + +# This option stands in the [Repository RemoteExample] section. # -# dynamic_folderfilter = False +# Determines if folderfilter will be invoked on each run (dynamic folder +# filtering) or filtering status will be determined at startup (default +# behaviour). +# +#dynamic_folderfilter = False -# You can specify which folders to sync using the folderfilter -# setting. You can provide any python function (e.g. a lambda function) -# which will be invoked for each foldername. If the filter function -# returns True, the folder will be synced, if it returns False, it. The -# folderfilter operates on the *UNTRANSLATED* name (before any nametrans + +# This option stands in the [Repository RemoteExample] section. +# +# You can specify which folders to sync using the folderfilter setting. You can +# provide any python function (e.g. a lambda function) which will be invoked for +# each foldername. If the filter function returns True, the folder will be +# synced, if it returns False, it. +# +# The folderfilter operates on the *UNTRANSLATED* name (before any nametrans # translation takes place). # # Example 1: synchronizing only INBOX and Sent. # -# folderfilter = lambda foldername: foldername in ['INBOX', 'Sent'] +#folderfilter = lambda foldername: foldername in ['INBOX', 'Sent'] # # Example 2: synchronizing everything except Trash. # -# folderfilter = lambda foldername: foldername not in ['Trash'] +#folderfilter = lambda foldername: foldername not in ['Trash'] # # Example 3: Using a regular expression to exclude Trash and all folders # containing the characters "Del". # -# folderfilter = lambda foldername: not re.search('(^Trash$|Del)', foldername) +#folderfilter = lambda foldername: not re.search('(^Trash$|Del)', foldername) # -# If folderfilter is not specified, ALL remote folders will be -# synchronized. +# If folderfilter is not specified, ALL remote folders will be synchronized. # -# You can span multiple lines by indenting the others. (Use backslashes -# at the end when required by Python syntax) For instance: +# You can span multiple lines by indenting the others. (Use backslashes at the +# end when required by Python syntax) For instance: # -# folderfilter = lambda foldername: foldername in -# ['INBOX', 'Sent Mail', 'Deleted Items', -# 'Received'] +#folderfilter = lambda foldername: foldername in [ +# 'INBOX', 'Sent Mail', +# 'Deleted Items', 'Received'] - -# You can specify folderincludes to include additional folders. It -# should return a Python list. This might be used to include a folder -# that was excluded by your folderfilter rule, to include a folder that -# your server does not specify with its LIST option, or to include a -# folder that is outside your basic reference. The 'reference' value -# will not be prefixed to this folder name, even if you have specified -# one. For example: -# folderincludes = ['debian.user', 'debian.personal'] +# This option stands in the [Repository RemoteExample] section. +# +# You can specify folderincludes to include additional folders. It should +# return a Python list. This might be used to include a folder that was +# excluded by your folderfilter rule, to include a folder that your server does +# not specify with its LIST option, or to include a folder that is outside your +# basic reference. +# +# The 'reference' value will not be prefixed to this folder name, even if you +# have specified one. For example: +# +#folderincludes = ['debian.user', 'debian.personal'] +# This option stands in the [Repository RemoteExample] section. +# # If you do not want to have any folders created on this repository, # set the createfolders variable to False, the default is True. Using # this feature you can e.g. disable the propagation of new folders to # the new repository. +# #createfolders = True -# You can specify 'foldersort' to determine how folders are sorted. -# This affects order of synchronization and mbnames. The expression -# should return -1, 0, or 1, as the default Python cmp() does. The two -# arguments, x and y, are strings representing the names of the folders -# to be sorted. The sorting is applied *AFTER* nametrans, if any. The -# default is to sort IMAP folders alphabetically -# (case-insensitive). Usually, you should never have to modify this. To -# eg. reverse the sort: +# This option stands in the [Repository RemoteExample] section. # -# foldersort = lambda x, y: -cmp(x, y) +# 'foldersort' determines how folders are sorted. +# +# This affects order of synchronization and mbnames. The expression should +# return -1, 0, or 1, as the default Python cmp() does. The two arguments, x +# and y, are strings representing the names of the folders to be sorted. The +# sorting is applied *AFTER* nametrans, if any. The default is to sort IMAP +# folders alphabetically (case-insensitive). Usually, you should never have to +# modify this. To eg. reverse the sort: +# +#foldersort = lambda x, y: -cmp(x, y) + +# This option stands in the [Repository RemoteExample] section. +# # Enable 1-way synchronization. When setting 'readonly' to True, this -# repository will not be modified during synchronization. Use to +# repository will not be modified during synchronization. Usefull to # e.g. backup an IMAP server. The readonly setting can be applied to any # type of Repository (Maildir, Imap, etc). # #readonly = False + [Repository GmailExample] -# A repository using Gmail's IMAP interface. Any configuration -# parameter of `IMAP` type repositories can be used here. Only -# `remoteuser` (or `remoteusereval` ) is mandatory. Default values -# for other parameters are OK, and you should not need fiddle with -# those. +# A repository using Gmail's IMAP interface. +# +# Any configuration parameter of "IMAP" type repositories can be used here. +# Only "remoteuser" (or "remoteusereval" ) is mandatory. Default values for +# other parameters are OK, and you should not need fiddle with those. +# +# The Gmail repository will use hard-coded values for "remotehost", +# "remoteport", "tunnel" and "ssl". Any attempt to set those parameters will be +# silently ignored. For details, see +# +# http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814 +# +# To enable GMail labels synchronisation, set the option "synclabels" +# in the corresponding "Account" section. # -# The Gmail repository will use hard-coded values for `remotehost`, -# `remoteport`, `tunnel` and `ssl`. (See -# http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814) -# Any attempt to set those parameters will be silently ignored. - type = Gmail + +# This option stands in the [Repository GmailExample] section. +# # Specify the Gmail user name. This is the only mandatory parameter. +# remoteuser = username@gmail.com -# The trash folder name may be different from [Gmail]/Trash -# for example on German Gmail, this setting should be -# -# trashfolder = [Gmail]/Papierkorb -# -# You should look for the localized names of the spam folder too: -# "spamfolder" tunable will help you to override the standard name. -# Enable 1-way synchronization. See above for explanation. +# This option stands in the [Repository GmailExample] section. # -#readonly = False +# The trash folder name may be different from [Gmail]/Trash due to localization. +# You should look for the localized names of the spam folder too: "spamfolder" +# tunable will help you to override the standard name. # -# To enable GMail labels synchronisation, set the option synclabels -# in the corresponding "Account" section. +# For example on German Gmail, this setting should be: +# +#trashfolder = [Gmail]/Papierkorb From 7b2fbeee747c55c40d0e9a34ef54a789cf3510e2 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 14 Jan 2015 18:00:15 +0100 Subject: [PATCH 702/817] repository: GmailMaildir: fix copyright Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/GmailMaildir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/repository/GmailMaildir.py b/offlineimap/repository/GmailMaildir.py index 61352f8..71f89ac 100644 --- a/offlineimap/repository/GmailMaildir.py +++ b/offlineimap/repository/GmailMaildir.py @@ -1,5 +1,5 @@ # Maildir repository support -# Copyright (C) 2002 John Goerzen +# Copyright (C) 2002-2015 John Goerzen & contributors # # # This program is free software; you can redistribute it and/or modify From 6d46070e11ebe488a164417ae4b62df59d85a1c9 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 12 Jan 2015 17:38:03 +0100 Subject: [PATCH 703/817] MAINTAINERS: add mailing list maintainers Signed-off-by: Nicolas Sebrecht --- MAINTAINERS | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index f237f00..53b54d8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10,10 +10,22 @@ Eygene Ryabinkin email: rea at freebsd.org github: konvpalto +Sebastian Spaeth + email: sebastian at sspaeth.de + github: spaetz + Nicolas Sebrecht email: nicolas.s-dev at laposte.net github: nicolas33 +MAILING LIST MAINTAINERS +======================== + +Eygene Ryabinkin + email: rea at freebsd.org + Sebastian Spaeth email: sebastian at sspaeth.de - github: spaetz + +Nicolas Sebrecht + email: nicolas.s-dev at laposte.net From c5eb4c9432b6b7f2ffe4f3201cea6e703f736f51 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 15 Jan 2015 11:41:35 +0100 Subject: [PATCH 704/817] COPYING: fix unexpected characters Signed-off-by: Nicolas Sebrecht --- COPYING | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/COPYING b/COPYING index 6526be4..bb94f18 100644 --- a/COPYING +++ b/COPYING @@ -65,7 +65,7 @@ patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. - + GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION @@ -120,7 +120,7 @@ above, provided that you also meet all of these conditions: License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) - + These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in @@ -178,7 +178,7 @@ access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. - + 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is @@ -235,7 +235,7 @@ impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - + 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License @@ -288,7 +288,7 @@ PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS - + How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest From f69613965f27401b7009fc50560dff0e05735f94 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Sun, 18 Jan 2015 10:45:46 +0300 Subject: [PATCH 705/817] Make OS-default CA certificate file to be requested explicitely This simplifies logics for the user, especially if he uses both fingerprint and certificate validation: it is hard to maintain the compatibility with the prior behaviour and to avoid getting default CA bundle to be disabled when fingerprint verification is requested. See http://thread.gmane.org/gmane.mail.imap.offlineimap.general/6695 for discussion about this change. Default CA bundle is requested via 'sslcertfile = OS-DEFAULT'. I had also enforced all cases where explicitely-requested CA bundles are non-existent to be hard errors: when users asks us to use CA bundle (and, thus, certificate validation), but we can't find one, we must error out rather than happily continue and downgrade to no validation. Reported-By: Edd Barrett Reviewed-By: Nicolas Sebrecht Signed-off-by: Eygene Ryabinkin --- offlineimap.conf | 22 ++++++++++++++---- offlineimap/imapserver.py | 8 +++---- offlineimap/repository/IMAP.py | 42 ++++++++++++++++++++++++++++------ offlineimap/utils/distro.py | 41 +++++++++++++++++++++++---------- 4 files changed, 86 insertions(+), 27 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index e0ef4f4..cfafb23 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -497,6 +497,17 @@ remotehost = examplehost # # Tilde and environment variable expansions will be performed. # +# Special value OS-DEFAULT makes OfflineIMAP to automatically +# determine system-wide location of standard trusted CA roots file +# for known OS distributions and use the first bundle encountered +# (if any). If no system-wide CA bundle is found, OfflineIMAP +# will refuse to continue; this is done to prevent creation +# of false security expectations ("I had configured CA bundle, +# thou certificate verification shalt be present"). +# +# You can also use fingerprint verification via cert_fingerprint. +# See below for more verbose explanation. +# #sslcacertfile = /path/to/cacertfile.crt @@ -506,10 +517,13 @@ remotehost = examplehost # specified, OfflineIMAP will refuse to sync as it connects to a server # with an unknown "fingerprint". If you are sure you connect to the # correct server, you can then configure the presented server -# fingerprint here. OfflineImap will verify that the server fingerprint -# has not changed on each connection and refuse to connect otherwise. -# You can also configure this in addition to CA certificate validation -# above and it will check both ways. +# fingerprint here. OfflineIMAP will verify that the server fingerprint +# has not changed on each connect and refuse to connect otherwise. +# +# You can also configure fingerprint validation in addition to +# CA certificate validation above and it will check both: +# OfflineIMAP fill verify certificate first and if things will be fine, +# fingerprint will be validated. # # Multiple fingerprints can be specified, separated by commas. # diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index bac681e..1275092 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -81,9 +81,10 @@ class IMAPServer: self.sslclientcert = repos.getsslclientcert() self.sslclientkey = repos.getsslclientkey() self.sslcacertfile = repos.getsslcacertfile() - self.sslversion = repos.getsslversion() if self.sslcacertfile is None: self.__verifycert = None # disable cert verification + self.fingerprint = repos.get_ssl_fingerprint() + self.sslversion = repos.getsslversion() self.delim = None self.root = None @@ -394,7 +395,6 @@ class IMAPServer: success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) - fingerprint = self.repos.get_ssl_fingerprint() imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, self.port, self.sslclientkey, @@ -403,7 +403,7 @@ class IMAPServer: self.__verifycert, self.sslversion, timeout=socket.getdefaulttimeout(), - fingerprint=fingerprint + fingerprint=self.fingerprint ) else: self.ui.connecting(self.hostname, self.port) @@ -468,7 +468,7 @@ class IMAPServer: (self.hostname, self.repos) raise OfflineImapError(reason, severity), None, exc_info()[2] - elif isinstance(e, SSLError) and e.errno == 1: + elif isinstance(e, SSLError) and e.errno == errno.EPERM: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 993: diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index c59b3a8..c0cd887 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -25,7 +25,7 @@ from offlineimap.repository.Base import BaseRepository from offlineimap import folder, imaputil, imapserver, OfflineImapError from offlineimap.folder.UIDMaps import MappedIMAPFolder from offlineimap.threadutil import ExitNotifyThread -from offlineimap.utils.distro import get_os_sslcertfile +from offlineimap.utils.distro import get_os_sslcertfile, get_os_sslcertfile_searchpath class IMAPRepository(BaseRepository): @@ -201,16 +201,44 @@ class IMAPRepository(BaseRepository): return self.getconf_xform('sslclientkey', xforms, None) def getsslcacertfile(self): - """Return the absolute path of the CA certfile to use, if any""" + """Determines CA bundle. + + Returns path to the CA bundle. It is either explicitely specified + or requested via "OS-DEFAULT" value (and we will search known + locations for the current OS and distribution). + + If search via "OS-DEFAULT" route yields nothing, we will throw an + exception to make our callers distinguish between not specified + value and non-existent default CA bundle. + + It is also an error to specify non-existent file via configuration: + it will error out later, but, perhaps, with less verbose explanation, + so we will also throw an exception. It is consistent with + the above behaviour, so any explicitely-requested configuration + that doesn't result in an existing file will give an exception. + """ + xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] - cacertfile = self.getconf_xform('sslcacertfile', xforms, - get_os_sslcertfile()) + cacertfile = self.getconf_xform('sslcacertfile', xforms, None) + if self.getconf('sslcacertfile', None) == "OS-DEFAULT": + cacertfile = get_os_sslcertfile() + if cacertfile == None: + searchpath = get_os_sslcertfile_searchpath() + if searchpath: + reason = "Default CA bundle was requested, "\ + "but no existing locations available. "\ + "Tried %s." % (", ".join(searchpath)) + else: + reason = "Default CA bundle was requested, "\ + "but OfflineIMAP doesn't know any for your "\ + "current operating system." + raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) if cacertfile is None: return None if not os.path.isfile(cacertfile): - raise SyntaxWarning("CA certfile for repository '%s' could " - "not be found. No such file: '%s'" \ - % (self.name, cacertfile)) + reason = "CA certfile for repository '%s' couldn't be found. "\ + "No such file: '%s'" % (self.name, cacertfile) + raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) return cacertfile def getsslversion(self): diff --git a/offlineimap/utils/distro.py b/offlineimap/utils/distro.py index 7c944b9..8cd2b79 100644 --- a/offlineimap/utils/distro.py +++ b/offlineimap/utils/distro.py @@ -14,7 +14,6 @@ import os __DEF_OS_LOCATIONS = { 'freebsd': '/usr/local/share/certs/ca-root-nss.crt', 'openbsd': '/etc/ssl/cert.pem', - 'netbsd': None, 'dragonfly': '/etc/ssl/cert.pem', 'darwin': [ # MacPorts, port curl-ca-bundle @@ -48,6 +47,26 @@ def get_os_name(): return OS +def get_os_sslcertfile_searchpath(): + """Returns search path for CA bundle for the current OS. + + We will return an iterable even if configuration has just + a single value: it is easier for our callers to be sure + that they can iterate over result. + + Returned value of None means that there is no search path + at all. + """ + + OS = get_os_name() + + l = None + if OS in __DEF_OS_LOCATIONS: + l = __DEF_OS_LOCATIONS[OS] + if not hasattr(l, '__iter__'): + l = (l, ) + return l + def get_os_sslcertfile(): """ @@ -57,18 +76,16 @@ def get_os_sslcertfile(): Returns the location of the file or None if there is no known CA certificate file or all known locations correspond to non-existing filesystem objects. - """ - OS = get_os_name() - if OS in __DEF_OS_LOCATIONS: - l = __DEF_OS_LOCATIONS[OS] - if not hasattr(l, '__iter__'): - l = (l, ) - for f in l: - assert (type(f) == type("")) - if os.path.exists(f) and \ - (os.path.isfile(f) or os.path.islink(f)): - return f + l = get_os_sslcertfile_searchpath() + if l == None: + return None + + for f in l: + assert (type(f) == type("")) + if os.path.exists(f) and \ + (os.path.isfile(f) or os.path.islink(f)): + return f return None From 17980baea61e15ade88186ee413757298ebc1249 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 8 Jan 2015 12:28:36 +0100 Subject: [PATCH 706/817] v6.5.7-rc2 Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 51 ++++++++++++++++++++++++++++++++++++++++- offlineimap/__init__.py | 2 +- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 5580e69..1954dbc 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,13 +5,62 @@ ChangeLog :website: http://offlineimap.org +OfflineIMAP v6.5.7-rc2 (2015-01-18) +=================================== + +Notes +----- + +This release candidate should be minor for most users. + +The best points are about SSL not falling back on other authentication methods +when failing, better RAM footprint and reduced I/O access. + +Documentation had our attention, too. + +There's some code cleanups and code refactoring, as usual. + +Features +-------- + +* Do not keep reloading pyhtonfile, make it stateful. +* HACKING: how to create tags. +* MANUAL: add minor sample on how to retrieve a password with a helper python file. + +Fixes +----- + +* Make OS-default CA certificate file to be requested explicitely. +* SSL: do not fallback on other authentication mode if it fails. +* Fix regression introduced while style patching. +* API documentation: properly auto-document main class, fixes. +* ui: Machine: remove offending param for a _printData() call. +* Drop caches after having processed folders. + +Changes +------- + +* Fix unexpected garbage code. +* Properly re-raise exception to save original tracebacks. +* Refactoring: avoid redefining various Python keywords. +* Code: improvements of comments and more style consistency. +* Configuration file: better design and other small improvements. +* nametrans documentation: fix minor error. +* Unused import removal. +* Add a note about the incorrect rendering of the docstring with Sphinx. +* Errors handling: log the messages with level ERROR. +* MAINTAINERS: add mailing list maintainers. +* Fixed copyright statement. +* COPYING: fix unexpected characters. + + OfflineIMAP v6.5.7-rc1 (2015-01-07) =================================== Notes ----- -I think it's time for a new release candidate. Our release cycle are long +I think it's time for a new release candidate. Our release cycles are long enough and users are asked to use the current TIP of the next branch to test our recent patches. diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index ff6acd9..441af44 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,7 +2,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.5.7" -__revision__ = "-rc1" +__revision__ = "-rc2" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" From 9e6b23933ace1b56065aaeb4b52302838aefb4f1 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 18 Jan 2015 21:45:15 +0100 Subject: [PATCH 707/817] Changelog: prepare for coming updates Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 1954dbc..5e499c1 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,6 +5,11 @@ ChangeLog :website: http://offlineimap.org +OfflineIMAP v6.5.7-rc3 (2015- - ) +=================================== + + + OfflineIMAP v6.5.7-rc2 (2015-01-18) =================================== @@ -163,7 +168,7 @@ OfflineIMAP v6.5.6 (2014-05-14) to Tomasz Żok) -OfflineIMAP v6.5.6-RC1 (2014-05-14) +OfflineIMAP v6.5.6-rc1 (2014-05-14) =================================== * Add knob to invoke folderfilter dynamically on each sync (GitHub#73) From 285295c4f251517018a5812cb1b76e4c011ce5bb Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 18 Jan 2015 03:49:15 +0100 Subject: [PATCH 708/817] folder: LocalStatus: revamp cachemessagelist() - Do not redefine "file". - break loop as soon as possible. Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatus.py | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index d3b6cf0..7190c0c 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -117,33 +117,33 @@ class LocalStatusFolder(BaseFolder): self.messagelist = {} return - # loop as many times as version, and update format - for i in range(1, self.cur_version+1): - file = open(self.filename, "rt") + # Loop as many times as version, and update format. + for i in range(1, self.cur_version + 1): self.messagelist = {} - line = file.readline().strip() + cache = open(self.filename, "rt") + line = cache.readline().strip() - # convert from format v1 - if line == (self.magicline % 1): - self.ui._msg('Upgrading LocalStatus cache from version 1 to version 2 for %s:%s' %\ - (self.repository, self)) - self.readstatus_v1(file) - file.close() + # Format is up to date. break. + if line == (self.magicline % self.cur_version): + break + + # Convert from format v1. + elif line == (self.magicline % 1): + self.ui._msg('Upgrading LocalStatus cache from version 1' + 'to version 2 for %s:%s'% (self.repository, self)) + self.readstatus_v1(cache) + cache.close() self.save() # NOTE: Add other format transitions here in the future. # elif line == (self.magicline % 2): - # self.ui._msg('Upgrading LocalStatus cache from version 2 to version 3 for %s:%s' %\ - # (self.repository, self)) - # self.readstatus_v2(file) - # file.close() - # file.save() + # self.ui._msg(u'Upgrading LocalStatus cache from version 2' + # 'to version 3 for %s:%s'% (self.repository, self)) + # self.readstatus_v2(cache) + # cache.close() + # cache.save() - # format is up to date. break - elif line == (self.magicline % self.cur_version): - break - - # something is wrong + # Something is wrong. else: errstr = "Unrecognized cache magicline in '%s'" % self.filename self.ui.warn(errstr) @@ -152,14 +152,14 @@ class LocalStatusFolder(BaseFolder): if not line: # The status file is empty - should not have happened, # but somehow did. - errstr = "Cache file '%s' is empty. Closing..." % self.filename + errstr = "Cache file '%s' is empty."% self.filename self.ui.warn(errstr) - file.close() + cache.close() return assert(line == (self.magicline % self.cur_version)) - self.readstatus(file) - file.close() + self.readstatus(cache) + cache.close() def dropmessagelistcache(self): self.messagelist = None From 2af3d93a852ef8806ee82d36396a7dcfa14cf4ad Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 16 Jan 2015 00:00:07 +0100 Subject: [PATCH 709/817] folder: LocalStatusSQLite: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 9d7e559..e1a24ab 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -15,15 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os -import re from sys import exc_info from threading import Lock -from .Base import BaseFolder try: import sqlite3 as sqlite except: pass #fail only if needed later on, not on import +from .Base import BaseFolder + class LocalStatusSQLiteFolder(BaseFolder): """LocalStatus backend implemented with an SQLite database From d0cb96781cc827ffde887ba77e2ae45d0ddc1ccc Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 16 Jan 2015 12:09:29 +0100 Subject: [PATCH 710/817] folder: LocalStatus(SQLite): avoid redefining unchanged Base method Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatus.py | 4 ---- offlineimap/folder/LocalStatusSQLite.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 7190c0c..d9cfa70 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -45,10 +45,6 @@ class LocalStatusFolder(BaseFolder): def isnewfolder(self): return not os.path.exists(self.filename) - # Interface from BaseFolder - def getname(self): - return self.name - # Interface from BaseFolder def getfullname(self): return self.filename diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index e1a24ab..7dceec1 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -88,9 +88,6 @@ class LocalStatusSQLiteFolder(BaseFolder): def storesmessages(self): return False - def getname(self): - return self.name - def getfullname(self): return self.filename From cb6790b8041ed917f5defee5977cfbf56b5634ec Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 15 Jan 2015 17:40:27 +0100 Subject: [PATCH 711/817] minor: fix indentation Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 8 +++++--- offlineimap/repository/__init__.py | 10 +++++----- offlineimap/ui/UIBase.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 7dceec1..4f5efb1 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -56,7 +56,8 @@ class LocalStatusSQLiteFolder(BaseFolder): if not os.path.exists(dirname): os.makedirs(dirname) if not os.path.isdir(dirname): - raise UserWarning("SQLite database path '%s' is not a directory." % dirname) + raise UserWarning("SQLite database path '%s' is not a directory."% + dirname) # dblock protects against concurrent writes in same connection self._dblock = Lock() @@ -67,14 +68,15 @@ class LocalStatusSQLiteFolder(BaseFolder): except NameError: # sqlite import had failed raise UserWarning('SQLite backend chosen, but no sqlite python ' - 'bindings available. Please install.'), None, exc_info()[2] + 'bindings available. Please install.'), None, exc_info()[2] #Make sure sqlite is in multithreading SERIALIZE mode assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' #Test if db version is current enough and if db is readable. try: - cursor = self.connection.execute("SELECT value from metadata WHERE key='db_version'") + cursor = self.connection.execute( + "SELECT value from metadata WHERE key='db_version'") except sqlite.DatabaseError: #db file missing or corrupt, recreate it. self.__create_db() diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py index 076255e..59a7bb6 100644 --- a/offlineimap/repository/__init__.py +++ b/offlineimap/repository/__init__.py @@ -44,13 +44,13 @@ class Repository(object): name = account.getconf('remoterepository') # We don't support Maildirs on the remote side. typemap = {'IMAP': IMAPRepository, - 'Gmail': GmailRepository} + 'Gmail': GmailRepository} elif reqtype == 'local': name = account.getconf('localrepository') typemap = {'IMAP': MappedIMAPRepository, - 'Maildir': MaildirRepository, - 'GmailMaildir': GmailMaildirRepository} + 'Maildir': MaildirRepository, + 'GmailMaildir': GmailMaildirRepository} elif reqtype == 'status': # create and return a LocalStatusRepository @@ -69,7 +69,7 @@ class Repository(object): errstr = ("Could not find section '%s' in configuration. Required " "for account '%s'." % ('Repository %s' % name, account)) raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \ - None, exc_info()[2] + None, exc_info()[2] try: repo = typemap[repostype] @@ -77,7 +77,7 @@ class Repository(object): errstr = "'%s' repository not supported for '%s' repositories." \ % (repostype, reqtype) raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \ - None, exc_info()[2] + None, exc_info()[2] return repo(name, account) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 3c6e499..41ce1a4 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -539,7 +539,7 @@ class UIBase(object): abortsleep = False while sleepsecs > 0 and not abortsleep: if account.get_abort_event(): - abortsleep = True + abortsleep = True else: abortsleep = self.sleeping(10, sleepsecs) sleepsecs -= 10 From 7ab779ab21dcf4f33b6a43d2fdcb50d415c0c0e7 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Mon, 19 Jan 2015 00:13:17 +0300 Subject: [PATCH 712/817] Imaplib2: use proper tracebacks for exceptions Get real tracebacks (at the point that calls Request.abort() and not from the point that handles collected abort requests) and pass them to our calling functions to ease debugging of user problems. Signed-off-by: Eygene Ryabinkin --- offlineimap/imaplib2.py | 46 +++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 8975145..6d2805a 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -48,7 +48,7 @@ __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" -import binascii, errno, os, Queue, random, re, select, socket, sys, time, threading, zlib +import binascii, errno, os, Queue, random, re, select, socket, sys, time, threading, traceback, zlib select_module = select @@ -160,21 +160,25 @@ class Request(object): self.data = None - def abort(self, typ, val): - self.aborted = (typ, val) + def abort_tb(self, typ, val, tb): + self.aborted = (typ, val, tb) self.deliver(None) + def abort(self, typ, val): + self.abort_tb(typ, val, traceback.extract_stack()) + + def get_response(self, exc_fmt=None): self.callback = None if __debug__: self.parent._log(3, '%s:%s.ready.wait' % (self.name, self.tag)) self.ready.wait() if self.aborted is not None: - typ, val = self.aborted + typ, val, tb = self.aborted if exc_fmt is None: exc_fmt = '%s - %%s' % typ - raise typ(exc_fmt % str(val)) + raise typ, typ(exc_fmt % str(val)), tb return self.response @@ -1661,7 +1665,7 @@ class IMAP4(object): if __debug__: self._log(1, 'starting') - typ, val = self.abort, 'connection terminated' + typ, val, tb = self.abort, 'connection terminated', None while not self.Terminate: @@ -1684,6 +1688,7 @@ class IMAP4(object): if resp_timeout is not None and self.tagged_commands: if __debug__: self._log(1, 'response timeout') typ, val = self.abort, 'no response after %s secs' % resp_timeout + tb = traceback.extract_stack() break continue if self.idle_timeout > time.time(): @@ -1695,23 +1700,33 @@ class IMAP4(object): if __debug__: self._log(1, 'inq None - terminating') break + # self.inq can contain tuple, it means we got an exception. + # For example of code that produces tuple see IMAP4._reader() + # and IMAP4._writer(). + # + # XXX: may be it will be more explicit to create own exception + # XXX: class and pass it instead of tuple. if not isinstance(line, basestring): - typ, val = line + typ, val, tb = line break try: self._put_response(line) except: typ, val = self.error, 'program error: %s - %s' % sys.exc_info()[:2] + tb = sys.exc_info()[2] break self.Terminate = True + if not tb: + tb = traceback.extract_stack() + if __debug__: self._log(1, 'terminating: %s' % repr(val)) while not self.ouq.empty(): try: - self.ouq.get_nowait().abort(typ, val) + self.ouq.get_nowait().abort_tb(typ, val, tb) except Queue.Empty: break self.ouq.put(None) @@ -1719,7 +1734,7 @@ class IMAP4(object): self.commands_lock.acquire() for name in self.tagged_commands.keys(): rqb = self.tagged_commands.pop(name) - rqb.abort(typ, val) + rqb.abort_tb(typ, val, tb) self.state_change_free.set() self.commands_lock.release() if __debug__: self._log(3, 'state_change_free.set') @@ -1801,7 +1816,7 @@ class IMAP4(object): self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) - self.inq.put((self.abort, reason)) + self.inq.put((self.abort, reason, sys.exc_info()[2])) break poll.unregister(self.read_fd) @@ -1865,7 +1880,7 @@ class IMAP4(object): self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) - self.inq.put((self.abort, reason)) + self.inq.put((self.abort, reason, sys.exc_info()[2])) break if __debug__: self._log(1, 'finished') @@ -1878,6 +1893,7 @@ class IMAP4(object): if __debug__: self._log(1, 'starting') reason = 'Terminated' + tb = None while not self.Terminate: rqb = self.ouq.get() @@ -1889,15 +1905,19 @@ class IMAP4(object): if __debug__: self._log(4, '> %s' % rqb.data) except: reason = 'socket error: %s - %s' % sys.exc_info()[:2] + tb = sys.exc_info()[2] if __debug__: if not self.Terminate: self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) - rqb.abort(self.abort, reason) + rqb.abort_tb(self.abort, reason, tb) break - self.inq.put((self.abort, reason)) + if not tb: + tb = traceback.extract_stack() + + self.inq.put((self.abort, reason, tb)) if __debug__: self._log(1, 'finished') From 5318af606e4ebd0906cf3c9cc5745a6dd2b150e3 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Tue, 10 Feb 2015 11:24:59 +0100 Subject: [PATCH 713/817] docs: expand documentation of the gmail labels syncing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Suggest that this works well together with email indexing. - Warn about modifying the labelsheader after a sync. - In the example replace X-Gmail-Keywords by X-Keywords. I think this is misleading, since no email client will pick up X-Gmail-Keywords, while X-Keywords (the default) is recognized by some. - add paragraph explaining special gmail labels. Merge-request-from: Abdó Roig-Maranges Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 16af514..7fc355a 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -450,11 +450,17 @@ Sync from Gmail to a local Maildir with labels This is an example of a setup where GMail gets synced with a local Maildir. It also keeps track of GMail labels, that get embedded into the messages -under the header configured in labelsheader, and syncs them back and forth -the same way as flags. -The header used for the labels may need to be set according to the email -client used. -Some choices that may be recognized by email clients are `X-Keywords` or `X-Labels`. +under the header configured in the labelsheader setting, and syncs them back +and forth the same way as flags. This is particularly useful with an email +client that indexes your email and recognizes the labels header, so that you +can sync a single "All Mail" folder, and navigate your email via searches. + +The header used to store the labels depends on the email client you plan to use. +Some choices that may be recognized by email clients are X-Keywords +(the default) or X-Labels. Note that if you need to change the label header +after the labels have already been synced, you will have to change the header +manually on all messages, otherwise offlineimap will not pick up the labels under +the old header. The first time OfflineIMAP runs with synclabels enabled on a large repository it may take some time as the labels are read / embedded on every message. @@ -466,9 +472,9 @@ much faster:: remoterepository = Gmailserver-mine synclabels = yes # This header is where labels go. Usually you will be fine - # with default value, but in case you want it different, - # here we go: - labelsheader = X-GMail-Keywords + # with default value (X-Keywords), but in case you want it + # different, here we go: + labelsheader = X-Keywords [Repository Gmailserver-mine] #This is the remote repository @@ -480,6 +486,14 @@ much faster:: #This is the 'local' repository type = GmailMaildir +There are some labels, that gmail treats in a special way. All internal gmail +labels start with "\\". Those labels include: \\Drafts, \\Important, \\Inbox, +\\Sent, \\Junk, \\Flagged, \\Trash. You can add and remove those labels +locally, and when synced, will have special actions on the gmail side. For instance, +adding the label \Trash to an email will move it to the trash, and be permanantly +deleted after some time. This is relevant, since gmail's IMAP prevents from removing +messages from the "All Mail" folder the usual way. + Selecting only a few folders to sync ------------------------------------ From 461554b7b1aa58655acdd5ac660e01a1ee17b7d8 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 14 Jan 2015 22:58:25 +0100 Subject: [PATCH 714/817] more style consistency Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 74 +++++++------- offlineimap/folder/Base.py | 57 +++++------ offlineimap/folder/Gmail.py | 11 +-- offlineimap/folder/IMAP.py | 52 +++++----- offlineimap/folder/LocalStatus.py | 10 +- offlineimap/folder/LocalStatusSQLite.py | 33 ++++--- offlineimap/folder/Maildir.py | 17 ++-- offlineimap/folder/UIDMaps.py | 20 ++-- offlineimap/imaplibutil.py | 5 +- offlineimap/imapserver.py | 47 ++++----- offlineimap/imaputil.py | 2 +- offlineimap/init.py | 2 +- offlineimap/repository/Base.py | 14 +-- offlineimap/repository/IMAP.py | 25 +++-- offlineimap/repository/LocalStatus.py | 12 +-- offlineimap/repository/Maildir.py | 2 +- offlineimap/repository/__init__.py | 8 +- offlineimap/threadutil.py | 14 ++- offlineimap/ui/Curses.py | 112 ++++++++++++--------- offlineimap/ui/UIBase.py | 123 +++++++++++++----------- 20 files changed, 343 insertions(+), 297 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 73b8ff1..62ed5c3 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -55,7 +55,7 @@ def AccountHashGenerator(customconfig): class Account(CustomConfig.ConfigHelperMixin): - """Represents an account (ie. 2 repositories) to sync + """Represents an account (ie. 2 repositories) to sync. Most of the time you will actually want to use the derived :class:`accounts.SyncableAccount` which contains all functions used @@ -71,8 +71,9 @@ class Account(CustomConfig.ConfigHelperMixin): :param config: Representing the offlineimap configuration file. :type config: :class:`offlineimap.CustomConfig.CustomConfigParser` - :param name: A string denoting the name of the Account - as configured""" + :param name: A (str) string denoting the name of the Account + as configured. + """ self.config = config self.name = name @@ -109,7 +110,7 @@ class Account(CustomConfig.ConfigHelperMixin): @classmethod def set_abort_event(cls, config, signum): - """Set skip sleep/abort event for all accounts + """Set skip sleep/abort event for all accounts. If we want to skip a current (or the next) sleep, or if we want to abort an autorefresh loop, the main thread can use @@ -121,6 +122,7 @@ class Account(CustomConfig.ConfigHelperMixin): This is a class method, it will send the signal to all accounts. """ + if signum == 1: # resync signal, set config option for all accounts for acctsection in getaccountlist(config): @@ -133,7 +135,7 @@ class Account(CustomConfig.ConfigHelperMixin): cls.abort_NOW_signal.set() def get_abort_event(self): - """Checks if an abort signal had been sent + """Checks if an abort signal had been sent. If the 'skipsleep' config option for this account had been set, with `set_abort_event(config, 1)` it will get cleared in this @@ -142,6 +144,7 @@ class Account(CustomConfig.ConfigHelperMixin): :returns: True, if the main thread had called :meth:`set_abort_event` earlier, otherwise 'False'. """ + skipsleep = self.getconfboolean("skipsleep", 0) if skipsleep: self.config.set(self.getsection(), "skipsleep", '0') @@ -149,12 +152,13 @@ class Account(CustomConfig.ConfigHelperMixin): Account.abort_NOW_signal.is_set() def _sleeper(self): - """Sleep if the account is set to autorefresh + """Sleep if the account is set to autorefresh. :returns: 0:timeout expired, 1: canceled the timer, 2:request to abort the program, 100: if configured to not sleep at all. """ + if not self.refreshperiod: return 100 @@ -184,7 +188,8 @@ class Account(CustomConfig.ConfigHelperMixin): return 0 def serverdiagnostics(self): - """Output diagnostics for all involved repositories""" + """Output diagnostics for all involved repositories.""" + remote_repo = Repository(self, 'remote') local_repo = Repository(self, 'local') #status_repo = Repository(self, 'status') @@ -194,7 +199,7 @@ class Account(CustomConfig.ConfigHelperMixin): class SyncableAccount(Account): - """A syncable email account connecting 2 repositories + """A syncable email account connecting 2 repositories. Derives from :class:`accounts.Account` but contains the additional functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`, @@ -203,11 +208,12 @@ class SyncableAccount(Account): def __init__(self, *args, **kwargs): Account.__init__(self, *args, **kwargs) self._lockfd = None - self._lockfilepath = os.path.join(self.config.getmetadatadir(), - "%s.lock" % self) + self._lockfilepath = os.path.join( + self.config.getmetadatadir(), "%s.lock"% self) def __lock(self): - """Lock the account, throwing an exception if it is locked already""" + """Lock the account, throwing an exception if it is locked already.""" + self._lockfd = open(self._lockfilepath, 'w') try: fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB) @@ -217,19 +223,19 @@ class SyncableAccount(Account): except IOError: self._lockfd.close() raise OfflineImapError("Could not lock account %s. Is another " - "instance using this account?" % self, - OfflineImapError.ERROR.REPO), \ - None, exc_info()[2] + "instance using this account?"% self, + OfflineImapError.ERROR.REPO), None, exc_info()[2] def _unlock(self): """Unlock the account, deleting the lock file""" + #If we own the lock file, delete it if self._lockfd and not self._lockfd.closed: self._lockfd.close() try: os.unlink(self._lockfilepath) except OSError: - pass #Failed to delete for some reason. + pass # Failed to delete for some reason. def syncrunner(self): self.ui.registerthread(self) @@ -265,8 +271,8 @@ class SyncableAccount(Account): raise self.ui.error(e, exc_info()[2]) except Exception as e: - self.ui.error(e, exc_info()[2], msg="While attempting to sync" - " account '%s'"% self) + self.ui.error(e, exc_info()[2], msg= + "While attempting to sync account '%s'"% self) else: # after success sync, reset the looping counter to 3 if self.refreshperiod: @@ -278,18 +284,19 @@ class SyncableAccount(Account): looping = 0 def get_local_folder(self, remotefolder): - """Return the corresponding local folder for a given remotefolder""" + """Return the corresponding local folder for a given remotefolder.""" + return self.localrepos.getfolder( remotefolder.getvisiblename(). replace(self.remoterepos.getsep(), self.localrepos.getsep())) def __sync(self): - """Synchronize the account once, then return + """Synchronize the account once, then return. Assumes that `self.remoterepos`, `self.localrepos`, and `self.statusrepos` has already been populated, so it should only - be called from the :meth:`syncrunner` function. - """ + be called from the :meth:`syncrunner` function.""" + folderthreads = [] hook = self.getconf('presynchook', '') @@ -383,12 +390,12 @@ class SyncableAccount(Account): stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) r = p.communicate() - self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n" % r) - self.ui.callhook("Hook return code: %d" % p.returncode) + self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n"% r) + self.ui.callhook("Hook return code: %d"% p.returncode) except (KeyboardInterrupt, SystemExit): raise except Exception as e: - self.ui.error(e, exc_info()[2], msg = "Calling hook") + self.ui.error(e, exc_info()[2], msg="Calling hook") def syncfolder(account, remotefolder, quick): """Synchronizes given remote folder for the specified account. @@ -407,12 +414,12 @@ def syncfolder(account, remotefolder, quick): # Write the mailboxes mbnames.add(account.name, localfolder.getname(), - localrepos.getlocalroot()) + localrepos.getlocalroot()) # Load status folder. - statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ - replace(remoterepos.getsep(), - statusrepos.getsep())) + statusfolder = statusrepos.getfolder(remotefolder.getvisiblename(). + replace(remoterepos.getsep(), statusrepos.getsep())) + if localfolder.get_uidvalidity() == None: # This is a new folder, so delete the status cache to be # sure we don't have a conflict. @@ -423,13 +430,13 @@ def syncfolder(account, remotefolder, quick): statusfolder.cachemessagelist() if quick: - if not localfolder.quickchanged(statusfolder) \ - and not remotefolder.quickchanged(statusfolder): + if (not localfolder.quickchanged(statusfolder) and + not remotefolder.quickchanged(statusfolder)): ui.skippingfolder(remotefolder) localrepos.restore_atime() return - # Load local folder + # Load local folder. ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) ui.loadmessagelist(localrepos, localfolder) localfolder.cachemessagelist() @@ -488,9 +495,8 @@ def syncfolder(account, remotefolder, quick): ui.error(e, exc_info()[2], msg = "Aborting sync, folder '%s' " "[acc: '%s']" % (localfolder, account)) except Exception as e: - ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ - (account, remotefolder.getvisiblename(), - traceback.format_exc())) + ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s"% + (account, remotefolder.getvisiblename(), traceback.format_exc())) finally: for folder in ["statusfolder", "localfolder", "remotefolder"]: if folder in locals(): diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 55b2bfc..3a04ef6 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -48,13 +48,11 @@ class BaseFolder(object): self.visiblename = '' self.config = repository.getconfig() - utime_from_message_global = \ - self.config.getdefaultboolean("general", - "utime_from_message", False) + utime_from_message_global = self.config.getdefaultboolean( + "general", "utime_from_message", False) repo = "Repository " + repository.name - self._utime_from_message = \ - self.config.getdefaultboolean(repo, - "utime_from_message", utime_from_message_global) + self._utime_from_message = self.config.getdefaultboolean(repo, + "utime_from_message", utime_from_message_global) # Determine if we're running static or dynamic folder filtering # and check filtering status @@ -78,16 +76,19 @@ class BaseFolder(object): return self.name def __str__(self): + # FIMXE: remove calls of this. We have getname(). return self.name @property def accountname(self): """Account name as string""" + return self.repository.accountname @property def sync_this(self): """Should this folder be synced or is it e.g. filtered out?""" + if not self._dynamic_folderfilter: return self._sync_this else: @@ -144,7 +145,7 @@ class BaseFolder(object): if self.name == self.visiblename: return self.name else: - return "%s [remote name %s]" % (self.visiblename, self.name) + return "%s [remote name %s]"% (self.visiblename, self.name) def getrepository(self): """Returns the repository object that this folder is within.""" @@ -172,9 +173,9 @@ class BaseFolder(object): if not self.name: basename = '.' - else: #avoid directory hierarchies and file names such as '/' + else: # Avoid directory hierarchies and file names such as '/'. basename = self.name.replace('/', '.') - # replace with literal 'dot' if final path name is '.' as '.' is + # Replace with literal 'dot' if final path name is '.' as '.' is # an invalid file name. basename = re.sub('(^|\/)\.$','\\1dot', basename) return basename @@ -196,7 +197,7 @@ class BaseFolder(object): return True def _getuidfilename(self): - """provides UIDVALIDITY cache filename for class internal purposes""" + """provides UIDVALIDITY cache filename for class internal purposes. return os.path.join(self.repository.getuiddir(), self.getfolderbasename()) @@ -228,7 +229,7 @@ class BaseFolder(object): uidfilename = self._getuidfilename() with open(uidfilename + ".tmp", "wt") as file: - file.write("%d\n" % newval) + file.write("%d\n"% newval) os.rename(uidfilename + ".tmp", uidfilename) self._base_saved_uidvalidity = newval @@ -252,6 +253,7 @@ class BaseFolder(object): def getmessagelist(self): """Gets the current message list. + You must call cachemessagelist() before calling this function!""" raise NotImplementedError @@ -272,6 +274,7 @@ class BaseFolder(object): def getmessageuidlist(self): """Gets a list of UIDs. + You may have to call cachemessagelist() before calling this function!""" return self.getmessagelist().keys() @@ -377,6 +380,7 @@ class BaseFolder(object): def getmessagelabels(self, uid): """Returns the labels for the specified message.""" + raise NotImplementedError def savemessagelabels(self, uid, labels, ignorelabels=set(), mtime=0): @@ -691,10 +695,10 @@ class BaseFolder(object): # load it up. if dstfolder.storesmessages(): message = self.getmessage(uid) - #Succeeded? -> IMAP actually assigned a UID. If newid - #remained negative, no server was willing to assign us an - #UID. If newid is 0, saving succeeded, but we could not - #retrieve the new UID. Ignore message in this case. + # Succeeded? -> IMAP actually assigned a UID. If newid + # remained negative, no server was willing to assign us an + # UID. If newid is 0, saving succeeded, but we could not + # retrieve the new UID. Ignore message in this case. new_uid = dstfolder.savemessage(uid, message, flags, rtime) if new_uid > 0: if new_uid != uid: @@ -728,7 +732,7 @@ class BaseFolder(object): raise #raise on unknown errors, so we can fix those def __syncmessagesto_copy(self, dstfolder, statusfolder): - """Pass1: Copy locally existing messages not on the other side + """Pass1: Copy locally existing messages not on the other side. This will copy messages to dstfolder that exist locally but are not in the statusfolder yet. The strategy is: @@ -738,18 +742,16 @@ class BaseFolder(object): - If dstfolder doesn't have it yet, add them to dstfolder. - Update statusfolder - This function checks and protects us from action in ryrun mode. - """ + This function checks and protects us from action in dryrun mode.""" threads = [] - copylist = filter(lambda uid: not \ - statusfolder.uidexists(uid), - self.getmessageuidlist()) + copylist = filter(lambda uid: not statusfolder.uidexists(uid), + self.getmessageuidlist()) num_to_copy = len(copylist) if num_to_copy and self.repository.account.dryrun: self.ui.info("[DRYRUN] Copy {0} messages from {1}[{2}] to {3}".format( - num_to_copy, self, self.repository, dstfolder.repository)) + num_to_copy, self, self.repository, dstfolder.repository)) return for num, uid in enumerate(copylist): # bail out on CTRL-C or SIGTERM @@ -773,7 +775,7 @@ class BaseFolder(object): thread.join() def __syncmessagesto_delete(self, dstfolder, statusfolder): - """Pass 2: Remove locally deleted messages on dst + """Pass 2: Remove locally deleted messages on dst. Get all UIDS in statusfolder but not self. These are messages that were deleted in 'self'. Delete those from dstfolder and @@ -782,9 +784,8 @@ class BaseFolder(object): This function checks and protects us from action in ryrun mode. """ - deletelist = filter(lambda uid: uid>=0 \ - and not self.uidexists(uid), - statusfolder.getmessageuidlist()) + deletelist = filter(lambda uid: uid >= 0 and not + self.uidexists(uid), statusfolder.getmessageuidlist()) if len(deletelist): self.ui.deletingmessages(deletelist, [dstfolder]) if self.repository.account.dryrun: @@ -795,7 +796,7 @@ class BaseFolder(object): folder.deletemessages(deletelist) def __syncmessagesto_flags(self, dstfolder, statusfolder): - """Pass 3: Flag synchronization + """Pass 3: Flag synchronization. Compare flag mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been @@ -904,7 +905,7 @@ class BaseFolder(object): def __eq__(self, other): """Comparisons work either on string comparing folder names or - on the same instance + on the same instance. MailDirFolder('foo') == 'foo' --> True a = MailDirFolder('foo'); a == b --> True diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index c12c0ff..1afbe47 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -22,11 +22,10 @@ from sys import exc_info from offlineimap import imaputil, OfflineImapError from offlineimap import imaplibutil import offlineimap.accounts - -"""Folder implementation to support features of the Gmail IMAP server. -""" from .IMAP import IMAPFolder +"""Folder implementation to support features of the Gmail IMAP server.""" + class GmailFolder(IMAPFolder): """Folder implementation to support features of the Gmail IMAP server. @@ -101,11 +100,11 @@ class GmailFolder(IMAPFolder): body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) if len(body)>200: - dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:]) + dbg_output = "%s...%s"% (str(body)[:150], str(body)[-50:]) else: dbg_output = body - self.ui.debug('imap', "Returned object from fetching %d: '%s'" % + self.ui.debug('imap', "Returned object from fetching %d: '%s'"% (uid, dbg_output)) return body @@ -139,7 +138,7 @@ class GmailFolder(IMAPFolder): # imaplib2 from quoting the sequence. # # NB: msgsToFetch are sequential numbers, not UID's - res_type, response = imapobj.fetch("'%s'" % msgsToFetch, + res_type, response = imapobj.fetch("'%s'"% msgsToFetch, '(FLAGS X-GM-LABELS UID)') if res_type != 'OK': raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " % \ diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index e0c0cc7..c7e6516 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -164,10 +164,10 @@ class IMAPFolder(BaseFolder): # By default examine all messages in this folder msgsToFetch = '1:*' - maxage = self.config.getdefaultint("Account %s"% self.accountname, - "maxage", -1) - maxsize = self.config.getdefaultint("Account %s"% self.accountname, - "maxsize", -1) + maxage = self.config.getdefaultint( + "Account %s"% self.accountname, "maxage", -1) + maxsize = self.config.getdefaultint( + "Account %s"% self.accountname, "maxsize", -1) # Build search condition if (maxage != -1) | (maxsize != -1): @@ -178,9 +178,9 @@ class IMAPFolder(BaseFolder): oldest_struct = time.gmtime(time.time() - (60*60*24*maxage)) if oldest_struct[0] < 1900: raise OfflineImapError("maxage setting led to year %d. " - "Abort syncing." % oldest_struct[0], - OfflineImapError.ERROR.REPO) - search_cond += "SINCE %02d-%s-%d" % ( + "Abort syncing."% oldest_struct[0], + OfflineImapError.ERROR.REPO) + search_cond += "SINCE %02d-%s-%d"% ( oldest_struct[2], MonthNames[oldest_struct[1]], oldest_struct[0]) @@ -188,7 +188,7 @@ class IMAPFolder(BaseFolder): if(maxsize != -1): if(maxage != -1): # There are two conditions, add space search_cond += " " - search_cond += "SMALLER %d" % maxsize + search_cond += "SMALLER %d"% maxsize search_cond += ")" @@ -225,10 +225,8 @@ class IMAPFolder(BaseFolder): msgsToFetch, '(FLAGS UID)') if res_type != 'OK': raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " - "Server responded '[%s] %s'"% ( - self.getrepository(), self, - res_type, response), - OfflineImapError.ERROR.FOLDER) + "Server responded '[%s] %s'"% (self.getrepository(), self, + res_type, response), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) @@ -259,7 +257,7 @@ class IMAPFolder(BaseFolder): # Interface from BaseFolder def getmessage(self, uid): - """Retrieve message with UID from the IMAP server (incl body) + """Retrieve message with UID from the IMAP server (incl body). After this function all CRLFs will be transformed to '\n'. @@ -280,7 +278,7 @@ class IMAPFolder(BaseFolder): data = data[0][1].replace(CRLF, "\n") if len(data)>200: - dbg_output = "%s...%s" % (str(data)[:150], str(data)[-50:]) + dbg_output = "%s...%s"% (str(data)[:150], str(data)[-50:]) else: dbg_output = data @@ -331,7 +329,8 @@ class IMAPFolder(BaseFolder): # Now find the UID it got. headervalue = imapobj._quote(headervalue) try: - matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0] + matchinguids = imapobj.uid('search', 'HEADER', + headername, headervalue)[1][0] except imapobj.error as err: # IMAP server doesn't implement search or had a problem. self.ui.debug('imap', "__savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s"% (err, headername)) @@ -396,8 +395,8 @@ class IMAPFolder(BaseFolder): result = imapobj.uid('FETCH', bytearray('%d:*'% start), 'rfc822.header') if result[0] != 'OK': - raise OfflineImapError('Error fetching mail headers: ' + '. '.join(result[1]), - OfflineImapError.ERROR.MESSAGE) + raise OfflineImapError('Error fetching mail headers: %s'% + '. '.join(result[1]), OfflineImapError.ERROR.MESSAGE) result = result[1] @@ -423,7 +422,8 @@ class IMAPFolder(BaseFolder): def __getmessageinternaldate(self, content, rtime=None): """Parses mail and returns an INTERNALDATE string - It will use information in the following order, falling back as an attempt fails: + It will use information in the following order, falling back as an + attempt fails: - rtime parameter - Date header of email @@ -475,21 +475,22 @@ class IMAPFolder(BaseFolder): "Server will use local time."% datetuple) return None - #produce a string representation of datetuple that works as - #INTERNALDATE + # Produce a string representation of datetuple that works as + # INTERNALDATE. num2mon = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'} - #tm_isdst coming from email.parsedate is not usable, we still use it here, mhh + # tm_isdst coming from email.parsedate is not usable, we still use it + # here, mhh. if datetuple.tm_isdst == '1': zone = -time.altzone else: zone = -time.timezone offset_h, offset_m = divmod(zone//60, 60) - internaldate = '"%02d-%s-%04d %02d:%02d:%02d %+03d%02d"' \ - % (datetuple.tm_mday, num2mon[datetuple.tm_mon], datetuple.tm_year, \ - datetuple.tm_hour, datetuple.tm_min, datetuple.tm_sec, offset_h, offset_m) + internaldate = '"%02d-%s-%04d %02d:%02d:%02d %+03d%02d"'% \ + (datetuple.tm_mday, num2mon[datetuple.tm_mon], datetuple.tm_year, \ + datetuple.tm_hour, datetuple.tm_min, datetuple.tm_sec, offset_h, offset_m) return internaldate @@ -554,7 +555,7 @@ class IMAPFolder(BaseFolder): content = self.addmessageheader(content, CRLF, headername, headervalue) if len(content)>200: - dbg_output = "%s...%s" % (content[:150], content[-50:]) + dbg_output = "%s...%s"% (content[:150], content[-50:]) else: dbg_output = content self.ui.debug('imap', "savemessage: date: %s, content: '%s'"% @@ -726,6 +727,7 @@ class IMAPFolder(BaseFolder): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + imapobj = self.imapserver.acquireconnection() try: result = self._store_to_imap(imapobj, str(uid), 'FLAGS', diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index d9cfa70..57c696d 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -21,8 +21,9 @@ import threading from .Base import BaseFolder + class LocalStatusFolder(BaseFolder): - """LocalStatus backend implemented as a plain text file""" + """LocalStatus backend implemented as a plain text file.""" cur_version = 2 magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT %d" @@ -53,12 +54,10 @@ class LocalStatusFolder(BaseFolder): if not self.isnewfolder(): os.unlink(self.filename) - # Interface from BaseFolder def msglist_item_initializer(self, uid): return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0} - def readstatus_v1(self, fp): """Read status folder in format version 1. @@ -80,7 +79,6 @@ class LocalStatusFolder(BaseFolder): self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags - def readstatus(self, fp): """Read status file in the current format. @@ -97,7 +95,7 @@ class LocalStatusFolder(BaseFolder): mtime = long(mtime) labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) except ValueError as e: - errstr = "Corrupt line '%s' in cache file '%s'" % \ + errstr = "Corrupt line '%s' in cache file '%s'"% \ (line, self.filename) self.ui.warn(errstr) raise ValueError(errstr), None, exc_info()[2] @@ -227,7 +225,6 @@ class LocalStatusFolder(BaseFolder): self.messagelist[uid]['flags'] = flags self.save() - def savemessagelabels(self, uid, labels, mtime=None): self.messagelist[uid]['labels'] = labels if mtime: self.messagelist[uid]['mtime'] = mtime @@ -263,7 +260,6 @@ class LocalStatusFolder(BaseFolder): def getmessagemtime(self, uid): return self.messagelist[uid]['mtime'] - # Interface from BaseFolder def deletemessage(self, uid): self.deletemessages([uid]) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 4f5efb1..8a3b9df 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -64,7 +64,7 @@ class LocalStatusSQLiteFolder(BaseFolder): #Try to establish connection, no need for threadsafety in __init__ try: - self.connection = sqlite.connect(self.filename, check_same_thread = False) + self.connection = sqlite.connect(self.filename, check_same_thread=False) except NameError: # sqlite import had failed raise UserWarning('SQLite backend chosen, but no sqlite python ' @@ -93,7 +93,6 @@ class LocalStatusSQLiteFolder(BaseFolder): def getfullname(self): return self.filename - # Interface from LocalStatusFolder def isnewfolder(self): return self._newfolder @@ -101,7 +100,8 @@ class LocalStatusSQLiteFolder(BaseFolder): # Interface from LocalStatusFolder def deletemessagelist(self): - """delete all messages in the db""" + """Delete all messages in the db.""" + self.__sql_write('DELETE FROM status') @@ -114,6 +114,7 @@ class LocalStatusSQLiteFolder(BaseFolder): :param executemany: bool indicating whether we want to perform conn.executemany() or conn.execute(). :returns: the Cursor() or raises an Exception""" + success = False while not success: self._dblock.acquire() @@ -153,8 +154,8 @@ class LocalStatusSQLiteFolder(BaseFolder): # Upgrade from database version 1 to version 2 # This change adds labels and mtime columns, to be used by Gmail IMAP and Maildir folders. if from_ver <= 1: - self.ui._msg('Upgrading LocalStatus cache from version 1 to version 2 for %s:%s' %\ - (self.repository, self)) + self.ui._msg('Upgrading LocalStatus cache from version 1 to version 2 for %s:%s'% + (self.repository, self)) self.connection.executescript("""ALTER TABLE status ADD mtime INTEGER DEFAULT 0; ALTER TABLE status ADD labels VARCHAR(256) DEFAULT ''; UPDATE metadata SET value='2' WHERE key='db_version'; @@ -167,12 +168,10 @@ class LocalStatusSQLiteFolder(BaseFolder): def __create_db(self): - """ - Create a new db file. + """Create a new db file. self.connection must point to the opened and valid SQlite - database connection. - """ + database connection.""" self.ui._msg('Creating new Local Status db for %s:%s' \ % (self.repository, self)) self.connection.executescript(""" @@ -212,6 +211,7 @@ class LocalStatusSQLiteFolder(BaseFolder): def saveall(self): """Saves the entire messagelist to the database.""" + data = [] for uid, msg in self.messagelist.items(): mtime = msg['mtime'] @@ -219,8 +219,9 @@ class LocalStatusSQLiteFolder(BaseFolder): labels = ', '.join(sorted(msg['labels'])) data.append((uid, flags, mtime, labels)) - self.__sql_write('INSERT OR REPLACE INTO status (id,flags,mtime,labels) VALUES (?,?,?,?)', - data, executemany=True) + self.__sql_write('INSERT OR REPLACE INTO status ' + '(id,flags,mtime,labels) VALUES (?,?,?,?)', + data, executemany=True) # Following some pure SQLite functions, where we chose to use @@ -267,14 +268,12 @@ class LocalStatusSQLiteFolder(BaseFolder): # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): - """ - Writes a new message, with the specified uid. + """Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not check against dryrun settings, so you need to ensure that - savemessage is never called in a dryrun mode. - - """ + savemessage is never called in a dryrun mode.""" + if uid < 0: # We cannot assign a uid. return uid @@ -352,6 +351,7 @@ class LocalStatusSQLiteFolder(BaseFolder): def savemessagesmtimebulk(self, mtimes): """Saves mtimes from the mtimes dictionary in a single database operation.""" + data = [(mt, uid) for uid, mt in mtimes.items()] self.__sql_write('UPDATE status SET mtime=? WHERE id=?', data, executemany=True) for uid, mt in mtimes.items(): @@ -376,6 +376,7 @@ class LocalStatusSQLiteFolder(BaseFolder): This function uses sqlites executemany() function which is much faster than iterating through deletemessage() when we have many messages to delete.""" + # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if uid in self.messagelist] if not len(uidlist): diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 3a19cc3..b35bfc2 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -69,7 +69,7 @@ class MaildirFolder(BaseFolder): "Account "+self.accountname, "maildir-windows-compatible", False) self.infosep = '!' if self.wincompatible else ':' """infosep is the separator between maildir name and flag appendix""" - self.re_flagmatch = re.compile('%s2,(\w*)' % self.infosep) + self.re_flagmatch = re.compile('%s2,(\w*)'% self.infosep) #self.ui is set in BaseFolder.init() # Everything up to the first comma or colon (or ! if Windows): self.re_prefixmatch = re.compile('([^'+ self.infosep + ',]*)') @@ -128,13 +128,14 @@ class MaildirFolder(BaseFolder): detected, we return an empty flags list. :returns: (prefix, UID, FMD5, flags). UID is a numeric "long" - type. flags is a set() of Maildir flags""" + type. flags is a set() of Maildir flags. + """ prefix, uid, fmd5, flags = None, None, None, set() prefixmatch = self.re_prefixmatch.match(filename) if prefixmatch: prefix = prefixmatch.group(1) - folderstr = ',FMD5=%s' % self._foldermd5 + folderstr = ',FMD5=%s'% self._foldermd5 foldermatch = folderstr in filename # If there was no folder MD5 specified, or if it mismatches, # assume it is a foreign (new) message and ret: uid, fmd5 = None, None @@ -154,7 +155,9 @@ class MaildirFolder(BaseFolder): Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F (flagged). - :returns: dict that can be used as self.messagelist""" + :returns: dict that can be used as self.messagelist. + """ + maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", None) maxsize = self.config.getdefaultint("Account " + self.accountname, @@ -254,9 +257,9 @@ class MaildirFolder(BaseFolder): :returns: String containing unique message filename""" timeval, timeseq = _gettimeseq() - return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s' % \ + return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \ (timeval, timeseq, os.getpid(), socket.gethostname(), - uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) + uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) def save_to_tmp_file(self, filename, content): @@ -393,7 +396,7 @@ class MaildirFolder(BaseFolder): """ if not uid in self.messagelist: - raise OfflineImapError("Cannot change unknown Maildir UID %s" % uid) + raise OfflineImapError("Cannot change unknown Maildir UID %s"% uid) if uid == new_uid: return oldfilename = self.messagelist[uid]['filename'] diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index eb4fbae..e8ca9a7 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -78,7 +78,7 @@ class MappedIMAPFolder(IMAPFolder): try: file = open(mapfilename + ".tmp", 'wt') for (key, value) in self.diskl2r.iteritems(): - file.write("%d:%d\n" % (key, value)) + file.write("%d:%d\n"% (key, value)) file.close() os.rename(mapfilename + '.tmp', mapfilename) finally: @@ -91,7 +91,7 @@ class MappedIMAPFolder(IMAPFolder): raise OfflineImapError("Could not find UID for msg '{0}' (f:'{1}'." " This is usually a bad thing and should be reported on the ma" "iling list.".format(e.args[0], self), - OfflineImapError.ERROR.MESSAGE), None, exc_info()[2] + OfflineImapError.ERROR.MESSAGE), None, exc_info()[2] # Interface from BaseFolder def cachemessagelist(self): @@ -215,8 +215,8 @@ class MappedIMAPFolder(IMAPFolder): newluid = self._mb.savemessage(-1, content, flags, rtime) if newluid < 1: - raise ValueError("Backend could not find uid for message, returned " - "%s" % newluid) + raise ValueError("Backend could not find uid for message, " + "returned %s"% newluid) self.maplock.acquire() try: self.diskl2r[newluid] = uid @@ -262,8 +262,8 @@ class MappedIMAPFolder(IMAPFolder): UID. The UIDMaps case handles this efficiently by simply changing the mappings file.""" if ruid not in self.r2l: - raise OfflineImapError("Cannot change unknown Maildir UID %s" % ruid, - OfflineImapError.ERROR.MESSAGE) + raise OfflineImapError("Cannot change unknown Maildir UID %s"% + ruid, OfflineImapError.ERROR.MESSAGE) if ruid == new_ruid: return # sanity check shortcut self.maplock.acquire() try: @@ -271,10 +271,10 @@ class MappedIMAPFolder(IMAPFolder): self.l2r[luid] = new_ruid del self.r2l[ruid] self.r2l[new_ruid] = luid - #TODO: diskl2r|r2l are a pain to sync and should be done away with - #diskl2r only contains positive UIDs, so wrap in ifs - if luid>0: self.diskl2r[luid] = new_ruid - if ruid>0: del self.diskr2l[ruid] + # TODO: diskl2r|r2l are a pain to sync and should be done away with + # diskl2r only contains positive UIDs, so wrap in ifs. + if luid > 0: self.diskl2r[luid] = new_ruid + if ruid > 0: del self.diskr2l[ruid] if new_ruid > 0: self.diskr2l[new_ruid] = luid self._savemaps(dolock = 0) finally: diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 917c726..83ffe9a 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -39,8 +39,9 @@ class UsefulIMAPMixIn(object): :returns: 'OK' on success, nothing if the folder was already selected or raises an :exc:`OfflineImapError`.""" - if self.__getselectedfolder() == mailbox and self.is_readonly == readonly \ - and not force: + if self.__getselectedfolder() == mailbox and \ + self.is_readonly == readonly and \ + not force: # No change; return. return # Wipe out all old responses, to maintain semantics with old imaplib2 diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 1275092..3d69426 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -45,7 +45,8 @@ class IMAPServer: Public instance variables are: self.: delim The server's folder delimiter. Only valid after acquireconnection() - """ + """ + GSS_STATE_STEP = 0 GSS_STATE_WRAP = 1 def __init__(self, repos): @@ -56,16 +57,16 @@ class IMAPServer: self.preauth_tunnel = repos.getpreauthtunnel() self.transport_tunnel = repos.gettransporttunnel() if self.preauth_tunnel and self.transport_tunnel: - raise OfflineImapError('%s: '% repos + \ - 'you must enable precisely one ' - 'type of tunnel (preauth or transport), ' - 'not both', OfflineImapError.ERROR.REPO) + raise OfflineImapError('%s: '% repos + + 'you must enable precisely one ' + 'type of tunnel (preauth or transport), ' + 'not both', OfflineImapError.ERROR.REPO) self.tunnel = \ - self.preauth_tunnel if self.preauth_tunnel \ - else self.transport_tunnel + self.preauth_tunnel if self.preauth_tunnel \ + else self.transport_tunnel self.username = \ - None if self.preauth_tunnel else repos.getuser() + None if self.preauth_tunnel else repos.getuser() self.user_identity = repos.get_remote_identity() self.authmechs = repos.get_auth_mechanisms() self.password = None @@ -74,7 +75,7 @@ class IMAPServer: self.usessl = repos.getssl() self.hostname = \ - None if self.preauth_tunnel else repos.gethost() + None if self.preauth_tunnel else repos.gethost() self.port = repos.getport() if self.port == None: self.port = 993 if self.usessl else 143 @@ -110,8 +111,8 @@ class IMAPServer: # get 1) configured password first 2) fall back to asking via UI self.password = self.repos.getpassword() or \ - self.ui.getpass(self.repos.getname(), self.config, - self.passworderror) + self.ui.getpass(self.repos.getname(), self.config, + self.passworderror) self.passworderror = None return self.password @@ -182,7 +183,7 @@ class IMAPServer: response = kerberos.authGSSClientResponse(self.gss_vc) rc = kerberos.authGSSClientStep(self.gss_vc, data) if rc != kerberos.AUTH_GSS_CONTINUE: - self.gss_step = self.GSS_STATE_WRAP + self.gss_step = self.GSS_STATE_WRAP elif self.gss_step == self.GSS_STATE_WRAP: rc = kerberos.authGSSClientUnwrap(self.gss_vc, data) response = kerberos.authGSSClientResponse(self.gss_vc) @@ -207,8 +208,8 @@ class IMAPServer: imapobj.starttls() except imapobj.error as e: raise OfflineImapError("Failed to start " - "TLS connection: %s" % str(e), - OfflineImapError.ERROR.REPO, None, exc_info()[2]) + "TLS connection: %s"% str(e), + OfflineImapError.ERROR.REPO, None, exc_info()[2]) ## All __authn_* procedures are helpers that do authentication. @@ -260,8 +261,8 @@ class IMAPServer: # (per RFC 2595) if 'LOGINDISABLED' in imapobj.capabilities: raise OfflineImapError("IMAP LOGIN is " - "disabled by server. Need to use SSL?", - OfflineImapError.ERROR.REPO) + "disabled by server. Need to use SSL?", + OfflineImapError.ERROR.REPO) else: self.__loginauth(imapobj) return True @@ -335,7 +336,7 @@ class IMAPServer: exc_stack )) raise OfflineImapError("All authentication types " - "failed:\n\t%s" % msg, OfflineImapError.ERROR.REPO) + "failed:\n\t%s"% msg, OfflineImapError.ERROR.REPO) if not tried_to_authn: methods = ", ".join(map( @@ -443,7 +444,7 @@ class IMAPServer: self.ui.warn(err) raise Exception(err) self.delim, self.root = \ - imaputil.imapsplit(listres[0])[1:] + imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) @@ -474,7 +475,7 @@ class IMAPServer: if self.port != 993: reason = "Could not connect via SSL to host '%s' and non-s"\ "tandard ssl port %d configured. Make sure you connect"\ - " to the correct port." % (self.hostname, self.port) + " to the correct port."% (self.hostname, self.port) else: reason = "Unknown SSL protocol connecting to host '%s' for "\ "repository '%s'. OpenSSL responded:\n%s"\ @@ -487,7 +488,7 @@ class IMAPServer: reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ - "network." % (self.hostname, self.port, self.repos) + "network."% (self.hostname, self.port, self.repos) raise OfflineImapError(reason, severity), None, exc_info()[2] # Could not acquire connection to the remote; # socket.error(last_error) raised @@ -709,15 +710,15 @@ class IdleThread(object): imapobj.idle(callback=callback) else: self.ui.warn("IMAP IDLE not supported on server '%s'." - "Sleep until next refresh cycle." % imapobj.identifier) + "Sleep until next refresh cycle."% imapobj.identifier) imapobj.noop() self.stop_sig.wait() # self.stop() or IDLE callback are invoked try: # End IDLE mode with noop, imapobj can point to a dropped conn. imapobj.noop() except imapobj.abort: - self.ui.warn('Attempting NOOP on dropped connection %s' % \ - imapobj.identifier) + self.ui.warn('Attempting NOOP on dropped connection %s'% + imapobj.identifier) self.parent.releaseconnection(imapobj, True) else: self.parent.releaseconnection(imapobj) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index e5eb541..f1f287b 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -211,7 +211,7 @@ def uid_sequence(uidlist): def getrange(start, end): if start == end: return(str(start)) - return "%s:%s" % (start, end) + return "%s:%s"% (start, end) if not len(uidlist): return '' # Empty list, return start, end = None, None diff --git a/offlineimap/init.py b/offlineimap/init.py index 8b20394..7f6a679 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -227,7 +227,7 @@ class OfflineImap: 'of %s'% ', '.join(UI_LIST.keys())) if options.diagnostics: ui_type = 'basic' # enforce basic UI for --info - #dry-run? Set [general]dry-run=True + # dry-run? Set [general]dry-run=True if options.dryrun: dryrun = config.set('general', 'dry-run', 'True') config.set_if_not_exists('general', 'dry-run', 'False') diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index d30d8a3..0cf44f8 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -115,7 +115,6 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): @property def readonly(self): """Is the repository readonly?""" - return self._readonly def getlocaleval(self): @@ -123,13 +122,11 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getfolders(self): """Returns a list of ALL folders on this server.""" - return [] def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" - pass def getsep(self): @@ -150,8 +147,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): self.getconfboolean('createfolders', True) def makefolder(self, foldername): - """Create a new folder""" - + """Create a new folder.""" raise NotImplementedError def deletefolder(self, foldername): @@ -200,8 +196,8 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): dst_haschanged = True # Need to refresh list except OfflineImapError as e: self.ui.error(e, exc_info()[2], - "Creating folder %s on repository %s" %\ - (src_name_t, dst_repo)) + "Creating folder %s on repository %s"% + (src_name_t, dst_repo)) raise status_repo.makefolder(src_name_t.replace(dst_repo.getsep(), status_repo.getsep())) @@ -218,8 +214,8 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): # case don't create it on it: if not self.should_sync_folder(dst_name_t): self.ui.debug('', "Not creating folder '%s' (repository '%s" - "') as it would be filtered out on that repository." % - (dst_name_t, self)) + "') as it would be filtered out on that repository."% + (dst_name_t, self)) continue # get IMAPFolder and see if the reverse nametrans # works fine TODO: getfolder() works only because we diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index c0cd887..b109546 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -38,7 +38,7 @@ class IMAPRepository(BaseRepository): self.folders = None if self.getconf('sep', None): self.ui.info("The 'sep' setting is being ignored for IMAP " - "repository '%s' (it's autodetected)" % self) + "repository '%s' (it's autodetected)"% self) def startkeepalive(self): keepalivetime = self.getkeepalive() @@ -85,7 +85,7 @@ class IMAPRepository(BaseRepository): acquireconnection() or it will still be `None`""" assert self.imapserver.delim != None, "'%s' " \ "repository called getsep() before the folder separator was " \ - "queried from the server" % self + "queried from the server"% self return self.imapserver.delim def gethost(self): @@ -101,10 +101,9 @@ class IMAPRepository(BaseRepository): try: host = self.localeval.eval(host) except Exception as e: - raise OfflineImapError("remotehosteval option for repository "\ - "'%s' failed:\n%s" % (self, e), - OfflineImapError.ERROR.REPO), \ - None, exc_info()[2] + raise OfflineImapError("remotehosteval option for repository " + "'%s' failed:\n%s"% (self, e), OfflineImapError.ERROR.REPO), \ + None, exc_info()[2] if host: self._host = host return self._host @@ -115,9 +114,8 @@ class IMAPRepository(BaseRepository): return self._host # no success - raise OfflineImapError("No remote host for repository "\ - "'%s' specified." % self, - OfflineImapError.ERROR.REPO) + raise OfflineImapError("No remote host for repository " + "'%s' specified."% self, OfflineImapError.ERROR.REPO) def get_remote_identity(self): """Remote identity is used for certain SASL mechanisms @@ -139,8 +137,8 @@ class IMAPRepository(BaseRepository): for m in mechs: if m not in supported: - raise OfflineImapError("Repository %s: " % self + \ - "unknown authentication mechanism '%s'" % m, + raise OfflineImapError("Repository %s: "% self + \ + "unknown authentication mechanism '%s'"% m, OfflineImapError.ERROR.REPO) self.ui.debug('imap', "Using authentication mechanisms %s" % mechs) @@ -431,9 +429,8 @@ class IMAPRepository(BaseRepository): result = imapobj.create(foldername) if result[0] != 'OK': raise OfflineImapError("Folder '%s'[%s] could not be created. " - "Server responded: %s" % \ - (foldername, self, str(result)), - OfflineImapError.ERROR.FOLDER) + "Server responded: %s"% (foldername, self, str(result)), + OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 170145b..fc34a55 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -28,13 +28,13 @@ class LocalStatusRepository(BaseRepository): # class and root for all backends self.backends = {} self.backends['sqlite'] = { - 'class': LocalStatusSQLiteFolder, - 'root': os.path.join(account.getaccountmeta(), 'LocalStatus-sqlite') + 'class': LocalStatusSQLiteFolder, + 'root': os.path.join(account.getaccountmeta(), 'LocalStatus-sqlite') } self.backends['plain'] = { - 'class': LocalStatusFolder, - 'root': os.path.join(account.getaccountmeta(), 'LocalStatus') + 'class': LocalStatusFolder, + 'root': os.path.join(account.getaccountmeta(), 'LocalStatus') } # Set class and root for the configured backend @@ -54,7 +54,7 @@ class LocalStatusRepository(BaseRepository): else: raise SyntaxWarning("Unknown status_backend '%s' for account '%s'"% - (backend, self.account.name)) + (backend, self.account.name)) def import_other_backend(self, folder): for bk, dic in self.backends.items(): @@ -101,7 +101,7 @@ class LocalStatusRepository(BaseRepository): folder = self.LocalStatusFolderClass(foldername, self) - # if folder is empty, try to import data from an other backend + # If folder is empty, try to import data from an other backend. if folder.isnewfolder(): self.import_other_backend(folder) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 58bf664..0262ba2 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -115,7 +115,7 @@ class MaildirRepository(BaseRepository): except OSError as e: if e.errno == 17 and os.path.isdir(full_path): self.debug("makefolder: '%s' already has subdir %s"% - (foldername, subdir)) + (foldername, subdir)) else: raise diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py index 59a7bb6..0fbbc13 100644 --- a/offlineimap/repository/__init__.py +++ b/offlineimap/repository/__init__.py @@ -53,7 +53,7 @@ class Repository(object): 'GmailMaildir': GmailMaildirRepository} elif reqtype == 'status': - # create and return a LocalStatusRepository + # create and return a LocalStatusRepository. name = account.getconf('localrepository') return LocalStatusRepository(name, account) @@ -61,7 +61,7 @@ class Repository(object): errstr = "Repository type %s not supported" % reqtype raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO) - # Get repository type + # Get repository type. config = account.getconfig() try: repostype = config.get('Repository ' + name, 'type').strip() @@ -74,8 +74,8 @@ class Repository(object): try: repo = typemap[repostype] except KeyError: - errstr = "'%s' repository not supported for '%s' repositories." \ - % (repostype, reqtype) + errstr = "'%s' repository not supported for '%s' repositories."% \ + (repostype, reqtype) raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \ None, exc_info()[2] diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 33dbd64..f69f8a6 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -32,6 +32,7 @@ from offlineimap.ui import getglobalui def semaphorereset(semaphore, originalstate): """Block until `semaphore` gets back to its original state, ie all acquired resources have been released.""" + for i in range(originalstate): semaphore.acquire() # Now release these. @@ -41,6 +42,7 @@ def semaphorereset(semaphore, originalstate): class threadlist: """Store the list of all threads in the software so it can be used to find out what's running and what's not.""" + def __init__(self): self.lock = Lock() self.list = [] @@ -98,6 +100,7 @@ def exitnotifymonitorloop(callback): while the other thread is waiting. :type callback: a callable function """ + global exitthreads do_loop = True while do_loop: @@ -116,6 +119,7 @@ def threadexited(thread): """Called when a thread exits. Main thread is aborted when this returns True.""" + ui = getglobalui() if thread.exit_exception: if isinstance(thread.exit_exception, SystemExit): @@ -139,8 +143,9 @@ class ExitNotifyThread(Thread): The thread can set instance variables self.exit_message for a human readable reason of the thread exit.""" + profiledir = None - """class variable that is set to the profile directory if required""" + """Class variable that is set to the profile directory if required.""" def __init__(self, *args, **kwargs): super(ExitNotifyThread, self).__init__(*args, **kwargs) @@ -167,7 +172,7 @@ class ExitNotifyThread(Thread): except SystemExit: pass prof.dump_stats(os.path.join(ExitNotifyThread.profiledir, - "%s_%s.prof" % (self.ident, self.getName()))) + "%s_%s.prof"% (self.ident, self.getName()))) except Exception as e: # Thread exited with Exception, store it tb = traceback.format_exc() @@ -179,6 +184,7 @@ class ExitNotifyThread(Thread): def set_exit_exception(self, exc, st=None): """Sets Exception and stacktrace of a thread, so that other threads can query its exit status""" + self._exit_exc = exc self._exit_stacktrace = st @@ -187,16 +193,19 @@ class ExitNotifyThread(Thread): """Returns the cause of the exit, one of: Exception() -- the thread aborted with this exception None -- normal termination.""" + return self._exit_exc @property def exit_stacktrace(self): """Returns a string representing the stack trace if set""" + return self._exit_stacktrace @classmethod def set_profiledir(cls, directory): """If set, will output profile information to 'directory'""" + cls.profiledir = directory @@ -210,6 +219,7 @@ instancelimitedlock = Lock() def initInstanceLimit(instancename, instancemax): """Initialize the instance-limited thread implementation to permit up to intancemax threads with the given instancename.""" + instancelimitedlock.acquire() if not instancename in instancelimitedsems: instancelimitedsems[instancename] = BoundedSemaphore(instancemax) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index fb3da80..ddc05ea 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -33,17 +33,19 @@ class CursesUtil: # iolock protects access to the self.iolock = RLock() self.tframe_lock = RLock() - """tframe_lock protects the self.threadframes manipulation to - only happen from 1 thread""" + # tframe_lock protects the self.threadframes manipulation to + # only happen from 1 thread. self.colormap = {} """dict, translating color string to curses color pair number""" def curses_colorpair(self, col_name): - """Return the curses color pair, that corresponds to the color""" + """Return the curses color pair, that corresponds to the color.""" + return curses.color_pair(self.colormap[col_name]) def init_colorpairs(self): - """initialize the curses color pairs available""" + """Initialize the curses color pairs available.""" + # set special colors 'gray' and 'banner' self.colormap['white'] = 0 #hardcoded by curses curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) @@ -66,24 +68,27 @@ class CursesUtil: curses.init_pair(i, fcol, bcol) def lock(self, block=True): - """Locks the Curses ui thread + """Locks the Curses ui thread. Can be invoked multiple times from the owning thread. Invoking from a non-owning thread blocks and waits until it has been unlocked by the owning thread.""" + return self.iolock.acquire(block) def unlock(self): - """Unlocks the Curses ui thread + """Unlocks the Curses ui thread. Decrease the lock counter by one and unlock the ui thread if the counter reaches 0. Only call this method when the calling thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked.""" + self.iolock.release() def exec_locked(self, target, *args, **kwargs): """Perform an operation with full locking.""" + self.lock() try: target(*args, **kwargs) @@ -113,31 +118,34 @@ class CursesAccountFrame: def __init__(self, ui, account): """ :param account: An Account() or None (for eg SyncrunnerThread)""" + self.children = [] self.account = account if account else '*Control' self.ui = ui self.window = None - """Curses window associated with this acc""" + # Curses window associated with this acc. self.acc_num = None - """Account number (& hotkey) associated with this acc""" + # Account number (& hotkey) associated with this acc. self.location = 0 - """length of the account prefix string""" + # length of the account prefix string def drawleadstr(self, secs = 0): - """Draw the account status string + """Draw the account status string. secs tells us how long we are going to sleep.""" - sleepstr = '%3d:%02d' % (secs // 60, secs % 60) if secs else 'active' - accstr = '%s: [%s] %12.12s: ' % (self.acc_num, sleepstr, self.account) + + sleepstr = '%3d:%02d'% (secs // 60, secs % 60) if secs else 'active' + accstr = '%s: [%s] %12.12s: '% (self.acc_num, sleepstr, self.account) self.ui.exec_locked(self.window.addstr, 0, 0, accstr) self.location = len(accstr) def setwindow(self, curses_win, acc_num): - """Register an curses win and a hotkey as Account window + """Register an curses win and a hotkey as Account window. :param curses_win: the curses window associated with an account :param acc_num: int denoting the hotkey associated with this account.""" + self.window = curses_win self.acc_num = acc_num self.drawleadstr() @@ -147,39 +155,43 @@ class CursesAccountFrame: self.location += 1 def get_new_tframe(self): - """Create a new ThreadFrame and append it to self.children + """Create a new ThreadFrame and append it to self.children. :returns: The new ThreadFrame""" + tf = CursesThreadFrame(self.ui, self.window, self.location, 0) self.location += 1 self.children.append(tf) return tf def sleeping(self, sleepsecs, remainingsecs): - """show how long we are going to sleep and sleep + """Show how long we are going to sleep and sleep. :returns: Boolean, whether we want to abort the sleep""" + self.drawleadstr(remainingsecs) self.ui.exec_locked(self.window.refresh) time.sleep(sleepsecs) return self.account.get_abort_event() def syncnow(self): - """Request that we stop sleeping asap and continue to sync""" + """Request that we stop sleeping asap and continue to sync.""" + # if this belongs to an Account (and not *Control), set the # skipsleep pref if isinstance(self.account, offlineimap.accounts.Account): - self.ui.info("Requested synchronization for acc: %s" % self.account) - self.account.config.set('Account %s' % self.account.name, - 'skipsleep', '1') + self.ui.info("Requested synchronization for acc: %s"% self.account) + self.account.config.set('Account %s'% self.account.name, + 'skipsleep', '1') class CursesThreadFrame: - """ - curses_color: current color pair for logging""" + """curses_color: current color pair for logging.""" + def __init__(self, ui, acc_win, x, y): """ :param ui: is a Blinkenlights() instance :param acc_win: curses Account window""" + self.ui = ui self.window = acc_win self.x = x @@ -188,7 +200,10 @@ class CursesThreadFrame: def setcolor(self, color, modifier=0): """Draw the thread symbol '@' in the specified color - :param modifier: Curses modified, such as curses.A_BOLD""" + + :param modifier: Curses modified, such as curses.A_BOLD + """ + self.curses_color = modifier | self.ui.curses_colorpair(color) self.colorname = color self.display() @@ -201,7 +216,8 @@ class CursesThreadFrame: self.ui.exec_locked(locked_display) def update(self, acc_win, x, y): - """Update the xy position of the '.' (and possibly the aframe)""" + """Update the xy position of the '.' (and possibly the aframe).""" + self.window = acc_win self.y = y self.x = x @@ -213,6 +229,7 @@ class CursesThreadFrame: class InputHandler(ExitNotifyThread): """Listens for input via the curses interfaces""" + #TODO, we need to use the ugly exitnotifythread (rather than simply #threading.Thread here, so exiting this thread via the callback #handler, kills off all parents too. Otherwise, they would simply @@ -222,17 +239,18 @@ class InputHandler(ExitNotifyThread): self.char_handler = None self.ui = ui self.enabled = Event() - """We will only parse input if we are enabled""" + # We will only parse input if we are enabled. self.inputlock = RLock() - """denotes whether we should be handling the next char.""" + # denotes whether we should be handling the next char. self.start() #automatically start the thread def get_next_char(self): - """return the key pressed or -1 + """Return the key pressed or -1. Wait until `enabled` and loop internally every stdscr.timeout() msecs, releasing the inputlock. :returns: char or None if disabled while in here""" + self.enabled.wait() while self.enabled.is_set(): with self.inputlock: @@ -247,13 +265,14 @@ class InputHandler(ExitNotifyThread): #curses.ungetch(char) def set_char_hdlr(self, callback): - """Sets a character callback handler + """Sets a character callback handler. If a key is pressed it will be passed to this handler. Keys include the curses.KEY_RESIZE key. callback is a function taking a single arg -- the char pressed. If callback is None, input will be ignored.""" + with self.inputlock: self.char_handler = callback # start or stop the parsing of things @@ -266,13 +285,14 @@ class InputHandler(ExitNotifyThread): """Call this method when you want exclusive input control. Make sure to call input_release afterwards! While this lockis - held, input can go to e.g. the getpass input. - """ + held, input can go to e.g. the getpass input.""" + self.enabled.clear() self.inputlock.acquire() def input_release(self): """Call this method when you are done getting input.""" + self.inputlock.release() self.enabled.set() @@ -301,7 +321,7 @@ class CursesLogHandler(logging.StreamHandler): self.ui.stdscr.refresh() class Blinkenlights(UIBase, CursesUtil): - """Curses-cased fancy UI + """Curses-cased fancy UI. Notable instance variables self. ....: @@ -319,7 +339,7 @@ class Blinkenlights(UIBase, CursesUtil): ################################################## UTILS def setup_consolehandler(self): - """Backend specific console handler + """Backend specific console handler. Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" @@ -337,7 +357,7 @@ class Blinkenlights(UIBase, CursesUtil): return ch def isusable(s): - """Returns true if the backend is usable ie Curses works""" + """Returns true if the backend is usable ie Curses works.""" # Not a terminal? Can't use curses. if not sys.stdout.isatty() and sys.stdin.isatty(): @@ -393,7 +413,7 @@ class Blinkenlights(UIBase, CursesUtil): self.info(offlineimap.banner) def acct(self, *args): - """Output that we start syncing an account (and start counting)""" + """Output that we start syncing an account (and start counting).""" self.gettf().setcolor('purple') super(Blinkenlights, self).acct(*args) @@ -458,7 +478,8 @@ class Blinkenlights(UIBase, CursesUtil): super(Blinkenlights, self).threadExited(thread) def gettf(self): - """Return the ThreadFrame() of the current thread""" + """Return the ThreadFrame() of the current thread.""" + cur_thread = currentThread() acc = self.getthreadaccount() #Account() or None @@ -504,7 +525,7 @@ class Blinkenlights(UIBase, CursesUtil): def sleep(self, sleepsecs, account): self.gettf().setcolor('red') - self.info("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) + self.info("Next sync in %d:%02d"% (sleepsecs / 60, sleepsecs % 60)) return super(Blinkenlights, self).sleep(sleepsecs, account) def sleeping(self, sleepsecs, remainingsecs): @@ -515,13 +536,14 @@ class Blinkenlights(UIBase, CursesUtil): return accframe.sleeping(sleepsecs, remainingsecs) def resizeterm(self): - """Resize the current windows""" + """Resize the current windows.""" + self.exec_locked(self.setupwindows, True) def mainException(self): UIBase.mainException(self) - def getpass(self, accountname, config, errmsg = None): + def getpass(self, accountname, config, errmsg=None): # disable the hotkeys inputhandler self.inputhandler.input_acquire() @@ -540,10 +562,11 @@ class Blinkenlights(UIBase, CursesUtil): return password def setupwindows(self, resize=False): - """Setup and draw bannerwin and logwin + """Setup and draw bannerwin and logwin. If `resize`, don't create new windows, just adapt size. This function should be invoked with CursesUtils.locked().""" + self.height, self.width = self.stdscr.getmaxyx() self.logheight = self.height - len(self.accframes) - 1 if resize: @@ -571,22 +594,24 @@ class Blinkenlights(UIBase, CursesUtil): curses.doupdate() def draw_bannerwin(self): - """Draw the top-line banner line""" + """Draw the top-line banner line.""" + if curses.has_colors(): color = curses.A_BOLD | self.curses_colorpair('banner') else: color = curses.A_REVERSE self.bannerwin.clear() # Delete old content (eg before resizes) self.bannerwin.bkgd(' ', color) # Fill background with that color - string = "%s %s" % (offlineimap.__productname__, - offlineimap.__bigversion__) + string = "%s %s"% (offlineimap.__productname__, + offlineimap.__bigversion__) self.bannerwin.addstr(0, 0, string, color) self.bannerwin.addstr(0, self.width -len(offlineimap.__copyright__) -1, offlineimap.__copyright__, color) self.bannerwin.noutrefresh() def draw_logwin(self): - """(Re)draw the current logwindow""" + """(Re)draw the current logwindow.""" + if curses.has_colors(): color = curses.color_pair(0) #default colors else: @@ -596,9 +621,10 @@ class Blinkenlights(UIBase, CursesUtil): self.logwin.bkgd(' ', color) def getaccountframe(self, acc_name): - """Return an AccountFrame() corresponding to acc_name + """Return an AccountFrame() corresponding to acc_name. Note that the *control thread uses acc_name `None`.""" + with self.aflock: # 1) Return existing or 2) create a new CursesAccountFrame. if acc_name in self.accframes: return self.accframes[acc_name] diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 41ce1a4..9285b51 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -36,14 +36,18 @@ debugtypes = {'':'Other offlineimap related sync messages', globalui = None def setglobalui(newui): - """Set the global ui object to be used for logging""" + """Set the global ui object to be used for logging.""" + global globalui globalui = newui + def getglobalui(): - """Return the current ui object""" + """Return the current ui object.""" + global globalui return globalui + class UIBase(object): def __init__(self, config, loglevel=logging.INFO): self.config = config @@ -65,11 +69,11 @@ class UIBase(object): self.logger = logging.getLogger('OfflineImap') self.logger.setLevel(loglevel) self._log_con_handler = self.setup_consolehandler() - """The console handler (we need access to be able to lock it)""" + """The console handler (we need access to be able to lock it).""" ################################################## UTILS def setup_consolehandler(self): - """Backend specific console handler + """Backend specific console handler. Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" @@ -86,10 +90,11 @@ class UIBase(object): return ch def setlogfile(self, logfile): - """Create file handler which logs to file""" + """Create file handler which logs to file.""" + fh = logging.FileHandler(logfile, 'at') file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " - "%(message)s", '%Y-%m-%d %H:%M:%S') + "%(message)s", '%Y-%m-%d %H:%M:%S') fh.setFormatter(file_formatter) self.logger.addHandler(fh) # write out more verbose initial info blurb on the log file @@ -107,13 +112,14 @@ class UIBase(object): def info(self, msg): """Display a message.""" + self.logger.info(msg) - def warn(self, msg, minor = 0): + def warn(self, msg, minor=0): self.logger.warning(msg) def error(self, exc, exc_traceback=None, msg=None): - """Log a message at severity level ERROR + """Log a message at severity level ERROR. Log Exception 'exc' to error log, possibly prepended by a preceding error "msg", detailing at what point the error occurred. @@ -134,7 +140,7 @@ class UIBase(object): One example of such a call might be: ui.error(exc, sys.exc_info()[2], msg="While syncing Folder %s in " - "repo %s") + "repo %s") """ if msg: self.logger.error("ERROR: %s\n %s" % (msg, exc)) @@ -160,8 +166,8 @@ class UIBase(object): "'%s')" % (cur_thread.getName(), self.getthreadaccount(cur_thread), account)) else: - self.debug('thread', "Register new thread '%s' (account '%s')" %\ - (cur_thread.getName(), account)) + self.debug('thread', "Register new thread '%s' (account '%s')"% + (cur_thread.getName(), account)) self.threadaccounts[cur_thread] = account def unregisterthread(self, thr): @@ -216,7 +222,7 @@ class UIBase(object): self.warn("Invalid debug type: %s" % debugtype) def getnicename(self, object): - """Return the type of a repository or Folder as string + """Return the type of a repository or Folder as string. (IMAP, Gmail, Maildir, etc...)""" @@ -234,49 +240,49 @@ class UIBase(object): ################################################## INPUT def getpass(self, accountname, config, errmsg = None): - raise NotImplementedError("Prompting for a password is not supported"\ - " in this UI backend.") + raise NotImplementedError("Prompting for a password is not supported" + " in this UI backend.") def folderlist(self, folder_list): return ', '.join(["%s[%s]"% \ - (self.getnicename(x), x.getname()) for x in folder_list]) + (self.getnicename(x), x.getname()) for x in folder_list]) ################################################## WARNINGS def msgtoreadonly(self, destfolder, uid, content, flags): if self.config.has_option('general', 'ignore-readonly') and \ - self.config.getboolean('general', 'ignore-readonly'): + self.config.getboolean('general', 'ignore-readonly'): return self.warn("Attempted to synchronize message %d to folder %s[%s], " - "but that folder is read-only. The message will not be " - "copied to that folder." % ( - uid, self.getnicename(destfolder), destfolder)) + "but that folder is read-only. The message will not be " + "copied to that folder."% ( + uid, self.getnicename(destfolder), destfolder)) def flagstoreadonly(self, destfolder, uidlist, flags): if self.config.has_option('general', 'ignore-readonly') and \ self.config.getboolean('general', 'ignore-readonly'): return self.warn("Attempted to modify flags for messages %s in folder %s[%s], " - "but that folder is read-only. No flags have been modified " - "for that message." % ( - str(uidlist), self.getnicename(destfolder), destfolder)) + "but that folder is read-only. No flags have been modified " + "for that message."% ( + str(uidlist), self.getnicename(destfolder), destfolder)) def labelstoreadonly(self, destfolder, uidlist, labels): if self.config.has_option('general', 'ignore-readonly') and \ self.config.getboolean('general', 'ignore-readonly'): return self.warn("Attempted to modify labels for messages %s in folder %s[%s], " - "but that folder is read-only. No labels have been modified " - "for that message." % ( - str(uidlist), self.getnicename(destfolder), destfolder)) + "but that folder is read-only. No labels have been modified " + "for that message."% ( + str(uidlist), self.getnicename(destfolder), destfolder)) def deletereadonly(self, destfolder, uidlist): if self.config.has_option('general', 'ignore-readonly') and \ self.config.getboolean('general', 'ignore-readonly'): return self.warn("Attempted to delete messages %s in folder %s[%s], but that " - "folder is read-only. No messages have been deleted in that " - "folder." % (str(uidlist), self.getnicename(destfolder), - destfolder)) + "folder is read-only. No messages have been deleted in that " + "folder."% (str(uidlist), self.getnicename(destfolder), + destfolder)) ################################################## MESSAGES @@ -293,7 +299,7 @@ class UIBase(object): if not self.logger.isEnabledFor(logging.INFO): return displaystr = '' hostname = hostname if hostname else '' - port = "%s" % port if port else '' + port = "%s"% port if port else '' if hostname: displaystr = ' to %s:%s' % (hostname, port) self.logger.info("Establishing connection%s" % displaystr) @@ -309,8 +315,8 @@ class UIBase(object): sec = time.time() - self.acct_startimes[account] del self.acct_startimes[account] - self.logger.info("*** Finished account '%s' in %d:%02d" % - (account, sec // 60, sec % 60)) + self.logger.info("*** Finished account '%s' in %d:%02d"% + (account, sec // 60, sec % 60)) def syncfolders(self, src_repo, dst_repo): """Log 'Copying folder structure...'.""" @@ -321,16 +327,18 @@ class UIBase(object): ############################## Folder syncing def makefolder(self, repo, foldername): - """Called when a folder is created""" + """Called when a folder is created.""" + prefix = "[DRYRUN] " if self.dryrun else "" - self.info("{0}Creating folder {1}[{2}]".format( - prefix, foldername, repo)) + self.info(("{0}Creating folder {1}[{2}]".format( + prefix, foldername, repo))) def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder): """Called when a folder sync operation is started.""" - self.logger.info("Syncing %s: %s -> %s" % (srcfolder, - self.getnicename(srcrepos), - self.getnicename(destrepos))) + + self.logger.info("Syncing %s: %s -> %s"% (srcfolder, + self.getnicename(srcrepos), + self.getnicename(destrepos))) def skippingfolder(self, folder): """Called when a folder sync operation is started.""" @@ -344,7 +352,7 @@ class UIBase(object): folder.get_saveduidvalidity(), folder.get_uidvalidity())) def loadmessagelist(self, repos, folder): - self.logger.debug(u"Loading message list for %s[%s]"% ( + self.logger.debug("Loading message list for %s[%s]"% ( self.getnicename(repos), folder)) @@ -369,8 +377,8 @@ class UIBase(object): ds = self.folderlist(destlist) prefix = "[DRYRUN] " if self.dryrun else "" self.info("{0}Deleting {1} messages ({2}) in {3}".format( - prefix, len(uidlist), - offlineimap.imaputil.uid_sequence(uidlist), ds)) + prefix, len(uidlist), + offlineimap.imaputil.uid_sequence(uidlist), ds)) def addingflags(self, uidlist, flags, dest): self.logger.info("Adding flag %s to %d messages on %s" % ( @@ -407,9 +415,8 @@ class UIBase(object): self.getnicename(repository))) try: if hasattr(repository, 'gethost'): # IMAP - self._msg("Host: %s Port: %s SSL: %s" % (repository.gethost(), - repository.getport(), - repository.getssl())) + self._msg("Host: %s Port: %s SSL: %s"% (repository.gethost(), + repository.getport(), repository.getssl())) try: conn = repository.imapserver.acquireconnection() except OfflineImapError as e: @@ -437,8 +444,8 @@ class UIBase(object): self._msg("nametrans= %s\n" % nametrans) folders = repository.getfolders() - foldernames = [(f.name, f.getvisiblename(), f.sync_this) \ - for f in folders] + foldernames = [(f.name, f.getvisiblename(), f.sync_this) + for f in folders] folders = [] for name, visiblename, sync_this in foldernames: syncstr = "" if sync_this else " (disabled)" @@ -454,8 +461,8 @@ class UIBase(object): def savemessage(self, debugtype, uid, flags, folder): """Output a log line stating that we save a msg.""" - self.debug(debugtype, u"Write mail '%s:%d' with flags %s"% - (folder, uid, repr(flags))) + self.debug(debugtype, "Write mail '%s:%d' with flags %s"% + (folder, uid, repr(flags))) ################################################## Threads @@ -465,8 +472,8 @@ class UIBase(object): % (len(self.debugmessages[thread]), thread.getName()) message += "\n".join(self.debugmessages[thread]) else: - message = "\nNo debug messages were logged for %s." % \ - thread.getName() + message = "\nNo debug messages were logged for %s."% \ + thread.getName() return message def delThreadDebugLog(self, thread): @@ -474,9 +481,9 @@ class UIBase(object): del self.debugmessages[thread] def getThreadExceptionString(self, thread): - message = u"Thread '%s' terminated with exception:\n%s"% \ - (thread.getName(), thread.exit_stacktrace) - message += u"\n" + self.getThreadDebugLog(thread) + message = "Thread '%s' terminated with exception:\n%s"% \ + (thread.getName(), thread.exit_stacktrace) + message += "\n" + self.getThreadDebugLog(thread) return message def threadException(self, thread): @@ -492,21 +499,21 @@ class UIBase(object): #print any exceptions that have occurred over the run if not self.exc_queue.empty(): - self.warn(u"ERROR: Exceptions occurred during the run!") + self.warn("ERROR: Exceptions occurred during the run!") while not self.exc_queue.empty(): msg, exc, exc_traceback = self.exc_queue.get() if msg: - self.warn(u"ERROR: %s\n %s"% (msg, exc)) + self.warn("ERROR: %s\n %s"% (msg, exc)) else: - self.warn(u"ERROR: %s"% (exc)) + self.warn("ERROR: %s"% (exc)) if exc_traceback: - self.warn(u"\nTraceback:\n%s"% "".join( + self.warn("\nTraceback:\n%s"% "".join( traceback.format_tb(exc_traceback))) if errormsg and errortitle: - self.warn(u'ERROR: %s\n\n%s\n'% (errortitle, errormsg)) + self.warn('ERROR: %s\n\n%s\n'% (errortitle, errormsg)) elif errormsg: - self.warn(u'%s\n' % errormsg) + self.warn('%s\n'% errormsg) sys.exit(exitstatus) def threadExited(self, thread): From 95eb8697f9ab757b796693517bfa84690ac1cd87 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 18 Jan 2015 22:09:41 +0100 Subject: [PATCH 715/817] folder: LocalStatus: avoid to redefine 'file' Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatus.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 57c696d..c753f63 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -114,8 +114,8 @@ class LocalStatusFolder(BaseFolder): # Loop as many times as version, and update format. for i in range(1, self.cur_version + 1): self.messagelist = {} - cache = open(self.filename, "rt") - line = cache.readline().strip() + cachefd = open(self.filename, "rt") + line = cachefd.readline().strip() # Format is up to date. break. if line == (self.magicline % self.cur_version): @@ -125,8 +125,8 @@ class LocalStatusFolder(BaseFolder): elif line == (self.magicline % 1): self.ui._msg('Upgrading LocalStatus cache from version 1' 'to version 2 for %s:%s'% (self.repository, self)) - self.readstatus_v1(cache) - cache.close() + self.readstatus_v1(cachefd) + cachefd.close() self.save() # NOTE: Add other format transitions here in the future. @@ -148,12 +148,12 @@ class LocalStatusFolder(BaseFolder): # but somehow did. errstr = "Cache file '%s' is empty."% self.filename self.ui.warn(errstr) - cache.close() + cachefd.close() return assert(line == (self.magicline % self.cur_version)) - self.readstatus(cache) - cache.close() + self.readstatus(cachefd) + cachefd.close() def dropmessagelistcache(self): self.messagelist = None @@ -167,16 +167,16 @@ class LocalStatusFolder(BaseFolder): """Saves the entire messagelist to disk.""" with self.savelock: - file = open(self.filename + ".tmp", "wt") - file.write((self.magicline % self.cur_version) + "\n") + cachefd = open(self.filename + ".tmp", "wt") + cachefd.write((self.magicline % self.cur_version) + "\n") for msg in self.messagelist.values(): flags = ''.join(sorted(msg['flags'])) labels = ', '.join(sorted(msg['labels'])) - file.write("%s|%s|%d|%s\n" % (msg['uid'], flags, msg['mtime'], labels)) - file.flush() + cachefd.write("%s|%s|%d|%s\n" % (msg['uid'], flags, msg['mtime'], labels)) + cachefd.flush() if self.doautosave: - os.fsync(file.fileno()) - file.close() + os.fsync(cachefd.fileno()) + cachefd.close() os.rename(self.filename + ".tmp", self.filename) if self.doautosave: From baee2b6fd99f8e9b0d58f9c915dd281c721883c3 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Feb 2015 17:06:27 +0100 Subject: [PATCH 716/817] fix: folder/Base: missing """ for doc string Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 3a04ef6..bdae5c3 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -197,7 +197,7 @@ class BaseFolder(object): return True def _getuidfilename(self): - """provides UIDVALIDITY cache filename for class internal purposes. + """provides UIDVALIDITY cache filename for class internal purposes.""" return os.path.join(self.repository.getuiddir(), self.getfolderbasename()) From 9e63fa3784b8adc7b1dd28871ec5e80404d32b12 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Feb 2015 17:02:33 +0100 Subject: [PATCH 717/817] fix: folder/*: never set self.messagelist to None Empty the list by setting an empty dict. Introduce BaseFolder().ismessagelistempty(). Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 11 ++++++++++- offlineimap/folder/GmailMaildir.py | 2 +- offlineimap/folder/IMAP.py | 4 ++-- offlineimap/folder/LocalStatus.py | 3 --- offlineimap/folder/LocalStatusSQLite.py | 2 +- offlineimap/folder/Maildir.py | 7 ++----- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index bdae5c3..df2e654 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -248,8 +248,17 @@ class BaseFolder(object): raise NotImplementedError + def ismessagelistempty(self): + """Empty everythings we know about messages.""" + + if len(self.messagelist.keys()) < 1: + return True + return False + def dropmessagelistcache(self): - raise NotImplementedException + """Empty everythings we know about messages.""" + + self.messagelist = {} def getmessagelist(self): """Gets the current message list. diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 5ca0e1f..894792d 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -65,7 +65,7 @@ class GmailMaildirFolder(MaildirFolder): def cachemessagelist(self): - if self.messagelist is None: + if self.ismessagelistempty(): self.messagelist = self._scanfolder() # Get mtimes diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index c7e6516..ccc83b0 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -48,7 +48,7 @@ class IMAPFolder(BaseFolder): self.expunge = repository.getexpunge() self.root = None # imapserver.root self.imapserver = imapserver - self.messagelist = None + self.messagelist = {} self.randomgenerator = random.Random() #self.ui is set in BaseFolder self.imap_query = ['BODY.PEEK[]'] @@ -249,7 +249,7 @@ class IMAPFolder(BaseFolder): self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} def dropmessagelistcache(self): - self.messagelist = None + self.messagelist = {} # Interface from BaseFolder def getmessagelist(self): diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index c753f63..78f2134 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -155,9 +155,6 @@ class LocalStatusFolder(BaseFolder): self.readstatus(cachefd) cachefd.close() - def dropmessagelistcache(self): - self.messagelist = None - def save(self): """Save changed data to disk. For this backend it is the same as saveall.""" diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 8a3b9df..a53b48e 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -202,7 +202,7 @@ class LocalStatusSQLiteFolder(BaseFolder): self.messagelist[uid]['mtime'] = row[2] def dropmessagelistcache(self): - self.messagelist = None + self.messagelist = {} # Interface from LocalStatusFolder def save(self): diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index b35bfc2..4bbfe7e 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -63,7 +63,7 @@ class MaildirFolder(BaseFolder): super(MaildirFolder, self).__init__(name, repository) self.dofsync = self.config.getdefaultboolean("general", "fsync", True) self.root = root - self.messagelist = None + self.messagelist = {} # check if we should use a different infosep to support Win file systems self.wincompatible = self.config.getdefaultboolean( "Account "+self.accountname, "maildir-windows-compatible", False) @@ -220,12 +220,9 @@ class MaildirFolder(BaseFolder): # Interface from BaseFolder def cachemessagelist(self): - if self.messagelist is None: + if self.ismessagelistempty(): self.messagelist = self._scanfolder() - def dropmessagelistcache(self): - self.messagelist = None - # Interface from BaseFolder def getmessagelist(self): return self.messagelist From 655c2b1fb9052b192bf7be32f8af716ed8c1396d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Feb 2015 17:18:07 +0100 Subject: [PATCH 718/817] sqlite: provide offending filename when open fails Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index a53b48e..c66ca44 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -67,8 +67,9 @@ class LocalStatusSQLiteFolder(BaseFolder): self.connection = sqlite.connect(self.filename, check_same_thread=False) except NameError: # sqlite import had failed - raise UserWarning('SQLite backend chosen, but no sqlite python ' - 'bindings available. Please install.'), None, exc_info()[2] + raise UserWarning("SQLite backend chosen, but cannot connect " + "with available bindings to '%s'. Is the sqlite3 package " + "installed?."% self.filename), None, exc_info()[2] #Make sure sqlite is in multithreading SERIALIZE mode assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' From 30499cf6803e4483b87b8fd7c759add3e778ab35 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Feb 2015 17:30:10 +0100 Subject: [PATCH 719/817] MANUAL: rename "KNOWN BUGS" TO "KNOWN ISSUES" Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 7fc355a..41eb6fa 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -349,8 +349,8 @@ how to use folder filters and name transformations. The documentation will be autogenerated by a "make doc" in the docs directory. It is also viewable at :ref:`folder_filtering_and_name_translation`. -KNOWN BUGS -========== +KNOWN ISSUES +============ * SSL3 write pending: users enabling SSL may hit a bug about "SSL3 write pending". If so, the From cd962d40a8febaacdb7ea5eaceaa79d840ef417c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Feb 2015 17:31:18 +0100 Subject: [PATCH 720/817] MANUAL: add known issues entry about socktimeout for suspended sessions Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 41eb6fa..5bdc067 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -393,6 +393,11 @@ KNOWN ISSUES * Use cygwin managed mount (not tested) - not available anymore since cygwin 1.7 +* OfflineIMAP confused after system suspend. + When resuming a suspended session, OfflineIMAP does not cleanly handles the + broken socket(s) if socktimeout option is not set. + You should enable this option with a value like 10. + .. _pitfalls: PITFALLS & ISSUES From 73952b8c2c76ae51a31d7746b9db30ae6992f09a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Feb 2015 17:43:29 +0100 Subject: [PATCH 721/817] offlineimap.conf: say what is the default value for the sep option Some style improvements. Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 3 ++- offlineimap/folder/LocalStatusSQLite.py | 26 ++++++++++++------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index cfafb23..3ffc2e6 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -403,8 +403,9 @@ localfolders = ~/Test # folders. It is inserted in-between the components of the tree. If you # want your folders to be nested directories, set it to "/". 'sep' is # ignored for IMAP repositories, as it is queried automatically. +# Otherwise, default value is ".". # -#sep = . +#sep = "." # This option stands in the [Repository LocalExample] section. diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index c66ca44..f54ba99 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -33,24 +33,24 @@ class LocalStatusSQLiteFolder(BaseFolder): connection and cursor for all operations. This is a big disadvantage and we might want to investigate if we cannot hold an object open for a thread somehow.""" - #though. According to sqlite docs, you need to commit() before - #the connection is closed or your changes will be lost!""" - #get db connection which autocommits - #connection = sqlite.connect(self.filename, isolation_level=None) - #cursor = connection.cursor() - #return connection, cursor + # Though. According to sqlite docs, you need to commit() before + # the connection is closed or your changes will be lost! + # get db connection which autocommits + # connection = sqlite.connect(self.filename, isolation_level=None) + # cursor = connection.cursor() + # return connection, cursor - #current version of our db format + # Current version of our db format. cur_version = 2 def __init__(self, name, repository): - self.sep = '.' #needs to be set before super.__init__() + self.sep = '.' # Needs to be set before super.__init__() super(LocalStatusSQLiteFolder, self).__init__(name, repository) self.root = repository.root self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self.messagelist = {} - self._newfolder = False # flag if the folder is new + self._newfolder = False # Flag if the folder is new. dirname = os.path.dirname(self.filename) if not os.path.exists(dirname): @@ -59,10 +59,10 @@ class LocalStatusSQLiteFolder(BaseFolder): raise UserWarning("SQLite database path '%s' is not a directory."% dirname) - # dblock protects against concurrent writes in same connection + # dblock protects against concurrent writes in same connection. self._dblock = Lock() - #Try to establish connection, no need for threadsafety in __init__ + # Try to establish connection, no need for threadsafety in __init__. try: self.connection = sqlite.connect(self.filename, check_same_thread=False) except NameError: @@ -71,10 +71,10 @@ class LocalStatusSQLiteFolder(BaseFolder): "with available bindings to '%s'. Is the sqlite3 package " "installed?."% self.filename), None, exc_info()[2] - #Make sure sqlite is in multithreading SERIALIZE mode + # Make sure sqlite is in multithreading SERIALIZE mode. assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' - #Test if db version is current enough and if db is readable. + # Test if db version is current enough and if db is readable. try: cursor = self.connection.execute( "SELECT value from metadata WHERE key='db_version'") From a1383da9b3828cea5ba9c6700839a5494f4d4e60 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 16 Feb 2015 10:31:21 +0100 Subject: [PATCH 722/817] LocalStatusSQLite: provide information on what is failing for OperationalError Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index f54ba99..bcce8ec 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -66,10 +66,16 @@ class LocalStatusSQLiteFolder(BaseFolder): try: self.connection = sqlite.connect(self.filename, check_same_thread=False) except NameError: - # sqlite import had failed + # sqlite import had failed. raise UserWarning("SQLite backend chosen, but cannot connect " "with available bindings to '%s'. Is the sqlite3 package " "installed?."% self.filename), None, exc_info()[2] + except sqlite.OperationalError as e: + # Operation had failed. + raise UserWarning("cannot open database file '%s': %s.\nYou might " + "want to check the rights to that file and if it cleanly opens " + "with the 'sqlite<3>' command."% + (self.filename, e)), None, exc_info()[2] # Make sure sqlite is in multithreading SERIALIZE mode. assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' From d36209a305306a46152b3e28323d31baa48edc6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=8F=E6=81=BA=28Xia=20Kai=29?= Date: Sun, 15 Feb 2015 22:16:20 +0800 Subject: [PATCH 723/817] add proxy support powered by PySocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read proxy option in imapserver, instantiate a class in imaplibutil using a self-defined keyword and a socket instance, and use this socket instance to substitute the default socket instance used in imaplib2. Signed-off-by: 夏恺(Xia Kai) Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 9 +++++ offlineimap/imaplibutil.py | 44 +++++++++++++++++++++++- offlineimap/imapserver.py | 69 ++++++++++++++++++++++++++++---------- 3 files changed, 104 insertions(+), 18 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 3ffc2e6..1b3214a 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -380,6 +380,15 @@ remoterepository = RemoteExample #filterheaders = X-Some-Weird-Header +# This option stands in the [Account Test] section. +# +# Use proxy connection for this account. Usefull to bypass the GFW in China. +# To specify a proxy connection, join proxy type, host and port with colons. +# Available proxy types are SOCKS5, SOCKS4, HTTP. +# You also need to install PySocks through pip. +# +#proxy = SOCKS5:localhost:9999 + [Repository LocalExample] # Each repository requires a "type" declaration. The types supported for diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 83ffe9a..47328bf 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -21,6 +21,7 @@ import subprocess from sys import exc_info import threading from hashlib import sha1 +import socket from offlineimap.ui import getglobalui from offlineimap import OfflineImapError @@ -69,6 +70,37 @@ class UsefulIMAPMixIn(object): def _mesg(self, s, tn=None, secs=None): new_mesg(self, s, tn, secs) + # Overrides private function from IMAP4 (@imaplib2) + def open_socket(self): + """open_socket() + Open socket choosing first address family available.""" + msg = (-1, 'could not open socket') + for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + # use socket of our own, possiblly socksified socket. + s = self.socket(af, socktype, proto) + except socket.error, msg: + continue + try: + for i in (0, 1): + try: + s.connect(sa) + break + except socket.error, msg: + if len(msg.args) < 2 or msg.args[0] != errno.EINTR: + raise + else: + raise socket.error(msg) + except socket.error, msg: + s.close() + continue + break + else: + raise socket.error(msg) + + return s + class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): """IMAP4 client class over a tunnel @@ -79,6 +111,9 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): The result will be in PREAUTH stage.""" def __init__(self, tunnelcmd, **kwargs): + if "use_socket" in kwargs: + self.socket = kwargs['use_socket'] + del kwargs['use_socket'] IMAP4.__init__(self, tunnelcmd, **kwargs) def open(self, host, port): @@ -141,6 +176,9 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): """Improved version of imaplib.IMAP4_SSL overriding select().""" def __init__(self, *args, **kwargs): + if "use_socket" in kwargs: + self.socket = kwargs['use_socket'] + del kwargs['use_socket'] self._fingerprint = kwargs.get('fingerprint', None) if type(self._fingerprint) != type([]): self._fingerprint = [self._fingerprint] @@ -171,7 +209,11 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): """Improved version of imaplib.IMAP4 overriding select().""" - pass + def __init__(self, *args, **kwargs): + if "use_socket" in kwargs: + self.socket = kwargs['use_socket'] + del kwargs['use_socket'] + IMAP4.__init__(self, *args, **kwargs) def Internaldate2epoch(resp): diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 3d69426..351ee9f 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -15,10 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError -from offlineimap.ui import getglobalui from threading import Lock, BoundedSemaphore, Thread, Event, currentThread -import offlineimap.accounts import hmac import socket import base64 @@ -28,6 +25,11 @@ from sys import exc_info from socket import gaierror from ssl import SSLError, cert_time_to_seconds +from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError +import offlineimap.accounts +from offlineimap.ui import getglobalui + + try: # do we have a recent pykerberos? have_gss = False @@ -101,6 +103,31 @@ class IMAPServer: self.gss_vc = None self.gssapi = False + # In order to support proxy connection, we have to override the + # default socket instance with our own socksified socket instance. + # We add this option to bypass the GFW in China. + _account_section = 'Account ' + self.repos.account.name + if not self.config.has_option(_account_section, 'proxy'): + self.proxied_socket = socket.socket + else: + proxy = self.config.get(_account_section, 'proxy') + # Powered by PySocks. + try: + import socks + proxy_type, host, port = proxy.split(":") + port = int(port) + socks.setdefaultproxy(getattr(socks, proxy_type), host, port) + self.proxied_socket = socks.socksocket + except ImportError: + self.ui.warn("PySocks not installed, ignoring proxy option.") + self.proxied_socket = socket.socket + except (AttributeError, ValueError) as e: + self.ui.warn("Bad proxy option %s for account %s: %s " + "Ignoring proxy option."% + (proxy, self.repos.account.name, e)) + self.proxied_socket = socket.socket + + def __getpassword(self): """Returns the server password or None""" if self.goodpassword != None: # use cached good one first @@ -391,25 +418,33 @@ class IMAPServer: # Generate a new connection. if self.tunnel: self.ui.connecting('tunnel', self.tunnel) - imapobj = imaplibutil.IMAP4_Tunnel(self.tunnel, - timeout=socket.getdefaulttimeout()) + imapobj = imaplibutil.IMAP4_Tunnel( + self.tunnel, + timeout=socket.getdefaulttimeout(), + use_socket=self.proxied_socket, + ) success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) - imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, - self.port, - self.sslclientkey, - self.sslclientcert, - self.sslcacertfile, - self.__verifycert, - self.sslversion, - timeout=socket.getdefaulttimeout(), - fingerprint=self.fingerprint - ) + imapobj = imaplibutil.WrappedIMAP4_SSL( + self.hostname, + self.port, + self.sslclientkey, + self.sslclientcert, + self.sslcacertfile, + self.__verifycert, + self.sslversion, + timeout=socket.getdefaulttimeout(), + fingerprint=self.fingerprint, + use_socket=self.proxied_socket, + ) else: self.ui.connecting(self.hostname, self.port) - imapobj = imaplibutil.WrappedIMAP4(self.hostname, self.port, - timeout=socket.getdefaulttimeout()) + imapobj = imaplibutil.WrappedIMAP4( + self.hostname, self.port, + timeout=socket.getdefaulttimeout(), + use_socket=self.proxied_socket, + ) if not self.preauth_tunnel: try: From ca06819e700dd1945f99af434b7755175e6d549d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 17 Feb 2015 10:33:56 +0100 Subject: [PATCH 724/817] imaplibutil: add missing errno import Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 47328bf..30aed9a 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -22,6 +22,7 @@ from sys import exc_info import threading from hashlib import sha1 import socket +import errno from offlineimap.ui import getglobalui from offlineimap import OfflineImapError From efc4df1bd73bb58fd170105f364fb0e80a0a2c97 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 22 Feb 2015 14:08:39 +0100 Subject: [PATCH 725/817] LocalStatusSQLite: labels: don't fail if database returns unexpected None value This requires more researches. See https://github.com/OfflineIMAP/offlineimap/issues/103 . Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 16 +++++++++++++++- offlineimap/folder/UIDMaps.py | 9 ++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index bcce8ec..64adcd1 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -203,7 +203,21 @@ class LocalStatusSQLiteFolder(BaseFolder): uid = row[0] self.messagelist[uid] = self.msglist_item_initializer(uid) flags = set(row[1]) - labels = set([lb.strip() for lb in row[3].split(',') if len(lb.strip()) > 0]) + try: + labels = set([lb.strip() for lb in + row[3].split(',') if len(lb.strip()) > 0]) + except AttributeError: + # FIXME: This except clause was introduced because row[3] from + # database can be found of unexpected type NoneType. See + # https://github.com/OfflineIMAP/offlineimap/issues/103 + # + # We are fixing the type here but this would require more + # researches to find the true root cause. row[3] is expected to + # be a (empty) string, not None. + # + # Also, since database might return None, we have to fix the + # database, too. + labels = set() self.messagelist[uid]['flags'] = flags self.messagelist[uid]['labels'] = labels self.messagelist[uid]['mtime'] = row[2] diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index e8ca9a7..04a986b 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -208,7 +208,7 @@ class MappedIMAPFolder(IMAPFolder): if uid < 0: return uid - #if msg uid already exists, just modify the flags + # If msg uid already exists, just modify the flags. if uid in self.r2l: self.savemessageflags(uid, flags) return uid @@ -238,11 +238,10 @@ class MappedIMAPFolder(IMAPFolder): # Interface from BaseFolder def savemessageflags(self, uid, flags): - """ - - Note that this function does not check against dryrun settings, + """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + self._mb.savemessageflags(self.r2l[uid], flags) # Interface from BaseFolder @@ -304,7 +303,7 @@ class MappedIMAPFolder(IMAPFolder): # Interface from BaseFolder def deletemessagesflags(self, uidlist, flags): self._mb.deletemessagesflags(self._uidlist(self.r2l, uidlist), - flags) + flags) # Interface from BaseFolder def deletemessage(self, uid): From 5ecd557dfbd8d734abe35b24dd30a813ab2bbadf Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 2 Mar 2015 10:49:51 +0100 Subject: [PATCH 726/817] doc: add IMAP RFCs Signed-off-by: Nicolas Sebrecht --- docs/rfcs/README.md | 6 + docs/rfcs/rfc1731.txt | 339 +++ docs/rfcs/rfc1732.txt | 283 ++ docs/rfcs/rfc1733.txt | 171 ++ docs/rfcs/rfc1939.txt | 1291 +++++++++ docs/rfcs/rfc2061.txt | 171 ++ docs/rfcs/rfc2062.txt | 451 +++ docs/rfcs/rfc2086.txt | 451 +++ docs/rfcs/rfc2087.txt | 283 ++ docs/rfcs/rfc2088.txt | 115 + docs/rfcs/rfc2095.txt | 283 ++ docs/rfcs/rfc2177.txt | 227 ++ docs/rfcs/rfc2180.txt | 787 ++++++ docs/rfcs/rfc2192.txt | 899 ++++++ docs/rfcs/rfc2193.txt | 507 ++++ docs/rfcs/rfc2195.txt | 283 ++ docs/rfcs/rfc2221.txt | 283 ++ docs/rfcs/rfc2244.txt | 4035 +++++++++++++++++++++++++++ docs/rfcs/rfc2342.txt | 563 ++++ docs/rfcs/rfc2359.txt | 339 +++ docs/rfcs/rfc2595.txt | 843 ++++++ docs/rfcs/rfc2683.txt | 1291 +++++++++ docs/rfcs/rfc2821.txt | 4427 ++++++++++++++++++++++++++++++ docs/rfcs/rfc2971.txt | 451 +++ docs/rfcs/rfc3028.txt | 2019 ++++++++++++++ docs/rfcs/rfc3348.txt | 339 +++ docs/rfcs/rfc3501.txt | 6051 +++++++++++++++++++++++++++++++++++++++++ docs/rfcs/rfc3502.txt | 395 +++ docs/rfcs/rfc3503.txt | 507 ++++ docs/rfcs/rfc3516.txt | 451 +++ docs/rfcs/rfc3656.txt | 1067 ++++++++ docs/rfcs/rfc3691.txt | 283 ++ docs/rfcs/rfc4314.txt | 1515 +++++++++++ docs/rfcs/rfc4315.txt | 451 +++ docs/rfcs/rfc4466.txt | 955 +++++++ docs/rfcs/rfc4467.txt | 1011 +++++++ docs/rfcs/rfc4469.txt | 731 +++++ docs/rfcs/rfc4549.txt | 1963 +++++++++++++ docs/rfcs/rfc4551.txt | 1403 ++++++++++ 39 files changed, 37920 insertions(+) create mode 100644 docs/rfcs/README.md create mode 100644 docs/rfcs/rfc1731.txt create mode 100644 docs/rfcs/rfc1732.txt create mode 100644 docs/rfcs/rfc1733.txt create mode 100644 docs/rfcs/rfc1939.txt create mode 100644 docs/rfcs/rfc2061.txt create mode 100644 docs/rfcs/rfc2062.txt create mode 100644 docs/rfcs/rfc2086.txt create mode 100644 docs/rfcs/rfc2087.txt create mode 100644 docs/rfcs/rfc2088.txt create mode 100644 docs/rfcs/rfc2095.txt create mode 100644 docs/rfcs/rfc2177.txt create mode 100644 docs/rfcs/rfc2180.txt create mode 100644 docs/rfcs/rfc2192.txt create mode 100644 docs/rfcs/rfc2193.txt create mode 100644 docs/rfcs/rfc2195.txt create mode 100644 docs/rfcs/rfc2221.txt create mode 100644 docs/rfcs/rfc2244.txt create mode 100644 docs/rfcs/rfc2342.txt create mode 100644 docs/rfcs/rfc2359.txt create mode 100644 docs/rfcs/rfc2595.txt create mode 100644 docs/rfcs/rfc2683.txt create mode 100644 docs/rfcs/rfc2821.txt create mode 100644 docs/rfcs/rfc2971.txt create mode 100644 docs/rfcs/rfc3028.txt create mode 100644 docs/rfcs/rfc3348.txt create mode 100644 docs/rfcs/rfc3501.txt create mode 100644 docs/rfcs/rfc3502.txt create mode 100644 docs/rfcs/rfc3503.txt create mode 100644 docs/rfcs/rfc3516.txt create mode 100644 docs/rfcs/rfc3656.txt create mode 100644 docs/rfcs/rfc3691.txt create mode 100644 docs/rfcs/rfc4314.txt create mode 100644 docs/rfcs/rfc4315.txt create mode 100644 docs/rfcs/rfc4466.txt create mode 100644 docs/rfcs/rfc4467.txt create mode 100644 docs/rfcs/rfc4469.txt create mode 100644 docs/rfcs/rfc4549.txt create mode 100644 docs/rfcs/rfc4551.txt diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md new file mode 100644 index 0000000..70264ee --- /dev/null +++ b/docs/rfcs/README.md @@ -0,0 +1,6 @@ + +All RFCs related to IMAP. + +TODO: +- rename the files to know what they are talking about. +- Add a brief introduction here to introduce the most important RFCs. diff --git a/docs/rfcs/rfc1731.txt b/docs/rfcs/rfc1731.txt new file mode 100644 index 0000000..9cced5d --- /dev/null +++ b/docs/rfcs/rfc1731.txt @@ -0,0 +1,339 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 1731 Carnegie Mellon +Category: Standards Track December 1994 + + + IMAP4 Authentication Mechanisms + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + + +1. Introduction + + The Internet Message Access Protocol, Version 4 [IMAP4] contains the + AUTHENTICATE command, for identifying and authenticating a user to an + IMAP4 server and for optionally negotiating a protection mechanism + for subsequent protocol interactions. This document describes + several authentication mechanisms for use by the IMAP4 AUTHENTICATE + command. + + +2. Kerberos version 4 authentication mechanism + + The authentication type associated with Kerberos version 4 is + "KERBEROS_V4". + + The data encoded in the first ready response contains a random 32-bit + number in network byte order. The client should respond with a + Kerberos ticket and an authenticator for the principal + "imap.hostname@realm", where "hostname" is the first component of the + host name of the server with all letters in lower case and where + "realm" is the Kerberos realm of the server. The encrypted checksum + field included within the Kerberos authenticator should contain the + server provided 32-bit number in network byte order. + + Upon decrypting and verifying the ticket and authenticator, the + server should verify that the contained checksum field equals the + original server provided random 32-bit number. Should the + verification be successful, the server must add one to the checksum + and construct 8 octets of data, with the first four octets containing + the incremented checksum in network byte order, the fifth octet + containing a bit-mask specifying the protection mechanisms supported + by the server, and the sixth through eighth octets containing, in + + + +Myers [Page 1] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + + network byte order, the maximum cipher-text buffer size the server is + able to receive. The server must encrypt the 8 octets of data in the + session key and issue that encrypted data in a second ready response. + The client should consider the server authenticated if the first four + octets the un-encrypted data is equal to one plus the checksum it + previously sent. + + The client must construct data with the first four octets containing + the original server-issued checksum in network byte order, the fifth + octet containing the bit-mask specifying the selected protection + mechanism, the sixth through eighth octets containing in network byte + order the maximum cipher-text buffer size the client is able to + receive, and the following octets containing a user name string. The + client must then append from one to eight octets so that the length + of the data is a multiple of eight octets. The client must then PCBC + encrypt the data with the session key and respond to the second ready + response with the encrypted data. The server decrypts the data and + verifies the contained checksum. The username field identifies the + user for whom subsequent IMAP operations are to be performed; the + server must verify that the principal identified in the Kerberos + ticket is authorized to connect as that user. After these + verifications, the authentication process is complete. + + The protection mechanisms and their corresponding bit-masks are as + follows: + + 1 No protection mechanism + 2 Integrity (krb_mk_safe) protection + 4 Privacy (krb_mk_priv) protection + + + EXAMPLE: The following are two Kerberos version 4 login scenarios + (note that the line breaks in the sample authenticators are for + editorial clarity and are not in real authenticators) + + S: * OK IMAP4 Server + C: A001 AUTHENTICATE KERBEROS_V4 + S: + AmFYig== + C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT + +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd + WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh + S: + or//EoAADZI= + C: DiAF5A4gA+oOIALuBkAAmw== + S: A001 OK Kerberos V4 authentication successful + + + + + + + +Myers [Page 2] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + + S: * OK IMAP4 Server + C: A001 AUTHENTICATE KERBEROS_V4 + S: + gcfgCA== + C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT + +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd + WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh + S: A001 NO Kerberos V4 authentication failed + + +3. GSSAPI authentication mechanism + + The authentication type associated with all mechanisms employing the + GSSAPI [RFC1508] is "GSSAPI". + + The first ready response issued by the server contains no data. The + client should call GSS_Init_sec_context, passing in 0 for + input_context_handle (initially) and a targ_name equal to output_name + from GSS_Import_Name called with input_name_type of NULL and + input_name_string of "SERVICE:imap@hostname" where "hostname" is the + fully qualified host name of the server with all letters in lower + case. The client must then respond with the resulting output_token. + If GSS_Init_sec_context returns GSS_CONTINUE_NEEDED, then the client + should expect the server to issue a token in a subsequent ready + response. The client must pass the token to another call to + GSS_Init_sec_context. + + If GSS_Init_sec_context returns GSS_COMPLETE, then the client should + respond with any resulting output_token. If there is no + output_token, the client should respond with no data. The client + should then expect the server to issue a token in a subsequent ready + response. The client should pass this token to GSS_Unseal and + interpret the first octet of resulting cleartext as a bit-mask + specifying the protection mechanisms supported by the server and the + second through fourth octets as the maximum size output_message to + send to the server. The client should construct data, with the first + octet containing the bit-mask specifying the selected protection + mechanism, the second through fourth octets containing in network + byte order the maximum size output_message the client is able to + receive, and the remaining octets containing a user name string. The + client must pass the data to GSS_Seal with conf_flag set to FALSE, + and respond with the generated output_message. The client can then + consider the server authenticated. + + The server must issue a ready response with no data and pass the + resulting client supplied token to GSS_Accept_sec_context as + input_token, setting acceptor_cred_handle to NULL (for "use default + credentials"), and 0 for input_context_handle (initially). If + GSS_Accept_sec_context returns GSS_CONTINUE_NEEDED, the server should + + + +Myers [Page 3] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + + return the generated output_token to the client in a ready response + and pass the resulting client supplied token to another call to + GSS_Accept_sec_context. + + If GSS_Accept_sec_context returns GSS_COMPLETE, then if an + output_token is returned, the server should return it to the client + in a ready response and expect a reply from the client with no data. + Whether or not an output_token was returned, the server then should + then construct 4 octets of data, with the first octet containing a + bit-mask specifying the protection mechanisms supported by the server + and the second through fourth octets containing in network byte order + the maximum size output_token the server is able to receive. The + server must then pass the plaintext to GSS_Seal with conf_flag set to + FALSE and issue the generated output_message to the client in a ready + response. The server must then pass the resulting client supplied + token to GSS_Unseal and interpret the first octet of resulting + cleartext as the bit-mask for the selected protection mechanism, the + second through fourth octets as the maximum size output_message to + send to the client, and the remaining octets as the user name. Upon + verifying the src_name is authorized to authenticate as the user + name, The server should then consider the client authenticated. + + The protection mechanisms and their corresponding bit-masks are as + follows: + + 1 No protection mechanism + 2 Integrity protection. + Sender calls GSS_Seal with conf_flag set to FALSE + 4 Privacy protection. + Sender calls GSS_Seal with conf_flag set to TRUE + + +4. S/Key authentication mechanism + + The authentication type associated with S/Key [SKEY] is "SKEY". + + The first ready response issued by the server contains no data. The + client responds with the user name string. + + The data encoded in the second ready response contains the decimal + sequence number followed by a single space and the seed string for + the indicated user. The client responds with the one-time-password, + as either a 64-bit value in network byte order or encoded in the "six + English words" format. + + Upon successful verification of the one-time-password, the server + should consider the client authenticated. + + + + +Myers [Page 4] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + + S/Key authentication does not provide for any protection mechanisms. + + + EXAMPLE: The following are two S/Key login scenarios. + + S: * OK IMAP4 Server + C: A001 AUTHENTICATE SKEY + S: + + C: bW9yZ2Fu + S: + OTUgUWE1ODMwOA== + C: Rk9VUiBNQU5OIFNPT04gRklSIFZBUlkgTUFTSA== + S: A001 OK S/Key authentication successful + + + S: * OK IMAP4 Server + C: A001 AUTHENTICATE SKEY + S: + + C: c21pdGg= + S: + OTUgUWE1ODMwOA== + C: BsAY3g4gBNo= + S: A001 NO S/Key authentication failed + + +5. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", + RFC 1730, University of Washington, December 1994. + + [RFC1508] Linn, J., "Generic Security Service Application Program + Interface", RFC 1508, Geer Zolot Associates, September 1993. + + [SKEY] Haller, Neil M. "The S/Key One-Time Password System", + Bellcore, Morristown, New Jersey, October 1993, + thumper.bellcore.com:pub/nmh/docs/ISOC.symp.ps + + + + + + + + + + + + + + + + + +Myers [Page 5] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + +6. Security Considerations + + Security issues are discussed throughout this memo. + + +7. Author's Address + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave. + Pittsburgh PA, 15213-3890 + + EMail: jgm+@cmu.edu + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Myers [Page 6] + diff --git a/docs/rfcs/rfc1732.txt b/docs/rfcs/rfc1732.txt new file mode 100644 index 0000000..cfae89c --- /dev/null +++ b/docs/rfcs/rfc1732.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 1732 University of Washington +Category: Informational December 1994 + + + IMAP4 COMPATIBILITY WITH IMAP2 AND IMAP2BIS + + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + +Introduction + + This is a summary of hints and recommendations to enable an IMAP4 + implementation to interoperate with implementations that conform to + earlier specifications. None of these hints and recommendations are + required by the IMAP4 specification; implementors must decide for + themselves whether they want their implementation to fail if it + encounters old software. + + IMAP4 has been designed to be upwards compatible with earlier + specifications. For the most part, IMAP4 facilities that were not in + earlier specifications should be invisible to clients unless the + client asks for the facility. + + In some cases, older servers may support some of the capabilities + listed as being "new in IMAP4" as experimental extensions to the + IMAP2 protocol described in RFC 1176. + + This information may not be complete; it reflects current knowledge + of server and client implementations as well as "folklore" acquired + in the evolution of the protocol. + + + + + + + + + + + + + + + + +Crispin [Page 1] + +RFC 1732 IMAP4 - Compatibility December 1994 + + +IMAP4 client interoperability with old servers + + In general, a client should be able to discover whether an IMAP2 + server supports a facility by trial-and-error; if an attempt to use a + facility generates a BAD response, the client can assume that the + server does not support the facility. + + A quick way to check whether a server implementation supports the + IMAP4 specification is to try the CAPABILITY command. An OK response + that includes the IMAP4 capability value indicates a server that + supports IMAP4; a BAD response or one without the IMAP4 capability + value indicates an older server. + + The following is a list of facilities that are only in IMAP4, and + suggestions for how new clients might interoperate with old servers: + + CAPABILITY command + A BAD response to this command indicates that the server + implements IMAP2 (or IMAP2bis) and not IMAP4. + + AUTHENTICATE command. + Use the LOGIN command. + + LSUB and LIST commands + Try the RFC 1176 FIND command. + + * in a sequence + Use the number of messages in the mailbox from the EXISTS + unsolicited response. + + SEARCH extensions (character set, additional criteria) + Reformulate the search request using only the searching + options listed in search_old in the IMAP4 grammar. This may + entail doing multiple searches to achieve the desired + results. + + BODYSTRUCTURE fetch data item + Try to fetch the non-extensible BODY data item. + + body section number 0 + Fetch the entire message and extract the header. + + RFC822.HEADER.LINES and RFC822.HEADER.LINES.NOT fetch data items + Use RFC822.HEADER and remove the unwanted information. + + BODY.PEEK[section], RFC822.PEEK, and RFC822.TEXT.PEEK fetch data + items Use the corresponding non-PEEK versions and manually + clear the \Seen flag as necessary. + + + +Crispin [Page 2] + +RFC 1732 IMAP4 - Compatibility December 1994 + + + UID fetch data item and the UID commands + No equivalent capabilitity exists in older servers. + + FLAGS.SILENT, +FLAGS.SILENT, and -FLAGS.SILENT store data items + Use the corresponding non-SILENT versions and ignore the + untagged FETCH responses which com eback. + + + The following IMAP4 facilities were introduced in the experimental + IMAP2bis revisions to RFC-1176, and may be present in a server that + does not support the CAPABILITY command: + + CREATE, DELETE, and RENAME commands + To test whether these commands are present, try a CREATE + INBOX command. If the response is NO, these commands are + supported by the server. If the response is BAD, they are + not. Older servers without the CREATE capability may sup- + port implicit creation of a mailbox by a COPY command with a + non-existant name as the destination. + + APPEND command + To test whether this command is present, try to append a + zero-length stream to a mailbox name that is known not to + exist (or at least, highly unlikely to exist) on the remote + system. + + SUBSCRIBE and UNSUBSCRIBE commands + Try the form of these commands with the optional MAILBOX + keyword. + + EXAMINE command + Use the SELECT command instead. + + flags and internal date argument to APPEND command + Try the APPEND without any flag list and internal date argu- + ments. + + BODY, BODY[section], and FULL fetch data items + Use RFC822.TEXT and ALL instead. Server does not support + MIME. + + PARTIAL command + Use the appropriate FETCH command and ignore the unwanted + data. + + + IMAP4 client implementations must accept all responses and data for- + mats documented in the IMAP4 specification, including those labeled + + + +Crispin [Page 3] + +RFC 1732 IMAP4 - Compatibility December 1994 + + + as obsolete. This includes the COPY and STORE unsolicited responses + and the old format of dates and times. In particular, client imple- + mentations must not treat a date/time as a fixed format string; nor + may they assume that the time begins at a particular octet. + + IMAP4 client implementations must not depend upon the presence of any + server extensions that are not in the base IMAP4 specification. + + The experimental IMAP2bis version specified that the TRYCREATE spe- + cial information token is sent as a separate unsolicited OK response + instead of inside the NO response. + + The FIND BBOARDS, FIND ALL.BBOARDS, and BBOARD commands of RFC 1176 + are removed from IMAP4. There is no equivalent to the bboard com- + mands, which provided a separate namespace with implicit restrictions + on what may be done in that namespace. + + Older server implementations may automatically create the destination + mailbox on COPY if that mailbox does not already exist. This was how + a new mailbox was created in older specifications. If the server + does not support the CREATE command (see above for how to test for + this), it will probably create a mailbox on COPY. + + Older server implementations may not preserve flags or internal dates + on COPY. Some server implementations may not permit the preservation + of certain flags on COPY or their setting with APPEND as site policy. + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin [Page 4] + +RFC 1732 IMAP4 - Compatibility December 1994 + + +IMAP4 server interoperability with old clients + + In general, there should be no interoperation problem between a + server conforming to the IMAP4 specification and a well-written + client that conforms to an earlier specification. Known problems are + noted below: + + Poor wording in the description of the CHECK command in earlier + specifications implied that a CHECK command is the way to get the + current number of messages in the mailbox. This is incorrect. A + CHECK command does not necessarily result in an EXISTS response. + Clients must remember the most recent EXISTS value sent from the + server, and should not generate unnecessary CHECK commands. + + An incompatibility exists with COPY in IMAP4. COPY in IMAP4 + servers does not automatically create the destination mailbox if + that mailbox does not already exist. This may cause problems with + old clients that expect automatic mailbox creation in COPY. + + The PREAUTH unsolicited response is new in IMAP4. It is highly + unlikely that an old client would ever see this response. + + The format of dates and times has changed due to the impending end + of the century. Clients that fail to accept a four-digit year or + a signed four-digit timezone value will not work properly with + IMAP4. + + An incompatibility exists with the use of "\" in quoted strings. + This is best avoided by using literals instead of quoted strings + if "\" or <"> is embedded in the string. + +Security Considerations + + Security issues are not discussed in this memo. + +Author's Address: + + Mark R. Crispin + Networks and Distributed Computing, JE-30 + University of Washington + Seattle, WA 98195 + + Phone: (206) 543-5762 + + EMail: MRC@CAC.Washington.EDU + + + + + + +Crispin [Page 5] + diff --git a/docs/rfcs/rfc1733.txt b/docs/rfcs/rfc1733.txt new file mode 100644 index 0000000..2ec385f --- /dev/null +++ b/docs/rfcs/rfc1733.txt @@ -0,0 +1,171 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 1733 University of Washington +Category: Informational December 1994 + + + DISTRIBUTED ELECTRONIC MAIL MODELS IN IMAP4 + + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + + +Distributed Electronic Mail Models + + There are three fundamental models of client/server email: offline, + online, and disconnected use. IMAP4 can be used in any one of these + three models. + + The offline model is the most familiar form of client/server email + today, and is used by protocols such as POP-3 (RFC 1225) and UUCP. + In this model, a client application periodically connects to a + server. It downloads all the pending messages to the client machine + and deletes these from the server. Thereafter, all mail processing + is local to the client. This model is store-and-forward; it moves + mail on demand from an intermediate server (maildrop) to a single + destination machine. + + The online model is most commonly used with remote filesystem + protocols such as NFS. In this model, a client application + manipulates mailbox data on a server machine. A connection to the + server is maintained throughout the session. No mailbox data are + kept on the client; the client retrieves data from the server as is + needed. IMAP4 introduces a form of the online model that requires + considerably less network bandwidth than a remote filesystem + protocol, and provides the opportunity for using the server for CPU + or I/O intensive functions such as parsing and searching. + + The disconnected use model is a hybrid of the offline and online + models, and is used by protocols such as PCMAIL (RFC 1056). In this + model, a client user downloads some set of messages from the server, + manipulates them offline, then at some later time uploads the + changes. The server remains the authoritative repository of the + messages. The problems of synchronization (particularly when + multiple clients are involved) are handled through the means of + unique identifiers for each message. + + + +Crispin [Page 1] + +RFC 1733 IMAP4 - Model December 1994 + + + Each of these models have their own strengths and weaknesses: + + Feature Offline Online Disc + ------- ------- ------ ---- + Can use multiple clients NO YES YES + Minimum use of server connect time YES NO YES + Minimum use of server resources YES NO NO + Minimum use of client disk resources NO YES NO + Multiple remote mailboxes NO YES YES + Fast startup NO YES NO + Mail processing when not online YES NO YES + + Although IMAP4 has its origins as a protocol designed to accommodate + the online model, it can support the other two models as well. This + makes possible the creation of clients that can be used in any of the + three models. For example, a user may wish to switch between the + online and disconnected models on a regular basis (e.g. owing to + travel). + + IMAP4 is designed to transmit message data on demand, and to provide + the facilities necessary for a client to decide what data it needs at + any particular time. There is generally no need to do a wholesale + transfer of an entire mailbox or even of the complete text of a + message. This makes a difference in situations where the mailbox is + large, or when the link to the server is slow. + + More specifically, IMAP4 supports server-based RFC 822 and MIME + processing. With this information, it is possible for a client to + determine in advance whether it wishes to retrieve a particular + message or part of a message. For example, a user connected to an + IMAP4 server via a dialup link can determine that a message has a + 2000 byte text segment and a 40 megabyte video segment, and elect to + fetch only the text segment. + + In IMAP4, the client/server relationship lasts only for the duration + of the TCP connection. There is no registration of clients. Except + for any unique identifiers used in disconnected use operation, the + client initially has no knowledge of mailbox state and learns it from + the IMAP4 server when a mailbox is selected. This initial transfer + is minimal; the client requests additional state data as it needs. + + As noted above, the choice for the location of mailbox data depends + upon the model chosen. The location of message state (e.g. whether + or not a message has been read or answered) is also determined by the + model, and is not necessarily the same as the location of the mailbox + data. For example, in the online model message state can be co- + located with mailbox data; it can also be located elsewhere (on the + client or on a third agent) using unique identifiers to achieve + + + +Crispin [Page 2] + +RFC 1733 IMAP4 - Model December 1994 + + + common reference across sessions. The latter is particularly useful + with a server that exports public data such as netnews and does not + maintain per-user state. + + The IMAP4 protocol provides the generality to implement these + different models. This is done by means of server and (especially) + client configuration, and not by requiring changes to the protocol or + the implementation of the protocol. + + +Security Considerations + + Security issues are not discussed in this memo. + + +Author's Address: + + Mark R. Crispin + Networks and Distributed Computing, JE-30 + University of Washington + Seattle, WA 98195 + + Phone: (206) 543-5762 + + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin [Page 3] + diff --git a/docs/rfcs/rfc1939.txt b/docs/rfcs/rfc1939.txt new file mode 100644 index 0000000..b11350e --- /dev/null +++ b/docs/rfcs/rfc1939.txt @@ -0,0 +1,1291 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 1939 Carnegie Mellon +STD: 53 M. Rose +Obsoletes: 1725 Dover Beach Consulting, Inc. +Category: Standards Track May 1996 + + + Post Office Protocol - Version 3 + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Table of Contents + + 1. Introduction ................................................ 2 + 2. A Short Digression .......................................... 2 + 3. Basic Operation ............................................. 3 + 4. The AUTHORIZATION State ..................................... 4 + QUIT Command ................................................ 5 + 5. The TRANSACTION State ....................................... 5 + STAT Command ................................................ 6 + LIST Command ................................................ 6 + RETR Command ................................................ 8 + DELE Command ................................................ 8 + NOOP Command ................................................ 9 + RSET Command ................................................ 9 + 6. The UPDATE State ............................................ 10 + QUIT Command ................................................ 10 + 7. Optional POP3 Commands ...................................... 11 + TOP Command ................................................. 11 + UIDL Command ................................................ 12 + USER Command ................................................ 13 + PASS Command ................................................ 14 + APOP Command ................................................ 15 + 8. Scaling and Operational Considerations ...................... 16 + 9. POP3 Command Summary ........................................ 18 + 10. Example POP3 Session ....................................... 19 + 11. Message Format ............................................. 19 + 12. References ................................................. 20 + 13. Security Considerations .................................... 20 + 14. Acknowledgements ........................................... 20 + 15. Authors' Addresses ......................................... 21 + Appendix A. Differences from RFC 1725 .......................... 22 + + + +Myers & Rose Standards Track [Page 1] + +RFC 1939 POP3 May 1996 + + + Appendix B. Command Index ...................................... 23 + +1. Introduction + + On certain types of smaller nodes in the Internet it is often + impractical to maintain a message transport system (MTS). For + example, a workstation may not have sufficient resources (cycles, + disk space) in order to permit a SMTP server [RFC821] and associated + local mail delivery system to be kept resident and continuously + running. Similarly, it may be expensive (or impossible) to keep a + personal computer interconnected to an IP-style network for long + amounts of time (the node is lacking the resource known as + "connectivity"). + + Despite this, it is often very useful to be able to manage mail on + these smaller nodes, and they often support a user agent (UA) to aid + the tasks of mail handling. To solve this problem, a node which can + support an MTS entity offers a maildrop service to these less endowed + nodes. The Post Office Protocol - Version 3 (POP3) is intended to + permit a workstation to dynamically access a maildrop on a server + host in a useful fashion. Usually, this means that the POP3 protocol + is used to allow a workstation to retrieve mail that the server is + holding for it. + + POP3 is not intended to provide extensive manipulation operations of + mail on the server; normally, mail is downloaded and then deleted. A + more advanced (and complex) protocol, IMAP4, is discussed in + [RFC1730]. + + For the remainder of this memo, the term "client host" refers to a + host making use of the POP3 service, while the term "server host" + refers to a host which offers the POP3 service. + +2. A Short Digression + + This memo does not specify how a client host enters mail into the + transport system, although a method consistent with the philosophy of + this memo is presented here: + + When the user agent on a client host wishes to enter a message + into the transport system, it establishes an SMTP connection to + its relay host and sends all mail to it. This relay host could + be, but need not be, the POP3 server host for the client host. Of + course, the relay host must accept mail for delivery to arbitrary + recipient addresses, that functionality is not required of all + SMTP servers. + + + + + +Myers & Rose Standards Track [Page 2] + +RFC 1939 POP3 May 1996 + + +3. Basic Operation + + Initially, the server host starts the POP3 service by listening on + TCP port 110. When a client host wishes to make use of the service, + it establishes a TCP connection with the server host. When the + connection is established, the POP3 server sends a greeting. The + client and POP3 server then exchange commands and responses + (respectively) until the connection is closed or aborted. + + Commands in the POP3 consist of a case-insensitive keyword, possibly + followed by one or more arguments. All commands are terminated by a + CRLF pair. Keywords and arguments consist of printable ASCII + characters. Keywords and arguments are each separated by a single + SPACE character. Keywords are three or four characters long. Each + argument may be up to 40 characters long. + + Responses in the POP3 consist of a status indicator and a keyword + possibly followed by additional information. All responses are + terminated by a CRLF pair. Responses may be up to 512 characters + long, including the terminating CRLF. There are currently two status + indicators: positive ("+OK") and negative ("-ERR"). Servers MUST + send the "+OK" and "-ERR" in upper case. + + Responses to certain commands are multi-line. In these cases, which + are clearly indicated below, after sending the first line of the + response and a CRLF, any additional lines are sent, each terminated + by a CRLF pair. When all lines of the response have been sent, a + final line is sent, consisting of a termination octet (decimal code + 046, ".") and a CRLF pair. If any line of the multi-line response + begins with the termination octet, the line is "byte-stuffed" by + pre-pending the termination octet to that line of the response. + Hence a multi-line response is terminated with the five octets + "CRLF.CRLF". When examining a multi-line response, the client checks + to see if the line begins with the termination octet. If so and if + octets other than CRLF follow, the first octet of the line (the + termination octet) is stripped away. If so and if CRLF immediately + follows the termination character, then the response from the POP + server is ended and the line containing ".CRLF" is not considered + part of the multi-line response. + + A POP3 session progresses through a number of states during its + lifetime. Once the TCP connection has been opened and the POP3 + server has sent the greeting, the session enters the AUTHORIZATION + state. In this state, the client must identify itself to the POP3 + server. Once the client has successfully done this, the server + acquires resources associated with the client's maildrop, and the + session enters the TRANSACTION state. In this state, the client + requests actions on the part of the POP3 server. When the client has + + + +Myers & Rose Standards Track [Page 3] + +RFC 1939 POP3 May 1996 + + + issued the QUIT command, the session enters the UPDATE state. In + this state, the POP3 server releases any resources acquired during + the TRANSACTION state and says goodbye. The TCP connection is then + closed. + + A server MUST respond to an unrecognized, unimplemented, or + syntactically invalid command by responding with a negative status + indicator. A server MUST respond to a command issued when the + session is in an incorrect state by responding with a negative status + indicator. There is no general method for a client to distinguish + between a server which does not implement an optional command and a + server which is unwilling or unable to process the command. + + A POP3 server MAY have an inactivity autologout timer. Such a timer + MUST be of at least 10 minutes' duration. The receipt of any command + from the client during that interval should suffice to reset the + autologout timer. When the timer expires, the session does NOT enter + the UPDATE state--the server should close the TCP connection without + removing any messages or sending any response to the client. + +4. The AUTHORIZATION State + + Once the TCP connection has been opened by a POP3 client, the POP3 + server issues a one line greeting. This can be any positive + response. An example might be: + + S: +OK POP3 server ready + + The POP3 session is now in the AUTHORIZATION state. The client must + now identify and authenticate itself to the POP3 server. Two + possible mechanisms for doing this are described in this document, + the USER and PASS command combination and the APOP command. Both + mechanisms are described later in this document. Additional + authentication mechanisms are described in [RFC1734]. While there is + no single authentication mechanism that is required of all POP3 + servers, a POP3 server must of course support at least one + authentication mechanism. + + Once the POP3 server has determined through the use of any + authentication command that the client should be given access to the + appropriate maildrop, the POP3 server then acquires an exclusive- + access lock on the maildrop, as necessary to prevent messages from + being modified or removed before the session enters the UPDATE state. + If the lock is successfully acquired, the POP3 server responds with a + positive status indicator. The POP3 session now enters the + TRANSACTION state, with no messages marked as deleted. If the + maildrop cannot be opened for some reason (for example, a lock can + not be acquired, the client is denied access to the appropriate + + + +Myers & Rose Standards Track [Page 4] + +RFC 1939 POP3 May 1996 + + + maildrop, or the maildrop cannot be parsed), the POP3 server responds + with a negative status indicator. (If a lock was acquired but the + POP3 server intends to respond with a negative status indicator, the + POP3 server must release the lock prior to rejecting the command.) + After returning a negative status indicator, the server may close the + connection. If the server does not close the connection, the client + may either issue a new authentication command and start again, or the + client may issue the QUIT command. + + After the POP3 server has opened the maildrop, it assigns a message- + number to each message, and notes the size of each message in octets. + The first message in the maildrop is assigned a message-number of + "1", the second is assigned "2", and so on, so that the nth message + in a maildrop is assigned a message-number of "n". In POP3 commands + and responses, all message-numbers and message sizes are expressed in + base-10 (i.e., decimal). + + Here is the summary for the QUIT command when used in the + AUTHORIZATION state: + + QUIT + + Arguments: none + + Restrictions: none + + Possible Responses: + +OK + + Examples: + C: QUIT + S: +OK dewey POP3 server signing off + +5. The TRANSACTION State + + Once the client has successfully identified itself to the POP3 server + and the POP3 server has locked and opened the appropriate maildrop, + the POP3 session is now in the TRANSACTION state. The client may now + issue any of the following POP3 commands repeatedly. After each + command, the POP3 server issues a response. Eventually, the client + issues the QUIT command and the POP3 session enters the UPDATE state. + + + + + + + + + + +Myers & Rose Standards Track [Page 5] + +RFC 1939 POP3 May 1996 + + + Here are the POP3 commands valid in the TRANSACTION state: + + STAT + + Arguments: none + + Restrictions: + may only be given in the TRANSACTION state + + Discussion: + The POP3 server issues a positive response with a line + containing information for the maildrop. This line is + called a "drop listing" for that maildrop. + + In order to simplify parsing, all POP3 servers are + required to use a certain format for drop listings. The + positive response consists of "+OK" followed by a single + space, the number of messages in the maildrop, a single + space, and the size of the maildrop in octets. This memo + makes no requirement on what follows the maildrop size. + Minimal implementations should just end that line of the + response with a CRLF pair. More advanced implementations + may include other information. + + NOTE: This memo STRONGLY discourages implementations + from supplying additional information in the drop + listing. Other, optional, facilities are discussed + later on which permit the client to parse the messages + in the maildrop. + + Note that messages marked as deleted are not counted in + either total. + + Possible Responses: + +OK nn mm + + Examples: + C: STAT + S: +OK 2 320 + + + LIST [msg] + + Arguments: + a message-number (optional), which, if present, may NOT + refer to a message marked as deleted + + + + + +Myers & Rose Standards Track [Page 6] + +RFC 1939 POP3 May 1996 + + + Restrictions: + may only be given in the TRANSACTION state + + Discussion: + If an argument was given and the POP3 server issues a + positive response with a line containing information for + that message. This line is called a "scan listing" for + that message. + + If no argument was given and the POP3 server issues a + positive response, then the response given is multi-line. + After the initial +OK, for each message in the maildrop, + the POP3 server responds with a line containing + information for that message. This line is also called a + "scan listing" for that message. If there are no + messages in the maildrop, then the POP3 server responds + with no scan listings--it issues a positive response + followed by a line containing a termination octet and a + CRLF pair. + + In order to simplify parsing, all POP3 servers are + required to use a certain format for scan listings. A + scan listing consists of the message-number of the + message, followed by a single space and the exact size of + the message in octets. Methods for calculating the exact + size of the message are described in the "Message Format" + section below. This memo makes no requirement on what + follows the message size in the scan listing. Minimal + implementations should just end that line of the response + with a CRLF pair. More advanced implementations may + include other information, as parsed from the message. + + NOTE: This memo STRONGLY discourages implementations + from supplying additional information in the scan + listing. Other, optional, facilities are discussed + later on which permit the client to parse the messages + in the maildrop. + + Note that messages marked as deleted are not listed. + + Possible Responses: + +OK scan listing follows + -ERR no such message + + Examples: + C: LIST + S: +OK 2 messages (320 octets) + S: 1 120 + + + +Myers & Rose Standards Track [Page 7] + +RFC 1939 POP3 May 1996 + + + S: 2 200 + S: . + ... + C: LIST 2 + S: +OK 2 200 + ... + C: LIST 3 + S: -ERR no such message, only 2 messages in maildrop + + + RETR msg + + Arguments: + a message-number (required) which may NOT refer to a + message marked as deleted + + Restrictions: + may only be given in the TRANSACTION state + + Discussion: + If the POP3 server issues a positive response, then the + response given is multi-line. After the initial +OK, the + POP3 server sends the message corresponding to the given + message-number, being careful to byte-stuff the termination + character (as with all multi-line responses). + + Possible Responses: + +OK message follows + -ERR no such message + + Examples: + C: RETR 1 + S: +OK 120 octets + S: + S: . + + + DELE msg + + Arguments: + a message-number (required) which may NOT refer to a + message marked as deleted + + Restrictions: + may only be given in the TRANSACTION state + + + + + + +Myers & Rose Standards Track [Page 8] + +RFC 1939 POP3 May 1996 + + + Discussion: + The POP3 server marks the message as deleted. Any future + reference to the message-number associated with the message + in a POP3 command generates an error. The POP3 server does + not actually delete the message until the POP3 session + enters the UPDATE state. + + Possible Responses: + +OK message deleted + -ERR no such message + + Examples: + C: DELE 1 + S: +OK message 1 deleted + ... + C: DELE 2 + S: -ERR message 2 already deleted + + + NOOP + + Arguments: none + + Restrictions: + may only be given in the TRANSACTION state + + Discussion: + The POP3 server does nothing, it merely replies with a + positive response. + + Possible Responses: + +OK + + Examples: + C: NOOP + S: +OK + + + RSET + + Arguments: none + + Restrictions: + may only be given in the TRANSACTION state + + Discussion: + If any messages have been marked as deleted by the POP3 + server, they are unmarked. The POP3 server then replies + + + +Myers & Rose Standards Track [Page 9] + +RFC 1939 POP3 May 1996 + + + with a positive response. + + Possible Responses: + +OK + + Examples: + C: RSET + S: +OK maildrop has 2 messages (320 octets) + +6. The UPDATE State + + When the client issues the QUIT command from the TRANSACTION state, + the POP3 session enters the UPDATE state. (Note that if the client + issues the QUIT command from the AUTHORIZATION state, the POP3 + session terminates but does NOT enter the UPDATE state.) + + If a session terminates for some reason other than a client-issued + QUIT command, the POP3 session does NOT enter the UPDATE state and + MUST not remove any messages from the maildrop. + + QUIT + + Arguments: none + + Restrictions: none + + Discussion: + The POP3 server removes all messages marked as deleted + from the maildrop and replies as to the status of this + operation. If there is an error, such as a resource + shortage, encountered while removing messages, the + maildrop may result in having some or none of the messages + marked as deleted be removed. In no case may the server + remove any messages not marked as deleted. + + Whether the removal was successful or not, the server + then releases any exclusive-access lock on the maildrop + and closes the TCP connection. + + Possible Responses: + +OK + -ERR some deleted messages not removed + + Examples: + C: QUIT + S: +OK dewey POP3 server signing off (maildrop empty) + ... + C: QUIT + + + +Myers & Rose Standards Track [Page 10] + +RFC 1939 POP3 May 1996 + + + S: +OK dewey POP3 server signing off (2 messages left) + ... + +7. Optional POP3 Commands + + The POP3 commands discussed above must be supported by all minimal + implementations of POP3 servers. + + The optional POP3 commands described below permit a POP3 client + greater freedom in message handling, while preserving a simple POP3 + server implementation. + + NOTE: This memo STRONGLY encourages implementations to support + these commands in lieu of developing augmented drop and scan + listings. In short, the philosophy of this memo is to put + intelligence in the part of the POP3 client and not the POP3 + server. + + TOP msg n + + Arguments: + a message-number (required) which may NOT refer to to a + message marked as deleted, and a non-negative number + of lines (required) + + Restrictions: + may only be given in the TRANSACTION state + + Discussion: + If the POP3 server issues a positive response, then the + response given is multi-line. After the initial +OK, the + POP3 server sends the headers of the message, the blank + line separating the headers from the body, and then the + number of lines of the indicated message's body, being + careful to byte-stuff the termination character (as with + all multi-line responses). + + Note that if the number of lines requested by the POP3 + client is greater than than the number of lines in the + body, then the POP3 server sends the entire message. + + Possible Responses: + +OK top of message follows + -ERR no such message + + Examples: + C: TOP 1 10 + S: +OK + + + +Myers & Rose Standards Track [Page 11] + +RFC 1939 POP3 May 1996 + + + S: + S: . + ... + C: TOP 100 3 + S: -ERR no such message + + + UIDL [msg] + + Arguments: + a message-number (optional), which, if present, may NOT + refer to a message marked as deleted + + Restrictions: + may only be given in the TRANSACTION state. + + Discussion: + If an argument was given and the POP3 server issues a positive + response with a line containing information for that message. + This line is called a "unique-id listing" for that message. + + If no argument was given and the POP3 server issues a positive + response, then the response given is multi-line. After the + initial +OK, for each message in the maildrop, the POP3 server + responds with a line containing information for that message. + This line is called a "unique-id listing" for that message. + + In order to simplify parsing, all POP3 servers are required to + use a certain format for unique-id listings. A unique-id + listing consists of the message-number of the message, + followed by a single space and the unique-id of the message. + No information follows the unique-id in the unique-id listing. + + The unique-id of a message is an arbitrary server-determined + string, consisting of one to 70 characters in the range 0x21 + to 0x7E, which uniquely identifies a message within a + maildrop and which persists across sessions. This + persistence is required even if a session ends without + entering the UPDATE state. The server should never reuse an + unique-id in a given maildrop, for as long as the entity + using the unique-id exists. + + Note that messages marked as deleted are not listed. + + While it is generally preferable for server implementations + to store arbitrarily assigned unique-ids in the maildrop, + + + +Myers & Rose Standards Track [Page 12] + +RFC 1939 POP3 May 1996 + + + this specification is intended to permit unique-ids to be + calculated as a hash of the message. Clients should be able + to handle a situation where two identical copies of a + message in a maildrop have the same unique-id. + + Possible Responses: + +OK unique-id listing follows + -ERR no such message + + Examples: + C: UIDL + S: +OK + S: 1 whqtswO00WBw418f9t5JxYwZ + S: 2 QhdPYR:00WBw1Ph7x7 + S: . + ... + C: UIDL 2 + S: +OK 2 QhdPYR:00WBw1Ph7x7 + ... + C: UIDL 3 + S: -ERR no such message, only 2 messages in maildrop + + + USER name + + Arguments: + a string identifying a mailbox (required), which is of + significance ONLY to the server + + Restrictions: + may only be given in the AUTHORIZATION state after the POP3 + greeting or after an unsuccessful USER or PASS command + + Discussion: + To authenticate using the USER and PASS command + combination, the client must first issue the USER + command. If the POP3 server responds with a positive + status indicator ("+OK"), then the client may issue + either the PASS command to complete the authentication, + or the QUIT command to terminate the POP3 session. If + the POP3 server responds with a negative status indicator + ("-ERR") to the USER command, then the client may either + issue a new authentication command or may issue the QUIT + command. + + The server may return a positive response even though no + such mailbox exists. The server may return a negative + response if mailbox exists, but does not permit plaintext + + + +Myers & Rose Standards Track [Page 13] + +RFC 1939 POP3 May 1996 + + + password authentication. + + Possible Responses: + +OK name is a valid mailbox + -ERR never heard of mailbox name + + Examples: + C: USER frated + S: -ERR sorry, no mailbox for frated here + ... + C: USER mrose + S: +OK mrose is a real hoopy frood + + + PASS string + + Arguments: + a server/mailbox-specific password (required) + + Restrictions: + may only be given in the AUTHORIZATION state immediately + after a successful USER command + + Discussion: + When the client issues the PASS command, the POP3 server + uses the argument pair from the USER and PASS commands to + determine if the client should be given access to the + appropriate maildrop. + + Since the PASS command has exactly one argument, a POP3 + server may treat spaces in the argument as part of the + password, instead of as argument separators. + + Possible Responses: + +OK maildrop locked and ready + -ERR invalid password + -ERR unable to lock maildrop + + Examples: + C: USER mrose + S: +OK mrose is a real hoopy frood + C: PASS secret + S: -ERR maildrop already locked + ... + C: USER mrose + S: +OK mrose is a real hoopy frood + C: PASS secret + S: +OK mrose's maildrop has 2 messages (320 octets) + + + +Myers & Rose Standards Track [Page 14] + +RFC 1939 POP3 May 1996 + + + APOP name digest + + Arguments: + a string identifying a mailbox and a MD5 digest string + (both required) + + Restrictions: + may only be given in the AUTHORIZATION state after the POP3 + greeting or after an unsuccessful USER or PASS command + + Discussion: + Normally, each POP3 session starts with a USER/PASS + exchange. This results in a server/user-id specific + password being sent in the clear on the network. For + intermittent use of POP3, this may not introduce a sizable + risk. However, many POP3 client implementations connect to + the POP3 server on a regular basis -- to check for new + mail. Further the interval of session initiation may be on + the order of five minutes. Hence, the risk of password + capture is greatly enhanced. + + An alternate method of authentication is required which + provides for both origin authentication and replay + protection, but which does not involve sending a password + in the clear over the network. The APOP command provides + this functionality. + + A POP3 server which implements the APOP command will + include a timestamp in its banner greeting. The syntax of + the timestamp corresponds to the `msg-id' in [RFC822], and + MUST be different each time the POP3 server issues a banner + greeting. For example, on a UNIX implementation in which a + separate UNIX process is used for each instance of a POP3 + server, the syntax of the timestamp might be: + + + + where `process-ID' is the decimal value of the process's + PID, clock is the decimal value of the system clock, and + hostname is the fully-qualified domain-name corresponding + to the host where the POP3 server is running. + + The POP3 client makes note of this timestamp, and then + issues the APOP command. The `name' parameter has + identical semantics to the `name' parameter of the USER + command. The `digest' parameter is calculated by applying + the MD5 algorithm [RFC1321] to a string consisting of the + timestamp (including angle-brackets) followed by a shared + + + +Myers & Rose Standards Track [Page 15] + +RFC 1939 POP3 May 1996 + + + secret. This shared secret is a string known only to the + POP3 client and server. Great care should be taken to + prevent unauthorized disclosure of the secret, as knowledge + of the secret will allow any entity to successfully + masquerade as the named user. The `digest' parameter + itself is a 16-octet value which is sent in hexadecimal + format, using lower-case ASCII characters. + + When the POP3 server receives the APOP command, it verifies + the digest provided. If the digest is correct, the POP3 + server issues a positive response, and the POP3 session + enters the TRANSACTION state. Otherwise, a negative + response is issued and the POP3 session remains in the + AUTHORIZATION state. + + Note that as the length of the shared secret increases, so + does the difficulty of deriving it. As such, shared + secrets should be long strings (considerably longer than + the 8-character example shown below). + + Possible Responses: + +OK maildrop locked and ready + -ERR permission denied + + Examples: + S: +OK POP3 server ready <1896.697170952@dbc.mtview.ca.us> + C: APOP mrose c4c9334bac560ecc979e58001b3e22fb + S: +OK maildrop has 1 message (369 octets) + + In this example, the shared secret is the string `tan- + staaf'. Hence, the MD5 algorithm is applied to the string + + <1896.697170952@dbc.mtview.ca.us>tanstaaf + + which produces a digest value of + + c4c9334bac560ecc979e58001b3e22fb + +8. Scaling and Operational Considerations + + Since some of the optional features described above were added to the + POP3 protocol, experience has accumulated in using them in large- + scale commercial post office operations where most of the users are + unrelated to each other. In these situations and others, users and + vendors of POP3 clients have discovered that the combination of using + the UIDL command and not issuing the DELE command can provide a weak + version of the "maildrop as semi-permanent repository" functionality + normally associated with IMAP. Of course the other capabilities of + + + +Myers & Rose Standards Track [Page 16] + +RFC 1939 POP3 May 1996 + + + IMAP, such as polling an existing connection for newly arrived + messages and supporting multiple folders on the server, are not + present in POP3. + + When these facilities are used in this way by casual users, there has + been a tendency for already-read messages to accumulate on the server + without bound. This is clearly an undesirable behavior pattern from + the standpoint of the server operator. This situation is aggravated + by the fact that the limited capabilities of the POP3 do not permit + efficient handling of maildrops which have hundreds or thousands of + messages. + + Consequently, it is recommended that operators of large-scale multi- + user servers, especially ones in which the user's only access to the + maildrop is via POP3, consider such options as: + + * Imposing a per-user maildrop storage quota or the like. + + A disadvantage to this option is that accumulation of messages may + result in the user's inability to receive new ones into the + maildrop. Sites which choose this option should be sure to inform + users of impending or current exhaustion of quota, perhaps by + inserting an appropriate message into the user's maildrop. + + * Enforce a site policy regarding mail retention on the server. + + Sites are free to establish local policy regarding the storage and + retention of messages on the server, both read and unread. For + example, a site might delete unread messages from the server after + 60 days and delete read messages after 7 days. Such message + deletions are outside the scope of the POP3 protocol and are not + considered a protocol violation. + + Server operators enforcing message deletion policies should take + care to make all users aware of the policies in force. + + Clients must not assume that a site policy will automate message + deletions, and should continue to explicitly delete messages using + the DELE command when appropriate. + + It should be noted that enforcing site message deletion policies + may be confusing to the user community, since their POP3 client + may contain configuration options to leave mail on the server + which will not in fact be supported by the server. + + One special case of a site policy is that messages may only be + downloaded once from the server, and are deleted after this has + been accomplished. This could be implemented in POP3 server + + + +Myers & Rose Standards Track [Page 17] + +RFC 1939 POP3 May 1996 + + + software by the following mechanism: "following a POP3 login by a + client which was ended by a QUIT, delete all messages downloaded + during the session with the RETR command". It is important not to + delete messages in the event of abnormal connection termination + (ie, if no QUIT was received from the client) because the client + may not have successfully received or stored the messages. + Servers implementing a download-and-delete policy may also wish to + disable or limit the optional TOP command, since it could be used + as an alternate mechanism to download entire messages. + +9. POP3 Command Summary + + Minimal POP3 Commands: + + USER name valid in the AUTHORIZATION state + PASS string + QUIT + + STAT valid in the TRANSACTION state + LIST [msg] + RETR msg + DELE msg + NOOP + RSET + QUIT + + Optional POP3 Commands: + + APOP name digest valid in the AUTHORIZATION state + + TOP msg n valid in the TRANSACTION state + UIDL [msg] + + POP3 Replies: + + +OK + -ERR + + Note that with the exception of the STAT, LIST, and UIDL commands, + the reply given by the POP3 server to any command is significant + only to "+OK" and "-ERR". Any text occurring after this reply + may be ignored by the client. + + + + + + + + + +Myers & Rose Standards Track [Page 18] + +RFC 1939 POP3 May 1996 + + +10. Example POP3 Session + + S: + C: + S: +OK POP3 server ready <1896.697170952@dbc.mtview.ca.us> + C: APOP mrose c4c9334bac560ecc979e58001b3e22fb + S: +OK mrose's maildrop has 2 messages (320 octets) + C: STAT + S: +OK 2 320 + C: LIST + S: +OK 2 messages (320 octets) + S: 1 120 + S: 2 200 + S: . + C: RETR 1 + S: +OK 120 octets + S: + S: . + C: DELE 1 + S: +OK message 1 deleted + C: RETR 2 + S: +OK 200 octets + S: + S: . + C: DELE 2 + S: +OK message 2 deleted + C: QUIT + S: +OK dewey POP3 server signing off (maildrop empty) + C: + S: + +11. Message Format + + All messages transmitted during a POP3 session are assumed to conform + to the standard for the format of Internet text messages [RFC822]. + + It is important to note that the octet count for a message on the + server host may differ from the octet count assigned to that message + due to local conventions for designating end-of-line. Usually, + during the AUTHORIZATION state of the POP3 session, the POP3 server + can calculate the size of each message in octets when it opens the + maildrop. For example, if the POP3 server host internally represents + end-of-line as a single character, then the POP3 server simply counts + each occurrence of this character in a message as two octets. Note + that lines in the message which start with the termination octet need + not (and must not) be counted twice, since the POP3 client will + remove all byte-stuffed termination characters when it receives a + multi-line response. + + + +Myers & Rose Standards Track [Page 19] + +RFC 1939 POP3 May 1996 + + +12. References + + [RFC821] Postel, J., "Simple Mail Transfer Protocol", STD 10, RFC + 821, USC/Information Sciences Institute, August 1982. + + [RFC822] Crocker, D., "Standard for the Format of ARPA-Internet Text + Messages", STD 11, RFC 822, University of Delaware, August 1982. + + [RFC1321] Rivest, R., "The MD5 Message-Digest Algorithm", RFC 1321, + MIT Laboratory for Computer Science, April 1992. + + [RFC1730] Crispin, M., "Internet Message Access Protocol - Version + 4", RFC 1730, University of Washington, December 1994. + + [RFC1734] Myers, J., "POP3 AUTHentication command", RFC 1734, + Carnegie Mellon, December 1994. + +13. Security Considerations + + It is conjectured that use of the APOP command provides origin + identification and replay protection for a POP3 session. + Accordingly, a POP3 server which implements both the PASS and APOP + commands should not allow both methods of access for a given user; + that is, for a given mailbox name, either the USER/PASS command + sequence or the APOP command is allowed, but not both. + + Further, note that as the length of the shared secret increases, so + does the difficulty of deriving it. + + Servers that answer -ERR to the USER command are giving potential + attackers clues about which names are valid. + + Use of the PASS command sends passwords in the clear over the + network. + + Use of the RETR and TOP commands sends mail in the clear over the + network. + + Otherwise, security issues are not discussed in this memo. + +14. Acknowledgements + + The POP family has a long and checkered history. Although primarily + a minor revision to RFC 1460, POP3 is based on the ideas presented in + RFCs 918, 937, and 1081. + + In addition, Alfred Grimstad, Keith McCloghrie, and Neil Ostroff + provided significant comments on the APOP command. + + + +Myers & Rose Standards Track [Page 20] + +RFC 1939 POP3 May 1996 + + +15. Authors' Addresses + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave + Pittsburgh, PA 15213 + + EMail: jgm+@cmu.edu + + + Marshall T. Rose + Dover Beach Consulting, Inc. + 420 Whisman Court + Mountain View, CA 94043-2186 + + EMail: mrose@dbc.mtview.ca.us + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Myers & Rose Standards Track [Page 21] + +RFC 1939 POP3 May 1996 + + +Appendix A. Differences from RFC 1725 + + This memo is a revision to RFC 1725, a Draft Standard. It makes the + following changes from that document: + + - clarifies that command keywords are case insensitive. + + - specifies that servers must send "+OK" and "-ERR" in + upper case. + + - specifies that the initial greeting is a positive response, + instead of any string which should be a positive response. + + - clarifies behavior for unimplemented commands. + + - makes the USER and PASS commands optional. + + - clarified the set of possible responses to the USER command. + + - reverses the order of the examples in the USER and PASS + commands, to reduce confusion. + + - clarifies that the PASS command may only be given immediately + after a successful USER command. + + - clarified the persistence requirements of UIDs and added some + implementation notes. + + - specifies a UID length limitation of one to 70 octets. + + - specifies a status indicator length limitation + of 512 octets, including the CRLF. + + - clarifies that LIST with no arguments on an empty mailbox + returns success. + + - adds a reference from the LIST command to the Message Format + section + + - clarifies the behavior of QUIT upon failure + + - clarifies the security section to not imply the use of the + USER command with the APOP command. + + - adds references to RFCs 1730 and 1734 + + - clarifies the method by which a UA may enter mail into the + transport system. + + + +Myers & Rose Standards Track [Page 22] + +RFC 1939 POP3 May 1996 + + + - clarifies that the second argument to the TOP command is a + number of lines. + + - changes the suggestion in the Security Considerations section + for a server to not accept both PASS and APOP for a given user + from a "must" to a "should". + + - adds a section on scaling and operational considerations + +Appendix B. Command Index + + APOP ....................................................... 15 + DELE ....................................................... 8 + LIST ....................................................... 6 + NOOP ....................................................... 9 + PASS ....................................................... 14 + QUIT ....................................................... 5 + QUIT ....................................................... 10 + RETR ....................................................... 8 + RSET ....................................................... 9 + STAT ....................................................... 6 + TOP ........................................................ 11 + UIDL ....................................................... 12 + USER ....................................................... 13 + + + + + + + + + + + + + + + + + + + + + + + + + + + +Myers & Rose Standards Track [Page 23] + diff --git a/docs/rfcs/rfc2061.txt b/docs/rfcs/rfc2061.txt new file mode 100644 index 0000000..7cb02bb --- /dev/null +++ b/docs/rfcs/rfc2061.txt @@ -0,0 +1,171 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 2061 University of Washington +Category: Informational December 1996 + + + IMAP4 COMPATIBILITY WITH IMAP2BIS + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + +Introduction + + The Internet Message Access Protocol (IMAP) has been through several + revisions and variants in its 10-year history. Many of these are + either extinct or extremely rare; in particular, several undocumented + variants and the variants described in RFC 1064, RFC 1176, and RFC + 1203 fall into this category. + + One variant, IMAP2bis, is at the time of this writing very common and + has been widely distributed with the Pine mailer. Unfortunately, + there is no definite document describing IMAP2bis. This document is + intended to be read along with RFC 1176 and the most recent IMAP4 + specification (RFC 2060) to assist implementors in creating an IMAP4 + implementation to interoperate with implementations that conform to + earlier specifications. Nothing in this document is required by the + IMAP4 specification; implementors must decide for themselves whether + they want their implementation to fail if it encounters old software. + + At the time of this writing, IMAP4 has been updated from the version + described in RFC 1730. An implementor who wishes to interoperate + with both RFC 1730 and RFC 2060 should refer to both documents. + + This information is not complete; it reflects current knowledge of + server and client implementations as well as "folklore" acquired in + the evolution of the protocol. It is NOT a description of how to + interoperate with all variants of IMAP, but rather with the old + variant that is most likely to be encountered. For detailed + information on interoperating with other old variants, refer to RFC + 1732. + +IMAP4 client interoperability with IMAP2bis servers + + A quick way to check whether a server implementation supports the + IMAP4 specification is to try the CAPABILITY command. An OK response + will indicate which variant(s) of IMAP4 are supported by the server. + + + +Crispin Informational [Page 1] + +RFC 2061 IMAP4 Compatibility December 1996 + + + If the client does not find any of its known variant in the response, + it should treat the server as IMAP2bis. A BAD response indicates an + IMAP2bis or older server. + + Most IMAP4 facilities are in IMAP2bis. The following exceptions + exist: + + CAPABILITY command + The absense of this command indicates IMAP2bis (or older). + + AUTHENTICATE command. + Use the LOGIN command. + + LSUB, SUBSCRIBE, and UNSUBSCRIBE commands + No direct functional equivalent. IMAP2bis had a concept + called "bboards" which is not in IMAP4. RFC 1176 supported + these with the BBOARD and FIND BBOARDS commands. IMAP2bis + augmented these with the FIND ALL.BBOARDS, SUBSCRIBE BBOARD, + and UNSUBSCRIBE BBOARD commands. It is recommended that + none of these commands be implemented in new software, + including servers that support old clients. + + LIST command + Use the command FIND ALL.MAILBOXES, which has a similar syn- + tax and response to the FIND MAILBOXES command described in + RFC 1176. The FIND MAILBOXES command is unlikely to produce + useful information. + + * in a sequence + Use the number of messages in the mailbox from the EXISTS + unsolicited response. + + SEARCH extensions (character set, additional criteria) + Reformulate the search request using only the RFC 1176 syn- + tax. This may entail doing multiple searches to achieve the + desired results. + + BODYSTRUCTURE fetch data item + Use the non-extensible BODY data item. + + body sections HEADER, TEXT, MIME, HEADER.FIELDS, HEADER.FIELDS.NOT + Use body section numbers only. + + BODY.PEEK[section] + Use BODY[section] and manually clear the \Seen flag as + necessary. + + + + + +Crispin Informational [Page 2] + +RFC 2061 IMAP4 Compatibility December 1996 + + + FLAGS.SILENT, +FLAGS.SILENT, and -FLAGS.SILENT store data items + Use the corresponding non-SILENT versions and ignore the + untagged FETCH responses which come back. + + UID fetch data item and the UID commands + No functional equivalent. + + CLOSE command + No functional equivalent. + + + In IMAP2bis, the TRYCREATE special information token is sent as a + separate unsolicited OK response instead of inside the NO response. + + IMAP2bis is ambiguous about whether or not flags or internal dates + are preserved on COPY. It is impossible to know what behavior is + supported by the server. + +IMAP4 server interoperability with IMAP2bis clients + + The only interoperability problem between an IMAP4 server and a + well-written IMAP2bis client is an incompatibility with the use of + "\" in quoted strings. This is best avoided by using literals + instead of quoted strings if "\" or <"> is embedded in the string. + +Security Considerations + + Security issues are not discussed in this memo. + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Aveneue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + +Crispin Informational [Page 3] + diff --git a/docs/rfcs/rfc2062.txt b/docs/rfcs/rfc2062.txt new file mode 100644 index 0000000..865d1da --- /dev/null +++ b/docs/rfcs/rfc2062.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 2062 University of Washington +Category: Informational December 1996 + + + Internet Message Access Protocol - Obsolete Syntax + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + +Abstract + + This document describes obsolete syntax which may be encountered by + IMAP4 implementations which deal with older versions of the Internet + Mail Access Protocol. IMAP4 implementations MAY implement this + syntax in order to maximize interoperability with older + implementations. + + This document repeats information from earlier documents, most + notably RFC 1176 and RFC 1730. + +Obsolete Commands and Fetch Data Items + + The following commands are OBSOLETE. It is NOT required to support + any of these commands or fetch data items in new server + implementations. These commands are documented here for the benefit + of implementors who may wish to support them for compatibility with + old client implementations. + + The section headings of these commands are intended to correspond + with where they would be located in the main document if they were + not obsoleted. + +6.3.OBS.1. FIND ALL.MAILBOXES Command + + Arguments: mailbox name with possible wildcards + + Data: untagged responses: MAILBOX + + Result: OK - find completed + NO - find failure: can't list that name + BAD - command unknown or arguments invalid + + + + + + +Crispin Informational [Page 1] + +RFC 2062 IMAP4 Obsolete December 1996 + + + The FIND ALL.MAILBOXES command returns a subset of names from the + complete set of all names available to the user. It returns zero + or more untagged MAILBOX replies. The mailbox argument to FIND + ALL.MAILBOXES is similar to that for LIST with an empty reference, + except that the characters "%" and "?" match a single character. + + Example: C: A002 FIND ALL.MAILBOXES * + S: * MAILBOX blurdybloop + S: * MAILBOX INBOX + S: A002 OK FIND ALL.MAILBOXES completed + +6.3.OBS.2. FIND MAILBOXES Command + + Arguments: mailbox name with possible wildcards + + Data: untagged responses: MAILBOX + + Result: OK - find completed + NO - find failure: can't list that name + BAD - command unknown or arguments invalid + + The FIND MAILBOXES command returns a subset of names from the set + of names that the user has declared as being "active" or + "subscribed". It returns zero or more untagged MAILBOX replies. + The mailbox argument to FIND MAILBOXES is similar to that for LSUB + with an empty reference, except that the characters "%" and "?" + match a single character. + + Example: C: A002 FIND MAILBOXES * + S: * MAILBOX blurdybloop + S: * MAILBOX INBOX + S: A002 OK FIND MAILBOXES completed + +6.3.OBS.3. SUBSCRIBE MAILBOX Command + + Arguments: mailbox name + + Data: no specific data for this command + + Result: OK - subscribe completed + NO - subscribe failure: can't subscribe to that name + BAD - command unknown or arguments invalid + + The SUBSCRIBE MAILBOX command is identical in effect to the + SUBSCRIBE command. A server which implements this command must be + able to distinguish between a SUBSCRIBE MAILBOX command and a + SUBSCRIBE command with a mailbox name argument of "MAILBOX". + + + + +Crispin Informational [Page 2] + +RFC 2062 IMAP4 Obsolete December 1996 + + + Example: C: A002 SUBSCRIBE MAILBOX #news.comp.mail.mime + S: A002 OK SUBSCRIBE MAILBOX to #news.comp.mail.mime + completed + C: A003 SUBSCRIBE MAILBOX + S: A003 OK SUBSCRIBE to MAILBOX completed + + +6.3.OBS.4. UNSUBSCRIBE MAILBOX Command + + Arguments: mailbox name + + Data: no specific data for this command + + Result: OK - unsubscribe completed + NO - unsubscribe failure: can't unsubscribe that name + BAD - command unknown or arguments invalid + + The UNSUBSCRIBE MAILBOX command is identical in effect to the + UNSUBSCRIBE command. A server which implements this command must + be able to distinguish between a UNSUBSCRIBE MAILBOX command and + an UNSUBSCRIBE command with a mailbox name argument of "MAILBOX". + + Example: C: A002 UNSUBSCRIBE MAILBOX #news.comp.mail.mime + S: A002 OK UNSUBSCRIBE MAILBOX from #news.comp.mail.mime + completed + C: A003 UNSUBSCRIBE MAILBOX + S: A003 OK UNSUBSCRIBE from MAILBOX completed + +6.4.OBS.1 PARTIAL Command + + Arguments: message sequence number + message data item name + position of first octet + number of octets + + Data: untagged responses: FETCH + + Result: OK - partial completed + NO - partial error: can't fetch that data + BAD - command unknown or arguments invalid + + The PARTIAL command is equivalent to the associated FETCH command, + with the added functionality that only the specified number of + octets, beginning at the specified starting octet, are returned. + Only a single message can be fetched at a time. The first octet + of a message, and hence the minimum for the starting octet, is + octet 1. + + + + +Crispin Informational [Page 3] + +RFC 2062 IMAP4 Obsolete December 1996 + + + The following FETCH items are valid data for PARTIAL: RFC822, + RFC822.HEADER, RFC822.TEXT, BODY[
], as well as any .PEEK + forms of these. + + Any partial fetch that attempts to read beyond the end of the text + is truncated as appropriate. If the starting octet is beyond the + end of the text, an empty string is returned. + + The data are returned with the FETCH response. There is no + indication of the range of the partial data in this response. It + is not possible to stream multiple PARTIAL commands of the same + data item without processing and synchronizing at each step, since + streamed commands may be executed out of order. + + There is no requirement that partial fetches follow any sequence. + For example, if a partial fetch of octets 1 through 10000 breaks + in an awkward place for BASE64 decoding, it is permitted to + continue with a partial fetch of 9987 through 19987, etc. + + The handling of the \Seen flag is the same as in the associated + FETCH command. + + Example: C: A005 PARTIAL 4 RFC822 1 1024 + S: * 1 FETCH (RFC822 {1024} + S: Return-Path: + S: ... + S: ......... FLAGS (\Seen)) + S: A005 OK PARTIAL completed + +6.4.5.OBS.1 Obsolete FETCH Data Items + + The following FETCH data items are obsolete: + + BODY[<...>0] A body part number of 0 is the [RFC-822] header of + the message. BODY[0] is functionally equivalent to + BODY[HEADER], differing in the syntax of the + resulting untagged FETCH data (BODY[0] is + returned). + + RFC822.HEADER.LINES + Functionally equivalent to BODY.PEEK[HEADER.LINES + ], differing in the syntax of the + resulting untagged FETCH data (RFC822.HEADER is + returned). + + + + + + + +Crispin Informational [Page 4] + +RFC 2062 IMAP4 Obsolete December 1996 + + + RFC822.HEADER.LINES.NOT + Functionally equivalent to + BODY.PEEK[HEADER.LINES.NOT ], + differing in the syntax of the resulting untagged + FETCH data (RFC822.HEADER is returned). + + RFC822.PEEK Functionally equivalent to BODY.PEEK[], except for + the syntax of the resulting untagged FETCH data + (RFC822 is returned). + + RFC822.TEXT.PEEK + Functionally equivalent to BODY.PEEK[TEXT], except + for the syntax of the resulting untagged FETCH data + (RFC822.TEXT is returned). + +Obsolete Responses + + The following responses are OBSOLETE. Except as noted below, these + responses MUST NOT be transmitted by new server implementations. + Client implementations SHOULD accept these responses. + + The section headings of these responses are intended to correspond + with where they would be located in the main document if they were + not obsoleted. + +7.2.OBS.1. MAILBOX Response + + Data: name + + The MAILBOX response MUST NOT be transmitted by server + implementations except in response to the obsolete FIND MAILBOXES + and FIND ALL.MAILBOXES commands. Client implementations that do + not use these commands MAY ignore this response. It is documented + here for the benefit of implementors who may wish to support it + for compatibility with old client implementations. + + This response occurs as a result of the FIND MAILBOXES and FIND + ALL.MAILBOXES commands. It returns a single name that matches the + FIND specification. There are no attributes or hierarchy + delimiter. + + Example: S: * MAILBOX blurdybloop + + + + + + + + + +Crispin Informational [Page 5] + +RFC 2062 IMAP4 Obsolete December 1996 + + +7.3.OBS.1. COPY Response + + Data: none + + The COPY response MUST NOT be transmitted by new server + implementations. Client implementations MUST ignore the COPY + response. It is documented here for the benefit of client + implementors who may encounter this response from old server + implementations. + + In some experimental versions of this protocol, this response was + returned in response to a COPY command to indicate on a + per-message basis that the message was copied successfully. + + Example: S: * 44 COPY + +7.3.OBS.2. STORE Response + + Data: message data + + The STORE response MUST NOT be transmitted by new server + implementations. Client implementations MUST treat the STORE + response as equivalent to the FETCH response. It is documented + here for the benefit of client implementors who may encounter this + response from old server implementations. + + In some experimental versions of this protocol, this response was + returned instead of FETCH in response to a STORE command to report + the new value of the flags. + + Example: S: * 69 STORE (FLAGS (\Deleted)) + +Formal Syntax of Obsolete Commands and Responses + + Each obsolete syntax rule that is suffixed with "_old" is added to + the corresponding name in the formal syntax. For example, + command_auth_old adds the FIND command to command_auth. + + command_auth_old ::= find + + command_select_old + ::= partial + + date_year_old ::= 2digit + ;; (year - 1900) + + date_time_old ::= <"> date_day_fixed "-" date_month "-" date_year + SPACE time "-" zone_name <"> + + + +Crispin Informational [Page 6] + +RFC 2062 IMAP4 Obsolete December 1996 + + + find ::= "FIND" SPACE ["ALL."] "MAILBOXES" SPACE + list_mailbox + + fetch_att_old ::= "RFC822.HEADER.LINES" [".NOT"] SPACE header_list / + fetch_text_old + + fetch_text_old ::= "BODY" [".PEEK"] section_old / + "RFC822" [".HEADER" / ".TEXT" [".PEEK"]] + + msg_data_old ::= "COPY" / ("STORE" SPACE msg_att) + + partial ::= "PARTIAL" SPACE nz_number SPACE fetch_text_old SPACE + number SPACE number + + section_old ::= "[" (number ["." number]) "]" + + subscribe_old ::= "SUBSCRIBE" SPACE "MAILBOX" SPACE mailbox + + unsubscribe_old ::= "UNSUBSCRIBE" SPACE "MAILBOX" SPACE mailbox + + zone_name ::= "UT" / "GMT" / "Z" / ;; +0000 + "AST" / "EDT" / ;; -0400 + "EST" / "CDT" / ;; -0500 + "CST" / "MDT" / ;; -0600 + "MST" / "PDT" / ;; -0700 + "PST" / "YDT" / ;; -0800 + "YST" / "HDT" / ;; -0900 + "HST" / "BDT" / ;; -1000 + "BST" / ;; -1100 + "A" / "B" / "C" / "D" / "E" / "F" / ;; +1 to +6 + "G" / "H" / "I" / "K" / "L" / "M" / ;; +7 to +12 + "N" / "O" / "P" / "Q" / "R" / "S" / ;; -1 to -6 + "T" / "U" / "V" / "W" / "X" / "Y" ;; -7 to -12 + +Security Considerations + + Security issues are not discussed in this memo. + + + + + + + + + + + + + + +Crispin Informational [Page 7] + +RFC 2062 IMAP4 Obsolete December 1996 + + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Aveneue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Informational [Page 8] + diff --git a/docs/rfcs/rfc2086.txt b/docs/rfcs/rfc2086.txt new file mode 100644 index 0000000..b6a98b5 --- /dev/null +++ b/docs/rfcs/rfc2086.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 2086 Carnegie Mellon +Category: Standards Track January 1997 + + + IMAP4 ACL extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + The ACL extension of the Internet Message Access Protocol [IMAP4] + permits access control lists to be manipulated through the IMAP + protocol. + +Table of Contents + + 1. Abstract............................................... 1 + 2. Conventions Used in this Document...................... 1 + 3. Introduction and Overview.............................. 2 + 4. Commands............................................... 3 + 4.1. SETACL................................................. 3 + 4.2. DELETEACL.............................................. 4 + 4.3. GETACL................................................. 4 + 4.4. LISTRIGHTS............................................. 4 + 4.5. MYRIGHTS............................................... 5 + 5. Responses.............................................. 5 + 5.1. ACL.................................................... 5 + 5.2. LISTRIGHTS............................................. 6 + 5.3. MYRIGHTS............................................... 6 + 6. Formal Syntax.......................................... 6 + 7. References............................................. 7 + 8. Security Considerations................................ 7 + 9. Author's Address....................................... 8 + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + + + + + +Myers Standards Track [Page 1] + +RFC 2086 ACL extension January 1997 + + +3. Introduction and Overview + + The ACL extension is present in any IMAP4 implementation which + returns "ACL" as one of the supported capabilities to the CAPABILITY + command. + + An access control list is a set of pairs. + + Identifier is a US-ASCII string. The identifier anyone is reserved + to refer to the universal identity (all authentications, including + anonymous). All user name strings accepted by the LOGIN or + AUTHENTICATE commands to authenticate to the IMAP server are reserved + as identifiers for the corresponding user. Identifiers starting with + a dash ("-") are reserved for "negative rights", described below. + All other identifier strings are interpreted in an implementation- + defined manner. + + Rights is a string listing a (possibly empty) set of alphanumeric + characters, each character listing a set of operations which is being + controlled. Letters are reserved for ``standard'' rights, listed + below. The set of standard rights may only be extended by a + standards-track document. Digits are reserved for implementation or + site defined rights. The currently defined standard rights are: + + l - lookup (mailbox is visible to LIST/LSUB commands) + r - read (SELECT the mailbox, perform CHECK, FETCH, PARTIAL, + SEARCH, COPY from mailbox) + s - keep seen/unseen information across sessions (STORE SEEN flag) + w - write (STORE flags other than SEEN and DELETED) + i - insert (perform APPEND, COPY into mailbox) + p - post (send mail to submission address for mailbox, + not enforced by IMAP4 itself) + c - create (CREATE new sub-mailboxes in any implementation-defined + hierarchy) + d - delete (STORE DELETED flag, perform EXPUNGE) + a - administer (perform SETACL) + + An implementation may tie rights together or may force rights to + always or never be granted to particular identifiers. For example, + in an implementation that uses unix mode bits, the rights "wisd" are + tied, the "a" right is always granted to the owner of a mailbox and + is never granted to another user. If rights are tied in an + implementation, the implementation must be conservative in granting + rights in response to SETACL commands--unless all rights in a tied + set are specified, none of that set should be included in the ACL + entry for that identifier. A client may discover the set of rights + which may be granted to a given identifier in the ACL for a given + mailbox by using the LISTRIGHTS command. + + + +Myers Standards Track [Page 2] + +RFC 2086 ACL extension January 1997 + + + It is possible for multiple identifiers in an access control list to + apply to a given user (or other authentication identity). For + example, an ACL may include rights to be granted to the identifier + matching the user, one or more implementation-defined identifiers + matching groups which include the user, and/or the identifier + "anyone". How these rights are combined to determine the user's + access is implementation-defined. An implementation may choose, for + example, to use the union of the rights granted to the applicable + identifiers. An implementation may instead choose, for example, to + only use those rights granted to the most specific identifier present + in the ACL. A client may determine the set of rights granted to the + logged-in user for a given mailbox by using the MYRIGHTS command. + + When an identifier in an ACL starts with a dash ("-"), that indicates + that associated rights are to be removed from the identifier that is + prefixed by the dash. For example, if the identifier "-fred" is + granted the "w" right, that indicates that the "w" right is to be + removed from users matching the identifier "fred". Implementations + need not support having identifiers which start with a dash in ACLs. + +4. Commands + +4.1. SETACL + + Arguments: mailbox name + authentication identifier + access right modification + + Data: no specific data for this command + + Result: OK - setacl completed + NO - setacl failure: can't set acl + BAD - command unknown or arguments invalid + + The SETACL command changes the access control list on the + specified mailbox so that the specified identifier is granted + permissions as specified in the third argument. + + The third argument is a string containing an optional plus ("+") + or minus ("-") prefix, followed by zero or more rights characters. + If the string starts with a plus, the following rights are added + to any existing rights for the identifier. If the string starts + with a minus, the following rights are removed from any existing + rights for the identifier. If the string does not start with a + plus or minus, the rights replace any existing rights for the + identifier. + + + + + +Myers Standards Track [Page 3] + +RFC 2086 ACL extension January 1997 + + +4.2. DELETEACL + + Arguments: mailbox name + authentication identifier + + Data: no specific data for this command + + Result: OK - deleteacl completed + NO - deleteacl failure: can't delete acl + BAD - command unknown or arguments invalid + + The DELETEACL command removes any pair for the + specified identifier from the access control list for the specified + mailbox. + +4.3. GETACL + + Arguments: mailbox name + + Data: untagged responses: ACL + + Result: OK - getacl completed + NO - getacl failure: can't get acl + BAD - command unknown or arguments invalid + + The GETACL command returns the access control list for mailbox in + an untagged ACL reply. + + Example: C: A002 GETACL INBOX + S: * ACL INBOX Fred rwipslda + S: A002 OK Getacl complete + +4.4. LISTRIGHTS + + Arguments: mailbox name + authentication identifier + + Data: untagged responses: LISTRIGHTS + + Result: OK - listrights completed + NO - listrights failure: can't get rights list + BAD - command unknown or arguments invalid + + The LISTRIGHTS command takes a mailbox name and an identifier and + returns information about what rights may be granted to the identifier + in the ACL for the mailbox. + + + + + +Myers Standards Track [Page 4] + +RFC 2086 ACL extension January 1997 + + + Example: C: a001 LISTRIGHTS ~/Mail/saved smith + S: * LISTRIGHTS ~/Mail/saved smith la r swicd + S: a001 OK Listrights completed + + + C: a005 LISTRIGHTS archive.imap anyone + S: * LISTRIGHTS archive.imap anyone "" l r s w i p c d a + 0 1 2 3 4 5 6 7 8 9 + +4.5. MYRIGHTS + + Arguments: mailbox name + + Data: untagged responses: MYRIGHTS + + Result: OK - myrights completed + NO - myrights failure: can't get rights + BAD - command unknown or arguments invalid + + The MYRIGHTS command returns the set of rights that the user has + to mailbox in an untagged MYRIGHTS reply. + + Example: C: A003 MYRIGHTS INBOX + S: * MYRIGHTS INBOX rwipslda + S: A003 OK Myrights complete + +5. Responses + +5.1. ACL + + Data: mailbox name + zero or more identifier rights pairs + + The ACL response occurs as a result of a GETACL command. The first + string is the mailbox name for which this ACL applies. This is + followed by zero or more pairs of strings, each pair contains the + identifier for which the entry applies followed by the set of + rights that the identifier has. + + + + + + + + + + + + + +Myers Standards Track [Page 5] + +RFC 2086 ACL extension January 1997 + + +5.2. LISTRIGHTS + + Data: mailbox name + identifier + required rights + list of optional rights + + The LISTRIGHTS response occurs as a result of a LISTRIGHTS + command. The first two strings are the mailbox name and identifier + for which this rights list applies. Following the identifier is a + string containing the (possibly empty) set of rights the + identifier will always be granted in the mailbox. + + Following this are zero or more strings each containing a set of + rights the identifier may be granted in the mailbox. Rights + mentioned in the same string are tied together--either all must be + granted to the identifier in the mailbox or none may be granted. + + The same right may not be listed more than once in the LISTRIGHTS + command. + +5.3. MYRIGHTS + + Data: mailbox name + rights + + The MYRIGHTS response occurs as a result of a MYRIGHTS command. + The first string is the mailbox name for which these rights apply. + The second string is the set of rights that the client has. + +6. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + + + + + + + + + +Myers Standards Track [Page 6] + +RFC 2086 ACL extension January 1997 + + + acl_data ::= "ACL" SPACE mailbox *(SPACE identifier SPACE + rights) + + deleteacl ::= "DELETEACL" SPACE mailbox SPACE identifier + + getacl ::= "GETACL" SPACE mailbox + + identifier ::= astring + + listrights ::= "LISTRIGHTS" SPACE mailbox SPACE identifier + + listrights_data ::= "LISTRIGHTS" SPACE mailbox SPACE identifier + SPACE rights *(SPACE rights) + + mod_rights ::= astring + ;; +rights to add, -rights to remove + ;; rights to replace + + myrights ::= "MYRIGHTS" SPACE mailbox + + myrights_data ::= "MYRIGHTS" SPACE mailbox SPACE rights + + rights ::= astring + + setacl ::= "SETACL" SPACE mailbox SPACE identifier + SPACE mod_rights + +7. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", + RFC 1730, University of Washington, December 1994. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text + Messages", STD 11, RFC 822. + +8. Security Considerations + + An implementation must make sure the ACL commands themselves do not + give information about mailboxes with appropriately restricted ACL's. + For example, a GETACL command on a mailbox for which the user has + insufficient rights should not admit the mailbox exists, much less + return the mailbox's ACL. + + + + + + + + + +Myers Standards Track [Page 7] + +RFC 2086 ACL extension January 1997 + + +9. Author's Address + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave. + Pittsburgh PA, 15213-3890 + + Email: jgm+@cmu.edu + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Myers Standards Track [Page 8] + diff --git a/docs/rfcs/rfc2087.txt b/docs/rfcs/rfc2087.txt new file mode 100644 index 0000000..1db5b57 --- /dev/null +++ b/docs/rfcs/rfc2087.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 2087 Carnegie Mellon +Category: Standards Track January 1997 + + + IMAP4 QUOTA extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + The QUOTA extension of the Internet Message Access Protocol [IMAP4] + permits administrative limits on resource usage (quotas) to be + manipulated through the IMAP protocol. + +Table of Contents + + 1. Abstract........................................... 1 + 2. Conventions Used in this Document.................. 1 + 3. Introduction and Overview.......................... 2 + 4. Commands........................................... 2 + 4.1. SETQUOTA Command................................... 2 + 4.2. GETQUOTA Command................................... 2 + 4.3. GETQUOTAROOT Command............................... 3 + 5. Responses.......................................... 3 + 5.1. QUOTA Response..................................... 3 + 5.2. QUOTAROOT Response................................. 4 + 6. Formal syntax...................................... 4 + 7. References......................................... 5 + 8. Security Considerations............................ 5 + 9. Author's Address................................... 5 + + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + + + + + + + +Myers Standards Track [Page 1] + +RFC 2087 QUOTA January 1997 + + +3. Introduction and Overview + + The QUOTA extension is present in any IMAP4 implementation which + returns "QUOTA" as one of the supported capabilities to the + CAPABILITY command. + + An IMAP4 server which supports the QUOTA capability may support + limits on any number of resources. Each resource has an atom name + and an implementation-defined interpretation which evaluates to an + integer. Examples of such resources are: + + Name Interpretation + + STORAGE Sum of messages' RFC822.SIZE, in units of 1024 octets + MESSAGE Number of messages + + + Each mailbox has zero or more implementation-defined named "quota + roots". Each quota root has zero or more resource limits. All + mailboxes that share the same named quota root share the resource + limits of the quota root. + + Quota root names do not necessarily have to match the names of + existing mailboxes. + +4. Commands + +4.1. SETQUOTA Command + + Arguments: quota root + list of resource limits + + Data: untagged responses: QUOTA + + Result: OK - setquota completed + NO - setquota error: can't set that data + BAD - command unknown or arguments invalid + + The SETQUOTA command takes the name of a mailbox quota root and a + list of resource limits. The resource limits for the named quota root + are changed to be the specified limits. Any previous resource limits + for the named quota root are discarded. + + If the named quota root did not previously exist, an implementation + may optionally create it and change the quota roots for any number of + existing mailboxes in an implementation-defined manner. + + + + + +Myers Standards Track [Page 2] + +RFC 2087 QUOTA January 1997 + + + Example: C: A001 SETQUOTA "" (STORAGE 512) + S: * QUOTA "" (STORAGE 10 512) + S: A001 OK Setquota completed + +4.2. GETQUOTA Command + + Arguments: quota root + + Data: untagged responses: QUOTA + + Result: OK - getquota completed + NO - getquota error: no such quota root, permission + denied + BAD - command unknown or arguments invalid + + The GETQUOTA command takes the name of a quota root and returns the + quota root's resource usage and limits in an untagged QUOTA response. + + Example: C: A003 GETQUOTA "" + S: * QUOTA "" (STORAGE 10 512) + S: A003 OK Getquota completed + +4.3. GETQUOTAROOT Command + + Arguments: mailbox name + + Data: untagged responses: QUOTAROOT, QUOTA + + Result: OK - getquota completed + NO - getquota error: no such mailbox, permission denied + BAD - command unknown or arguments invalid + + The GETQUOTAROOT command takes the name of a mailbox and returns the + list of quota roots for the mailbox in an untagged QUOTAROOT + response. For each listed quota root, it also returns the quota + root's resource usage and limits in an untagged QUOTA response. + + Example: C: A003 GETQUOTAROOT INBOX + S: * QUOTAROOT INBOX "" + S: * QUOTA "" (STORAGE 10 512) + S: A003 OK Getquota completed + + + + + + + + + + +Myers Standards Track [Page 3] + +RFC 2087 QUOTA January 1997 + + +5. Responses + +5.1. QUOTA Response + + Data: quota root name + list of resource names, usages, and limits + + This response occurs as a result of a GETQUOTA or GETQUOTAROOT + command. The first string is the name of the quota root for which + this quota applies. + + The name is followed by a S-expression format list of the resource + usage and limits of the quota root. The list contains zero or + more triplets. Each triplet conatins a resource name, the current + usage of the resource, and the resource limit. + + Resources not named in the list are not limited in the quota root. + Thus, an empty list means there are no administrative resource + limits in the quota root. + + Example: S: * QUOTA "" (STORAGE 10 512) + +5.2. QUOTAROOT Response + + Data: mailbox name + zero or more quota root names + + This response occurs as a result of a GETQUOTAROOT command. The + first string is the mailbox and the remaining strings are the + names of the quota roots for the mailbox. + + Example: S: * QUOTAROOT INBOX "" + S: * QUOTAROOT comp.mail.mime + +6. Formal syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in RFC 822 with one exception; the + delimiter used with the "#" construct is a single space (SP) and not + one or more commas. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + + + + + +Myers Standards Track [Page 4] + +RFC 2087 QUOTA January 1997 + + + getquota ::= "GETQUOTA" SP astring + + getquotaroot ::= "GETQUOTAROOT" SP astring + + quota_list ::= "(" #quota_resource ")" + + quota_resource ::= atom SP number SP number + + quota_response ::= "QUOTA" SP astring SP quota_list + + quotaroot_response + ::= "QUOTAROOT" SP astring *(SP astring) + + setquota ::= "SETQUOTA" SP astring SP setquota_list + + setquota_list ::= "(" 0#setquota_resource ")" + + setquota_resource ::= atom SP number + +7. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", + RFC 1730, University of Washington, December 1994. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet + Text Messages", STD 11, RFC 822. + +8. Security Considerations + + Implementors should be careful to make sure the implementation of + these commands does not violate the site's security policy. The + resource usage of other users is likely to be considered confidential + information and should not be divulged to unauthorized persons. + +9. Author's Address + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave. + Pittsburgh PA, 15213-3890 + + EMail: jgm+@cmu.edu + + + + + + + + + +Myers Standards Track [Page 5] + diff --git a/docs/rfcs/rfc2088.txt b/docs/rfcs/rfc2088.txt new file mode 100644 index 0000000..f36cc76 --- /dev/null +++ b/docs/rfcs/rfc2088.txt @@ -0,0 +1,115 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 2088 Carnegie Mellon +Cateogry: Standards Track January 1997 + + + IMAP4 non-synchronizing literals + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + The Internet Message Access Protocol [IMAP4] contains the "literal" + syntactic construct for communicating strings. When sending a + literal from client to server, IMAP4 requires the client to wait for + the server to send a command continuation request between sending the + octet count and the string data. This document specifies an + alternate form of literal which does not require this network round + trip. + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + +3. Specification + + The non-synchronizing literal is added an alternate form of literal, + and may appear in communication from client to server instead of the + IMAP4 form of literal. The IMAP4 form of literal, used in + communication from client to server, is referred to as a + synchronizing literal. + + Non-synchronizing literals may be used with any IMAP4 server + implementation which returns "LITERAL+" as one of the supported + capabilities to the CAPABILITY command. If the server does not + advertise the LITERAL+ capability, the client must use synchronizing + literals instead. + + The non-synchronizing literal is distinguished from the original + synchronizing literal by having a plus ('+') between the octet count + and the closing brace ('}'). The server does not generate a command + continuation request in response to a non-synchronizing literal, and + + + +Myers Standards Track [Page 1] + +RFC 2088 LITERAL January 1997 + + + clients are not required to wait before sending the octets of a non- + synchronizing literal. + + The protocol receiver of an IMAP4 server must check the end of every + received line for an open brace ('{') followed by an octet count, a + plus ('+'), and a close brace ('}') immediately preceeding the CRLF. + If it finds this sequence, it is the octet count of a non- + synchronizing literal and the server MUST treat the specified number + of following octets and the following line as part of the same + command. A server MAY still process commands and reject errors on a + line-by-line basis, as long as it checks for non-synchronizing + literals at the end of each line. + + Example: C: A001 LOGIN {11+} + C: FRED FOOBAR {7+} + C: fat man + S: A001 OK LOGIN completed + +4. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + literal ::= "{" number ["+"] "}" CRLF *CHAR8 + ;; Number represents the number of CHAR8 octets + +6. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", + draft-crispin-imap-base-XX.txt, University of Washington, April 1996. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text + Messages", STD 11, RFC 822. + +7. Security Considerations + + There are no known security issues with this extension. + +8. Author's Address + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave. + Pittsburgh PA, 15213-3890 + + Email: jgm+@cmu.edu + + + +Myers Standards Track [Page 2] + diff --git a/docs/rfcs/rfc2095.txt b/docs/rfcs/rfc2095.txt new file mode 100644 index 0000000..f1167f6 --- /dev/null +++ b/docs/rfcs/rfc2095.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group J. Klensin +Request for Comments: 2095 R. Catoe +Category: Standards Track P. Krumviede + MCI + January 1997 + + + IMAP/POP AUTHorize Extension for Simple Challenge/Response + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + While IMAP4 supports a number of strong authentication mechanisms as + described in RFC 1731, it lacks any mechanism that neither passes + cleartext, reusable passwords across the network nor requires either + a significant security infrastructure or that the mail server update + a mail-system-wide user authentication file on each mail access. + This specification provides a simple challenge-response + authentication protocol that is suitable for use with IMAP4. Since + it utilizes Keyed-MD5 digests and does not require that the secret be + stored in the clear on the server, it may also constitute an + improvement on APOP for POP3 use as specified in RFC 1734. + +1. Introduction + + Existing Proposed Standards specify an AUTHENTICATE mechanism for the + IMAP4 protocol [IMAP, IMAP-AUTH] and a parallel AUTH mechanism for + the POP3 protocol [POP3-AUTH]. The AUTHENTICATE mechanism is + intended to be extensible; the four methods specified in [IMAP-AUTH] + are all fairly powerful and require some security infrastructure to + support. The base POP3 specification [POP3] also contains a + lightweight challenge-response mechanism called APOP. APOP is + associated with most of the risks associated with such protocols: in + particular, it requires that both the client and server machines have + access to the shared secret in cleartext form. CRAM offers a method + for avoiding such cleartext storage while retaining the algorithmic + simplicity of APOP in using only MD5, though in a "keyed" method. + + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 1] + +RFC 2095 IMAP/POP AUTHorize Extension January 1997 + + + At present, IMAP [IMAP] lacks any facility corresponding to APOP. + The only alternative to the strong mechanisms identified in [IMAP- + AUTH] is a presumably cleartext username and password, supported + through the LOGIN command in [IMAP]. This document describes a + simple challenge-response mechanism, similar to APOP and PPP CHAP + [PPP], that can be used with IMAP (and, in principle, with POP3). + + This mechanism also has the advantage over some possible alternatives + of not requiring that the server maintain information about email + "logins" on a per-login basis. While mechanisms that do require such + per-login history records may offer enhanced security, protocols such + as IMAP, which may have several connections between a given client + and server open more or less simultaneous, may make their + implementation particularly challenging. + +2. Challenge-Response Authentication Mechanism (CRAM) + + The authentication type associated with CRAM is "CRAM-MD5". + + The data encoded in the first ready response contains an + presumptively arbitrary string of random digits, a timestamp, and the + fully-qualified primary host name of the server. The syntax of the + unencoded form must correspond to that of an RFC 822 'msg-id' + [RFC822] as described in [POP3]. + + The client makes note of the data and then responds with a string + consisting of the user name, a space, and a 'digest'. The latter is + computed by applying the keyed MD5 algorithm from [KEYED-MD5] where + the key is a shared secret and the digested text is the timestamp + (including angle-brackets). + + This shared secret is a string known only to the client and server. + The `digest' parameter itself is a 16-octet value which is sent in + hexadecimal format, using lower-case ASCII characters. + + When the server receives this client response, it verifies the digest + provided. If the digest is correct, the server should consider the + client authenticated and respond appropriately. + + Keyed MD5 is chosen for this application because of the greater + security imparted to authentication of short messages. In addition, + the use of the techniques described in [KEYED-MD5] for precomputation + of intermediate results make it possible to avoid explicit cleartext + storage of the shared secret on the server system by instead storing + the intermediate results which are known as "contexts". + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 2] + +RFC 2095 IMAP/POP AUTHorize Extension January 1997 + + + CRAM does not support a protection mechanism. + + Example: + + The examples in this document show the use of the CRAM mechanism with + the IMAP4 AUTHENTICATE command [IMAP-AUTH]. The base64 encoding of + the challenges and responses is part of the IMAP4 AUTHENTICATE + command, not part of the CRAM specification itself. + + S: * OK IMAP4 Server + C: A0001 AUTHENTICATE CRAM-MD5 + S: + PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2UucmVzdG9uLm1jaS5uZXQ+ + C: dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw + S: A0001 OK CRAM authentication successful + + In this example, the shared secret is the string + 'tanstaaftanstaaf'. Hence, the Keyed MD5 digest is produced by + calculating + + MD5((tanstaaftanstaaf XOR opad), + MD5((tanstaaftanstaaf XOR ipad), + <1896.697170952@postoffice.reston.mci.net>)) + + where ipad and opad are as defined in the keyed-MD5 Work in + Progress [KEYED-MD5] and the string shown in the challenge is the + base64 encoding of <1896.697170952@postoffice.reston.mci.net>. The + shared secret is null-padded to a length of 64 bytes. If the + shared secret is longer than 64 bytes, the MD5 digest of the + shared secret is used as a 16 byte input to the keyed MD5 + calculation. + + This produces a digest value (in hexadecimal) of + + b913a602c7eda7a495b4e6e7334d3890 + + The user name is then prepended to it, forming + + tim b913a602c7eda7a495b4e6e7334d3890 + + Which is then base64 encoded to meet the requirements of the IMAP4 + AUTHENTICATE command (or the similar POP3 AUTH command), yielding + + dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw + + + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 3] + +RFC 2095 IMAP/POP AUTHorize Extension January 1997 + + +3. References + + [CHAP] Lloyd, B., and W. Simpson, "PPP Authentication Protocols", + RFC 1334, October 1992. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanisms", + RFC 1731, Carnegie Mellon, December 1994. + + [KEYED-MD5] Krawczyk, H., "HMAC-MD5: Keyed-MD5 for Message + Authentication", Work in Progess. + + [MD5] Rivest, R., "The MD5 Message Digest Algorithm", + RFC 1321, MIT Laboratory for Computer Science, April 1992. + + [POP3] Myers, J., and M. Rose, "Post Office Protocol - Version 3", + STD 53, RFC 1939, Carnegie Mellon, May 1996. + + [POP3-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, + Carnegie Mellon, December, 1994. + +4. Security Considerations + + It is conjectured that use of the CRAM authentication mechanism + provides origin identification and replay protection for a session. + Accordingly, a server that implements both a cleartext password + command and this authentication type should not allow both methods of + access for a given user. + + While the saving, on the server, of "contexts" (see section 2) is + marginally better than saving the shared secrets in cleartext as is + required by CHAP [CHAP] and APOP [POP3], it is not sufficient to + protect the secrets if the server itself is compromised. + Consequently, servers that store the secrets or contexts must both be + protected to a level appropriate to the potential information value + in user mailboxes and identities. + + As the length of the shared secret increases, so does the difficulty + of deriving it. + + While there are now suggestions in the literature that the use of MD5 + and keyed MD5 in authentication procedures probably has a limited + effective lifetime, the technique is now widely deployed and widely + understood. It is believed that this general understanding may + assist with the rapid replacement, by CRAM-MD5, of the current uses + of permanent cleartext passwords in IMAP. This document has been + + + +Klensin, Catoe & Krumviede Standards Track [Page 4] + +RFC 2095 IMAP/POP AUTHorize Extension January 1997 + + + deliberately written to permit easy upgrading to use SHA (or whatever + alternatives emerge) when they are considered to be widely available + and adequately safe. + + Even with the use of CRAM, users are still vulnerable to active + attacks. An example of an increasingly common active attack is 'TCP + Session Hijacking' as described in CERT Advisory CA-95:01 [CERT95]. + + See section 1 above for additional discussion. + +5. Acknowledgements + + This memo borrows ideas and some text liberally from [POP3] and + [RFC-1731] and thanks are due the authors of those documents. Ran + Atkinson made a number of valuable technical and editorial + contributions to the document. + +6. Authors' Addresses + + John C. Klensin + MCI Telecommunications + 800 Boylston St, 7th floor + Boston, MA 02199 + USA + + EMail: klensin@mci.net + Phone: +1 617 960 1011 + + Randy Catoe + MCI Telecommunications + 2100 Reston Parkway + Reston, VA 22091 + USA + + EMail: randy@mci.net + Phone: +1 703 715 7366 + + Paul Krumviede + MCI Telecommunications + 2100 Reston Parkway + Reston, VA 22091 + USA + + EMail: paul@mci.net + Phone: +1 703 715 7251 + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 5] + diff --git a/docs/rfcs/rfc2177.txt b/docs/rfcs/rfc2177.txt new file mode 100644 index 0000000..c11c765 --- /dev/null +++ b/docs/rfcs/rfc2177.txt @@ -0,0 +1,227 @@ + + + + + + +Network Working Group B. Leiba +Request for Comments: 2177 IBM T.J. Watson Research Center +Category: Standards Track June 1997 + + + IMAP4 IDLE command + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + The Internet Message Access Protocol [IMAP4] requires a client to + poll the server for changes to the selected mailbox (new mail, + deletions). It's often more desirable to have the server transmit + updates to the client in real time. This allows a user to see new + mail immediately. It also helps some real-time applications based on + IMAP, which might otherwise need to poll extremely often (such as + every few seconds). (While the spec actually does allow a server to + push EXISTS responses aysynchronously, a client can't expect this + behaviour and must poll.) + + This document specifies the syntax of an IDLE command, which will + allow a client to tell the server that it's ready to accept such + real-time updates. + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as described in RFC 2060 + [IMAP4]. + +3. Specification + + IDLE Command + + Arguments: none + + Responses: continuation data will be requested; the client sends + the continuation data "DONE" to end the command + + + +Leiba Standards Track [Page 1] + +RFC 2177 IMAP4 IDLE command June 1997 + + + + Result: OK - IDLE completed after client sent "DONE" + NO - failure: the server will not allow the IDLE + command at this time + BAD - command unknown or arguments invalid + + The IDLE command may be used with any IMAP4 server implementation + that returns "IDLE" as one of the supported capabilities to the + CAPABILITY command. If the server does not advertise the IDLE + capability, the client MUST NOT use the IDLE command and must poll + for mailbox updates. In particular, the client MUST continue to be + able to accept unsolicited untagged responses to ANY command, as + specified in the base IMAP specification. + + The IDLE command is sent from the client to the server when the + client is ready to accept unsolicited mailbox update messages. The + server requests a response to the IDLE command using the continuation + ("+") response. The IDLE command remains active until the client + responds to the continuation, and as long as an IDLE command is + active, the server is now free to send untagged EXISTS, EXPUNGE, and + other messages at any time. + + The IDLE command is terminated by the receipt of a "DONE" + continuation from the client; such response satisfies the server's + continuation request. At that point, the server MAY send any + remaining queued untagged responses and then MUST immediately send + the tagged response to the IDLE command and prepare to process other + commands. As in the base specification, the processing of any new + command may cause the sending of unsolicited untagged responses, + subject to the ambiguity limitations. The client MUST NOT send a + command while the server is waiting for the DONE, since the server + will not be able to distinguish a command from a continuation. + + The server MAY consider a client inactive if it has an IDLE command + running, and if such a server has an inactivity timeout it MAY log + the client off implicitly at the end of its timeout period. Because + of that, clients using IDLE are advised to terminate the IDLE and + re-issue it at least every 29 minutes to avoid being logged off. + This still allows a client to receive immediate mailbox updates even + though it need only "poll" at half hour intervals. + + + + + + + + + + + +Leiba Standards Track [Page 2] + +RFC 2177 IMAP4 IDLE command June 1997 + + + Example: C: A001 SELECT INBOX + S: * FLAGS (Deleted Seen) + S: * 3 EXISTS + S: * 0 RECENT + S: * OK [UIDVALIDITY 1] + S: A001 OK SELECT completed + C: A002 IDLE + S: + idling + ...time passes; new mail arrives... + S: * 4 EXISTS + C: DONE + S: A002 OK IDLE terminated + ...another client expunges message 2 now... + C: A003 FETCH 4 ALL + S: * 4 FETCH (...) + S: A003 OK FETCH completed + C: A004 IDLE + S: * 2 EXPUNGE + S: * 3 EXISTS + S: + idling + ...time passes; another client expunges message 3... + S: * 3 EXPUNGE + S: * 2 EXISTS + ...time passes; new mail arrives... + S: * 3 EXISTS + C: DONE + S: A004 OK IDLE terminated + C: A005 FETCH 3 ALL + S: * 3 FETCH (...) + S: A005 OK FETCH completed + C: A006 IDLE + +4. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + command_auth ::= append / create / delete / examine / list / lsub / + rename / select / status / subscribe / unsubscribe + / idle + ;; Valid only in Authenticated or Selected state + + idle ::= "IDLE" CRLF "DONE" + + + + + + +Leiba Standards Track [Page 3] + +RFC 2177 IMAP4 IDLE command June 1997 + + +5. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + +6. Security Considerations + + There are no known security issues with this extension. + +7. Author's Address + + Barry Leiba + IBM T.J. Watson Research Center + 30 Saw Mill River Road + Hawthorne, NY 10532 + + Email: leiba@watson.ibm.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Leiba Standards Track [Page 4] + diff --git a/docs/rfcs/rfc2180.txt b/docs/rfcs/rfc2180.txt new file mode 100644 index 0000000..5760700 --- /dev/null +++ b/docs/rfcs/rfc2180.txt @@ -0,0 +1,787 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 2180 Microsoft +Category: Informational July 1997 + + + IMAP4 Multi-Accessed Mailbox Practice + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + +1. Abstract + + IMAP4[RFC-2060] is rich client/server protocol that allows a client + to access and manipulate electronic mail messages on a server. + Within the protocol framework, it is possible to have differing + results for particular client/server interactions. If a protocol does + not allow for this, it is often unduly restrictive. + + For example, when multiple clients are accessing a mailbox and one + attempts to delete the mailbox, an IMAP4 server may choose to + implement a solution based upon server architectural constraints or + individual preference. + + With this flexibility comes greater client responsibility. It is not + sufficient for a client to be written based upon the behavior of a + particular IMAP server. Rather the client must be based upon the + behavior allowed by the protocol. + + By documenting common IMAP4 server practice for the case of + simultaneous client access to a mailbox, we hope to ensure the widest + amount of inter-operation between IMAP4 clients and servers. + + The behavior described in this document reflects the practice of some + existing servers or behavior that the consensus of the IMAP mailing + list has deemed to be reasonable. The behavior described within this + document is believed to be [RFC-2060] compliant. However, this + document is not meant to define IMAP4 compliance, nor is it an + exhaustive list of valid IMAP4 behavior. [RFC-2060] must always be + consulted to determine IMAP4 compliance, especially for server + behavior not described within this document. + + + + + + + + +Gahrns Informational [Page 1] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +2. Conventions used in this document + + In examples,"C1:", "C2:" and "C3:" indicate lines sent by 3 different + clients (client #1, client #2 and client #3) that are connected to a + server. "S1:", "S2:" and "S3:" indicated lines sent by the server to + client #1, client #2 and client #3 respectively. + + A shared mailbox, is a mailbox that can be used by multiple users. + + A multi-accessed mailbox, is a mailbox that has multiple clients + simultaneously accessing it. + + A client is said to have accessed a mailbox after a successful SELECT + or EXAMINE command. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + + +3. Deletion/Renaming of a multi-accessed mailbox + + If an external agent or multiple clients are accessing a mailbox, + care must be taken when handling the deletion or renaming of the + mailbox. Following are some strategies an IMAP server may choose to + use when dealing with this situation. + + +3.1. The server MAY fail the DELETE/RENAME command of a multi-accessed + mailbox + + In some cases, this behavior may not be practical. For example, if a + large number of clients are accessing a shared mailbox, the window in + which no clients have the mailbox accessed may be small or non- + existent, effectively rendering the mailbox undeletable or + unrenamable. + + Example: + + + + C1: A001 DELETE FOO + S1: A001 NO Mailbox FOO is in use by another user. + + + + + + + +Gahrns Informational [Page 2] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +3.2. The server MAY allow the DELETE command of a multi-accessed + mailbox, but keep the information in the mailbox available for + those clients that currently have access to the mailbox. + + When all clients have finished accessing the mailbox, it is + permanently removed. For clients that do not already have access to + the mailbox, the 'ghosted' mailbox would not be available. For + example, it would not be returned to these clients in a subsequent + LIST or LSUB command and would not be a valid mailbox argument to any + other IMAP command until the reference count of clients accessing the + mailbox reached 0. + + In some cases, this behavior may not be desirable. For example if + someone created a mailbox with offensive or sensitive information, + one might prefer to have the mailbox deleted and all access to the + information contained within removed immediately, rather than + continuing to allow access until the client closes the mailbox. + + Furthermore, this behavior, may prevent 'recycling' of the same + mailbox name until all clients have finished accessing the original + mailbox. + + Example: + + + + C1: A001 DELETE FOO + S1: A001 OK Mailbox FOO is deleted. + + + + C2: B001 STORE 1 +FLAGS (\Seen) + S2: * 1 FETCH FLAGS (\Seen) + S2: B001 OK STORE completed + + + + C3: C001 STATUS FOO (MESSAGES) + S3: C001 NO Mailbox does not exist + + + + C3: C002 CREATE FOO + S3: C002 NO Mailbox FOO is still in use. Try again later. + + + + +Gahrns Informational [Page 3] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + + + C2: B002 CLOSE + S2: B002 OK CLOSE Completed + + + + C3: C003 CREATE FOO + S3: C003 OK CREATE Completed + + +3.3. The server MAY allow the DELETE/RENAME of a multi-accessed + mailbox, but disconnect all other clients who have the mailbox + accessed by sending a untagged BYE response. + + A server may often choose to disconnect clients in the DELETE case, + but may choose to implement a "friendlier" method for the RENAME + case. + + Example: + + + + C1: A002 DELETE FOO + S1: A002 OK DELETE completed. + + + S2: * BYE Mailbox FOO has been deleted. + + +3.4. The server MAY allow the RENAME of a multi-accessed mailbox by + simply changing the name attribute on the mailbox. + + Other clients that have access to the mailbox can continue issuing + commands such as FETCH that do not reference the mailbox name. + Clients would discover the renaming the next time they referred to + the old mailbox name. Some servers MAY choose to include the + [NEWNAME] response code in their tagged NO response to a command that + contained the old mailbox name, as a hint to the client that the + operation can succeed if the command is issued with the new mailbox + name. + + + + + + + +Gahrns Informational [Page 4] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + Example: + + + + C1: A001 RENAME FOO BAR + S1: A001 OK RENAME completed. + + + + C2: B001 FETCH 2:4 (FLAGS) + S2: * 2 FETCH . . . + S2: * 3 FETCH . . . + S2: * 4 FETCH . . . + S2: B001 OK FETCH completed + + + + C2: B002 APPEND FOO {300} C2: Date: Mon, 7 Feb 1994 + 21:52:25 0800 (PST) C2: . . . S2: B002 NO [NEWNAME FOO + BAR] Mailbox has been renamed + + +4. Expunging of messages on a multi-accessed mailbox + + If an external agent or multiple clients are accessing a mailbox, + care must be taken when handling the EXPUNGE of messages. Other + clients accessing the mailbox may be in the midst of issuing a + command that depends upon message sequence numbers. Because an + EXPUNGE response can not be sent while responding to a FETCH, STORE + or SEARCH command, it is not possible to immediately notify the + client of the EXPUNGE. This can result in ambiguity if the client + issues a FETCH, STORE or SEARCH operation on a message that has been + EXPUNGED. + + +4.1. Fetching of expunged messages + + Following are some strategies an IMAP server may choose to use when + dealing with a FETCH command on expunged messages. + + + + + + + + + +Gahrns Informational [Page 5] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + Consider the following scenario: + + - Client #1 and Client #2 have mailbox FOO selected. + - There are 7 messages in the mailbox. + - Messages 4:7 are marked for deletion. + - Client #1 issues an EXPUNGE, to expunge messages 4:7 + + +4.1.1. The server MAY allow the EXPUNGE of a multi-accessed mailbox but + keep the messages available to satisfy subsequent FETCH commands + until it is able to send an EXPUNGE response to each client. + + In some cases, the behavior of keeping "ghosted" messages may not be + desirable. For example if a message contained offensive or sensitive + information, one might prefer to instantaneously remove all access to + the information, regardless of whether another client is in the midst + of accessing it. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 FETCH 4:7 RFC822 + S2: * 4 FETCH RFC822 . . . (RFC822 info returned) + S2: * 5 FETCH RFC822 . . . (RFC822 info returned) + S2: * 6 FETCH RFC822 . . . (RFC822 info returned) + S2: * 7 FETCH RFC822 . . . (RFC822 info returned) + S2: B001 OK FETCH Completed + + + + C2: B002 NOOP + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 3 EXISTS + S2: B002 OK NOOP Complete + + + + C2: B003 FETCH 4:7 RFC822 + S2: B003 NO Messages 4:7 are no longer available. + + + + + + +Gahrns Informational [Page 6] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.1.2 The server MAY allow the EXPUNGE of a multi-accessed mailbox, + and on subsequent FETCH commands return FETCH responses only for + non-expunged messages and a tagged NO. + + After receiving a tagged NO FETCH response, the client SHOULD issue a + NOOP command so that it will be informed of any pending EXPUNGE + responses. The client may then either reissue the failed FETCH + command, or by examining the EXPUNGE response from the NOOP and the + FETCH response from the FETCH, determine that the FETCH failed + because of pending expunges. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 FETCH 3:5 ENVELOPE + S2: * 3 FETCH ENVELOPE . . . (ENVELOPE info returned) + S2: B001 NO Some of the requested messages no longer exist + + + + C2: B002 NOOP + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 3 EXISTS + S2: B002 OK NOOP Completed. + + + + + + + + + + + + + + + + + + +Gahrns Informational [Page 7] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.1.3 The server MAY allow the EXPUNGE of a multi-accessed mailbox, and + on subsequent FETCH commands return the usual FETCH responses for + non-expunged messages, "NIL FETCH Responses" for expunged + messages, and a tagged OK response. + + If all of the messages in the subsequent FETCH command have been + expunged, the server SHOULD return only a tagged NO. In this case, + the client SHOULD issue a NOOP command so that it will be informed of + any pending EXPUNGE responses. The client may then either reissue + the failed FETCH command, or by examining the EXPUNGE response from + the NOOP, determine that the FETCH failed because of pending + expunges. + + "NIL FETCH responses" are a representation of empty data as + appropriate for the FETCH argument specified. + + Example: + + * 1 FETCH (ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL)) + * 1 FETCH (FLAGS ()) + * 1 FETCH (INTERNALDATE "00-Jan-0000 00:00:00 +0000") + * 1 FETCH (RFC822 "") + * 1 FETCH (RFC822.HEADER "") + * 1 FETCH (RFC822.TEXT "") + * 1 FETCH (RFC822.SIZE 0) + * 1 FETCH (BODY ("TEXT" "PLAIN" NIL NIL NIL "7BIT" 0 0) + * 1 FETCH (BODYSTRUCTURE ("TEXT" "PLAIN" NIL NIL NIL "7BIT" 0 0) + * 1 FETCH (BODY[
] "") + * 1 FETCH (BODY[
] "") + + In some cases, a client may not be able to distinguish between "NIL + FETCH responses" received because a message was expunged and those + received because the data actually was NIL. For example, a * 5 + FETCH (FLAGS ()) response could be received if no flags were set on + message 5, or because message 5 was expunged. In a case of potential + ambiguity, the client SHOULD issue a command such as NOOP to force + the sending of the EXPUNGE responses to resolve any ambiguity. + + Example: (Building upon the scenario outlined in 4.1.) + + + + + + + + + + +Gahrns Informational [Page 8] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + C2: B002 FETCH 3:5 ENVELOPE + S2: * 3 FETCH ENVELOPE . . . (ENVELOPE info returned) + S2: * 4 FETCH ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL + NIL NIL) + S2: * 5 FETCH ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL + NIL NIL) + S2: B002 OK FETCH Completed + + + + C2: B002 FETCH 4:7 ENVELOPE + S2: B002 NO Messages 4:7 have been expunged. + + +4.1.4 To avoid the situation altogether, the server MAY fail the + EXPUNGE of a multi-accessed mailbox + + In some cases, this behavior may not be practical. For example, if a + large number of clients are accessing a shared mailbox, the window in + which no clients have the mailbox accessed may be small or non- + existent, effectively rendering the message unexpungeable. + + +4.2. Storing of expunged messages + + Following are some strategies an IMAP server may choose to use when + dealing with a STORE command on expunged messages. + + +4.2.1 If the ".SILENT" suffix is used, and the STORE completed + successfully for all the non-expunged messages, the server SHOULD + return a tagged OK. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 STORE 1:7 +FLAGS.SILENT (\SEEN) + S2: B001 OK + + + + + + + + + +Gahrns Informational [Page 9] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.2.2. If the ".SILENT" suffix is not used, and only expunged messages + are referenced, the server SHOULD return only a tagged NO. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 STORE 5:7 +FLAGS (\SEEN) + S2: B001 NO Messages have been expunged + + +4.2.3. If the ".SILENT" suffix is not used, and a mixture of expunged + and non-expunged messages are referenced, the server MAY set the + flags and return a FETCH response for the non-expunged messages + along with a tagged NO. + + After receiving a tagged NO STORE response, the client SHOULD issue a + NOOP command so that it will be informed of any pending EXPUNGE + responses. The client may then either reissue the failed STORE + command, or by examining the EXPUNGE responses from the NOOP and + FETCH responses from the STORE, determine that the STORE failed + because of pending expunges. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 STORE 1:7 +FLAGS (\SEEN) + S2: * FETCH 1 FLAGS (\SEEN) + S2: * FETCH 2 FLAGS (\SEEN) + S2: * FETCH 3 FLAGS (\SEEN) + S2: B001 NO Some of the messages no longer exist. + + C2: B002 NOOP + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 3 EXISTS + S2: B002 OK NOOP Completed. + + + + + + + + +Gahrns Informational [Page 10] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.2.4. If the ".SILENT" suffix is not used, and a mixture of expunged + and non-expunged messages are referenced, the server MAY return + an untagged NO and not set any flags. + + After receiving a tagged NO STORE response, the client SHOULD issue a + NOOP command so that it will be informed of any pending EXPUNGE + responses. The client would then re-issue the STORE command after + updating its message list per any EXPUNGE response. + + If a large number of clients are accessing a shared mailbox, the + window in which there are no pending expunges may be small or non- + existent, effectively disallowing a client from setting the flags on + all messages at once. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 STORE 1:7 +FLAGS (\SEEN) + S2: B001 NO Some of the messages no longer exist. + + + + C2: B002 NOOP + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 3 EXISTS + S2: B002 OK NOOP Completed. + + + + C2: B003 STORE 1:3 +FLAGS (\SEEN) S2: * FETCH 1 FLAGS + (\SEEN) S2: * FETCH 2 FLAGS (\SEEN) S2: * FETCH 3 FLAGS + (\SEEN) S2: B003 OK STORE Completed + + +4.3. Searching of expunged messages + + A server MAY simply not return a search response for messages that + have been expunged and it has not been able to inform the client + about. If a client was expecting a particular message to be returned + in a search result, and it was not, the client SHOULD issue a NOOP + command to see if the message was expunged by another client. + + + + +Gahrns Informational [Page 11] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.4 Copying of expunged messages + + COPY is the only IMAP4 sequence number command that is safe to allow + an EXPUNGE response on. This is because a client is not permitted to + cascade several COPY commands together. A client is required to wait + and confirm that the copy worked before issuing another one. + +4.4.1 The server MAY disallow the COPY of messages in a multi-access + mailbox that contains expunged messages. + + Pending EXPUNGE response(s) MUST be returned to the COPY command. + + Example: + + C: A001 COPY 2,4,6,8 FRED + S: * 4 EXPUNGE + S: A001 NO COPY rejected, because some of the requested + messages were expunged + + Note: Non of the above messages are copied because if a COPY command + is unsuccessful, the server MUST restore the destination mailbox to + its state before the COPY attempt. + + +4.4.2 The server MAY allow the COPY of messages in a multi-access + mailbox that contains expunged messages. + + Pending EXPUNGE response(s) MUST be returned to the COPY command. + Messages that are copied are messages corresponding to sequence + numbers before any EXPUNGE response. + + Example: + + C: A001 COPY 2,4,6,8 FRED + S: * 3 EXPUNGE + S: A001 OK COPY completed + + In the above example, the messages that are copied to FRED are + messages 2,4,6,8 at the start of the COPY command. These are + equivalent to messages 2,3,5,7 at the end of the COPY command. The + EXPUNGE response can't take place until after the messages from the + COPY command are identified (because of the "no expunge while no + commands in progress" rule). + + + + + + + + +Gahrns Informational [Page 12] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + Example: + + C: A001 COPY 2,4,6,8 FRED + S: * 4 EXPUNGE + S: A001 OK COPY completed + + In the above example, message 4 was copied before it was expunged, + and MUST appear in the destination mailbox FRED. + + +5. Security Considerations + + This document describes behavior of servers that use the IMAP4 + protocol, and as such, has the same security considerations as + described in [RFC-2060]. + + In particular, some described server behavior does not allow for the + immediate deletion of information when a mailbox is accessed by + multiple clients. This may be a consideration when dealing with + sensitive information where immediate deletion would be preferred. + + +6. References + + [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, Harvard University, March 1997. + + +7. Acknowledgments + + This document is the result of discussions on the IMAP4 mailing list + and is meant to reflect consensus of this group. In particular, + Raymond Cheng, Mark Crispin, Jim Evans, Erik Forsberg, Steve Hole, + Mark Keasling, Barry Leiba, Syd Logan, John Mani, Pat Moran, Larry + Osterman, Chris Newman, Bart Schaefer, Vladimir Vulovic, and Jack De + Winter were active participants in this discussion or made + suggestions to this document. + + + + + + + + + + + +Gahrns Informational [Page 13] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +8. Author's Address + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98072 + + Phone: (206) 936-9833 + EMail: mikega@microsoft.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gahrns Informational [Page 14] + diff --git a/docs/rfcs/rfc2192.txt b/docs/rfcs/rfc2192.txt new file mode 100644 index 0000000..1b5a1d4 --- /dev/null +++ b/docs/rfcs/rfc2192.txt @@ -0,0 +1,899 @@ + + + + + + +Network Working Group C. Newman +Request for Comments: 2192 Innosoft +Category: Standards Track September 1997 + + + IMAP URL Scheme + + +Status of this memo + + This document specifies an Internet standards track protocol for + the Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is + unlimited. + + +Abstract + + IMAP [IMAP4] is a rich protocol for accessing remote message + stores. It provides an ideal mechanism for accessing public + mailing list archives as well as private and shared message stores. + This document defines a URL scheme for referencing objects on an + IMAP server. + + +1. Conventions used in this document + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in "Key words for + use in RFCs to Indicate Requirement Levels" [KEYWORDS]. + + +2. IMAP scheme + + The IMAP URL scheme is used to designate IMAP servers, mailboxes, + messages, MIME bodies [MIME], and search programs on Internet hosts + accessible using the IMAP protocol. + + The IMAP URL follows the common Internet scheme syntax as defined + in RFC 1738 [BASIC-URL] except that clear text passwords are not + permitted. If : is omitted, the port defaults to 143. + + + + + + + + +Newman Standards Track [Page 1] + +RFC 2192 IMAP URL Scheme September 1997 + + + An IMAP URL takes one of the following forms: + + imap:/// + imap:///;TYPE= + imap:///[uidvalidity][?] + imap:///[uidvalidity][isection] + + The first form is used to refer to an IMAP server, the second form + refers to a list of mailboxes, the third form refers to the + contents of a mailbox or a set of messages resulting from a search, + and the final form refers to a specific message or message part. + Note that the syntax here is informal. The authoritative formal + syntax for IMAP URLs is defined in section 11. + + +3. IMAP User Name and Authentication Mechanism + + A user name and/or authentication mechanism may be supplied. They + are used in the "LOGIN" or "AUTHENTICATE" commands after making the + connection to the IMAP server. If no user name or authentication + mechanism is supplied, the user name "anonymous" is used with the + "LOGIN" command and the password is supplied as the Internet e-mail + address of the end user accessing the resource. If the URL doesn't + supply a user name, the program interpreting the IMAP URL SHOULD + request one from the user if necessary. + + An authentication mechanism can be expressed by adding + ";AUTH=" to the end of the user name. When such an + is indicated, the client SHOULD request appropriate + credentials from that mechanism and use the "AUTHENTICATE" command + instead of the "LOGIN" command. If no user name is specified, one + SHOULD be obtained from the mechanism or requested from the user as + appropriate. + + The string ";AUTH=*" indicates that the client SHOULD select an + appropriate authentication mechanism. It MAY use any mechanism + listed in the CAPABILITY command or use an out of band security + service resulting in a PREAUTH connection. If no user name is + specified and no appropriate authentication mechanisms are + available, the client SHOULD fall back to anonymous login as + described above. This allows a URL which grants read-write access + to authorized users, and read-only anonymous access to other users. + + If a user name is included with no authentication mechanism, then + ";AUTH=*" is assumed. + + + + + + +Newman Standards Track [Page 2] + +RFC 2192 IMAP URL Scheme September 1997 + + + Since URLs can easily come from untrusted sources, care must be + taken when resolving a URL which requires or requests any sort of + authentication. If authentication credentials are supplied to the + wrong server, it may compromise the security of the user's account. + The program resolving the URL should make sure it meets at least + one of the following criteria in this case: + + (1) The URL comes from a trusted source, such as a referral server + which the client has validated and trusts according to site policy. + Note that user entry of the URL may or may not count as a trusted + source, depending on the experience level of the user and site + policy. + (2) Explicit local site policy permits the client to connect to the + server in the URL. For example, if the client knows the site + domain name, site policy may dictate that any hostname ending in + that domain is trusted. + (3) The user confirms that connecting to that domain name with the + specified credentials and/or mechanism is permitted. + (4) A mechanism is used which validates the server before passing + potentially compromising client credentials. + (5) An authentication mechanism is used which will not reveal + information to the server which could be used to compromise future + connections. + + URLs which do not include a user name must be treated with extra + care, since they are more likely to compromise the user's primary + account. A URL containing ";AUTH=*" must also be treated with + extra care since it might fall back on a weaker security mechanism. + Finally, clients are discouraged from using a plain text password + as a fallback with ";AUTH=*" unless the connection has strong + encryption (e.g. a key length of greater than 56 bits). + + A program interpreting IMAP URLs MAY cache open connections to an + IMAP server for later re-use. If a URL contains a user name, only + connections authenticated as that user may be re-used. If a URL + does not contain a user name or authentication mechanism, then only + an anonymous connection may be re-used. If a URL contains an + authentication mechanism without a user name, then any non- + anonymous connection may be re-used. + + Note that if unsafe or reserved characters such as " " or ";" are + present in the user name or authentication mechanism, they MUST be + encoded as described in RFC 1738 [BASIC-URL]. + + + + + + + + +Newman Standards Track [Page 3] + +RFC 2192 IMAP URL Scheme September 1997 + + +4. IMAP server + + An IMAP URL referring to an IMAP server has the following form: + + imap:/// + + A program interpreting this URL would issue the standard set of + commands it uses to present a view of the contents of an IMAP + server. This is likely to be semanticly equivalent to one of the + following URLs: + + imap:///;TYPE=LIST + imap:///;TYPE=LSUB + + The program interpreting this URL SHOULD use the LSUB form if it + supports mailbox subscriptions. + + +5. Lists of mailboxes + + An IMAP URL referring to a list of mailboxes has the following + form: + + imap:///;TYPE= + + The may be either "LIST" or "LSUB", and is case + insensitive. The field ";TYPE=" MUST be included. + + The is any argument suitable for the + list_mailbox field of the IMAP [IMAP4] LIST or LSUB commands. The + field may be omitted, in which case the program + interpreting the IMAP URL may use "*" or "%" as the + . The program SHOULD use "%" if it supports a + hierarchical view, otherwise it SHOULD use "*". + + Note that if unsafe or reserved characters such as " " or "%" are + present in they MUST be encoded as described in + RFC 1738 [BASIC-URL]. If the character "/" is present in + enc_list_mailbox, it SHOULD NOT be encoded. + + +6. Lists of messages + + An IMAP URL referring to a list of messages has the following form: + + imap:///[uidvalidity][?] + + + + + +Newman Standards Track [Page 4] + +RFC 2192 IMAP URL Scheme September 1997 + + + The field is used as the argument to the IMAP4 + "SELECT" command. Note that if unsafe or reserved characters such + as " ", ";", or "?" are present in they MUST be + encoded as described in RFC 1738 [BASIC-URL]. If the character "/" + is present in enc_mailbox, it SHOULD NOT be encoded. + + The [uidvalidity] field is optional. If it is present, it MUST be + the argument to the IMAP4 UIDVALIDITY status response at the time + the URL was created. This SHOULD be used by the program + interpreting the IMAP URL to determine if the URL is stale. + + The [?] field is optional. If it is not present, the + contents of the mailbox SHOULD be presented by the program + interpreting the URL. If it is present, it SHOULD be used as the + arguments following an IMAP4 SEARCH command with unsafe characters + such as " " (which are likely to be present in the ) + encoded as described in RFC 1738 [BASIC-URL]. + + +7. A specific message or message part + + An IMAP URL referring to a specific message or message part has the + following form: + + imap:///[uidvalidity][isection] + + The and [uidvalidity] are as defined above. + + If [uidvalidity] is present in this form, it SHOULD be used by the + program interpreting the URL to determine if the URL is stale. + + The refers to an IMAP4 message UID, and SHOULD be used as + the argument to the IMAP4 "UID FETCH" command. + + The [isection] field is optional. If not present, the URL refers + to the entire Internet message as returned by the IMAP command "UID + FETCH BODY.PEEK[]". If present, the URL refers to the object + returned by a "UID FETCH BODY.PEEK[
]" command. The + type of the object may be determined with a "UID FETCH + BODYSTRUCTURE" command and locating the appropriate part in the + resulting BODYSTRUCTURE. Note that unsafe characters in [isection] + MUST be encoded as described in [BASIC-URL]. + + + + + + + + + +Newman Standards Track [Page 5] + +RFC 2192 IMAP URL Scheme September 1997 + + +8. Relative IMAP URLs + + Relative IMAP URLs are permitted and are resolved according to the + rules defined in RFC 1808 [REL-URL] with one exception. In IMAP + URLs, parameters are treated as part of the normal path with + respect to relative URL resolution. This is believed to be the + behavior of the installed base and is likely to be documented in a + future revision of the relative URL specification. + + The following observations are also important: + + The grammar element is considered part of the user name for + purposes of resolving relative IMAP URLs. This means that unless a + new login/server specification is included in the relative URL, the + authentication mechanism is inherited from a base IMAP URL. + + URLs always use "/" as the hierarchy delimiter for the purpose of + resolving paths in relative URLs. IMAP4 permits the use of any + hierarchy delimiter in mailbox names. For this reason, relative + mailbox paths will only work if the mailbox uses "/" as the + hierarchy delimiter. Relative URLs may be used on mailboxes which + use other delimiters, but in that case, the entire mailbox name + MUST be specified in the relative URL or inherited as a whole from + the base URL. + + The base URL for a list of mailboxes or messages which was referred + to by an IMAP URL is always the referring IMAP URL itself. The + base URL for a message or message part which was referred to by an + IMAP URL may be more complicated to determine. The program + interpreting the relative URL will have to check the headers of the + MIME entity and any enclosing MIME entities in order to locate the + "Content-Base" and "Content-Location" headers. These headers are + used to determine the base URL as defined in [HTTP]. For example, + if the referring IMAP URL contains a "/;SECTION=1.2" parameter, + then the MIME headers for section 1.2, for section 1, and for the + enclosing message itself SHOULD be checked in that order for + "Content-Base" or "Content-Location" headers. + + +9. Multinational Considerations + + IMAP4 [IMAP4] section 5.1.3 includes a convention for encoding + non-US-ASCII characters in IMAP mailbox names. Because this + convention is private to IMAP, it is necessary to convert IMAP's + encoding to one that can be more easily interpreted by a URL + display program. For this reason, IMAP's modified UTF-7 encoding + for mailboxes MUST be converted to UTF-8 [UTF8]. Since 8-bit + characters are not permitted in URLs, the UTF-8 characters are + + + +Newman Standards Track [Page 6] + +RFC 2192 IMAP URL Scheme September 1997 + + + encoded as required by the URL specification [BASIC-URL]. Sample + code is included in Appendix A to demonstrate this conversion. + + +10. Examples + + The following examples demonstrate how an IMAP4 client program + might translate various IMAP4 URLs into a series of IMAP4 commands. + Commands sent from the client to the server are prefixed with "C:", + and responses sent from the server to the client are prefixed with + "S:". + + The URL: + + + + Results in the following client commands: + + + C: A001 LOGIN ANONYMOUS sheridan@babylon5.org + C: A002 SELECT gray-council + + C: A003 UID FETCH 20 BODY.PEEK[] + + The URL: + + + + Results in the following client commands: + + + + C: A001 LOGIN MICHAEL zipper + C: A002 LIST "" users.* + + The URL: + + + + Results in the following client commands: + + + C: A001 LOGIN ANONYMOUS bester@psycop.psicorp.org + C: A002 SELECT ~peter/&ZeVnLIqe-/&U,BTFw- + + + + + + +Newman Standards Track [Page 7] + +RFC 2192 IMAP URL Scheme September 1997 + + + The URL: + + + + Results in the following client commands: + + + C: A001 AUTHENTICATE KERBEROS_V4 + + C: A002 SELECT gray-council + C: A003 UID FETCH 20 BODY.PEEK[1.2] + + If the following relative URL is located in that body part: + + <;section=1.4> + + This could result in the following client commands: + + C: A004 UID FETCH 20 (BODY.PEEK[1.2.MIME] + BODY.PEEK[1.MIME] + BODY.PEEK[HEADER.FIELDS (Content-Base Content-Location)]) + + C: A005 UID FETCH 20 BODY.PEEK[1.4] + + The URL: + + + + Could result in the following: + + + C: A001 CAPABILITY + S: * CAPABILITY IMAP4rev1 AUTH=GSSAPI + S: A001 OK + C: A002 AUTHENTICATE GSSAPI + + S: A002 OK user lennier authenticated + C: A003 SELECT "gray council" + ... + C: A004 SEARCH SUBJECT shadows + S: * SEARCH 8 10 13 14 15 16 + S: A004 OK SEARCH completed + C: A005 FETCH 8,10,13:16 ALL + ... + + + + + +Newman Standards Track [Page 8] + +RFC 2192 IMAP URL Scheme September 1997 + + + NOTE: In this final example, the client has implementation + dependent choices. The authentication mechanism could be anything, + including PREAUTH. And the final FETCH command could fetch more or + less information about the messages, depending on what it wishes to + display to the user. + + +11. Security Considerations + + Security considerations discussed in the IMAP specification [IMAP4] + and the URL specification [BASIC-URL] are relevant. Security + considerations related to authenticated URLs are discussed in + section 3 of this document. + + Many email clients store the plain text password for later use + after logging into an IMAP server. Such clients MUST NOT use a + stored password in response to an IMAP URL without explicit + permission from the user to supply that password to the specified + host name. + + +12. ABNF for IMAP URL scheme + + This uses ABNF as defined in RFC 822 [IMAIL]. Terminals from the + BNF for IMAP [IMAP4] and URLs [BASIC-URL] are also used. Strings + are not case sensitive and free insertion of linear-white-space is + not permitted. + + achar = uchar / "&" / "=" / "~" + ; see [BASIC-URL] for "uchar" definition + + bchar = achar / ":" / "@" / "/" + + enc_auth_type = 1*achar + ; encoded version of [IMAP-AUTH] "auth_type" + + enc_list_mailbox = 1*bchar + ; encoded version of [IMAP4] "list_mailbox" + + enc_mailbox = 1*bchar + ; encoded version of [IMAP4] "mailbox" + + enc_search = 1*bchar + ; encoded version of search_program below + + enc_section = 1*bchar + ; encoded version of section below + + + + +Newman Standards Track [Page 9] + +RFC 2192 IMAP URL Scheme September 1997 + + + enc_user = 1*achar + ; encoded version of [IMAP4] "userid" + + imapurl = "imap://" iserver "/" [ icommand ] + + iauth = ";AUTH=" ( "*" / enc_auth_type ) + + icommand = imailboxlist / imessagelist / imessagepart + + imailboxlist = [enc_list_mailbox] ";TYPE=" list_type + + imessagelist = enc_mailbox [ "?" enc_search ] [uidvalidity] + + imessagepart = enc_mailbox [uidvalidity] iuid [isection] + + isection = "/;SECTION=" enc_section + + iserver = [iuserauth "@"] hostport + ; See [BASIC-URL] for "hostport" definition + + iuid = "/;UID=" nz_number + ; See [IMAP4] for "nz_number" definition + + iuserauth = enc_user [iauth] / [enc_user] iauth + + list_type = "LIST" / "LSUB" + + search_program = ["CHARSET" SPACE astring SPACE] + search_key *(SPACE search_key) + ; IMAP4 literals may not be used + ; See [IMAP4] for "astring" and "search_key" + + section = section_text / (nz_number *["." nz_number] + ["." (section_text / "MIME")]) + ; See [IMAP4] for "section_text" and "nz_number" + + uidvalidity = ";UIDVALIDITY=" nz_number + ; See [IMAP4] for "nz_number" definition + +13. References + + [BASIC-URL] Berners-Lee, Masinter, McCahill, "Uniform Resource + Locators (URL)", RFC 1738, CERN, Xerox Corporation, University of + Minnesota, December 1994. + + + + + + + +Newman Standards Track [Page 10] + +RFC 2192 IMAP URL Scheme September 1997 + + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + + + [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanism", RFC 1731, + Carnegie-Mellon University, December 1994. + + + + [HTTP] Fielding, Gettys, Mogul, Frystyk, Berners-Lee, "Hypertext + Transfer Protocol -- HTTP/1.1", RFC 2068, UC Irvine, DEC, MIT/LCS, + January 1997. + + + + [IMAIL] Crocker, "Standard for the Format of ARPA Internet Text + Messages", STD 11, RFC 822, University of Delaware, August 1982. + + + + [KEYWORDS] Bradner, "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, Harvard University, March 1997. + + + + [MIME] Freed, N., Borenstein, N., "Multipurpose Internet Mail + Extensions", RFC 2045, Innosoft, First Virtual, November 1996. + + + + [REL-URL] Fielding, "Relative Uniform Resource Locators", RFC 1808, + UC Irvine, June 1995. + + + + [UTF8] Yergeau, F. "UTF-8, a transformation format of Unicode and + ISO 10646", RFC 2044, Alis Technologies, October 1996. + + + +14. Author's Address + + Chris Newman + Innosoft International, Inc. + 1050 Lakes Drive + West Covina, CA 91790 USA + EMail: chris.newman@innosoft.com + + + +Newman Standards Track [Page 11] + +RFC 2192 IMAP URL Scheme September 1997 + + +Appendix A. Sample code + +Here is sample C source code to convert between URL paths and IMAP +mailbox names, taking into account mapping between IMAP's modified UTF-7 +[IMAP4] and hex-encoded UTF-8 which is more appropriate for URLs. This +code has not been rigorously tested nor does it necessarily behave +reasonably with invalid input, but it should serve as a useful example. +This code just converts the mailbox portion of the URL and does not deal +with parameters, query or server components of the URL. + +#include +#include + +/* hexadecimal lookup table */ +static char hex[] = "0123456789ABCDEF"; + +/* URL unsafe printable characters */ +static char urlunsafe[] = " \"#%&+:;<=>?@[\\]^`{|}"; + +/* UTF7 modified base64 alphabet */ +static char base64chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; +#define UNDEFINED 64 + +/* UTF16 definitions */ +#define UTF16MASK 0x03FFUL +#define UTF16SHIFT 10 +#define UTF16BASE 0x10000UL +#define UTF16HIGHSTART 0xD800UL +#define UTF16HIGHEND 0xDBFFUL +#define UTF16LOSTART 0xDC00UL +#define UTF16LOEND 0xDFFFUL + +/* Convert an IMAP mailbox to a URL path + * dst needs to have roughly 4 times the storage space of src + * Hex encoding can triple the size of the input + * UTF-7 can be slightly denser than UTF-8 + * (worst case: 8 octets UTF-7 becomes 9 octets UTF-8) + */ +void MailboxToURL(char *dst, char *src) +{ + unsigned char c, i, bitcount; + unsigned long ucs4, utf16, bitbuf; + unsigned char base64[256], utf8[6]; + + + + + + + +Newman Standards Track [Page 12] + +RFC 2192 IMAP URL Scheme September 1997 + + + /* initialize modified base64 decoding table */ + memset(base64, UNDEFINED, sizeof (base64)); + for (i = 0; i < sizeof (base64chars); ++i) { + base64[base64chars[i]] = i; + } + + /* loop until end of string */ + while (*src != '\0') { + c = *src++; + /* deal with literal characters and &- */ + if (c != '&' || *src == '-') { + if (c < ' ' || c > '~' || strchr(urlunsafe, c) != NULL) { + /* hex encode if necessary */ + dst[0] = '%'; + dst[1] = hex[c >> 4]; + dst[2] = hex[c & 0x0f]; + dst += 3; + } else { + /* encode literally */ + *dst++ = c; + } + /* skip over the '-' if this is an &- sequence */ + if (c == '&') ++src; + } else { + /* convert modified UTF-7 -> UTF-16 -> UCS-4 -> UTF-8 -> HEX */ + bitbuf = 0; + bitcount = 0; + ucs4 = 0; + while ((c = base64[(unsigned char) *src]) != UNDEFINED) { + ++src; + bitbuf = (bitbuf << 6) | c; + bitcount += 6; + /* enough bits for a UTF-16 character? */ + if (bitcount >= 16) { + bitcount -= 16; + utf16 = (bitcount ? bitbuf >> bitcount + : bitbuf) & 0xffff; + /* convert UTF16 to UCS4 */ + if + (utf16 >= UTF16HIGHSTART && utf16 <= UTF16HIGHEND) { + ucs4 = (utf16 - UTF16HIGHSTART) << UTF16SHIFT; + continue; + } else if + (utf16 >= UTF16LOSTART && utf16 <= UTF16LOEND) { + ucs4 += utf16 - UTF16LOSTART + UTF16BASE; + } else { + ucs4 = utf16; + } + + + +Newman Standards Track [Page 13] + +RFC 2192 IMAP URL Scheme September 1997 + + + /* convert UTF-16 range of UCS4 to UTF-8 */ + if (ucs4 <= 0x7fUL) { + utf8[0] = ucs4; + i = 1; + } else if (ucs4 <= 0x7ffUL) { + utf8[0] = 0xc0 | (ucs4 >> 6); + utf8[1] = 0x80 | (ucs4 & 0x3f); + i = 2; + } else if (ucs4 <= 0xffffUL) { + utf8[0] = 0xe0 | (ucs4 >> 12); + utf8[1] = 0x80 | ((ucs4 >> 6) & 0x3f); + utf8[2] = 0x80 | (ucs4 & 0x3f); + i = 3; + } else { + utf8[0] = 0xf0 | (ucs4 >> 18); + utf8[1] = 0x80 | ((ucs4 >> 12) & 0x3f); + utf8[2] = 0x80 | ((ucs4 >> 6) & 0x3f); + utf8[3] = 0x80 | (ucs4 & 0x3f); + i = 4; + } + /* convert utf8 to hex */ + for (c = 0; c < i; ++c) { + dst[0] = '%'; + dst[1] = hex[utf8[c] >> 4]; + dst[2] = hex[utf8[c] & 0x0f]; + dst += 3; + } + } + } + /* skip over trailing '-' in modified UTF-7 encoding */ + if (*src == '-') ++src; + } + } + /* terminate destination string */ + *dst = '\0'; +} + +/* Convert hex coded UTF-8 URL path to modified UTF-7 IMAP mailbox + * dst should be about twice the length of src to deal with non-hex + * coded URLs + */ +void URLtoMailbox(char *dst, char *src) +{ + unsigned int utf8pos, utf8total, i, c, utf7mode, bitstogo, utf16flag; + unsigned long ucs4, bitbuf; + unsigned char hextab[256]; + + /* initialize hex lookup table */ + + + +Newman Standards Track [Page 14] + +RFC 2192 IMAP URL Scheme September 1997 + + + memset(hextab, 0, sizeof (hextab)); + for (i = 0; i < sizeof (hex); ++i) { + hextab[hex[i]] = i; + if (isupper(hex[i])) hextab[tolower(hex[i])] = i; + } + + utf7mode = 0; + utf8total = 0; + bitstogo = 0; + while ((c = *src) != '\0') { + ++src; + /* undo hex-encoding */ + if (c == '%' && src[0] != '\0' && src[1] != '\0') { + c = (hextab[src[0]] << 4) | hextab[src[1]]; + src += 2; + } + /* normal character? */ + if (c >= ' ' && c <= '~') { + /* switch out of UTF-7 mode */ + if (utf7mode) { + if (bitstogo) { + *dst++ = base64chars[(bitbuf << (6 - bitstogo)) & 0x3F]; + } + *dst++ = '-'; + utf7mode = 0; + } + *dst++ = c; + /* encode '&' as '&-' */ + if (c == '&') { + *dst++ = '-'; + } + continue; + } + /* switch to UTF-7 mode */ + if (!utf7mode) { + *dst++ = '&'; + utf7mode = 1; + } + /* Encode US-ASCII characters as themselves */ + if (c < 0x80) { + ucs4 = c; + utf8total = 1; + } else if (utf8total) { + /* save UTF8 bits into UCS4 */ + ucs4 = (ucs4 << 6) | (c & 0x3FUL); + if (++utf8pos < utf8total) { + continue; + } + + + +Newman Standards Track [Page 15] + +RFC 2192 IMAP URL Scheme September 1997 + + + } else { + utf8pos = 1; + if (c < 0xE0) { + utf8total = 2; + ucs4 = c & 0x1F; + } else if (c < 0xF0) { + utf8total = 3; + ucs4 = c & 0x0F; + } else { + /* NOTE: can't convert UTF8 sequences longer than 4 */ + utf8total = 4; + ucs4 = c & 0x03; + } + continue; + } + /* loop to split ucs4 into two utf16 chars if necessary */ + utf8total = 0; + do { + if (ucs4 >= UTF16BASE) { + ucs4 -= UTF16BASE; + bitbuf = (bitbuf << 16) | ((ucs4 >> UTF16SHIFT) + + UTF16HIGHSTART); + ucs4 = (ucs4 & UTF16MASK) + UTF16LOSTART; + utf16flag = 1; + } else { + bitbuf = (bitbuf << 16) | ucs4; + utf16flag = 0; + } + bitstogo += 16; + /* spew out base64 */ + while (bitstogo >= 6) { + bitstogo -= 6; + *dst++ = base64chars[(bitstogo ? (bitbuf >> bitstogo) + : bitbuf) + & 0x3F]; + } + } while (utf16flag); + } + /* if in UTF-7 mode, finish in ASCII */ + if (utf7mode) { + if (bitstogo) { + *dst++ = base64chars[(bitbuf << (6 - bitstogo)) & 0x3F]; + } + *dst++ = '-'; + } + /* tie off string */ + *dst = '\0'; +} + + + +Newman Standards Track [Page 16] + diff --git a/docs/rfcs/rfc2193.txt b/docs/rfcs/rfc2193.txt new file mode 100644 index 0000000..2fec58d --- /dev/null +++ b/docs/rfcs/rfc2193.txt @@ -0,0 +1,507 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 2193 Microsoft +Category: Standards Track September 1997 + + + IMAP4 Mailbox Referrals + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + When dealing with large amounts of users, messages and geographically + dispersed IMAP4 [RFC-2060] servers, it is often desirable to + distribute messages amongst different servers within an organization. + For example an administrator may choose to store user's personal + mailboxes on a local IMAP4 server, while storing shared mailboxes + remotely on another server. This type of configuration is common + when it is uneconomical to store all data centrally due to limited + bandwidth or disk resources. + + Mailbox referrals allow clients to seamlessly access mailboxes that + are distributed across several IMAP4 servers. + + A referral mechanism can provide efficiencies over the alternative + "proxy method", in which the local IMAP4 server contacts the remote + server on behalf of the client, and then transfers the data from the + remote server to itself, and then on to the client. The referral + mechanism's direct client connection to the remote server is often a + more efficient use of bandwidth, and does not require the local + server to impersonate the client when authenticating to the remote + server. + +2. Conventions used in this document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + A home server, is an IMAP4 server that contains the user's inbox. + + A remote mailbox is a mailbox that is not hosted on the user's home + server. + + + + +Gahrns Standards Track [Page 1] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + A remote server is a server that contains remote mailboxes. + + A shared mailbox, is a mailbox that multiple users have access to. + + An IMAP mailbox referral is when the server directs the client to + another IMAP mailbox. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + +3. Introduction and Overview + + IMAP4 servers that support this extension MUST list the keyword + MAILBOX-REFERRALS in their CAPABILITY response. No client action is + needed to invoke the MAILBOX-REFERRALS capability in a server. + + A MAILBOX-REFERRALS capable IMAP4 server MUST NOT return referrals + that result in a referrals loop. + + A referral response consists of a tagged NO response and a REFERRAL + response code. The REFERRAL response code MUST contain as an + argument a one or more valid URLs separated by a space as defined in + [RFC-1738]. If a server replies with multiple URLs for a particular + object, they MUST all be of the same type. In this case, the URL MUST + be an IMAP URL as defined in [RFC-2192]. A client that supports the + REFERRALS extension MUST be prepared for a URL of any type, but it + need only be able to process IMAP URLs. + + A server MAY respond with multiple IMAP mailbox referrals if there is + more than one replica of the mailbox. This allows the implementation + of a load balancing or failover scheme. How a server keeps multiple + replicas of a mailbox in sync is not addressed by this document. + + If the server has a preferred order in which the client should + attempt to access the URLs, the preferred URL SHOULD be listed in the + first, with the remaining URLs presented in descending order of + preference. If multiple referrals are given for a mailbox, a server + should be aware that there are synchronization issues for a client if + the UIDVALIDITY of the referred mailboxes are different. + + An IMAP mailbox referral may be given in response to an IMAP command + that specifies a mailbox as an argument. + + + + + + + + +Gahrns Standards Track [Page 2] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + Example: + + A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/REMOTE]Remote Mailbox + + NOTE: user;AUTH=* is specified as required by [RFC-2192] to avoid a + client falling back to anonymous login. + + Remote mailboxes and their inferiors, that are accessible only via + referrals SHOULD NOT appear in LIST and LSUB responses issued against + the user's home server. They MUST appear in RLIST and RLSUB + responses issued against the user's home server. Hierarchy referrals, + in which a client would be required to connect to the remote server + to issue a LIST to discover the inferiors of a mailbox are not + addressed in this document. + + For example, if shared mailboxes were only accessible via referrals + on a remote server, a RLIST "" "#SHARED/%" command would return the + same response if issued against the user's home server or the remote + server. + + Note: Mailboxes that are available on the user's home server do not + need to be available on the remote server. In addition, there may be + additional mailboxes available on the remote server, but they will + not accessible to the client via referrals unless they appear in the + LIST response to the RLIST command against the user's home server. + + A MAILBOX-REFERRALS capable client will issue the RLIST and RLSUB + commands in lieu of LIST and LSUB. The RLIST and RLSUB commands + behave identically to their LIST and LSUB counterparts, except remote + mailboxes are returned in addition to local mailboxes in the LIST and + LSUB responses. This avoids displaying to a non MAILBOX-REFERRALS + enabled client inaccessible remote mailboxes. + +4.1. SELECT, EXAMINE, DELETE, SUBSCRIBE, UNSUBSCRIBE, STATUS and APPEND + Referrals + + An IMAP4 server MAY respond to the SELECT, EXAMINE, DELETE, + SUBSCRIBE, UNSUBSCRIBE, STATUS or APPEND command with one or more + IMAP mailbox referrals to indicate to the client that the mailbox is + hosted on a remote server. + + When a client processes an IMAP mailbox referral, it will open a new + connection or use an existing connection to the remote server so that + it is able to issue the commands necessary to process the remote + mailbox. + + + + + + +Gahrns Standards Track [Page 3] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + Example: + + C: A001 DELETE "SHARED/FOO" + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] + Remote mailbox. Try SERVER2. + + + + S: * OK IMAP4rev1 server ready + C: B001 AUTHENTICATE KERBEROS_V4 + + S: B001 OK user is authenticated + + C: B002 DELETE "SHARED/FOO" + S: B002 OK DELETE completed + + Example: + + C: A001 SELECT REMOTE + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/REMOTE + IMAP://user;AUTH=*@SERVER3/REMOTE] Remote mailbox. + Try SERVER2 or SERVER3. + + + + S: * OK IMAP4rev1 server ready + C: B001 AUTHENTICATE KERBEROS_V4 + + S: B001 OK user is authenticated + + C: B002 SELECT REMOTE + S: * 12 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 10] Message 10 is first unseen + S: * OK [UIDVALIDITY 123456789] + S: * FLAGS (Answered Flagged Deleted Seen Draft) + S: * OK [PERMANENTFLAGS (Answered Deleted Seen ] + S: B002 OK [READ-WRITE] Selected completed + + C: B003 FETCH 10:12 RFC822 + S: * 10 FETCH . . . + S: * 11 FETCH . . . + S: * 12 FETCH . . . + S: B003 OK FETCH Completed + + + + +Gahrns Standards Track [Page 4] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + + + C: B004 LOGOUT + S: * BYE IMAP4rev1 server logging out + S: B004 OK LOGOUT Completed + + + + C: A002 SELECT INBOX + S: * 16 EXISTS + S: * 2 RECENT + S: * OK [UNSEEN 10] Message 10 is first unseen + S: * OK [UIDVALIDITY 123456789] + S: * FLAGS (Answered Flagged Deleted Seen Draft) + S: * OK [PERMANENTFLAGS (Answered Deleted Seen ] + S: A002 OK [READ-WRITE] Selected completed + +4.2. CREATE Referrals + + An IMAP4 server MAY respond to the CREATE command with one or more + IMAP mailbox referrals, if it wishes to direct the client to issue + the CREATE against another server. The server can employ any means, + such as examining the hierarchy of the specified mailbox name, in + determining which server the mailbox should be created on. + + Example: + + C: A001 CREATE "SHARED/FOO" + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] + Mailbox should be created on remote server + + Alternatively, because a home server is required to maintain a + listing of referred remote mailboxes, a server MAY allow the creation + of a mailbox that will ultimately reside on a remote server against + the home server, and provide referrals on subsequent commands that + manipulate the mailbox. + + Example: + + C: A001 CREATE "SHARED/FOO" + S: A001 OK CREATE succeeded + + C: A002 SELECT "SHARED/FOO" + S: A002 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] + Remote mailbox. Try SERVER2 + + + + + +Gahrns Standards Track [Page 5] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + +4.3. RENAME Referrals + + An IMAP4 server MAY respond to the RENAME command with one or more + pairs of IMAP mailbox referrals. In each pair of IMAP mailbox + referrals, the first one is an URL to the existing mailbox name and + the second is an URL to the requested new mailbox name. + + If within an IMAP mailbox referral pair, the existing and new mailbox + URLs are on different servers, the remote servers are unable to + perform the RENAME operation. To achieve the same behavior of + server RENAME, the client MAY issue the constituent CREATE, FETCH, + APPEND, and DELETE commands against both servers. + + If within an IMAP mailbox referral pair, the existing and new mailbox + URLs are on the same server it is an indication that the currently + connected server is unable to perform the operation. The client can + simply re-issue the RENAME command on the remote server. + + Example: + + C: A001 RENAME FOO BAR + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER1/FOO + IMAP://user;AUTH=*@SERVER2/BAR] Unable to rename mailbox + across servers + + Since the existing and new mailbox names are on different servers, + the client would be required to make a connection to both servers and + issue the constituent commands require to achieve the RENAME. + + Example: + + C: A001 RENAME FOO BAR + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/FOO + IMAP://user;AUTH=*@SERVER2/BAR] Unable to rename mailbox + located on SERVER2 + + Since both the existing and new mailbox are on the same remote + server, the client can simply make a connection to the remote server + and re-issue the RENAME command. + +4.4. COPY Referrals + + An IMAP4 server MAY respond to the COPY command with one or more IMAP + mailbox referrals. This indicates that the destination mailbox is on + a remote server. To achieve the same behavior of a server COPY, the + client MAY issue the constituent FETCH and APPEND commands against + both servers. + + + + +Gahrns Standards Track [Page 6] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + Example: + + C: A001 COPY 1 "SHARED/STUFF" + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/STUFF] + Unable to copy message(s) to SERVER2. + +5.1 RLIST command + + Arguments: reference name + mailbox name with possible wildcards + + Responses: untagged responses: LIST + + Result: OK - RLIST Completed + NO - RLIST Failure + BAD - command unknown or arguments invalid + + The RLIST command behaves identically to its LIST counterpart, except + remote mailboxes are returned in addition to local mailboxes in the + LIST responses. + +5.2 RLSUB Command + + Arguments: reference name + mailbox name with possible wildcards + + Responses: untagged responses: LSUB + + Result: OK - RLSUB Completed + NO - RLSUB Failure + BAD - command unknown or arguments invalid + + The RLSUB command behaves identically to its LSUB counterpart, except + remote mailboxes are returned in addition to local mailboxes in the + LSUB responses. + +6. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. + + list_mailbox = as defined in [RFC-2060] + + mailbox = as defined in [RFC-2060] + + mailbox_referral = SPACE "NO" SPACE + (text / text_mime2) + ; See [RFC-2060] for , text and text_mime2 definition + + + +Gahrns Standards Track [Page 7] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + referral_response_code = "[" "REFERRAL" 1*(SPACE ) "]" + ; See [RFC-1738] for definition + + rlist = "RLIST" SPACE mailbox SPACE list_mailbox + + rlsub = "RLSUB" SPACE mailbox SPACE list_mailbox + +6. Security Considerations + + The IMAP4 referral mechanism makes use of IMAP URLs, and as such, + have the same security considerations as general internet URLs [RFC- + 1738], and in particular IMAP URLs [RFC-2192]. + + With the MAILBOX-REFERRALS capability, it is potentially easier to + write a rogue server that injects a bogus referral response that + directs a user to an incorrect mailbox. Although referrals reduce + the effort to write such a server, the referral response makes + detection of the intrusion easier. + +7. References + + [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + [RFC-2192], Newman, C., "IMAP URL Scheme", RFC 2192, Innosoft, + September 1997. + + [RFC-1738], Berners-Lee, T., Masinter, L., and M. McCahill, "Uniform + Resource Locators (URL)", RFC 1738, CERN, Xerox Corporation, + University of Minnesota, December 1994. + + [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, Harvard University, March 1997. + + [ABNF], DRUMS working group, Dave Crocker Editor, "Augmented BNF for + Syntax Specifications: ABNF", Work in Progress, Internet Mail + Consortium, April 1997. + +8. Acknowledgments + + Many valuable suggestions were received from private discussions and + the IMAP4 mailing list. In particular, Raymond Cheng, Mark Crispin, + Mark Keasling, Chris Newman and Larry Osterman made significant + contributions to this document. + + + + + + + +Gahrns Standards Track [Page 8] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + +9. Author's Address + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98072 + + Phone: (206) 936-9833 + EMail: mikega@microsoft.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gahrns Standards Track [Page 9] + diff --git a/docs/rfcs/rfc2195.txt b/docs/rfcs/rfc2195.txt new file mode 100644 index 0000000..4a2725b --- /dev/null +++ b/docs/rfcs/rfc2195.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group J. Klensin +Request for Comments: 2195 R. Catoe +Category: Standards Track P. Krumviede +Obsoletes: 2095 MCI + September 1997 + + + IMAP/POP AUTHorize Extension for Simple Challenge/Response + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + While IMAP4 supports a number of strong authentication mechanisms as + described in RFC 1731, it lacks any mechanism that neither passes + cleartext, reusable passwords across the network nor requires either + a significant security infrastructure or that the mail server update + a mail-system-wide user authentication file on each mail access. + This specification provides a simple challenge-response + authentication protocol that is suitable for use with IMAP4. Since + it utilizes Keyed-MD5 digests and does not require that the secret be + stored in the clear on the server, it may also constitute an + improvement on APOP for POP3 use as specified in RFC 1734. + +1. Introduction + + Existing Proposed Standards specify an AUTHENTICATE mechanism for the + IMAP4 protocol [IMAP, IMAP-AUTH] and a parallel AUTH mechanism for + the POP3 protocol [POP3-AUTH]. The AUTHENTICATE mechanism is + intended to be extensible; the four methods specified in [IMAP-AUTH] + are all fairly powerful and require some security infrastructure to + support. The base POP3 specification [POP3] also contains a + lightweight challenge-response mechanism called APOP. APOP is + associated with most of the risks associated with such protocols: in + particular, it requires that both the client and server machines have + access to the shared secret in cleartext form. CRAM offers a method + for avoiding such cleartext storage while retaining the algorithmic + simplicity of APOP in using only MD5, though in a "keyed" method. + + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 1] + +RFC 2195 IMAP/POP AUTHorize Extension September 1997 + + + At present, IMAP [IMAP] lacks any facility corresponding to APOP. + The only alternative to the strong mechanisms identified in [IMAP- + AUTH] is a presumably cleartext username and password, supported + through the LOGIN command in [IMAP]. This document describes a + simple challenge-response mechanism, similar to APOP and PPP CHAP + [PPP], that can be used with IMAP (and, in principle, with POP3). + + This mechanism also has the advantage over some possible alternatives + of not requiring that the server maintain information about email + "logins" on a per-login basis. While mechanisms that do require such + per-login history records may offer enhanced security, protocols such + as IMAP, which may have several connections between a given client + and server open more or less simultaneous, may make their + implementation particularly challenging. + +2. Challenge-Response Authentication Mechanism (CRAM) + + The authentication type associated with CRAM is "CRAM-MD5". + + The data encoded in the first ready response contains an + presumptively arbitrary string of random digits, a timestamp, and the + fully-qualified primary host name of the server. The syntax of the + unencoded form must correspond to that of an RFC 822 'msg-id' + [RFC822] as described in [POP3]. + + The client makes note of the data and then responds with a string + consisting of the user name, a space, and a 'digest'. The latter is + computed by applying the keyed MD5 algorithm from [KEYED-MD5] where + the key is a shared secret and the digested text is the timestamp + (including angle-brackets). + + This shared secret is a string known only to the client and server. + The `digest' parameter itself is a 16-octet value which is sent in + hexadecimal format, using lower-case ASCII characters. + + When the server receives this client response, it verifies the digest + provided. If the digest is correct, the server should consider the + client authenticated and respond appropriately. + + Keyed MD5 is chosen for this application because of the greater + security imparted to authentication of short messages. In addition, + the use of the techniques described in [KEYED-MD5] for precomputation + of intermediate results make it possible to avoid explicit cleartext + storage of the shared secret on the server system by instead storing + the intermediate results which are known as "contexts". + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 2] + +RFC 2195 IMAP/POP AUTHorize Extension September 1997 + + + CRAM does not support a protection mechanism. + + Example: + + The examples in this document show the use of the CRAM mechanism with + the IMAP4 AUTHENTICATE command [IMAP-AUTH]. The base64 encoding of + the challenges and responses is part of the IMAP4 AUTHENTICATE + command, not part of the CRAM specification itself. + + S: * OK IMAP4 Server + C: A0001 AUTHENTICATE CRAM-MD5 + S: + PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2UucmVzdG9uLm1jaS5uZXQ+ + C: dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw + S: A0001 OK CRAM authentication successful + + In this example, the shared secret is the string + 'tanstaaftanstaaf'. Hence, the Keyed MD5 digest is produced by + calculating + + MD5((tanstaaftanstaaf XOR opad), + MD5((tanstaaftanstaaf XOR ipad), + <1896.697170952@postoffice.reston.mci.net>)) + + where ipad and opad are as defined in the keyed-MD5 Work in + Progress [KEYED-MD5] and the string shown in the challenge is the + base64 encoding of <1896.697170952@postoffice.reston.mci.net>. The + shared secret is null-padded to a length of 64 bytes. If the + shared secret is longer than 64 bytes, the MD5 digest of the + shared secret is used as a 16 byte input to the keyed MD5 + calculation. + + This produces a digest value (in hexadecimal) of + + b913a602c7eda7a495b4e6e7334d3890 + + The user name is then prepended to it, forming + + tim b913a602c7eda7a495b4e6e7334d3890 + + Which is then base64 encoded to meet the requirements of the IMAP4 + AUTHENTICATE command (or the similar POP3 AUTH command), yielding + + dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw + + + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 3] + +RFC 2195 IMAP/POP AUTHorize Extension September 1997 + + +3. References + + [CHAP] Lloyd, B., and W. Simpson, "PPP Authentication Protocols", + RFC 1334, October 1992. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanisms", + RFC 1731, Carnegie Mellon, December 1994. + + [KEYED-MD5] Krawczyk, Bellare, Canetti, "HMAC: Keyed-Hashing for + Message Authentication", RFC 2104, February 1997. + + [MD5] Rivest, R., "The MD5 Message Digest Algorithm", + RFC 1321, MIT Laboratory for Computer Science, April 1992. + + [POP3] Myers, J., and M. Rose, "Post Office Protocol - Version 3", + STD 53, RFC 1939, Carnegie Mellon, May 1996. + + [POP3-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, + Carnegie Mellon, December, 1994. + +4. Security Considerations + + It is conjectured that use of the CRAM authentication mechanism + provides origin identification and replay protection for a session. + Accordingly, a server that implements both a cleartext password + command and this authentication type should not allow both methods of + access for a given user. + + While the saving, on the server, of "contexts" (see section 2) is + marginally better than saving the shared secrets in cleartext as is + required by CHAP [CHAP] and APOP [POP3], it is not sufficient to + protect the secrets if the server itself is compromised. + Consequently, servers that store the secrets or contexts must both be + protected to a level appropriate to the potential information value + in user mailboxes and identities. + + As the length of the shared secret increases, so does the difficulty + of deriving it. + + While there are now suggestions in the literature that the use of MD5 + and keyed MD5 in authentication procedures probably has a limited + effective lifetime, the technique is now widely deployed and widely + understood. It is believed that this general understanding may + assist with the rapid replacement, by CRAM-MD5, of the current uses + of permanent cleartext passwords in IMAP. This document has been + + + +Klensin, Catoe & Krumviede Standards Track [Page 4] + +RFC 2195 IMAP/POP AUTHorize Extension September 1997 + + + deliberately written to permit easy upgrading to use SHA (or whatever + alternatives emerge) when they are considered to be widely available + and adequately safe. + + Even with the use of CRAM, users are still vulnerable to active + attacks. An example of an increasingly common active attack is 'TCP + Session Hijacking' as described in CERT Advisory CA-95:01 [CERT95]. + + See section 1 above for additional discussion. + +5. Acknowledgements + + This memo borrows ideas and some text liberally from [POP3] and + [RFC-1731] and thanks are due the authors of those documents. Ran + Atkinson made a number of valuable technical and editorial + contributions to the document. + +6. Authors' Addresses + + John C. Klensin + MCI Telecommunications + 800 Boylston St, 7th floor + Boston, MA 02199 + USA + + EMail: klensin@mci.net + Phone: +1 617 960 1011 + + Randy Catoe + MCI Telecommunications + 2100 Reston Parkway + Reston, VA 22091 + USA + + EMail: randy@mci.net + Phone: +1 703 715 7366 + + Paul Krumviede + MCI Telecommunications + 2100 Reston Parkway + Reston, VA 22091 + USA + + EMail: paul@mci.net + Phone: +1 703 715 7251 + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 5] + diff --git a/docs/rfcs/rfc2221.txt b/docs/rfcs/rfc2221.txt new file mode 100644 index 0000000..81d0062 --- /dev/null +++ b/docs/rfcs/rfc2221.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 2221 Microsoft +Category: Standards Track October 1997 + + + IMAP4 Login Referrals + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1997). All Rights Reserved. + +1. Abstract + + When dealing with large amounts of users and many IMAP4 [RFC-2060] + servers, it is often necessary to move users from one IMAP4 server to + another. For example, hardware failures or organizational changes + may dictate such a move. + + Login referrals allow clients to transparently connect to an + alternate IMAP4 server, if their home IMAP4 server has changed. + + A referral mechanism can provide efficiencies over the alternative + 'proxy method', in which the local IMAP4 server contacts the remote + server on behalf of the client, and then transfers the data from the + remote server to itself, and then on to the client. The referral + mechanism's direct client connection to the remote server is often a + more efficient use of bandwidth, and does not require the local + server to impersonate the client when authenticating to the remote + server. + +2. Conventions used in this document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + A home server, is an IMAP4 server that contains the user's inbox. + + A remote server is a server that contains remote mailboxes. + + + + + +Gahrns Standards Track [Page 1] + +RFC 2221 IMAP4 Login Referrals October 1997 + + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + +3. Introduction and Overview + + IMAP4 servers that support this extension MUST list the keyword + LOGIN-REFERRALS in their CAPABILITY response. No client action is + needed to invoke the LOGIN-REFERRALS capability in a server. + + A LOGIN-REFERRALS capable IMAP4 server SHOULD NOT return a referral + to a server that will return a referral. A client MUST NOT follow + more than 10 levels of referral without consulting the user. + + A LOGIN-REFERRALS response code MUST contain as an argument a valid + IMAP server URL as defined in [IMAP-URL]. + + A home server referral consists of either a tagged NO or OK, or an + untagged BYE response that contains a LOGIN-REFERRALS response code. + + Example: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/] Remote Server + + NOTE: user;AUTH=* is specified as required by [IMAP-URL] to avoid a + client falling back to anonymous login. + +4. Home Server Referrals + + A home server referral may be returned in response to an AUTHENTICATE + or LOGIN command, or it may appear in the connection startup banner. + If a server returns a home server referral in a tagged NO response, + that server does not contain any mailboxes that are accessible to the + user. If a server returns a home server referral in a tagged OK + response, it indicates that the user's personal mailboxes are + elsewhere, but the server contains public mailboxes which are + readable by the user. After receiving a home server referral, the + client can not make any assumptions as to whether this was a + permanent or temporary move of the user. + +4.1. LOGIN and AUTHENTICATE Referrals + + An IMAP4 server MAY respond to a LOGIN or AUTHENTICATE command with a + home server referral if it wishes to direct the user to another IMAP4 + server. + + Example: C: A001 LOGIN MIKE PASSWORD + S: A001 NO [REFERRAL IMAP://MIKE@SERVER2/] Specified user + is invalid on this server. Try SERVER2. + + + + +Gahrns Standards Track [Page 2] + +RFC 2221 IMAP4 Login Referrals October 1997 + + + Example: C: A001 LOGIN MATTHEW PASSWORD + S: A001 OK [REFERRAL IMAP://MATTHEW@SERVER2/] Specified + user's personal mailboxes located on Server2, but + public mailboxes are available. + + Example: C: A001 AUTHENTICATE GSSAPI + + S: A001 NO [REFERRAL IMAP://user;AUTH=GSSAPI@SERVER2/] + Specified user is invalid on this server. Try + SERVER2. + +4.2. BYE at connection startup referral + + An IMAP4 server MAY respond with an untagged BYE and a REFERRAL + response code that contains an IMAP URL to a home server if it is not + willing to accept connections and wishes to direct the client to + another IMAP4 server. + + Example: S: * BYE [REFERRAL IMAP://user;AUTH=*@SERVER2/] Server not + accepting connections. Try SERVER2 + +5. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. + + This amends the "resp_text_code" element of the IMAP4 grammar + described in [RFC-2060] + + resp_text_code =/ "REFERRAL" SPACE + ; See [IMAP-URL] for definition of + ; See [RFC-2060] for base definition of resp_text_code + +6. Security Considerations + + The IMAP4 login referral mechanism makes use of IMAP URLs, and as + such, have the same security considerations as general internet URLs + [RFC-1738], and in particular IMAP URLs [IMAP-URL]. + + A server MUST NOT give a login referral if authentication for that + user fails. This is to avoid revealing information about the user's + account to an unauthorized user. + + With the LOGIN-REFERRALS capability, it is potentially easier to + write a rogue 'password catching' server that collects login data and + then refers the client to their actual IMAP4 server. Although + referrals reduce the effort to write such a server, the referral + response makes detection of the intrusion easier. + + + +Gahrns Standards Track [Page 3] + +RFC 2221 IMAP4 Login Referrals October 1997 + + +7. References + + [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [IMAP-URL], Newman, C., "IMAP URL Scheme", RFC 2192, Innosoft, + September 1997. + + [RFC-1738], Berners-Lee, T., Masinter, L. and M. McCahill, "Uniform + Resource Locators (URL)", RFC 1738, December 1994. + + [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, March 1997. + + [ABNF], DRUMS working group, Dave Crocker Editor, "Augmented BNF for + Syntax Specifications: ABNF", Work in Progress. + +8. Acknowledgments + + Many valuable suggestions were received from private discussions and + the IMAP4 mailing list. In particular, Raymond Cheng, Mark Crispin, + Mark Keasling Chris Newman and Larry Osterman made significant + contributions to this document. + +9. Author's Address + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98072 + + Phone: (206) 936-9833 + EMail: mikega@microsoft.com + + + + + + + + + + + + + + + + + + +Gahrns Standards Track [Page 4] + +RFC 2221 IMAP4 Login Referrals October 1997 + + +10. Full Copyright Statement + + Copyright (C) The Internet Society (1997). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implmentation may be prepared, copied, published + andand distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE." + + + + + + + + + + + + + + + + + + + + + + + + +Gahrns Standards Track [Page 5] + diff --git a/docs/rfcs/rfc2244.txt b/docs/rfcs/rfc2244.txt new file mode 100644 index 0000000..ecf9492 --- /dev/null +++ b/docs/rfcs/rfc2244.txt @@ -0,0 +1,4035 @@ + + + + + + +Network Working Group C. Newman +Request for Comments: 2244 Innosoft +Category: Standards Track J. G. Myers + Netscape + November 1997 + + + ACAP -- Application Configuration Access Protocol + + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society 1997. All Rights Reserved. + +Abstract + + The Application Configuration Access Protocol (ACAP) is designed to + support remote storage and access of program option, configuration + and preference information. The data store model is designed to + allow a client relatively simple access to interesting data, to allow + new information to be easily added without server re-configuration, + and to promote the use of both standardized data and custom or + proprietary data. Key features include "inheritance" which can be + used to manage default values for configuration settings and access + control lists which allow interesting personal information to be + shared and group information to be restricted. + + + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page i] + +RFC 2244 ACAP November 1997 + + + + + + Table of Contents + + + +Status of this Memo ............................................... i +Copyright Notice .................................................. i +Abstract .......................................................... i +ACAP Protocol Specification ....................................... 1 +1. Introduction ............................................. 1 +1.1. Conventions Used in this Document ........................ 1 +1.2. ACAP Data Model .......................................... 1 +1.3. ACAP Design Goals ........................................ 1 +1.4. Validation ............................................... 2 +1.5. Definitions .............................................. 2 +1.6. ACAP Command Overview .................................... 4 +2. Protocol Framework ....................................... 4 +2.1. Link Level ............................................... 4 +2.2. Commands and Responses ................................... 4 +2.2.1. Client Protocol Sender and Server Protocol Receiver ...... 4 +2.2.2. Server Protocol Sender and Client Protocol Receiver ...... 5 +2.3. Server States ............................................ 6 +2.3.1. Non-Authenticated State .................................. 6 +2.3.2. Authenticated State ...................................... 6 +2.3.3. Logout State ............................................. 6 +2.4. Operational Considerations ............................... 7 +2.4.1. Untagged Status Updates .................................. 7 +2.4.2. Response when No Command in Progress ..................... 7 +2.4.3. Auto-logout Timer ........................................ 7 +2.4.4. Multiple Commands in Progress ............................ 8 +2.5. Server Command Continuation Request ...................... 8 +2.6. Data Formats ............................................. 8 +2.6.1. Atom ..................................................... 9 +2.6.2. Number ................................................... 9 +2.6.3. String ................................................... 9 +2.6.3.1. 8-bit and Binary Strings ................................. 10 +2.6.4. Parenthesized List ....................................... 10 +2.6.5. NIL ...................................................... 10 +3. Protocol Elements ........................................ 10 +3.1. Entries and Attributes ................................... 10 +3.1.1. Predefined Attributes .................................... 11 +3.1.2. Attribute Metadata ....................................... 12 +3.2. ACAP URL scheme .......................................... 13 +3.2.1. ACAP URL User Name and Authentication Mechanism .......... 13 +3.2.2. Relative ACAP URLs ....................................... 14 +3.3. Contexts ................................................. 14 + + + +Newman & Myers Standards Track [Page ii] + +RFC 2244 ACAP November 1997 + + +3.4. Comparators .............................................. 15 +3.5. Access Control Lists (ACLs) .............................. 17 +3.6. Server Response Codes .................................... 18 +4. Namespace Conventions .................................... 21 +4.1. Dataset Namespace ........................................ 21 +4.2. Attribute Namespace ...................................... 21 +4.3. Formal Syntax for Dataset and Attribute Namespace ........ 22 +5. Dataset Management ....................................... 23 +5.1. Dataset Inheritance ...................................... 23 +5.2. Dataset Attributes ....................................... 24 +5.3. Dataset Creation ......................................... 25 +5.4. Dataset Class Capabilities ............................... 25 +5.5. Dataset Quotas ........................................... 26 +6. Command and Response Specifications ...................... 26 +6.1. Initial Connection ....................................... 26 +6.1.1. ACAP Untagged Response ................................... 26 +6.2. Any State ................................................ 27 +6.2.1. NOOP Command ............................................. 27 +6.2.2. LANG Command ............................................. 28 +6.2.3. LANG Intermediate Response ............................... 28 +6.2.4. LOGOUT Command ........................................... 29 +6.2.5. OK Response .............................................. 29 +6.2.6. NO Response .............................................. 29 +6.2.7. BAD Response ............................................. 30 +6.2.8. BYE Untagged Response .................................... 30 +6.2.9. ALERT Untagged Response .................................. 31 +6.3. Non-Authenticated State .................................. 31 +6.3.1. AUTHENTICATE Command ..................................... 31 +6.4. Searching ................................................ 33 +6.4.1. SEARCH Command ........................................... 33 +6.4.2. ENTRY Intermediate Response .............................. 37 +6.4.3. MODTIME Intermediate Response ............................ 38 +6.4.4. REFER Intermediate Response .............................. 38 +6.4.5. Search Examples .......................................... 38 +6.5. Contexts ................................................. 39 +6.5.1. FREECONTEXT Command ...................................... 39 +6.5.2. UPDATECONTEXT Command .................................... 40 +6.5.3. ADDTO Untagged Response .................................. 40 +6.5.4. REMOVEFROM Untagged Response ............................. 41 +6.5.5. CHANGE Untagged Response ................................. 41 +6.5.6. MODTIME Untagged Response ................................ 42 +6.6. Dataset modification ..................................... 42 +6.6.1. STORE Command ............................................ 42 +6.6.2. DELETEDSINCE Command ..................................... 45 +6.6.3. DELETED Intermediate Response ............................ 45 +6.7. Access Control List Commands ............................. 45 +6.7.1. SETACL Command ........................................... 46 +6.7.2. DELETEACL Command ........................................ 46 + + + +Newman & Myers Standards Track [Page iii] + +RFC 2244 ACAP November 1997 + + +6.7.3. MYRIGHTS Command ......................................... 47 +6.7.4. MYRIGHTS Intermediate Response ........................... 47 +6.7.5. LISTRIGHTS Command ....................................... 47 +6.7.6. LISTRIGHTS Intermediate Response ......................... 48 +6.8. Quotas ................................................... 48 +6.8.1. GETQUOTA Command ......................................... 48 +6.8.3. QUOTA Untagged Response .................................. 49 +6.9. Extensions ............................................... 49 +7. Registration Procedures .................................. 49 +7.1. ACAP Capabilities ........................................ 50 +7.2. ACAP Response Codes ...................................... 50 +7.3. Dataset Classes .......................................... 51 +7.4. Vendor Subtree ........................................... 51 +8. Formal Syntax ............................................ 52 +9. Multi-lingual Considerations ............................. 61 +10. Security Considerations .................................. 62 +11. Acknowledgments .......................................... 63 +12. Authors' Addresses ....................................... 63 +Appendices ........................................................ 64 +A. References ............................................... 64 +B. ACAP Keyword Index ....................................... 66 +C. Full Copyright Statement + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page iv] + RFC 2244 ACAP November 1997 + + +ACAP Protocol Specification + +1. Introduction + +1.1. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If such lines are wrapped without a new "C:" or + "S:" label, then the wrapping is for editorial clarity and is not + part of the command. + + The key words "REQUIRED", "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", + and "MAY" in this document are to be interpreted as described in "Key + words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. + +1.2. ACAP Data Model + + An ACAP server exports a hierarchical tree of entries. Each level of + the tree is called a dataset, and each dataset is made up of a list + of entries. Each entry has a unique name and may contain any number + of named attributes. Each attribute within an entry may be single + valued or multi-valued and may have associated metadata to assist + access and interpretation of the value. + + The rules with which a client interprets the data within a portion of + ACAP's tree of entries are called a dataset class. + +1.3. ACAP Design Goals + + ACAP's primary purpose is to allow users access to their + configuration data from multiple network-connected computers. Users + can then sit down in front of any network-connected computer, run any + ACAP-enabled application and have access to their own configuration + data. Because it is hoped that many applications will become ACAP- + enabled, client simplicity was preferred to server or protocol + simplicity whenever reasonable. + + ACAP is designed to be easily manageable. For this reason, it + includes "inheritance" which allows one dataset to inherit default + attributes from another dataset. In addition, access control lists + are included to permit delegation of management and quotas are + included to control storage. Finally, an ACAP server which is + conformant to this base specification should be able to support most + dataset classes defined in the future without requiring a server + reconfiguration or upgrade. + + + + + + +Newman & Myers Standards Track [Page 1] + +RFC 2244 ACAP November 1997 + + + ACAP is designed to operate well with a client that only has + intermittent access to an ACAP server. For this reason, each entry + has a server maintained modification time so that the client may + detect changes. In addition, the client may ask the server for a + list of entries which have been removed since it last accessed the + server. + + ACAP presumes that a dataset may be potentially large and/or the + client's network connection may be slow, and thus offers server + sorting, selective fetching and change notification for entries + within a dataset. + + As required for most Internet protocols, security, scalability and + internationalization were important design goals. + + Given these design goals, an attempt was made to keep ACAP as simple + as possible. It is a traditional Internet text based protocol which + massively simplifies protocol debugging. It was designed based on + the successful IMAP [IMAP4] protocol framework, with a few + refinements. + +1.4. Validation + + By default, any value may be stored in any attribute for which the + user has appropriate permission and quota. This rule is necessary to + allow the addition of new simple dataset classes without + reconfiguring or upgrading the server. + + In some cases, such as when the value has special meaning to the + server, it is useful to have the server enforce validation by + returning the INVALID response code to a STORE command. These cases + MUST be explicitly identified in the dataset class specification + which SHOULD include specific fixed rules for validation. Since a + given ACAP server may be unaware of any particular dataset class + specification, clients MUST NOT depend on the presence of enforced + validation on the server. + +1.5. Definitions + + + access control list (ACL) + A set of identifier, rights pairs associated with an object. An + ACL is used to determine which operations a user is permitted to + perform on that object. See section 3.5. + + attribute + A named value within an entry. See section 3.1. + + + + +Newman & Myers Standards Track [Page 2] + +RFC 2244 ACAP November 1997 + + + comparator + A named function which can be used to perform one or more of + three comparison operations: ordering, equality and substring + matching. See section 3.4. + + context + An ordered subset of entries in a dataset, created by a SEARCH + command with a MAKECONTEXT modifier. See section 3.3. + + dataset + One level of hierarchy in ACAP's tree of entries. + + dataset class specification + The rules which allow a client to interpret the data within a + portion of ACAP's tree of entries. + + entry + A set of attributes with a unique entry name. See section 3.1. + + metadata + Information describing an attribute, its value and any access + controls associated with that attribute. See section 3.1.2. + + NIL This represents the non-existence of a particular data item. + + NUL A control character encoded as 0 in US-ASCII [US-ASCII]. + + octet + An 8-bit value. On most modern computer systems, an octet is + one byte. + + SASL Simple Authentication and Security Layer [SASL]. + + UTC Universal Coordinated Time as maintained by the Bureau + International des Poids et Mesures (BIPM). + + UTF-8 + An 8-bit transformation format of the Universal Character Set + [UTF8]. Note that an incompatible change was made to the coded + character set referenced by [UTF8], so for the purpose of this + document, UTF-8 refers to the UTF-8 encoding as defined by + version 2.0 of Unicode [UNICODE-2], or ISO 10646 [ISO-10646] + including amendments one through seven. + + + + + + + + +Newman & Myers Standards Track [Page 3] + +RFC 2244 ACAP November 1997 + + +1.6. ACAP Command Overview + + The AUTHENTICATE, NOOP, LANG and LOGOUT commands provide basic + protocol services. The SEARCH command is used to select, sort, fetch + and monitor changes to attribute values and metadata. The + UPDATECONTEXT and FREECONTEXT commands are also used to assist in + monitoring changes in attribute values and metadata. The STORE + command is used to add, modify and delete entries and attributes. + The DELETEDSINCE command is used to assist a client in + re-synchronizing a cache with the server. The GETQUOTA, SETACL, + DELETEACL, LISTRIGHTS and MYRIGHTS commands are used to examine + storage quotas and examine or modify access permissions. + +2. Protocol Framework + +2.1. Link Level + + The ACAP protocol assumes a reliable data stream such as provided by + TCP. When TCP is used, an ACAP server listens on port 674. + +2.2. Commands and Responses + + An ACAP session consists of the establishment of a client/server + connection, an initial greeting from the server, and client/server + interactions. These client/server interactions consist of a client + command, server data, and a server completion result. + + ACAP is a text-based line-oriented protocol. In general, + interactions transmitted by clients and servers are in the form of + lines; that is, sequences of characters that end with a CRLF. The + protocol receiver of an ACAP client or server is either reading a + line, or is reading a sequence of octets with a known count (a + literal) followed by a line. Both clients and servers must be + capable of handling lines of arbitrary length. + +2.2.1. Client Protocol Sender and Server Protocol Receiver + + The client command begins an operation. Each client command is + prefixed with a identifier (an alphanumeric string of no more than 32 + characters, e.g., A0001, A0002, etc.) called a "tag". A different + tag SHOULD be generated by the client for each command. + + There are two cases in which a line from the client does not + represent a complete command. In one case, a command argument is + quoted with an octet count (see the description of literal in section + 2.6.3); in the other case, the command arguments require server + + + + + +Newman & Myers Standards Track [Page 4] + +RFC 2244 ACAP November 1997 + + + feedback (see the AUTHENTICATE command). In some of these cases, the + server sends a command continuation request if it is ready for the + next part of the command. This response is prefixed with the token + "+". + + Note: If, instead, the server detected an error in a + command, it sends a BAD completion response with tag + matching the command (as described below) to reject the + command and prevent the client from sending any more of the + command. + + It is also possible for the server to send a completion or + intermediate response for some other command (if multiple + commands are in progress), or untagged data. In either + case, the command continuation request is still pending; + the client takes the appropriate action for the response, + and reads another response from the server. + + The ACAP server reads a command line from the client, parses the + command and its arguments, and transmits server data and a server + command completion result. + +2.2.2. Server Protocol Sender and Client Protocol Receiver + + Data transmitted by the server to the client come in four forms: + command continuation requests, command completion results, + intermediate responses, and untagged responses. + + A command continuation request is prefixed with the token "+". + + A command completion result indicates the success or failure of the + operation. It is tagged with the same tag as the client command + which began the operation. Thus, if more than one command is in + progress, the tag in a server completion response identifies the + command to which the response applies. There are three possible + server completion responses: OK (indicating success), NO (indicating + failure), or BAD (indicating protocol error such as unrecognized + command or command syntax error). + + An intermediate response returns data which can only be interpreted + within the context of a command in progress. It is tagged with the + same tag as the client command which began the operation. Thus, if + more than one command is in progress, the tag in an intermediate + response identifies the command to which the response applies. A + tagged response other than "OK", "NO", or "BAD" is an intermediate + response. + + + + + +Newman & Myers Standards Track [Page 5] + +RFC 2244 ACAP November 1997 + + + An untagged response returns data or status messages which may be + interpreted outside the context of a command in progress. It is + prefixed with the token "*". Untagged data may be sent as a result + of a client command, or may be sent unilaterally by the server. + There is no syntactic difference between untagged data that resulted + from a specific command and untagged data that were sent + unilaterally. + + The protocol receiver of an ACAP client reads a response line from + the server. It then takes action on the response based upon the + first token of the response, which may be a tag, a "*", or a "+" as + described above. + + A client MUST be prepared to accept any server response at all times. + This includes untagged data that it may not have requested. + + This topic is discussed in greater detail in the Server Responses + section. + +2.3. Server States + + An ACAP server is in one of three states. Most commands are valid in + only certain states. It is a protocol error for the client to + attempt a command while the server is in an inappropriate state for + that command. In this case, a server will respond with a BAD command + completion result. + +2.3.1. Non-Authenticated State + + In non-authenticated state, the user must supply authentication + credentials before most commands will be permitted. This state is + entered when a connection starts. + +2.3.2. Authenticated State + + In authenticated state, the user is authenticated and most commands + will be permitted. This state is entered when acceptable + authentication credentials have been provided. + +2.3.3. Logout State + + In logout state, the session is being terminated, and the server will + close the connection. This state can be entered as a result of a + client request or by unilateral server decision. + + + + + + + +Newman & Myers Standards Track [Page 6] + +RFC 2244 ACAP November 1997 + + + +--------------------------------------+ + |initial connection and server greeting| + +--------------------------------------+ + || (1) || (2) + VV || + +-----------------+ || + |non-authenticated| || + +-----------------+ || + || (4) || (3) || + || VV || + || +----------------+ || + || | authenticated | || + || +----------------+ || + || || (4) || + VV VV VV + +--------------------------------------+ + | logout and close connection | + +--------------------------------------+ + + (1) connection (ACAP greeting) + (2) rejected connection (BYE greeting) + (3) successful AUTHENTICATE command + (4) LOGOUT command, server shutdown, or connection closed + +2.4. Operational Considerations + +2.4.1. Untagged Status Updates + + At any time, a server can send data that the client did not request. + +2.4.2. Response when No Command in Progress + + Server implementations are permitted to send an untagged response + while there is no command in progress. Server implementations that + send such responses MUST deal with flow control considerations. + Specifically, they must either (1) verify that the size of the data + does not exceed the underlying transport's available window size, or + (2) use non-blocking writes. + +2.4.3. Auto-logout Timer + + If a server has an inactivity auto-logout timer, that timer MUST be + of at least 30 minutes duration. The receipt of ANY command from the + client during that interval MUST suffice to reset the auto-logout + timer. + + + + + + +Newman & Myers Standards Track [Page 7] + +RFC 2244 ACAP November 1997 + + +2.4.4. Multiple Commands in Progress + + The client is not required to wait for the completion result of a + command before sending another command, subject to flow control + constraints on the underlying data stream. Similarly, a server is + not required to process a command to completion before beginning + processing of the next command, unless an ambiguity would result + because of a command that would affect the results of other commands. + If there is such an ambiguity, the server executes commands to + completion in the order given by the client. + +2.5. Server Command Continuation Request + + The command continuation request is indicated by a "+" token instead + of a tag. This indicates that the server is ready to accept the + continuation of a command from the client. + + This response is used in the AUTHENTICATE command to transmit server + data to the client, and request additional client data. This + response is also used if an argument to any command is a + synchronizing literal (see section 2.6.3). + + The client is not permitted to send the octets of a synchronizing + literal unless the server indicates that it expects it. This permits + the server to process commands and reject errors on a line-by-line + basis, assuming it checks for non-synchronizing literals at the end + of each line. The remainder of the command, including the CRLF that + terminates a command, follows the octets of the literal. If there + are any additional command arguments the literal octets are followed + by a space and those arguments. + + Example: C: A099 FREECONTEXT {10} + S: + "Ready for additional command text" + C: FRED + C: FOOB + S: A099 OK "FREECONTEXT completed" + C: A044 BLURDYBLOOP {102856} + S: A044 BAD "No such command as 'BLURDYBLOOP'" + + +2.6. Data Formats + + ACAP uses textual commands and responses. Data in ACAP can be in one + of five forms: atom, number, string, parenthesized list or NIL. + + + + + + + +Newman & Myers Standards Track [Page 8] + +RFC 2244 ACAP November 1997 + + +2.6.1. Atom + + An atom consists of one to 1024 non-special characters. It must + begin with a letter. Atoms are used for protocol keywords. + +2.6.2. Number + + A number consists of one or more digit characters, and represents a + numeric value. Numbers are restricted to the range of an unsigned + 32-bit integer: 0 < number < 4,294,967,296. + +2.6.3. String + + A string is in one of two forms: literal and quoted string. The + literal form is the general form of string. The quoted string form + is an alternative that avoids the overhead of processing a literal at + the cost of restrictions of what may be in a quoted string. + + A literal is a sequence of zero or more octets (including CR and LF), + prefix-quoted with an octet count in the form of an open brace ("{"), + the number of octets, close brace ("}"), and CRLF. In the case of + literals transmitted from server to client, the CRLF is immediately + followed by the octet data. + + There are two forms of literals transmitted from client to server. + The form where the open brace ("{") and number of octets is + immediately followed by a close brace ("}") and CRLF is called a + synchronizing literal. When sending a synchronizing literal, the + client must wait to receive a command continuation request before + sending the octet data (and the remainder of the command). The other + form of literal, the non-synchronizing literal, is used to transmit a + string from client to server without waiting for a command + continuation request. The non-synchronizing literal differs from the + synchronizing literal by having a plus ("+") between the number of + octets and the close brace ("}") and by having the octet data + immediately following the CRLF. + + A quoted string is a sequence of zero to 1024 octets excluding NUL, + CR and LF, with double quote (<">) characters at each end. + + The empty string is represented as "" (a quoted string with zero + characters between double quotes), as {0} followed by CRLF (a + synchronizing literal with an octet count of 0), or as {0+} followed + by a CRLF (a non-synchronizing literal with an octet count of 0). + + Note: Even if the octet count is 0, a client transmitting a + synchronizing literal must wait to receive a command + continuation request. + + + +Newman & Myers Standards Track [Page 9] + +RFC 2244 ACAP November 1997 + + +2.6.3.1. 8-bit and Binary Strings + + Most strings in ACAP are restricted to UTF-8 characters and may not + contain NUL octets. Attribute values MAY contain any octets + including NUL. + +2.6.4. Parenthesized List + + Data structures are represented as a "parenthesized list"; a sequence + of data items, delimited by space, and bounded at each end by + parentheses. A parenthesized list can contain other parenthesized + lists, using multiple levels of parentheses to indicate nesting. + + The empty list is represented as () -- a parenthesized list with no + members. + +2.6.5. NIL + + The special atom "NIL" represents the non-existence of a particular + data item that is represented as a string or parenthesized list, as + distinct from the empty string "" or the empty parenthesized list (). + +3. Protocol Elements + + This section defines data formats and other protocol elements used + throughout the ACAP protocol. + +3.1. Entries and Attributes + + Within a dataset, each entry name is made up of zero or more UTF-8 + characters other than slash ("/"). A slash separated list of + entries, one at each level of the hierarchy, forms the full path to + an entry. + + Each entry is made up of a set of attributes. Each attribute has a + hierarchical name in UTF-8, with each component of the name separated + by a period ("."). + + The value of an attribute is either single or multi-valued. A single + value is NIL (has no value), or a string of zero or more octets. A + multi-value is a list of zero or more strings, each of zero or more + octets. + + Attribute names are not permitted to contain asterisk ("*") or + percent ("%") and MUST be valid UTF-8 strings which do not contain + NUL. Invalid attribute names result in a BAD response. Entry names + + + + + +Newman & Myers Standards Track [Page 10] + +RFC 2244 ACAP November 1997 + + + are not permitted to begin with "." or contain slash ("/") and MUST + be valid UTF-8 strings which do not contain NUL. Invalid entry names + in the entry field of a command result in a BAD response. + + Use of non-visible UTF-8 characters in attribute and entry names is + discouraged. + +3.1.1. Predefined Attributes + + Attribute names which do not contain a dot (".") are reserved for + standardized attributes which have meaning in any dataset. The + following attributes are defined by the ACAP protocol. + + entry + Contains the name of the entry. MUST be single valued. + Attempts to use illegal or multi-valued values for the entry + attribute are protocol errors and MUST result in a BAD + completion response. This is a special case. + + modtime + Contains the date and time any read-write metadata in the entry + was last modified. This value MUST be in UTC, MUST be + automatically updated by the server. + + The value consists of 14 or more US-ASCII digits. The first + four indicate the year, the next two indicate the month, the + next two indicate the day of month, the next two indicate the + hour (0 - 23), the next two indicate the minute, and the next + two indicate the second. Any further digits indicate fractions + of a second. + + The time, particularly fractions of a second, need not be + accurate. It is REQUIRED, however, that any two entries in a + dataset changed by successive modifications have strictly + ascending modtime values. In addition, each STORE command + within a dataset (including simultaneous stores from different + connections) MUST use different modtime values. + + This attribute has enforced validation, so any attempt to STORE + a value in this attribute MAY result in a NO response with an + INVALID response code. + + subdataset + If this attribute is set, it indicates the existence of a sub- + dataset of this entry. + + + + + + +Newman & Myers Standards Track [Page 11] + +RFC 2244 ACAP November 1997 + + + The value consists of a list of relative ACAP URLs (see section + 3.2) which may be used to locate the sub-dataset. The base URL + is the full path to the entry followed by a slash ("/"). The + value "." indicates a subdataset is located directly under this + one. Multiple values indicate replicated copies of the + subdataset. + + For example, if the dataset "/folder/site/" has an entry + "public-folder" with a subdataset attribute of ".", then there + exists a dataset "/folder/site/public-folder/". If the value of + the subdataset attribute was instead + "//other.acap.domain//folder/site/public-folder/", that would + indicate the dataset is actually located on a different ACAP + server. + + A dataset can be created by storing a "subdataset" attribute + including ".", and a sub-hierarchy of datasets is deleted by + storing a NIL value to the "subdataset" attribute on the entry + in the parent dataset. + + This attribute has enforced syntax validation. Specifically, if + an attempt is made to STORE a non-list value (other than NIL), + an empty list, or one of the values does not follow the URL + syntax rules [BASIC-URL, REL-URL], then this will result in a NO + response with an INVALID response code. + +3.1.2. Attribute Metadata + + Each attribute is made up of metadata items which describe that + attribute, its value and any associated access controls. Metadata + items may be either read-only, in which case the client is never + permitted to modify the item, or read-write, in which case the client + may modify the item if the access control list (ACL) permits. + + The following metadata items are defined in this specification: + + acl The access control list for the attribute, if one exists. If + the attribute does not have an ACL, NIL is returned. + Read-write. See section 3.5 for the contents of an ACL. + + attribute + The attribute name. Read-only. + + myrights + The set of rights that the client has to the attribute. + Read-only. See section 3.5 for the possible rights. + + + + + +Newman & Myers Standards Track [Page 12] + +RFC 2244 ACAP November 1997 + + + size This is the length of the value. In the case of a + multi-value, this is a list of lengths for each of the values. + Read-only. + + value The value. For a multi-value, this is a list of single + values. Read-write. + + Additional items of metadata may be defined in extensions to this + protocol. Servers MUST respond to unrecognized metadata by returning + a BAD command completion result. + +3.2. ACAP URL scheme + + ACAP URLs are used within the ACAP protocol for the "subdataset" + attribute, referrals and inheritance. They provide a convenient + syntax for referring to other ACAP datasets. The ACAP URL follows + the common Internet scheme syntax as defined in [BASIC-URL] except + that plaintext passwords are not permitted. If : is omitted, + the port defaults to 674. + + An ACAP URL has the following general form: + + url-acap = "acap://" url-server "/" url-enc-entry [url-filter] + [url-extension] + + The element includes the hostname, and optional user + name, authentication mechanism and port number. The + element contains the name of an entry path encoded according to the + rules in [BASIC-URL]. + + The element is an optional list of interesting attribute + names. If omitted, the URL refers to all attributes of the named + entry. The element is reserved for extensions to + this URL scheme. + + Note that unsafe or reserved characters such as " " or "?" MUST be + hex encoded as described in the URL specification [BASIC-URL]. Hex + encoded octets are interpreted according to UTF-8 [UTF8]. + +3.2.1. ACAP URL User Name and Authentication Mechanism + + A user name and/or authentication mechanism may be supplied. They + are used in the "AUTHENTICATE" command after making the connection to + the ACAP server. If no user name or authentication mechanism is + supplied, then the SASL ANONYMOUS [SASL-ANON] mechanism is used by + default. If an authentication mechanism is supplied without a user + + + + + +Newman & Myers Standards Track [Page 13] + +RFC 2244 ACAP November 1997 + + + name, then one SHOULD be obtained from the specified mechanism or + requested from the user as appropriate. If a user name is supplied + without an authentication mechanism then ";AUTH=*" is assumed. + + The ";AUTH=" authentication parameter is interpreted as described in + the IMAP URL Scheme [IMAP-URL]. + + Note that if unsafe or reserved characters such as " " or ";" are + present in the user name or authentication mechanism, they MUST be + encoded as described in the URL specification [BASIC-URL]. + +3.2.2. Relative ACAP URLs + + Because ACAP uses "/" as the hierarchy separator for dataset paths, + it works well with the relative URL rules defined in the relative URL + specification [REL-URL]. + + The grammar element is considered part of the user name for + purposes of resolving relative ACAP URLs. + + The base URL for a relative URL stored in an attribute's value is + formed by taking the path to the dataset containing that attribute, + appending a "/" followed by the entry name of the entry containing + that attribute followed by "/". + +3.3. Contexts + + A context is subset of entries in a dataset or datasets, created by a + SEARCH command with a MAKECONTEXT modifier. Context names are + client-generated strings and must not start with the slash ('/') + character. + + When a client creates a context, it may request automatic + notification of changes. A client may also request enumeration of + entries within a context. Enumeration simplifies the implementation + of a "virtual scrollbar" by the client. + + A context exists only within the ACAP session in which it was + created. When the connection is closed, all contexts associated with + that connection are automatically discarded. A server is required to + support at least 100 active contexts within a session. If the server + supports a larger limit it must advertise it in a CONTEXTLIMIT + capability. + + + + + + + + +Newman & Myers Standards Track [Page 14] + +RFC 2244 ACAP November 1997 + + +3.4. Comparators + + A comparator is a named function which takes two input values and can + be used to perform one or more of four comparison operations: + ordering, equality, prefix and substring matching. + + The ordering operation is used both for the SORT search modifier and + the COMPARE and COMPARESTRICT search keys. Ordering comparators can + determine the ordinal precedence of any two values. When used for + ordering, a comparator's name can be prefixed with "+" or "-" to + indicate that the ordering should be normal order or reversed order + respectively. If no prefix is included, "+" is assumed. + + For the purpose of ordering, a comparator may designate certain + values as having an undefined ordinal precedence. Such values always + collate with equal value after all other values regardless of whether + normal or reversed ordering is used. Unless the comparator + definition specifies otherwise, multi-values and NIL values have an + undefined ordinal precedence. + + The equality operation is used for the EQUAL search modifier, and + simply determines if the two values are considered equal under the + comparator function. When comparing a single value to a multi-value, + the two are considered equal if any one of the multiple values is + equal to the single value. + + The prefix match operation is used for the PREFIX search modifier, + and simply determines if the search value is a prefix of the item + being searched. In the case of prefix search on a multi-value, the + match is successful if the value is a prefix of any one of the + multiple values. + + The substring match operation is used for the SUBSTRING search + modifier, and simply determines if search value is a substring of the + item being searched. In the case of substring search on a multi- + value, the match is successful if the value is a substring of any one + of the multiple values. + + Rules for naming and registering comparators will be defined in a + future specification. Servers MUST respond to unknown or improperly + used comparators with a BAD command completion result. + + + + + + + + + + +Newman & Myers Standards Track [Page 15] + +RFC 2244 ACAP November 1997 + + + The following comparators are defined by this standard and MUST be + implemented: + + i;octet + Operations: Ordering, Equality, Prefix match, Substring match + + For collation, the i;octet comparator interprets the value of + an attribute as a series of unsigned octets with ordinal + values from 0 to 255. When ordering two strings, each octet + pair is compared in sequence until the octets are unequal or + the end of the string is reached. When collating two strings + where the shorter is a prefix of the longer, the shorter + string is interpreted as having a smaller ordinal value. The + "i;octet" or "+i;octet" forms collate smaller ordinal values + earlier, and the "-i;octet" form collates larger ordinal + values earlier. + + For the equality function, two strings are equal if they are + the same length and contain the same octets in the same + order. NIL is equal only to itself. + + For non-binary, non-nil single values, i;octet ordering is + equivalent to the ANSI C [ISO-C] strcmp() function applied to + C string representations of the values. For non-binary, + non-nil single values, i;octet substring match is equivalent + to the ANSI C strstr() function applied to the C string + representations of the values. + + i;ascii-casemap + Operations: Ordering, Equality, Prefix match, Substring match + + The i;ascii-casemap comparator first applies a mapping to the + attribute values which translates all US-ASCII letters to + uppercase (octet values 0x61 to 0x7A are translated to octet + values 0x41 to 0x5A respectively), then applies the i;octet + comparator as described above. With this function the values + "hello" and "HELLO" have the same ordinal value and are + considered equal. + + i;ascii-numeric + Operations: Ordering, Equality + + The i;ascii-numeric comparator interprets strings as decimal + positive integers represented as US-ASCII digits. All values + which do not begin with a US-ASCII digit are considered equal + with an ordinal value higher than all non-NIL single-valued + + + + + +Newman & Myers Standards Track [Page 16] + +RFC 2244 ACAP November 1997 + + + attributes. Otherwise, all US-ASCII digits (octet values + 0x30 to 0x39) are interpreted starting from the beginning of + the string to the first non-digit or the end of the string. + + +3.5. Access Control Lists (ACLs) + + An access control list is a set of identifier, rights pairs used to + restrict access to a given dataset, attribute or attribute within an + entry. An ACL is represented by a multi-value with each value + containing an identifier followed by a tab character followed by the + rights. The syntax is defined by the "acl" rule in the formal syntax + in section 8. + + Identifier is a UTF-8 string. The identifier "anyone" is reserved to + refer to the universal identity (all authentications, including + anonymous). All user name strings accepted by the AUTHENTICATE + command to authenticate to the ACAP server are reserved as + identifiers for the corresponding user. Identifiers starting with a + slash ("/") character are reserved for authorization groups which + will be defined in a future specification. Identifiers MAY be + prefixed with a dash ("-") to indicate a revocation of rights. All + other identifiers have implementation-defined meanings. + + Rights is a string listing a (possibly empty) set of alphanumeric + characters, each character listing a set of operations which is being + controlled. Letters are reserved for "standard" rights, listed + below. The set of standard rights may only be extended by a + standards-track or IESG approved experimental RFC. Digits are + reserved for implementation or site defined rights. The currently + defined standard rights are: + + x - search (use EQUAL search key with i;octet comparator) + r - read (access with SEARCH command) + w - write (modify with STORE command) + i - insert (perform STORE on a previously NIL value) + a - administer (perform SETACL or STORE on ACL attribute/metadata) + + An implementation may force rights to always or never be granted. In + particular, implementations are expected to grant implicit read and + administer rights to a user's personal dataset storage in order to + avoid denial of service problems. Rights are never tied, unlike the + IMAP ACL extension [IMAP-ACL]. + + It is possible for multiple identifiers in an access control list to + apply to a given user (or other authentication identity). For + example, an ACL may include rights to be granted to the identifier + matching the user, one or more implementation-defined identifiers + + + +Newman & Myers Standards Track [Page 17] + +RFC 2244 ACAP November 1997 + + + matching groups which include the user, and/or the identifier + "anyone". These rights are combined by taking the union of all + positive rights which apply to a given user and subtracting the union + of all negative rights which apply to that user. A client MAY avoid + this calculation by using the MYRIGHTS command and metadata items. + + Each attribute of each entry of a dataset may potentially have an + ACL. If an attribute in an entry does not have an ACL, then access + is controlled by a default ACL for that attribute in the dataset, if + it exists. If there is no default ACL for that attribute in the + dataset, access is controlled by a default ACL for that dataset. The + default ACL for a dataset must exist. + + In order to perform any access or manipulation on an entry in a + dataset, the client must have 'r' rights on the "entry" attribute of + the entry. Implementations should take care not to reveal via error + messages the existence of an entry for which the client does not have + 'r' rights. A client does not need access to the "subdataset" + attribute of the parent dataset in order to access the contents of a + dataset. + + Many of the ACL commands and responses include an "acl object" + parameter, for specifying what the ACL applies to. This is a + parenthesized list. The list contains just the dataset name when + referring to the default ACL for a dataset. The list contains a + dataset name and an attribute name when referring to the default ACL + for an attribute in a dataset. The list contains a dataset name, an + attribute name, and an entry name when referring to the ACL for an + attribute of an entry of a dataset. + + +3.6. Server Response Codes + + An OK, NO, BAD, ALERT or BYE response from the server MAY contain a + response code to describe the event in a more detailed machine + parsable fashion. A response code consists of data inside + parentheses in the form of an atom, possibly followed by a space and + arguments. Response codes are defined when there is a specific + action that a client can take based upon the additional information. + In order to support future extension, the response code is + represented as a slash-separated hierarchy with each level of + hierarchy representing increasing detail about the error. Clients + MUST tolerate additional hierarchical response code detail which they + don't understand. + + The currently defined response codes are: + + + + + +Newman & Myers Standards Track [Page 18] + +RFC 2244 ACAP November 1997 + + + AUTH-TOO-WEAK + This response code is returned on a tagged NO result from an + AUTHENTICATE command. It indicates that site security policy + forbids the use of the requested mechanism for the specified + authentication identity. + + ENCRYPT-NEEDED + This response code is returned on a tagged NO result from an + AUTHENTICATE command. It indicates that site security policy + requires the use of a strong encryption mechanism for the + specified authentication identity and mechanism. + + INVALID + This response code indicates that a STORE command included + data which the server implementation does not permit. It + MUST NOT be used unless the dataset class specification for + the attribute in question explicitly permits enforced server + validation. The argument is the attribute which was invalid. + + MODIFIED + This response code indicates that a conditional store failed + because the modtime on the entry is later than the modtime + specified with the STORE command UNCHANGEDSINCE modifier. + The argument is the entry which had been modified. + + NOEXIST + This response code indicates that a search or NOCREATE store + failed because a specified dataset did not exist. The + argument is the dataset which does not exist. + + PERMISSION + A command failed due to insufficient permission based on the + access control list or implicit rights. The argument is the + acl-object which caused the permission failure. + + QUOTA + A STORE or SETACL command which would have increased the size + of the dataset failed due to insufficient quota. + + REFER + This response code may be returned in a tagged NO response to + any command that takes a dataset name as a parameter. It has + one or more arguments with the syntax of relative URLs. It + is a referral, indicating that the command should be retried + using one of the relative URLs. + + + + + + +Newman & Myers Standards Track [Page 19] + +RFC 2244 ACAP November 1997 + + + SASL This response code can occur in the tagged OK response to a + successful AUTHENTICATE command and includes the optional + final server response data from the server as specified by + SASL [SASL]. + + TOOMANY + This response code may be returned in a tagged OK response to + a SEARCH command which includes the LIMIT modifier. The + argument returns the total number of matching entries. + + TOOOLD + The modtime specified in the DELETEDSINCE command is too old, + so deletedsince information is no longer available. + + TRANSITION-NEEDED + This response code occurs on a NO response to an AUTHENTICATE + command. It indicates that the user name is valid, but the + entry in the authentication database needs to be updated in + order to permit authentication with the specified mechanism. + This can happen if a user has an entry in a system + authentication database such as Unix /etc/passwd, but does + not have credentials suitable for use by the specified + mechanism. + + TRYLATER + A command failed due to a temporary server failure. The + client MAY continue using local information and try the + command later. + + TRYFREECONTEXT + This response code may be returned in a tagged NO response to + a SEARCH command which includes the MAKECONTEXT modifier. It + indicates that a new context may not be created due to the + server's limit on the number of existing contexts. + + WAYTOOMANY + This response code may be returned in a tagged NO response to + a SEARCH command which includes a HARDLIMIT search modifier. + It indicates that the SEARCH would have returned more entries + than the HARDLIMIT permitted. + + Additional response codes MUST be registered with IANA according + to the proceedures in section 7.2. Client implementations MUST + tolerate response codes that they do not recognize. + + + + + + + +Newman & Myers Standards Track [Page 20] + +RFC 2244 ACAP November 1997 + + +4. Namespace Conventions + +4.1. Dataset Namespace + + The dataset namespace is a slash-separated hierarchy. The first + component of the dataset namespace is a dataset class. Dataset + classes MUST have a vendor prefix (vendor.) or be + specified in a standards track or IESG approved experimental RFC. + See section 7.3 for the registration template. + + The second component of the dataset name is "site", "group", "host", + or "user" referring to server-wide data, administrative group data, + per-host data and per-user data respectively. + + For "group", "host", and "user" areas, the third component of the + path is the group name, the fully qualified host domain name, or the + user name. A path of the form "//~/" is a convenient + abbreviation for "//user//". + + Dataset names which begin with "/byowner/" are reserved as an + alternate view of the namespace. This provides a way to see all the + dataset classes which a particular owner uses. For example, + "/byowner/~//" is an alternate name for + "//~/". Byowner provides a way to view a list of + dataset classes owned by a given user; this is done using the dataset + "/byowner/user//" with the NOINHERIT SEARCH modifier. + + The dataset "/" may be used to find all dataset classes visible to + the current user. A dataset of the form "//user/" may + be used to find all users which have made a dataset or entry of that + class visible to the current user. + + The formal syntax for a dataset name is defined by the "dataset-name" + rule in section 4.3. + +4.2. Attribute Namespace + + Attribute names which do not contain a dot (".") are reserved for + standardized attributes which have meaning in any dataset. In order + to simplify client implementations, the attribute namespace is + intended to be unique across all datasets. To achieve this, + attribute names are prefixed with the dataset class name followed by + a dot ("."). Attributes which affect management of the dataset are + prefixed with "dataset.". In addition, a subtree of the "vendor." + attribute namespace may be registered with IANA according to the + rules in section 7.4. ACAP implementors are encouraged to help + define interoperable dataset classes specifications rather than using + the private attribute namespace. + + + +Newman & Myers Standards Track [Page 21] + +RFC 2244 ACAP November 1997 + + + Some users or sites may wish to add their own private attributes to + certain dataset classes. In order to enable this, the "user.." and "site." subtrees of the attribute namespace are reserved + for user-specific and site-specific attributes respectively and will + not be standardized. Such attributes are not interoperable so are + discouraged in favor of defining standard attributes. A future + extension is expected to permit discovery of syntax for user or + site-specific attributes. Clients wishing to support display of user + or site-specific attributes should display the value of any non-NIL + single-valued "user.." or "site." attribute which has + valid UTF-8 syntax. + + The formal syntax for an attribute name is defined by the + "attribute-name" rule in the next section. + +4.3. Formal Syntax for Dataset and Attribute Namespace + + The naming conventions for datasets and attributes are defined by the + following ABNF. Note that this grammar is not part of the ACAP + protocol syntax in section 8, as dataset names and attribute names + are encoded as strings within the ACAP protocol. + + attribute-dacl = "dataset.acl" *("." name-component) + + attribute-dset = dataset-std 1*("." name-component) + ;; MUST be defined in a dataset class specification + + attribute-name = attribute-std / attr-site / attr-user / vendor-name + + attribute-std = "entry" / "subdataset" / "modtime" / + "dataset.inherit" / attribute-dacl / attribute-dset + + attr-site = "site" 1*("." name-component) + + attr-user = "user." name-component 1*("." name-component) + + byowner = "/byowner/" owner "/" + [dataset-class "/" dataset-sub] + + dataset-class = dataset-std / vendor-name + + dataset-normal = "/" [dataset-class "/" + (owner-prefix / dataset-tail)] + + dataset-name = byowner / dataset-normal + + + + + + +Newman & Myers Standards Track [Page 22] + +RFC 2244 ACAP November 1997 + + + dataset-std = name-component + ;; MUST be registered with IANA and the spec MUST + ;; be published as a standards track or + ;; IESG-approved experimental RFC + + dataset-sub = *(dname-component "/") + ;; The rules for this portion of the namespace may + ;; be further restricted by the dataset class + ;; specification. + + dataset-tail = owner "/" dataset-sub + + dname-component = 1*UTF8-CHAR + ;; MUST NOT begin with "." or contain "/" + + name-component = 1*UTF8-CHAR + ;; MUST NOT contain ".", "/", "%", or "*" + + owner = "site" / owner-host / owner-group / + owner-user / "~" + + owner-group = "group/" dname-component + + owner-host = "host/" dname-component + + owner-prefix = "group/" / "host/" / "user/" + + owner-user = "user/" dname-component + + vendor-name = vendor-token *("." name-component) + + vendor-token = "vendor." name-component + ;; MUST be registered with IANA + +5. Dataset Management + + The entry with an empty name ("") in the dataset is used to hold + management information for the dataset as a whole. + +5.1. Dataset Inheritance + + It is possible for one dataset to inherit data from another. The + dataset from which the data is inherited is called the base dataset. + Data in the base dataset appears in the inheriting dataset, except + when overridden by a STORE to the inheriting dataset. + + + + + + +Newman & Myers Standards Track [Page 23] + +RFC 2244 ACAP November 1997 + + + The base dataset is usually a system-wide or group-wide set of + defaults. A system-wide dataset usually has one inheriting dataset + per user, allowing each user to add to or modify the defaults as + appropriate. + + An entry which exists in both the inheriting and base dataset + inherits a modtime equal to the greater of the two modtimes. An + attribute in such an entry is inherited from the base dataset if it + was never modified by a STORE command in the inheriting dataset or if + DEFAULT was stored to that attribute. This permits default entries + to be amended rather than replaced in the inheriting dataset. + + The "subdataset" attribute is not directly inherited. If the base + dataset includes a "subdataset" attribute and the inheriting dataset + does not, then the "subdataset" attribute will inherit a virtual + value of a list containing a ".". The subdataset at that node is + said to be a "virtual" dataset as it is simply a virtual copy of the + appropriate base dataset with all "subdataset" attributes changed to + a list containing a ".". A virtual dataset is not visible if + NOINHERIT is specified on the SEARCH command. + + Servers MUST support at least two levels of inheritance. This + permits a user's dataset such as "/options/user/fred/common" to + inherit from a group dataset such as "/options/group/dinosaur + operators/common" which in turn inherits from a server-wide dataset + such as "/options/site/common". + +5.2. Dataset Attributes + + The following attributes apply to management of the dataset when + stored in the "" entry of a dataset. These attributes are not + inherited. + + dataset.acl + This holds the default access control list for the dataset. + This attribute is validated, so an invalid access control list + in a STORE command will result in a NO response with an INVALID + response code. + + dataset.acl. + This holds the default access control list for an attribute + within the dataset. This attribute is validated, so an invalid + access control list in a STORE command will result in a NO + response with an INVALID response code. + + dataset.inherit + This holds the name of a dataset from which to inherit according + to the rules in the previous section. This attribute MAY refer + + + +Newman & Myers Standards Track [Page 24] + +RFC 2244 ACAP November 1997 + + + to a non-existent dataset, in which case nothing is inherited. + This attribute is validated, so illegal dataset syntax or an + attempt to store a multi-value will result in a NO response with + an INVALID response code. + +5.3. Dataset Creation + + When a dataset is first created (by storing a "." in the subdataset + attribute or storing an entry in a previously non-existent dataset), + the dataset attributes are initialized with the values from the + parent dataset in the "/byowner/" hierarchy. In the case of the + "dataset.inherit" attribute, the appropriate hierarchy component is + added. For example, given the following entry (note that \t refers + to the US-ASCII horizontal tab character): + + entry path "/byowner/user/joe/" + dataset.acl ("joe\txrwia" "fred\txr") + dataset.inherit "/byowner/site" + + If a new dataset class "/byowner/user/joe/new" is created, it will + have the following dataset attributes: + + entry path "/byowner/user/joe/new/" + dataset.acl ("joe\txrwia" "fred\txr") + dataset.inherit "/byowner/site/new" + + Note that the dataset "/byowner/user/joe/new/" is equivalent to + "/new/user/joe/". + +5.4. Dataset Class Capabilities + + Certain dataset classes or dataset class features may only be useful + if there is an active updating client or integrated server support + for the feature. The dataset class "capability" is reserved to allow + clients or servers to advertise such features. The "entry" attribute + within this dataset class is the name of the dataset class whose + features are being described. The attributes are prefixed with + "capability.." and are defined by the appropriate + dataset class specification. + + Since it is possible for an unprivileged user to run an active client + for himself, a per-user capability dataset is useful. The dataset + "/capability/~/" holds information about all features available to + the user (via inheritance), and the dataset "/capability/site/" holds + information about all features supported by the site. + + + + + + +Newman & Myers Standards Track [Page 25] + +RFC 2244 ACAP November 1997 + + +5.5. Dataset Quotas + + Management and scope of quotas is implementation dependent. Clients + can check the applicable quota limit and usage (in bytes) with the + GETQUOTA command. Servers can notify the client of a low quota + situation with the QUOTA untagged response. + +6. Command and Response Specifications + + ACAP commands and responses are described in this section. Commands + are organized first by the state in which the command is permitted, + then by a general category of command type. + + Command arguments, identified by "Arguments:" in the command + descriptions below, are described by function, not by syntax. The + precise syntax of command arguments is described in the Formal Syntax + section. + + Some commands cause specific server data to be returned; these are + identified by "Data:" in the command descriptions below. See the + response descriptions in the Responses section for information on + these responses, and the Formal Syntax section for the precise syntax + of these responses. It is possible for server data to be transmitted + as a result of any command; thus, commands that do not specifically + require server data specify "no specific data for this command" + instead of "none". + + The "Result:" in the command description refers to the possible + tagged status responses to a command, and any special interpretation + of these status responses. + +6.1. Initial Connection + + Upon session startup, the server sends one of two untagged responses: + ACAP or BYE. The untagged BYE response is described in section + 6.2.8. + +6.1.1. ACAP Untagged Response + + Data: capability list + + The untagged ACAP response indicates the session is ready to + accept commands and contains a space-separated listing of + capabilities that the server supports. Each capability is + represented by a list containing the capability name optionally + followed by capability specific string arguments. + + + + + +Newman & Myers Standards Track [Page 26] + +RFC 2244 ACAP November 1997 + + + ACAP capability names MUST be registered with IANA according to + the rules in section 7.1. + + Client implementations SHOULD NOT require any capability name + beyond those defined in this specification, and MUST tolerate any + unknown capability names. A client implementation MAY be + configurable to require SASL mechanisms other than CRAM-MD5 + [CRAM-MD5] for site security policy reasons. + + The following initial capabilities are defined: + + CONTEXTLIMIT + The CONTEXTLIMIT capability has one argument which is a + number describing the maximum number of contexts the server + supports per connection. The number 0 indicates the server + has no limit, otherwise this number MUST be greater than + 100. + + IMPLEMENTATION + The IMPLEMENTATION capability has one argument which is a + string describing the server implementation. ACAP clients + MUST NOT alter their behavior based on this value. It is + intended primarily for debugging purposes. + + SASL The SASL capability includes a list of the authentication + mechanisms supported by the server. See section 6.3.1. + + + Example: S: * ACAP (IMPLEMENTATION "ACME v3.5") + (SASL "CRAM-MD5") (CONTEXTLIMIT "200") + +6.2. Any State + + The following commands and responses are valid in any state. + +6.2.1. NOOP Command + + Arguments: none + + Data: no specific data for this command (but see below) + + Result: OK - noop completed + BAD - command unknown or arguments invalid + + The NOOP command always succeeds. It does nothing. It can be + used to reset any inactivity auto-logout timer on the server. + + Example: C: a002 NOOP + + + +Newman & Myers Standards Track [Page 27] + +RFC 2244 ACAP November 1997 + + + S: a002 OK "NOOP completed" + + +6.2.2. LANG Command + + Arguments: list of language preferences + + Data: intermediate response: LANG + + Result: OK - lang completed + NO - no matching language available + BAD - command unknown or arguments invalid + + One or more arguments are supplied to indicate the client's + preferred languages [LANG-TAGS] for error messages. The server + will match each client preference in order against its internal + table of available error string languages. For a client + preference to match a server language, the client's language tag + MUST be a prefix of the server's tag and match up to a "-" or the + end of string. If a match is found, the server returns an + intermediate LANG response and an OK response. The LANG response + indicates the actual language selected and appropriate comparators + for use with the languages listed in the LANG command. + + If no LANG command is issued, all error text strings MUST be in + the registered language "i-default" [CHARSET-LANG-POLICY], + intended for an international audience. + + Example: C: A003 LANG "fr-ca" "fr" "en-ca" "en-uk" + S: A003 LANG "fr-ca" "i;octet" "i;ascii-numeric" + "i;ascii-casemap" "en;primary" "fr;primary" + S: A003 OK "Bonjour" + + +6.2.3. LANG Intermediate Response + + Data: language for error responses + appropriate comparators + + The LANG response indicates the language which will be used for + error responses and the comparators which are appropriate for the + languages listed in the LANG command. The comparators SHOULD be + in approximate order from most efficient (usually "i;octet") to + most appropriate for human text in the preferred language. + + + + + + + +Newman & Myers Standards Track [Page 28] + +RFC 2244 ACAP November 1997 + + +6.2.4. LOGOUT Command + + Arguments: none + + Data: mandatory untagged response: BYE + + Result: OK - logout completed + BAD - command unknown or arguments invalid + + The LOGOUT command informs the server that the client is done with + the session. The server must send a BYE untagged response before + the (tagged) OK response, and then close the network connection. + + Example: C: A023 LOGOUT + S: * BYE "ACAP Server logging out" + S: A023 OK "LOGOUT completed" + (Server and client then close the connection) + + +6.2.5. OK Response + + Data: optional response code + human-readable text + + The OK response indicates an information message from the server. + When tagged, it indicates successful completion of the associated + command. The human-readable text may be presented to the user as + an information message. The untagged form indicates an + information-only message; the nature of the information MAY be + indicated by a response code. + + Example: S: * OK "Master ACAP server is back up" + + +6.2.6. NO Response + + Data: optional response code + human-readable text + + The NO response indicates an operational error message from the + server. When tagged, it indicates unsuccessful completion of the + associated command. The untagged form indicates a warning; the + command may still complete successfully. The human-readable text + describes the condition. + + Example: C: A010 SEARCH "/addressbook/" DEPTH 3 RETURN ("*") + EQUAL "entry" "+i;octet" "bozo" + S: * NO "Master ACAP server is down, your data may + + + +Newman & Myers Standards Track [Page 29] + +RFC 2244 ACAP November 1997 + + + be out of date." + S: A010 OK "search done" + ... + C: A222 STORE ("/folder/site/comp.mail.misc" + "folder.creation-time" "19951206103412") + S: A222 NO (PERMISSION ("/folder/site/")) "Permission + denied" + + +6.2.7. BAD Response + + Data: optional response code + human-readable text + + The BAD response indicates an error message from the server. When + tagged, it reports a protocol-level error in the client's command; + the tag indicates the command that caused the error. The untagged + form indicates a protocol-level error for which the associated + command can not be determined; it may also indicate an internal + server failure. The human-readable text describes the condition. + + Example: C: ...empty line... + S: * BAD "Empty command line" + C: A443 BLURDYBLOOP + S: A443 BAD "Unknown command" + C: A444 NOOP Hello + S: A444 BAD "invalid arguments" + + +6.2.8. BYE Untagged Response + + Data: optional response code + human-readable text + + The untagged BYE response indicates that the server is about to + close the connection. The human-readable text may be displayed to + the user in a status report by the client. The BYE response may + be sent as part of a normal logout sequence, or as a panic + shutdown announcement by the server. It is also used by some + server implementations as an announcement of an inactivity auto- + logout. + + This response is also used as one of two possible greetings at + session startup. It indicates that the server is not willing to + accept a session from this client. + + Example: S: * BYE "Auto-logout; idle for too long" + + + + +Newman & Myers Standards Track [Page 30] + +RFC 2244 ACAP November 1997 + + +6.2.9. ALERT Untagged Response + + Data: optional response code + human-readable text + + The human-readable text contains a special human generated alert + message that MUST be presented to the user in a fashion that calls + the user's attention to the message. This is intended to be used + for vital messages from the server administrator to the user, such + as a warning that the server will soon be shut down for + maintenance. + + Example: S: * ALERT "This ACAP server will be shut down in + 10 minutes for system maintenance." + + +6.3. Non-Authenticated State + + In non-authenticated state, the AUTHENTICATE command establishes + authentication and enters authenticated state. The AUTHENTICATE + command provides a general mechanism for a variety of authentication + techniques. + + Server implementations may allow non-authenticated access to certain + information by supporting the SASL ANONYMOUS [SASL-ANON] mechanism. + + Once authenticated (including as anonymous), it is not possible to + re-enter non-authenticated state. + + Only the any-state commands (NOOP, LANG and LOGOUT) and the + AUTHENTICATE command are valid in non-authenticated state. + + +6.3.1. AUTHENTICATE Command + + Arguments: SASL mechanism name + optional initial response + + Data: continuation data may be requested + + Result: OK - authenticate completed, now in authenticated state + NO - authenticate failure: unsupported authentication + mechanism, credentials rejected + BAD - command unknown or arguments invalid, + authentication exchange cancelled + + + + + + +Newman & Myers Standards Track [Page 31] + +RFC 2244 ACAP November 1997 + + + The AUTHENTICATE command indicates a SASL [SASL] authentication + mechanism to the server. If the server supports the requested + authentication mechanism, it performs an authentication protocol + exchange to authenticate and identify the user. Optionally, it + also negotiates a security layer for subsequent protocol + interactions. If the requested authentication mechanism is not + supported, the server rejects the AUTHENTICATE command by sending + a tagged NO response. + + The authentication protocol exchange consists of a series of + server challenges and client answers that are specific to the + authentication mechanism. A server challenge consists of a + command continuation request with the "+" token followed by a + string. The client answer consists of a line consisting of a + string. If the client wishes to cancel an authentication + exchange, it should issue a line with a single unquoted "*". If + the server receives such an answer, it must reject the + AUTHENTICATE command by sending a tagged BAD response. + + The optional initial-response argument to the AUTHENTICATE command + is used to save a round trip when using authentication mechanisms + that are defined to send no data in the initial challenge. When + the initial-response argument is used with such a mechanism, the + initial empty challenge is not sent to the client and the server + uses the data in the initial-response argument as if it were sent + in response to the empty challenge. If the initial-response + argument to the AUTHENTICATE command is used with a mechanism that + sends data in the initial challenge, the server rejects the + AUTHENTICATE command by sending a tagged NO response. + + The service name specified by this protocol's profile of SASL is + "acap". + + If a security layer is negotiated through the SASL authentication + exchange, it takes effect immediately following the CRLF that + concludes the authentication exchange for the client, and the CRLF + of the tagged OK response for the server. + + All ACAP implementations MUST implement the CRAM-MD5 SASL + mechanism [CRAM-MD5], although they MAY offer a configuration + option to disable it if site security policy dictates. The + example below is the same example described in the CRAM-MD5 + specification. + + If an AUTHENTICATE command fails with a NO response, the client + may try another authentication mechanism by issuing another + AUTHENTICATE command. In other words, the client may request + authentication types in decreasing order of preference. + + + +Newman & Myers Standards Track [Page 32] + +RFC 2244 ACAP November 1997 + + + Example: S: * ACAP (IMPLEMENTATION "Blorfysoft v3.5") + (SASL "CRAM-MD5" "KERBEROS_V4") + C: A001 AUTHENTICATE "CRAM-MD5" + S: + "<1896.697170952@postoffice.reston.mci.net>" + C: "tim b913a602c7eda7a495b4e6e7334d3890" + S: A001 OK "CRAM-MD5 authentication successful" + + +6.4. Searching + + This section describes the SEARCH command, for retrieving data from + datasets. + + +6.4.1. SEARCH Command + + Arguments: dataset or context name + optional list of modifiers + search criteria + + Data: intermediate responses: ENTRY, MODTIME, REFER + untagged responses: ADDTO, REMOVEFROM, CHANGE, MODTIME + + Result: OK - search completed + NO - search failure: can't perform search + BAD - command unknown or arguments invalid + + The SEARCH command identifies a subset of entries in a dataset and + returns information on that subset to the client. Inherited + entries and attributes are included in the search unless the + NOINHERIT search modifier is included or the user does not have + permission to read the attributes in the base dataset. + + The first argument to SEARCH identifies what is to be searched. + If the string begins with a slash ("/"), it is the name of a + dataset to be searched, otherwise it is a name of a context that + was created by a SEARCH command given previously in the session. + + A successful SEARCH command MAY result in intermediate ENTRY + responses and MUST result in a MODTIME intermediate response. + + Following that are zero or more modifiers to the search. Each + modifier may be specified at most once. The defined modifiers + are: + + + + + + + +Newman & Myers Standards Track [Page 33] + +RFC 2244 ACAP November 1997 + + + DEPTH number + The SEARCH command will traverse the dataset tree up to the + specified depth. ENTRY responses will include the full path + to the entry. A value of "0" indicates that the search + should traverse the entire tree. A value of "1" is the + default and indicates only the specified dataset should be + searched. If a dataset is traversed which is not located on + the current server, then a REFER intermediate response is + returned for that subtree and the search continues. + + HARDLIMIT number + If the SEARCH command would result in more than number + entries, the SEARCH fails with a NO completion result with a + WAYTOOMANY response code. + + LIMIT number number + Limits the number of intermediate ENTRY responses that the + search may generate. The first numeric argument specifies + the limit, the second number specifies the number of entries + to return if the number of matches exceeds the limit. If the + limit is exceeded, the SEARCH command still succeeds, + returning the total number of matches in a TOOMANY response + code in the tagged OK response. + + MAKECONTEXT [ENUMERATE] [NOTIFY] context + Causes the SEARCH command to create a context with the name + given in the argument to refer to the matching entries. If + the SEARCH is successful, the context name may then be given + as an argument to subsequent SEARCH commands to search the + set of matching entries. If a context with the specified + name already exists, it is first freed. If a new context may + not be created due to the server's limit on the number of + existing contexts, the command fails, returning a + TRYFREECONTEXT response code in the NO completion response. + + The optional "ENUMERATE" and "NOTIFY" arguments may be + included to request enumeration of the context (for virtual + scroll bars) or change notifications for the context. If + "NOTIFY" is not requested, the context represents a snapshot + of the entries at the time the SEARCH was issued. + + ENUMERATE requests that the contents of the context be + ordered according to the SORT modifier and that sequential + numbers, starting with one, be assigned to the entries in the + context. This permits the RANGE modifier to be used to fetch + portions of the ordered context. + + + + + +Newman & Myers Standards Track [Page 34] + +RFC 2244 ACAP November 1997 + + + NOTIFY requests that the server send untagged ADDTO, + REMOVEFROM, CHANGE, and MODTIME responses while the context + created by this SEARCH command exists. The server MAY issue + untagged ADDTO, REMOVEFROM, CHANGE and MODTIME notifications + for a context at any time between the issuing of the SEARCH + command with MAKECONTEXT NOTIFY and the completion of a + FREECONTEXT command for the context. Notifications are only + issued for changes which occur after the server receives the + SEARCH command which created the context. After issuing a + sequence of ADDTO, REMOVEFROM or CHANGE notifications, the + server MUST issue an untagged MODTIME notification indicating + that the client has all updates to the entries in the context + up to and including the given modtime value. Servers are + permitted a reasonable delay to batch change notifications + before sending them to the client. + + The position arguments of the ADDTO, REMOVEFROM and CHANGE + notifications are 0 if ENUMERATE is not requested. + + NOINHERIT + This causes the SEARCH command to operate without + inheritance. It can be used to tell which values are + explicit overrides. If MAKECONTEXT is also specified, the + created context is also not affected by inheritance. + + RETURN (metadata...) + Specifies what is to be returned in intermediate ENTRY + responses. If this modifier is not specified, no + intermediate ENTRY responses are returned. + + Inside the parentheses is an optional list of attributes, + each optionally followed by a parenthesized list of metadata. + If the parenthesized list of metadata is not specified, it + defaults to "(value)". + + An attribute name with a trailing "*" requests all attributes + with that prefix. A "*" by itself requests all attributes. + If the parenthesized list of metadata is not specified for an + attribute with a trailing "*", it defaults to "(attribute + value)". Results matching such an attribute pattern are + grouped in parentheses. + + Following the last intermediate ENTRY response, the server + returns a single intermediate MODTIME response. + + + + + + + +Newman & Myers Standards Track [Page 35] + +RFC 2244 ACAP November 1997 + + + SORT (attribute comparator ...) + Specifies the order in which any resulting ENTRY replies are + to be returned to the client. The SORT modifier takes as an + argument a parenthesized list of one or more + attribute/comparator pairs. Attribute lists the attribute to + sort on, comparator specifies the name of the collation rule + to apply to the values of the attribute. Successive + attribute/comparator pairs are used to order two entries only + when all preceding pairs indicate the two entries collate the + same. + + If the SORT modifier is used in conjunction with the + MAKECONTEXT modifier, the SORT modifier specifies the + ordering of entries in the created context. + + If no SORT modifier is specified, or none of the + attribute/comparator pairs indicates an order for the two + entries, the server uses the order of the entries that exists + in the context or dataset being searched. + + + Following the modifiers is the search criteria. Searching + criteria consist of one or more search keys. Search keys may be + combined using the AND, and OR search keys. For example, the + criteria (the newline is for readability and not part of the + criteria): + AND COMPARE "modtime" "+i;octet" "19951206103400" + COMPARE "modtime" "-i;octet" "19960112000000" + refers to all entries modified between 10:34 December 6 1995 and + midnight January 12, 1996 UTC. + + The currently defined search keys are as follows. + + ALL This matches all entries. + + AND search-key1 search-key2 + Entries that match both search keys. + + COMPARE attribute comparator value + Entries for which the value of the specified attribute + collates using the specified comparator the same or later + than the specified value. + + COMPARESTRICT attribute comparator value + Entries for which the specified attribute collates using the + specified comparator later than the specified value. + + + + + +Newman & Myers Standards Track [Page 36] + +RFC 2244 ACAP November 1997 + + + EQUAL attribute comparator value + Entries for which the value of the attribute is equal to the + specified value using the specified comparator. + + NOT search-key + Entries that do not match the specified search key. + + OR search-key1 search-key2 + Entries that match either search key. + + PREFIX attribute comparator value + Entries which begin with the specified value using the + specified comparator. If the specified comparator doesn't + support substring matching, a BAD response is returned. + + RANGE start end time + Entries which are within the specified range of the + enumerated context's ordering. The lowest-ordered entry in + the context is assigned number one, the next lowest entry is + assigned number two, and so on. The numeric arguments + specify the lowest and highest numbers to match. The time + specifies that the client has processed notifications for the + context up to the specified time. If the context has been + modified since then, the server MUST either return a NO with + a MODIFIED response code, or return the results that the + SEARCH would have returned if none of the changes since that + time had been made. + + RANGE is only permitted on enumerated contexts. If RANGE is + used with a dataset or non-enumerated context, the server + MUST return a BAD response. + + SUBSTRING attribute comparator value + Entries which contain the specified value, using the + specified comparator. If the specified comparator doesn't + support substring matching, a BAD response is returned. + + +6.4.2. ENTRY Intermediate Response + + Data: entry name + entry data + + The ENTRY intermediate response occurs as a result of a SEARCH or + STORE command. This is the means by which dataset entries are + returned to the client. + + + + + +Newman & Myers Standards Track [Page 37] + +RFC 2244 ACAP November 1997 + + + The ENTRY response begins with the entry name, if a SEARCH command + without the DEPTH modifier was issued, or the entry path in other + cases. This is followed by a set of zero or more items, one for + each metadata item in the RETURN search modifier. Results + matching an attribute pattern or returning multiple metadata items + are grouped in parentheses. + +6.4.3. MODTIME Intermediate Response + + Data: modtime value + + The MODTIME intermediate response occurs as a result of a SEARCH + command. It indicates that the just created context or the + previously returned ENTRY responses include all updates to the + returned entries up to and including the modtime value in the + argument. + +6.4.4. REFER Intermediate Response + + Data: dataset path + relative ACAP URLs + + The REFER intermediate response occurs as a result of a + multi-level SEARCH where one of the levels is located on a + different server. The response indicates the dataset which is not + located on the current server and one or more relative ACAP URLs + for where that dataset may be found. + +6.4.5. Search Examples + + Here are some SEARCH command exchanges between the client and server: + + C: A046 SEARCH "/addressbook/" DEPTH 3 RETURN ("addressbook.Alias" + "addressbook.Email" "addressbook.List") OR NOT EQUAL + "addressbook.Email" "i;octet" NIL NOT EQUAL + "addressbook.List" "i;octet" NIL + S: A046 ENTRY "/addressbook/user/joe/A0345" "fred" + "fred@stone.org" NIL + S: A046 ENTRY "/addressbook/user/fred/A0537" "joe" "joe@stone.org" + NIL + S: A046 ENTRY "/addressbook/group/Dinosaur Operators/A423" + "saurians" NIL "1" + S: A046 MODTIME "19970728105252" + S: A046 OK "SEARCH completed" + + C: A047 SEARCH "/addressbook/user/fred/" RETURN ("*") EQUAL "entry" + "i;octet" "A0345" + S: A047 ENTRY "A0345" (("modtime" "19970728102226") + + + +Newman & Myers Standards Track [Page 38] + +RFC 2244 ACAP November 1997 + + + ("addressbook.Alias" "fred") ("addressbook.Email" + "fred@stone.org") ("addressbook.CommonName" + "Fred Flintstone") ("addressbook.Surname" "Flintstone") + ("addressbook.GivenName" "Fred")) + S: A047 MODTIME "19970728105258" + S: A047 OK "SEARCH completed" + + C: A048 SEARCH "/options/~/vendor.example/" RETURN + ("option.value"("size" "value" "myrights")) + SORT ("entry" "i;octet") COMPARE "modtime" "i;octet" + "19970727123225" + S: A048 ENTRY "blurdybloop" (5 "ghoti" "rwia") + S: A048 ENTRY "buckybits" (2 "10" "rwia") + S: A048 ENTRY "windowSize" (7 "100x100" "rwia") + S: A048 MODTIME "19970728105304" + S: A048 OK "SEARCH completed" + + C: A049 SEARCH "/addressbook/~/public" RETURN ("addressbook.Alias" + "addressbook.Email") MAKECONTEXT ENUMERATE "blob" LIMIT 100 1 + SORT ("addressbook.Alias" "i;octet") NOT EQUAL + "addressbook.Email" NIL + S: A049 ENTRY "A437" "aaguy" "aaguy@stone.org" + S: A049 MODTIME "19970728105308" + S: A049 OK (TOOMANY 347) "Context 'blob' created" + + C: A050 SEARCH "blob" RANGE 2 2 "19970728105308" ALL + S: A050 ENTRY "A238" "abguy" "abguy@stone.org" + S: A050 MODTIME "19970728105310" + S: A050 OK "SEARCH Completed" + +6.5. Contexts + + The following commands use contexts created by a SEARCH command with + a MAKECONTEXT modifier. + + +6.5.1. FREECONTEXT Command + + Arguments: context name + + Data: no specific data for this command + + Result: OK - freecontext completed + NO - freecontext failure: no such context + BAD - command unknown or arguments invalid + + + + + + +Newman & Myers Standards Track [Page 39] + +RFC 2244 ACAP November 1997 + + + The FREECONTEXT command causes the server to free all state + associated with the named context. The context may no longer be + searched and the server will no longer issue any untagged + responses for the context. The context is no longer counted + against the server's limit on the number of contexts. + + Example: C: A683 FREECONTEXT "blurdybloop" + S: A683 OK "Freecontext completed" + + +6.5.2. UPDATECONTEXT Command + + Arguments: list of context names + + Data: untagged responses: ADDTO REMOVEFROM CHANGE MODTIME + + Result: OK - Updatecontext completed: all updates completed + NO - Updatecontext failed: no such context + not a notify context + BAD - command unknown or arguments invalid + + The UPDATECONTEXT command causes the server to ensure that the + client is notified of all changes known to the server for the + contexts listed as arguments up to the current time. The contexts + listed in the arguments must have been previously given to a + successful SEARCH command with a MAKECONTEXT NOTIFY modifier. A + MODTIME untagged response MUST be returned if any read-write + metadata in the context changed since the last MODTIME for that + context. This includes metadata which is not listed in the RETURN + modifier for the context. + + While a server may issue untagged ADDTO, REMOVEFROM, CHANGE, and + MODTIME at any time, the UPDATECONTEXT command is used to "prod" + the server to send any notifications it has not sent yet. + + The UPDATECONTEXT command SHOULD NOT be used to poll for updates. + + Example: C: Z4S9 UPDATECONTEXT "blurdybloop" "blarfl" + S: Z4S9 OK "client has been notified of all changes" + + +6.5.3. ADDTO Untagged Response + + Data: context name + entry name + position + metadata list + + + + +Newman & Myers Standards Track [Page 40] + +RFC 2244 ACAP November 1997 + + + The untagged ADDTO response informs the client that an entry has + been added to a context. The response includes the position + number of the added entry (the first entry in the context is + numbered 1) and those metadata contained in the entry which match + the RETURN statement when the context was created. + + For enumerated contexts, the ADDTO response implicitly adds one to + the position of all members of the context which had position + numbers that were greater than or equal to the ADDTO position + number. For non-enumerated contexts, the position field is always + 0. + + Example: S: * ADDTO "blurdybloop" "fred" 15 + ("addressbook.Email" "fred@stone.org") + + +6.5.4. REMOVEFROM Untagged Response + + Data: context name + entry name + old position + + The untagged REMOVEFROM response informs the client that an entry + has been removed from a context. The response includes the + position number that the removed entry used to have (the first + entry in the context is numbered 1). + + For enumerated contexts, the REMOVEFROM response implicitly + subtracts one from the position numbers of all members of the + context which had position numbers greater than the REMOVEFROM + position number. For non-enumerated contexts, the position field + is always 0. + + Example: S: * REMOVEFROM "blurdybloop" "fred" 15 + + +6.5.5. CHANGE Untagged Response + + Data: context name + entry name + old position + new position + metadata list + + The untagged CHANGE response informs the client that an entry in a + context has either changed position in the context or has changed + the values of one or more of the attributes specified in the + RETURN modifier when the context was created. + + + +Newman & Myers Standards Track [Page 41] + +RFC 2244 ACAP November 1997 + + + The response includes the previous and current position numbers of + the entry (which are 0 if ENUMERATE was not specified on the + context) and the attribute metadata requested in the RETURN + modifier when the context was created. + + For enumerated contexts, the CHANGE response implicitly changes + the position numbers of all entries which had position numbers + between the old and new position. If old position is less than + new position, than one is subtracted from all entries which had + position numbers in that range. Otherwise one is added to all + entries which had position numbers in that range. If the old + position and new position are the same, then no implicit position + renumbering occurs. + + CHANGE responses are not issued for entries which have changed + position implicitly due to another ADDTO, REMOVEFROM or CHANGE + response. + + Example: S: * CHANGE "blurdybloop" "fred" 15 10 + ("addressbook.Email" "fred@stone.org") + + +6.5.6. MODTIME Untagged Response + + Data: context name + modtime value + + The untagged MODTIME response informs the client that it has + received all updates to entries in the context which have modtime + values less than or equal to the modtime value in the argument. + + Example: S: * MODTIME mycontext "19970320162338" + +6.6. Dataset modification + + The following commands and responses handle modification of datasets. + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 42] + +RFC 2244 ACAP November 1997 + + +6.6.1. STORE Command + + Arguments: entry store list + + Data: intermediate responses: ENTRY + + Result: OK - store completed + NO - store failure: can't store that name + UNCHANGEDSINCE specified and entry changed + BAD - command unknown or arguments invalid + invalid UTF-8 syntax in attribute name + + + Creates, modifies, or deletes the named entries in the named + datasets. The values of metadata not specified in the command are + not changed. Setting the "value" metadata of an attribute to NIL + removes that attribute from the entry. Setting the "value" of the + "entry" attribute to NIL removes that entry from the dataset and + cancels inheritance for the entire entry. Setting the "value" of + the "entry" attribute to DEFAULT removes that entry from the + inheriting dataset and reverts the entry and its attributes to + inherited values, if any. Changing the value of the "entry" + attribute renames the entry. + + Storing DEFAULT to the "value" metadata of an attribute is + equivalent to storing NIL, except that inheritance is enabled for + that attribute. If a non-NIL value is inherited then an ENTRY + intermediate response is generated to notify the client of the + this change. The ENTRY response includes the entry-path and the + attribute name and value metadata for each attribute which + reverted to a non-NIL inherited setting. + + Storing NIL to the "value" metadata of an attribute MAY be treated + equivalent to storing DEFAULT to that "value" if there is a NIL + value in the base dataset. + + The STORE command is followed by one or more entry store lists. + Each entry store list begins with an entry path followed by STORE + modifiers, followed by zero or more attribute store items. Each + attribute store item is made up of the attribute name followed by + NIL (to remove the attribute's value), DEFAULT (to revert the item + to any inherited value), a single value (to set the attribute's + single value), or a list of metadata items to modify. The + following STORE modifiers may be specified: + + + + + + + +Newman & Myers Standards Track [Page 43] + +RFC 2244 ACAP November 1997 + + + NOCREATE + By default, the server MUST create any datasets necessary to + store the entry, including multiple hierarchy levels. If + NOCREATE is specified, the STORE command will fail with a + NOEXIST error unless the parent dataset already exists. + + UNCHANGEDSINCE + If the "modtime" of the entry is later than the + unchangedsince time, then the store fails with a MODIFIED + response code. Use of UNCHANGEDSINCE with a time of + "00000101000000" will always fail if the entry exists. + Clients writing to a shared dataset are encouraged to use + UNCHANGEDSINCE when modifying an existing entry. + + + The server MUST either make all the changes specified in a single + STORE command or make none of them. If successful, the server + MUST update the "modtime" attribute for every entry which was + changed. + + It is illegal to list any metadata item within an attribute twice, + any attribute within an entry twice or any entry path twice. The + server MUST return a BAD response if this happens. + + The server MAY re-order the strings in a multi-value on STORE and + MAY remove duplicate strings. However, SEARCH MUST return multi- + values and the associated size list metadata in a consistant + order. + + + Example: C: A342 STORE ("/addressbook/user/fred/ABC547" + "addressbook.TelephoneNumber" "555-1234" + "addressbook.CommonName" "Barney Rubble" + "addressbook.AlternateNames" ("value" + ("Barnacus Rubble" "Coco Puffs Thief")) + "addressbook.Email" NIL) + S: A342 OK "Store completed" + C: A343 STORE ("/addressbook/user/joe/ABD42" + UNCHANGEDSINCE "19970320162338" + "user.joe.hair-length" "10 inches") + S: A343 NO (MODIFIED) "'ABD42' has been changed + by somebody else." + C: A344 STORE ("/addressbook/group/Developers/ACD54" + "entry" NIL) + S: A344 OK "Store completed" + C: A345 STORE ("/option/~/common/SMTPserver" + "option.value" DEFAULT) + S: A345 ENTRY "/option/~/common/SMTPserver" + + + +Newman & Myers Standards Track [Page 44] + +RFC 2244 ACAP November 1997 + + + "option.value" "smtp.server.do.main" + S: A345 OK "Store completed" + C: A347 STORE ("/addressbook/~/" "dataset.inherit" + "/addressbook/group/Developers") + S: A347 OK "Store completed" + + +6.6.2. DELETEDSINCE Command + + Arguments: dataset name + time + + Data: intermediate response: DELETED + + Result: OK - DELETEDSINCE completed + NO - DELETEDSINCE failure: can't read dataset + date too far in the past + BAD - command unknown or arguments invalid + + The DELETEDSINCE command returns in intermediate DELETED replies + the names of entries that have been deleted from the named dataset + since the given time. + + Servers may impose a limit on the number or age of deleted entry + names they keep track of. If the server does not have information + going back to the specified time, the command fails, returning a + TOOOLD response code in the tagged NO response. + + Example: C: Z4S9 DELETEDSINCE "/folder/site/" 19951205103412 + S: Z4S9 DELETED "blurdybloop" + S: Z4S9 DELETED "anteaters" + S: Z4S9 OK "DELETEDSINCE completed" + C: Z4U3 DELETEDSINCE "/folder/site/" 19951009040854 + S: Z4U3 NO (TOOOLD) "Don't have that information" + + +6.6.3. DELETED Intermediate Response + + Data: entry name + + The intermediate DELETED response occurs as a result of a + DELETEDSINCE command. It returns an entry that has been deleted + from the dataset specified in the DELETEDSINCE command. + +6.7. Access Control List Commands + + The commands in this section are used to manage access control lists. + + + + +Newman & Myers Standards Track [Page 45] + +RFC 2244 ACAP November 1997 + + +6.7.1. SETACL Command + + Arguments: acl object + authentication identifier + access rights + + Data: no specific data for this command + + Result: OK - setacl completed + NO - setacl failure: can't set acl + BAD - command unknown or arguments invalid + + The SETACL command changes the access control list on the + specified object so that the specified identifier is granted the + permissions enumerated in rights. If the object did not + previously have an access control list, one is created. + + + Example: C: A123 SETACL ("/addressbook/~/public/") "anyone" "r" + S: A123 OK "Setacl complete" + C: A124 SETACL ("/folder/site/") "B1FF" "rwa" + S: A124 NO (PERMISSION ("/folder/site/")) "'B1FF' not + permitted to modify access rights + for '/folder/site/'" + + + +6.7.2. DELETEACL Command + + Arguments: acl object + optional authentication identifier + + Data: no specific data for this command + + Result: OK - deleteacl completed + NO - deleteacl failure: can't delete acl + BAD - command unknown or arguments invalid + + If given the optional identifier argument, the DELETEACL command + removes any portion of the access control list on the specified + object for the specified identifier. + + If not given the optional identifier argument, the DELETEACL + command removes the ACL from the object entirely, causing access + to be controlled by a higher-level default ACL. This form of the + DELETEACL command is not permitted on the default ACL for a + dataset and servers MUST return a BAD. + + + + +Newman & Myers Standards Track [Page 46] + +RFC 2244 ACAP November 1997 + + + Example: C: A223 DELETEACL ("/addressbook/~/public") "anyone" + S: A223 OK "Deleteacl complete" + C: A224 DELETEACL ("/folder/site") + S: A224 BAD "Can't delete ACL from dataset" + C: A225 DELETEACL ("/addressbook/user/fred" + "addressbook.Email" "barney") + S: A225 OK "Deleteacl complete" + + +6.7.3. MYRIGHTS Command + + Arguments: acl object + + Data: intermediate responses: MYRIGHTS + + Result: OK - myrights completed + NO - myrights failure: can't get rights + BAD - command unknown or arguments invalid + + The MYRIGHTS command returns the set of rights that the client has + to the given dataset or dataset attribute. + + + Example: C: A003 MYRIGHTS ("/folder/site") + S: A003 MYRIGHTS "r" + S: A003 OK "Myrights complete" + + +6.7.4. MYRIGHTS Intermediate Response + + Data: rights + + The MYRIGHTS response occurs as a result of a MYRIGHTS command. + The argument is the set of rights that the client has for the + object referred to in the MYRIGHTS command. + +6.7.5. LISTRIGHTS Command + + Arguments: acl object + authentication identifier + + Data: untagged responses: LISTRIGHTS + + Result: OK - listrights completed + NO - listrights failure: can't get rights list + BAD - command unknown or arguments invalid + + + + + +Newman & Myers Standards Track [Page 47] + +RFC 2244 ACAP November 1997 + + + The LISTRIGHTS command takes an object and an identifier and + returns information about what rights the current user may revoke + or grant to that identifier in the ACL for that object. + + Example: C: a001 LISTRIGHTS ("/folder/~/") "smith" + S: a001 LISTRIGHTS "xra" "w" "i" + S: a001 OK Listrights completed + C: a005 LISTRIGHTS ("/folder/site/archive/imap") "anyone" + S: a005 LISTRIGHTS "" "x" "r" "w" "i" + S: a005 OK Listrights completed + + + +6.7.6. LISTRIGHTS Intermediate Response + + Data: required rights + list of optional rights + + The LISTRIGHTS response occurs as a result of a LISTRIGHTS + command. The first argument is a string containing the (possibly + empty) set of rights the identifier will always be granted on the + dataset or attribute. + + Following this are zero or more strings each containing a single + right which the current user may revoke or grant to the identifier + in the dataset or attribute. + + The same right MUST NOT be listed more than once in the LISTRIGHTS + response. + + +6.8. Quotas + + The section defines the commands and responses relating to quotas. + + +6.8.1. GETQUOTA Command + + Arguments: dataset + + Data: untagged responses: QUOTA + + Result: OK - Quota information returned + NO - Quota failure: can't access resource limit + no resource limit + BAD - command unknown or arguments invalid + + + + + +Newman & Myers Standards Track [Page 48] + +RFC 2244 ACAP November 1997 + + + The GETQUOTA command takes the name of a dataset, and returns in + an untagged QUOTA response the name of the dataset, quota limit in + bytes that applies to that dataset and the quota usage within that + limit. The scope of a quota limit is implementation dependent. + + Example: C: A043 GETQUOTA "/option/user/fred/common" + S: * QUOTA "/option/user/fred/common" 1048576 2475 + S: A043 OK "Getquota completed" + + +6.8.3. QUOTA Untagged Response + + Data: dataset + quota limit in bytes + amount of quota limit used + extension data + + The QUOTA untagged response is generated as a result of a GETQUOTA + command or MAY be generated by the server in response to a SEARCH + or STORE command to warn about high usage of a quota. It includes + the name of the applicable dataset, the quota limit in bytes, the + quota usage and some optional extension data. Clients MUST + tolerate the extension data as its use is reserved for a future + extension. + +6.9. Extensions + + In order to simplify the process of extending the protocol, clients + MUST tolerate unknown server responses which meet the syntax of + response-extend. In addition, clients MUST tolerate unknown server + response codes which meet the syntax of resp-code-ext. Availability + of new commands MUST be announced via a capability on the initial + greeting line and such commands SHOULD meet the syntax of + command-extend. + + Servers MUST respond to unknown commands with a BAD command + completion result. Servers MUST skip over non-synchronizing literals + contained in an unknown command. This may be done by assuming the + unknown command matches the command-extend syntax, or by reading a + line at a time and checking for the non-synchronizing literal syntax + at the end of the line. + +7. Registration Procedures + + ACAP's usefulness comes from providing a structured storage model for + all sorts of configuration data. However, for its potential to be + achieved, it is important that the Internet community strives for the + following goals: + + + +Newman & Myers Standards Track [Page 49] + +RFC 2244 ACAP November 1997 + + + (1) Standardization. It is very important to standardize dataset + classes. The authors hope that ACAP achieves the success that SNMP + has seen with the definition of numerous standards track MIBs. + + (2) Community Review. In the absence of standardization, it is + important to get community review on a proposal to improve its + engineering quality. Community review is strongly recommended prior + to registration. The ACAP implementors mailing list + should be used for this purpose. + + (3) Registration. Registration serves a two-fold purpose. First it + prevents use of the same name for different purposes, and second it + provides a one-stop list which can be used to locate existing + extensions or dataset classes to prevent duplicate work. + + The following registration templates may be used to register ACAP + protocol elements with the Internet Assigned Numbers Authority + (IANA). + +7.1. ACAP Capabilities + + New ACAP capabilities MUST be registered prior to use. Careful + consideration should be made before extending the protocol, as it can + lead to complexity or interoperability problems. Review of proposals + on the acap implementors mailing list is strongly encouraged prior to + registration. + + To: iana@iana.org + Subject: Registration of ACAP capability + + Capability name: + + Capability keyword: + + Capability arguments: + + Published Specification(s): + + (Optional, but strongly encouraged) + + Person and email address to contact for further information: + +7.2. ACAP Response Codes + + ACAP response codes are registered on a first come, first served + basis. Review of proposals on the acap implementors mailing list is + strongly encouraged prior to registration. + + + + +Newman & Myers Standards Track [Page 50] + +RFC 2244 ACAP November 1997 + + + To: iana@iana.org + Subject: Registration of ACAP response code + + Response Code: + + Arguments (use ABNF to specify syntax): + + Purpose: + + Published Specification(s): + + (Optional, but strongly encouraged) + + Person and email address to contact for further information: + +7.3. Dataset Classes + + A dataset class provides a core set of attributes for use in a + specified hierarchy. It may also define rules for the dataset + hierarchy underneath that class. Dataset class specifications must + be standards track or IESG approved experimental RFCs. + + To: iana@iana.org + Subject: Registration of ACAP dataset class + + Dataset class name/attribute prefix: + + Purpose: + + Published Specification(s): + + (Standards track or IESG approved experimental RFC) + + Person and email address to contact for further information: + +7.4. Vendor Subtree + + Vendors may reserve a portion of the ACAP namespace for private use. + Dataset class names beginning with "vendor.." + are reserved for use by that company or product. In addition, all + attribute names beginning with "vendor.." are + reserved for use by that company or product once registered. + Registration is on a first come, first served basis. Whenever + possible, private attributes and dataset classes should be avoided in + favor of improving interoperable dataset class definitions. + + + + + + +Newman & Myers Standards Track [Page 51] + +RFC 2244 ACAP November 1997 + + + To: iana@iana.org + Subject: Registration of ACAP vendor subtree + + Private Prefix: vendor.. + + Person and email address to contact for further information: + + (company names and addresses should be included when appropriate) + +8. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [ABNF]. This uses the ABNF core + rules as specified in Appendix A of the ABNF specification [ABNF]. + + Except as noted otherwise, all alphabetic characters are + case-insensitive. The use of upper or lower case characters to + define token strings is for editorial clarity only. Implementations + MUST accept these strings in a case-insensitive fashion. + + The "initial-greeting" rule below defines the initial ACAP greeting + from the server. The "command" rule below defines the syntax for + commands sent by the client. The "response" rule below defines the + syntax for responses sent by the server. + + ATOM-CHAR = "!" / %x23-27 / %x2A-5B / %x5D-7A / %x7C-7E + ;; Any CHAR except ATOM-SPECIALS + + ATOM-SPECIALS = "(" / ")" / "{" / SP / CTL / QUOTED-SPECIALS + + CHAR = %x01-7F + + DIGIT-NZ = %x31-39 + ; non-zero digits ("1" - "9") + + QUOTED-CHAR = SAFE-UTF8-CHAR / "\" QUOTED-SPECIALS + + QUOTED-SPECIALS = <"> / "\" + + SAFE-CHAR = %x01-09 / %x0B-0C / %x0E-21 / + %x23-5B / %x5D-7F + ;; any TEXT-CHAR except QUOTED-SPECIALS + + SAFE-UTF8-CHAR = SAFE-CHAR / UTF8-2 / UTF8-3 / UTF8-4 / + UTF8-5 / UTF8-6 + + TAG-CHAR = %x21 / %x23-27 / %x2C-5B / %x5D-7A / %x7C-7E + ;; Any ATOM-CHAR except "*" or "+" + + + +Newman & Myers Standards Track [Page 52] + +RFC 2244 ACAP November 1997 + + + TEXT-CHAR = %x01-09 / %x0B-0C / %x0E-7F + ;; any CHAR except CR and LF + + TEXT-UTF8-CHAR = SAFE-UTF8-CHAR / QUOTED-SPECIALS + + UTF8-1 = %x80-BF + + UTF8-2 = %xC0-DF UTF8-1 + + UTF8-3 = %xE0-EF 2UTF8-1 + + UTF8-4 = %xF0-F7 3UTF8-1 + + UTF8-5 = %xF8-FB 4UTF8-1 + + UTF8-6 = %xFC-FD 5UTF8-1 + + UTF8-CHAR = TEXT-UTF8-CHAR / CR / LF + + acl = "(" [acl-identrights *(SP acl-identrights)] ")" + *(SPACE acl-identrights)] ")" + + acl-identifier = string-utf8 + ;; MUST NOT contain HTAB + + acl-identrights = string-utf8 + ;; The identifier followed by a HTAB, + ;; followed by the rights. + + acl-delobject = "(" dataset SP attribute [SP entry-name] ")" + + acl-object = "(" dataset [SP attribute [SP entry-name]] ")" + + acl-rights = quoted + + atom = ALPHA *1023ATOM-CHAR + + attribute = string-utf8 + ;; dot-separated attribute name + ;; MUST NOT contain "*" or "%" + + attribute-store = attribute SP (value-nildef / + "(" 1*(metadata-write-q SP value-store) ")") + ;; MUST NOT include the same metadata twice + + auth-type = <"> auth-type-name <"> + + + + + +Newman & Myers Standards Track [Page 53] + +RFC 2244 ACAP November 1997 + + + auth-type-name = iana-token + ;; as defined in SASL [SASL] + + command = tag SP (command-any / command-auth / + command-nonauth) CRLF + ;; Modal based on state + + command-authent = "AUTHENTICATE" SP auth-type + [SP string] *(CRLF string) + + command-any = "NOOP" / command-lang / "LOGOUT" / + command-extend + + command-auth = command-delacl / command-dsince / + command-freectx / command-getquota / + command-lrights / command-myrights / + command-search / command-setacl / + command-store + ;; only valid in authenticated state + + command-delacl = "DELETEACL" SP acl-delobject [SP acl-identifier] + + command-dsince = "DELETEDSINCE" SP dataset SP time + + command-extend = extend-token [SP extension-data] + + command-freectx = "FREECONTEXT" SP context + + command-getquota = "GETQUOTA" SP dataset + + command-lang = "LANG" *(SP lang-tag) + + command-lrights = "LISTRIGHTS" SP acl-object + + command-myrights = "MYRIGHTS" SP acl-object + + command-nonauth = command-authent + ;; only valid in non-authenticated state + + command-search = "SEARCH" SP (dataset / context) + *(SP search-modifier) SP search-criteria + ;; MUST NOT include same search-modifier twice + + command-setacl = "SETACL" SP acl-object SP acl-identifier + SP acl-rights + + command-store = "STORE" SP store-entry-list + + + + +Newman & Myers Standards Track [Page 54] + +RFC 2244 ACAP November 1997 + + + comparator = <"> comparator-name <"> + + comparator-name = ["+" / "-"] iana-token + + context = string-utf8 + ;; MUST NOT begin with slash ("/") + + dataset = string-utf8 + ;; slash-separated dataset name + ;; begins with slash + + entry = entry-name / entry-path + + entry-name = string-utf8 + ;; entry name MUST NOT contain slash + ;; MUST NOT begin with "." + + entry-path = string-utf8 + ;; slash-separated path to entry + ;; begins with slash + + entry-relative = string-utf8 + ;; potentially relative path to entry + + extend-token = atom + ;; MUST be defined by a standards track or + ;; IESG approved experimental protocol extension + + extension-data = extension-item *(SP extension-item) + + extension-item = extend-token / string / number / + "(" [extension-data] ")" + + iana-token = atom + ;; MUST be registered with IANA + + initial-greeting = "*" SP "ACAP" *(SP "(" init-capability ")") CRLF + + init-capability = init-cap-context / init-cap-extend / + init-cap-implem / init-cap-sasl + + init-cap-context = "CONTEXTLIMIT" SP string + + init-cap-extend = iana-token [SP string-list] + + init-cap-implem = "IMPLEMENTATION" SP string + + init-cap-sasl = "SASL" SP string-list + + + +Newman & Myers Standards Track [Page 55] + +RFC 2244 ACAP November 1997 + + + lang-tag = <"> Language-Tag <"> + ;; Language-Tag rule is defined in [LANG-TAGS] + + literal = "{" number [ "+" ] "}" CRLF *OCTET + ;; The number represents the number of octets + ;; MUST be literal-utf8 except for values + + literal-utf8 = "{" number [ "+" ] "}" CRLF *UTF8-CHAR + ;; The number represents the number of octets + ;; not the number of characters + + metadata = attribute [ "(" metadata-type-list ")" ] + ;; attribute MAY end in "*" as wildcard. + + metadata-list = metadata *(SP metadata) + + metadata-type = "attribute" / "myrights" / "size" / + "count" / metadata-write + + metadata-type-q = <"> metadata-type <"> + + metadata-type-list = metadata-type-q *(SP metadata-type-q) + + metadata-write = "value" / "acl" + + metadata-write-q = <"> metadata-write <"> + + nil = "NIL" + + number = *DIGIT + ;; A 32-bit unsigned number. + ;; (0 <= n < 4,294,967,296) + + nz-number = DIGIT-NZ *DIGIT + ;; A 32-bit unsigned non-zero number. + ;; (0 < n < 4,294,967,296) + + position = number + ;; "0" if context is not enumerated + ;; otherwise this is non-zero + + quota-limit = number + + quota-usage = number + + quoted = <"> *QUOTED-CHAR <"> + ;; limited to 1024 octets between the <">s + + + + +Newman & Myers Standards Track [Page 56] + +RFC 2244 ACAP November 1997 + + + response = response-addto / response-alert / response-bye / + response-change / response-cont / + response-deleted / response-done / + response-entry / response-extend / + response-listr / response-lang / + response-mtimei / response-mtimeu / + response-myright / response-quota / + response-refer / response-remove / response-stat + + response-addto = "*" SP "ADDTO" SP context SP entry-name + SP position SP return-data-list + + response-alert = "*" SP "ALERT" SP resp-body CRLF + ;; Client MUST display alert text to user + + response-bye = "*" SP "BYE" SP resp-body CRLF + ;; Server will disconnect condition + + response-change = "*" SP "CHANGE" SP context SP entry-name + SP position SP position SP return-data-list + + response-cont = "+" SP string + + response-deleted = tag SP "DELETED" SP entry-name + + response-done = tag SP resp-cond-state CRLF + + response-entry = tag SP "ENTRY" SP entry SP return-data-list + + response-extend = (tag / "*") SP extend-token [SP extension-data] + + response-lang = "*" SP "LANG" SP lang-tag 1*(SP comparator) + + response-listr = tag SP "LISTRIGHTS" SP acl-rights + *(SP acl-rights) + + response-mtimei = tag SP "MODTIME" SP time + + response-mtimeu = "*" SP "MODTIME" SP context SP time + + response-myright = tag SP "MYRIGHTS" SP acl-rights + + response-quota = "*" SP "QUOTA" SP dataset SP quota-limit + SP quota-usage [SP extension-data] + + response-refer = tag SP "REFER" SP dataset + 1*(SP <"> url-relative <">) + + + + +Newman & Myers Standards Track [Page 57] + +RFC 2244 ACAP November 1997 + + + response-remove = "*" SP "REMOVEFROM" SP context SP + entry-name SP position + + response-stat = "*" SP resp-cond-state CRLF + + resp-body = ["(" resp-code ")" SP] quoted + + resp-code = "AUTH-TOO-WEAK" / "ENCRYPT-NEEDED" / + resp-code-inval / resp-code-mod / + resp-code-noexist / resp-code-perm / "QUOTA" / + resp-code-refer / resp-code-sasl / + resp-code-toomany / "TOOOLD" / + "TRANSITION-NEEDED" / "TRYFREECONTEXT" / + "TRYLATER" / "WAYTOOMANY" / resp-code-ext + + resp-code-ext = iana-token [SP extension-data] + ;; unknown codes MUST be tolerated by the client + + resp-code-inval = "INVALID" 1*(SP entry-path SP attribute) + + resp-code-mod = "MODIFIED" SP entry-path + + resp-code-noexist = "NOEXIST" SP dataset + + resp-code-perm = "PERMISSION" SP acl-object + + resp-code-refer = "REFER" 1*(SP <"> url-relative <">) + + resp-code-sasl = "SASL" SP string + + resp-code-toomany = "TOOMANY" SP nz-number + + resp-cond-state = ("OK" / "NO" / "BAD") SP resp-body + ;; Status condition + + return-attr-list = "(" return-metalist *(SP return-metalist) ")" + ;; occurs when "*" in RETURN pattern on SEARCH + + return-data = return-metadata / return-metalist / + return-attr-list + + return-data-list = return-data *(SP return-data) + + return-metalist = "(" return-metadata *(SP return-metadata) ")" + ;; occurs when multiple metadata items requested + + return-metadata = nil / string / value-list / acl + + + + +Newman & Myers Standards Track [Page 58] + +RFC 2244 ACAP November 1997 + + + searchkey-equal = "EQUAL" SP attribute SP comparator SP value-nil + + searchkey-comp = "COMPARE" SP attribute SP comparator SP value + + searchkey-prefix = "PREFIX" SP attribute SP comparator SP value + + searchkey-range = "RANGE" SP nz-number SP nz-number SP time + + searchkey-strict = "COMPARESTRICT" SP attribute SP comparator + SP value + + searchkey-substr = "SUBSTRING" SP attribute SP comparator SP value + + searchmod-depth = "DEPTH" SP number + + searchmod-hard = "HARDLIMIT" SP nz-number + + searchmod-limit = "LIMIT" SP number SP number + + searchmod-make = "MAKECONTEXT" [SP "ENUMERATE"] + [SP "NOTIFY"] SP context + + searchmod-ninh = "NOINHERIT" + + searchmod-return = "RETURN" SP "(" [metadata-list] ")" + + searchmod-sort = "SORT" SP "(" sort-list ")" + + search-criteria = "ALL" / searchkey-equal / searchkey-comp / + searchkey-strict / searchkey-range / + searchkey-prefix / searchkey-substr / + "NOT" SP search-criteria / + "OR" SP search-criteria SP search-criteria / + "AND" SP search-criteria SP search-criteria + + search-modifier = searchmod-depth / searchmod-hard / + searchmod-limit / searchmod-make / + searchmod-ninh / searchmod-return / + searchmod-sort + + sort-list = sort-item *(SP sort-item) + + sort-item = attribute SP comparator + + store-entry = "(" entry-path *(SP store-modifier) + *(SP attribute-store) ")" + ;; MUST NOT include same store-modifier twice + ;; MUST NOT include same attribute twice + + + +Newman & Myers Standards Track [Page 59] + +RFC 2244 ACAP November 1997 + + + store-entry-list = store-entry *(SP store-entry) + ;; MUST NOT include same entry twice + + store-modifier = store-mod-unchang / store-mod-nocreate + + store-mod-nocreate = "NOCREATE" + + store-mod-unchang = "UNCHANGEDSINCE" SP time + + string = quoted / literal + + string-list = string *(SP string) + + string-utf8 = quoted / literal-utf8 + + tag = 1*32TAG-CHAR + + time = <"> time-year time-month time-day time-hour + time-minute time-second time-subsecond <"> + ;; Timestamp in UTC + + time-day = 2DIGIT ;; 01-31 + + time-hour = 2DIGIT ;; 00-23 + + time-minute = 2DIGIT ;; 00-59 + + time-month = 2DIGIT ;; 01-12 + + time-second = 2DIGIT ;; 00-60 + + time-subsecond = *DIGIT + + time-year = 4DIGIT + + value = string + + value-list = "(" [value *(SP value)] ")" + + value-nil = value / nil + + value-nildef = value-nil / "DEFAULT" + + value-store = value-nildef / value-list / acl + + url-acap = "acap://" url-server "/" url-enc-entry + [url-filter] [url-extension] + ;; url-enc-entry interpreted relative to "/" + + + +Newman & Myers Standards Track [Page 60] + +RFC 2244 ACAP November 1997 + + + url-attr-list = url-enc-attr *("&" url-enc-attr) + + url-auth = ";AUTH=" ("*" / url-enc-auth) + + url-achar = uchar / "&" / "=" / "~" + ;; See RFC 1738 for definition of "uchar" + + url-char = uchar / "=" / "~" / ":" / "@" / "/" + ;; See RFC 1738 for definition of "uchar" + + url-enc-attr = 1*url-char + ;; encoded version of attribute name + + url-enc-auth = 1*url-achar + ;; encoded version of auth-type-name above + + url-enc-entry = 1*url-char + ;; encoded version of entry-relative above + + url-enc-user = *url-achar + ;; encoded version of login userid + + url-extension = *("?" 1*url-char) + + url-filter = "?" url-attr-list + + url-relative = url-acap / [url-enc-entry] [url-filter] + ;; url-enc-entry is relative to base URL + + url-server = [url-enc-user [url-auth] "@"] hostport + ;; See RFC 1738 for definition of "hostport" + +9. Multi-lingual Considerations + + The IAB charset workshop [IAB-CHARSET] came to a number of + conclusions which influenced the design of ACAP. The decision to use + UTF-8 as the character encoding scheme was based on that work. The + LANG command to negotiate a language for error messages is also + included. + + Section 3.4.5 of the IAB charset workshop report states that there + should be a way to identify the natural language for human readable + strings. Several promising proposals have been made for use within + ACAP, but no clear consensus on a single method is apparent at this + stage. The following rules are likely to permit the addition of + multi-lingual support in the future: + + + + + +Newman & Myers Standards Track [Page 61] + +RFC 2244 ACAP November 1997 + + + (1) A work in progress called Multi-Lingual String Format (MLSF) + proposes a layer on top of UTF-8 which uses otherwise illegal UTF-8 + sequences to store language tags. In order to permit its addition to + a future version of this standard, client-side UTF-8 interpreters + MUST be able to silently ignore illegal multi-byte UTF-8 characters, + and treat illegal single-byte UTF-8 characters as end of string + markers. Servers, for the time being, MUST be able to silently + accept illegal UTF-8 characters, except in attribute names and entry + names. Clients MUST NOT send illegal UTF-8 characters to the server + unless a future standard changes this rule. + + (2) There is a proposal to add language tags to Unicode. To support + this, servers MUST be able to store UTF-8 characters of up to 20 bits + of data. + + (3) The metadata item "language" is reserved for future use. + +10. Security Considerations + + The AUTHENTICATE command uses SASL [SASL] to provide basic + authentication, authorization, integrity and privacy services. This + is described in section 6.3.1. + + When the CRAM-MD5 mechanism is used, the security considerations for + the CRAM-MD5 SASL mechanism [CRAM-MD5] apply. The CRAM-MD5 mechanism + is also susceptible to passive dictionary attacks. This means that + if an authentication session is recorded by a passive observer, that + observer can try common passwords through the CRAM-MD5 mechanism and + see if the results match. This attack is reduced by using hard to + guess passwords. Sites are encouraged to educate users and have the + password change service test candidate passwords against a + dictionary. ACAP implementations of CRAM-MD5 SHOULD permit passwords + of at least 64 characters in length. + + ACAP protocol transactions are susceptible to passive observers or + man in the middle attacks which alter the data, unless the optional + encryption and integrity services of the AUTHENTICATE command are + enabled, or an external security mechanism is used for protection. + It may be useful to allow configuration of both clients and servers + to refuse to transfer sensitive information in the absence of strong + encryption. + + ACAP access control lists provide fine grained authorization for + access to attributes. A number of related security issues are + described in section 3.5. + + ACAP URLs have the same security considerations as IMAP URLs + [IMAP-URL]. + + + +Newman & Myers Standards Track [Page 62] + +RFC 2244 ACAP November 1997 + + + ACAP clients are encouraged to consider the security problems + involved with a lab computer situation. Specifically, a client cache + of ACAP configuration information MUST NOT allow access by an + unauthorized user. One way to assure this is for an ACAP client to + be able to completely flush any non-public cached configuration data + when a user leaves. + + As laptop computers can be easily stolen and a cache of configuration + data may contain sensitive information, a disconnected mode ACAP + client may wish to encrypt and password protect cached configuration + information. + +11. Acknowledgments + + Many thanks to the follow people who have contributed to ACAP over + the past four years: Wallace Colyer, Mark Crispin, Jack DeWinter, Rob + Earhart, Ned Freed, Randy Gellens, Terry Gray, J. S. Greenfield, + Steve Dorner, Steve Hole, Steve Hubert, Dave Roberts, Bart Schaefer, + Matt Wall and other participants of the IETF ACAP working group. + +12. Authors' Addresses + + Chris Newman + Innosoft International, Inc. + 1050 Lakes Drive + West Covina, CA 91790 USA + + Email: chris.newman@innosoft.com + + + John Gardiner Myers + Netscape Communications + 501 East Middlefield Road + Mail Stop MV-029 + Mountain View, CA 94043 + + Email: jgmyers@netscape.com + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 63] + +RFC 2244 ACAP November 1997 + + +Appendices + +A. References + + [ABNF] Crocker, Overell, "Augmented BNF for Syntax Specifications: + ABNF", RFC 2234, Internet Mail Consortium, Demon Internet Ltd, + November 1997. + + + + [BASIC-URL] Berners-Lee, Masinter, McCahill, "Uniform Resource + Locators (URL)", RFC 1738, CERN, Xerox Coproration, University of + Minnesota, December 1994. + + + + [CHARSET-LANG-POLICY] Alvestrand, "IETF Policy on Character Sets and + Languages", work in progress. + + [CRAM-MD5] Klensin, Catoe, Krumviede, "IMAP/POP AUTHorize Extension + for Simple Challenge/Response", RFC 2195, MCI, September 1997. + + + + [IAB-CHARSET] Weider, Preston, Simonsen, Alvestrand, Atkinson, + Crispin, Svanberg, "The Report of the IAB Character Set Workshop held + 29 February - 1 March, 1996", RFC 2130, April 1997. + + + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + + + [IMAP-ACL] Myers, J., "IMAP4 ACL extension", RFC 2086, Carnegie + Mellon, January 1997. + + + + [IMAP-URL] Newman, "IMAP URL Scheme", RFC 2192, Innosoft, July 1997. + + + + [ISO-10646] ISO/IEC 10646-1:1993(E) "Information Technology-- + Universal Multiple-octet Coded Character Set (UCS)." See also + amendments 1 through 7, plus editorial corrections. + + + + +Newman & Myers Standards Track [Page 64] + +RFC 2244 ACAP November 1997 + + + [ISO-C] "Programming languages -- C", ISO/IEC 9899:1990, + International Organization for Standardization. This is effectively + the same as ANSI C standard X3.159-1989. + + [KEYWORDS] Bradner, "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, Harvard University, March 1997. + + + + [LANG-TAGS] Alvestrand, H., "Tags for the Identification of + Languages", RFC 1766. + + + + [REL-URL] Fielding, "Relative Uniform Resource Locators", RFC 1808, + UC Irvine, June 1995. + + + + [SASL] Myers, J., "Simple Authentication and Security Layer (SASL)", + RFC 2222, Netscape Communications, October 1997. + + + + [SASL-ANON] Newman, C., "Anonymous SASL Mechanism", RFC 2245, + November 1997. + + [UNICODE-2] The Unicode Consortium, "The Unicode Standard, Version + 2.0", Addison-Wesley, 1996. ISBN 0-201-48345-9. + + [US-ASCII] "USA Standard Code for Information Interchange," X3.4. + American National Standards Institute: New York (1968). + + [UTF8] Yergeau, F. "UTF-8, a transformation format of Unicode and ISO + 10646", RFC 2044, Alis Technologies, October 1996. + + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 65] + +RFC 2244 ACAP November 1997 + + +B. ACAP Keyword Index + + + ACAP (untagged response) ................................... 26 + ADDTO (untagged response) .................................. 40 + ALERT (untagged response) .................................. 31 + ALL (search keyword) ....................................... 36 + AND (search keyword) ....................................... 36 + AUTH-TOO-WEAK (response code) .............................. 19 + AUTHENTICATE (command) ..................................... 31 + BAD (response) ............................................. 30 + BYE (untagged response) .................................... 30 + CHANGE (untagged response) ................................. 41 + COMPARE (search keyword) ................................... 36 + COMPARESTRICT (search keyword) ............................. 36 + CONTEXTLIMIT (ACAP capability) ............................. 27 + DELETEACL (command) ........................................ 46 + DELETED (intermediate response) ............................ 45 + DELETEDSINCE (command) ..................................... 45 + DEPTH (search modifier) .................................... 34 + ENCRYPT-NEEDED (response code) ............................. 19 + ENTRY (intermediate response) .............................. 37 + EQUAL (search keyword) ..................................... 37 + FREECONTEXT (command) ...................................... 39 + GETQUOTA (command) ......................................... 48 + HARDLIMIT (search modifier) ................................ 34 + IMPLEMENTATION (ACAP capability) ........................... 27 + INVALID (response code) .................................... 19 + LANG (command) ............................................. 28 + LANG (intermediate response) ............................... 28 + LIMIT (search modifier) .................................... 34 + LISTRIGHTS (command) ....................................... 47 + LISTRIGHTS (intermediate response) ......................... 48 + LOGOUT (command) ........................................... 29 + MAKECONTEXT (search modifier) .............................. 34 + MODIFIED (response code) ................................... 19 + MODTIME (intermediate response) ............................ 38 + MODTIME (untagged response) ................................ 42 + MYRIGHTS (command) ......................................... 47 + MYRIGHTS (intermediate response) ........................... 47 + NO (response) .............................................. 29 + NOCREATE (store modifier) .................................. 44 + NOEXIST (response code) .................................... 19 + NOINHERIT (search modifier) ................................ 35 + NOOP (command) ............................................. 27 + NOT (search keyword) ....................................... 37 + OK (response) .............................................. 29 + OR (search keyword) ........................................ 37 + PERMISSION (response code) ................................. 19 + + + +Newman & Myers Standards Track [Page 66] + +RFC 2244 ACAP November 1997 + + + + PREFIX (search keyword) .................................... 37 + QUOTA (response code) ...................................... 19 + QUOTA (untagged response) .................................. 49 + RANGE (search keyword) ..................................... 37 + REFER (intermediate response) .............................. 38 + REFER (response code) ...................................... 19 + REMOVEFROM (untagged response) ............................. 41 + RETURN (search modifier) ................................... 35 + SASL (ACAP capability) ..................................... 27 + SASL (response code) ....................................... 20 + SEARCH (command) ........................................... 33 + SETACL (command) ........................................... 46 + SORT (search modifier) ..................................... 36 + STORE (command) ............................................ 42 + SUBSTRING (search keyword) ................................. 37 + TOOMANY (response code) .................................... 20 + TOOOLD (response code) ..................................... 20 + TRANSITION-NEEDED (response code) .......................... 20 + TRYFREECONTEXT (response code) ............................. 20 + TRYLATER (response code) ................................... 20 + UNCHANGEDSINCE (store modifier) ............................ 44 + UPDATECONTEXT (command) .................................... 40 + WAYTOOMANY (response code) ................................. 20 + acl (attribute metadata) ................................... 12 + anyone (ACL identifier) .................................... 17 + attribute (attribute metadata) ............................. 12 + dataset.acl (dataset attribute) ............................ 24 + dataset.acl. (dataset attribute) ................ 24 + dataset.inherit (dataset attribute) ........................ 24 + entry (predefined attribute) ............................... 11 + i;ascii-casemap (comparator) ............................... 16 + i;ascii-numeric (comparator) ............................... 16 + i;octet (comparator) ....................................... 16 + modtime (predefined attribute) ............................. 11 + myrights (attribute metadata) .............................. 12 + size (attribute metadata) .................................. 13 + subdataset (predefined attribute) .......................... 11 + value (attribute metadata) ................................. 13 + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 67] + +RFC 2244 ACAP November 1997 + + +C. Full Copyright Statement + + Copyright (C) The Internet Society 1997. All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implmentation may be prepared, copied, published and + distributed, in whole or in part, without restriction of any kind, + provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of developing + Internet standards in which case the procedures for copyrights defined + in the Internet Standards process must be followed, or as required to + translate it into languages other than English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT + NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN + WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + + + + + + + + + + + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 68] diff --git a/docs/rfcs/rfc2342.txt b/docs/rfcs/rfc2342.txt new file mode 100644 index 0000000..0926646 --- /dev/null +++ b/docs/rfcs/rfc2342.txt @@ -0,0 +1,563 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 2342 Microsoft +Category: Standards Track C. Newman + Innosoft + May 1998 + + + IMAP4 Namespace + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1998). All Rights Reserved. + +1. Abstract + + IMAP4 [RFC-2060] does not define a default server namespace. As a + result, two common namespace models have evolved: + + The "Personal Mailbox" model, in which the default namespace that is + presented consists of only the user's personal mailboxes. To access + shared mailboxes, the user must use an escape mechanism to reach + another namespace. + + The "Complete Hierarchy" model, in which the default namespace that + is presented includes the user's personal mailboxes along with any + other mailboxes they have access to. + + These two models, create difficulties for certain client operations. + This document defines a NAMESPACE command that allows a client to + discover the prefixes of namespaces used by a server for personal + mailboxes, other users' mailboxes, and shared mailboxes. This allows + a client to avoid much of the manual user configuration that is now + necessary when mixing and matching IMAP4 clients and servers. + +2. Conventions used in this document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If such lines are wrapped without a new "C:" or + "S:" label, then the wrapping is for editorial clarity and is not + part of the command. + + + +Gahrns & Newman Standards Track [Page 1] + +RFC 2342 IMAP4 Namespace May 1998 + + + Personal Namespace: A namespace that the server considers within the + personal scope of the authenticated user on a particular connection. + Typically, only the authenticated user has access to mailboxes in + their Personal Namespace. It is the part of the namespace that + belongs to the user that is allocated for mailboxes. If an INBOX + exists for a user, it MUST appear within the user's personal + namespace. In the typical case, there SHOULD be only one Personal + Namespace on a server. + + Other Users' Namespace: A namespace that consists of mailboxes from + the Personal Namespaces of other users. To access mailboxes in the + Other Users' Namespace, the currently authenticated user MUST be + explicitly granted access rights. For example, it is common for a + manager to grant to their secretary access rights to their mailbox. + In the typical case, there SHOULD be only one Other Users' Namespace + on a server. + + Shared Namespace: A namespace that consists of mailboxes that are + intended to be shared amongst users and do not exist within a user's + Personal Namespace. + + The namespaces a server uses MAY differ on a per-user basis. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + +3. Introduction and Overview + + Clients often attempt to create mailboxes for such purposes as + maintaining a record of sent messages (e.g. "Sent Mail") or + temporarily saving messages being composed (e.g. "Drafts"). For + these clients to inter-operate correctly with the variety of IMAP4 + servers available, the user must enter the prefix of the Personal + Namespace used by the server. Using the NAMESPACE command, a client + is able to automatically discover this prefix without manual user + configuration. + + In addition, users are often required to manually enter the prefixes + of various namespaces in order to view the mailboxes located there. + For example, they might be required to enter the prefix of #shared to + view the shared mailboxes namespace. The NAMESPACE command allows a + client to automatically discover the namespaces that are available on + a server. This allows a client to present the available namespaces to + the user in what ever manner it deems appropriate. For example, a + + + + + + +Gahrns & Newman Standards Track [Page 2] + +RFC 2342 IMAP4 Namespace May 1998 + + + client could choose to initially display only personal mailboxes, or + it may choose to display the complete list of mailboxes available, + and initially position the user at the root of their Personal + Namespace. + + A server MAY choose to make available to the NAMESPACE command only a + subset of the complete set of namespaces the server supports. To + provide the ability to access these namespaces, a client SHOULD allow + the user the ability to manually enter a namespace prefix. + +4. Requirements + + IMAP4 servers that support this extension MUST list the keyword + NAMESPACE in their CAPABILITY response. + + The NAMESPACE command is valid in the Authenticated and Selected + state. + +5. NAMESPACE Command + + Arguments: none + + Response: an untagged NAMESPACE response that contains the prefix + and hierarchy delimiter to the server's Personal + Namespace(s), Other Users' Namespace(s), and Shared + Namespace(s) that the server wishes to expose. The + response will contain a NIL for any namespace class + that is not available. Namespace_Response_Extensions + MAY be included in the response. + Namespace_Response_Extensions which are not on the IETF + standards track, MUST be prefixed with an "X-". + + Result: OK - Command completed + NO - Error: Can't complete command + BAD - argument invalid + + Example 5.1: + =========== + + < A server that supports a single personal namespace. No leading + prefix is used on personal mailboxes and "/" is the hierarchy + delimiter.> + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) NIL NIL + S: A001 OK NAMESPACE command completed + + + + + +Gahrns & Newman Standards Track [Page 3] + +RFC 2342 IMAP4 Namespace May 1998 + + + Example 5.2: + =========== + + < A user logged on anonymously to a server. No personal mailboxes + are associated with the anonymous user and the user does not have + access to the Other Users' Namespace. No prefix is required to + access shared mailboxes and the hierarchy delimiter is "." > + + C: A001 NAMESPACE + S: * NAMESPACE NIL NIL (("" ".")) + S: A001 OK NAMESPACE command completed + + Example 5.3: + =========== + + < A server that contains a Personal Namespace and a single Shared + Namespace. > + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) NIL (("Public Folders/" "/")) + S: A001 OK NAMESPACE command completed + + Example 5.4: + =========== + + < A server that contains a Personal Namespace, Other Users' + Namespace and multiple Shared Namespaces. Note that the hierarchy + delimiter used within each namespace can be different. > + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) (("~" "/")) (("#shared/" "/") + ("#public/" "/")("#ftp/" "/")("#news." ".")) + S: A001 OK NAMESPACE command completed + + The prefix string allows a client to do things such as automatically + creating personal mailboxes or LISTing all available mailboxes within + a namespace. + + Example 5.5: + =========== + + < A server that supports only the Personal Namespace, with a + leading prefix of INBOX to personal mailboxes and a hierarchy + delimiter of "."> + + C: A001 NAMESPACE + S: * NAMESPACE (("INBOX." ".")) NIL NIL + S: A001 OK NAMESPACE command completed + + + +Gahrns & Newman Standards Track [Page 4] + +RFC 2342 IMAP4 Namespace May 1998 + + + < Automatically create a mailbox to store sent items.> + + C: A002 CREATE "INBOX.Sent Mail" + S: A002 OK CREATE command completed + + Although typically a server will support only a single Personal + Namespace, and a single Other User's Namespace, circumstances exist + where there MAY be multiples of these, and a client MUST be prepared + for them. If a client is configured such that it is required to + create a certain mailbox, there can be circumstances where it is + unclear which Personal Namespaces it should create the mailbox in. + In these situations a client SHOULD let the user select which + namespaces to create the mailbox in. + + Example 5.6: + =========== + + < In this example, a server supports 2 Personal Namespaces. In + addition to the regular Personal Namespace, the user has an + additional personal namespace to allow access to mailboxes in an + MH format mailstore. > + + < The client is configured to save a copy of all mail sent by the + user into a mailbox called 'Sent Mail'. Furthermore, after a + message is deleted from a mailbox, the client is configured to + move that message to a mailbox called 'Deleted Items'.> + + < Note that this example demonstrates how some extension flags can + be passed to further describe the #mh namespace. > + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")("#mh/" "/" "X-PARAM" ("FLAG1" "FLAG2"))) + NIL NIL + S: A001 OK NAMESPACE command completed + + < It is desired to keep only one copy of sent mail. It is unclear + which Personal Namespace the client should use to create the 'Sent + Mail' mailbox. The user is prompted to select a namespace and + only one 'Sent Mail' mailbox is created. > + + C: A002 CREATE "Sent Mail" + S: A002 OK CREATE command completed + + < The client is designed so that it keeps two 'Deleted Items' + mailboxes, one for each namespace. > + + C: A003 CREATE "Delete Items" + S: A003 OK CREATE command completed + + + +Gahrns & Newman Standards Track [Page 5] + +RFC 2342 IMAP4 Namespace May 1998 + + + C: A004 CREATE "#mh/Deleted Items" + S: A004 OK CREATE command completed + + The next level of hierarchy following the Other Users' Namespace + prefix SHOULD consist of , where is a user name + as per the IMAP4 LOGIN or AUTHENTICATE command. + + A client can construct a LIST command by appending a "%" to the Other + Users' Namespace prefix to discover the Personal Namespaces of other + users that are available to the currently authenticated user. + + In response to such a LIST command, a server SHOULD NOT return user + names that have not granted access to their personal mailboxes to the + user in question. + + A server MAY return a LIST response containing only the names of + users that have explicitly granted access to the user in question. + + Alternatively, a server MAY return NO to such a LIST command, + requiring that a user name be included with the Other Users' + Namespace prefix before listing any other user's mailboxes. + + Example 5.7: + =========== + + < A server that supports providing a list of other user's + mailboxes that are accessible to the currently logged on user. > + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) (("Other Users/" "/")) NIL + S: A001 OK NAMESPACE command completed + + C: A002 LIST "" "Other Users/%" + S: * LIST () "/" "Other Users/Mike" + S: * LIST () "/" "Other Users/Karen" + S: * LIST () "/" "Other Users/Matthew" + S: * LIST () "/" "Other Users/Tesa" + S: A002 OK LIST command completed + + Example 5.8: + =========== + + < A server that does not support providing a list of other user's + mailboxes that are accessible to the currently logged on user. + The mailboxes are listable if the client includes the name of the + other user with the Other Users' Namespace prefix. > + + + + + +Gahrns & Newman Standards Track [Page 6] + +RFC 2342 IMAP4 Namespace May 1998 + + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) (("#Users/" "/")) NIL + S: A001 OK NAMESPACE command completed + + < In this example, the currently logged on user has access to the + Personal Namespace of user Mike, but the server chose to suppress + this information in the LIST response. However, by appending the + user name Mike (received through user input) to the Other Users' + Namespace prefix, the client is able to get a listing of the + personal mailboxes of user Mike. > + + C: A002 LIST "" "#Users/%" + S: A002 NO The requested item could not be found. + + C: A003 LIST "" "#Users/Mike/%" + S: * LIST () "/" "#Users/Mike/INBOX" + S: * LIST () "/" "#Users/Mike/Foo" + S: A003 OK LIST command completed. + + A prefix string might not contain a hierarchy delimiter, because + in some cases it is not needed as part of the prefix. + + Example 5.9: + =========== + + < A server that allows access to the Other Users' Namespace by + prefixing the others' mailboxes with a '~' followed by , + where is a user name as per the IMAP4 LOGIN or + AUTHENTICATE command.> + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) (("~" "/")) NIL + S: A001 OK NAMESPACE command completed + + < List the mailboxes for user mark > + + C: A002 LIST "" "~mark/%" + S: * LIST () "/" "~mark/INBOX" + S: * LIST () "/" "~mark/foo" + S: A002 OK LIST command completed + + Historical convention has been to start all namespaces with the "#" + character. Namespaces that include the "#" character are not IMAP + URL [IMAP-URL] friendly requiring the "#" character to be represented + as %23 when within URLs. As such, server implementers MAY instead + consider using namespace prefixes that do not contain the "#" + character. + + + + +Gahrns & Newman Standards Track [Page 7] + +RFC 2342 IMAP4 Namespace May 1998 + + +6. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. + + atom = + ; as defined in [RFC-2060] + + Namespace = nil / "(" 1*( "(" string SP (<"> QUOTED_CHAR <"> / + nil) *(Namespace_Response_Extension) ")" ) ")" + + Namespace_Command = "NAMESPACE" + + Namespace_Response_Extension = SP string SP "(" string *(SP string) + ")" + + Namespace_Response = "*" SP "NAMESPACE" SP Namespace SP Namespace SP + Namespace + + ; The first Namespace is the Personal Namespace(s) + ; The second Namespace is the Other Users' Namespace(s) + ; The third Namespace is the Shared Namespace(s) + + nil = + ; as defined in [RFC-2060] + + QUOTED_CHAR = + ; as defined in [RFC-2060] + + string = + ; as defined in [RFC-2060] + ; Note that the namespace prefix is to a mailbox and following + ; IMAP4 convention, any international string in the NAMESPACE + ; response MUST be of modified UTF-7 format as described in + ; [RFC-2060]. + +7. Security Considerations + + In response to a LIST command containing an argument of the Other + Users' Namespace prefix, a server SHOULD NOT list users that have not + granted list access to their personal mailboxes to the currently + authenticated user. Providing such a list, could compromise security + by potentially disclosing confidential information of who is located + on the server, or providing a starting point of a list of user + accounts to attack. + + + + + + +Gahrns & Newman Standards Track [Page 8] + +RFC 2342 IMAP4 Namespace May 1998 + + +8. References + + [RFC-2060], Crispin, M., "Internet Message Access Protocol Version + 4rev1", RFC 2060, December 1996. + + [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [ABNF] Crocker, D., Editor, and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [IMAP-URL], Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. + +9. Acknowledgments + + Many people have participated in the discussion of IMAP namespaces on + the IMAP mailing list. In particular, the authors would like to + thank Mark Crispin for many of the concepts relating to the Personal + Namespace and accessing the Personal Namespace of other users, Steve + Hole for summarizing the two namespace models, John Myers and Jack De + Winter for their work in a preceding effort trying to define a + standardized personal namespace, and Larry Osterman for his review + and collaboration on this document. + +11. Authors' Addresses + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98072, USA + + Phone: (425) 936-9833 + EMail: mikega@microsoft.com + + + Chris Newman + Innosoft International, Inc. + 1050 East Garvey Ave. South + West Covina, CA, 91790, USA + + EMail: chris.newman@innosoft.com + + + + + + + + + + +Gahrns & Newman Standards Track [Page 9] + +RFC 2342 IMAP4 Namespace May 1998 + + +12. Full Copyright Statement + + Copyright (C) The Internet Society (1998). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + + + + + + + + + + + + + + + + + + + + + + + + +Gahrns & Newman Standards Track [Page 10] + diff --git a/docs/rfcs/rfc2359.txt b/docs/rfcs/rfc2359.txt new file mode 100644 index 0000000..460ac4b --- /dev/null +++ b/docs/rfcs/rfc2359.txt @@ -0,0 +1,339 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 2359 Netscape Communications +Category: Standards Track June 1998 + + + IMAP4 UIDPLUS extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1998). All Rights Reserved. + +IESG NOTE + + The IMAP extension described here assumes a particular means of using + IMAP to support disconnected operation. However, this means of + supporting disconnected operation is not yet documented. Also, there + are multiple theories about how best to do disconnected operation in + IMAP, and as yet, there is no consensus on which one should be + adopted as a standard. + + This document is being approved as a Proposed Standard because it + does not appear to have technical flaws in itelf. However, approval + of this document as a Proposed Standard should not be considered an + IETF endorsement of any particular means of doing disconnected + operation in IMAP. + +Table of Contents + + 1. Abstract .............................................. 2 + 2. Conventions Used in this Document ..................... 2 + 3. Introduction and Overview ............................. 2 + 4. Features .............................................. 2 + 4.1. UID EXPUNGE Command ................................... 2 + 4.2. APPENDUID response code ............................... 3 + 4.3. COPYUID response code ................................. 4 + 5. Formal Syntax ......................................... 4 + 6. References ............................................ 4 + 7. Security Considerations ............................... 5 + 8. Author's Address ...................................... 5 + 9. Full Copyright Statement .............................. 6 + + + +Myers Standards Track [Page 1] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + +1. Abstract + + The UIDPLUS extension of the Internet Message Access Protocol [IMAP4] + provides a set of features intended to reduce the amount of time and + resources used by some client operations. The features in UIDPLUS + are primarily intended for disconnected-use clients. + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in "Key words for + use in RFCs to Indicate Requirement Levels" [KEYWORDS]. + +3. Introduction and Overview + + The UIDPLUS extension is present in any IMAP4 server implementation + which returns "UIDPLUS" as one of the supported capabilities to the + CAPABILITY command. The UIDPLUS extension contains one additional + command and additional data returned with successful APPEND and COPY + commands. + + Clients that wish to use the new command in UIDPLUS must of course + first test for the presence of the extension by issuing a CAPABILITY + command. Each of the features in UIDPLUS are optimizations; clients + can provide the same functionality, albeit more slowly, by using + commands in the base protocol. With each feature, this document + recommends a fallback approach to take when the UIDPLUS extension is + not supported by the server. + +4. Features + +4.1. UID EXPUNGE Command + + Arguments: message set + + Data: untagged responses: EXPUNGE + + Result: OK - expunge completed + NO - expunge failure (e.g. permission denied) + BAD - command unknown or arguments invalid + + + + + + + + +Myers Standards Track [Page 2] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + + The UID EXPUNGE command permanently removes from the currently + selected mailbox all messages that both have the \Deleted flag set + and have a UID that is included in the specified message set. If + a message either does not have the \Deleted flag set or is has a + UID that is not included in the specified message set, it is not + affected. + + This command may be used to ensure that a replayed EXPUNGE command + does not remove any messages that have been marked as \Deleted + between the time that the user requested the expunge operation and + the time the server processes the command. + + If the server does not support the UIDPLUS capability, the client + should fall back to using the STORE command to temporarily remove + the \Deleted flag from messages it does not want to remove. The + client could alternatively fall back to using the EXPUNGE command, + risking the unintended removal of some messages. + + Example: C: A003 UID EXPUNGE 3000:3002 + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: A003 OK UID EXPUNGE completed + +4.2. APPENDUID response code + + Successful APPEND commands return an APPENDUID response code in the + tagged OK response. The APPENDUID response code contains as + arguments the UIDVALIDITY of the destination mailbox and the UID + assigned to the appended message. + + If the server does not support the UIDPLUS capability, the client can + only discover this information by selecting the destination mailbox + and issuing FETCH commands. + + Example: C: A003 APPEND saved-messages (\Seen) {310} + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 38505 3955] APPEND completed + + + + +Myers Standards Track [Page 3] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + +4.3. COPYUID response code + + Successful COPY and UID COPY commands return a COPYUID response code + in the tagged OK response whenever at least one message was copied. + The COPYUID response code contains as an argument the UIDVALIDITY of + the appended-to mailbox, a message set containing the UIDs of the + messages copied to the destination mailbox, in the order they were + copied, and a message containing the UIDs assigned to the copied + messages, in the order they were assigned. Neither of the message + sets may contain extraneous UIDs or the symbol '*'. + + If the server does not support the UIDPLUS capability, the client can + only discover this information by selecting the destination mailbox + and issuing FETCH commands. + + Example: C: A003 COPY 2:4 MEETING + S: A003 OK [COPYUID 38505 304,319:320 3956:3958] Done + C: A003 UID COPY 305:310 MEETING + S: A003 OK Done + +5. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + resp_code_apnd ::= "APPENDUID" SPACE nz_number SPACE uniqueid + + resp_code_copy ::= "COPYUID" SPACE nz_number SPACE set SPACE set + + uid_expunge ::= "UID" SPACE "EXPUNGE" SPACE set + +6. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - + Version 4rev1", RFC 2060, December 1996. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet + Text Messages", STD 11, RFC 822, August 1982. + + + +Myers Standards Track [Page 4] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + +7. Security Considerations + + There are no known security issues with this extension. + +8. Author's Address + + John Gardiner Myers + Netscape Communications + 501 East Middlefield Road + Mail Stop MV-029 + Mountain View, CA 94043 + + EMail: jgmyers@netscape.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Myers Standards Track [Page 5] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + +9. Full Copyright Statement + + Copyright (C) The Internet Society (1998). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + + + + + + + + + + + + + + + + + + + + + + + + +Myers Standards Track [Page 6] + diff --git a/docs/rfcs/rfc2595.txt b/docs/rfcs/rfc2595.txt new file mode 100644 index 0000000..66897cd --- /dev/null +++ b/docs/rfcs/rfc2595.txt @@ -0,0 +1,843 @@ + + + + + + +Network Working Group C. Newman +Request for Comments: 2595 Innosoft +Category: Standards Track June 1999 + + + Using TLS with IMAP, POP3 and ACAP + + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1999). All Rights Reserved. + +1. Motivation + + The TLS protocol (formerly known as SSL) provides a way to secure an + application protocol from tampering and eavesdropping. The option of + using such security is desirable for IMAP, POP and ACAP due to common + connection eavesdropping and hijacking attacks [AUTH]. Although + advanced SASL authentication mechanisms can provide a lightweight + version of this service, TLS is complimentary to simple + authentication-only SASL mechanisms or deployed clear-text password + login commands. + + Many sites have a high investment in authentication infrastructure + (e.g., a large database of a one-way-function applied to user + passwords), so a privacy layer which is not tightly bound to user + authentication can protect against network eavesdropping attacks + without requiring a new authentication infrastructure and/or forcing + all users to change their password. Recognizing that such sites will + desire simple password authentication in combination with TLS + encryption, this specification defines the PLAIN SASL mechanism for + use with protocols which lack a simple password authentication + command such as ACAP and SMTP. (Note there is a separate RFC for the + STARTTLS command in SMTP [SMTPTLS].) + + There is a strong desire in the IETF to eliminate the transmission of + clear-text passwords over unencrypted channels. While SASL can be + used for this purpose, TLS provides an additional tool with different + deployability characteristics. A server supporting both TLS with + + + + +Newman Standards Track [Page 1] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + simple passwords and a challenge/response SASL mechanism is likely to + interoperate with a wide variety of clients without resorting to + unencrypted clear-text passwords. + + The STARTTLS command rectifies a number of the problems with using a + separate port for a "secure" protocol variant. Some of these are + mentioned in section 7. + +1.1. Conventions Used in this Document + + The key words "REQUIRED", "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", + "MAY", and "OPTIONAL" in this document are to be interpreted as + described in "Key words for use in RFCs to Indicate Requirement + Levels" [KEYWORDS]. + + Terms related to authentication are defined in "On Internet + Authentication" [AUTH]. + + Formal syntax is defined using ABNF [ABNF]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + +2. Basic Interoperability and Security Requirements + + The following requirements apply to all implementations of the + STARTTLS extension for IMAP, POP3 and ACAP. + +2.1. Cipher Suite Requirements + + Implementation of the TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA [TLS] cipher + suite is REQUIRED. This is important as it assures that any two + compliant implementations can be configured to interoperate. + + All other cipher suites are OPTIONAL. + +2.2. Privacy Operational Mode Security Requirements + + Both clients and servers SHOULD have a privacy operational mode which + refuses authentication unless successful activation of an encryption + layer (such as that provided by TLS) occurs prior to or at the time + of authentication and which will terminate the connection if that + encryption layer is deactivated. Implementations are encouraged to + have flexability with respect to the minimal encryption strength or + cipher suites permitted. A minimalist approach to this + recommendation would be an operational mode where the + TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA cipher suite is mandatory prior to + permitting authentication. + + + +Newman Standards Track [Page 2] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + Clients MAY have an operational mode which uses encryption only when + it is advertised by the server, but authentication continues + regardless. For backwards compatibility, servers SHOULD have an + operational mode where only the authentication mechanisms required by + the relevant base protocol specification are needed to successfully + authenticate. + +2.3. Clear-Text Password Requirements + + Clients and servers which implement STARTTLS MUST be configurable to + refuse all clear-text login commands or mechanisms (including both + standards-track and nonstandard mechanisms) unless an encryption + layer of adequate strength is active. Servers which allow + unencrypted clear-text logins SHOULD be configurable to refuse + clear-text logins both for the entire server, and on a per-user + basis. + +2.4. Server Identity Check + + During the TLS negotiation, the client MUST check its understanding + of the server hostname against the server's identity as presented in + the server Certificate message, in order to prevent man-in-the-middle + attacks. Matching is performed according to these rules: + + - The client MUST use the server hostname it used to open the + connection as the value to compare against the server name as + expressed in the server certificate. The client MUST NOT use any + form of the server hostname derived from an insecure remote source + (e.g., insecure DNS lookup). CNAME canonicalization is not done. + + - If a subjectAltName extension of type dNSName is present in the + certificate, it SHOULD be used as the source of the server's + identity. + + - Matching is case-insensitive. + + - A "*" wildcard character MAY be used as the left-most name + component in the certificate. For example, *.example.com would + match a.example.com, foo.example.com, etc. but would not match + example.com. + + - If the certificate contains multiple names (e.g. more than one + dNSName field), then a match with any one of the fields is + considered acceptable. + + If the match fails, the client SHOULD either ask for explicit user + confirmation, or terminate the connection and indicate the server's + identity is suspect. + + + +Newman Standards Track [Page 3] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +2.5. TLS Security Policy Check + + Both the client and server MUST check the result of the STARTTLS + command and subsequent TLS negotiation to see whether acceptable + authentication or privacy was achieved. Ignoring this step + completely invalidates using TLS for security. The decision about + whether acceptable authentication or privacy was achieved is made + locally, is implementation-dependent, and is beyond the scope of this + document. + +3. IMAP STARTTLS extension + + When the TLS extension is present in IMAP, "STARTTLS" is listed as a + capability in response to the CAPABILITY command. This extension + adds a single command, "STARTTLS" to the IMAP protocol which is used + to begin a TLS negotiation. + +3.1. STARTTLS Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - begin TLS negotiation + BAD - command unknown or arguments invalid + + A TLS negotiation begins immediately after the CRLF at the end of + the tagged OK response from the server. Once a client issues a + STARTTLS command, it MUST NOT issue further commands until a + server response is seen and the TLS negotiation is complete. + + The STARTTLS command is only valid in non-authenticated state. + The server remains in non-authenticated state, even if client + credentials are supplied during the TLS negotiation. The SASL + [SASL] EXTERNAL mechanism MAY be used to authenticate once TLS + client credentials are successfully exchanged, but servers + supporting the STARTTLS command are not required to support the + EXTERNAL mechanism. + + Once TLS has been started, the client MUST discard cached + information about server capabilities and SHOULD re-issue the + CAPABILITY command. This is necessary to protect against + man-in-the-middle attacks which alter the capabilities list prior + to STARTTLS. The server MAY advertise different capabilities + after STARTTLS. + + The formal syntax for IMAP is amended as follows: + + + + +Newman Standards Track [Page 4] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + command_any =/ "STARTTLS" + + Example: C: a001 CAPABILITY + S: * CAPABILITY IMAP4rev1 STARTTLS LOGINDISABLED + S: a001 OK CAPABILITY completed + C: a002 STARTTLS + S: a002 OK Begin TLS negotiation now + + C: a003 CAPABILITY + S: * CAPABILITY IMAP4rev1 AUTH=EXTERNAL + S: a003 OK CAPABILITY completed + C: a004 LOGIN joe password + S: a004 OK LOGIN completed + +3.2. IMAP LOGINDISABLED capability + + The current IMAP protocol specification (RFC 2060) requires the + implementation of the LOGIN command which uses clear-text passwords. + Many sites may choose to disable this command unless encryption is + active for security reasons. An IMAP server MAY advertise that the + LOGIN command is disabled by including the LOGINDISABLED capability + in the capability response. Such a server will respond with a tagged + "NO" response to any attempt to use the LOGIN command. + + An IMAP server which implements STARTTLS MUST implement support for + the LOGINDISABLED capability on unencrypted connections. + + An IMAP client which complies with this specification MUST NOT issue + the LOGIN command if this capability is present. + + This capability is useful to prevent clients compliant with this + specification from sending an unencrypted password in an environment + subject to passive attacks. It has no impact on an environment + subject to active attacks as a man-in-the-middle attacker can remove + this capability. Therefore this does not relieve clients of the need + to follow the privacy mode recommendation in section 2.2. + + Servers advertising this capability will fail to interoperate with + many existing compliant IMAP clients and will be unable to prevent + those clients from disclosing the user's password. + +4. POP3 STARTTLS extension + + The POP3 STARTTLS extension adds the STLS command to POP3 servers. + If this is implemented, the POP3 extension mechanism [POP3EXT] MUST + also be implemented to avoid the need for client probing of multiple + commands. The capability name "STLS" indicates this command is + present and permitted in the current state. + + + +Newman Standards Track [Page 5] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + STLS + + Arguments: none + + Restrictions: + Only permitted in AUTHORIZATION state. + + Discussion: + A TLS negotiation begins immediately after the CRLF at the + end of the +OK response from the server. A -ERR response + MAY result if a security layer is already active. Once a + client issues a STLS command, it MUST NOT issue further + commands until a server response is seen and the TLS + negotiation is complete. + + The STLS command is only permitted in AUTHORIZATION state + and the server remains in AUTHORIZATION state, even if + client credentials are supplied during the TLS negotiation. + The AUTH command [POP-AUTH] with the EXTERNAL mechanism + [SASL] MAY be used to authenticate once TLS client + credentials are successfully exchanged, but servers + supporting the STLS command are not required to support the + EXTERNAL mechanism. + + Once TLS has been started, the client MUST discard cached + information about server capabilities and SHOULD re-issue + the CAPA command. This is necessary to protect against + man-in-the-middle attacks which alter the capabilities list + prior to STLS. The server MAY advertise different + capabilities after STLS. + + Possible Responses: + +OK -ERR + + Examples: + C: STLS + S: +OK Begin TLS negotiation + + ... + C: STLS + S: -ERR Command not permitted when TLS active + + + + + + + + + + +Newman Standards Track [Page 6] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +5. ACAP STARTTLS extension + + When the TLS extension is present in ACAP, "STARTTLS" is listed as a + capability in the ACAP greeting. No arguments to this capability are + defined at this time. This extension adds a single command, + "STARTTLS" to the ACAP protocol which is used to begin a TLS + negotiation. + +5.1. STARTTLS Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - begin TLS negotiation + BAD - command unknown or arguments invalid + + A TLS negotiation begins immediately after the CRLF at the end of + the tagged OK response from the server. Once a client issues a + STARTTLS command, it MUST NOT issue further commands until a + server response is seen and the TLS negotiation is complete. + + The STARTTLS command is only valid in non-authenticated state. + The server remains in non-authenticated state, even if client + credentials are supplied during the TLS negotiation. The SASL + [SASL] EXTERNAL mechanism MAY be used to authenticate once TLS + client credentials are successfully exchanged, but servers + supporting the STARTTLS command are not required to support the + EXTERNAL mechanism. + + After the TLS layer is established, the server MUST re-issue an + untagged ACAP greeting. This is necessary to protect against + man-in-the-middle attacks which alter the capabilities list prior + to STARTTLS. The client MUST discard cached capability + information and replace it with the information from the new ACAP + greeting. The server MAY advertise different capabilities after + STARTTLS. + + The formal syntax for ACAP is amended as follows: + + command_any =/ "STARTTLS" + + Example: S: * ACAP (SASL "CRAM-MD5") (STARTTLS) + C: a002 STARTTLS + S: a002 OK "Begin TLS negotiation now" + + S: * ACAP (SASL "CRAM-MD5" "PLAIN" "EXTERNAL") + + + + +Newman Standards Track [Page 7] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +6. PLAIN SASL mechanism + + Clear-text passwords are simple, interoperate with almost all + existing operating system authentication databases, and are useful + for a smooth transition to a more secure password-based + authentication mechanism. The drawback is that they are unacceptable + for use over an unencrypted network connection. + + This defines the "PLAIN" SASL mechanism for use with ACAP and other + protocols with no clear-text login command. The PLAIN SASL mechanism + MUST NOT be advertised or used unless a strong encryption layer (such + as the provided by TLS) is active or backwards compatibility dictates + otherwise. + + The mechanism consists of a single message from the client to the + server. The client sends the authorization identity (identity to + login as), followed by a US-ASCII NUL character, followed by the + authentication identity (identity whose password will be used), + followed by a US-ASCII NUL character, followed by the clear-text + password. The client may leave the authorization identity empty to + indicate that it is the same as the authentication identity. + + The server will verify the authentication identity and password with + the system authentication database and verify that the authentication + credentials permit the client to login as the authorization identity. + If both steps succeed, the user is logged in. + + The server MAY also use the password to initialize any new + authentication database, such as one suitable for CRAM-MD5 + [CRAM-MD5]. + + Non-US-ASCII characters are permitted as long as they are represented + in UTF-8 [UTF-8]. Use of non-visible characters or characters which + a user may be unable to enter on some keyboards is discouraged. + + The formal grammar for the client message using Augmented BNF [ABNF] + follows. + + message = [authorize-id] NUL authenticate-id NUL password + authenticate-id = 1*UTF8-SAFE ; MUST accept up to 255 octets + authorize-id = 1*UTF8-SAFE ; MUST accept up to 255 octets + password = 1*UTF8-SAFE ; MUST accept up to 255 octets + NUL = %x00 + UTF8-SAFE = %x01-09 / %x0B-0C / %x0E-7F / UTF8-2 / + UTF8-3 / UTF8-4 / UTF8-5 / UTF8-6 + UTF8-1 = %x80-BF + UTF8-2 = %xC0-DF UTF8-1 + UTF8-3 = %xE0-EF 2UTF8-1 + + + +Newman Standards Track [Page 8] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + UTF8-4 = %xF0-F7 3UTF8-1 + UTF8-5 = %xF8-FB 4UTF8-1 + UTF8-6 = %xFC-FD 5UTF8-1 + + Here is an example of how this might be used to initialize a CRAM-MD5 + authentication database for ACAP: + + Example: S: * ACAP (SASL "CRAM-MD5") (STARTTLS) + C: a001 AUTHENTICATE "CRAM-MD5" + S: + "<1896.697170952@postoffice.reston.mci.net>" + C: "tim b913a602c7eda7a495b4e6e7334d3890" + S: a001 NO (TRANSITION-NEEDED) + "Please change your password, or use TLS to login" + C: a002 STARTTLS + S: a002 OK "Begin TLS negotiation now" + + S: * ACAP (SASL "CRAM-MD5" "PLAIN" "EXTERNAL") + C: a003 AUTHENTICATE "PLAIN" {21+} + C: timtanstaaftanstaaf + S: a003 OK CRAM-MD5 password initialized + + Note: In this example, represents a single ASCII NUL octet. + +7. imaps and pop3s ports + + Separate "imaps" and "pop3s" ports were registered for use with SSL. + Use of these ports is discouraged in favor of the STARTTLS or STLS + commands. + + A number of problems have been observed with separate ports for + "secure" variants of protocols. This is an attempt to enumerate some + of those problems. + + - Separate ports lead to a separate URL scheme which intrudes into + the user interface in inappropriate ways. For example, many web + pages use language like "click here if your browser supports SSL." + This is a decision the browser is often more capable of making than + the user. + + - Separate ports imply a model of either "secure" or "not secure." + This can be misleading in a number of ways. First, the "secure" + port may not in fact be acceptably secure as an export-crippled + cipher suite might be in use. This can mislead users into a false + sense of security. Second, the normal port might in fact be + secured by using a SASL mechanism which includes a security layer. + Thus the separate port distinction makes the complex topic of + security policy even more confusing. One common result of this + confusion is that firewall administrators are often misled into + + + +Newman Standards Track [Page 9] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + permitting the "secure" port and blocking the standard port. This + could be a poor choice given the common use of SSL with a 40-bit + key encryption layer and plain-text password authentication is less + secure than strong SASL mechanisms such as GSSAPI with Kerberos 5. + + - Use of separate ports for SSL has caused clients to implement only + two security policies: use SSL or don't use SSL. The desirable + security policy "use TLS when available" would be cumbersome with + the separate port model, but is simple with STARTTLS. + + - Port numbers are a limited resource. While they are not yet in + short supply, it is unwise to set a precedent that could double (or + worse) the speed of their consumption. + + +8. IANA Considerations + + This constitutes registration of the "STARTTLS" and "LOGINDISABLED" + IMAP capabilities as required by section 7.2.1 of RFC 2060 [IMAP]. + + The registration for the POP3 "STLS" capability follows: + + CAPA tag: STLS + Arguments: none + Added commands: STLS + Standard commands affected: May enable USER/PASS as a side-effect. + CAPA command SHOULD be re-issued after successful completion. + Announced states/Valid states: AUTHORIZATION state only. + Specification reference: this memo + + The registration for the ACAP "STARTTLS" capability follows: + + Capability name: STARTTLS + Capability keyword: STARTTLS + Capability arguments: none + Published Specification(s): this memo + Person and email address for further information: + see author's address section below + + The registration for the PLAIN SASL mechanism follows: + + SASL mechanism name: PLAIN + Security Considerations: See section 9 of this memo + Published specification: this memo + Person & email address to contact for further information: + see author's address section below + Intended usage: COMMON + Author/Change controller: see author's address section below + + + +Newman Standards Track [Page 10] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +9. Security Considerations + + TLS only provides protection for data sent over a network connection. + Messages transferred over IMAP or POP3 are still available to server + administrators and usually subject to eavesdropping, tampering and + forgery when transmitted through SMTP or NNTP. TLS is no substitute + for an end-to-end message security mechanism using MIME security + multiparts [MIME-SEC]. + + A man-in-the-middle attacker can remove STARTTLS from the capability + list or generate a failure response to the STARTTLS command. In + order to detect such an attack, clients SHOULD warn the user when + session privacy is not active and/or be configurable to refuse to + proceed without an acceptable level of security. + + A man-in-the-middle attacker can always cause a down-negotiation to + the weakest authentication mechanism or cipher suite available. For + this reason, implementations SHOULD be configurable to refuse weak + mechanisms or cipher suites. + + Any protocol interactions prior to the TLS handshake are performed in + the clear and can be modified by a man-in-the-middle attacker. For + this reason, clients MUST discard cached information about server + capabilities advertised prior to the start of the TLS handshake. + + Clients are encouraged to clearly indicate when the level of + encryption active is known to be vulnerable to attack using modern + hardware (such as encryption keys with 56 bits of entropy or less). + + The LOGINDISABLED IMAP capability (discussed in section 3.2) only + reduces the potential for passive attacks, it provides no protection + against active attacks. The responsibility remains with the client + to avoid sending a password over a vulnerable channel. + + The PLAIN mechanism relies on the TLS encryption layer for security. + When used without TLS, it is vulnerable to a common network + eavesdropping attack. Therefore PLAIN MUST NOT be advertised or used + unless a suitable TLS encryption layer is active or backwards + compatibility dictates otherwise. + + When the PLAIN mechanism is used, the server gains the ability to + impersonate the user to all services with the same password + regardless of any encryption provided by TLS or other network privacy + mechanisms. While many other authentication mechanisms have similar + weaknesses, stronger SASL mechanisms such as Kerberos address this + issue. Clients are encouraged to have an operational mode where all + mechanisms which are likely to reveal the user's password to the + server are disabled. + + + +Newman Standards Track [Page 11] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + The security considerations for TLS apply to STARTTLS and the + security considerations for SASL apply to the PLAIN mechanism. + Additional security requirements are discussed in section 2. + +10. References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [AUTH] Haller, N. and R. Atkinson, "On Internet Authentication", + RFC 1704, October 1994. + + [CRAM-MD5] Klensin, J., Catoe, R. and P. Krumviede, "IMAP/POP + AUTHorize Extension for Simple Challenge/Response", RFC + 2195, September 1997. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MIME-SEC] Galvin, J., Murphy, S., Crocker, S. and N. Freed, + "Security Multiparts for MIME: Multipart/Signed and + Multipart/Encrypted", RFC 1847, October 1995. + + [POP3] Myers, J. and M. Rose, "Post Office Protocol - Version 3", + STD 53, RFC 1939, May 1996. + + [POP3EXT] Gellens, R., Newman, C. and L. Lundblade, "POP3 Extension + Mechanism", RFC 2449, November 1998. + + [POP-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, + December 1994. + + [SASL] Myers, J., "Simple Authentication and Security Layer + (SASL)", RFC 2222, October 1997. + + [SMTPTLS] Hoffman, P., "SMTP Service Extension for Secure SMTP over + TLS", RFC 2487, January 1999. + + [TLS] Dierks, T. and C. Allen, "The TLS Protocol Version 1.0", + RFC 2246, January 1999. + + + + + +Newman Standards Track [Page 12] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", RFC 2279, January 1998. + + +11. Author's Address + + Chris Newman + Innosoft International, Inc. + 1050 Lakes Drive + West Covina, CA 91790 USA + + EMail: chris.newman@innosoft.com + + +A. Appendix -- Compliance Checklist + + An implementation is not compliant if it fails to satisfy one or more + of the MUST requirements for the protocols it implements. An + implementation that satisfies all the MUST and all the SHOULD + requirements for its protocols is said to be "unconditionally + compliant"; one that satisfies all the MUST requirements but not all + the SHOULD requirements for its protocols is said to be + "conditionally compliant". + + Rules Section + ----- ------- + Mandatory-to-implement Cipher Suite 2.1 + SHOULD have mode where encryption required 2.2 + server SHOULD have mode where TLS not required 2.2 + MUST be configurable to refuse all clear-text login + commands or mechanisms 2.3 + server SHOULD be configurable to refuse clear-text + login commands on entire server and on per-user basis 2.3 + client MUST check server identity 2.4 + client MUST use hostname used to open connection 2.4 + client MUST NOT use hostname from insecure remote lookup 2.4 + client SHOULD support subjectAltName of dNSName type 2.4 + client SHOULD ask for confirmation or terminate on fail 2.4 + MUST check result of STARTTLS for acceptable privacy 2.5 + client MUST NOT issue commands after STARTTLS + until server response and negotiation done 3.1,4,5.1 + client MUST discard cached information 3.1,4,5.1,9 + client SHOULD re-issue CAPABILITY/CAPA command 3.1,4 + IMAP server with STARTTLS MUST implement LOGINDISABLED 3.2 + IMAP client MUST NOT issue LOGIN if LOGINDISABLED 3.2 + POP server MUST implement POP3 extensions 4 + ACAP server MUST re-issue ACAP greeting 5.1 + + + + +Newman Standards Track [Page 13] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + client SHOULD warn when session privacy not active and/or + refuse to proceed without acceptable security level 9 + SHOULD be configurable to refuse weak mechanisms or + cipher suites 9 + + The PLAIN mechanism is an optional part of this specification. + However if it is implemented the following rules apply: + + Rules Section + ----- ------- + MUST NOT use PLAIN unless strong encryption active + or backwards compatibility dictates otherwise 6,9 + MUST use UTF-8 encoding for characters in PLAIN 6 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Newman Standards Track [Page 14] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +Full Copyright Statement + + Copyright (C) The Internet Society (1999). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Newman Standards Track [Page 15] + diff --git a/docs/rfcs/rfc2683.txt b/docs/rfcs/rfc2683.txt new file mode 100644 index 0000000..d92e340 --- /dev/null +++ b/docs/rfcs/rfc2683.txt @@ -0,0 +1,1291 @@ + + + + + + +Network Working Group B. Leiba +Request for Comments: 2683 IBM T.J. Watson Research Center +Category: Informational September 1999 + + + IMAP4 Implementation Recommendations + +Status of this Memo + + This memo provides information for the Internet community. It does + not specify an Internet standard of any kind. Distribution of this + memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1999). All Rights Reserved. + +1. Abstract + + The IMAP4 specification [RFC-2060] describes a rich protocol for use + in building clients and servers for storage, retrieval, and + manipulation of electronic mail. Because the protocol is so rich and + has so many implementation choices, there are often trade-offs that + must be made and issues that must be considered when designing such + clients and servers. This document attempts to outline these issues + and to make recommendations in order to make the end products as + interoperable as possible. + +2. Conventions used in this document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The words "must", "must not", "should", "should not", and "may" are + used with specific meaning in this document; since their meaning is + somewhat different from that specified in RFC 2119, we do not put + them in all caps here. Their meaning is as follows: + + must -- This word means that the action described is necessary + to ensure interoperability. The recommendation should + not be ignored. + must not -- This phrase means that the action described will be + almost certain to hurt interoperability. The + recommendation should not be ignored. + + + + + + + +Leiba Informational [Page 1] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + should -- This word means that the action described is strongly + recommended and will enhance interoperability or + usability. The recommendation should not be ignored + without careful consideration. + should not -- This phrase means that the action described is strongly + recommended against, and might hurt interoperability or + usability. The recommendation should not be ignored + without careful consideration. + may -- This word means that the action described is an + acceptable implementation choice. No specific + recommendation is implied; this word is used to point + out a choice that might not be obvious, or to let + implementors know what choices have been made by + existing implementations. + +3. Interoperability Issues and Recommendations + +3.1. Accessibility + + This section describes the issues related to access to servers and + server resources. Concerns here include data sharing and maintenance + of client/server connections. + +3.1.1. Multiple Accesses of the Same Mailbox + + One strong point of IMAP4 is that, unlike POP3, it allows for + multiple simultaneous access to a single mailbox. A user can, thus, + read mail from a client at home while the client in the office is + still connected; or the help desk staff can all work out of the same + inbox, all seeing the same pool of questions. An important point + about this capability, though is that NO SERVER IS GUARANTEED TO + SUPPORT THIS. If you are selecting an IMAP server and this facility + is important to you, be sure that the server you choose to install, + in the configuration you choose to use, supports it. + + If you are designing a client, you must not assume that you can + access the same mailbox more than once at a time. That means + + 1. you must handle gracefully the failure of a SELECT command if the + server refuses the second SELECT, + 2. you must handle reasonably the severing of your connection (see + "Severed Connections", below) if the server chooses to allow the + second SELECT by forcing the first off, + 3. you must avoid making multiple connections to the same mailbox in + your own client (for load balancing or other such reasons), and + 4. you must avoid using the STATUS command on a mailbox that you have + selected (with some server implementations the STATUS command has + the same problems with multiple access as do the SELECT and + + + +Leiba Informational [Page 2] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + EXAMINE commands). + + A further note about STATUS: The STATUS command is sometimes used to + check a non-selected mailbox for new mail. This mechanism must not + be used to check for new mail in the selected mailbox; section 5.2 of + [RFC-2060] specifically forbids this in its last paragraph. Further, + since STATUS takes a mailbox name it is an independent operation, not + operating on the selected mailbox. Because of this, the information + it returns is not necessarily in synchronization with the selected + mailbox state. + +3.1.2. Severed Connections + + The client/server connection may be severed for one of three reasons: + the client severs the connection, the server severs the connection, + or the connection is severed by outside forces beyond the control of + the client and the server (a telephone line drops, for example). + Clients and servers must both deal with these situations. + + When the client wants to sever a connection, it's usually because it + has finished the work it needed to do on that connection. The client + should send a LOGOUT command, wait for the tagged response, and then + close the socket. But note that, while this is what's intended in + the protocol design, there isn't universal agreement here. Some + contend that sending the LOGOUT and waiting for the two responses + (untagged BYE and tagged OK) is wasteful and unnecessary, and that + the client can simply close the socket. The server should interpret + the closed socket as a log out by the client. The counterargument is + that it's useful from the standpoint of cleanup, problem + determination, and the like, to have an explicit client log out, + because otherwise there is no way for the server to tell the + difference between "closed socket because of log out" and "closed + socket because communication was disrupted". If there is a + client/server interaction problem, a client which routinely + terminates a session by breaking the connection without a LOGOUT will + make it much more difficult to determine the problem. + + Because of this disagreement, server designers must be aware that + some clients might close the socket without sending a LOGOUT. In any + case, whether or not a LOGOUT was sent, the server should not + implicitly expunge any messages from the selected mailbox. If a + client wants the server to do so, it must send a CLOSE or EXPUNGE + command explicitly. + + When the server wants to sever a connection it's usually due to an + inactivity timeout or is because a situation has arisen that has + changed the state of the mail store in a way that the server can not + communicate to the client. The server should send an untagged BYE + + + +Leiba Informational [Page 3] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + response to the client and then close the socket. Sending an + untagged BYE response before severing allows the server to send a + human-readable explanation of the problem to the client, which the + client may then log, display to the user, or both (see section 7.1.5 + of [RFC-2060]). + + Regarding inactivity timeouts, there is some controversy. Unlike + POP, for which the design is for a client to connect, retrieve mail, + and log out, IMAP's design encourages long-lived (and mostly + inactive) client/server sessions. As the number of users grows, this + can use up a lot of server resources, especially with clients that + are designed to maintain sessions for mailboxes that the user has + finished accessing. To alleviate this, a server may implement an + inactivity timeout, unilaterally closing a session (after first + sending an untagged BYE, as noted above). Some server operators have + reported dramatic improvements in server performance after doing + this. As specified in [RFC-2060], if such a timeout is done it must + not be until at least 30 minutes of inactivity. The reason for this + specification is to prevent clients from sending commands (such as + NOOP) to the server at frequent intervals simply to avert a too-early + timeout. If the client knows that the server may not time out the + session for at least 30 minutes, then the client need not poll at + intervals more frequent than, say, 25 minutes. + +3.2. Scaling + + IMAP4 has many features that allow for scalability, as mail stores + become larger and more numerous. Large numbers of users, mailboxes, + and messages, and very large messages require thought to handle + efficiently. This document will not address the administrative + issues involved in large numbers of users, but we will look at the + other items. + +3.2.1. Flood Control + + There are three situations when a client can make a request that will + result in a very large response - too large for the client reasonably + to deal with: there are a great many mailboxes available, there are a + great many messages in the selected mailbox, or there is a very large + message part. The danger here is that the end user will be stuck + waiting while the server sends (and the client processes) an enormous + response. In all of these cases there are things a client can do to + reduce that danger. + + There is also the case where a client can flood a server, by sending + an arbitratily long command. We'll discuss that issue, too, in this + section. + + + + +Leiba Informational [Page 4] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.2.1.1. Listing Mailboxes + + Some servers present Usenet newsgroups to IMAP users. Newsgroups, + and other such hierarchical mailbox structures, can be very numerous + but may have only a few entries at the top level of hierarchy. Also, + some servers are built against mail stores that can, unbeknownst to + the server, have circular hierarchies - that is, it's possible for + "a/b/c/d" to resolve to the same file structure as "a", which would + then mean that "a/b/c/d/b" is the same as "a/b", and the hierarchy + will never end. The LIST response in this case will be unlimited. + + Clients that will have trouble with this are those that use + + C: 001 LIST "" * + + to determine the mailbox list. Because of this, clients should not + use an unqualified "*" that way in the LIST command. A safer + approach is to list each level of hierarchy individually, allowing + the user to traverse the tree one limb at a time, thus: + + C: 001 LIST "" % + S: * LIST () "/" Banana + S: * LIST ...etc... + S: 001 OK done + + and then + + C: 002 LIST "" Banana/% + S: * LIST () "/" Banana/Apple + S: * LIST ...etc... + S: 002 OK done + + Using this technique the client's user interface can give the user + full flexibility without choking on the voluminous reply to "LIST *". + + Of course, it is still possible that the reply to + + C: 005 LIST "" alt.fan.celebrity.% + + may be thousands of entries long, and there is, unfortunately, + nothing the client can do to protect itself from that. This has not + yet been a notable problem. + + Servers that may export circular hierarchies (any server that + directly presents a UNIX file system, for instance) should limit the + hierarchy depth to prevent unlimited LIST responses. A suggested + depth limit is 20 hierarchy levels. + + + + +Leiba Informational [Page 5] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.2.1.2. Fetching the List of Messages + + When a client selects a mailbox, it is given a count, in the untagged + EXISTS response, of the messages in the mailbox. This number can be + very large. In such a case it might be unwise to use + + C: 004 FETCH 1:* ALL + + to populate the user's view of the mailbox. One good method to avoid + problems with this is to batch the requests, thus: + + C: 004 FETCH 1:50 ALL + S: * 1 FETCH ...etc... + S: 004 OK done + C: 005 FETCH 51:100 ALL + S: * 51 FETCH ...etc... + S: 005 OK done + C: 006 FETCH 101:150 ALL + ...etc... + + Using this method, another command, such as "FETCH 6 BODY[1]" can be + inserted as necessary, and the client will not have its access to the + server blocked by a storm of FETCH replies. (Such a method could be + reversed to fetch the LAST 50 messages first, then the 50 prior to + that, and so on.) + + As a smart extension of this, a well designed client, prepared for + very large mailboxes, will not automatically fetch data for all + messages AT ALL. Rather, the client will populate the user's view + only as the user sees it, possibly pre-fetching selected information, + and only fetching other information as the user scrolls to it. For + example, to select only those messages beginning with the first + unseen one: + + C: 003 SELECT INBOX + S: * 10000 EXISTS + S: * 80 RECENT + S: * FLAGS (\Answered \Flagged \Deleted \Draft \Seen) + S: * OK [UIDVALIDITY 824708485] UID validity status + S: * OK [UNSEEN 9921] First unseen message + S: 003 OK [READ-WRITE] SELECT completed + C: 004 FETCH 9921:* ALL + ... etc... + + If the server does not return an OK [UNSEEN] response, the client may + use SEARCH UNSEEN to obtain that value. + + + + + +Leiba Informational [Page 6] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + This mechanism is good as a default presentation method, but only + works well if the default message order is acceptable. A client may + want to present various sort orders to the user (by subject, by date + sent, by sender, and so on) and in that case (lacking a SORT + extension on the server side) the client WILL have to retrieve all + message descriptors. A client that provides this service should not + do it by default and should inform the user of the costs of choosing + this option for large mailboxes. + +3.2.1.3. Fetching a Large Body Part + + The issue here is similar to the one for a list of messages. In the + BODYSTRUCTURE response the client knows the size, in bytes, of the + body part it plans to fetch. Suppose this is a 70 MB video clip. The + client can use partial fetches to retrieve the body part in pieces, + avoiding the problem of an uninterruptible 70 MB literal coming back + from the server: + + C: 022 FETCH 3 BODY[1]<0.20000> + S: * 3 FETCH (FLAGS(\Seen) BODY[1]<0> {20000} + S: ...data...) + S: 022 OK done + C: 023 FETCH 3 BODY[1]<20001.20000> + S: * 3 FETCH (BODY[1]<20001> {20000} + S: ...data...) + S: 023 OK done + C: 024 FETCH 3 BODY[1]<40001.20000> + ...etc... + +3.2.1.4. BODYSTRUCTURE vs. Entire Messages + + Because FETCH BODYSTRUCTURE is necessary in order to determine the + number of body parts, and, thus, whether a message has "attachments", + clients often use FETCH FULL as their normal method of populating the + user's view of a mailbox. The benefit is that the client can display + a paperclip icon or some such indication along with the normal + message summary. However, this comes at a significant cost with some + server configurations. The parsing needed to generate the FETCH + BODYSTRUCTURE response may be time-consuming compared with that + needed for FETCH ENVELOPE. The client developer should consider this + issue when deciding whether the ability to add a paperclip icon is + worth the tradeoff in performance, especially with large mailboxes. + + Some clients, rather than using FETCH BODYSTRUCTURE, use FETCH BODY[] + (or the equivalent FETCH RFC822) to retrieve the entire message. + They then do the MIME parsing in the client. This may give the + client slightly more flexibility in some areas (access, for instance, + to header fields that aren't returned in the BODYSTRUCTURE and + + + +Leiba Informational [Page 7] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + ENVELOPE responses), but it can cause severe performance problems by + forcing the transfer of all body parts when the user might only want + to see some of them - a user logged on by modem and reading a small + text message with a large ZIP file attached may prefer to read the + text only and save the ZIP file for later. Therefore, a client + should not normally retrieve entire messages and should retrieve + message body parts selectively. + +3.2.1.5. Long Command Lines + + A client can wind up building a very long command line in an effort to + try to be efficient about requesting information from a server. This + can typically happen when a client builds a message set from selected + messages and doesn't recognise that contiguous blocks of messages may + be group in a range. Suppose a user selects all 10,000 messages in a + large mailbox and then unselects message 287. The client could build + that message set as "1:286,288:10000", but a client that doesn't + handle that might try to enumerate each message individually and build + "1,2,3,4, [and so on] ,9999,10000". Adding that to the fetch command + results in a command line that's almost 49,000 octets long, and, + clearly, one can construct a command line that's even longer. + + A client should limit the length of the command lines it generates to + approximately 1000 octets (including all quoted strings but not + including literals). If the client is unable to group things into + ranges so that the command line is within that length, it should + split the request into multiple commands. The client should use + literals instead of long quoted strings, in order to keep the command + length down. + + For its part, a server should allow for a command line of at least + 8000 octets. This provides plenty of leeway for accepting reasonable + length commands from clients. The server should send a BAD response + to a command that does not end within the server's maximum accepted + command length. + +3.2.2. Subscriptions + + The client isn't the only entity that can get flooded: the end user, + too, may need some flood control. The IMAP4 protocol provides such + control in the form of subscriptions. Most servers support the + SUBSCRIBE, UNSUBSCRIBE, and LSUB commands, and many users choose to + narrow down a large list of available mailboxes by subscribing to the + ones that they usually want to see. Clients, with this in mind, + should give the user a way to see only subscribed mailboxes. A + client that never uses the LSUB command takes a significant usability + feature away from the user. Of course, the client would not want to + hide the LIST command completely; the user needs to have a way to + + + +Leiba Informational [Page 8] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + choose between LIST and LSUB. The usual way to do this is to provide + a setting like "show which mailboxes?: [] all [] subscribed only". + +3.2.3. Searching + + IMAP SEARCH commands can become particularly troublesome (that is, + slow) on mailboxes containing a large number of messages. So let's + put a few things in perspective in that regard. + + The flag searches should be fast. The flag searches (ALL, [UN]SEEN, + [UN]ANSWERED, [UN]DELETED, [UN]DRAFT, [UN]FLAGGED, NEW, OLD, RECENT) + are known to be used by clients for the client's own use (for + instance, some clients use "SEARCH UNSEEN" to find unseen mail and + "SEARCH DELETED" to warn the user before expunging messages). + + Other searches, particularly the text searches (HEADER, TEXT, BODY) + are initiated by the user, rather than by the client itself, and + somewhat slower performance can be tolerated, since the user is aware + that the search is being done (and is probably aware that it might be + time-consuming). A smart server might use dynamic indexing to speed + commonly used text searches. + + The client may allow other commands to be sent to the server while a + SEARCH is in progress, but at the time of this writing there is + little or no server support for parallel processing of multiple + commands in the same session (and see "Multiple Accesses of the Same + Mailbox" above for a description of the dangers of trying to work + around this by doing your SEARCH in another session). + + Another word about text searches: some servers, built on database + back-ends with indexed search capabilities, may return search results + that do not match the IMAP spec's "case-insensitive substring" + requirements. While these servers are in violation of the protocol, + there is little harm in the violation as long as the search results + are used only in response to a user's request. Still, developers of + such servers should be aware that they ARE violating the protocol, + should think carefully about that behaviour, and must be certain that + their servers respond accurately to the flag searches for the reasons + outlined above. + + In addition, servers should support CHARSET UTF-8 [UTF-8] in + searches. + + + + + + + + + +Leiba Informational [Page 9] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.3 Avoiding Invalid Requests + + IMAP4 provides ways for a server to tell a client in advance what is + and isn't permitted in some circumstances. Clients should use these + features to avoid sending requests that a well designed client would + know to be invalid. This section explains this in more detail. + +3.3.1. The CAPABILITY Command + + All IMAP4 clients should use the CAPABILITY command to determine what + version of IMAP and what optional features a server supports. The + client should not send IMAP4rev1 commands and arguments to a server + that does not advertize IMAP4rev1 in its CAPABILITY response. + Similarly, the client should not send IMAP4 commands that no longer + exist in IMAP4rev1 to a server that does not advertize IMAP4 in its + CAPABILITY response. An IMAP4rev1 server is NOT required to support + obsolete IMAP4 or IMAP2bis commands (though some do; do not let this + fact lull you into thinking that it's valid to send such commands to + an IMAP4rev1 server). + + A client should not send commands to probe for the existance of + certain extensions. All standard and standards-track extensions + include CAPABILITY tokens indicating their presense. All private and + experimental extensions should do the same, and clients that take + advantage of them should use the CAPABILITY response to determine + whether they may be used or not. + +3.3.2. Don't Do What the Server Says You Can't + + In many cases, the server, in response to a command, will tell the + client something about what can and can't be done with a particular + mailbox. The client should pay attention to this information and + should not try to do things that it's been told it can't do. + + Examples: + + * Do not try to SELECT a mailbox that has the \Noselect flag set. + * Do not try to CREATE a sub-mailbox in a mailbox that has the + \Noinferiors flag set. + * Do not respond to a failing COPY or APPEND command by trying to + CREATE the target mailbox if the server does not respond with a + [TRYCREATE] response code. + * Do not try to expunge a mailbox that has been selected with the + [READ-ONLY] response code. + + + + + + + +Leiba Informational [Page 10] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.4. Miscellaneous Protocol Considerations + + We describe here a number of important protocol-related issues, the + misunderstanding of which has caused significant interoperability + problems in IMAP4 implementations. One general item is that every + implementer should be certain to take note of and to understand + section 2.2.2 and the preamble to section 7 of the IMAP4rev1 spec + [RFC-2060]. + +3.4.1. Well Formed Protocol + + We cannot stress enough the importance of adhering strictly to the + protocol grammar. The specification of the protocol is quite rigid; + do not assume that you can insert blank space for "readability" if + none is called for. Keep in mind that there are parsers out there + that will crash if there are protocol errors. There are clients that + will report every parser burp to the user. And in any case, + information that cannot be parsed is information that is lost. Be + careful in your protocol generation. And see "A Word About Testing", + below. + + In particular, note that the string in the INTERNALDATE response is + NOT an RFC-822 date string - that is, it is not in the same format as + the first string in the ENVELOPE response. Since most clients will, + in fact, accept an RFC-822 date string in the INTERNALDATE response, + it's easy to miss this in your interoperability testing. But it will + cause a problem with some client, so be sure to generate the correct + string for this field. + +3.4.2. Special Characters + + Certain characters, currently the double-quote and the backslash, may + not be sent as-is inside a quoted string. These characters must be + preceded by the escape character if they are in a quoted string, or + else the string must be sent as a literal. Both clients and servers + must handle this, both on output (they must send these characters + properly) and on input (they must be able to receive escaped + characters in quoted strings). Example: + + C: 001 LIST "" % + S: * LIST () "" INBOX + S: * LIST () "\\" TEST + S: * LIST () "\\" {12} + S: "My" mailbox + S: 001 OK done + C: 002 LIST "" "\"My\" mailbox\\%" + S: * LIST () "\\" {17} + S: "My" mailbox\Junk + + + +Leiba Informational [Page 11] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + S: 002 OK done + + Note that in the example the server sent the hierarchy delimiter as + an escaped character in the quoted string and sent the mailbox name + containing imbedded double-quotes as a literal. The client used only + quoted strings, escaping both the backslash and the double-quote + characters. + + The CR and LF characters may be sent ONLY in literals; they are not + allowed, even if escaped, inside quoted strings. + + And while we're talking about special characters: the IMAP spec, in + the section titled "Mailbox International Naming Convention", + describes how to encode mailbox names in modified UTF-7 [UTF-7 and + RFC-2060]. Implementations must adhere to this in order to be + interoperable in the international market, and servers should + validate mailbox names sent by client and reject names that do not + conform. + + As to special characters in userids and passwords: clients must not + restrict what a user may type in for a userid or a password. The + formal grammar specifies that these are "astrings", and an astring + can be a literal. A literal, in turn can contain any 8-bit + character, and clients must allow users to enter all 8-bit characters + here, and must pass them, unchanged, to the server (being careful to + send them as literals when necessary). In particular, some server + configurations use "@" in user names, and some clients do not allow + that character to be entered; this creates a severe interoperability + problem. + +3.4.3. UIDs and UIDVALIDITY + + Servers that support existing back-end mail stores often have no good + place to save UIDs for messages. Often the existing mail store will + not have the concept of UIDs in the sense that IMAP has: strictly + increasing, never re-issued, 32-bit integers. Some servers solve + this by storing the UIDs in a place that's accessible to end users, + allowing for the possibility that the users will delete them. Others + solve it by re-assigning UIDs every time a mailbox is selected. + + The server should maintain UIDs permanently for all messages if it + can. If that's not possible, the server must change the UIDVALIDITY + value for the mailbox whenever any of the UIDs may have become + invalid. Clients must recognize that the UIDVALIDITY has changed and + must respond to that condition by throwing away any information that + they have saved about UIDs in that mailbox. There have been many + problems in this area when clients have failed to do this; in the + worst case it will result in loss of mail when a client deletes the + + + +Leiba Informational [Page 12] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + wrong piece of mail by using a stale UID. + + It seems to be a common misunderstanding that "the UIDVALIDITY and + the UID, taken together, form a 64-bit identifier that uniquely + identifies a message on a server". This is absolutely NOT TRUE. + There is no assurance that the UIDVALIDITY values of two mailboxes be + different, so the UIDVALIDITY in no way identifies a mailbox. The + ONLY purpose of UIDVALIDITY is, as its name indicates, to give the + client a way to check the validity of the UIDs it has cached. While + it is a valid implementation choice to put these values together to + make a 64-bit identifier for the message, the important concept here + is that UIDs are not unique between mailboxes; they are only unique + WITHIN a given mailbox. + + Some server implementations have attempted to make UIDs unique across + the entire server. This is inadvisable, in that it limits the life + of UIDs unnecessarily. The UID is a 32-bit number and will run out + in reasonably finite time if it's global across the server. If you + assign UIDs sequentially in one mailbox, you will not have to start + re-using them until you have had, at one time or another, 2**32 + different messages in that mailbox. In the global case, you will + have to reuse them once you have had, at one time or another, 2**32 + different messages in the entire mail store. Suppose your server has + around 8000 users registered (2**13). That gives an average of 2**19 + UIDs per user. Suppose each user gets 32 messages (2**5) per day. + That gives you 2**14 days (16000+ days = about 45 years) before you + run out. That may seem like enough, but multiply the usage just a + little (a lot of spam, a lot of mailing list subscriptions, more + users) and you limit yourself too much. + + What's worse is that if you have to wrap the UIDs, and, thus, you + have to change UIDVALIDITY and invalidate the UIDs in the mailbox, + you have to do it for EVERY mailbox in the system, since they all + share the same UID pool. If you assign UIDs per mailbox and you have + a problem, you only have to kill the UIDs for that one mailbox. + + Under extreme circumstances (and this is extreme, indeed), the server + may have to invalidate UIDs while a mailbox is in use by a client - + that is, the UIDs that the client knows about in its active mailbox + are no longer valid. In that case, the server must immediately + change the UIDVALIDITY and must communicate this to the client. The + server may do this by sending an unsolicited UIDVALIDITY message, in + the same form as in response to the SELECT command. Clients must be + prepared to handle such a message and the possibly coincident failure + of the command in process. For example: + + + + + + +Leiba Informational [Page 13] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + C: 032 UID STORE 382 +Flags.silent \Deleted + S: * OK [UIDVALIDITY 12345] New UIDVALIDITY value! + S: 032 NO UID command rejected because UIDVALIDITY changed! + C: ...invalidates local information and re-fetches... + C: 033 FETCH 1:* UID + ...etc... + + At the time of the writing of this document, the only server known to + do this does so only under the following condition: the client + selects INBOX, but there is not yet a physical INBOX file created. + Nonetheless, the SELECT succeeds, exporting an empty INBOX with a + temporary UIDVALIDITY of 1. While the INBOX remains selected, mail + is delivered to the user, which creates the real INBOX file and + assigns a permanent UIDVALIDITY (that is likely not to be 1). The + server reports the change of UIDVALIDITY, but as there were no + messages before, so no UIDs have actually changed, all the client + must do is accept the change in UIDVALIDITY. + + Alternatively, a server may force the client to re-select the + mailbox, at which time it will obtain a new UIDVALIDITY value. To do + this, the server closes this client session (see "Severed + Connections" above) and the client then reconnects and gets back in + synch. Clients must be prepared for either of these behaviours. + + We do not know of, nor do we anticipate the future existance of, a + server that changes UIDVALIDITY while there are existing messages, + but clients must be prepared to handle this eventuality. + +3.4.4. FETCH Responses + + When a client asks for certain information in a FETCH command, the + server may return the requested information in any order, not + necessarily in the order that it was requested. Further, the server + may return the information in separate FETCH responses and may also + return information that was not explicitly requested (to reflect to + the client changes in the state of the subject message). Some + examples: + + C: 001 FETCH 1 UID FLAGS INTERNALDATE + S: * 5 FETCH (FLAGS (\Deleted)) + S: * 1 FETCH (FLAGS (\Seen) INTERNALDATE "..." UID 345) + S: 001 OK done + + (In this case, the responses are in a different order. Also, the + server returned a flag update for message 5, which wasn't part of the + client's request.) + + + + + +Leiba Informational [Page 14] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + C: 002 FETCH 2 UID FLAGS INTERNALDATE + S: * 2 FETCH (INTERNALDATE "...") + S: * 2 FETCH (UID 399) + S: * 2 FETCH (FLAGS ()) + S: 002 OK done + + (In this case, the responses are in a different order and were + returned in separate responses.) + + C: 003 FETCH 2 BODY[1] + S: * 2 FETCH (FLAGS (\Seen) BODY[1] {14} + S: Hello world! + S: ) + S: 003 OK done + + (In this case, the FLAGS response was added by the server, since + fetching the body part caused the server to set the \Seen flag.) + + Because of this characteristic a client must be ready to receive any + FETCH response at any time and should use that information to update + its local information about the message to which the FETCH response + refers. A client must not assume that any FETCH responses will come + in any particular order, or even that any will come at all. If after + receiving the tagged response for a FETCH command the client finds + that it did not get all of the information requested, the client + should send a NOOP command to the server to ensure that the server + has an opportunity to send any pending EXPUNGE responses to the + client (see [RFC-2180]). + +3.4.5. RFC822.SIZE + + Some back-end mail stores keep the mail in a canonical form, rather + than retaining the original MIME format of the messages. This means + that the server must reassemble the message to produce a MIME stream + when a client does a fetch such as RFC822 or BODY[], requesting the + entire message. It also may mean that the server has no convenient + way to know the RFC822.SIZE of the message. Often, such a server + will actually have to build the MIME stream to compute the size, only + to throw the stream away and report the size to the client. + + When this is the case, some servers have chosen to estimate the size, + rather than to compute it precisely. Such an estimate allows the + client to display an approximate size to the user and to use the + estimate in flood control considerations (q.v.), but requires that + the client not use the size for things such as allocation of buffers, + because those buffers might then be too small to hold the actual MIME + stream. Instead, a client should use the size that's returned in the + literal when you fetch the data. + + + +Leiba Informational [Page 15] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + The protocol requires that the RFC822.SIZE value returned by the + server be EXACT. Estimating the size is a protocol violation, and + server designers must be aware that, despite the performance savings + they might realize in using an estimate, this practice will cause + some clients to fail in various ways. If possible, the server should + compute the RFC822.SIZE for a particular message once, and then save + it for later retrieval. If that's not possible, the server must + compute the value exactly every time. Incorrect estimates do cause + severe interoperability problems with some clients. + +3.4.6. Expunged Messages + + If the server allows multiple connections to the same mailbox, it is + often possible for messages to be expunged in one client unbeknownst + to another client. Since the server is not allowed to tell the + client about these expunged messages in response to a FETCH command, + the server may have to deal with the issue of how to return + information about an expunged message. There was extensive + discussion about this issue, and the results of that discussion are + summarized in [RFC-2180]. See that reference for a detailed + explanation and for recommendations. + +3.4.7. The Namespace Issue + + Namespaces are a very muddy area in IMAP4 implementation right now + (see [NAMESPACE] for a proposal to clear the water a bit). Until the + issue is resolved, the important thing for client developers to + understand is that some servers provide access through IMAP to more + than just the user's personal mailboxes, and, in fact, the user's + personal mailboxes may be "hidden" somewhere in the user's default + hierarchy. The client, therefore, should provide a setting wherein + the user can specify a prefix to be used when accessing mailboxes. If + the user's mailboxes are all in "~/mail/", for instance, then the + user can put that string in the prefix. The client would then put + the prefix in front of any name pattern in the LIST and LSUB + commands: + + C: 001 LIST "" ~/mail/% + + (See also "Reference Names in the LIST Command" below.) + +3.4.8. Creating Special-Use Mailboxes + + It may seem at first that this is part of the namespace issue; it is + not, and is only indirectly related to it. A number of clients like + to create special-use mailboxes with particular names. Most + commonly, clients with a "trash folder" model of message deletion + want to create a mailbox with the name "Trash" or "Deleted". Some + + + +Leiba Informational [Page 16] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + clients want to create a "Drafts" mailbox, an "Outbox" mailbox, or a + "Sent Mail" mailbox. And so on. There are two major + interoperability problems with this practice: + + 1. different clients may use different names for mailboxes with + similar functions (such as "Trash" and "Deleted"), or may manage + the same mailboxes in different ways, causing problems if a user + switches between clients and + 2. there is no guarantee that the server will allow the creation of + the desired mailbox. + + The client developer is, therefore, well advised to consider + carefully the creation of any special-use mailboxes on the server, + and, further, the client must not require such mailbox creation - + that is, if you do decide to do this, you must handle gracefully the + failure of the CREATE command and behave reasonably when your + special-use mailboxes do not exist and can not be created. + + In addition, the client developer should provide a convenient way for + the user to select the names for any special-use mailboxes, allowing + the user to make these names the same in all clients used and to put + them where the user wants them. + +3.4.9. Reference Names in the LIST Command + + Many implementers of both clients and servers are confused by the + "reference name" on the LIST command. The reference name is intended + to be used in much the way a "cd" (change directory) command is used + on Unix, PC DOS, Windows, and OS/2 systems. That is, the mailbox + name is interpreted in much the same way as a file of that name would + be found if one had done a "cd" command into the directory specified + by the reference name. For example, in Unix we have the following: + + > cd /u/jones/junk + > vi banana [file is "/u/jones/junk/banana"] + > vi stuff/banana [file is "/u/jones/junk/stuff/banana"] + > vi /etc/hosts [file is "/etc/hosts"] + + In the past, there have been several interoperability problems with + this. First, while some IMAP servers are built on Unix or PC file + systems, many others are not, and the file system semantics do not + make sense in those configurations. Second, while some IMAP servers + expose the underlying file system to the clients, others allow access + only to the user's personal mailboxes, or to some other limited set + of files, making such file-system-like semantics less meaningful. + Third, because the IMAP spec leaves the interpretation of the + reference name as "implementation-dependent", in the past the various + server implementations handled it in vastly differing ways. + + + +Leiba Informational [Page 17] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + The following recommendations are the result of significant + operational experience, and are intended to maximize + interoperability. + + Server implementations must implement the reference argument in a way + that matches the intended "change directory" operation as closely as + possible. As a minimum implementation, the reference argument may be + prepended to the mailbox name (while suppressing double delimiters; + see the next paragraph). Even servers that do not provide a way to + break out of the current hierarchy (see "breakout facility" below) + must provide a reasonable implementation of the reference argument, + as described here, so that they will interoperate with clients that + use it. + + Server implementations that prepend the reference argument to the + mailbox name should insert a hierarchy delimiter between them, and + must not insert a second if one is already present: + + C: A001 LIST ABC DEF + S: * LIST () "/" ABC/DEF <=== should do this + S: A001 OK done + + C: A002 LIST ABC/ /DEF + S: * LIST () "/" ABC//DEF <=== must not do this + S: A002 OK done + + On clients, the reference argument is chiefly used to implement a + "breakout facility", wherein the user may directly access a mailbox + outside the "current directory" hierarchy. Client implementations + should have an operational mode that does not use the reference + argument. This is to interoperate with older servers that did not + implement the reference argument properly. While it's a good idea to + give the user access to a breakout facility, clients that do not + intend to do so should not use the reference argument at all. + + Client implementations should always place a trailing hierarchy + delimiter on the reference argument. This is because some servers + prepend the reference argument to the mailbox name without inserting + a hierarchy delimiter, while others do insert a hierarchy delimiter + if one is not already present. A client that puts the delimiter in + will work with both varieties of server. + + Client implementations that implement a breakout facility should + allow the user to choose whether or not to use a leading hierarchy + delimiter on the mailbox argument. This is because the handling of a + leading mailbox hierarchy delimiter also varies from server to + server, and even between different mailstores on the same server. In + some cases, a leading hierarchy delimiter means "discard the + + + +Leiba Informational [Page 18] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + reference argument" (implementing the intended breakout facility), + thus: + + C: A001 LIST ABC/ /DEF + S: * LIST () "/" /DEF + S: A001 OK done + + In other cases, however, the two are catenated and the extra + hierarchy delimiter is discarded, thus: + + C: A001 LIST ABC/ /DEF + S: * LIST () "/" ABC/DEF + S: A001 OK done + + Client implementations must not assume that the server supports a + breakout facility, but may provide a way for the user to use one if + it is available. Any breakout facility should be exported to the + user interface. Note that there may be other "breakout" characters + besides the hierarchy delimiter (for instance, UNIX filesystem + servers are likely to use a leading "~" as well), and that their + interpretation is server-dependent. + +3.4.10. Mailbox Hierarchy Delimiters + + The server's selection of what to use as a mailbox hierarchy + delimiter is a difficult one, involving several issues: What + characters do users expect to see? What characters can they enter + for a hierarchy delimiter if it is desired (or required) that the + user enter it? What character can be used for the hierarchy + delimiter, noting that the chosen character can not otherwise be used + in the mailbox name? + + Because some interfaces show users the hierarchy delimiters or allow + users to enter qualified mailbox names containing them, server + implementations should use delimiter characters that users generally + expect to see as name separators. The most common characters used + for this are "/" (as in Unix file names), "\" (as in OS/2 and Windows + file names), and "." (as in news groups). There is little to choose + among these apart from what users may expect or what is dictated by + the underlying file system, if any. One consideration about using + "\" is that it's also a special character in the IMAP protocol. While + the use of other hierarchy delimiter characters is permissible, A + DESIGNER IS WELL ADVISED TO STAY WITH ONE FROM THIS SET unless the + server is intended for special purposes only. Implementers might be + thinking about using characters such as "-", "_", ";", "&", "#", "@", + and "!", but they should be aware of the surprise to the user as well + as of the effect on URLs and other external specifications (since + some of these characters have special meanings there). Also, a + + + +Leiba Informational [Page 19] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + server that uses "\" (and clients of such a server) must remember to + escape that character in quoted strings or to send literals instead. + Literals are recommended over escaped characters in quoted strings in + order to maintain compatibility with older IMAP versions that did not + allow escaped characters in quoted strings (but check the grammar to + see where literals are allowed): + + C: 001 LIST "" {13} + S: + send literal + C: this\%\%\%\h* + S: * LIST () "\\" {27} + S: this\is\a\mailbox\hierarchy + S: 001 OK LIST complete + + In any case, a server should not use normal alpha-numeric characters + (such as "X" or "0") as delimiters; a user would be very surprised to + find that "EXPENDITURES" actually represented a two-level hierarchy. + And a server should not use characters that are non-printable or + difficult or impossible to enter on a standard US keyboard. Control + characters, box-drawing characters, and characters from non-US + alphabets fit into this category. Their use presents + interoperability problems that are best avoided. + + The UTF-7 encoding of mailbox names also raises questions about what + to do with the hierarchy delimiters in encoded names: do we encode + each hierarchy level and separate them with delimiters, or do we + encode the fully qualified name, delimiters and all? The answer for + IMAP is the former: encode each hierarchy level separately, and + insert delimiters between. This makes it particularly important not + to use as a hierarchy delimiter a character that might cause + confusion with IMAP's modified UTF-7 [UTF-7 and RFC-2060] encoding. + + To repeat: a server should use "/", "\", or "." as its hierarchy + delimiter. The use of any other character is likely to cause + problems and is STRONGLY DISCOURAGED. + +3.4.11. ALERT Response Codes + + The protocol spec is very clear on the matter of what to do with + ALERT response codes, and yet there are many clients that violate it + so it needs to be said anyway: "The human-readable text contains a + special alert that must be presented to the user in a fashion that + calls the user's attention to the message." That should be clear + enough, but I'll repeat it here: Clients must present ALERT text + clearly to the user. + + + + + + +Leiba Informational [Page 20] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.4.12. Deleting Mailboxes + + The protocol does not guarantee that a client may delete a mailbox + that is not empty, though on some servers it is permissible and is, + in fact, much faster than the alternative or deleting all the + messages from the client. If the client chooses to try to take + advantage of this possibility it must be prepared to use the other + method in the even that the more convenient one fails. Further, a + client should not try to delete the mailbox that it has selected, but + should first close that mailbox; some servers do not permit the + deletion of the selected mailbox. + + That said, a server should permit the deletion of a non-empty + mailbox; there's little reason to pass this work on to the client. + Moreover, forbidding this prevents the deletion of a mailbox that for + some reason can not be opened or expunged, leading to possible + denial-of-service problems. + + Example: + + [User tells the client to delete mailbox BANANA, which is + currently selected...] + C: 008 CLOSE + S: 008 OK done + C: 009 DELETE BANANA + S: 009 NO Delete failed; mailbox is not empty. + C: 010 SELECT BANANA + S: * ... untagged SELECT responses + S: 010 OK done + C: 011 STORE 1:* +FLAGS.SILENT \DELETED + S: 011 OK done + C: 012 CLOSE + S: 012 OK done + C: 013 DELETE BANANA + S: 013 OK done + +3.5. A Word About Testing + + Since the whole point of IMAP is interoperability, and since + interoperability can not be tested in a vacuum, the final + recommendation of this treatise is, "Test against EVERYTHING." Test + your client against every server you can get an account on. Test + your server with every client you can get your hands on. Many + clients make limited test versions available on the Web for the + downloading. Many server owners will give serious client developers + guest accounts for testing. Contact them and ask. NEVER assume that + because your client works with one or two servers, or because your + server does fine with one or two clients, you will interoperate well + + + +Leiba Informational [Page 21] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + in general. + + In particular, in addition to everything else, be sure to test + against the reference implementations: the PINE client, the + University of Washington server, and the Cyrus server. + + See the following URLs on the web for more information here: + + IMAP Products and Sources: http://www.imap.org/products.html + IMC MailConnect: http://www.imc.org/imc-mailconnect + +4. Security Considerations + + This document describes behaviour of clients and servers that use the + IMAP4 protocol, and as such, has the same security considerations as + described in [RFC-2060]. + +5. References + + [RFC-2060] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [RFC-2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC-2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC + 2180, July 1997. + + [UTF-8] Yergeau, F., " UTF-8, a transformation format of Unicode + and ISO 10646", RFC 2044, October 1996. + + [UTF-7] Goldsmith, D. and M. Davis, "UTF-7, a Mail-Safe + Transformation Format of Unicode", RFC 2152, May 1997. + + [NAMESPACE] Gahrns, M. and C. Newman, "IMAP4 Namespace", Work in + Progress. + +6. Author's Address + + Barry Leiba + IBM T.J. Watson Research Center + 30 Saw Mill River Road + Hawthorne, NY 10532 + + Phone: 1-914-784-7941 + EMail: leiba@watson.ibm.com + + + + + +Leiba Informational [Page 22] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +7. Full Copyright Statement + + Copyright (C) The Internet Society (1999). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Leiba Informational [Page 23] + diff --git a/docs/rfcs/rfc2821.txt b/docs/rfcs/rfc2821.txt new file mode 100644 index 0000000..0eac911 --- /dev/null +++ b/docs/rfcs/rfc2821.txt @@ -0,0 +1,4427 @@ + + + + + + +Network Working Group J. Klensin, Editor +Request for Comments: 2821 AT&T Laboratories +Obsoletes: 821, 974, 1869 April 2001 +Updates: 1123 +Category: Standards Track + + + Simple Mail Transfer Protocol + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2001). All Rights Reserved. + +Abstract + + This document is a self-contained specification of the basic protocol + for the Internet electronic mail transport. It consolidates, updates + and clarifies, but doesn't add new or change existing functionality + of the following: + + - the original SMTP (Simple Mail Transfer Protocol) specification of + RFC 821 [30], + + - domain name system requirements and implications for mail + transport from RFC 1035 [22] and RFC 974 [27], + + - the clarifications and applicability statements in RFC 1123 [2], + and + + - material drawn from the SMTP Extension mechanisms [19]. + + It obsoletes RFC 821, RFC 974, and updates RFC 1123 (replaces the + mail transport materials of RFC 1123). However, RFC 821 specifies + some features that were not in significant use in the Internet by the + mid-1990s and (in appendices) some additional transport models. + Those sections are omitted here in the interest of clarity and + brevity; readers needing them should refer to RFC 821. + + + + + + +Klensin Standards Track [Page 1] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + It also includes some additional material from RFC 1123 that required + amplification. This material has been identified in multiple ways, + mostly by tracking flaming on various lists and newsgroups and + problems of unusual readings or interpretations that have appeared as + the SMTP extensions have been deployed. Where this specification + moves beyond consolidation and actually differs from earlier + documents, it supersedes them technically as well as textually. + + Although SMTP was designed as a mail transport and delivery protocol, + this specification also contains information that is important to its + use as a 'mail submission' protocol, as recommended for POP [3, 26] + and IMAP [6]. Additional submission issues are discussed in RFC 2476 + [15]. + + Section 2.3 provides definitions of terms specific to this document. + Except when the historical terminology is necessary for clarity, this + document uses the current 'client' and 'server' terminology to + identify the sending and receiving SMTP processes, respectively. + + A companion document [32] discusses message headers, message bodies + and formats and structures for them, and their relationship. + +Table of Contents + + 1. Introduction .................................................. 4 + 2. The SMTP Model ................................................ 5 + 2.1 Basic Structure .............................................. 5 + 2.2 The Extension Model .......................................... 7 + 2.2.1 Background ................................................. 7 + 2.2.2 Definition and Registration of Extensions .................. 8 + 2.3 Terminology .................................................. 9 + 2.3.1 Mail Objects ............................................... 10 + 2.3.2 Senders and Receivers ...................................... 10 + 2.3.3 Mail Agents and Message Stores ............................. 10 + 2.3.4 Host ....................................................... 11 + 2.3.5 Domain ..................................................... 11 + 2.3.6 Buffer and State Table ..................................... 11 + 2.3.7 Lines ...................................................... 12 + 2.3.8 Originator, Delivery, Relay, and Gateway Systems ........... 12 + 2.3.9 Message Content and Mail Data .............................. 13 + 2.3.10 Mailbox and Address ....................................... 13 + 2.3.11 Reply ..................................................... 13 + 2.4 General Syntax Principles and Transaction Model .............. 13 + 3. The SMTP Procedures: An Overview .............................. 15 + 3.1 Session Initiation ........................................... 15 + 3.2 Client Initiation ............................................ 16 + 3.3 Mail Transactions ............................................ 16 + 3.4 Forwarding for Address Correction or Updating ................ 19 + + + +Klensin Standards Track [Page 2] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + 3.5 Commands for Debugging Addresses ............................. 20 + 3.5.1 Overview ................................................... 20 + 3.5.2 VRFY Normal Response ....................................... 22 + 3.5.3 Meaning of VRFY or EXPN Success Response ................... 22 + 3.5.4 Semantics and Applications of EXPN ......................... 23 + 3.6 Domains ...................................................... 23 + 3.7 Relaying ..................................................... 24 + 3.8 Mail Gatewaying .............................................. 25 + 3.8.1 Header Fields in Gatewaying ................................ 26 + 3.8.2 Received Lines in Gatewaying ............................... 26 + 3.8.3 Addresses in Gatewaying .................................... 26 + 3.8.4 Other Header Fields in Gatewaying .......................... 27 + 3.8.5 Envelopes in Gatewaying .................................... 27 + 3.9 Terminating Sessions and Connections ......................... 27 + 3.10 Mailing Lists and Aliases ................................... 28 + 3.10.1 Alias ..................................................... 28 + 3.10.2 List ...................................................... 28 + 4. The SMTP Specifications ....................................... 29 + 4.1 SMTP Commands ................................................ 29 + 4.1.1 Command Semantics and Syntax ............................... 29 + 4.1.1.1 Extended HELLO (EHLO) or HELLO (HELO) ................... 29 + 4.1.1.2 MAIL (MAIL) .............................................. 31 + 4.1.1.3 RECIPIENT (RCPT) ......................................... 31 + 4.1.1.4 DATA (DATA) .............................................. 33 + 4.1.1.5 RESET (RSET) ............................................. 34 + 4.1.1.6 VERIFY (VRFY) ............................................ 35 + 4.1.1.7 EXPAND (EXPN) ............................................ 35 + 4.1.1.8 HELP (HELP) .............................................. 35 + 4.1.1.9 NOOP (NOOP) .............................................. 35 + 4.1.1.10 QUIT (QUIT) ............................................. 36 + 4.1.2 Command Argument Syntax .................................... 36 + 4.1.3 Address Literals ........................................... 38 + 4.1.4 Order of Commands .......................................... 39 + 4.1.5 Private-use Commands ....................................... 40 + 4.2 SMTP Replies ................................................ 40 + 4.2.1 Reply Code Severities and Theory ........................... 42 + 4.2.2 Reply Codes by Function Groups ............................. 44 + 4.2.3 Reply Codes in Numeric Order .............................. 45 + 4.2.4 Reply Code 502 ............................................. 46 + 4.2.5 Reply Codes After DATA and the Subsequent . .... 46 + 4.3 Sequencing of Commands and Replies ........................... 47 + 4.3.1 Sequencing Overview ........................................ 47 + 4.3.2 Command-Reply Sequences .................................... 48 + 4.4 Trace Information ............................................ 49 + 4.5 Additional Implementation Issues ............................. 53 + 4.5.1 Minimum Implementation ..................................... 53 + 4.5.2 Transparency ............................................... 53 + 4.5.3 Sizes and Timeouts ......................................... 54 + + + +Klensin Standards Track [Page 3] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + 4.5.3.1 Size limits and minimums ................................. 54 + 4.5.3.2 Timeouts ................................................. 56 + 4.5.4 Retry Strategies ........................................... 57 + 4.5.4.1 Sending Strategy ......................................... 58 + 4.5.4.2 Receiving Strategy ....................................... 59 + 4.5.5 Messages with a null reverse-path .......................... 59 + 5. Address Resolution and Mail Handling .......................... 60 + 6. Problem Detection and Handling ................................ 62 + 6.1 Reliable Delivery and Replies by Email ....................... 62 + 6.2 Loop Detection ............................................... 63 + 6.3 Compensating for Irregularities .............................. 63 + 7. Security Considerations ....................................... 64 + 7.1 Mail Security and Spoofing ................................... 64 + 7.2 "Blind" Copies ............................................... 65 + 7.3 VRFY, EXPN, and Security ..................................... 65 + 7.4 Information Disclosure in Announcements ...................... 66 + 7.5 Information Disclosure in Trace Fields ....................... 66 + 7.6 Information Disclosure in Message Forwarding ................. 67 + 7.7 Scope of Operation of SMTP Servers ........................... 67 + 8. IANA Considerations ........................................... 67 + 9. References .................................................... 68 + 10. Editor's Address ............................................. 70 + 11. Acknowledgments .............................................. 70 + Appendices ....................................................... 71 + A. TCP Transport Service ......................................... 71 + B. Generating SMTP Commands from RFC 822 Headers ................. 71 + C. Source Routes ................................................. 72 + D. Scenarios ..................................................... 73 + E. Other Gateway Issues .......................................... 76 + F. Deprecated Features of RFC 821 ................................ 76 + Full Copyright Statement ......................................... 79 + +1. Introduction + + The objective of the Simple Mail Transfer Protocol (SMTP) is to + transfer mail reliably and efficiently. + + SMTP is independent of the particular transmission subsystem and + requires only a reliable ordered data stream channel. While this + document specifically discusses transport over TCP, other transports + are possible. Appendices to RFC 821 describe some of them. + + An important feature of SMTP is its capability to transport mail + across networks, usually referred to as "SMTP mail relaying" (see + section 3.8). A network consists of the mutually-TCP-accessible + hosts on the public Internet, the mutually-TCP-accessible hosts on a + firewall-isolated TCP/IP Intranet, or hosts in some other LAN or WAN + environment utilizing a non-TCP transport-level protocol. Using + + + +Klensin Standards Track [Page 4] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + SMTP, a process can transfer mail to another process on the same + network or to some other network via a relay or gateway process + accessible to both networks. + + In this way, a mail message may pass through a number of intermediate + relay or gateway hosts on its path from sender to ultimate recipient. + The Mail eXchanger mechanisms of the domain name system [22, 27] (and + section 5 of this document) are used to identify the appropriate + next-hop destination for a message being transported. + +2. The SMTP Model + +2.1 Basic Structure + + The SMTP design can be pictured as: + + +----------+ +----------+ + +------+ | | | | + | User |<-->| | SMTP | | + +------+ | Client- |Commands/Replies| Server- | + +------+ | SMTP |<-------------->| SMTP | +------+ + | File |<-->| | and Mail | |<-->| File | + |System| | | | | |System| + +------+ +----------+ +----------+ +------+ + SMTP client SMTP server + + When an SMTP client has a message to transmit, it establishes a two- + way transmission channel to an SMTP server. The responsibility of an + SMTP client is to transfer mail messages to one or more SMTP servers, + or report its failure to do so. + + The means by which a mail message is presented to an SMTP client, and + how that client determines the domain name(s) to which mail messages + are to be transferred is a local matter, and is not addressed by this + document. In some cases, the domain name(s) transferred to, or + determined by, an SMTP client will identify the final destination(s) + of the mail message. In other cases, common with SMTP clients + associated with implementations of the POP [3, 26] or IMAP [6] + protocols, or when the SMTP client is inside an isolated transport + service environment, the domain name determined will identify an + intermediate destination through which all mail messages are to be + relayed. SMTP clients that transfer all traffic, regardless of the + target domain names associated with the individual messages, or that + do not maintain queues for retrying message transmissions that + initially cannot be completed, may otherwise conform to this + specification but are not considered fully-capable. Fully-capable + SMTP implementations, including the relays used by these less capable + + + + +Klensin Standards Track [Page 5] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + ones, and their destinations, are expected to support all of the + queuing, retrying, and alternate address functions discussed in this + specification. + + The means by which an SMTP client, once it has determined a target + domain name, determines the identity of an SMTP server to which a + copy of a message is to be transferred, and then performs that + transfer, is covered by this document. To effect a mail transfer to + an SMTP server, an SMTP client establishes a two-way transmission + channel to that SMTP server. An SMTP client determines the address + of an appropriate host running an SMTP server by resolving a + destination domain name to either an intermediate Mail eXchanger host + or a final target host. + + An SMTP server may be either the ultimate destination or an + intermediate "relay" (that is, it may assume the role of an SMTP + client after receiving the message) or "gateway" (that is, it may + transport the message further using some protocol other than SMTP). + SMTP commands are generated by the SMTP client and sent to the SMTP + server. SMTP replies are sent from the SMTP server to the SMTP + client in response to the commands. + + In other words, message transfer can occur in a single connection + between the original SMTP-sender and the final SMTP-recipient, or can + occur in a series of hops through intermediary systems. In either + case, a formal handoff of responsibility for the message occurs: the + protocol requires that a server accept responsibility for either + delivering a message or properly reporting the failure to do so. + + Once the transmission channel is established and initial handshaking + completed, the SMTP client normally initiates a mail transaction. + Such a transaction consists of a series of commands to specify the + originator and destination of the mail and transmission of the + message content (including any headers or other structure) itself. + When the same message is sent to multiple recipients, this protocol + encourages the transmission of only one copy of the data for all + recipients at the same destination (or intermediate relay) host. + + The server responds to each command with a reply; replies may + indicate that the command was accepted, that additional commands are + expected, or that a temporary or permanent error condition exists. + Commands specifying the sender or recipients may include server- + permitted SMTP service extension requests as discussed in section + 2.2. The dialog is purposely lock-step, one-at-a-time, although this + can be modified by mutually-agreed extension requests such as command + pipelining [13]. + + + + + +Klensin Standards Track [Page 6] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + Once a given mail message has been transmitted, the client may either + request that the connection be shut down or may initiate other mail + transactions. In addition, an SMTP client may use a connection to an + SMTP server for ancillary services such as verification of email + addresses or retrieval of mailing list subscriber addresses. + + As suggested above, this protocol provides mechanisms for the + transmission of mail. This transmission normally occurs directly + from the sending user's host to the receiving user's host when the + two hosts are connected to the same transport service. When they are + not connected to the same transport service, transmission occurs via + one or more relay SMTP servers. An intermediate host that acts as + either an SMTP relay or as a gateway into some other transmission + environment is usually selected through the use of the domain name + service (DNS) Mail eXchanger mechanism. + + Usually, intermediate hosts are determined via the DNS MX record, not + by explicit "source" routing (see section 5 and appendices C and + F.2). + +2.2 The Extension Model + +2.2.1 Background + + In an effort that started in 1990, approximately a decade after RFC + 821 was completed, the protocol was modified with a "service + extensions" model that permits the client and server to agree to + utilize shared functionality beyond the original SMTP requirements. + The SMTP extension mechanism defines a means whereby an extended SMTP + client and server may recognize each other, and the server can inform + the client as to the service extensions that it supports. + + Contemporary SMTP implementations MUST support the basic extension + mechanisms. For instance, servers MUST support the EHLO command even + if they do not implement any specific extensions and clients SHOULD + preferentially utilize EHLO rather than HELO. (However, for + compatibility with older conforming implementations, SMTP clients and + servers MUST support the original HELO mechanisms as a fallback.) + Unless the different characteristics of HELO must be identified for + interoperability purposes, this document discusses only EHLO. + + SMTP is widely deployed and high-quality implementations have proven + to be very robust. However, the Internet community now considers + some services to be important that were not anticipated when the + protocol was first designed. If support for those services is to be + added, it must be done in a way that permits older implementations to + continue working acceptably. The extension framework consists of: + + + + +Klensin Standards Track [Page 7] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + - The SMTP command EHLO, superseding the earlier HELO, + + - a registry of SMTP service extensions, + + - additional parameters to the SMTP MAIL and RCPT commands, and + + - optional replacements for commands defined in this protocol, such + as for DATA in non-ASCII transmissions [33]. + + SMTP's strength comes primarily from its simplicity. Experience with + many protocols has shown that protocols with few options tend towards + ubiquity, whereas protocols with many options tend towards obscurity. + + Each and every extension, regardless of its benefits, must be + carefully scrutinized with respect to its implementation, deployment, + and interoperability costs. In many cases, the cost of extending the + SMTP service will likely outweigh the benefit. + +2.2.2 Definition and Registration of Extensions + + The IANA maintains a registry of SMTP service extensions. A + corresponding EHLO keyword value is associated with each extension. + Each service extension registered with the IANA must be defined in a + formal standards-track or IESG-approved experimental protocol + document. The definition must include: + + - the textual name of the SMTP service extension; + + - the EHLO keyword value associated with the extension; + + - the syntax and possible values of parameters associated with the + EHLO keyword value; + + - any additional SMTP verbs associated with the extension + (additional verbs will usually be, but are not required to be, the + same as the EHLO keyword value); + + - any new parameters the extension associates with the MAIL or RCPT + verbs; + + - a description of how support for the extension affects the + behavior of a server and client SMTP; and, + + - the increment by which the extension is increasing the maximum + length of the commands MAIL and/or RCPT, over that specified in + this standard. + + + + + +Klensin Standards Track [Page 8] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + In addition, any EHLO keyword value starting with an upper or lower + case "X" refers to a local SMTP service extension used exclusively + through bilateral agreement. Keywords beginning with "X" MUST NOT be + used in a registered service extension. Conversely, keyword values + presented in the EHLO response that do not begin with "X" MUST + correspond to a standard, standards-track, or IESG-approved + experimental SMTP service extension registered with IANA. A + conforming server MUST NOT offer non-"X"-prefixed keyword values that + are not described in a registered extension. + + Additional verbs and parameter names are bound by the same rules as + EHLO keywords; specifically, verbs beginning with "X" are local + extensions that may not be registered or standardized. Conversely, + verbs not beginning with "X" must always be registered. + +2.3 Terminology + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described below. + + 1. MUST This word, or the terms "REQUIRED" or "SHALL", mean that + the definition is an absolute requirement of the specification. + + 2. MUST NOT This phrase, or the phrase "SHALL NOT", mean that the + definition is an absolute prohibition of the specification. + + 3. SHOULD This word, or the adjective "RECOMMENDED", mean that + there may exist valid reasons in particular circumstances to + ignore a particular item, but the full implications must be + understood and carefully weighed before choosing a different + course. + + 4. SHOULD NOT This phrase, or the phrase "NOT RECOMMENDED" mean + that there may exist valid reasons in particular circumstances + when the particular behavior is acceptable or even useful, but the + full implications should be understood and the case carefully + weighed before implementing any behavior described with this + label. + + 5. MAY This word, or the adjective "OPTIONAL", mean that an item is + truly optional. One vendor may choose to include the item because + a particular marketplace requires it or because the vendor feels + that it enhances the product while another vendor may omit the + same item. An implementation which does not include a particular + option MUST be prepared to interoperate with another + implementation which does include the option, though perhaps with + reduced functionality. In the same vein an implementation which + + + +Klensin Standards Track [Page 9] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + does include a particular option MUST be prepared to interoperate + with another implementation which does not include the option + (except, of course, for the feature the option provides.) + +2.3.1 Mail Objects + + SMTP transports a mail object. A mail object contains an envelope + and content. + + The SMTP envelope is sent as a series of SMTP protocol units + (described in section 3). It consists of an originator address (to + which error reports should be directed); one or more recipient + addresses; and optional protocol extension material. Historically, + variations on the recipient address specification command (RCPT TO) + could be used to specify alternate delivery modes, such as immediate + display; those variations have now been deprecated (see appendix F, + section F.6). + + The SMTP content is sent in the SMTP DATA protocol unit and has two + parts: the headers and the body. If the content conforms to other + contemporary standards, the headers form a collection of field/value + pairs structured as in the message format specification [32]; the + body, if structured, is defined according to MIME [12]. The content + is textual in nature, expressed using the US-ASCII repertoire [1]. + Although SMTP extensions (such as "8BITMIME" [20]) may relax this + restriction for the content body, the content headers are always + encoded using the US-ASCII repertoire. A MIME extension [23] defines + an algorithm for representing header values outside the US-ASCII + repertoire, while still encoding them using the US-ASCII repertoire. + +2.3.2 Senders and Receivers + + In RFC 821, the two hosts participating in an SMTP transaction were + described as the "SMTP-sender" and "SMTP-receiver". This document + has been changed to reflect current industry terminology and hence + refers to them as the "SMTP client" (or sometimes just "the client") + and "SMTP server" (or just "the server"), respectively. Since a + given host may act both as server and client in a relay situation, + "receiver" and "sender" terminology is still used where needed for + clarity. + +2.3.3 Mail Agents and Message Stores + + Additional mail system terminology became common after RFC 821 was + published and, where convenient, is used in this specification. In + particular, SMTP servers and clients provide a mail transport service + and therefore act as "Mail Transfer Agents" (MTAs). "Mail User + Agents" (MUAs or UAs) are normally thought of as the sources and + + + +Klensin Standards Track [Page 10] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + targets of mail. At the source, an MUA might collect mail to be + transmitted from a user and hand it off to an MTA; the final + ("delivery") MTA would be thought of as handing the mail off to an + MUA (or at least transferring responsibility to it, e.g., by + depositing the message in a "message store"). However, while these + terms are used with at least the appearance of great precision in + other environments, the implied boundaries between MUAs and MTAs + often do not accurately match common, and conforming, practices with + Internet mail. Hence, the reader should be cautious about inferring + the strong relationships and responsibilities that might be implied + if these terms were used elsewhere. + +2.3.4 Host + + For the purposes of this specification, a host is a computer system + attached to the Internet (or, in some cases, to a private TCP/IP + network) and supporting the SMTP protocol. Hosts are known by names + (see "domain"); identifying them by numerical address is discouraged. + +2.3.5 Domain + + A domain (or domain name) consists of one or more dot-separated + components. These components ("labels" in DNS terminology [22]) are + restricted for SMTP purposes to consist of a sequence of letters, + digits, and hyphens drawn from the ASCII character set [1]. Domain + names are used as names of hosts and of other entities in the domain + name hierarchy. For example, a domain may refer to an alias (label + of a CNAME RR) or the label of Mail eXchanger records to be used to + deliver mail instead of representing a host name. See [22] and + section 5 of this specification. + + The domain name, as described in this document and in [22], is the + entire, fully-qualified name (often referred to as an "FQDN"). A + domain name that is not in FQDN form is no more than a local alias. + Local aliases MUST NOT appear in any SMTP transaction. + +2.3.6 Buffer and State Table + + SMTP sessions are stateful, with both parties carefully maintaining a + common view of the current state. In this document we model this + state by a virtual "buffer" and a "state table" on the server which + may be used by the client to, for example, "clear the buffer" or + "reset the state table," causing the information in the buffer to be + discarded and the state to be returned to some previous state. + + + + + + + +Klensin Standards Track [Page 11] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +2.3.7 Lines + + SMTP commands and, unless altered by a service extension, message + data, are transmitted in "lines". Lines consist of zero or more data + characters terminated by the sequence ASCII character "CR" (hex value + 0D) followed immediately by ASCII character "LF" (hex value 0A). + This termination sequence is denoted as in this document. + Conforming implementations MUST NOT recognize or generate any other + character or character sequence as a line terminator. Limits MAY be + imposed on line lengths by servers (see section 4.5.3). + + In addition, the appearance of "bare" "CR" or "LF" characters in text + (i.e., either without the other) has a long history of causing + problems in mail implementations and applications that use the mail + system as a tool. SMTP client implementations MUST NOT transmit + these characters except when they are intended as line terminators + and then MUST, as indicated above, transmit them only as a + sequence. + +2.3.8 Originator, Delivery, Relay, and Gateway Systems + + This specification makes a distinction among four types of SMTP + systems, based on the role those systems play in transmitting + electronic mail. An "originating" system (sometimes called an SMTP + originator) introduces mail into the Internet or, more generally, + into a transport service environment. A "delivery" SMTP system is + one that receives mail from a transport service environment and + passes it to a mail user agent or deposits it in a message store + which a mail user agent is expected to subsequently access. A + "relay" SMTP system (usually referred to just as a "relay") receives + mail from an SMTP client and transmits it, without modification to + the message data other than adding trace information, to another SMTP + server for further relaying or for delivery. + + A "gateway" SMTP system (usually referred to just as a "gateway") + receives mail from a client system in one transport environment and + transmits it to a server system in another transport environment. + Differences in protocols or message semantics between the transport + environments on either side of a gateway may require that the gateway + system perform transformations to the message that are not permitted + to SMTP relay systems. For the purposes of this specification, + firewalls that rewrite addresses should be considered as gateways, + even if SMTP is used on both sides of them (see [11]). + + + + + + + + +Klensin Standards Track [Page 12] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +2.3.9 Message Content and Mail Data + + The terms "message content" and "mail data" are used interchangeably + in this document to describe the material transmitted after the DATA + command is accepted and before the end of data indication is + transmitted. Message content includes message headers and the + possibly-structured message body. The MIME specification [12] + provides the standard mechanisms for structured message bodies. + +2.3.10 Mailbox and Address + + As used in this specification, an "address" is a character string + that identifies a user to whom mail will be sent or a location into + which mail will be deposited. The term "mailbox" refers to that + depository. The two terms are typically used interchangeably unless + the distinction between the location in which mail is placed (the + mailbox) and a reference to it (the address) is important. An + address normally consists of user and domain specifications. The + standard mailbox naming convention is defined to be "local- + part@domain": contemporary usage permits a much broader set of + applications than simple "user names". Consequently, and due to a + long history of problems when intermediate hosts have attempted to + optimize transport by modifying them, the local-part MUST be + interpreted and assigned semantics only by the host specified in the + domain part of the address. + +2.3.11 Reply + + An SMTP reply is an acknowledgment (positive or negative) sent from + receiver to sender via the transmission channel in response to a + command. The general form of a reply is a numeric completion code + (indicating failure or success) usually followed by a text string. + The codes are for use by programs and the text is usually intended + for human users. Recent work [34] has specified further structuring + of the reply strings, including the use of supplemental and more + specific completion codes. + +2.4 General Syntax Principles and Transaction Model + + SMTP commands and replies have a rigid syntax. All commands begin + with a command verb. All Replies begin with a three digit numeric + code. In some commands and replies, arguments MUST follow the verb + or reply code. Some commands do not accept arguments (after the + verb), and some reply codes are followed, sometimes optionally, by + free form text. In both cases, where text appears, it is separated + from the verb or reply code by a space character. Complete + definitions of commands and replies appear in section 4. + + + + +Klensin Standards Track [Page 13] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + Verbs and argument values (e.g., "TO:" or "to:" in the RCPT command + and extension name keywords) are not case sensitive, with the sole + exception in this specification of a mailbox local-part (SMTP + Extensions may explicitly specify case-sensitive elements). That is, + a command verb, an argument value other than a mailbox local-part, + and free form text MAY be encoded in upper case, lower case, or any + mixture of upper and lower case with no impact on its meaning. This + is NOT true of a mailbox local-part. The local-part of a mailbox + MUST BE treated as case sensitive. Therefore, SMTP implementations + MUST take care to preserve the case of mailbox local-parts. Mailbox + domains are not case sensitive. In particular, for some hosts the + user "smith" is different from the user "Smith". However, exploiting + the case sensitivity of mailbox local-parts impedes interoperability + and is discouraged. + + A few SMTP servers, in violation of this specification (and RFC 821) + require that command verbs be encoded by clients in upper case. + Implementations MAY wish to employ this encoding to accommodate those + servers. + + The argument field consists of a variable length character string + ending with the end of the line, i.e., with the character sequence + . The receiver will take no action until this sequence is + received. + + The syntax for each command is shown with the discussion of that + command. Common elements and parameters are shown in section 4.1.2. + + Commands and replies are composed of characters from the ASCII + character set [1]. When the transport service provides an 8-bit byte + (octet) transmission channel, each 7-bit character is transmitted + right justified in an octet with the high order bit cleared to zero. + More specifically, the unextended SMTP service provides seven bit + transport only. An originating SMTP client which has not + successfully negotiated an appropriate extension with a particular + server MUST NOT transmit messages with information in the high-order + bit of octets. If such messages are transmitted in violation of this + rule, receiving SMTP servers MAY clear the high-order bit or reject + the message as invalid. In general, a relay SMTP SHOULD assume that + the message content it has received is valid and, assuming that the + envelope permits doing so, relay it without inspecting that content. + Of course, if the content is mislabeled and the data path cannot + accept the actual content, this may result in ultimate delivery of a + severely garbled message to the recipient. Delivery SMTP systems MAY + reject ("bounce") such messages rather than deliver them. No sending + SMTP system is permitted to send envelope commands in any character + + + + + +Klensin Standards Track [Page 14] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + set other than US-ASCII; receiving systems SHOULD reject such + commands, normally using "500 syntax error - invalid character" + replies. + + Eight-bit message content transmission MAY be requested of the server + by a client using extended SMTP facilities, notably the "8BITMIME" + extension [20]. 8BITMIME SHOULD be supported by SMTP servers. + However, it MUST not be construed as authorization to transmit + unrestricted eight bit material. 8BITMIME MUST NOT be requested by + senders for material with the high bit on that is not in MIME format + with an appropriate content-transfer encoding; servers MAY reject + such messages. + + The metalinguistic notation used in this document corresponds to the + "Augmented BNF" used in other Internet mail system documents. The + reader who is not familiar with that syntax should consult the ABNF + specification [8]. Metalanguage terms used in running text are + surrounded by pointed brackets (e.g., ) for clarity. + +3. The SMTP Procedures: An Overview + + This section contains descriptions of the procedures used in SMTP: + session initiation, the mail transaction, forwarding mail, verifying + mailbox names and expanding mailing lists, and the opening and + closing exchanges. Comments on relaying, a note on mail domains, and + a discussion of changing roles are included at the end of this + section. Several complete scenarios are presented in appendix D. + +3.1 Session Initiation + + An SMTP session is initiated when a client opens a connection to a + server and the server responds with an opening message. + + SMTP server implementations MAY include identification of their + software and version information in the connection greeting reply + after the 220 code, a practice that permits more efficient isolation + and repair of any problems. Implementations MAY make provision for + SMTP servers to disable the software and version announcement where + it causes security concerns. While some systems also identify their + contact point for mail problems, this is not a substitute for + maintaining the required "postmaster" address (see section 4.5.1). + + The SMTP protocol allows a server to formally reject a transaction + while still allowing the initial connection as follows: a 554 + response MAY be given in the initial connection opening message + instead of the 220. A server taking this approach MUST still wait + for the client to send a QUIT (see section 4.1.1.10) before closing + the connection and SHOULD respond to any intervening commands with + + + +Klensin Standards Track [Page 15] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + "503 bad sequence of commands". Since an attempt to make an SMTP + connection to such a system is probably in error, a server returning + a 554 response on connection opening SHOULD provide enough + information in the reply text to facilitate debugging of the sending + system. + +3.2 Client Initiation + + Once the server has sent the welcoming message and the client has + received it, the client normally sends the EHLO command to the + server, indicating the client's identity. In addition to opening the + session, use of EHLO indicates that the client is able to process + service extensions and requests that the server provide a list of the + extensions it supports. Older SMTP systems which are unable to + support service extensions and contemporary clients which do not + require service extensions in the mail session being initiated, MAY + use HELO instead of EHLO. Servers MUST NOT return the extended + EHLO-style response to a HELO command. For a particular connection + attempt, if the server returns a "command not recognized" response to + EHLO, the client SHOULD be able to fall back and send HELO. + + In the EHLO command the host sending the command identifies itself; + the command may be interpreted as saying "Hello, I am " (and, + in the case of EHLO, "and I support service extension requests"). + +3.3 Mail Transactions + + There are three steps to SMTP mail transactions. The transaction + starts with a MAIL command which gives the sender identification. + (In general, the MAIL command may be sent only when no mail + transaction is in progress; see section 4.1.4.) A series of one or + more RCPT commands follows giving the receiver information. Then a + DATA command initiates transfer of the mail data and is terminated by + the "end of mail" data indicator, which also confirms the + transaction. + + The first step in the procedure is the MAIL command. + + MAIL FROM: [SP ] + + This command tells the SMTP-receiver that a new mail transaction is + starting and to reset all its state tables and buffers, including any + recipients or mail data. The portion of the first or + only argument contains the source mailbox (between "<" and ">" + brackets), which can be used to report errors (see section 4.2 for a + discussion of error reporting). If accepted, the SMTP server returns + a 250 OK reply. If the mailbox specification is not acceptable for + some reason, the server MUST return a reply indicating whether the + + + +Klensin Standards Track [Page 16] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + failure is permanent (i.e., will occur again if the client tries to + send the same address again) or temporary (i.e., the address might be + accepted if the client tries again later). Despite the apparent + scope of this requirement, there are circumstances in which the + acceptability of the reverse-path may not be determined until one or + more forward-paths (in RCPT commands) can be examined. In those + cases, the server MAY reasonably accept the reverse-path (with a 250 + reply) and then report problems after the forward-paths are received + and examined. Normally, failures produce 550 or 553 replies. + + Historically, the can contain more than just a + mailbox, however, contemporary systems SHOULD NOT use source routing + (see appendix C). + + The optional are associated with negotiated SMTP + service extensions (see section 2.2). + + The second step in the procedure is the RCPT command. + + RCPT TO: [ SP ] + + The first or only argument to this command includes a forward-path + (normally a mailbox and domain, always surrounded by "<" and ">" + brackets) identifying one recipient. If accepted, the SMTP server + returns a 250 OK reply and stores the forward-path. If the recipient + is known not to be a deliverable address, the SMTP server returns a + 550 reply, typically with a string such as "no such user - " and the + mailbox name (other circumstances and reply codes are possible). + This step of the procedure can be repeated any number of times. + + The can contain more than just a mailbox. + Historically, the can be a source routing list of + hosts and the destination mailbox, however, contemporary SMTP clients + SHOULD NOT utilize source routes (see appendix C). Servers MUST be + prepared to encounter a list of source routes in the forward path, + but SHOULD ignore the routes or MAY decline to support the relaying + they imply. Similarly, servers MAY decline to accept mail that is + destined for other hosts or systems. These restrictions make a + server useless as a relay for clients that do not support full SMTP + functionality. Consequently, restricted-capability clients MUST NOT + assume that any SMTP server on the Internet can be used as their mail + processing (relaying) site. If a RCPT command appears without a + previous MAIL command, the server MUST return a 503 "Bad sequence of + commands" response. The optional are associated + with negotiated SMTP service extensions (see section 2.2). + + The third step in the procedure is the DATA command (or some + alternative specified in a service extension). + + + +Klensin Standards Track [Page 17] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + DATA + + If accepted, the SMTP server returns a 354 Intermediate reply and + considers all succeeding lines up to but not including the end of + mail data indicator to be the message text. When the end of text is + successfully received and stored the SMTP-receiver sends a 250 OK + reply. + + Since the mail data is sent on the transmission channel, the end of + mail data must be indicated so that the command and reply dialog can + be resumed. SMTP indicates the end of the mail data by sending a + line containing only a "." (period or full stop). A transparency + procedure is used to prevent this from interfering with the user's + text (see section 4.5.2). + + The end of mail data indicator also confirms the mail transaction and + tells the SMTP server to now process the stored recipients and mail + data. If accepted, the SMTP server returns a 250 OK reply. The DATA + command can fail at only two points in the protocol exchange: + + - If there was no MAIL, or no RCPT, command, or all such commands + were rejected, the server MAY return a "command out of sequence" + (503) or "no valid recipients" (554) reply in response to the DATA + command. If one of those replies (or any other 5yz reply) is + received, the client MUST NOT send the message data; more + generally, message data MUST NOT be sent unless a 354 reply is + received. + + - If the verb is initially accepted and the 354 reply issued, the + DATA command should fail only if the mail transaction was + incomplete (for example, no recipients), or if resources were + unavailable (including, of course, the server unexpectedly + becoming unavailable), or if the server determines that the + message should be rejected for policy or other reasons. + + However, in practice, some servers do not perform recipient + verification until after the message text is received. These servers + SHOULD treat a failure for one or more recipients as a "subsequent + failure" and return a mail message as discussed in section 6. Using + a "550 mailbox not found" (or equivalent) reply code after the data + are accepted makes it difficult or impossible for the client to + determine which recipients failed. + + When RFC 822 format [7, 32] is being used, the mail data include the + memo header items such as Date, Subject, To, Cc, From. Server SMTP + systems SHOULD NOT reject messages based on perceived defects in the + RFC 822 or MIME [12] message header or message body. In particular, + + + + +Klensin Standards Track [Page 18] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + they MUST NOT reject messages in which the numbers of Resent-fields + do not match or Resent-to appears without Resent-from and/or Resent- + date. + + Mail transaction commands MUST be used in the order discussed above. + +3.4 Forwarding for Address Correction or Updating + + Forwarding support is most often required to consolidate and simplify + addresses within, or relative to, some enterprise and less frequently + to establish addresses to link a person's prior address with current + one. Silent forwarding of messages (without server notification to + the sender), for security or non-disclosure purposes, is common in + the contemporary Internet. + + In both the enterprise and the "new address" cases, information + hiding (and sometimes security) considerations argue against exposure + of the "final" address through the SMTP protocol as a side-effect of + the forwarding activity. This may be especially important when the + final address may not even be reachable by the sender. Consequently, + the "forwarding" mechanisms described in section 3.2 of RFC 821, and + especially the 251 (corrected destination) and 551 reply codes from + RCPT must be evaluated carefully by implementers and, when they are + available, by those configuring systems. + + In particular: + + * Servers MAY forward messages when they are aware of an address + change. When they do so, they MAY either provide address-updating + information with a 251 code, or may forward "silently" and return + a 250 code. But, if a 251 code is used, they MUST NOT assume that + the client will actually update address information or even return + that information to the user. + + Alternately, + + * Servers MAY reject or bounce messages when they are not + deliverable when addressed. When they do so, they MAY either + provide address-updating information with a 551 code, or may + reject the message as undeliverable with a 550 code and no + address-specific information. But, if a 551 code is used, they + MUST NOT assume that the client will actually update address + information or even return that information to the user. + + SMTP server implementations that support the 251 and/or 551 reply + codes are strongly encouraged to provide configuration mechanisms so + that sites which conclude that they would undesirably disclose + information can disable or restrict their use. + + + +Klensin Standards Track [Page 19] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +3.5 Commands for Debugging Addresses + +3.5.1 Overview + + SMTP provides commands to verify a user name or obtain the content of + a mailing list. This is done with the VRFY and EXPN commands, which + have character string arguments. Implementations SHOULD support VRFY + and EXPN (however, see section 3.5.2 and 7.3). + + For the VRFY command, the string is a user name or a user name and + domain (see below). If a normal (i.e., 250) response is returned, + the response MAY include the full name of the user and MUST include + the mailbox of the user. It MUST be in either of the following + forms: + + User Name + local-part@domain + + When a name that is the argument to VRFY could identify more than one + mailbox, the server MAY either note the ambiguity or identify the + alternatives. In other words, any of the following are legitimate + response to VRFY: + + 553 User ambiguous + + or + + 553- Ambiguous; Possibilities are + 553-Joe Smith + 553-Harry Smith + 553 Melvin Smith + + or + + 553-Ambiguous; Possibilities + 553- + 553- + 553 + + Under normal circumstances, a client receiving a 553 reply would be + expected to expose the result to the user. Use of exactly the forms + given, and the "user ambiguous" or "ambiguous" keywords, possibly + supplemented by extended reply codes such as those described in [34], + will facilitate automated translation into other languages as needed. + Of course, a client that was highly automated or that was operating + in another language than English, might choose to try to translate + the response, to return some other indication to the user than the + + + + +Klensin Standards Track [Page 20] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + literal text of the reply, or to take some automated action such as + consulting a directory service for additional information before + reporting to the user. + + For the EXPN command, the string identifies a mailing list, and the + successful (i.e., 250) multiline response MAY include the full name + of the users and MUST give the mailboxes on the mailing list. + + In some hosts the distinction between a mailing list and an alias for + a single mailbox is a bit fuzzy, since a common data structure may + hold both types of entries, and it is possible to have mailing lists + containing only one mailbox. If a request is made to apply VRFY to a + mailing list, a positive response MAY be given if a message so + addressed would be delivered to everyone on the list, otherwise an + error SHOULD be reported (e.g., "550 That is a mailing list, not a + user" or "252 Unable to verify members of mailing list"). If a + request is made to expand a user name, the server MAY return a + positive response consisting of a list containing one name, or an + error MAY be reported (e.g., "550 That is a user name, not a mailing + list"). + + In the case of a successful multiline reply (normal for EXPN) exactly + one mailbox is to be specified on each line of the reply. The case + of an ambiguous request is discussed above. + + "User name" is a fuzzy term and has been used deliberately. An + implementation of the VRFY or EXPN commands MUST include at least + recognition of local mailboxes as "user names". However, since + current Internet practice often results in a single host handling + mail for multiple domains, hosts, especially hosts that provide this + functionality, SHOULD accept the "local-part@domain" form as a "user + name"; hosts MAY also choose to recognize other strings as "user + names". + + The case of expanding a mailbox list requires a multiline reply, such + as: + + C: EXPN Example-People + S: 250-Jon Postel + S: 250-Fred Fonebone + S: 250 Sam Q. Smith + + or + + C: EXPN Executive-Washroom-List + S: 550 Access Denied to You. + + + + + +Klensin Standards Track [Page 21] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + The character string arguments of the VRFY and EXPN commands cannot + be further restricted due to the variety of implementations of the + user name and mailbox list concepts. On some systems it may be + appropriate for the argument of the EXPN command to be a file name + for a file containing a mailing list, but again there are a variety + of file naming conventions in the Internet. Similarly, historical + variations in what is returned by these commands are such that the + response SHOULD be interpreted very carefully, if at all, and SHOULD + generally only be used for diagnostic purposes. + +3.5.2 VRFY Normal Response + + When normal (2yz or 551) responses are returned from a VRFY or EXPN + request, the reply normally includes the mailbox name, i.e., + "", where "domain" is a fully qualified domain + name, MUST appear in the syntax. In circumstances exceptional enough + to justify violating the intent of this specification, free-form text + MAY be returned. In order to facilitate parsing by both computers + and people, addresses SHOULD appear in pointed brackets. When + addresses, rather than free-form debugging information, are returned, + EXPN and VRFY MUST return only valid domain addresses that are usable + in SMTP RCPT commands. Consequently, if an address implies delivery + to a program or other system, the mailbox name used to reach that + target MUST be given. Paths (explicit source routes) MUST NOT be + returned by VRFY or EXPN. + + Server implementations SHOULD support both VRFY and EXPN. For + security reasons, implementations MAY provide local installations a + way to disable either or both of these commands through configuration + options or the equivalent. When these commands are supported, they + are not required to work across relays when relaying is supported. + Since they were both optional in RFC 821, they MUST be listed as + service extensions in an EHLO response, if they are supported. + +3.5.3 Meaning of VRFY or EXPN Success Response + + A server MUST NOT return a 250 code in response to a VRFY or EXPN + command unless it has actually verified the address. In particular, + a server MUST NOT return 250 if all it has done is to verify that the + syntax given is valid. In that case, 502 (Command not implemented) + or 500 (Syntax error, command unrecognized) SHOULD be returned. As + stated elsewhere, implementation (in the sense of actually validating + addresses and returning information) of VRFY and EXPN are strongly + recommended. Hence, implementations that return 500 or 502 for VRFY + are not in full compliance with this specification. + + + + + + +Klensin Standards Track [Page 22] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + There may be circumstances where an address appears to be valid but + cannot reasonably be verified in real time, particularly when a + server is acting as a mail exchanger for another server or domain. + "Apparent validity" in this case would normally involve at least + syntax checking and might involve verification that any domains + specified were ones to which the host expected to be able to relay + mail. In these situations, reply code 252 SHOULD be returned. These + cases parallel the discussion of RCPT verification discussed in + section 2.1. Similarly, the discussion in section 3.4 applies to the + use of reply codes 251 and 551 with VRFY (and EXPN) to indicate + addresses that are recognized but that would be forwarded or bounced + were mail received for them. Implementations generally SHOULD be + more aggressive about address verification in the case of VRFY than + in the case of RCPT, even if it takes a little longer to do so. + +3.5.4 Semantics and Applications of EXPN + + EXPN is often very useful in debugging and understanding problems + with mailing lists and multiple-target-address aliases. Some systems + have attempted to use source expansion of mailing lists as a means of + eliminating duplicates. The propagation of aliasing systems with + mail on the Internet, for hosts (typically with MX and CNAME DNS + records), for mailboxes (various types of local host aliases), and in + various proxying arrangements, has made it nearly impossible for + these strategies to work consistently, and mail systems SHOULD NOT + attempt them. + +3.6 Domains + + Only resolvable, fully-qualified, domain names (FQDNs) are permitted + when domain names are used in SMTP. In other words, names that can + be resolved to MX RRs or A RRs (as discussed in section 5) are + permitted, as are CNAME RRs whose targets can be resolved, in turn, + to MX or A RRs. Local nicknames or unqualified names MUST NOT be + used. There are two exceptions to the rule requiring FQDNs: + + - The domain name given in the EHLO command MUST BE either a primary + host name (a domain name that resolves to an A RR) or, if the host + has no name, an address literal as described in section 4.1.1.1. + + - The reserved mailbox name "postmaster" may be used in a RCPT + command without domain qualification (see section 4.1.1.3) and + MUST be accepted if so used. + + + + + + + + +Klensin Standards Track [Page 23] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +3.7 Relaying + + In general, the availability of Mail eXchanger records in the domain + name system [22, 27] makes the use of explicit source routes in the + Internet mail system unnecessary. Many historical problems with + their interpretation have made their use undesirable. SMTP clients + SHOULD NOT generate explicit source routes except under unusual + circumstances. SMTP servers MAY decline to act as mail relays or to + accept addresses that specify source routes. When route information + is encountered, SMTP servers are also permitted to ignore the route + information and simply send to the final destination specified as the + last element in the route and SHOULD do so. There has been an + invalid practice of using names that do not appear in the DNS as + destination names, with the senders counting on the intermediate + hosts specified in source routing to resolve any problems. If source + routes are stripped, this practice will cause failures. This is one + of several reasons why SMTP clients MUST NOT generate invalid source + routes or depend on serial resolution of names. + + When source routes are not used, the process described in RFC 821 for + constructing a reverse-path from the forward-path is not applicable + and the reverse-path at the time of delivery will simply be the + address that appeared in the MAIL command. + + A relay SMTP server is usually the target of a DNS MX record that + designates it, rather than the final delivery system. The relay + server may accept or reject the task of relaying the mail in the same + way it accepts or rejects mail for a local user. If it accepts the + task, it then becomes an SMTP client, establishes a transmission + channel to the next SMTP server specified in the DNS (according to + the rules in section 5), and sends it the mail. If it declines to + relay mail to a particular address for policy reasons, a 550 response + SHOULD be returned. + + Many mail-sending clients exist, especially in conjunction with + facilities that receive mail via POP3 or IMAP, that have limited + capability to support some of the requirements of this specification, + such as the ability to queue messages for subsequent delivery + attempts. For these clients, it is common practice to make private + arrangements to send all messages to a single server for processing + and subsequent distribution. SMTP, as specified here, is not ideally + suited for this role, and work is underway on standardized mail + submission protocols that might eventually supercede the current + practices. In any event, because these arrangements are private and + fall outside the scope of this specification, they are not described + here. + + + + + +Klensin Standards Track [Page 24] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + It is important to note that MX records can point to SMTP servers + which act as gateways into other environments, not just SMTP relays + and final delivery systems; see sections 3.8 and 5. + + If an SMTP server has accepted the task of relaying the mail and + later finds that the destination is incorrect or that the mail cannot + be delivered for some other reason, then it MUST construct an + "undeliverable mail" notification message and send it to the + originator of the undeliverable mail (as indicated by the reverse- + path). Formats specified for non-delivery reports by other standards + (see, for example, [24, 25]) SHOULD be used if possible. + + This notification message must be from the SMTP server at the relay + host or the host that first determines that delivery cannot be + accomplished. Of course, SMTP servers MUST NOT send notification + messages about problems transporting notification messages. One way + to prevent loops in error reporting is to specify a null reverse-path + in the MAIL command of a notification message. When such a message + is transmitted the reverse-path MUST be set to null (see section + 4.5.5 for additional discussion). A MAIL command with a null + reverse-path appears as follows: + + MAIL FROM:<> + + As discussed in section 2.4.1, a relay SMTP has no need to inspect or + act upon the headers or body of the message data and MUST NOT do so + except to add its own "Received:" header (section 4.4) and, + optionally, to attempt to detect looping in the mail system (see + section 6.2). + +3.8 Mail Gatewaying + + While the relay function discussed above operates within the Internet + SMTP transport service environment, MX records or various forms of + explicit routing may require that an intermediate SMTP server perform + a translation function between one transport service and another. As + discussed in section 2.3.8, when such a system is at the boundary + between two transport service environments, we refer to it as a + "gateway" or "gateway SMTP". + + Gatewaying mail between different mail environments, such as + different mail formats and protocols, is complex and does not easily + yield to standardization. However, some general requirements may be + given for a gateway between the Internet and another mail + environment. + + + + + + +Klensin Standards Track [Page 25] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +3.8.1 Header Fields in Gatewaying + + Header fields MAY be rewritten when necessary as messages are + gatewayed across mail environment boundaries. This may involve + inspecting the message body or interpreting the local-part of the + destination address in spite of the prohibitions in section 2.4.1. + + Other mail systems gatewayed to the Internet often use a subset of + RFC 822 headers or provide similar functionality with a different + syntax, but some of these mail systems do not have an equivalent to + the SMTP envelope. Therefore, when a message leaves the Internet + environment, it may be necessary to fold the SMTP envelope + information into the message header. A possible solution would be to + create new header fields to carry the envelope information (e.g., + "X-SMTP-MAIL:" and "X-SMTP-RCPT:"); however, this would require + changes in mail programs in foreign environments and might risk + disclosure of private information (see section 7.2). + +3.8.2 Received Lines in Gatewaying + + When forwarding a message into or out of the Internet environment, a + gateway MUST prepend a Received: line, but it MUST NOT alter in any + way a Received: line that is already in the header. + + "Received:" fields of messages originating from other environments + may not conform exactly to this specification. However, the most + important use of Received: lines is for debugging mail faults, and + this debugging can be severely hampered by well-meaning gateways that + try to "fix" a Received: line. As another consequence of trace + fields arising in non-SMTP environments, receiving systems MUST NOT + reject mail based on the format of a trace field and SHOULD be + extremely robust in the light of unexpected information or formats in + those fields. + + The gateway SHOULD indicate the environment and protocol in the "via" + clauses of Received field(s) that it supplies. + +3.8.3 Addresses in Gatewaying + + From the Internet side, the gateway SHOULD accept all valid address + formats in SMTP commands and in RFC 822 headers, and all valid RFC + 822 messages. Addresses and headers generated by gateways MUST + conform to applicable Internet standards (including this one and RFC + 822). Gateways are, of course, subject to the same rules for + handling source routes as those described for other SMTP systems in + section 3.3. + + + + + +Klensin Standards Track [Page 26] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +3.8.4 Other Header Fields in Gatewaying + + The gateway MUST ensure that all header fields of a message that it + forwards into the Internet mail environment meet the requirements for + Internet mail. In particular, all addresses in "From:", "To:", + "Cc:", etc., fields MUST be transformed (if necessary) to satisfy RFC + 822 syntax, MUST reference only fully-qualified domain names, and + MUST be effective and useful for sending replies. The translation + algorithm used to convert mail from the Internet protocols to another + environment's protocol SHOULD ensure that error messages from the + foreign mail environment are delivered to the return path from the + SMTP envelope, not to the sender listed in the "From:" field (or + other fields) of the RFC 822 message. + +3.8.5 Envelopes in Gatewaying + + Similarly, when forwarding a message from another environment into + the Internet, the gateway SHOULD set the envelope return path in + accordance with an error message return address, if supplied by the + foreign environment. If the foreign environment has no equivalent + concept, the gateway must select and use a best approximation, with + the message originator's address as the default of last resort. + +3.9 Terminating Sessions and Connections + + An SMTP connection is terminated when the client sends a QUIT + command. The server responds with a positive reply code, after which + it closes the connection. + + An SMTP server MUST NOT intentionally close the connection except: + + - After receiving a QUIT command and responding with a 221 reply. + + - After detecting the need to shut down the SMTP service and + returning a 421 response code. This response code can be issued + after the server receives any command or, if necessary, + asynchronously from command receipt (on the assumption that the + client will receive it after the next command is issued). + + In particular, a server that closes connections in response to + commands that are not understood is in violation of this + specification. Servers are expected to be tolerant of unknown + commands, issuing a 500 reply and awaiting further instructions from + the client. + + + + + + + +Klensin Standards Track [Page 27] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + An SMTP server which is forcibly shut down via external means SHOULD + attempt to send a line containing a 421 response code to the SMTP + client before exiting. The SMTP client will normally read the 421 + response code after sending its next command. + + SMTP clients that experience a connection close, reset, or other + communications failure due to circumstances not under their control + (in violation of the intent of this specification but sometimes + unavoidable) SHOULD, to maintain the robustness of the mail system, + treat the mail transaction as if a 451 response had been received and + act accordingly. + +3.10 Mailing Lists and Aliases + + An SMTP-capable host SHOULD support both the alias and the list + models of address expansion for multiple delivery. When a message is + delivered or forwarded to each address of an expanded list form, the + return address in the envelope ("MAIL FROM:") MUST be changed to be + the address of a person or other entity who administers the list. + However, in this case, the message header [32] MUST be left + unchanged; in particular, the "From" field of the message header is + unaffected. + + An important mail facility is a mechanism for multi-destination + delivery of a single message, by transforming (or "expanding" or + "exploding") a pseudo-mailbox address into a list of destination + mailbox addresses. When a message is sent to such a pseudo-mailbox + (sometimes called an "exploder"), copies are forwarded or + redistributed to each mailbox in the expanded list. Servers SHOULD + simply utilize the addresses on the list; application of heuristics + or other matching rules to eliminate some addresses, such as that of + the originator, is strongly discouraged. We classify such a pseudo- + mailbox as an "alias" or a "list", depending upon the expansion + rules. + +3.10.1 Alias + + To expand an alias, the recipient mailer simply replaces the pseudo- + mailbox address in the envelope with each of the expanded addresses + in turn; the rest of the envelope and the message body are left + unchanged. The message is then delivered or forwarded to each + expanded address. + +3.10.2 List + + A mailing list may be said to operate by "redistribution" rather than + by "forwarding". To expand a list, the recipient mailer replaces the + pseudo-mailbox address in the envelope with all of the expanded + + + +Klensin Standards Track [Page 28] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + addresses. The return address in the envelope is changed so that all + error messages generated by the final deliveries will be returned to + a list administrator, not to the message originator, who generally + has no control over the contents of the list and will typically find + error messages annoying. + +4. The SMTP Specifications + +4.1 SMTP Commands + +4.1.1 Command Semantics and Syntax + + The SMTP commands define the mail transfer or the mail system + function requested by the user. SMTP commands are character strings + terminated by . The commands themselves are alphabetic + characters terminated by if parameters follow and + otherwise. (In the interest of improved interoperability, SMTP + receivers are encouraged to tolerate trailing white space before the + terminating .) The syntax of the local part of a mailbox must + conform to receiver site conventions and the syntax specified in + section 4.1.2. The SMTP commands are discussed below. The SMTP + replies are discussed in section 4.2. + + A mail transaction involves several data objects which are + communicated as arguments to different commands. The reverse-path is + the argument of the MAIL command, the forward-path is the argument of + the RCPT command, and the mail data is the argument of the DATA + command. These arguments or data objects must be transmitted and + held pending the confirmation communicated by the end of mail data + indication which finalizes the transaction. The model for this is + that distinct buffers are provided to hold the types of data objects, + that is, there is a reverse-path buffer, a forward-path buffer, and a + mail data buffer. Specific commands cause information to be appended + to a specific buffer, or cause one or more buffers to be cleared. + + Several commands (RSET, DATA, QUIT) are specified as not permitting + parameters. In the absence of specific extensions offered by the + server and accepted by the client, clients MUST NOT send such + parameters and servers SHOULD reject commands containing them as + having invalid syntax. + +4.1.1.1 Extended HELLO (EHLO) or HELLO (HELO) + + These commands are used to identify the SMTP client to the SMTP + server. The argument field contains the fully-qualified domain name + of the SMTP client if one is available. In situations in which the + SMTP client system does not have a meaningful domain name (e.g., when + its address is dynamically allocated and no reverse mapping record is + + + +Klensin Standards Track [Page 29] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + available), the client SHOULD send an address literal (see section + 4.1.3), optionally followed by information that will help to identify + the client system. y The SMTP server identifies itself to the SMTP + client in the connection greeting reply and in the response to this + command. + + A client SMTP SHOULD start an SMTP session by issuing the EHLO + command. If the SMTP server supports the SMTP service extensions it + will give a successful response, a failure response, or an error + response. If the SMTP server, in violation of this specification, + does not support any SMTP service extensions it will generate an + error response. Older client SMTP systems MAY, as discussed above, + use HELO (as specified in RFC 821) instead of EHLO, and servers MUST + support the HELO command and reply properly to it. In any event, a + client MUST issue HELO or EHLO before starting a mail transaction. + + These commands, and a "250 OK" reply to one of them, confirm that + both the SMTP client and the SMTP server are in the initial state, + that is, there is no transaction in progress and all state tables and + buffers are cleared. + + Syntax: + + ehlo = "EHLO" SP Domain CRLF + helo = "HELO" SP Domain CRLF + + Normally, the response to EHLO will be a multiline reply. Each line + of the response contains a keyword and, optionally, one or more + parameters. Following the normal syntax for multiline replies, these + keyworks follow the code (250) and a hyphen for all but the last + line, and the code and a space for the last line. The syntax for a + positive response, using the ABNF notation and terminal symbols of + [8], is: + + ehlo-ok-rsp = ( "250" domain [ SP ehlo-greet ] CRLF ) + / ( "250-" domain [ SP ehlo-greet ] CRLF + *( "250-" ehlo-line CRLF ) + "250" SP ehlo-line CRLF ) + + ehlo-greet = 1*(%d0-9 / %d11-12 / %d14-127) + ; string of any characters other than CR or LF + + ehlo-line = ehlo-keyword *( SP ehlo-param ) + + ehlo-keyword = (ALPHA / DIGIT) *(ALPHA / DIGIT / "-") + ; additional syntax of ehlo-params depends on + ; ehlo-keyword + + + + +Klensin Standards Track [Page 30] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + ehlo-param = 1*(%d33-127) + ; any CHAR excluding and all + ; control characters (US-ASCII 0-31 inclusive) + + Although EHLO keywords may be specified in upper, lower, or mixed + case, they MUST always be recognized and processed in a case- + insensitive manner. This is simply an extension of practices + specified in RFC 821 and section 2.4.1. + +4.1.1.2 MAIL (MAIL) + + This command is used to initiate a mail transaction in which the mail + data is delivered to an SMTP server which may, in turn, deliver it to + one or more mailboxes or pass it on to another system (possibly using + SMTP). The argument field contains a reverse-path and may contain + optional parameters. In general, the MAIL command may be sent only + when no mail transaction is in progress, see section 4.1.4. + + The reverse-path consists of the sender mailbox. Historically, that + mailbox might optionally have been preceded by a list of hosts, but + that behavior is now deprecated (see appendix C). In some types of + reporting messages for which a reply is likely to cause a mail loop + (for example, mail delivery and nondelivery notifications), the + reverse-path may be null (see section 3.7). + + This command clears the reverse-path buffer, the forward-path buffer, + and the mail data buffer; and inserts the reverse-path information + from this command into the reverse-path buffer. + + If service extensions were negotiated, the MAIL command may also + carry parameters associated with a particular service extension. + + Syntax: + + "MAIL FROM:" ("<>" / Reverse-Path) + [SP Mail-parameters] CRLF + +4.1.1.3 RECIPIENT (RCPT) + + This command is used to identify an individual recipient of the mail + data; multiple recipients are specified by multiple use of this + command. The argument field contains a forward-path and may contain + optional parameters. + + The forward-path normally consists of the required destination + mailbox. Sending systems SHOULD not generate the optional list of + hosts known as a source route. Receiving systems MUST recognize + + + + +Klensin Standards Track [Page 31] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + source route syntax but SHOULD strip off the source route + specification and utilize the domain name associated with the mailbox + as if the source route had not been provided. + + Similarly, relay hosts SHOULD strip or ignore source routes, and + names MUST NOT be copied into the reverse-path. When mail reaches + its ultimate destination (the forward-path contains only a + destination mailbox), the SMTP server inserts it into the destination + mailbox in accordance with its host mail conventions. + + For example, mail received at relay host xyz.com with envelope + commands + + MAIL FROM: + RCPT TO:<@hosta.int,@jkl.org:userc@d.bar.org> + + will normally be sent directly on to host d.bar.org with envelope + commands + + MAIL FROM: + RCPT TO: + + As provided in appendix C, xyz.com MAY also choose to relay the + message to hosta.int, using the envelope commands + + MAIL FROM: + RCPT TO:<@hosta.int,@jkl.org:userc@d.bar.org> + + or to jkl.org, using the envelope commands + + MAIL FROM: + RCPT TO:<@jkl.org:userc@d.bar.org> + + Of course, since hosts are not required to relay mail at all, xyz.com + may also reject the message entirely when the RCPT command is + received, using a 550 code (since this is a "policy reason"). + + If service extensions were negotiated, the RCPT command may also + carry parameters associated with a particular service extension + offered by the server. The client MUST NOT transmit parameters other + than those associated with a service extension offered by the server + in its EHLO response. + +Syntax: + "RCPT TO:" ("" / "" / Forward-Path) + [SP Rcpt-parameters] CRLF + + + + + +Klensin Standards Track [Page 32] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +4.1.1.4 DATA (DATA) + + The receiver normally sends a 354 response to DATA, and then treats + the lines (strings ending in sequences, as described in + section 2.3.7) following the command as mail data from the sender. + This command causes the mail data to be appended to the mail data + buffer. The mail data may contain any of the 128 ASCII character + codes, although experience has indicated that use of control + characters other than SP, HT, CR, and LF may cause problems and + SHOULD be avoided when possible. + + The mail data is terminated by a line containing only a period, that + is, the character sequence "." (see section 4.5.2). This + is the end of mail data indication. Note that the first of + this terminating sequence is also the that ends the final line + of the data (message text) or, if there was no data, ends the DATA + command itself. An extra MUST NOT be added, as that would + cause an empty line to be added to the message. The only exception + to this rule would arise if the message body were passed to the + originating SMTP-sender with a final "line" that did not end in + ; in that case, the originating SMTP system MUST either reject + the message as invalid or add in order to have the receiving + SMTP server recognize the "end of data" condition. + + The custom of accepting lines ending only in , as a concession to + non-conforming behavior on the part of some UNIX systems, has proven + to cause more interoperability problems than it solves, and SMTP + server systems MUST NOT do this, even in the name of improved + robustness. In particular, the sequence "." (bare line + feeds, without carriage returns) MUST NOT be treated as equivalent to + . as the end of mail data indication. + + Receipt of the end of mail data indication requires the server to + process the stored mail transaction information. This processing + consumes the information in the reverse-path buffer, the forward-path + buffer, and the mail data buffer, and on the completion of this + command these buffers are cleared. If the processing is successful, + the receiver MUST send an OK reply. If the processing fails the + receiver MUST send a failure reply. The SMTP model does not allow + for partial failures at this point: either the message is accepted by + the server for delivery and a positive response is returned or it is + not accepted and a failure reply is returned. In sending a positive + completion reply to the end of data indication, the receiver takes + full responsibility for the message (see section 6.1). Errors that + are diagnosed subsequently MUST be reported in a mail message, as + discussed in section 4.4. + + + + + +Klensin Standards Track [Page 33] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + When the SMTP server accepts a message either for relaying or for + final delivery, it inserts a trace record (also referred to + interchangeably as a "time stamp line" or "Received" line) at the top + of the mail data. This trace record indicates the identity of the + host that sent the message, the identity of the host that received + the message (and is inserting this time stamp), and the date and time + the message was received. Relayed messages will have multiple time + stamp lines. Details for formation of these lines, including their + syntax, is specified in section 4.4. + + Additional discussion about the operation of the DATA command appears + in section 3.3. + + Syntax: + "DATA" CRLF + +4.1.1.5 RESET (RSET) + + This command specifies that the current mail transaction will be + aborted. Any stored sender, recipients, and mail data MUST be + discarded, and all buffers and state tables cleared. The receiver + MUST send a "250 OK" reply to a RSET command with no arguments. A + reset command may be issued by the client at any time. It is + effectively equivalent to a NOOP (i.e., if has no effect) if issued + immediately after EHLO, before EHLO is issued in the session, after + an end-of-data indicator has been sent and acknowledged, or + immediately before a QUIT. An SMTP server MUST NOT close the + connection as the result of receiving a RSET; that action is reserved + for QUIT (see section 4.1.1.10). + + Since EHLO implies some additional processing and response by the + server, RSET will normally be more efficient than reissuing that + command, even though the formal semantics are the same. + + There are circumstances, contrary to the intent of this + specification, in which an SMTP server may receive an indication that + the underlying TCP connection has been closed or reset. To preserve + the robustness of the mail system, SMTP servers SHOULD be prepared + for this condition and SHOULD treat it as if a QUIT had been received + before the connection disappeared. + + Syntax: + "RSET" CRLF + + + + + + + + +Klensin Standards Track [Page 34] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +4.1.1.6 VERIFY (VRFY) + + This command asks the receiver to confirm that the argument + identifies a user or mailbox. If it is a user name, information is + returned as specified in section 3.5. + + This command has no effect on the reverse-path buffer, the forward- + path buffer, or the mail data buffer. + + Syntax: + "VRFY" SP String CRLF + +4.1.1.7 EXPAND (EXPN) + + This command asks the receiver to confirm that the argument + identifies a mailing list, and if so, to return the membership of + that list. If the command is successful, a reply is returned + containing information as described in section 3.5. This reply will + have multiple lines except in the trivial case of a one-member list. + + This command has no effect on the reverse-path buffer, the forward- + path buffer, or the mail data buffer and may be issued at any time. + + Syntax: + "EXPN" SP String CRLF + +4.1.1.8 HELP (HELP) + + This command causes the server to send helpful information to the + client. The command MAY take an argument (e.g., any command name) + and return more specific information as a response. + + This command has no effect on the reverse-path buffer, the forward- + path buffer, or the mail data buffer and may be issued at any time. + + SMTP servers SHOULD support HELP without arguments and MAY support it + with arguments. + + Syntax: + "HELP" [ SP String ] CRLF + +4.1.1.9 NOOP (NOOP) + + This command does not affect any parameters or previously entered + commands. It specifies no action other than that the receiver send + an OK reply. + + + + + +Klensin Standards Track [Page 35] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + This command has no effect on the reverse-path buffer, the forward- + path buffer, or the mail data buffer and may be issued at any time. + If a parameter string is specified, servers SHOULD ignore it. + + Syntax: + "NOOP" [ SP String ] CRLF + +4.1.1.10 QUIT (QUIT) + + This command specifies that the receiver MUST send an OK reply, and + then close the transmission channel. + + The receiver MUST NOT intentionally close the transmission channel + until it receives and replies to a QUIT command (even if there was an + error). The sender MUST NOT intentionally close the transmission + channel until it sends a QUIT command and SHOULD wait until it + receives the reply (even if there was an error response to a previous + command). If the connection is closed prematurely due to violations + of the above or system or network failure, the server MUST cancel any + pending transaction, but not undo any previously completed + transaction, and generally MUST act as if the command or transaction + in progress had received a temporary error (i.e., a 4yz response). + + The QUIT command may be issued at any time. + + Syntax: + "QUIT" CRLF + +4.1.2 Command Argument Syntax + + The syntax of the argument fields of the above commands (using the + syntax specified in [8] where applicable) is given below. Some of + the productions given below are used only in conjunction with source + routes as described in appendix C. Terminals not defined in this + document, such as ALPHA, DIGIT, SP, CR, LF, CRLF, are as defined in + the "core" syntax [8 (section 6)] or in the message format syntax + [32]. + + Reverse-path = Path + Forward-path = Path + Path = "<" [ A-d-l ":" ] Mailbox ">" + A-d-l = At-domain *( "," A-d-l ) + ; Note that this form, the so-called "source route", + ; MUST BE accepted, SHOULD NOT be generated, and SHOULD be + ; ignored. + At-domain = "@" domain + Mail-parameters = esmtp-param *(SP esmtp-param) + Rcpt-parameters = esmtp-param *(SP esmtp-param) + + + +Klensin Standards Track [Page 36] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + esmtp-param = esmtp-keyword ["=" esmtp-value] + esmtp-keyword = (ALPHA / DIGIT) *(ALPHA / DIGIT / "-") + esmtp-value = 1*(%d33-60 / %d62-127) + ; any CHAR excluding "=", SP, and control characters + Keyword = Ldh-str + Argument = Atom + Domain = (sub-domain 1*("." sub-domain)) / address-literal + sub-domain = Let-dig [Ldh-str] + + address-literal = "[" IPv4-address-literal / + IPv6-address-literal / + General-address-literal "]" + ; See section 4.1.3 + + Mailbox = Local-part "@" Domain + + Local-part = Dot-string / Quoted-string + ; MAY be case-sensitive + + Dot-string = Atom *("." Atom) + + Atom = 1*atext + + Quoted-string = DQUOTE *qcontent DQUOTE + + String = Atom / Quoted-string + + While the above definition for Local-part is relatively permissive, + for maximum interoperability, a host that expects to receive mail + SHOULD avoid defining mailboxes where the Local-part requires (or + uses) the Quoted-string form or where the Local-part is case- + sensitive. For any purposes that require generating or comparing + Local-parts (e.g., to specific mailbox names), all quoted forms MUST + be treated as equivalent and the sending system SHOULD transmit the + form that uses the minimum quoting possible. + + Systems MUST NOT define mailboxes in such a way as to require the use + in SMTP of non-ASCII characters (octets with the high order bit set + to one) or ASCII "control characters" (decimal value 0-31 and 127). + These characters MUST NOT be used in MAIL or RCPT commands or other + commands that require mailbox names. + + Note that the backslash, "\", is a quote character, which is used to + indicate that the next character is to be used literally (instead of + its normal interpretation). For example, "Joe\,Smith" indicates a + single nine character user field with the comma being the fourth + character of the field. + + + + +Klensin Standards Track [Page 37] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + To promote interoperability and consistent with long-standing + guidance about conservative use of the DNS in naming and applications + (e.g., see section 2.3.1 of the base DNS document, RFC1035 [22]), + characters outside the set of alphas, digits, and hyphen MUST NOT + appear in domain name labels for SMTP clients or servers. In + particular, the underscore character is not permitted. SMTP servers + that receive a command in which invalid character codes have been + employed, and for which there are no other reasons for rejection, + MUST reject that command with a 501 response. + +4.1.3 Address Literals + + Sometimes a host is not known to the domain name system and + communication (and, in particular, communication to report and repair + the error) is blocked. To bypass this barrier a special literal form + of the address is allowed as an alternative to a domain name. For + IPv4 addresses, this form uses four small decimal integers separated + by dots and enclosed by brackets such as [123.255.37.2], which + indicates an (IPv4) Internet Address in sequence-of-octets form. For + IPv6 and other forms of addressing that might eventually be + standardized, the form consists of a standardized "tag" that + identifies the address syntax, a colon, and the address itself, in a + format specified as part of the IPv6 standards [17]. + + Specifically: + + IPv4-address-literal = Snum 3("." Snum) + IPv6-address-literal = "IPv6:" IPv6-addr + General-address-literal = Standardized-tag ":" 1*dcontent + Standardized-tag = Ldh-str + ; MUST be specified in a standards-track RFC + ; and registered with IANA + + Snum = 1*3DIGIT ; representing a decimal integer + ; value in the range 0 through 255 + Let-dig = ALPHA / DIGIT + Ldh-str = *( ALPHA / DIGIT / "-" ) Let-dig + + IPv6-addr = IPv6-full / IPv6-comp / IPv6v4-full / IPv6v4-comp + IPv6-hex = 1*4HEXDIG + IPv6-full = IPv6-hex 7(":" IPv6-hex) + IPv6-comp = [IPv6-hex *5(":" IPv6-hex)] "::" [IPv6-hex *5(":" + IPv6-hex)] + ; The "::" represents at least 2 16-bit groups of zeros + ; No more than 6 groups in addition to the "::" may be + ; present + IPv6v4-full = IPv6-hex 5(":" IPv6-hex) ":" IPv4-address-literal + IPv6v4-comp = [IPv6-hex *3(":" IPv6-hex)] "::" + + + +Klensin Standards Track [Page 38] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + [IPv6-hex *3(":" IPv6-hex) ":"] IPv4-address-literal + ; The "::" represents at least 2 16-bit groups of zeros + ; No more than 4 groups in addition to the "::" and + ; IPv4-address-literal may be present + +4.1.4 Order of Commands + + There are restrictions on the order in which these commands may be + used. + + A session that will contain mail transactions MUST first be + initialized by the use of the EHLO command. An SMTP server SHOULD + accept commands for non-mail transactions (e.g., VRFY or EXPN) + without this initialization. + + An EHLO command MAY be issued by a client later in the session. If + it is issued after the session begins, the SMTP server MUST clear all + buffers and reset the state exactly as if a RSET command had been + issued. In other words, the sequence of RSET followed immediately by + EHLO is redundant, but not harmful other than in the performance cost + of executing unnecessary commands. + + If the EHLO command is not acceptable to the SMTP server, 501, 500, + or 502 failure replies MUST be returned as appropriate. The SMTP + server MUST stay in the same state after transmitting these replies + that it was in before the EHLO was received. + + The SMTP client MUST, if possible, ensure that the domain parameter + to the EHLO command is a valid principal host name (not a CNAME or MX + name) for its host. If this is not possible (e.g., when the client's + address is dynamically assigned and the client does not have an + obvious name), an address literal SHOULD be substituted for the + domain name and supplemental information provided that will assist in + identifying the client. + + An SMTP server MAY verify that the domain name parameter in the EHLO + command actually corresponds to the IP address of the client. + However, the server MUST NOT refuse to accept a message for this + reason if the verification fails: the information about verification + failure is for logging and tracing only. + + The NOOP, HELP, EXPN, VRFY, and RSET commands can be used at any time + during a session, or without previously initializing a session. SMTP + servers SHOULD process these normally (that is, not return a 503 + code) even if no EHLO command has yet been received; clients SHOULD + open a session with EHLO before sending these commands. + + + + + +Klensin Standards Track [Page 39] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + If these rules are followed, the example in RFC 821 that shows "550 + access denied to you" in response to an EXPN command is incorrect + unless an EHLO command precedes the EXPN or the denial of access is + based on the client's IP address or other authentication or + authorization-determining mechanisms. + + The MAIL command (or the obsolete SEND, SOML, or SAML commands) + begins a mail transaction. Once started, a mail transaction consists + of a transaction beginning command, one or more RCPT commands, and a + DATA command, in that order. A mail transaction may be aborted by + the RSET (or a new EHLO) command. There may be zero or more + transactions in a session. MAIL (or SEND, SOML, or SAML) MUST NOT be + sent if a mail transaction is already open, i.e., it should be sent + only if no mail transaction had been started in the session, or it + the previous one successfully concluded with a successful DATA + command, or if the previous one was aborted with a RSET. + + If the transaction beginning command argument is not acceptable, a + 501 failure reply MUST be returned and the SMTP server MUST stay in + the same state. If the commands in a transaction are out of order to + the degree that they cannot be processed by the server, a 503 failure + reply MUST be returned and the SMTP server MUST stay in the same + state. + + The last command in a session MUST be the QUIT command. The QUIT + command cannot be used at any other time in a session, but SHOULD be + used by the client SMTP to request connection closure, even when no + session opening command was sent and accepted. + +4.1.5 Private-use Commands + + As specified in section 2.2.2, commands starting in "X" may be used + by bilateral agreement between the client (sending) and server + (receiving) SMTP agents. An SMTP server that does not recognize such + a command is expected to reply with "500 Command not recognized". An + extended SMTP server MAY list the feature names associated with these + private commands in the response to the EHLO command. + + Commands sent or accepted by SMTP systems that do not start with "X" + MUST conform to the requirements of section 2.2.2. + +4.2 SMTP Replies + + Replies to SMTP commands serve to ensure the synchronization of + requests and actions in the process of mail transfer and to guarantee + that the SMTP client always knows the state of the SMTP server. + Every command MUST generate exactly one reply. + + + + +Klensin Standards Track [Page 40] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + The details of the command-reply sequence are described in section + 4.3. + + An SMTP reply consists of a three digit number (transmitted as three + numeric characters) followed by some text unless specified otherwise + in this document. The number is for use by automata to determine + what state to enter next; the text is for the human user. The three + digits contain enough encoded information that the SMTP client need + not examine the text and may either discard it or pass it on to the + user, as appropriate. Exceptions are as noted elsewhere in this + document. In particular, the 220, 221, 251, 421, and 551 reply codes + are associated with message text that must be parsed and interpreted + by machines. In the general case, the text may be receiver dependent + and context dependent, so there are likely to be varying texts for + each reply code. A discussion of the theory of reply codes is given + in section 4.2.1. Formally, a reply is defined to be the sequence: a + three-digit code, , one line of text, and , or a multiline + reply (as defined in section 4.2.1). Since, in violation of this + specification, the text is sometimes not sent, clients which do not + receive it SHOULD be prepared to process the code alone (with or + without a trailing space character). Only the EHLO, EXPN, and HELP + commands are expected to result in multiline replies in normal + circumstances, however, multiline replies are allowed for any + command. + + In ABNF, server responses are: + + Greeting = "220 " Domain [ SP text ] CRLF + Reply-line = Reply-code [ SP text ] CRLF + + where "Greeting" appears only in the 220 response that announces that + the server is opening its part of the connection. + + An SMTP server SHOULD send only the reply codes listed in this + document. An SMTP server SHOULD use the text shown in the examples + whenever appropriate. + + An SMTP client MUST determine its actions only by the reply code, not + by the text (except for the "change of address" 251 and 551 and, if + necessary, 220, 221, and 421 replies); in the general case, any text, + including no text at all (although senders SHOULD NOT send bare + codes), MUST be acceptable. The space (blank) following the reply + code is considered part of the text. Whenever possible, a receiver- + SMTP SHOULD test the first digit (severity indication) of the reply + code. + + + + + + +Klensin Standards Track [Page 41] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + The list of codes that appears below MUST NOT be construed as + permanent. While the addition of new codes should be a rare and + significant activity, with supplemental information in the textual + part of the response being preferred, new codes may be added as the + result of new Standards or Standards-track specifications. + Consequently, a sender-SMTP MUST be prepared to handle codes not + specified in this document and MUST do so by interpreting the first + digit only. + +4.2.1 Reply Code Severities and Theory + + The three digits of the reply each have a special significance. The + first digit denotes whether the response is good, bad or incomplete. + An unsophisticated SMTP client, or one that receives an unexpected + code, will be able to determine its next action (proceed as planned, + redo, retrench, etc.) by examining this first digit. An SMTP client + that wants to know approximately what kind of error occurred (e.g., + mail system error, command syntax error) may examine the second + digit. The third digit and any supplemental information that may be + present is reserved for the finest gradation of information. + + There are five values for the first digit of the reply code: + + 1yz Positive Preliminary reply + The command has been accepted, but the requested action is being + held in abeyance, pending confirmation of the information in this + reply. The SMTP client should send another command specifying + whether to continue or abort the action. Note: unextended SMTP + does not have any commands that allow this type of reply, and so + does not have continue or abort commands. + + 2yz Positive Completion reply + The requested action has been successfully completed. A new + request may be initiated. + + 3yz Positive Intermediate reply + The command has been accepted, but the requested action is being + held in abeyance, pending receipt of further information. The + SMTP client should send another command specifying this + information. This reply is used in command sequence groups (i.e., + in DATA). + + 4yz Transient Negative Completion reply + The command was not accepted, and the requested action did not + occur. However, the error condition is temporary and the action + may be requested again. The sender should return to the beginning + of the command sequence (if any). It is difficult to assign a + meaning to "transient" when two different sites (receiver- and + + + +Klensin Standards Track [Page 42] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + sender-SMTP agents) must agree on the interpretation. Each reply + in this category might have a different time value, but the SMTP + client is encouraged to try again. A rule of thumb to determine + whether a reply fits into the 4yz or the 5yz category (see below) + is that replies are 4yz if they can be successful if repeated + without any change in command form or in properties of the sender + or receiver (that is, the command is repeated identically and the + receiver does not put up a new implementation.) + + 5yz Permanent Negative Completion reply + The command was not accepted and the requested action did not + occur. The SMTP client is discouraged from repeating the exact + request (in the same sequence). Even some "permanent" error + conditions can be corrected, so the human user may want to direct + the SMTP client to reinitiate the command sequence by direct + action at some point in the future (e.g., after the spelling has + been changed, or the user has altered the account status). + + The second digit encodes responses in specific categories: + + x0z Syntax: These replies refer to syntax errors, syntactically + correct commands that do not fit any functional category, and + unimplemented or superfluous commands. + + x1z Information: These are replies to requests for information, + such as status or help. + + x2z Connections: These are replies referring to the transmission + channel. + + x3z Unspecified. + + x4z Unspecified. + + x5z Mail system: These replies indicate the status of the receiver + mail system vis-a-vis the requested transfer or other mail system + action. + + The third digit gives a finer gradation of meaning in each category + specified by the second digit. The list of replies illustrates this. + Each reply text is recommended rather than mandatory, and may even + change according to the command with which it is associated. On the + other hand, the reply codes must strictly follow the specifications + in this section. Receiver implementations should not invent new + codes for slightly different situations from the ones described here, + but rather adapt codes already defined. + + + + + +Klensin Standards Track [Page 43] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + For example, a command such as NOOP, whose successful execution does + not offer the SMTP client any new information, will return a 250 + reply. The reply is 502 when the command requests an unimplemented + non-site-specific action. A refinement of that is the 504 reply for + a command that is implemented, but that requests an unimplemented + parameter. + + The reply text may be longer than a single line; in these cases the + complete text must be marked so the SMTP client knows when it can + stop reading the reply. This requires a special format to indicate a + multiple line reply. + + The format for multiline replies requires that every line, except the + last, begin with the reply code, followed immediately by a hyphen, + "-" (also known as minus), followed by text. The last line will + begin with the reply code, followed immediately by , optionally + some text, and . As noted above, servers SHOULD send the + if subsequent text is not sent, but clients MUST be prepared for it + to be omitted. + + For example: + + 123-First line + 123-Second line + 123-234 text beginning with numbers + 123 The last line + + In many cases the SMTP client then simply needs to search for a line + beginning with the reply code followed by or and ignore + all preceding lines. In a few cases, there is important data for the + client in the reply "text". The client will be able to identify + these cases from the current context. + +4.2.2 Reply Codes by Function Groups + + 500 Syntax error, command unrecognized + (This may include errors such as command line too long) + 501 Syntax error in parameters or arguments + 502 Command not implemented (see section 4.2.4) + 503 Bad sequence of commands + 504 Command parameter not implemented + + 211 System status, or system help reply + 214 Help message + (Information on how to use the receiver or the meaning of a + particular non-standard command; this reply is useful only + to the human user) + + + + +Klensin Standards Track [Page 44] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + 220 Service ready + 221 Service closing transmission channel + 421 Service not available, closing transmission channel + (This may be a reply to any command if the service knows it + must shut down) + + 250 Requested mail action okay, completed + 251 User not local; will forward to + (See section 3.4) + 252 Cannot VRFY user, but will accept message and attempt + delivery + (See section 3.5.3) + 450 Requested mail action not taken: mailbox unavailable + (e.g., mailbox busy) + 550 Requested action not taken: mailbox unavailable + (e.g., mailbox not found, no access, or command rejected + for policy reasons) + 451 Requested action aborted: error in processing + 551 User not local; please try + (See section 3.4) + 452 Requested action not taken: insufficient system storage + 552 Requested mail action aborted: exceeded storage allocation + 553 Requested action not taken: mailbox name not allowed + (e.g., mailbox syntax incorrect) + 354 Start mail input; end with . + 554 Transaction failed (Or, in the case of a connection-opening + response, "No SMTP service here") + +4.2.3 Reply Codes in Numeric Order + + 211 System status, or system help reply + 214 Help message + (Information on how to use the receiver or the meaning of a + particular non-standard command; this reply is useful only + to the human user) + 220 Service ready + 221 Service closing transmission channel + 250 Requested mail action okay, completed + 251 User not local; will forward to + (See section 3.4) + 252 Cannot VRFY user, but will accept message and attempt + delivery + (See section 3.5.3) + + 354 Start mail input; end with . + + + + + + +Klensin Standards Track [Page 45] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + 421 Service not available, closing transmission channel + (This may be a reply to any command if the service knows it + must shut down) + 450 Requested mail action not taken: mailbox unavailable + (e.g., mailbox busy) + 451 Requested action aborted: local error in processing + 452 Requested action not taken: insufficient system storage + 500 Syntax error, command unrecognized + (This may include errors such as command line too long) + 501 Syntax error in parameters or arguments + 502 Command not implemented (see section 4.2.4) + 503 Bad sequence of commands + 504 Command parameter not implemented + 550 Requested action not taken: mailbox unavailable + (e.g., mailbox not found, no access, or command rejected + for policy reasons) + 551 User not local; please try + (See section 3.4) + 552 Requested mail action aborted: exceeded storage allocation + 553 Requested action not taken: mailbox name not allowed + (e.g., mailbox syntax incorrect) + 554 Transaction failed (Or, in the case of a connection-opening + response, "No SMTP service here") + +4.2.4 Reply Code 502 + + Questions have been raised as to when reply code 502 (Command not + implemented) SHOULD be returned in preference to other codes. 502 + SHOULD be used when the command is actually recognized by the SMTP + server, but not implemented. If the command is not recognized, code + 500 SHOULD be returned. Extended SMTP systems MUST NOT list + capabilities in response to EHLO for which they will return 502 (or + 500) replies. + +4.2.5 Reply Codes After DATA and the Subsequent . + + When an SMTP server returns a positive completion status (2yz code) + after the DATA command is completed with ., it accepts + responsibility for: + + - delivering the message (if the recipient mailbox exists), or + + - if attempts to deliver the message fail due to transient + conditions, retrying delivery some reasonable number of times at + intervals as specified in section 4.5.4. + + + + + + +Klensin Standards Track [Page 46] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + - if attempts to deliver the message fail due to permanent + conditions, or if repeated attempts to deliver the message fail + due to transient conditions, returning appropriate notification to + the sender of the original message (using the address in the SMTP + MAIL command). + + When an SMTP server returns a permanent error status (5yz) code after + the DATA command is completed with ., it MUST NOT make + any subsequent attempt to deliver that message. The SMTP client + retains responsibility for delivery of that message and may either + return it to the user or requeue it for a subsequent attempt (see + section 4.5.4.1). + + The user who originated the message SHOULD be able to interpret the + return of a transient failure status (by mail message or otherwise) + as a non-delivery indication, just as a permanent failure would be + interpreted. I.e., if the client SMTP successfully handles these + conditions, the user will not receive such a reply. + + When an SMTP server returns a permanent error status (5yz) code after + the DATA command is completely with ., it MUST NOT make + any subsequent attempt to deliver the message. As with temporary + error status codes, the SMTP client retains responsibility for the + message, but SHOULD not again attempt delivery to the same server + without user review and intervention of the message. + +4.3 Sequencing of Commands and Replies + +4.3.1 Sequencing Overview + + The communication between the sender and receiver is an alternating + dialogue, controlled by the sender. As such, the sender issues a + command and the receiver responds with a reply. Unless other + arrangements are negotiated through service extensions, the sender + MUST wait for this response before sending further commands. + + One important reply is the connection greeting. Normally, a receiver + will send a 220 "Service ready" reply when the connection is + completed. The sender SHOULD wait for this greeting message before + sending any commands. + + Note: all the greeting-type replies have the official name (the + fully-qualified primary domain name) of the server host as the first + word following the reply code. Sometimes the host will have no + meaningful name. See 4.1.3 for a discussion of alternatives in these + situations. + + + + + +Klensin Standards Track [Page 47] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + For example, + + 220 ISIF.USC.EDU Service ready + or + 220 mail.foo.com SuperSMTP v 6.1.2 Service ready + or + 220 [10.0.0.1] Clueless host service ready + + The table below lists alternative success and failure replies for + each command. These SHOULD be strictly adhered to: a receiver may + substitute text in the replies, but the meaning and action implied by + the code numbers and by the specific command reply sequence cannot be + altered. + +4.3.2 Command-Reply Sequences + + Each command is listed with its usual possible replies. The prefixes + used before the possible replies are "I" for intermediate, "S" for + success, and "E" for error. Since some servers may generate other + replies under special circumstances, and to allow for future + extension, SMTP clients SHOULD, when possible, interpret only the + first digit of the reply and MUST be prepared to deal with + unrecognized reply codes by interpreting the first digit only. + Unless extended using the mechanisms described in section 2.2, SMTP + servers MUST NOT transmit reply codes to an SMTP client that are + other than three digits or that do not start in a digit between 2 and + 5 inclusive. + + These sequencing rules and, in principle, the codes themselves, can + be extended or modified by SMTP extensions offered by the server and + accepted (requested) by the client. + + In addition to the codes listed below, any SMTP command can return + any of the following codes if the corresponding unusual circumstances + are encountered: + + 500 For the "command line too long" case or if the command name was + not recognized. Note that producing a "command not recognized" + error in response to the required subset of these commands is a + violation of this specification. + + 501 Syntax error in command or arguments. In order to provide for + future extensions, commands that are specified in this document as + not accepting arguments (DATA, RSET, QUIT) SHOULD return a 501 + message if arguments are supplied in the absence of EHLO- + advertised extensions. + + 421 Service shutting down and closing transmission channel + + + +Klensin Standards Track [Page 48] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + Specific sequences are: + + CONNECTION ESTABLISHMENT + S: 220 + E: 554 + EHLO or HELO + S: 250 + E: 504, 550 + MAIL + S: 250 + E: 552, 451, 452, 550, 553, 503 + RCPT + S: 250, 251 (but see section 3.4 for discussion of 251 and 551) + E: 550, 551, 552, 553, 450, 451, 452, 503, 550 + DATA + I: 354 -> data -> S: 250 + E: 552, 554, 451, 452 + E: 451, 554, 503 + RSET + S: 250 + VRFY + S: 250, 251, 252 + E: 550, 551, 553, 502, 504 + EXPN + S: 250, 252 + E: 550, 500, 502, 504 + HELP + S: 211, 214 + E: 502, 504 + NOOP + S: 250 + QUIT + S: 221 + +4.4 Trace Information + + When an SMTP server receives a message for delivery or further + processing, it MUST insert trace ("time stamp" or "Received") + information at the beginning of the message content, as discussed in + section 4.1.1.4. + + This line MUST be structured as follows: + + - The FROM field, which MUST be supplied in an SMTP environment, + SHOULD contain both (1) the name of the source host as presented + in the EHLO command and (2) an address literal containing the IP + address of the source, determined from the TCP connection. + + + + +Klensin Standards Track [Page 49] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + - The ID field MAY contain an "@" as suggested in RFC 822, but this + is not required. + + - The FOR field MAY contain a list of entries when multiple + RCPT commands have been given. This may raise some security + issues and is usually not desirable; see section 7.2. + + An Internet mail program MUST NOT change a Received: line that was + previously added to the message header. SMTP servers MUST prepend + Received lines to messages; they MUST NOT change the order of + existing lines or insert Received lines in any other location. + + As the Internet grows, comparability of Received fields is important + for detecting problems, especially slow relays. SMTP servers that + create Received fields SHOULD use explicit offsets in the dates + (e.g., -0800), rather than time zone names of any type. Local time + (with an offset) is preferred to UT when feasible. This formulation + allows slightly more information about local circumstances to be + specified. If UT is needed, the receiver need merely do some simple + arithmetic to convert the values. Use of UT loses information about + the time zone-location of the server. If it is desired to supply a + time zone name, it SHOULD be included in a comment. + + When the delivery SMTP server makes the "final delivery" of a + message, it inserts a return-path line at the beginning of the mail + data. This use of return-path is required; mail systems MUST support + it. The return-path line preserves the information in the from the MAIL command. Here, final delivery means the message + has left the SMTP environment. Normally, this would mean it had been + delivered to the destination user or an associated mail drop, but in + some cases it may be further processed and transmitted by another + mail system. + + It is possible for the mailbox in the return path to be different + from the actual sender's mailbox, for example, if error responses are + to be delivered to a special error handling mailbox rather than to + the message sender. When mailing lists are involved, this + arrangement is common and useful as a means of directing errors to + the list maintainer rather than the message originator. + + The text above implies that the final mail data will begin with a + return path line, followed by one or more time stamp lines. These + lines will be followed by the mail data headers and body [32]. + + It is sometimes difficult for an SMTP server to determine whether or + not it is making final delivery since forwarding or other operations + may occur after the message is accepted for delivery. Consequently, + + + + +Klensin Standards Track [Page 50] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + any further (forwarding, gateway, or relay) systems MAY remove the + return path and rebuild the MAIL command as needed to ensure that + exactly one such line appears in a delivered message. + + A message-originating SMTP system SHOULD NOT send a message that + already contains a Return-path header. SMTP servers performing a + relay function MUST NOT inspect the message data, and especially not + to the extent needed to determine if Return-path headers are present. + SMTP servers making final delivery MAY remove Return-path headers + before adding their own. + + The primary purpose of the Return-path is to designate the address to + which messages indicating non-delivery or other mail system failures + are to be sent. For this to be unambiguous, exactly one return path + SHOULD be present when the message is delivered. Systems using RFC + 822 syntax with non-SMTP transports SHOULD designate an unambiguous + address, associated with the transport envelope, to which error + reports (e.g., non-delivery messages) should be sent. + + Historical note: Text in RFC 822 that appears to contradict the use + of the Return-path header (or the envelope reverse path address from + the MAIL command) as the destination for error messages is not + applicable on the Internet. The reverse path address (as copied into + the Return-path) MUST be used as the target of any mail containing + delivery error messages. + + In particular: + + - a gateway from SMTP->elsewhere SHOULD insert a return-path header, + unless it is known that the "elsewhere" transport also uses + Internet domain addresses and maintains the envelope sender + address separately. + + - a gateway from elsewhere->SMTP SHOULD delete any return-path + header present in the message, and either copy that information to + the SMTP envelope or combine it with information present in the + envelope of the other transport system to construct the reverse + path argument to the MAIL command in the SMTP envelope. + + The server must give special treatment to cases in which the + processing following the end of mail data indication is only + partially successful. This could happen if, after accepting several + recipients and the mail data, the SMTP server finds that the mail + data could be successfully delivered to some, but not all, of the + recipients. In such cases, the response to the DATA command MUST be + an OK reply. However, the SMTP server MUST compose and send an + "undeliverable mail" notification message to the originator of the + message. + + + +Klensin Standards Track [Page 51] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + A single notification listing all of the failed recipients or + separate notification messages MUST be sent for each failed + recipient. For economy of processing by the sender, the former is + preferred when possible. All undeliverable mail notification + messages are sent using the MAIL command (even if they result from + processing the obsolete SEND, SOML, or SAML commands) and use a null + return path as discussed in section 3.7. + + The time stamp line and the return path line are formally defined as + follows: + +Return-path-line = "Return-Path:" FWS Reverse-path + +Time-stamp-line = "Received:" FWS Stamp + +Stamp = From-domain By-domain Opt-info ";" FWS date-time + + ; where "date-time" is as defined in [32] + ; but the "obs-" forms, especially two-digit + ; years, are prohibited in SMTP and MUST NOT be used. + +From-domain = "FROM" FWS Extended-Domain CFWS + +By-domain = "BY" FWS Extended-Domain CFWS + +Extended-Domain = Domain / + ( Domain FWS "(" TCP-info ")" ) / + ( Address-literal FWS "(" TCP-info ")" ) + +TCP-info = Address-literal / ( Domain FWS Address-literal ) + ; Information derived by server from TCP connection + ; not client EHLO. + +Opt-info = [Via] [With] [ID] [For] + +Via = "VIA" FWS Link CFWS + +With = "WITH" FWS Protocol CFWS + +ID = "ID" FWS String / msg-id CFWS + +For = "FOR" FWS 1*( Path / Mailbox ) CFWS + +Link = "TCP" / Addtl-Link +Addtl-Link = Atom + ; Additional standard names for links are registered with the + ; Internet Assigned Numbers Authority (IANA). "Via" is + ; primarily of value with non-Internet transports. SMTP + + + +Klensin Standards Track [Page 52] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + ; servers SHOULD NOT use unregistered names. +Protocol = "ESMTP" / "SMTP" / Attdl-Protocol +Attdl-Protocol = Atom + ; Additional standard names for protocols are registered with the + ; Internet Assigned Numbers Authority (IANA). SMTP servers + ; SHOULD NOT use unregistered names. + +4.5 Additional Implementation Issues + +4.5.1 Minimum Implementation + + In order to make SMTP workable, the following minimum implementation + is required for all receivers. The following commands MUST be + supported to conform to this specification: + + EHLO + HELO + MAIL + RCPT + DATA + RSET + NOOP + QUIT + VRFY + + Any system that includes an SMTP server supporting mail relaying or + delivery MUST support the reserved mailbox "postmaster" as a case- + insensitive local name. This postmaster address is not strictly + necessary if the server always returns 554 on connection opening (as + described in section 3.1). The requirement to accept mail for + postmaster implies that RCPT commands which specify a mailbox for + postmaster at any of the domains for which the SMTP server provides + mail service, as well as the special case of "RCPT TO:" + (with no domain specification), MUST be supported. + + SMTP systems are expected to make every reasonable effort to accept + mail directed to Postmaster from any other system on the Internet. + In extreme cases --such as to contain a denial of service attack or + other breach of security-- an SMTP server may block mail directed to + Postmaster. However, such arrangements SHOULD be narrowly tailored + so as to avoid blocking messages which are not part of such attacks. + +4.5.2 Transparency + + Without some provision for data transparency, the character sequence + "." ends the mail text and cannot be sent by the user. + In general, users are not aware of such "forbidden" sequences. To + + + + +Klensin Standards Track [Page 53] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + allow all user composed text to be transmitted transparently, the + following procedures are used: + + - Before sending a line of mail text, the SMTP client checks the + first character of the line. If it is a period, one additional + period is inserted at the beginning of the line. + + - When a line of mail text is received by the SMTP server, it checks + the line. If the line is composed of a single period, it is + treated as the end of mail indicator. If the first character is a + period and there are other characters on the line, the first + character is deleted. + + The mail data may contain any of the 128 ASCII characters. All + characters are to be delivered to the recipient's mailbox, including + spaces, vertical and horizontal tabs, and other control characters. + If the transmission channel provides an 8-bit byte (octet) data + stream, the 7-bit ASCII codes are transmitted right justified in the + octets, with the high order bits cleared to zero. See 3.7 for + special treatment of these conditions in SMTP systems serving a relay + function. + + In some systems it may be necessary to transform the data as it is + received and stored. This may be necessary for hosts that use a + different character set than ASCII as their local character set, that + store data in records rather than strings, or which use special + character sequences as delimiters inside mailboxes. If such + transformations are necessary, they MUST be reversible, especially if + they are applied to mail being relayed. + +4.5.3 Sizes and Timeouts + +4.5.3.1 Size limits and minimums + + There are several objects that have required minimum/maximum sizes. + Every implementation MUST be able to receive objects of at least + these sizes. Objects larger than these sizes SHOULD be avoided when + possible. However, some Internet mail constructs such as encoded + X.400 addresses [16] will often require larger objects: clients MAY + attempt to transmit these, but MUST be prepared for a server to + reject them if they cannot be handled by it. To the maximum extent + possible, implementation techniques which impose no limits on the + length of these objects should be used. + + local-part + The maximum total length of a user name or other local-part is 64 + characters. + + + + +Klensin Standards Track [Page 54] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + domain + The maximum total length of a domain name or number is 255 + characters. + + path + The maximum total length of a reverse-path or forward-path is 256 + characters (including the punctuation and element separators). + + command line + The maximum total length of a command line including the command + word and the is 512 characters. SMTP extensions may be + used to increase this limit. + + reply line + The maximum total length of a reply line including the reply code + and the is 512 characters. More information may be + conveyed through multiple-line replies. + + text line + The maximum total length of a text line including the is + 1000 characters (not counting the leading dot duplicated for + transparency). This number may be increased by the use of SMTP + Service Extensions. + + message content + The maximum total length of a message content (including any + message headers as well as the message body) MUST BE at least 64K + octets. Since the introduction of Internet standards for + multimedia mail [12], message lengths on the Internet have grown + dramatically, and message size restrictions should be avoided if + at all possible. SMTP server systems that must impose + restrictions SHOULD implement the "SIZE" service extension [18], + and SMTP client systems that will send large messages SHOULD + utilize it when possible. + + recipients buffer + The minimum total number of recipients that must be buffered is + 100 recipients. Rejection of messages (for excessive recipients) + with fewer than 100 RCPT commands is a violation of this + specification. The general principle that relaying SMTP servers + MUST NOT, and delivery SMTP servers SHOULD NOT, perform validation + tests on message headers suggests that rejecting a message based + on the total number of recipients shown in header fields is to be + discouraged. A server which imposes a limit on the number of + recipients MUST behave in an orderly fashion, such as to reject + additional addresses over its limit rather than silently + discarding addresses previously accepted. A client that needs to + + + + +Klensin Standards Track [Page 55] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + deliver a message containing over 100 RCPT commands SHOULD be + prepared to transmit in 100-recipient "chunks" if the server + declines to accept more than 100 recipients in a single message. + + Errors due to exceeding these limits may be reported by using the + reply codes. Some examples of reply codes are: + + 500 Line too long. + or + 501 Path too long + or + 452 Too many recipients (see below) + or + 552 Too much mail data. + + RFC 821 [30] incorrectly listed the error where an SMTP server + exhausts its implementation limit on the number of RCPT commands + ("too many recipients") as having reply code 552. The correct reply + code for this condition is 452. Clients SHOULD treat a 552 code in + this case as a temporary, rather than permanent, failure so the logic + below works. + + When a conforming SMTP server encounters this condition, it has at + least 100 successful RCPT commands in its recipients buffer. If the + server is able to accept the message, then at least these 100 + addresses will be removed from the SMTP client's queue. When the + client attempts retransmission of those addresses which received 452 + responses, at least 100 of these will be able to fit in the SMTP + server's recipients buffer. Each retransmission attempt which is + able to deliver anything will be able to dispose of at least 100 of + these recipients. + + If an SMTP server has an implementation limit on the number of RCPT + commands and this limit is exhausted, it MUST use a response code of + 452 (but the client SHOULD also be prepared for a 552, as noted + above). If the server has a configured site-policy limitation on the + number of RCPT commands, it MAY instead use a 5XX response code. + This would be most appropriate if the policy limitation was intended + to apply if the total recipient count for a particular message body + were enforced even if that message body was sent in multiple mail + transactions. + +4.5.3.2 Timeouts + + An SMTP client MUST provide a timeout mechanism. It MUST use per- + command timeouts rather than somehow trying to time the entire mail + transaction. Timeouts SHOULD be easily reconfigurable, preferably + without recompiling the SMTP code. To implement this, a timer is set + + + +Klensin Standards Track [Page 56] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + for each SMTP command and for each buffer of the data transfer. The + latter means that the overall timeout is inherently proportional to + the size of the message. + + Based on extensive experience with busy mail-relay hosts, the minimum + per-command timeout values SHOULD be as follows: + + Initial 220 Message: 5 minutes + An SMTP client process needs to distinguish between a failed TCP + connection and a delay in receiving the initial 220 greeting + message. Many SMTP servers accept a TCP connection but delay + delivery of the 220 message until their system load permits more + mail to be processed. + + MAIL Command: 5 minutes + + RCPT Command: 5 minutes + A longer timeout is required if processing of mailing lists and + aliases is not deferred until after the message was accepted. + + DATA Initiation: 2 minutes + This is while awaiting the "354 Start Input" reply to a DATA + command. + + Data Block: 3 minutes + This is while awaiting the completion of each TCP SEND call + transmitting a chunk of data. + + DATA Termination: 10 minutes. + This is while awaiting the "250 OK" reply. When the receiver gets + the final period terminating the message data, it typically + performs processing to deliver the message to a user mailbox. A + spurious timeout at this point would be very wasteful and would + typically result in delivery of multiple copies of the message, + since it has been successfully sent and the server has accepted + responsibility for delivery. See section 6.1 for additional + discussion. + + An SMTP server SHOULD have a timeout of at least 5 minutes while it + is awaiting the next command from the sender. + +4.5.4 Retry Strategies + + The common structure of a host SMTP implementation includes user + mailboxes, one or more areas for queuing messages in transit, and one + or more daemon processes for sending and receiving mail. The exact + structure will vary depending on the needs of the users on the host + + + + +Klensin Standards Track [Page 57] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + and the number and size of mailing lists supported by the host. We + describe several optimizations that have proved helpful, particularly + for mailers supporting high traffic levels. + + Any queuing strategy MUST include timeouts on all activities on a + per-command basis. A queuing strategy MUST NOT send error messages + in response to error messages under any circumstances. + +4.5.4.1 Sending Strategy + + The general model for an SMTP client is one or more processes that + periodically attempt to transmit outgoing mail. In a typical system, + the program that composes a message has some method for requesting + immediate attention for a new piece of outgoing mail, while mail that + cannot be transmitted immediately MUST be queued and periodically + retried by the sender. A mail queue entry will include not only the + message itself but also the envelope information. + + The sender MUST delay retrying a particular destination after one + attempt has failed. In general, the retry interval SHOULD be at + least 30 minutes; however, more sophisticated and variable strategies + will be beneficial when the SMTP client can determine the reason for + non-delivery. + + Retries continue until the message is transmitted or the sender gives + up; the give-up time generally needs to be at least 4-5 days. The + parameters to the retry algorithm MUST be configurable. + + A client SHOULD keep a list of hosts it cannot reach and + corresponding connection timeouts, rather than just retrying queued + mail items. + + Experience suggests that failures are typically transient (the target + system or its connection has crashed), favoring a policy of two + connection attempts in the first hour the message is in the queue, + and then backing off to one every two or three hours. + + The SMTP client can shorten the queuing delay in cooperation with the + SMTP server. For example, if mail is received from a particular + address, it is likely that mail queued for that host can now be sent. + Application of this principle may, in many cases, eliminate the + requirement for an explicit "send queues now" function such as ETRN + [9]. + + The strategy may be further modified as a result of multiple + addresses per host (see below) to optimize delivery time vs. resource + usage. + + + + +Klensin Standards Track [Page 58] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + An SMTP client may have a large queue of messages for each + unavailable destination host. If all of these messages were retried + in every retry cycle, there would be excessive Internet overhead and + the sending system would be blocked for a long period. Note that an + SMTP client can generally determine that a delivery attempt has + failed only after a timeout of several minutes and even a one-minute + timeout per connection will result in a very large delay if retries + are repeated for dozens, or even hundreds, of queued messages to the + same host. + + At the same time, SMTP clients SHOULD use great care in caching + negative responses from servers. In an extreme case, if EHLO is + issued multiple times during the same SMTP connection, different + answers may be returned by the server. More significantly, 5yz + responses to the MAIL command MUST NOT be cached. + + When a mail message is to be delivered to multiple recipients, and + the SMTP server to which a copy of the message is to be sent is the + same for multiple recipients, then only one copy of the message + SHOULD be transmitted. That is, the SMTP client SHOULD use the + command sequence: MAIL, RCPT, RCPT,... RCPT, DATA instead of the + sequence: MAIL, RCPT, DATA, ..., MAIL, RCPT, DATA. However, if there + are very many addresses, a limit on the number of RCPT commands per + MAIL command MAY be imposed. Implementation of this efficiency + feature is strongly encouraged. + + Similarly, to achieve timely delivery, the SMTP client MAY support + multiple concurrent outgoing mail transactions. However, some limit + may be appropriate to protect the host from devoting all its + resources to mail. + +4.5.4.2 Receiving Strategy + + The SMTP server SHOULD attempt to keep a pending listen on the SMTP + port at all times. This requires the support of multiple incoming + TCP connections for SMTP. Some limit MAY be imposed but servers that + cannot handle more than one SMTP transaction at a time are not in + conformance with the intent of this specification. + + As discussed above, when the SMTP server receives mail from a + particular host address, it could activate its own SMTP queuing + mechanisms to retry any mail pending for that host address. + +4.5.5 Messages with a null reverse-path + + There are several types of notification messages which are required + by existing and proposed standards to be sent with a null reverse + path, namely non-delivery notifications as discussed in section 3.7, + + + +Klensin Standards Track [Page 59] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + other kinds of Delivery Status Notifications (DSNs) [24], and also + Message Disposition Notifications (MDNs) [10]. All of these kinds of + messages are notifications about a previous message, and they are + sent to the reverse-path of the previous mail message. (If the + delivery of such a notification message fails, that usually indicates + a problem with the mail system of the host to which the notification + message is addressed. For this reason, at some hosts the MTA is set + up to forward such failed notification messages to someone who is + able to fix problems with the mail system, e.g., via the postmaster + alias.) + + All other types of messages (i.e., any message which is not required + by a standards-track RFC to have a null reverse-path) SHOULD be sent + with with a valid, non-null reverse-path. + + Implementors of automated email processors should be careful to make + sure that the various kinds of messages with null reverse-path are + handled correctly, in particular such systems SHOULD NOT reply to + messages with null reverse-path. + +5. Address Resolution and Mail Handling + + Once an SMTP client lexically identifies a domain to which mail will + be delivered for processing (as described in sections 3.6 and 3.7), a + DNS lookup MUST be performed to resolve the domain name [22]. The + names are expected to be fully-qualified domain names (FQDNs): + mechanisms for inferring FQDNs from partial names or local aliases + are outside of this specification and, due to a history of problems, + are generally discouraged. The lookup first attempts to locate an MX + record associated with the name. If a CNAME record is found instead, + the resulting name is processed as if it were the initial name. If + no MX records are found, but an A RR is found, the A RR is treated as + if it was associated with an implicit MX RR, with a preference of 0, + pointing to that host. If one or more MX RRs are found for a given + name, SMTP systems MUST NOT utilize any A RRs associated with that + name unless they are located using the MX RRs; the "implicit MX" rule + above applies only if there are no MX records present. If MX records + are present, but none of them are usable, this situation MUST be + reported as an error. + + When the lookup succeeds, the mapping can result in a list of + alternative delivery addresses rather than a single address, because + of multiple MX records, multihoming, or both. To provide reliable + mail transmission, the SMTP client MUST be able to try (and retry) + each of the relevant addresses in this list in order, until a + delivery attempt succeeds. However, there MAY also be a configurable + limit on the number of alternate addresses that can be tried. In any + case, the SMTP client SHOULD try at least two addresses. + + + +Klensin Standards Track [Page 60] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + Two types of information is used to rank the host addresses: multiple + MX records, and multihomed hosts. + + Multiple MX records contain a preference indication that MUST be used + in sorting (see below). Lower numbers are more preferred than higher + ones. If there are multiple destinations with the same preference + and there is no clear reason to favor one (e.g., by recognition of an + easily-reached address), then the sender-SMTP MUST randomize them to + spread the load across multiple mail exchangers for a specific + organization. + + The destination host (perhaps taken from the preferred MX record) may + be multihomed, in which case the domain name resolver will return a + list of alternative IP addresses. It is the responsibility of the + domain name resolver interface to have ordered this list by + decreasing preference if necessary, and SMTP MUST try them in the + order presented. + + Although the capability to try multiple alternative addresses is + required, specific installations may want to limit or disable the use + of alternative addresses. The question of whether a sender should + attempt retries using the different addresses of a multihomed host + has been controversial. The main argument for using the multiple + addresses is that it maximizes the probability of timely delivery, + and indeed sometimes the probability of any delivery; the counter- + argument is that it may result in unnecessary resource use. Note + that resource use is also strongly determined by the sending strategy + discussed in section 4.5.4.1. + + If an SMTP server receives a message with a destination for which it + is a designated Mail eXchanger, it MAY relay the message (potentially + after having rewritten the MAIL FROM and/or RCPT TO addresses), make + final delivery of the message, or hand it off using some mechanism + outside the SMTP-provided transport environment. Of course, neither + of the latter require that the list of MX records be examined + further. + + If it determines that it should relay the message without rewriting + the address, it MUST sort the MX records to determine candidates for + delivery. The records are first ordered by preference, with the + lowest-numbered records being most preferred. The relay host MUST + then inspect the list for any of the names or addresses by which it + might be known in mail transactions. If a matching record is found, + all records at that preference level and higher-numbered ones MUST be + discarded from consideration. If there are no records left at that + point, it is an error condition, and the message MUST be returned as + undeliverable. If records do remain, they SHOULD be tried, best + preference first, as described above. + + + +Klensin Standards Track [Page 61] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +6. Problem Detection and Handling + +6.1 Reliable Delivery and Replies by Email + + When the receiver-SMTP accepts a piece of mail (by sending a "250 OK" + message in response to DATA), it is accepting responsibility for + delivering or relaying the message. It must take this responsibility + seriously. It MUST NOT lose the message for frivolous reasons, such + as because the host later crashes or because of a predictable + resource shortage. + + If there is a delivery failure after acceptance of a message, the + receiver-SMTP MUST formulate and mail a notification message. This + notification MUST be sent using a null ("<>") reverse path in the + envelope. The recipient of this notification MUST be the address + from the envelope return path (or the Return-Path: line). However, + if this address is null ("<>"), the receiver-SMTP MUST NOT send a + notification. Obviously, nothing in this section can or should + prohibit local decisions (i.e., as part of the same system + environment as the receiver-SMTP) to log or otherwise transmit + information about null address events locally if that is desired. If + the address is an explicit source route, it MUST be stripped down to + its final hop. + + For example, suppose that an error notification must be sent for a + message that arrived with: + + MAIL FROM:<@a,@b:user@d> + + The notification message MUST be sent using: + + RCPT TO: + + Some delivery failures after the message is accepted by SMTP will be + unavoidable. For example, it may be impossible for the receiving + SMTP server to validate all the delivery addresses in RCPT command(s) + due to a "soft" domain system error, because the target is a mailing + list (see earlier discussion of RCPT), or because the server is + acting as a relay and has no immediate access to the delivering + system. + + To avoid receiving duplicate messages as the result of timeouts, a + receiver-SMTP MUST seek to minimize the time required to respond to + the final . end of data indicator. See RFC 1047 [28] for + a discussion of this problem. + + + + + + +Klensin Standards Track [Page 62] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +6.2 Loop Detection + + Simple counting of the number of "Received:" headers in a message has + proven to be an effective, although rarely optimal, method of + detecting loops in mail systems. SMTP servers using this technique + SHOULD use a large rejection threshold, normally at least 100 + Received entries. Whatever mechanisms are used, servers MUST contain + provisions for detecting and stopping trivial loops. + +6.3 Compensating for Irregularities + + Unfortunately, variations, creative interpretations, and outright + violations of Internet mail protocols do occur; some would suggest + that they occur quite frequently. The debate as to whether a well- + behaved SMTP receiver or relay should reject a malformed message, + attempt to pass it on unchanged, or attempt to repair it to increase + the odds of successful delivery (or subsequent reply) began almost + with the dawn of structured network mail and shows no signs of + abating. Advocates of rejection claim that attempted repairs are + rarely completely adequate and that rejection of bad messages is the + only way to get the offending software repaired. Advocates of + "repair" or "deliver no matter what" argue that users prefer that + mail go through it if at all possible and that there are significant + market pressures in that direction. In practice, these market + pressures may be more important to particular vendors than strict + conformance to the standards, regardless of the preference of the + actual developers. + + The problems associated with ill-formed messages were exacerbated by + the introduction of the split-UA mail reading protocols [3, 26, 5, + 21]. These protocols have encouraged the use of SMTP as a posting + protocol, and SMTP servers as relay systems for these client hosts + (which are often only intermittently connected to the Internet). + Historically, many of those client machines lacked some of the + mechanisms and information assumed by SMTP (and indeed, by the mail + format protocol [7]). Some could not keep adequate track of time; + others had no concept of time zones; still others could not identify + their own names or addresses; and, of course, none could satisfy the + assumptions that underlay RFC 822's conception of authenticated + addresses. + + In response to these weak SMTP clients, many SMTP systems now + complete messages that are delivered to them in incomplete or + incorrect form. This strategy is generally considered appropriate + when the server can identify or authenticate the client, and there + are prior agreements between them. By contrast, there is at best + great concern about fixes applied by a relay or delivery SMTP server + that has little or no knowledge of the user or client machine. + + + +Klensin Standards Track [Page 63] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + The following changes to a message being processed MAY be applied + when necessary by an originating SMTP server, or one used as the + target of SMTP as an initial posting protocol: + + - Addition of a message-id field when none appears + + - Addition of a date, time or time zone when none appears + + - Correction of addresses to proper FQDN format + + The less information the server has about the client, the less likely + these changes are to be correct and the more caution and conservatism + should be applied when considering whether or not to perform fixes + and how. These changes MUST NOT be applied by an SMTP server that + provides an intermediate relay function. + + In all cases, properly-operating clients supplying correct + information are preferred to corrections by the SMTP server. In all + cases, documentation of actions performed by the servers (in trace + fields and/or header comments) is strongly encouraged. + +7. Security Considerations + +7.1 Mail Security and Spoofing + + SMTP mail is inherently insecure in that it is feasible for even + fairly casual users to negotiate directly with receiving and relaying + SMTP servers and create messages that will trick a naive recipient + into believing that they came from somewhere else. Constructing such + a message so that the "spoofed" behavior cannot be detected by an + expert is somewhat more difficult, but not sufficiently so as to be a + deterrent to someone who is determined and knowledgeable. + Consequently, as knowledge of Internet mail increases, so does the + knowledge that SMTP mail inherently cannot be authenticated, or + integrity checks provided, at the transport level. Real mail + security lies only in end-to-end methods involving the message + bodies, such as those which use digital signatures (see [14] and, + e.g., PGP [4] or S/MIME [31]). + + Various protocol extensions and configuration options that provide + authentication at the transport level (e.g., from an SMTP client to + an SMTP server) improve somewhat on the traditional situation + described above. However, unless they are accompanied by careful + handoffs of responsibility in a carefully-designed trust environment, + they remain inherently weaker than end-to-end mechanisms which use + digitally signed messages rather than depending on the integrity of + the transport system. + + + + +Klensin Standards Track [Page 64] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + Efforts to make it more difficult for users to set envelope return + path and header "From" fields to point to valid addresses other than + their own are largely misguided: they frustrate legitimate + applications in which mail is sent by one user on behalf of another + or in which error (or normal) replies should be directed to a special + address. (Systems that provide convenient ways for users to alter + these fields on a per-message basis should attempt to establish a + primary and permanent mailbox address for the user so that Sender + fields within the message data can be generated sensibly.) + + This specification does not further address the authentication issues + associated with SMTP other than to advocate that useful functionality + not be disabled in the hope of providing some small margin of + protection against an ignorant user who is trying to fake mail. + +7.2 "Blind" Copies + + Addresses that do not appear in the message headers may appear in the + RCPT commands to an SMTP server for a number of reasons. The two + most common involve the use of a mailing address as a "list exploder" + (a single address that resolves into multiple addresses) and the + appearance of "blind copies". Especially when more than one RCPT + command is present, and in order to avoid defeating some of the + purpose of these mechanisms, SMTP clients and servers SHOULD NOT copy + the full set of RCPT command arguments into the headers, either as + part of trace headers or as informational or private-extension + headers. Since this rule is often violated in practice, and cannot + be enforced, sending SMTP systems that are aware of "bcc" use MAY + find it helpful to send each blind copy as a separate message + transaction containing only a single RCPT command. + + There is no inherent relationship between either "reverse" (from + MAIL, SAML, etc., commands) or "forward" (RCPT) addresses in the SMTP + transaction ("envelope") and the addresses in the headers. Receiving + systems SHOULD NOT attempt to deduce such relationships and use them + to alter the headers of the message for delivery. The popular + "Apparently-to" header is a violation of this principle as well as a + common source of unintended information disclosure and SHOULD NOT be + used. + +7.3 VRFY, EXPN, and Security + + As discussed in section 3.5, individual sites may want to disable + either or both of VRFY or EXPN for security reasons. As a corollary + to the above, implementations that permit this MUST NOT appear to + have verified addresses that are not, in fact, verified. If a site + + + + + +Klensin Standards Track [Page 65] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + disables these commands for security reasons, the SMTP server MUST + return a 252 response, rather than a code that could be confused with + successful or unsuccessful verification. + + Returning a 250 reply code with the address listed in the VRFY + command after having checked it only for syntax violates this rule. + Of course, an implementation that "supports" VRFY by always returning + 550 whether or not the address is valid is equally not in + conformance. + + Within the last few years, the contents of mailing lists have become + popular as an address information source for so-called "spammers." + The use of EXPN to "harvest" addresses has increased as list + administrators have installed protections against inappropriate uses + of the lists themselves. Implementations SHOULD still provide + support for EXPN, but sites SHOULD carefully evaluate the tradeoffs. + As authentication mechanisms are introduced into SMTP, some sites may + choose to make EXPN available only to authenticated requestors. + +7.4 Information Disclosure in Announcements + + There has been an ongoing debate about the tradeoffs between the + debugging advantages of announcing server type and version (and, + sometimes, even server domain name) in the greeting response or in + response to the HELP command and the disadvantages of exposing + information that might be useful in a potential hostile attack. The + utility of the debugging information is beyond doubt. Those who + argue for making it available point out that it is far better to + actually secure an SMTP server rather than hope that trying to + conceal known vulnerabilities by hiding the server's precise identity + will provide more protection. Sites are encouraged to evaluate the + tradeoff with that issue in mind; implementations are strongly + encouraged to minimally provide for making type and version + information available in some way to other network hosts. + +7.5 Information Disclosure in Trace Fields + + In some circumstances, such as when mail originates from within a LAN + whose hosts are not directly on the public Internet, trace + ("Received") fields produced in conformance with this specification + may disclose host names and similar information that would not + normally be available. This ordinarily does not pose a problem, but + sites with special concerns about name disclosure should be aware of + it. Also, the optional FOR clause should be supplied with caution or + not at all when multiple recipients are involved lest it + inadvertently disclose the identities of "blind copy" recipients to + others. + + + + +Klensin Standards Track [Page 66] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +7.6 Information Disclosure in Message Forwarding + + As discussed in section 3.4, use of the 251 or 551 reply codes to + identify the replacement address associated with a mailbox may + inadvertently disclose sensitive information. Sites that are + concerned about those issues should ensure that they select and + configure servers appropriately. + +7.7 Scope of Operation of SMTP Servers + + It is a well-established principle that an SMTP server may refuse to + accept mail for any operational or technical reason that makes sense + to the site providing the server. However, cooperation among sites + and installations makes the Internet possible. If sites take + excessive advantage of the right to reject traffic, the ubiquity of + email availability (one of the strengths of the Internet) will be + threatened; considerable care should be taken and balance maintained + if a site decides to be selective about the traffic it will accept + and process. + + In recent years, use of the relay function through arbitrary sites + has been used as part of hostile efforts to hide the actual origins + of mail. Some sites have decided to limit the use of the relay + function to known or identifiable sources, and implementations SHOULD + provide the capability to perform this type of filtering. When mail + is rejected for these or other policy reasons, a 550 code SHOULD be + used in response to EHLO, MAIL, or RCPT as appropriate. + +8. IANA Considerations + + IANA will maintain three registries in support of this specification. + The first consists of SMTP service extensions with the associated + keywords, and, as needed, parameters and verbs. As specified in + section 2.2.2, no entry may be made in this registry that starts in + an "X". Entries may be made only for service extensions (and + associated keywords, parameters, or verbs) that are defined in + standards-track or experimental RFCs specifically approved by the + IESG for this purpose. + + The second registry consists of "tags" that identify forms of domain + literals other than those for IPv4 addresses (specified in RFC 821 + and in this document) and IPv6 addresses (specified in this + document). Additional literal types require standardization before + being used; none are anticipated at this time. + + The third, established by RFC 821 and renewed by this specification, + is a registry of link and protocol identifiers to be used with the + "via" and "with" subclauses of the time stamp ("Received: header") + + + +Klensin Standards Track [Page 67] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + described in section 4.4. Link and protocol identifiers in addition + to those specified in this document may be registered only by + standardization or by way of an RFC-documented, IESG-approved, + Experimental protocol extension. + +9. References + + [1] American National Standards Institute (formerly United States of + America Standards Institute), X3.4, 1968, "USA Code for + Information Interchange". ANSI X3.4-1968 has been replaced by + newer versions with slight modifications, but the 1968 version + remains definitive for the Internet. + + [2] Braden, R., "Requirements for Internet hosts - application and + support", STD 3, RFC 1123, October 1989. + + [3] Butler, M., Chase, D., Goldberger, J., Postel, J. and J. + Reynolds, "Post Office Protocol - version 2", RFC 937, February + 1985. + + [4] Callas, J., Donnerhacke, L., Finney, H. and R. Thayer, "OpenPGP + Message Format", RFC 2440, November 1998. + + [5] Crispin, M., "Interactive Mail Access Protocol - Version 2", RFC + 1176, August 1990. + + [6] Crispin, M., "Internet Message Access Protocol - Version 4", RFC + 2060, December 1996. + + [7] Crocker, D., "Standard for the Format of ARPA Internet Text + Messages", RFC 822, August 1982. + + [8] Crocker, D. and P. Overell, Eds., "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [9] De Winter, J., "SMTP Service Extension for Remote Message Queue + Starting", RFC 1985, August 1996. + + [10] Fajman, R., "An Extensible Message Format for Message + Disposition Notifications", RFC 2298, March 1998. + + [11] Freed, N, "Behavior of and Requirements for Internet Firewalls", + RFC 2979, October 2000. + + [12] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message Bodies", + RFC 2045, December 1996. + + + + +Klensin Standards Track [Page 68] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + [13] Freed, N., "SMTP Service Extension for Command Pipelining", RFC + 2920, September 2000. + + [14] Galvin, J., Murphy, S., Crocker, S. and N. Freed, "Security + Multiparts for MIME: Multipart/Signed and Multipart/Encrypted", + RFC 1847, October 1995. + + [15] Gellens, R. and J. Klensin, "Message Submission", RFC 2476, + December 1998. + + [16] Kille, S., "Mapping between X.400 and RFC822/MIME", RFC 2156, + January 1998. + + [17] Hinden, R and S. Deering, Eds. "IP Version 6 Addressing + Architecture", RFC 2373, July 1998. + + [18] Klensin, J., Freed, N. and K. Moore, "SMTP Service Extension for + Message Size Declaration", STD 10, RFC 1870, November 1995. + + [19] Klensin, J., Freed, N., Rose, M., Stefferud, E. and D. Crocker, + "SMTP Service Extensions", STD 10, RFC 1869, November 1995. + + [20] Klensin, J., Freed, N., Rose, M., Stefferud, E. and D. Crocker, + "SMTP Service Extension for 8bit-MIMEtransport", RFC 1652, July + 1994. + + [21] Lambert, M., "PCMAIL: A distributed mail system for personal + computers", RFC 1056, July 1988. + + [22] Mockapetris, P., "Domain names - implementation and + specification", STD 13, RFC 1035, November 1987. + + Mockapetris, P., "Domain names - concepts and facilities", STD + 13, RFC 1034, November 1987. + + [23] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part + Three: Message Header Extensions for Non-ASCII Text", RFC 2047, + December 1996. + + [24] Moore, K., "SMTP Service Extension for Delivery Status + Notifications", RFC 1891, January 1996. + + [25] Moore, K., and G. Vaudreuil, "An Extensible Message Format for + Delivery Status Notifications", RFC 1894, January 1996. + + [26] Myers, J. and M. Rose, "Post Office Protocol - Version 3", STD + 53, RFC 1939, May 1996. + + + + +Klensin Standards Track [Page 69] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + [27] Partridge, C., "Mail routing and the domain system", RFC 974, + January 1986. + + [28] Partridge, C., "Duplicate messages and SMTP", RFC 1047, February + 1988. + + [29] Postel, J., ed., "Transmission Control Protocol - DARPA Internet + Program Protocol Specification", STD 7, RFC 793, September 1981. + + [30] Postel, J., "Simple Mail Transfer Protocol", RFC 821, August + 1982. + + [31] Ramsdell, B., Ed., "S/MIME Version 3 Message Specification", RFC + 2633, June 1999. + + [32] Resnick, P., Ed., "Internet Message Format", RFC 2822, April + 2001. + + [33] Vaudreuil, G., "SMTP Service Extensions for Transmission of + Large and Binary MIME Messages", RFC 1830, August 1995. + + [34] Vaudreuil, G., "Enhanced Mail System Status Codes", RFC 1893, + January 1996. + +10. Editor's Address + + John C. Klensin + AT&T Laboratories + 99 Bedford St + Boston, MA 02111 USA + + Phone: 617-574-3076 + EMail: klensin@research.att.com + +11. Acknowledgments + + Many people worked long and hard on the many iterations of this + document. There was wide-ranging debate in the IETF DRUMS Working + Group, both on its mailing list and in face to face discussions, + about many technical issues and the role of a revised standard for + Internet mail transport, and many contributors helped form the + wording in this specification. The hundreds of participants in the + many discussions since RFC 821 was produced are too numerous to + mention, but they all helped this document become what it is. + + + + + + + +Klensin Standards Track [Page 70] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +APPENDICES + +A. TCP Transport Service + + The TCP connection supports the transmission of 8-bit bytes. The + SMTP data is 7-bit ASCII characters. Each character is transmitted + as an 8-bit byte with the high-order bit cleared to zero. Service + extensions may modify this rule to permit transmission of full 8-bit + data bytes as part of the message body, but not in SMTP commands or + responses. + +B. Generating SMTP Commands from RFC 822 Headers + + Some systems use RFC 822 headers (only) in a mail submission + protocol, or otherwise generate SMTP commands from RFC 822 headers + when such a message is handed to an MTA from a UA. While the MTA-UA + protocol is a private matter, not covered by any Internet Standard, + there are problems with this approach. For example, there have been + repeated problems with proper handling of "bcc" copies and + redistribution lists when information that conceptually belongs to a + mail envelopes is not separated early in processing from header + information (and kept separate). + + It is recommended that the UA provide its initial ("submission + client") MTA with an envelope separate from the message itself. + However, if the envelope is not supplied, SMTP commands SHOULD be + generated as follows: + + 1. Each recipient address from a TO, CC, or BCC header field SHOULD + be copied to a RCPT command (generating multiple message copies if + that is required for queuing or delivery). This includes any + addresses listed in a RFC 822 "group". Any BCC fields SHOULD then + be removed from the headers. Once this process is completed, the + remaining headers SHOULD be checked to verify that at least one + To:, Cc:, or Bcc: header remains. If none do, then a bcc: header + with no additional information SHOULD be inserted as specified in + [32]. + + 2. The return address in the MAIL command SHOULD, if possible, be + derived from the system's identity for the submitting (local) + user, and the "From:" header field otherwise. If there is a + system identity available, it SHOULD also be copied to the Sender + header field if it is different from the address in the From + header field. (Any Sender field that was already there SHOULD be + removed.) Systems may provide a way for submitters to override + the envelope return address, but may want to restrict its use to + privileged users. This will not prevent mail forgery, but may + lessen its incidence; see section 7.1. + + + +Klensin Standards Track [Page 71] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + When an MTA is being used in this way, it bears responsibility for + ensuring that the message being transmitted is valid. The mechanisms + for checking that validity, and for handling (or returning) messages + that are not valid at the time of arrival, are part of the MUA-MTA + interface and not covered by this specification. + + A submission protocol based on Standard RFC 822 information alone + MUST NOT be used to gateway a message from a foreign (non-SMTP) mail + system into an SMTP environment. Additional information to construct + an envelope must come from some source in the other environment, + whether supplemental headers or the foreign system's envelope. + + Attempts to gateway messages using only their header "to" and "cc" + fields have repeatedly caused mail loops and other behavior adverse + to the proper functioning of the Internet mail environment. These + problems have been especially common when the message originates from + an Internet mailing list and is distributed into the foreign + environment using envelope information. When these messages are then + processed by a header-only remailer, loops back to the Internet + environment (and the mailing list) are almost inevitable. + +C. Source Routes + + Historically, the was a reverse source routing list of + hosts and a source mailbox. The first host in the + SHOULD be the host sending the MAIL command. Similarly, the + may be a source routing lists of hosts and a + destination mailbox. However, in general, the SHOULD + contain only a mailbox and domain name, relying on the domain name + system to supply routing information if required. The use of source + routes is deprecated; while servers MUST be prepared to receive and + handle them as discussed in section 3.3 and F.2, clients SHOULD NOT + transmit them and this section was included only to provide context. + + For relay purposes, the forward-path may be a source route of the + form "@ONE,@TWO:JOE@THREE", where ONE, TWO, and THREE MUST BE fully- + qualified domain names. This form is used to emphasize the + distinction between an address and a route. The mailbox is an + absolute address, and the route is information about how to get + there. The two concepts should not be confused. + + If source routes are used, RFC 821 and the text below should be + consulted for the mechanisms for constructing and updating the + forward- and reverse-paths. + + + + + + + +Klensin Standards Track [Page 72] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + The SMTP server transforms the command arguments by moving its own + identifier (its domain name or that of any domain for which it is + acting as a mail exchanger), if it appears, from the forward-path to + the beginning of the reverse-path. + + Notice that the forward-path and reverse-path appear in the SMTP + commands and replies, but not necessarily in the message. That is, + there is no need for these paths and especially this syntax to appear + in the "To:" , "From:", "CC:", etc. fields of the message header. + Conversely, SMTP servers MUST NOT derive final message delivery + information from message header fields. + + When the list of hosts is present, it is a "reverse" source route and + indicates that the mail was relayed through each host on the list + (the first host in the list was the most recent relay). This list is + used as a source route to return non-delivery notices to the sender. + As each relay host adds itself to the beginning of the list, it MUST + use its name as known in the transport environment to which it is + relaying the mail rather than that of the transport environment from + which the mail came (if they are different). + +D. Scenarios + + This section presents complete scenarios of several types of SMTP + sessions. In the examples, "C:" indicates what is said by the SMTP + client, and "S:" indicates what is said by the SMTP server. + +D.1 A Typical SMTP Transaction Scenario + + This SMTP example shows mail sent by Smith at host bar.com, to Jones, + Green, and Brown at host foo.com. Here we assume that host bar.com + contacts host foo.com directly. The mail is accepted for Jones and + Brown. Green does not have a mailbox at host foo.com. + + S: 220 foo.com Simple Mail Transfer Service Ready + C: EHLO bar.com + S: 250-foo.com greets bar.com + S: 250-8BITMIME + S: 250-SIZE + S: 250-DSN + S: 250 HELP + C: MAIL FROM: + S: 250 OK + C: RCPT TO: + S: 250 OK + C: RCPT TO: + S: 550 No such user here + C: RCPT TO: + + + +Klensin Standards Track [Page 73] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + S: 250 OK + C: DATA + S: 354 Start mail input; end with . + C: Blah blah blah... + C: ...etc. etc. etc. + C: . + S: 250 OK + C: QUIT + S: 221 foo.com Service closing transmission channel + +D.2 Aborted SMTP Transaction Scenario + + S: 220 foo.com Simple Mail Transfer Service Ready + C: EHLO bar.com + S: 250-foo.com greets bar.com + S: 250-8BITMIME + S: 250-SIZE + S: 250-DSN + S: 250 HELP + C: MAIL FROM: + S: 250 OK + C: RCPT TO: + S: 250 OK + C: RCPT TO: + S: 550 No such user here + C: RSET + S: 250 OK + C: QUIT + S: 221 foo.com Service closing transmission channel + +D.3 Relayed Mail Scenario + + Step 1 -- Source Host to Relay Host + + S: 220 foo.com Simple Mail Transfer Service Ready + C: EHLO bar.com + S: 250-foo.com greets bar.com + S: 250-8BITMIME + S: 250-SIZE + S: 250-DSN + S: 250 HELP + C: MAIL FROM: + S: 250 OK + C: RCPT TO:<@foo.com:Jones@XYZ.COM> + S: 250 OK + C: DATA + S: 354 Start mail input; end with . + C: Date: Thu, 21 May 1998 05:33:29 -0700 + + + +Klensin Standards Track [Page 74] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + C: From: John Q. Public + C: Subject: The Next Meeting of the Board + C: To: Jones@xyz.com + C: + C: Bill: + C: The next meeting of the board of directors will be + C: on Tuesday. + C: John. + C: . + S: 250 OK + C: QUIT + S: 221 foo.com Service closing transmission channel + + Step 2 -- Relay Host to Destination Host + + S: 220 xyz.com Simple Mail Transfer Service Ready + C: EHLO foo.com + S: 250 xyz.com is on the air + C: MAIL FROM:<@foo.com:JQP@bar.com> + S: 250 OK + C: RCPT TO: + S: 250 OK + C: DATA + S: 354 Start mail input; end with . + C: Received: from bar.com by foo.com ; Thu, 21 May 1998 + C: 05:33:29 -0700 + C: Date: Thu, 21 May 1998 05:33:22 -0700 + C: From: John Q. Public + C: Subject: The Next Meeting of the Board + C: To: Jones@xyz.com + C: + C: Bill: + C: The next meeting of the board of directors will be + C: on Tuesday. + C: John. + C: . + S: 250 OK + C: QUIT + S: 221 foo.com Service closing transmission channel + +D.4 Verifying and Sending Scenario + + S: 220 foo.com Simple Mail Transfer Service Ready + C: EHLO bar.com + S: 250-foo.com greets bar.com + S: 250-8BITMIME + S: 250-SIZE + S: 250-DSN + + + +Klensin Standards Track [Page 75] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + + S: 250-VRFY + S: 250 HELP + C: VRFY Crispin + S: 250 Mark Crispin + C: SEND FROM: + S: 250 OK + C: RCPT TO: + S: 250 OK + C: DATA + S: 354 Start mail input; end with . + C: Blah blah blah... + C: ...etc. etc. etc. + C: . + S: 250 OK + C: QUIT + S: 221 foo.com Service closing transmission channel + +E. Other Gateway Issues + + In general, gateways between the Internet and other mail systems + SHOULD attempt to preserve any layering semantics across the + boundaries between the two mail systems involved. Gateway- + translation approaches that attempt to take shortcuts by mapping, + (such as envelope information from one system to the message headers + or body of another) have generally proven to be inadequate in + important ways. Systems translating between environments that do not + support both envelopes and headers and Internet mail must be written + with the understanding that some information loss is almost + inevitable. + +F. Deprecated Features of RFC 821 + + A few features of RFC 821 have proven to be problematic and SHOULD + NOT be used in Internet mail. + +F.1 TURN + + This command, described in RFC 821, raises important security issues + since, in the absence of strong authentication of the host requesting + that the client and server switch roles, it can easily be used to + divert mail from its correct destination. Its use is deprecated; + SMTP systems SHOULD NOT use it unless the server can authenticate the + client. + + + + + + + + +Klensin Standards Track [Page 76] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +F.2 Source Routing + + RFC 821 utilized the concept of explicit source routing to get mail + from one host to another via a series of relays. The requirement to + utilize source routes in regular mail traffic was eliminated by the + introduction of the domain name system "MX" record and the last + significant justification for them was eliminated by the + introduction, in RFC 1123, of a clear requirement that addresses + following an "@" must all be fully-qualified domain names. + Consequently, the only remaining justifications for the use of source + routes are support for very old SMTP clients or MUAs and in mail + system debugging. They can, however, still be useful in the latter + circumstance and for routing mail around serious, but temporary, + problems such as problems with the relevant DNS records. + + SMTP servers MUST continue to accept source route syntax as specified + in the main body of this document and in RFC 1123. They MAY, if + necessary, ignore the routes and utilize only the target domain in + the address. If they do utilize the source route, the message MUST + be sent to the first domain shown in the address. In particular, a + server MUST NOT guess at shortcuts within the source route. + + Clients SHOULD NOT utilize explicit source routing except under + unusual circumstances, such as debugging or potentially relaying + around firewall or mail system configuration errors. + +F.3 HELO + + As discussed in sections 3.1 and 4.1.1, EHLO is strongly preferred to + HELO when the server will accept the former. Servers must continue + to accept and process HELO in order to support older clients. + +F.4 #-literals + + RFC 821 provided for specifying an Internet address as a decimal + integer host number prefixed by a pound sign, "#". In practice, that + form has been obsolete since the introduction of TCP/IP. It is + deprecated and MUST NOT be used. + +F.5 Dates and Years + + When dates are inserted into messages by SMTP clients or servers + (e.g., in trace fields), four-digit years MUST BE used. Two-digit + years are deprecated; three-digit years were never permitted in the + Internet mail system. + + + + + + +Klensin Standards Track [Page 77] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +F.6 Sending versus Mailing + + In addition to specifying a mechanism for delivering messages to + user's mailboxes, RFC 821 provided additional, optional, commands to + deliver messages directly to the user's terminal screen. These + commands (SEND, SAML, SOML) were rarely implemented, and changes in + workstation technology and the introduction of other protocols may + have rendered them obsolete even where they are implemented. + + Clients SHOULD NOT provide SEND, SAML, or SOML as services. Servers + MAY implement them. If they are implemented by servers, the + implementation model specified in RFC 821 MUST be used and the + command names MUST be published in the response to the EHLO command. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Klensin Standards Track [Page 78] + +RFC 2821 Simple Mail Transfer Protocol April 2001 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2001). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Klensin Standards Track [Page 79] + diff --git a/docs/rfcs/rfc2971.txt b/docs/rfcs/rfc2971.txt new file mode 100644 index 0000000..9e7264d --- /dev/null +++ b/docs/rfcs/rfc2971.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group T. Showalter +Request for Comments: 2971 Mirapoint, Inc. +Category: Standards Track October 2000 + + + IMAP4 ID extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2000). All Rights Reserved. + +Abstract + + The ID extension to the Internet Message Access Protocol - Version + 4rev1 (IMAP4rev1) protocol allows the server and client to exchange + identification information on their implementation in order to make + bug reports and usage statistics more complete. + +1. Introduction + + The IMAP4rev1 protocol described in [IMAP4rev1] provides a method for + accessing remote mail stores, but it provides no facility to + advertise what program a client or server uses to provide service. + This makes it difficult for implementors to get complete bug reports + from users, as it is frequently difficult to know what client or + server is in use. + + Additionally, some sites may wish to assemble usage statistics based + on what clients are used, but in an an environment where users are + permitted to obtain and maintain their own clients this is difficult + to accomplish. + + The ID command provides a facility to advertise information on what + programs are being used along with contact information (should bugs + ever occur). + + + + + + + + +Showalter Standards Track [Page 1] + +RFC 2971 IMAP4 ID extension October 2000 + + +2. Conventions Used in this Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [KEYWORDS]. + + The conventions used in this document are the same as specified in + [IMAP4rev1]. In examples, "C:" and "S:" indicate lines sent by the + client and server respectively. Line breaks have been inserted for + readability. + +3. Specification + + The sole purpose of the ID extension is to enable clients and servers + to exchange information on their implementations for the purposes of + statistical analysis and problem determination. + + This information is be submitted to a server by any client wishing to + provide information for statistical purposes, provided the server + advertises its willingness to take the information with the atom "ID" + included in the list of capabilities returned by the CAPABILITY + command. + + Implementations MUST NOT make operational changes based on the data + sent as part of the ID command or response. The ID command is for + human consumption only, and is not to be used in improving the + performance of clients or servers. + + This includes, but is not limited to, the following: + + Servers MUST NOT attempt to work around client bugs by using + information from the ID command. Clients MUST NOT attempt to work + around server bugs based on the ID response. + + Servers MUST NOT provide features to a client or otherwise + optimize for a particular client by using information from the ID + command. Clients MUST NOT provide features to a server or + otherwise optimize for a particular server based on the ID + response. + + Servers MUST NOT deny access to or refuse service for a client + based on information from the ID command. Clients MUST NOT refuse + to operate or limit their operation with a server based on the ID + response. + + + + + + + +Showalter Standards Track [Page 2] + +RFC 2971 IMAP4 ID extension October 2000 + + + Rationale: It is imperative that this extension not supplant IMAP's + CAPABILITY mechanism with a ad-hoc approach where implementations + guess each other's features based on who they claim to be. + + Implementations MUST NOT send false information in an ID command. + + Implementations MAY send less information than they have available or + no information at all. Such behavior may be useful to preserve user + privacy. See Security Considerations, section 7. + +3.1. ID Command + + Arguments: client parameter list or NIL + + Responses: OPTIONAL untagged response: ID + + Result: OK identification information accepted + BAD command unknown or arguments invalid + + Implementation identification information is sent by the client with + the ID command. + + This command is valid in any state. + + The information sent is in the form of a list of field/value pairs. + Fields are permitted to be any IMAP4 string, and values are permitted + to be any IMAP4 string or NIL. A value of NIL indicates that the + client can not or will not specify this information. The client may + also send NIL instead of the list, indicating that it wants to send + no information, but would still accept a server response. + + The available fields are defined in section 3.3. + + Example: C: a023 ID ("name" "sodr" "version" "19.34" "vendor" + "Pink Floyd Music Limited") + S: * ID NIL + S: a023 OK ID completed + +3.2. ID Response + + Contents: server parameter list + + In response to an ID command issued by the client, the server replies + with a tagged response containing information on its implementation. + The format is the same as the client list. + + + + + + +Showalter Standards Track [Page 3] + +RFC 2971 IMAP4 ID extension October 2000 + + + Example: C: a042 ID NIL + S: * ID ("name" "Cyrus" "version" "1.5" "os" "sunos" + "os-version" "5.5" "support-url" + "mailto:cyrus-bugs+@andrew.cmu.edu") + S: a042 OK ID command completed + + A server MUST send a tagged ID response to an ID command. However, a + server MAY send NIL in place of the list. + +3.3. Defined Field Values + + Any string may be sent as a field, but the following are defined to + describe certain values that might be sent. Implementations are free + to send none, any, or all of these. Strings are not case-sensitive. + Field strings MUST NOT be longer than 30 octets. Value strings MUST + NOT be longer than 1024 octets. Implementations MUST NOT send more + than 30 field-value pairs. + + name Name of the program + version Version number of the program + os Name of the operating system + os-version Version of the operating system + vendor Vendor of the client/server + support-url URL to contact for support + address Postal address of contact/vendor + date Date program was released, specified as a date-time + in IMAP4rev1 + command Command used to start the program + arguments Arguments supplied on the command line, if any + if any + environment Description of environment, i.e., UNIX environment + variables or Windows registry settings + + Implementations MUST NOT use contact information to submit automatic + bug reports. Implementations may include information from an ID + response in a report automatically prepared, but are prohibited from + sending the report without user authorization. + + It is preferable to find the name and version of the underlying + operating system at runtime in cases where this is possible. + + Information sent via an ID response may violate user privacy. See + Security Considerations, section 7. + + Implementations MUST NOT send the same field name more than once. + + + + + + +Showalter Standards Track [Page 4] + +RFC 2971 IMAP4 ID extension October 2000 + + +4. Formal Syntax + + This syntax is intended to augment the grammar specified in + [IMAP4rev1] in order to provide for the ID command. This + specification uses the augmented Backus-Naur Form (BNF) notation as + used in [IMAP4rev1]. + + command_any ::= "CAPABILITY" / "LOGOUT" / "NOOP" / x_command / id + ;; adds id command to command_any in [IMAP4rev1] + + id ::= "ID" SPACE id_params_list + + id_response ::= "ID" SPACE id_params_list + + id_params_list ::= "(" #(string SPACE nstring) ")" / nil + ;; list of field value pairs + + response_data ::= "*" SPACE (resp_cond_state / resp_cond_bye / + mailbox_data / message_data / capability_data / id_response) + +5. Use of the ID extension with Firewalls and Other Intermediaries + + There exist proxies, firewalls, and other intermediary systems that + can intercept an IMAP session and make changes to the data exchanged + in the session. Such intermediaries are not anticipated by the IMAP4 + protocol design and are not within the scope of the IMAP4 standard. + However, in order for the ID command to be useful in the presence of + such intermediaries, those intermediaries need to take special note + of the ID command and response. In particular, if an intermediary + changes any part of the IMAP session it must also change the ID + command to advertise its presence. + + A firewall MAY act to block transmission of specific information + fields in the ID command and response that it believes reveal + information that could expose a security vulnerability. However, a + firewall SHOULD NOT disable the extension, when present, entirely, + and SHOULD NOT unconditionally remove either the client or server + list. + + Finally, it should be noted that a firewall, when handling a + CAPABILITY response, MUST NOT allow the names of extensions to be + returned to the client that the firewall has no knowledge of. + + + + + + + + + +Showalter Standards Track [Page 5] + +RFC 2971 IMAP4 ID extension October 2000 + + +6. References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, March 1997. + + [IMAP4rev1] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, October 1996. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet + Text Messages", STD 11, RFC 822, August 1982. + +7. Security Considerations + + This extension has the danger of violating the privacy of users if + misused. Clients and servers should notify users that they implement + and enable the ID command. + + It is highly desirable that implementations provide a method of + disabling ID support, perhaps by not sending ID at all, or by sending + NIL as the argument to the ID command or response. + + Implementors must exercise extreme care in adding fields sent as part + of an ID command or response. Some fields, including a processor ID + number, Ethernet address, or other unique (or mostly unique) + identifier allow tracking of users in ways that violate user privacy + expectations. + + Having implementation information of a given client or server may + make it easier for an attacker to gain unauthorized access due to + security holes. + + Since this command includes arbitrary data and does not require the + user to authenticate, server implementations are cautioned to guard + against an attacker sending arbitrary garbage data in order to fill + up the ID log. In particular, if a server naively logs each ID + command to disk without inspecting it, an attacker can simply fire up + thousands of connections and send a few kilobytes of random data. + Servers have to guard against this. Methods include truncating + abnormally large responses; collating responses by storing only a + single copy, then keeping a counter of the number of times that + response has been seen; keeping only particularly interesting parts + of responses; and only logging responses of users who actually log + in. + + Security is affected by firewalls which modify the IMAP protocol + stream; see section 5, Use of the ID Extension with Firewalls and + Other Intermediaries, for more information. + + + + +Showalter Standards Track [Page 6] + +RFC 2971 IMAP4 ID extension October 2000 + + +8. Author's Address + + Tim Showalter + Mirapoint, Inc. + 909 Hermosa Ct. + Sunnyvale, CA 94095 + + EMail: tjs@mirapoint.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Showalter Standards Track [Page 7] + +RFC 2971 IMAP4 ID extension October 2000 + + +9. Full Copyright Statement + + Copyright (C) The Internet Society (2000). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Showalter Standards Track [Page 8] + diff --git a/docs/rfcs/rfc3028.txt b/docs/rfcs/rfc3028.txt new file mode 100644 index 0000000..d909a54 --- /dev/null +++ b/docs/rfcs/rfc3028.txt @@ -0,0 +1,2019 @@ + + + + + + +Network Working Group T. Showalter +Request for Comments: 3028 Mirapoint, Inc. +Category: Standards Track January 2001 + + + Sieve: A Mail Filtering Language + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2001). All Rights Reserved. + +Abstract + + This document describes a language for filtering e-mail messages at + time of final delivery. It is designed to be implementable on either + a mail client or mail server. It is meant to be extensible, simple, + and independent of access protocol, mail architecture, and operating + system. It is suitable for running on a mail server where users may + not be allowed to execute arbitrary programs, such as on black box + Internet Message Access Protocol (IMAP) servers, as it has no + variables, loops, or ability to shell out to external programs. + +Table of Contents + + 1. Introduction ........................................... 3 + 1.1. Conventions Used in This Document ..................... 4 + 1.2. Example mail messages ................................. 4 + 2. Design ................................................. 5 + 2.1. Form of the Language .................................. 5 + 2.2. Whitespace ............................................ 5 + 2.3. Comments .............................................. 6 + 2.4. Literal Data .......................................... 6 + 2.4.1. Numbers ............................................... 6 + 2.4.2. Strings ............................................... 7 + 2.4.2.1. String Lists .......................................... 7 + 2.4.2.2. Headers ............................................... 8 + 2.4.2.3. Addresses ............................................. 8 + 2.4.2.4. MIME Parts ............................................ 9 + 2.5. Tests ................................................. 9 + 2.5.1. Test Lists ............................................ 9 + + + +Showalter Standards Track [Page 1] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + 2.6. Arguments ............................................. 9 + 2.6.1. Positional Arguments .................................. 9 + 2.6.2. Tagged Arguments ...................................... 10 + 2.6.3. Optional Arguments .................................... 10 + 2.6.4. Types of Arguments .................................... 10 + 2.7. String Comparison ..................................... 11 + 2.7.1. Match Type ............................................ 11 + 2.7.2. Comparisons Across Character Sets ..................... 12 + 2.7.3. Comparators ........................................... 12 + 2.7.4. Comparisons Against Addresses ......................... 13 + 2.8. Blocks ................................................ 14 + 2.9. Commands .............................................. 14 + 2.10. Evaluation ............................................ 15 + 2.10.1. Action Interaction .................................... 15 + 2.10.2. Implicit Keep ......................................... 15 + 2.10.3. Message Uniqueness in a Mailbox ....................... 15 + 2.10.4. Limits on Numbers of Actions .......................... 16 + 2.10.5. Extensions and Optional Features ...................... 16 + 2.10.6. Errors ................................................ 17 + 2.10.7. Limits on Execution ................................... 17 + 3. Control Commands ....................................... 17 + 3.1. Control Structure If .................................. 18 + 3.2. Control Structure Require ............................. 19 + 3.3. Control Structure Stop ................................ 19 + 4. Action Commands ........................................ 19 + 4.1. Action reject ......................................... 20 + 4.2. Action fileinto ....................................... 20 + 4.3. Action redirect ....................................... 21 + 4.4. Action keep ........................................... 21 + 4.5. Action discard ........................................ 22 + 5. Test Commands .......................................... 22 + 5.1. Test address .......................................... 23 + 5.2. Test allof ............................................ 23 + 5.3. Test anyof ............................................ 24 + 5.4. Test envelope ......................................... 24 + 5.5. Test exists ........................................... 25 + 5.6. Test false ............................................ 25 + 5.7. Test header ........................................... 25 + 5.8. Test not .............................................. 26 + 5.9. Test size ............................................. 26 + 5.10. Test true ............................................. 26 + 6. Extensibility .......................................... 26 + 6.1. Capability String ..................................... 27 + 6.2. IANA Considerations ................................... 28 + 6.2.1. Template for Capability Registrations ................. 28 + 6.2.2. Initial Capability Registrations ...................... 28 + 6.3. Capability Transport .................................. 29 + 7. Transmission ........................................... 29 + + + +Showalter Standards Track [Page 2] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + 8. Parsing ................................................ 30 + 8.1. Lexical Tokens ........................................ 30 + 8.2. Grammar ............................................... 31 + 9. Extended Example ....................................... 32 + 10. Security Considerations ................................ 34 + 11. Acknowledgments ........................................ 34 + 12. Author's Address ....................................... 34 + 13. References ............................................. 34 + 14. Full Copyright Statement ............................... 36 + +1. Introduction + + This memo documents a language that can be used to create filters for + electronic mail. It is not tied to any particular operating system or + mail architecture. It requires the use of [IMAIL]-compliant + messages, but should otherwise generalize to many systems. + + The language is powerful enough to be useful but limited in order to + allow for a safe server-side filtering system. The intention is to + make it impossible for users to do anything more complex (and + dangerous) than write simple mail filters, along with facilitating + the use of GUIs for filter creation and manipulation. The language is + not Turing-complete: it provides no way to write a loop or a function + and variables are not provided. + + Scripts written in Sieve are executed during final delivery, when the + message is moved to the user-accessible mailbox. In systems where + the MTA does final delivery, such as traditional Unix mail, it is + reasonable to sort when the MTA deposits mail into the user's + mailbox. + + There are a number of reasons to use a filtering system. Mail + traffic for most users has been increasing due to increased usage of + e-mail, the emergence of unsolicited email as a form of advertising, + and increased usage of mailing lists. + + Experience at Carnegie Mellon has shown that if a filtering system is + made available to users, many will make use of it in order to file + messages from specific users or mailing lists. However, many others + did not make use of the Andrew system's FLAMES filtering language + [FLAMES] due to difficulty in setting it up. + + Because of the expectation that users will make use of filtering if + it is offered and easy to use, this language has been made simple + enough to allow many users to make use of it, but rich enough that it + can be used productively. However, it is expected that GUI-based + editors will be the preferred way of editing filters for a large + number of users. + + + +Showalter Standards Track [Page 3] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +1.1. Conventions Used in This Document + + In the sections of this document that discuss the requirements of + various keywords and operators, the following conventions have been + adopted. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and + "MAY" in this document are to be interpreted as defined in + [KEYWORDS]. + + Each section on a command (test, action, or control structure) has a + line labeled "Syntax:". This line describes the syntax of the + command, including its name and its arguments. Required arguments + are listed inside angle brackets ("<" and ">"). Optional arguments + are listed inside square brackets ("[" and "]"). Each argument is + followed by its type, so "" represents an argument + called "key" that is a string. Literal strings are represented with + double-quoted strings. Alternatives are separated with slashes, and + parenthesis are used for grouping, similar to [ABNF]. + + In the "Syntax" line, there are three special pieces of syntax that + are frequently repeated, MATCH-TYPE, COMPARATOR, and ADDRESS-PART. + These are discussed in sections 2.7.1, 2.7.3, and 2.7.4, + respectively. + + The formal grammar for these commands in section 10 and is the + authoritative reference on how to construct commands, but the formal + grammar does not specify the order, semantics, number or types of + arguments to commands, nor the legal command names. The intent is to + allow for extension without changing the grammar. + +1.2. Example mail messages + + The following mail messages will be used throughout this document in + examples. + + Message A + ----------------------------------------------------------- + Date: Tue, 1 Apr 1997 09:06:31 -0800 (PST) + From: coyote@desert.example.org + To: roadrunner@acme.example.com + Subject: I have a present for you + + Look, I'm sorry about the whole anvil thing, and I really + didn't mean to try and drop it on you from the top of the + cliff. I want to try to make it up to you. I've got some + great birdseed over here at my place--top of the line + + + + +Showalter Standards Track [Page 4] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + stuff--and if you come by, I'll have it all wrapped up + for you. I'm really sorry for all the problems I've caused + for you over the years, but I know we can work this out. + -- + Wile E. Coyote "Super Genius" coyote@desert.example.org + ----------------------------------------------------------- + + Message B + ----------------------------------------------------------- + From: youcouldberich!@reply-by-postal-mail.invalid + Sender: b1ff@de.res.example.com + To: rube@landru.example.edu + Date: Mon, 31 Mar 1997 18:26:10 -0800 + Subject: $$$ YOU, TOO, CAN BE A MILLIONAIRE! $$$ + + YOU MAY HAVE ALREADY WON TEN MILLION DOLLARS, BUT I DOUBT + IT! SO JUST POST THIS TO SIX HUNDRED NEWSGROUPS! IT WILL + GUARANTEE THAT YOU GET AT LEAST FIVE RESPONSES WITH MONEY! + MONEY! MONEY! COLD HARD CASH! YOU WILL RECEIVE OVER + $20,000 IN LESS THAN TWO MONTHS! AND IT'S LEGAL!!!!!!!!! + !!!!!!!!!!!!!!!!!!111111111!!!!!!!11111111111!!1 JUST + SEND $5 IN SMALL, UNMARKED BILLS TO THE ADDRESSES BELOW! + ----------------------------------------------------------- + +2. Design + +2.1. Form of the Language + + The language consists of a set of commands. Each command consists of + a set of tokens delimited by whitespace. The command identifier is + the first token and it is followed by zero or more argument tokens. + Arguments may be literal data, tags, blocks of commands, or test + commands. + + The language is represented in UTF-8, as specified in [UTF-8]. + + Tokens in the ASCII range are considered case-insensitive. + +2.2. Whitespace + + Whitespace is used to separate tokens. Whitespace is made up of + tabs, newlines (CRLF, never just CR or LF), and the space character. + The amount of whitespace used is not significant. + + + + + + + + +Showalter Standards Track [Page 5] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.3. Comments + + Two types of comments are offered. Comments are semantically + equivalent to whitespace and can be used anyplace that whitespace is + (with one exception in multi-line strings, as described in the + grammar). + + Hash comments begin with a "#" character that is not contained within + a string and continue until the next CRLF. + + Example: if size :over 100K { # this is a comment + discard; + } + + Bracketed comments begin with the token "/*" and end with "*/" outside + of a string. Bracketed comments may span multiple lines. Bracketed + comments do not nest. + + Example: if size :over 100K { /* this is a comment + this is still a comment */ discard /* this is a comment + */ ; + } + +2.4. Literal Data + + Literal data means data that is not executed, merely evaluated "as + is", to be used as arguments to commands. Literal data is limited to + numbers and strings. + +2.4.1. Numbers + + Numbers are given as ordinary decimal numbers. However, those + numbers that have a tendency to be fairly large, such as message + sizes, MAY have a "K", "M", or "G" appended to indicate a multiple of + a power of two. To be comparable with the power-of-two-based + versions of SI units that computers frequently use, K specifies + kibi-, or 1,024 (2^10) times the value of the number; M specifies + mebi-, or 1,048,576 (2^20) times the value of the number; and G + specifies tebi-, or 1,073,741,824 (2^30) times the value of the + number [BINARY-SI]. + + Implementations MUST provide 31 bits of magnitude in numbers, but MAY + provide more. + + Only positive integers are permitted by this specification. + + + + + + +Showalter Standards Track [Page 6] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.4.2. Strings + + Scripts involve large numbers of strings as they are used for pattern + matching, addresses, textual bodies, etc. Typically, short quoted + strings suffice for most uses, but a more convenient form is provided + for longer strings such as bodies of messages. + + A quoted string starts and ends with a single double quote (the <"> + character, ASCII 34). A backslash ("\", ASCII 92) inside of a quoted + string is followed by either another backslash or a double quote. + This two-character sequence represents a single backslash or double- + quote within the string, respectively. + + No other characters should be escaped with a single backslash. + + An undefined escape sequence (such as "\a" in a context where "a" has + no special meaning) is interpreted as if there were no backslash (in + this case, "\a" is just "a"). + + Non-printing characters such as tabs, CR and LF, and control + characters are permitted in quoted strings. Quoted strings MAY span + multiple lines. NUL (ASCII 0) is not allowed in strings. + + For entering larger amounts of text, such as an email message, a + multi-line form is allowed. It starts with the keyword "text:", + followed by a CRLF, and ends with the sequence of a CRLF, a single + period, and another CRLF. In order to allow the message to contain + lines with a single-dot, lines are dot-stuffed. That is, when + composing a message body, an extra `.' is added before each line + which begins with a `.'. When the server interprets the script, + these extra dots are removed. Note that a line that begins with a + dot followed by a non-dot character is not interpreted dot-stuffed; + that is, ".foo" is interpreted as ".foo". However, because this is + potentially ambiguous, scripts SHOULD be properly dot-stuffed so such + lines do not appear. + + Note that a hashed comment or whitespace may occur in between the + "text:" and the CRLF, but not within the string itself. Bracketed + comments are not allowed here. + +2.4.2.1. String Lists + + When matching patterns, it is frequently convenient to match against + groups of strings instead of single strings. For this reason, a list + of strings is allowed in many tests, implying that if the test is + true using any one of the strings, then the test is true. + Implementations are encouraged to use short-circuit evaluation in + these cases. + + + +Showalter Standards Track [Page 7] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + For instance, the test `header :contains ["To", "Cc"] + ["me@example.com", "me00@landru.example.edu"]' is true if either the + To header or Cc header of the input message contains either of the + e-mail addresses "me@example.com" or "me00@landru.example.edu". + + Conversely, in any case where a list of strings is appropriate, a + single string is allowed without being a member of a list: it is + equivalent to a list with a single member. This means that the test + `exists "To"' is equivalent to the test `exists ["To"]'. + +2.4.2.2. Headers + + Headers are a subset of strings. In the Internet Message + Specification [IMAIL] [RFC1123], each header line is allowed to have + whitespace nearly anywhere in the line, including after the field + name and before the subsequent colon. Extra spaces between the + header name and the ":" in a header field are ignored. + + A header name never contains a colon. The "From" header refers to a + line beginning "From:" (or "From :", etc.). No header will match + the string "From:" due to the trailing colon. + + Folding of long header lines (as described in [IMAIL] 3.4.8) is + removed prior to interpretation of the data. The folding syntax (the + CRLF that ends a line plus any leading whitespace at the beginning of + the next line that indicates folding) are interpreted as if they were + a single space. + +2.4.2.3. Addresses + + A number of commands call for email addresses, which are also a + subset of strings. When these addresses are used in outbound + contexts, addresses must be compliant with [IMAIL], but are further + constrained. Using the symbols defined in [IMAIL], section 6.1, the + syntax of an address is: + + sieve-address = addr-spec ; simple address + / phrase "<" addr-spec ">" ; name & addr-spec + + That is, routes and group syntax are not permitted. If multiple + addresses are required, use a string list. Named groups are not used + here. + + Implementations MUST ensure that the addresses are syntactically + valid, but need not ensure that they actually identify an email + recipient. + + + + + +Showalter Standards Track [Page 8] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.4.2.4. MIME Parts + + In a few places, [MIME] body parts are represented as strings. These + parts include MIME headers and the body. This provides a way of + embedding typed data within a Sieve script so that, among other + things, character sets other than UTF-8 can be used for output + messages. + +2.5. Tests + + Tests are given as arguments to commands in order to control their + actions. In this document, tests are given to if/elsif/else to + decide which block of code is run. + + Tests MUST NOT have side effects. That is, a test cannot affect the + state of the filter or message. No tests in this specification have + side effects, and side effects are forbidden in extension tests as + well. + + The rationale for this is that tests with side effects impair + readability and maintainability and are difficult to represent in a + graphic interface for generating scripts. Side effects are confined + to actions where they are clearer. + +2.5.1. Test Lists + + Some tests ("allof" and "anyof", which implement logical "and" and + logical "or", respectively) may require more than a single test as an + argument. The test-list syntax element provides a way of grouping + tests. + + Example: if anyof (not exists ["From", "Date"], + header :contains "from" "fool@example.edu") { + discard; + } + +2.6. Arguments + + In order to specify what to do, most commands take arguments. There + are three types of arguments: positional, tagged, and optional. + +2.6.1. Positional Arguments + + Positional arguments are given to a command which discerns their + meaning based on their order. When a command takes positional + arguments, all positional arguments must be supplied and must be in + the order prescribed. + + + + +Showalter Standards Track [Page 9] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.6.2. Tagged Arguments + + This document provides for tagged arguments in the style of + CommonLISP. These are also similar to flags given to commands in + most command-line systems. + + A tagged argument is an argument for a command that begins with ":" + followed by a tag naming the argument, such as ":contains". This + argument means that zero or more of the next tokens have some + particular meaning depending on the argument. These next tokens may + be numbers or strings but they are never blocks. + + Tagged arguments are similar to positional arguments, except that + instead of the meaning being derived from the command, it is derived + from the tag. + + Tagged arguments must appear before positional arguments, but they + may appear in any order with other tagged arguments. For simplicity + of the specification, this is not expressed in the syntax definitions + with commands, but they still may be reordered arbitrarily provided + they appear before positional arguments. Tagged arguments may be + mixed with optional arguments. + + To simplify this specification, tagged arguments SHOULD NOT take + tagged arguments as arguments. + +2.6.3. Optional Arguments + + Optional arguments are exactly like tagged arguments except that they + may be left out, in which case a default value is implied. Because + optional arguments tend to result in shorter scripts, they have been + used far more than tagged arguments. + + One particularly noteworthy case is the ":comparator" argument, which + allows the user to specify which [ACAP] comparator will be used to + compare two strings, since different languages may impose different + orderings on UTF-8 [UTF-8] characters. + +2.6.4. Types of Arguments + + Abstractly, arguments may be literal data, tests, or blocks of + commands. In this way, an "if" control structure is merely a command + that happens to take a test and a block as arguments and may execute + the block of code. + + However, this abstraction is ambiguous from a parsing standpoint. + The grammar in section 9.2 presents a parsable version of this: + Arguments are string-lists, numbers, and tags, which may be followed + + + +Showalter Standards Track [Page 10] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + by a test or a test-list, which may be followed by a block of + commands. No more than one test or test list, nor more than one + block of commands, may be used, and commands that end with blocks of + commands do not end with semicolons. + +2.7. String Comparison + + When matching one string against another, there are a number of ways + of performing the match operation. These are accomplished with three + types of matches: an exact match, a substring match, and a wildcard + glob-style match. These are described below. + + In order to provide for matches between character sets and case + insensitivity, Sieve borrows ACAP's comparator registry. + + However, when a string represents the name of a header, the + comparator is never user-specified. Header comparisons are always + done with the "i;ascii-casemap" operator, i.e., case-insensitive + comparisons, because this is the way things are defined in the + message specification [IMAIL]. + +2.7.1. Match Type + + There are three match types describing the matching used in this + specification: ":is", ":contains", and ":matches". Match type + arguments are supplied to those commands which allow them to specify + what kind of match is to be performed. + + These are used as tagged arguments to tests that perform string + comparison. + + The ":contains" match type describes a substring match. If the value + argument contains the key argument as a substring, the match is true. + For instance, the string "frobnitzm" contains "frob" and "nit", but + not "fbm". The null key ("") is contained in all values. + + The ":is" match type describes an absolute match; if the contents of + the first string are absolutely the same as the contents of the + second string, they match. Only the string "frobnitzm" is the string + "frobnitzm". The null key ":is" and only ":is" the null value. + + The ":matches" version specifies a wildcard match using the + characters "*" and "?". "*" matches zero or more characters, and "?" + matches a single character. "?" and "*" may be escaped as "\\?" and + "\\*" in strings to match against themselves. The first backslash + escapes the second backslash; together, they escape the "*". This is + awkward, but it is commonplace in several programming languages that + use globs and regular expressions. + + + +Showalter Standards Track [Page 11] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + In order to specify what type of match is supposed to happen, + commands that support matching take optional tagged arguments + ":matches", ":is", and ":contains". Commands default to using ":is" + matching if no match type argument is supplied. Note that these + modifiers may interact with comparators; in particular, some + comparators are not suitable for matching with ":contains" or + ":matches". It is an error to use a comparator with ":contains" or + ":matches" that is not compatible with it. + + It is an error to give more than one of these arguments to a given + command. + + For convenience, the "MATCH-TYPE" syntax element is defined here as + follows: + + Syntax: ":is" / ":contains" / ":matches" + +2.7.2. Comparisons Across Character Sets + + All Sieve scripts are represented in UTF-8, but messages may involve + a number of character sets. In order for comparisons to work across + character sets, implementations SHOULD implement the following + behavior: + + Implementations decode header charsets to UTF-8. Two strings are + considered equal if their UTF-8 representations are identical. + Implementations should decode charsets represented in the forms + specified by [MIME] for both message headers and bodies. + Implementations must be capable of decoding US-ASCII, ISO-8859-1, + the ASCII subset of ISO-8859-* character sets, and UTF-8. + + If implementations fail to support the above behavior, they MUST + conform to the following: + + No two strings can be considered equal if one contains octets + greater than 127. + +2.7.3. Comparators + + In order to allow for language-independent, case-independent matches, + the match type may be coupled with a comparator name. Comparators + are described for [ACAP]; a registry is defined for ACAP, and this + specification uses that registry. + + ACAP defines multiple comparator types. Only equality types are used + in this specification. + + + + + +Showalter Standards Track [Page 12] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + All implementations MUST support the "i;octet" comparator (simply + compares octets) and the "i;ascii-casemap" comparator (which treats + uppercase and lowercase characters in the ASCII subset of UTF-8 as + the same). If left unspecified, the default is "i;ascii-casemap". + + Some comparators may not be usable with substring matches; that is, + they may only work with ":is". It is an error to try and use a + comparator with ":matches" or ":contains" that is not compatible with + it. + + A comparator is specified by the ":comparator" option with commands + that support matching. This option is followed by a string providing + the name of the comparator to be used. For convenience, the syntax + of a comparator is abbreviated to "COMPARATOR", and (repeated in + several tests) is as follows: + + Syntax: ":comparator" + + So in this example, + + Example: if header :contains :comparator "i;octet" "Subject" + "MAKE MONEY FAST" { + discard; + } + + would discard any message with subjects like "You can MAKE MONEY + FAST", but not "You can Make Money Fast", since the comparator used + is case-sensitive. + + Comparators other than i;octet and i;ascii-casemap must be declared + with require, as they are extensions. If a comparator declared with + require is not known, it is an error, and execution fails. If the + comparator is not declared with require, it is also an error, even if + the comparator is supported. (See 2.10.5.) + + Both ":matches" and ":contains" match types are compatible with the + "i;octet" and "i;ascii-casemap" comparators and may be used with + them. + + It is an error to give more than one of these arguments to a given + command. + +2.7.4. Comparisons Against Addresses + + Addresses are one of the most frequent things represented as strings. + These are structured, and being able to compare against the local- + part or the domain of an address is useful, so some tests that act + + + + +Showalter Standards Track [Page 13] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + exclusively on addresses take an additional optional argument that + specifies what the test acts on. + + These optional arguments are ":localpart", ":domain", and ":all", + which act on the local-part (left-side), the domain part (right- + side), and the whole address. + + The kind of comparison done, such as whether or not the test done is + case-insensitive, is specified as a comparator argument to the test. + + If an optional address-part is omitted, the default is ":all". + + It is an error to give more than one of these arguments to a given + command. + + For convenience, the "ADDRESS-PART" syntax element is defined here as + follows: + + Syntax: ":localpart" / ":domain" / ":all" + +2.8. Blocks + + Blocks are sets of commands enclosed within curly braces. Blocks are + supplied to commands so that the commands can implement control + commands. + + A control structure is a command that happens to take a test and a + block as one of its arguments; depending on the result of the test + supplied as another argument, it runs the code in the block some + number of times. + + With the commands supplied in this memo, there are no loops. The + control structures supplied--if, elsif, and else--run a block either + once or not at all. So there are two arguments, the test and the + block. + +2.9. Commands + + Sieve scripts are sequences of commands. Commands can take any of + the tokens above as arguments, and arguments may be either tagged or + positional arguments. Not all commands take all arguments. + + There are three kinds of commands: test commands, action commands, + and control commands. + + The simplest is an action command. An action command is an + identifier followed by zero or more arguments, terminated by a + semicolon. Action commands do not take tests or blocks as arguments. + + + +Showalter Standards Track [Page 14] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + A control command is similar, but it takes a test as an argument, and + ends with a block instead of a semicolon. + + A test command is used as part of a control command. It is used to + specify whether or not the block of code given to the control command + is executed. + +2.10. Evaluation + +2.10.1. Action Interaction + + Some actions cannot be used with other actions because the result + would be absurd. These restrictions are noted throughout this memo. + + Extension actions MUST state how they interact with actions defined + in this specification. + +2.10.2. Implicit Keep + + Previous experience with filtering systems suggests that cases tend + to be missed in scripts. To prevent errors, Sieve has an "implicit + keep". + + An implicit keep is a keep action (see 4.4) performed in absence of + any action that cancels the implicit keep. + + An implicit keep is performed if a message is not written to a + mailbox, redirected to a new address, or explicitly thrown out. That + is, if a fileinto, a keep, a redirect, or a discard is performed, an + implicit keep is not. + + Some actions may be defined to not cancel the implicit keep. These + actions may not directly affect the delivery of a message, and are + used for their side effects. None of the actions specified in this + document meet that criteria, but extension actions will. + + For instance, with any of the short messages offered above, the + following script produces no actions. + + Example: if size :over 500K { discard; } + + As a result, the implicit keep is taken. + +2.10.3. Message Uniqueness in a Mailbox + + Implementations SHOULD NOT deliver a message to the same folder more + than once, even if a script explicitly asks for a message to be + written to a mailbox twice. + + + +Showalter Standards Track [Page 15] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + The test for equality of two messages is implementation-defined. + + If a script asks for a message to be written to a mailbox twice, it + MUST NOT be treated as an error. + +2.10.4. Limits on Numbers of Actions + + Site policy MAY limit numbers of actions taken and MAY impose + restrictions on which actions can be used together. In the event + that a script hits a policy limit on the number of actions taken for + a particular message, an error occurs. + + Implementations MUST prohibit more than one reject. + + Implementations MUST allow at least one keep or one fileinto. If + fileinto is not implemented, implementations MUST allow at least one + keep. + + Implementations SHOULD prohibit reject when used with other actions. + +2.10.5. Extensions and Optional Features + + Because of the differing capabilities of many mail systems, several + features of this specification are optional. Before any of these + extensions can be executed, they must be declared with the "require" + action. + + If an extension is not enabled with "require", implementations MUST + treat it as if they did not support it at all. + + If a script does not understand an extension declared with require, + the script must not be used at all. Implementations MUST NOT execute + scripts which require unknown capability names. + + Note: The reason for this restriction is that prior experiences with + languages such as LISP and Tcl suggest that this is a workable + way of noting that a given script uses an extension. + + Experience with PostScript suggests that mechanisms that allow + a script to work around missing extensions are not used in + practice. + + Extensions which define actions MUST state how they interact with + actions discussed in the base specification. + + + + + + + +Showalter Standards Track [Page 16] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.10.6. Errors + + In any programming language, there are compile-time and run-time + errors. + + Compile-time errors are ones in syntax that are detectable if a + syntax check is done. + + Run-time errors are not detectable until the script is run. This + includes transient failures like disk full conditions, but also + includes issues like invalid combinations of actions. + + When an error occurs in a Sieve script, all processing stops. + + Implementations MAY choose to do a full parse, then evaluate the + script, then do all actions. Implementations might even go so far as + to ensure that execution is atomic (either all actions are executed + or none are executed). + + Other implementations may choose to parse and run at the same time. + Such implementations are simpler, but have issues with partial + failure (some actions happen, others don't). + + Implementations might even go so far as to ensure that scripts can + never execute an invalid set of actions (e.g., reject + fileinto) + before execution, although this could involve solving the Halting + Problem. + + This specification allows any of these approaches. Solving the + Halting Problem is considered extra credit. + + When an error happens, implementations MUST notify the user that an + error occurred, which actions (if any) were taken, and do an implicit + keep. + +2.10.7. Limits on Execution + + Implementations may limit certain constructs. However, this + specification places a lower bound on some of these limits. + + Implementations MUST support fifteen levels of nested blocks. + + Implementations MUST support fifteen levels of nested test lists. + +3. Control Commands + + Control structures are needed to allow for multiple and conditional + actions. + + + +Showalter Standards Track [Page 17] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +3.1. Control Structure If + + There are three pieces to if: "if", "elsif", and "else". Each is + actually a separate command in terms of the grammar. However, an + elsif MUST only follow an if, and an else MUST follow only either an + if or an elsif. An error occurs if these conditions are not met. + + Syntax: if + + Syntax: elsif + + Syntax: else + + The semantics are similar to those of any of the many other + programming languages these control commands appear in. When the + interpreter sees an "if", it evaluates the test associated with it. + If the test is true, it executes the block associated with it. + + If the test of the "if" is false, it evaluates the test of the first + "elsif" (if any). If the test of "elsif" is true, it runs the + elsif's block. An elsif may be followed by an elsif, in which case, + the interpreter repeats this process until it runs out of elsifs. + + When the interpreter runs out of elsifs, there may be an "else" case. + If there is, and none of the if or elsif tests were true, the + interpreter runs the else case. + + This provides a way of performing exactly one of the blocks in the + chain. + + In the following example, both Message A and B are dropped. + + Example: require "fileinto"; + if header :contains "from" "coyote" { + discard; + } elsif header :contains ["subject"] ["$$$"] { + discard; + } else { + fileinto "INBOX"; + } + + + When the script below is run over message A, it redirects the message + to acm@example.edu; message B, to postmaster@example.edu; any other + message is redirected to field@example.edu. + + + + + + +Showalter Standards Track [Page 18] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Example: if header :contains ["From"] ["coyote"] { + redirect "acm@example.edu"; + } elsif header :contains "Subject" "$$$" { + redirect "postmaster@example.edu"; + } else { + redirect "field@example.edu"; + } + + Note that this definition prohibits the "... else if ..." sequence + used by C. This is intentional, because this construct produces a + shift-reduce conflict. + +3.2. Control Structure Require + + Syntax: require + + The require action notes that a script makes use of a certain + extension. Such a declaration is required to use the extension, as + discussed in section 2.10.5. Multiple capabilities can be declared + with a single require. + + The require command, if present, MUST be used before anything other + than a require can be used. An error occurs if a require appears + after a command other than require. + + Example: require ["fileinto", "reject"]; + + Example: require "fileinto"; + require "vacation"; + +3.3. Control Structure Stop + + Syntax: stop + + The "stop" action ends all processing. If no actions have been + executed, then the keep action is taken. + +4. Action Commands + + This document supplies five actions that may be taken on a message: + keep, fileinto, redirect, reject, and discard. + + Implementations MUST support the "keep", "discard", and "redirect" + actions. + + Implementations SHOULD support "reject" and "fileinto". + + + + + +Showalter Standards Track [Page 19] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Implementations MAY limit the number of certain actions taken (see + section 2.10.4). + +4.1. Action reject + + Syntax: reject + + The optional "reject" action refuses delivery of a message by sending + back an [MDN] to the sender. It resends the message to the sender, + wrapping it in a "reject" form, noting that it was rejected by the + recipient. In the following script, message A is rejected and + returned to the sender. + + Example: if header :contains "from" "coyote@desert.example.org" { + reject "I am not taking mail from you, and I don't want + your birdseed, either!"; + } + + A reject message MUST take the form of a failure MDN as specified by + [MDN]. The human-readable portion of the message, the first + component of the MDN, contains the human readable message describing + the error, and it SHOULD contain additional text alerting the + original sender that mail was refused by a filter. This part of the + MDN might appear as follows: + + ------------------------------------------------------------ + Message was refused by recipient's mail filtering program. Reason + given was as follows: + + I am not taking mail from you, and I don't want your birdseed, + either! + ------------------------------------------------------------ + + The MDN action-value field as defined in the MDN specification MUST + be "deleted" and MUST have the MDN-sent-automatically and automatic- + action modes set. + + Because some implementations can not or will not implement the reject + command, it is optional. The capability string to be used with the + require command is "reject". + +4.2. Action fileinto + + Syntax: fileinto + + The "fileinto" action delivers the message into the specified folder. + Implementations SHOULD support fileinto, but in some environments + this may be impossible. + + + +Showalter Standards Track [Page 20] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + The capability string for use with the require command is "fileinto". + + In the following script, message A is filed into folder + "INBOX.harassment". + + Example: require "fileinto"; + if header :contains ["from"] "coyote" { + fileinto "INBOX.harassment"; + } + +4.3. Action redirect + + Syntax: redirect + + The "redirect" action is used to send the message to another user at + a supplied address, as a mail forwarding feature does. The + "redirect" action makes no changes to the message body or existing + headers, but it may add new headers. The "redirect" modifies the + envelope recipient. + + The redirect command performs an MTA-style "forward"--that is, what + you get from a .forward file using sendmail under UNIX. The address + on the SMTP envelope is replaced with the one on the redirect command + and the message is sent back out. (This is not an MUA-style forward, + which creates a new message with a different sender and message ID, + wrapping the old message in a new one.) + + A simple script can be used for redirecting all mail: + + Example: redirect "bart@example.edu"; + + Implementations SHOULD take measures to implement loop control, + possibly including adding headers to the message or counting received + headers. If an implementation detects a loop, it causes an error. + +4.4. Action keep + + Syntax: keep + + The "keep" action is whatever action is taken in lieu of all other + actions, if no filtering happens at all; generally, this simply means + to file the message into the user's main mailbox. This command + provides a way to execute this action without needing to know the + name of the user's main mailbox, providing a way to call it without + needing to understand the user's setup, or the underlying mail + system. + + + + + +Showalter Standards Track [Page 21] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + For instance, in an implementation where the IMAP server is running + scripts on behalf of the user at time of delivery, a keep command is + equivalent to a fileinto "INBOX". + + Example: if size :under 1M { keep; } else { discard; } + + Note that the above script is identical to the one below. + + Example: if not size :under 1M { discard; } + +4.5. Action discard + + Syntax: discard + + Discard is used to silently throw away the message. It does so by + simply canceling the implicit keep. If discard is used with other + actions, the other actions still happen. Discard is compatible with + all other actions. (For instance fileinto+discard is equivalent to + fileinto.) + + Discard MUST be silent; that is, it MUST NOT return a non-delivery + notification of any kind ([DSN], [MDN], or otherwise). + + In the following script, any mail from "idiot@example.edu" is thrown + out. + + Example: if header :contains ["from"] ["idiot@example.edu"] { + discard; + } + + While an important part of this language, "discard" has the potential + to create serious problems for users: Students who leave themselves + logged in to an unattended machine in a public computer lab may find + their script changed to just "discard". In order to protect users in + this situation (along with similar situations), implementations MAY + keep messages destroyed by a script for an indefinite period, and MAY + disallow scripts that throw out all mail. + +5. Test Commands + + Tests are used in conditionals to decide which part(s) of the + conditional to execute. + + Implementations MUST support these tests: "address", "allof", + "anyof", "exists", "false", "header", "not", "size", and "true". + + Implementations SHOULD support the "envelope" test. + + + + +Showalter Standards Track [Page 22] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +5.1. Test address + + Syntax: address [ADDRESS-PART] [COMPARATOR] [MATCH-TYPE] + + + The address test matches Internet addresses in structured headers + that contain addresses. It returns true if any header contains any + key in the specified part of the address, as modified by the + comparator and the match keyword. + + Like envelope and header, this test returns true if any combination + of the header-list and key-list arguments match. + + Internet email addresses [IMAIL] have the somewhat awkward + characteristic that the local-part to the left of the at-sign is + considered case sensitive, and the domain-part to the right of the + at-sign is case insensitive. The "address" command does not deal + with this itself, but provides the ADDRESS-PART argument for allowing + users to deal with it. + + The address primitive never acts on the phrase part of an email + address, nor on comments within that address. It also never acts on + group names, although it does act on the addresses within the group + construct. + + Implementations MUST restrict the address test to headers that + contain addresses, but MUST include at least From, To, Cc, Bcc, + Sender, Resent-From, Resent-To, and SHOULD include any other header + that utilizes an "address-list" structured header body. + + Example: if address :is :all "from" "tim@example.com" { + discard; + +5.2. Test allof + + Syntax: allof + + The allof test performs a logical AND on the tests supplied to it. + + Example: allof (false, false) => false + allof (false, true) => false + allof (true, true) => true + + The allof test takes as its argument a test-list. + + + + + + + +Showalter Standards Track [Page 23] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +5.3. Test anyof + + Syntax: anyof + + The anyof test performs a logical OR on the tests supplied to it. + + Example: anyof (false, false) => false + anyof (false, true) => true + anyof (true, true) => true + +5.4. Test envelope + + Syntax: envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] + + + The "envelope" test is true if the specified part of the SMTP (or + equivalent) envelope matches the specified key. + + If one of the envelope-part strings is (case insensitive) "from", + then matching occurs against the FROM address used in the SMTP MAIL + command. + + If one of the envelope-part strings is (case insensitive) "to", then + matching occurs against the TO address used in the SMTP RCPT command + that resulted in this message getting delivered to this user. Note + that only the most recent TO is available, and only the one relevant + to this user. + + The envelope-part is a string list and may contain more than one + parameter, in which case all of the strings specified in the key-list + are matched against all parts given in the envelope-part list. + + Like address and header, this test returns true if any combination of + the envelope-part and key-list arguments is true. + + All tests against envelopes MUST drop source routes. + + If the SMTP transaction involved several RCPT commands, only the data + from the RCPT command that caused delivery to this user is available + in the "to" part of the envelope. + + If a protocol other than SMTP is used for message transport, + implementations are expected to adapt this command appropriately. + + The envelope command is optional. Implementations SHOULD support it, + but the necessary information may not be available in all cases. + + + + + +Showalter Standards Track [Page 24] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Example: require "envelope"; + if envelope :all :is "from" "tim@example.com" { + discard; + } + +5.5. Test exists + + Syntax: exists + + The "exists" test is true if the headers listed in the header-names + argument exist within the message. All of the headers must exist or + the test is false. + + The following example throws out mail that doesn't have a From header + and a Date header. + + Example: if not exists ["From","Date"] { + discard; + } + +5.6. Test false + + Syntax: false + + The "false" test always evaluates to false. + +5.7. Test header + + Syntax: header [COMPARATOR] [MATCH-TYPE] + + + The "header" test evaluates to true if any header name matches any + key. The type of match is specified by the optional match argument, + which defaults to ":is" if not specified, as specified in section + 2.6. + + Like address and envelope, this test returns true if any combination + of the string-list and key-list arguments match. + + If a header listed in the header-names argument exists, it contains + the null key (""). However, if the named header is not present, it + does not contain the null key. So if a message contained the header + + X-Caffeine: C8H10N4O2 + + + + + + + +Showalter Standards Track [Page 25] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + these tests on that header evaluate as follows: + + header :is ["X-Caffeine"] [""] => false + header :contains ["X-Caffeine"] [""] => true + +5.8. Test not + + Syntax: not + + The "not" test takes some other test as an argument, and yields the + opposite result. "not false" evaluates to "true" and "not true" + evaluates to "false". + +5.9. Test size + + Syntax: size <":over" / ":under"> + + The "size" test deals with the size of a message. It takes either a + tagged argument of ":over" or ":under", followed by a number + representing the size of the message. + + If the argument is ":over", and the size of the message is greater + than the number provided, the test is true; otherwise, it is false. + + If the argument is ":under", and the size of the message is less than + the number provided, the test is true; otherwise, it is false. + + Exactly one of ":over" or ":under" must be specified, and anything + else is an error. + + The size of a message is defined to be the number of octets from the + initial header until the last character in the message body. + + Note that for a message that is exactly 4,000 octets, the message is + neither ":over" 4000 octets or ":under" 4000 octets. + +5.10. Test true + + Syntax: true + + The "true" test always evaluates to true. + +6. Extensibility + + New control structures, actions, and tests can be added to the + language. Sites must make these features known to their users; this + document does not define a way to discover the list of extensions + supported by the server. + + + +Showalter Standards Track [Page 26] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Any extensions to this language MUST define a capability string that + uniquely identifies that extension. If a new version of an extension + changes the functionality of a previously defined extension, it MUST + use a different name. + + In a situation where there is a submission protocol and an extension + advertisement mechanism aware of the details of this language, + scripts submitted can be checked against the mail server to prevent + use of an extension that the server does not support. + + Extensions MUST state how they interact with constraints defined in + section 2.10, e.g., whether they cancel the implicit keep, and which + actions they are compatible and incompatible with. + +6.1. Capability String + + Capability strings are typically short strings describing what + capabilities are supported by the server. + + Capability strings beginning with "vnd." represent vendor-defined + extensions. Such extensions are not defined by Internet standards or + RFCs, but are still registered with IANA in order to prevent + conflicts. Extensions starting with "vnd." SHOULD be followed by the + name of the vendor and product, such as "vnd.acme.rocket-sled". + + The following capability strings are defined by this document: + + envelope The string "envelope" indicates that the implementation + supports the "envelope" command. + + fileinto The string "fileinto" indicates that the implementation + supports the "fileinto" command. + + reject The string "reject" indicates that the implementation + supports the "reject" command. + + comparator- The string "comparator-elbonia" is provided if the + implementation supports the "elbonia" comparator. + Therefore, all implementations have at least the + "comparator-i;octet" and "comparator-i;ascii-casemap" + capabilities. However, these comparators may be used + without being declared with require. + + + + + + + + + +Showalter Standards Track [Page 27] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +6.2. IANA Considerations + + In order to provide a standard set of extensions, a registry is + provided by IANA. Capability names may be registered on a first- + come, first-served basis. Extensions designed for interoperable use + SHOULD be defined as standards track or IESG approved experimental + RFCs. + +6.2.1. Template for Capability Registrations + + The following template is to be used for registering new Sieve + extensions with IANA. + + To: iana@iana.org + Subject: Registration of new Sieve extension + + Capability name: + Capability keyword: + Capability arguments: + Standards Track/IESG-approved experimental RFC number: + Person and email address to contact for further information: + +6.2.2. Initial Capability Registrations + + The following are to be added to the IANA registry for Sieve + extensions as the initial contents of the capability registry. + + Capability name: fileinto + Capability keyword: fileinto + Capability arguments: fileinto + Standards Track/IESG-approved experimental RFC number: + RFC 3028 (Sieve base spec) + Person and email address to contact for further information: + Tim Showalter + tjs@mirapoint.com + + Capability name: reject + Capability keyword: reject + Capability arguments: reject + Standards Track/IESG-approved experimental RFC number: + RFC 3028 (Sieve base spec) + Person and email address to contact for further information: + Tim Showalter + tjs@mirapoint.com + + + + + + + +Showalter Standards Track [Page 28] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Capability name: envelope + Capability keyword: envelope + Capability arguments: + envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] + + Standards Track/IESG-approved experimental RFC number: + RFC 3028 (Sieve base spec) + Person and email address to contact for further information: + Tim Showalter + tjs@mirapoint.com + + Capability name: comparator-* + Capability keyword: + comparator-* (anything starting with "comparator-") + Capability arguments: (none) + Standards Track/IESG-approved experimental RFC number: + RFC 3028, Sieve, by reference of + RFC 2244, Application Configuration Access Protocol + Person and email address to contact for further information: + Tim Showalter + tjs@mirapoint.com + +6.3. Capability Transport + + As the range of mail systems that this document is intended to apply + to is quite varied, a method of advertising which capabilities an + implementation supports is difficult due to the wide range of + possible implementations. Such a mechanism, however, should have + property that the implementation can advertise the complete set of + extensions that it supports. + +7. Transmission + + The MIME type for a Sieve script is "application/sieve". + + The registration of this type for RFC 2048 requirements is as + follows: + + Subject: Registration of MIME media type application/sieve + + MIME media type name: application + MIME subtype name: sieve + Required parameters: none + Optional parameters: none + Encoding considerations: Most sieve scripts will be textual, + written in UTF-8. When non-7bit characters are used, + quoted-printable is appropriate for transport systems + that require 7bit encoding. + + + +Showalter Standards Track [Page 29] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Security considerations: Discussed in section 10 of RFC 3028. + Interoperability considerations: Discussed in section 2.10.5 + of RFC 3028. + Published specification: RFC 3028. + Applications which use this media type: sieve-enabled mail servers + Additional information: + Magic number(s): + File extension(s): .siv + Macintosh File Type Code(s): + Person & email address to contact for further information: + See the discussion list at ietf-mta-filters@imc.org. + Intended usage: + COMMON + Author/Change controller: + See Author information in RFC 3028. + +8. Parsing + + The Sieve grammar is separated into tokens and a separate grammar as + most programming languages are. + +8.1. Lexical Tokens + + Sieve scripts are encoded in UTF-8. The following assumes a valid + UTF-8 encoding; special characters in Sieve scripts are all ASCII. + + The following are tokens in Sieve: + + - identifiers + - tags + - numbers + - quoted strings + - multi-line strings + - other separators + + Blanks, horizontal tabs, CRLFs, and comments ("white space") are + ignored except as they separate tokens. Some white space is required + to separate otherwise adjacent tokens and in specific places in the + multi-line strings. + + The other separators are single individual characters, and are + mentioned explicitly in the grammar. + + The lexical structure of sieve is defined in the following BNF (as + described in [ABNF]): + + + + + + +Showalter Standards Track [Page 30] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + bracket-comment = "/*" *(CHAR-NOT-STAR / ("*" CHAR-NOT-SLASH)) "*/" + ;; No */ allowed inside a comment. + ;; (No * is allowed unless it is the last character, + ;; or unless it is followed by a character that isn't a + ;; slash.) + + CHAR-NOT-DOT = (%x01-09 / %x0b-0c / %x0e-2d / %x2f-ff) + ;; no dots, no CRLFs + + CHAR-NOT-CRLF = (%x01-09 / %x0b-0c / %x0e-ff) + + CHAR-NOT-SLASH = (%x00-57 / %x58-ff) + + CHAR-NOT-STAR = (%x00-51 / %x53-ff) + + comment = bracket-comment / hash-comment + + hash-comment = ( "#" *CHAR-NOT-CRLF CRLF ) + + identifier = (ALPHA / "_") *(ALPHA DIGIT "_") + + tag = ":" identifier + + number = 1*DIGIT [QUANTIFIER] + + QUANTIFIER = "K" / "M" / "G" + + quoted-string = DQUOTE *CHAR DQUOTE + ;; in general, \ CHAR inside a string maps to CHAR + ;; so \" maps to " and \\ maps to \ + ;; note that newlines and other characters are all allowed + ;; strings + + multi-line = "text:" *(SP / HTAB) (hash-comment / CRLF) + *(multi-line-literal / multi-line-dotstuff) + "." CRLF + multi-line-literal = [CHAR-NOT-DOT *CHAR-NOT-CRLF] CRLF + multi-line-dotstuff = "." 1*CHAR-NOT-CRLF CRLF + ;; A line containing only "." ends the multi-line. + ;; Remove a leading '.' if followed by another '.'. + + white-space = 1*(SP / CRLF / HTAB) / comment + +8.2. Grammar + + The following is the grammar of Sieve after it has been lexically + interpreted. No white space or comments appear below. The start + symbol is "start". + + + +Showalter Standards Track [Page 31] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + argument = string-list / number / tag + + arguments = *argument [test / test-list] + + block = "{" commands "}" + + command = identifier arguments ( ";" / block ) + + commands = *command + + start = commands + + string = quoted-string / multi-line + + string-list = "[" string *("," string) "]" / string ;; if + there is only a single string, the brackets are optional + + test = identifier arguments + + test-list = "(" test *("," test) ")" + +9. Extended Example + + The following is an extended example of a Sieve script. Note that it + does not make use of the implicit keep. + + # + # Example Sieve Filter + # Declare any optional features or extension used by the script + # + require ["fileinto", "reject"]; + + # + # Reject any large messages (note that the four leading dots get + # "stuffed" to three) + # + if size :over 1M + { + reject text: + Please do not send me large attachments. + Put your file on a server and send me the URL. + Thank you. + .... Fred + . + ; + stop; + } + # + + + +Showalter Standards Track [Page 32] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + # Handle messages from known mailing lists + # Move messages from IETF filter discussion list to filter folder + # + if header :is "Sender" "owner-ietf-mta-filters@imc.org" + { + fileinto "filter"; # move to "filter" folder + } + # + # Keep all messages to or from people in my company + # + elsif address :domain :is ["From", "To"] "example.com" + { + keep; # keep in "In" folder + } + + # + # Try and catch unsolicited email. If a message is not to me, + # or it contains a subject known to be spam, file it away. + # + elsif anyof (not address :all :contains + ["To", "Cc", "Bcc"] "me@example.com", + header :matches "subject" + ["*make*money*fast*", "*university*dipl*mas*"]) + { + # If message header does not contain my address, + # it's from a list. + fileinto "spam"; # move to "spam" folder + } + else + { + # Move all other (non-company) mail to "personal" + # folder. + fileinto "personal"; + } + + + + + + + + + + + + + + + + + +Showalter Standards Track [Page 33] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +10. Security Considerations + + Users must get their mail. It is imperative that whatever method + implementations use to store the user-defined filtering scripts be + secure. + + It is equally important that implementations sanity-check the user's + scripts, and not allow users to create on-demand mailbombs. For + instance, an implementation that allows a user to reject or redirect + multiple times to a single message might also allow a user to create + a mailbomb triggered by mail from a specific user. Site- or + implementation-defined limits on actions are useful for this. + + Several commands, such as "discard", "redirect", and "fileinto" allow + for actions to be taken that are potentially very dangerous. + + Implementations SHOULD take measures to prevent languages from + looping. + +11. Acknowledgments + + I am very thankful to Chris Newman for his support and his ABNF + syntax checker, to John Myers and Steve Hole for outlining the + requirements for the original drafts, to Larry Greenfield for nagging + me about the grammar and finally fixing it, to Greg Sereda for + repeatedly fixing and providing examples, to Ned Freed for fixing + everything else, to Rob Earhart for an early implementation and a + great deal of help, and to Randall Gellens for endless amounts of + proofreading. I am grateful to Carnegie Mellon University where most + of the work on this document was done. I am also indebted to all of + the readers of the ietf-mta-filters@imc.org mailing list. + +12. Author's Address + + Tim Showalter + Mirapoint, Inc. + 909 Hermosa Court + Sunnyvale, CA 94085 + + EMail: tjs@mirapoint.com + +13. References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + + + + + +Showalter Standards Track [Page 34] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + [ACAP] Newman, C. and J. G. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [BINARY-SI] "Standard IEC 60027-2: Letter symbols to be used in + electrical technology - Part 2: Telecommunications and + electronics", January 1999. + + [DSN] Moore, K. and G. Vaudreuil, "An Extensible Message Format + for Delivery Status Notifications", RFC 1894, January + 1996. + + [FLAMES] Borenstein, N, and C. Thyberg, "Power, Ease of Use, and + Cooperative Work in a Practical Multimedia Message + System", Int. J. of Man-Machine Studies, April, 1991. + Reprinted in Computer-Supported Cooperative Work and + Groupware, Saul Greenberg, editor, Harcourt Brace + Jovanovich, 1991. Reprinted in Readings in Groupware and + Computer-Supported Cooperative Work, Ronald Baecker, + editor, Morgan Kaufmann, 1993. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP] Crispin, M., "Internet Message Access Protocol - version + 4rev1", RFC 2060, December 1996. + + [IMAIL] Crocker, D., "Standard for the Format of ARPA Internet + Text Messages", STD 11, RFC 822, August 1982. + + [MIME] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [MDN] Fajman, R., "An Extensible Message Format for Message + Disposition Notifications", RFC 2298, March 1998. + + [RFC1123] Braden, R., "Requirements for Internet Hosts -- + Application and Support", STD 3, RFC 1123, November 1989. + + [SMTP] Postel, J., "Simple Mail Transfer Protocol", STD 10, RFC + 821, August 1982. + + [UTF-8] Yergeau, F., "UTF-8, a transformation format of Unicode + and ISO 10646", RFC 2044, October 1996. + + + + + + + +Showalter Standards Track [Page 35] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +14. Full Copyright Statement + + Copyright (C) The Internet Society (2001). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Showalter Standards Track [Page 36] + diff --git a/docs/rfcs/rfc3348.txt b/docs/rfcs/rfc3348.txt new file mode 100644 index 0000000..500871c --- /dev/null +++ b/docs/rfcs/rfc3348.txt @@ -0,0 +1,339 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 3348 R. Cheng +Category: Informational Microsoft + July 2002 + + + The Internet Message Action Protocol (IMAP4) + Child Mailbox Extension + +Status of this Memo + + This memo provides information for the Internet community. It does + not specify an Internet standard of any kind. Distribution of this + memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2002). All Rights Reserved. + +Abstract + + The Internet Message Action Protocol (IMAP4) CHILDREN extension + provides a mechanism for a client to efficiently determine if a + particular mailbox has children, without issuing a LIST "" * or a + LIST "" % for each mailbox. + +1. Conventions used in this document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If such lines are wrapped without a new "C:" or + "S:" label, then the wrapping is for editorial clarity and is not + part of the command. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + +2. Introduction and Overview + + Many IMAP4 [RFC-2060] clients present to the user a hierarchical view + of the mailboxes that a user has access to. Rather than initially + presenting to the user the entire mailbox hierarchy, it is often + preferable to show to the user a collapsed outline list of the + mailbox hierarchy (particularly if there is a large number of + mailboxes). The user can then expand the collapsed outline hierarchy + as needed. It is common to include within the collapsed hierarchy a + + + + + +Gahrns, et al. Informational [Page 1] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + + visual clue (such as a "+") to indicate that there are child + mailboxes under a particular mailbox. When the visual clue is + clicked the hierarchy list is expanded to show the child mailboxes. + + Several IMAP vendors implemented this proposal, and it is proposed to + document this behavior and functionality as an Informational RFC. + + There is interest in addressing the general extensibility of the IMAP + LIST command through an IMAP LIST Extension draft. Similar + functionality to the \HasChildren and \HasNoChildren flags could be + incorporated into this new LIST Extension. It is proposed that the + more general LIST Extension draft proceed on the standards track with + this proposal being relegated to informational status only. + + If the functionality of the \HasChildren and \HasNoChildren flags + were incorporated into a more general LIST extension, this would have + the advantage that a client could then have the opportunity to + request whether or not the server should return this information. + This would be an advantage over the current draft for servers where + this information is expensive to compute, since the server would only + need to compute the information when it knew that the client + requesting the information was able to consume it. + +3. Requirements + + IMAP4 servers that support this extension MUST list the keyword + CHILDREN in their CAPABILITY response. + + The CHILDREN extension defines two new attributes that MAY be + returned within a LIST response. + + \HasChildren - The presence of this attribute indicates that the + mailbox has child mailboxes. + + Servers SHOULD NOT return \HasChildren if child mailboxes exist, but + none will be displayed to the current user in a LIST response (as + should be the case where child mailboxes exist, but a client does not + have permissions to access them.) In this case, \HasNoChildren + SHOULD be used. + + In many cases, however, a server may not be able to efficiently + compute whether a user has access to all child mailboxes, or multiple + users may be accessing the same account and simultaneously changing + the mailbox hierarchy. As such a client MUST be prepared to accept + the \HasChildren attribute as a hint. That is, a mailbox MAY be + flagged with the \HasChildren attribute, but no child mailboxes will + appear in a subsequent LIST response. + + + + +Gahrns, et al. Informational [Page 2] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + + Example 3.1: + ============ + + /*** Consider a server that has the following mailbox hierarchy: + + INBOX + ITEM_1 + ITEM_1A + ITEM_2 + TOP_SECRET + + Where INBOX, ITEM_1 and ITEM_2 are top level mailboxes. ITEM_1A is a + child mailbox of ITEM_1 and TOP_SECRET is a child mailbox of ITEM_2 + that the currently logged on user does NOT have access to. + + Note that in this case, the server is not able to efficiently compute + access rights to child mailboxes and responds with a \HasChildren + attribute for mailbox ITEM_2, even though ITEM_2/TOP_SECRET does not + appear in the list response. ***/ + + C: A001 LIST "" * + S: * LIST (\HasNoChildren) "/" INBOX + S: * LIST (\HasChildren) "/" ITEM_1 + S: * LIST (\HasNoChildren) "/" ITEM_1/ITEM_1A + S: * LIST (\HasChildren) "/" ITEM_2 + S: A001 OK LIST Completed + + \HasNoChildren - The presence of this attribute indicates that the + mailbox has NO child mailboxes that are accessible to the currently + authenticated user. If a mailbox has the \Noinferiors attribute, the + \HasNoChildren attribute is redundant and SHOULD be omitted in the + LIST response. + + In some instances a server that supports the CHILDREN extension MAY + NOT be able to determine whether a mailbox has children. For example + it may have difficulty determining whether there are child mailboxes + when LISTing mailboxes while operating in a particular namespace. + + In these cases, a server MAY exclude both the \HasChildren and + \HasNoChildren attributes in the LIST response. As such, a client + can not make any assumptions about whether a mailbox has children + based upon the absence of a single attribute. + + It is an error for the server to return both a \HasChildren and a + \HasNoChildren attribute in a LIST response. + + + + + + +Gahrns, et al. Informational [Page 3] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + + It is an error for the server to return both a \HasChildren and a + \NoInferiors attribute in a LIST response. + + Note: the \HasNoChildren attribute should not be confused with the + IMAP4 [RFC-2060] defined attribute \Noinferiors which indicates that + no child mailboxes exist now and none can be created in the future. + + The \HasChildren and \HasNoChildren attributes might not be returned + in response to a LSUB response. Many servers maintain a simple + mailbox subscription list that is not updated when the underlying + mailbox structure is changed. A client MUST NOT assume that + hierarchy information will be maintained in the subscription list. + + RLIST is a command defined in [RFC-2193] that includes in a LIST + response mailboxes that are accessible only via referral. That is, a + client must explicitly issue an RLIST command to see a list of these + mailboxes. Thus in the case where a mailbox has child mailboxes that + are available only via referral, the mailboxes would appear as + \HasNoChildren in response to the LIST command, and \HasChildren in + response to the RLIST command. + +5. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. + + Two new mailbox attributes are defined as flag_extensions to the + IMAP4 mailbox_list response: + + HasChildren = "\HasChildren" + + HasNoChildren = "\HasNoChildren" + +6. Security Considerations + + This extension provides a client a more efficient means of + determining whether a particular mailbox has children. If a mailbox + has children, but the currently authenticated user does not have + access to any of them, the server SHOULD respond with a + \HasNoChildren attribute. In many cases, however, a server may not + be able to efficiently compute whether a user has access to all child + mailboxes. If such a server responds with a \HasChildren attribute, + when in fact the currently authenticated user does not have access to + any child mailboxes, potentially more information is conveyed about + the mailbox than intended. A server designed with such levels of + security in mind SHOULD NOT attach the \HasChildren attribute to a + mailbox unless the server is certain that the user has access to at + least one of the child mailboxes. + + + +Gahrns, et al. Informational [Page 4] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + +7. References + + [RFC-2060] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [RFC-2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC-2234] Crocker, D. and P. Overell, Editors, "Augmented BNF for + Syntax Specifications: ABNF", RFC 2234, November 1997. + + [RFC-2193] Gahrns, M., "IMAP4 Mailbox Referrals", RFC 2193, September + 1997. + +8. Acknowledgments + + The authors would like to thank the participants of several IMC Mail + Connect events for their input when this idea was originally + presented and refined. + +9. Author's Address + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98052 + Phone: (425) 936-9833 + EMail: mikega@microsoft.com + + Raymond Cheng + Microsoft + One Microsoft Way + Redmond, WA, 98052 + Phone: (425) 703-4913 + EMail: raych@microsoft.com + + + + + + + + + + + + + + + + +Gahrns, et al. Informational [Page 5] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + +10. Full Copyright Statement + + Copyright (C) The Internet Society (2002). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Gahrns, et al. Informational [Page 6] + diff --git a/docs/rfcs/rfc3501.txt b/docs/rfcs/rfc3501.txt new file mode 100644 index 0000000..5ab48e1 --- /dev/null +++ b/docs/rfcs/rfc3501.txt @@ -0,0 +1,6051 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 3501 University of Washington +Obsoletes: 2060 March 2003 +Category: Standards Track + + + INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1 + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + The Internet Message Access Protocol, Version 4rev1 (IMAP4rev1) + allows a client to access and manipulate electronic mail messages on + a server. IMAP4rev1 permits manipulation of mailboxes (remote + message folders) in a way that is functionally equivalent to local + folders. IMAP4rev1 also provides the capability for an offline + client to resynchronize with the server. + + IMAP4rev1 includes operations for creating, deleting, and renaming + mailboxes, checking for new messages, permanently removing messages, + setting and clearing flags, RFC 2822 and RFC 2045 parsing, searching, + and selective fetching of message attributes, texts, and portions + thereof. Messages in IMAP4rev1 are accessed by the use of numbers. + These numbers are either message sequence numbers or unique + identifiers. + + IMAP4rev1 supports a single server. A mechanism for accessing + configuration information to support multiple IMAP4rev1 servers is + discussed in RFC 2244. + + IMAP4rev1 does not specify a means of posting mail; this function is + handled by a mail transfer protocol such as RFC 2821. + + + + + + + + +Crispin Standards Track [Page 1] + +RFC 3501 IMAPv4 March 2003 + + +Table of Contents + + IMAP4rev1 Protocol Specification ................................ 4 + 1. How to Read This Document ............................... 4 + 1.1. Organization of This Document ........................... 4 + 1.2. Conventions Used in This Document ....................... 4 + 1.3. Special Notes to Implementors ........................... 5 + 2. Protocol Overview ....................................... 6 + 2.1. Link Level .............................................. 6 + 2.2. Commands and Responses .................................. 6 + 2.2.1. Client Protocol Sender and Server Protocol Receiver ..... 6 + 2.2.2. Server Protocol Sender and Client Protocol Receiver ..... 7 + 2.3. Message Attributes ...................................... 8 + 2.3.1. Message Numbers ......................................... 8 + 2.3.1.1. Unique Identifier (UID) Message Attribute ....... 8 + 2.3.1.2. Message Sequence Number Message Attribute ....... 10 + 2.3.2. Flags Message Attribute ................................. 11 + 2.3.3. Internal Date Message Attribute ......................... 12 + 2.3.4. [RFC-2822] Size Message Attribute ....................... 12 + 2.3.5. Envelope Structure Message Attribute .................... 12 + 2.3.6. Body Structure Message Attribute ........................ 12 + 2.4. Message Texts ........................................... 13 + 3. State and Flow Diagram .................................. 13 + 3.1. Not Authenticated State ................................. 13 + 3.2. Authenticated State ..................................... 13 + 3.3. Selected State .......................................... 13 + 3.4. Logout State ............................................ 14 + 4. Data Formats ............................................ 16 + 4.1. Atom .................................................... 16 + 4.2. Number .................................................. 16 + 4.3. String .................................................. 16 + 4.3.1. 8-bit and Binary Strings ................................ 17 + 4.4. Parenthesized List ...................................... 17 + 4.5. NIL ..................................................... 17 + 5. Operational Considerations .............................. 18 + 5.1. Mailbox Naming .......................................... 18 + 5.1.1. Mailbox Hierarchy Naming ................................ 19 + 5.1.2. Mailbox Namespace Naming Convention ..................... 19 + 5.1.3. Mailbox International Naming Convention ................. 19 + 5.2. Mailbox Size and Message Status Updates ................. 21 + 5.3. Response when no Command in Progress .................... 21 + 5.4. Autologout Timer ........................................ 22 + 5.5. Multiple Commands in Progress ........................... 22 + 6. Client Commands ........................................ 23 + 6.1. Client Commands - Any State ............................ 24 + 6.1.1. CAPABILITY Command ..................................... 24 + 6.1.2. NOOP Command ........................................... 25 + 6.1.3. LOGOUT Command ......................................... 26 + + + +Crispin Standards Track [Page 2] + +RFC 3501 IMAPv4 March 2003 + + + 6.2. Client Commands - Not Authenticated State .............. 26 + 6.2.1. STARTTLS Command ....................................... 27 + 6.2.2. AUTHENTICATE Command ................................... 28 + 6.2.3. LOGIN Command .......................................... 30 + 6.3. Client Commands - Authenticated State .................. 31 + 6.3.1. SELECT Command ......................................... 32 + 6.3.2. EXAMINE Command ........................................ 34 + 6.3.3. CREATE Command ......................................... 34 + 6.3.4. DELETE Command ......................................... 35 + 6.3.5. RENAME Command ......................................... 37 + 6.3.6. SUBSCRIBE Command ...................................... 39 + 6.3.7. UNSUBSCRIBE Command .................................... 39 + 6.3.8. LIST Command ........................................... 40 + 6.3.9. LSUB Command ........................................... 43 + 6.3.10. STATUS Command ......................................... 44 + 6.3.11. APPEND Command ......................................... 46 + 6.4. Client Commands - Selected State ....................... 47 + 6.4.1. CHECK Command .......................................... 47 + 6.4.2. CLOSE Command .......................................... 48 + 6.4.3. EXPUNGE Command ........................................ 49 + 6.4.4. SEARCH Command ......................................... 49 + 6.4.5. FETCH Command .......................................... 54 + 6.4.6. STORE Command .......................................... 58 + 6.4.7. COPY Command ........................................... 59 + 6.4.8. UID Command ............................................ 60 + 6.5. Client Commands - Experimental/Expansion ............... 62 + 6.5.1. X Command ........................................ 62 + 7. Server Responses ....................................... 62 + 7.1. Server Responses - Status Responses .................... 63 + 7.1.1. OK Response ............................................ 65 + 7.1.2. NO Response ............................................ 66 + 7.1.3. BAD Response ........................................... 66 + 7.1.4. PREAUTH Response ....................................... 67 + 7.1.5. BYE Response ........................................... 67 + 7.2. Server Responses - Server and Mailbox Status ........... 68 + 7.2.1. CAPABILITY Response .................................... 68 + 7.2.2. LIST Response .......................................... 69 + 7.2.3. LSUB Response .......................................... 70 + 7.2.4 STATUS Response ........................................ 70 + 7.2.5. SEARCH Response ........................................ 71 + 7.2.6. FLAGS Response ......................................... 71 + 7.3. Server Responses - Mailbox Size ........................ 71 + 7.3.1. EXISTS Response ........................................ 71 + 7.3.2. RECENT Response ........................................ 72 + 7.4. Server Responses - Message Status ...................... 72 + 7.4.1. EXPUNGE Response ....................................... 72 + 7.4.2. FETCH Response ......................................... 73 + 7.5. Server Responses - Command Continuation Request ........ 79 + + + +Crispin Standards Track [Page 3] + +RFC 3501 IMAPv4 March 2003 + + + 8. Sample IMAP4rev1 connection ............................ 80 + 9. Formal Syntax .......................................... 81 + 10. Author's Note .......................................... 92 + 11. Security Considerations ................................ 92 + 11.1. STARTTLS Security Considerations ....................... 92 + 11.2. Other Security Considerations .......................... 93 + 12. IANA Considerations .................................... 94 + Appendices ..................................................... 95 + A. References ............................................. 95 + B. Changes from RFC 2060 .................................. 97 + C. Key Word Index ......................................... 103 + Author's Address ............................................... 107 + Full Copyright Statement ....................................... 108 + +IMAP4rev1 Protocol Specification + +1. How to Read This Document + +1.1. Organization of This Document + + This document is written from the point of view of the implementor of + an IMAP4rev1 client or server. Beyond the protocol overview in + section 2, it is not optimized for someone trying to understand the + operation of the protocol. The material in sections 3 through 5 + provides the general context and definitions with which IMAP4rev1 + operates. + + Sections 6, 7, and 9 describe the IMAP commands, responses, and + syntax, respectively. The relationships among these are such that it + is almost impossible to understand any of them separately. In + particular, do not attempt to deduce command syntax from the command + section alone; instead refer to the Formal Syntax section. + +1.2. Conventions Used in This Document + + "Conventions" are basic principles or procedures. Document + conventions are noted in this section. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to + be interpreted as described in [KEYWORDS]. + + The word "can" (not "may") is used to refer to a possible + circumstance or situation, as opposed to an optional facility of the + protocol. + + + +Crispin Standards Track [Page 4] + +RFC 3501 IMAPv4 March 2003 + + + "User" is used to refer to a human user, whereas "client" refers to + the software being run by the user. + + "Connection" refers to the entire sequence of client/server + interaction from the initial establishment of the network connection + until its termination. + + "Session" refers to the sequence of client/server interaction from + the time that a mailbox is selected (SELECT or EXAMINE command) until + the time that selection ends (SELECT or EXAMINE of another mailbox, + CLOSE command, or connection termination). + + Characters are 7-bit US-ASCII unless otherwise specified. Other + character sets are indicated using a "CHARSET", as described in + [MIME-IMT] and defined in [CHARSET]. CHARSETs have important + additional semantics in addition to defining character set; refer to + these documents for more detail. + + There are several protocol conventions in IMAP. These refer to + aspects of the specification which are not strictly part of the IMAP + protocol, but reflect generally-accepted practice. Implementations + need to be aware of these conventions, and avoid conflicts whether or + not they implement the convention. For example, "&" may not be used + as a hierarchy delimiter since it conflicts with the Mailbox + International Naming Convention, and other uses of "&" in mailbox + names are impacted as well. + +1.3. Special Notes to Implementors + + Implementors of the IMAP protocol are strongly encouraged to read the + IMAP implementation recommendations document [IMAP-IMPLEMENTATION] in + conjunction with this document, to help understand the intricacies of + this protocol and how best to build an interoperable product. + + IMAP4rev1 is designed to be upwards compatible from the [IMAP2] and + unpublished IMAP2bis protocols. IMAP4rev1 is largely compatible with + the IMAP4 protocol described in RFC 1730; the exception being in + certain facilities added in RFC 1730 that proved problematic and were + subsequently removed. In the course of the evolution of IMAP4rev1, + some aspects in the earlier protocols have become obsolete. Obsolete + commands, responses, and data formats which an IMAP4rev1 + implementation can encounter when used with an earlier implementation + are described in [IMAP-OBSOLETE]. + + Other compatibility issues with IMAP2bis, the most common variant of + the earlier protocol, are discussed in [IMAP-COMPAT]. A full + discussion of compatibility issues with rare (and presumed extinct) + + + + +Crispin Standards Track [Page 5] + +RFC 3501 IMAPv4 March 2003 + + + variants of [IMAP2] is in [IMAP-HISTORICAL]; this document is + primarily of historical interest. + + IMAP was originally developed for the older [RFC-822] standard, and + as a consequence several fetch items in IMAP incorporate "RFC822" in + their name. With the exception of RFC822.SIZE, there are more modern + replacements; for example, the modern version of RFC822.HEADER is + BODY.PEEK[HEADER]. In all cases, "RFC822" should be interpreted as a + reference to the updated [RFC-2822] standard. + +2. Protocol Overview + +2.1. Link Level + + The IMAP4rev1 protocol assumes a reliable data stream such as that + provided by TCP. When TCP is used, an IMAP4rev1 server listens on + port 143. + +2.2. Commands and Responses + + An IMAP4rev1 connection consists of the establishment of a + client/server network connection, an initial greeting from the + server, and client/server interactions. These client/server + interactions consist of a client command, server data, and a server + completion result response. + + All interactions transmitted by client and server are in the form of + lines, that is, strings that end with a CRLF. The protocol receiver + of an IMAP4rev1 client or server is either reading a line, or is + reading a sequence of octets with a known count followed by a line. + +2.2.1. Client Protocol Sender and Server Protocol Receiver + + The client command begins an operation. Each client command is + prefixed with an identifier (typically a short alphanumeric string, + e.g., A0001, A0002, etc.) called a "tag". A different tag is + generated by the client for each command. + + Clients MUST follow the syntax outlined in this specification + strictly. It is a syntax error to send a command with missing or + extraneous spaces or arguments. + + There are two cases in which a line from the client does not + represent a complete command. In one case, a command argument is + quoted with an octet count (see the description of literal in String + under Data Formats); in the other case, the command arguments require + server feedback (see the AUTHENTICATE command). In either case, the + + + + +Crispin Standards Track [Page 6] + +RFC 3501 IMAPv4 March 2003 + + + server sends a command continuation request response if it is ready + for the octets (if appropriate) and the remainder of the command. + This response is prefixed with the token "+". + + Note: If instead, the server detected an error in the + command, it sends a BAD completion response with a tag + matching the command (as described below) to reject the + command and prevent the client from sending any more of the + command. + + It is also possible for the server to send a completion + response for some other command (if multiple commands are + in progress), or untagged data. In either case, the + command continuation request is still pending; the client + takes the appropriate action for the response, and reads + another response from the server. In all cases, the client + MUST send a complete command (including receiving all + command continuation request responses and command + continuations for the command) before initiating a new + command. + + The protocol receiver of an IMAP4rev1 server reads a command line + from the client, parses the command and its arguments, and transmits + server data and a server command completion result response. + +2.2.2. Server Protocol Sender and Client Protocol Receiver + + Data transmitted by the server to the client and status responses + that do not indicate command completion are prefixed with the token + "*", and are called untagged responses. + + Server data MAY be sent as a result of a client command, or MAY be + sent unilaterally by the server. There is no syntactic difference + between server data that resulted from a specific command and server + data that were sent unilaterally. + + The server completion result response indicates the success or + failure of the operation. It is tagged with the same tag as the + client command which began the operation. Thus, if more than one + command is in progress, the tag in a server completion response + identifies the command to which the response applies. There are + three possible server completion responses: OK (indicating success), + NO (indicating failure), or BAD (indicating a protocol error such as + unrecognized command or command syntax error). + + Servers SHOULD enforce the syntax outlined in this specification + strictly. Any client command with a protocol syntax error, including + (but not limited to) missing or extraneous spaces or arguments, + + + +Crispin Standards Track [Page 7] + +RFC 3501 IMAPv4 March 2003 + + + SHOULD be rejected, and the client given a BAD server completion + response. + + The protocol receiver of an IMAP4rev1 client reads a response line + from the server. It then takes action on the response based upon the + first token of the response, which can be a tag, a "*", or a "+". + + A client MUST be prepared to accept any server response at all times. + This includes server data that was not requested. Server data SHOULD + be recorded, so that the client can reference its recorded copy + rather than sending a command to the server to request the data. In + the case of certain server data, the data MUST be recorded. + + This topic is discussed in greater detail in the Server Responses + section. + +2.3. Message Attributes + + In addition to message text, each message has several attributes + associated with it. These attributes can be retrieved individually + or in conjunction with other attributes or message texts. + +2.3.1. Message Numbers + + Messages in IMAP4rev1 are accessed by one of two numbers; the unique + identifier or the message sequence number. + + +2.3.1.1. Unique Identifier (UID) Message Attribute + + A 32-bit value assigned to each message, which when used with the + unique identifier validity value (see below) forms a 64-bit value + that MUST NOT refer to any other message in the mailbox or any + subsequent mailbox with the same name forever. Unique identifiers + are assigned in a strictly ascending fashion in the mailbox; as each + message is added to the mailbox it is assigned a higher UID than the + message(s) which were added previously. Unlike message sequence + numbers, unique identifiers are not necessarily contiguous. + + The unique identifier of a message MUST NOT change during the + session, and SHOULD NOT change between sessions. Any change of + unique identifiers between sessions MUST be detectable using the + UIDVALIDITY mechanism discussed below. Persistent unique identifiers + are required for a client to resynchronize its state from a previous + session with the server (e.g., disconnected or offline access + clients); this is discussed further in [IMAP-DISC]. + + + + + +Crispin Standards Track [Page 8] + +RFC 3501 IMAPv4 March 2003 + + + Associated with every mailbox are two values which aid in unique + identifier handling: the next unique identifier value and the unique + identifier validity value. + + The next unique identifier value is the predicted value that will be + assigned to a new message in the mailbox. Unless the unique + identifier validity also changes (see below), the next unique + identifier value MUST have the following two characteristics. First, + the next unique identifier value MUST NOT change unless new messages + are added to the mailbox; and second, the next unique identifier + value MUST change whenever new messages are added to the mailbox, + even if those new messages are subsequently expunged. + + Note: The next unique identifier value is intended to + provide a means for a client to determine whether any + messages have been delivered to the mailbox since the + previous time it checked this value. It is not intended to + provide any guarantee that any message will have this + unique identifier. A client can only assume, at the time + that it obtains the next unique identifier value, that + messages arriving after that time will have a UID greater + than or equal to that value. + + The unique identifier validity value is sent in a UIDVALIDITY + response code in an OK untagged response at mailbox selection time. + If unique identifiers from an earlier session fail to persist in this + session, the unique identifier validity value MUST be greater than + the one used in the earlier session. + + Note: Ideally, unique identifiers SHOULD persist at all + times. Although this specification recognizes that failure + to persist can be unavoidable in certain server + environments, it STRONGLY ENCOURAGES message store + implementation techniques that avoid this problem. For + example: + + 1) Unique identifiers MUST be strictly ascending in the + mailbox at all times. If the physical message store is + re-ordered by a non-IMAP agent, this requires that the + unique identifiers in the mailbox be regenerated, since + the former unique identifiers are no longer strictly + ascending as a result of the re-ordering. + + 2) If the message store has no mechanism to store unique + identifiers, it must regenerate unique identifiers at + each session, and each session must have a unique + UIDVALIDITY value. + + + + +Crispin Standards Track [Page 9] + +RFC 3501 IMAPv4 March 2003 + + + 3) If the mailbox is deleted and a new mailbox with the + same name is created at a later date, the server must + either keep track of unique identifiers from the + previous instance of the mailbox, or it must assign a + new UIDVALIDITY value to the new instance of the + mailbox. A good UIDVALIDITY value to use in this case + is a 32-bit representation of the creation date/time of + the mailbox. It is alright to use a constant such as + 1, but only if it guaranteed that unique identifiers + will never be reused, even in the case of a mailbox + being deleted (or renamed) and a new mailbox by the + same name created at some future time. + + 4) The combination of mailbox name, UIDVALIDITY, and UID + must refer to a single immutable message on that server + forever. In particular, the internal date, [RFC-2822] + size, envelope, body structure, and message texts + (RFC822, RFC822.HEADER, RFC822.TEXT, and all BODY[...] + fetch data items) must never change. This does not + include message numbers, nor does it include attributes + that can be set by a STORE command (e.g., FLAGS). + + +2.3.1.2. Message Sequence Number Message Attribute + + A relative position from 1 to the number of messages in the mailbox. + This position MUST be ordered by ascending unique identifier. As + each new message is added, it is assigned a message sequence number + that is 1 higher than the number of messages in the mailbox before + that new message was added. + + Message sequence numbers can be reassigned during the session. For + example, when a message is permanently removed (expunged) from the + mailbox, the message sequence number for all subsequent messages is + decremented. The number of messages in the mailbox is also + decremented. Similarly, a new message can be assigned a message + sequence number that was once held by some other message prior to an + expunge. + + In addition to accessing messages by relative position in the + mailbox, message sequence numbers can be used in mathematical + calculations. For example, if an untagged "11 EXISTS" is received, + and previously an untagged "8 EXISTS" was received, three new + messages have arrived with message sequence numbers of 9, 10, and 11. + Another example, if message 287 in a 523 message mailbox has UID + 12345, there are exactly 286 messages which have lesser UIDs and 236 + messages which have greater UIDs. + + + + +Crispin Standards Track [Page 10] + +RFC 3501 IMAPv4 March 2003 + + +2.3.2. Flags Message Attribute + + A list of zero or more named tokens associated with the message. A + flag is set by its addition to this list, and is cleared by its + removal. There are two types of flags in IMAP4rev1. A flag of + either type can be permanent or session-only. + + A system flag is a flag name that is pre-defined in this + specification. All system flags begin with "\". Certain system + flags (\Deleted and \Seen) have special semantics described + elsewhere. The currently-defined system flags are: + + \Seen + Message has been read + + \Answered + Message has been answered + + \Flagged + Message is "flagged" for urgent/special attention + + \Deleted + Message is "deleted" for removal by later EXPUNGE + + \Draft + Message has not completed composition (marked as a draft). + + \Recent + Message is "recently" arrived in this mailbox. This session + is the first session to have been notified about this + message; if the session is read-write, subsequent sessions + will not see \Recent set for this message. This flag can not + be altered by the client. + + If it is not possible to determine whether or not this + session is the first session to be notified about a message, + then that message SHOULD be considered recent. + + If multiple connections have the same mailbox selected + simultaneously, it is undefined which of these connections + will see newly-arrived messages with \Recent set and which + will see it without \Recent set. + + A keyword is defined by the server implementation. Keywords do not + begin with "\". Servers MAY permit the client to define new keywords + in the mailbox (see the description of the PERMANENTFLAGS response + code for more information). + + + + +Crispin Standards Track [Page 11] + +RFC 3501 IMAPv4 March 2003 + + + A flag can be permanent or session-only on a per-flag basis. + Permanent flags are those which the client can add or remove from the + message flags permanently; that is, concurrent and subsequent + sessions will see any change in permanent flags. Changes to session + flags are valid only in that session. + + Note: The \Recent system flag is a special case of a + session flag. \Recent can not be used as an argument in a + STORE or APPEND command, and thus can not be changed at + all. + +2.3.3. Internal Date Message Attribute + + The internal date and time of the message on the server. This + is not the date and time in the [RFC-2822] header, but rather a + date and time which reflects when the message was received. In + the case of messages delivered via [SMTP], this SHOULD be the + date and time of final delivery of the message as defined by + [SMTP]. In the case of messages delivered by the IMAP4rev1 COPY + command, this SHOULD be the internal date and time of the source + message. In the case of messages delivered by the IMAP4rev1 + APPEND command, this SHOULD be the date and time as specified in + the APPEND command description. All other cases are + implementation defined. + +2.3.4. [RFC-2822] Size Message Attribute + + The number of octets in the message, as expressed in [RFC-2822] + format. + +2.3.5. Envelope Structure Message Attribute + + A parsed representation of the [RFC-2822] header of the message. + Note that the IMAP Envelope structure is not the same as an + [SMTP] envelope. + +2.3.6. Body Structure Message Attribute + + A parsed representation of the [MIME-IMB] body structure + information of the message. + + + + + + + + + + + +Crispin Standards Track [Page 12] + +RFC 3501 IMAPv4 March 2003 + + +2.4. Message Texts + + In addition to being able to fetch the full [RFC-2822] text of a + message, IMAP4rev1 permits the fetching of portions of the full + message text. Specifically, it is possible to fetch the + [RFC-2822] message header, [RFC-2822] message body, a [MIME-IMB] + body part, or a [MIME-IMB] header. + +3. State and Flow Diagram + + Once the connection between client and server is established, an + IMAP4rev1 connection is in one of four states. The initial + state is identified in the server greeting. Most commands are + only valid in certain states. It is a protocol error for the + client to attempt a command while the connection is in an + inappropriate state, and the server will respond with a BAD or + NO (depending upon server implementation) command completion + result. + +3.1. Not Authenticated State + + In the not authenticated state, the client MUST supply + authentication credentials before most commands will be + permitted. This state is entered when a connection starts + unless the connection has been pre-authenticated. + +3.2. Authenticated State + + In the authenticated state, the client is authenticated and MUST + select a mailbox to access before commands that affect messages + will be permitted. This state is entered when a + pre-authenticated connection starts, when acceptable + authentication credentials have been provided, after an error in + selecting a mailbox, or after a successful CLOSE command. + +3.3. Selected State + + In a selected state, a mailbox has been selected to access. + This state is entered when a mailbox has been successfully + selected. + + + + + + + + + + + +Crispin Standards Track [Page 13] + +RFC 3501 IMAPv4 March 2003 + + +3.4. Logout State + + In the logout state, the connection is being terminated. This + state can be entered as a result of a client request (via the + LOGOUT command) or by unilateral action on the part of either + the client or server. + + If the client requests the logout state, the server MUST send an + untagged BYE response and a tagged OK response to the LOGOUT + command before the server closes the connection; and the client + MUST read the tagged OK response to the LOGOUT command before + the client closes the connection. + + A server MUST NOT unilaterally close the connection without + sending an untagged BYE response that contains the reason for + having done so. A client SHOULD NOT unilaterally close the + connection, and instead SHOULD issue a LOGOUT command. If the + server detects that the client has unilaterally closed the + connection, the server MAY omit the untagged BYE response and + simply close its connection. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 14] + +RFC 3501 IMAPv4 March 2003 + + + +----------------------+ + |connection established| + +----------------------+ + || + \/ + +--------------------------------------+ + | server greeting | + +--------------------------------------+ + || (1) || (2) || (3) + \/ || || + +-----------------+ || || + |Not Authenticated| || || + +-----------------+ || || + || (7) || (4) || || + || \/ \/ || + || +----------------+ || + || | Authenticated |<=++ || + || +----------------+ || || + || || (7) || (5) || (6) || + || || \/ || || + || || +--------+ || || + || || |Selected|==++ || + || || +--------+ || + || || || (7) || + \/ \/ \/ \/ + +--------------------------------------+ + | Logout | + +--------------------------------------+ + || + \/ + +-------------------------------+ + |both sides close the connection| + +-------------------------------+ + + (1) connection without pre-authentication (OK greeting) + (2) pre-authenticated connection (PREAUTH greeting) + (3) rejected connection (BYE greeting) + (4) successful LOGIN or AUTHENTICATE command + (5) successful SELECT or EXAMINE command + (6) CLOSE command, or failed SELECT or EXAMINE command + (7) LOGOUT command, server shutdown, or connection closed + + + + + + + + + + +Crispin Standards Track [Page 15] + +RFC 3501 IMAPv4 March 2003 + + +4. Data Formats + + IMAP4rev1 uses textual commands and responses. Data in + IMAP4rev1 can be in one of several forms: atom, number, string, + parenthesized list, or NIL. Note that a particular data item + may take more than one form; for example, a data item defined as + using "astring" syntax may be either an atom or a string. + +4.1. Atom + + An atom consists of one or more non-special characters. + +4.2. Number + + A number consists of one or more digit characters, and + represents a numeric value. + +4.3. String + + A string is in one of two forms: either literal or quoted + string. The literal form is the general form of string. The + quoted string form is an alternative that avoids the overhead of + processing a literal at the cost of limitations of characters + which may be used. + + A literal is a sequence of zero or more octets (including CR and + LF), prefix-quoted with an octet count in the form of an open + brace ("{"), the number of octets, close brace ("}"), and CRLF. + In the case of literals transmitted from server to client, the + CRLF is immediately followed by the octet data. In the case of + literals transmitted from client to server, the client MUST wait + to receive a command continuation request (described later in + this document) before sending the octet data (and the remainder + of the command). + + A quoted string is a sequence of zero or more 7-bit characters, + excluding CR and LF, with double quote (<">) characters at each + end. + + The empty string is represented as either "" (a quoted string + with zero characters between double quotes) or as {0} followed + by CRLF (a literal with an octet count of 0). + + Note: Even if the octet count is 0, a client transmitting a + literal MUST wait to receive a command continuation request. + + + + + + +Crispin Standards Track [Page 16] + +RFC 3501 IMAPv4 March 2003 + + +4.3.1. 8-bit and Binary Strings + + 8-bit textual and binary mail is supported through the use of a + [MIME-IMB] content transfer encoding. IMAP4rev1 implementations MAY + transmit 8-bit or multi-octet characters in literals, but SHOULD do + so only when the [CHARSET] is identified. + + Although a BINARY body encoding is defined, unencoded binary strings + are not permitted. A "binary string" is any string with NUL + characters. Implementations MUST encode binary data into a textual + form, such as BASE64, before transmitting the data. A string with an + excessive amount of CTL characters MAY also be considered to be + binary. + +4.4. Parenthesized List + + Data structures are represented as a "parenthesized list"; a sequence + of data items, delimited by space, and bounded at each end by + parentheses. A parenthesized list can contain other parenthesized + lists, using multiple levels of parentheses to indicate nesting. + + The empty list is represented as () -- a parenthesized list with no + members. + +4.5. NIL + + The special form "NIL" represents the non-existence of a particular + data item that is represented as a string or parenthesized list, as + distinct from the empty string "" or the empty parenthesized list (). + + Note: NIL is never used for any data item which takes the + form of an atom. For example, a mailbox name of "NIL" is a + mailbox named NIL as opposed to a non-existent mailbox + name. This is because mailbox uses "astring" syntax which + is an atom or a string. Conversely, an addr-name of NIL is + a non-existent personal name, because addr-name uses + "nstring" syntax which is NIL or a string, but never an + atom. + + + + + + + + + + + + + +Crispin Standards Track [Page 17] + +RFC 3501 IMAPv4 March 2003 + + +5. Operational Considerations + + The following rules are listed here to ensure that all IMAP4rev1 + implementations interoperate properly. + +5.1. Mailbox Naming + + Mailbox names are 7-bit. Client implementations MUST NOT attempt to + create 8-bit mailbox names, and SHOULD interpret any 8-bit mailbox + names returned by LIST or LSUB as UTF-8. Server implementations + SHOULD prohibit the creation of 8-bit mailbox names, and SHOULD NOT + return 8-bit mailbox names in LIST or LSUB. See section 5.1.3 for + more information on how to represent non-ASCII mailbox names. + + Note: 8-bit mailbox names were undefined in earlier + versions of this protocol. Some sites used a local 8-bit + character set to represent non-ASCII mailbox names. Such + usage is not interoperable, and is now formally deprecated. + + The case-insensitive mailbox name INBOX is a special name reserved to + mean "the primary mailbox for this user on this server". The + interpretation of all other names is implementation-dependent. + + In particular, this specification takes no position on case + sensitivity in non-INBOX mailbox names. Some server implementations + are fully case-sensitive; others preserve case of a newly-created + name but otherwise are case-insensitive; and yet others coerce names + to a particular case. Client implementations MUST interact with any + of these. If a server implementation interprets non-INBOX mailbox + names as case-insensitive, it MUST treat names using the + international naming convention specially as described in section + 5.1.3. + + There are certain client considerations when creating a new mailbox + name: + + 1) Any character which is one of the atom-specials (see the Formal + Syntax) will require that the mailbox name be represented as a + quoted string or literal. + + 2) CTL and other non-graphic characters are difficult to represent + in a user interface and are best avoided. + + 3) Although the list-wildcard characters ("%" and "*") are valid + in a mailbox name, it is difficult to use such mailbox names + with the LIST and LSUB commands due to the conflict with + wildcard interpretation. + + + + +Crispin Standards Track [Page 18] + +RFC 3501 IMAPv4 March 2003 + + + 4) Usually, a character (determined by the server implementation) + is reserved to delimit levels of hierarchy. + + 5) Two characters, "#" and "&", have meanings by convention, and + should be avoided except when used in that convention. + +5.1.1. Mailbox Hierarchy Naming + + If it is desired to export hierarchical mailbox names, mailbox names + MUST be left-to-right hierarchical using a single character to + separate levels of hierarchy. The same hierarchy separator character + is used for all levels of hierarchy within a single name. + +5.1.2. Mailbox Namespace Naming Convention + + By convention, the first hierarchical element of any mailbox name + which begins with "#" identifies the "namespace" of the remainder of + the name. This makes it possible to disambiguate between different + types of mailbox stores, each of which have their own namespaces. + + For example, implementations which offer access to USENET + newsgroups MAY use the "#news" namespace to partition the + USENET newsgroup namespace from that of other mailboxes. + Thus, the comp.mail.misc newsgroup would have a mailbox + name of "#news.comp.mail.misc", and the name + "comp.mail.misc" can refer to a different object (e.g., a + user's private mailbox). + +5.1.3. Mailbox International Naming Convention + + By convention, international mailbox names in IMAP4rev1 are specified + using a modified version of the UTF-7 encoding described in [UTF-7]. + Modified UTF-7 may also be usable in servers that implement an + earlier version of this protocol. + + In modified UTF-7, printable US-ASCII characters, except for "&", + represent themselves; that is, characters with octet values 0x20-0x25 + and 0x27-0x7e. The character "&" (0x26) is represented by the + two-octet sequence "&-". + + All other characters (octet values 0x00-0x1f and 0x7f-0xff) are + represented in modified BASE64, with a further modification from + [UTF-7] that "," is used instead of "/". Modified BASE64 MUST NOT be + used to represent any printing US-ASCII character which can represent + itself. + + + + + + +Crispin Standards Track [Page 19] + +RFC 3501 IMAPv4 March 2003 + + + "&" is used to shift to modified BASE64 and "-" to shift back to + US-ASCII. There is no implicit shift from BASE64 to US-ASCII, and + null shifts ("-&" while in BASE64; note that "&-" while in US-ASCII + means "&") are not permitted. However, all names start in US-ASCII, + and MUST end in US-ASCII; that is, a name that ends with a non-ASCII + ISO-10646 character MUST end with a "-"). + + The purpose of these modifications is to correct the following + problems with UTF-7: + + 1) UTF-7 uses the "+" character for shifting; this conflicts with + the common use of "+" in mailbox names, in particular USENET + newsgroup names. + + 2) UTF-7's encoding is BASE64 which uses the "/" character; this + conflicts with the use of "/" as a popular hierarchy delimiter. + + 3) UTF-7 prohibits the unencoded usage of "\"; this conflicts with + the use of "\" as a popular hierarchy delimiter. + + 4) UTF-7 prohibits the unencoded usage of "~"; this conflicts with + the use of "~" in some servers as a home directory indicator. + + 5) UTF-7 permits multiple alternate forms to represent the same + string; in particular, printable US-ASCII characters can be + represented in encoded form. + + Although modified UTF-7 is a convention, it establishes certain + requirements on server handling of any mailbox name with an + embedded "&" character. In particular, server implementations + MUST preserve the exact form of the modified BASE64 portion of a + modified UTF-7 name and treat that text as case-sensitive, even if + names are otherwise case-insensitive or case-folded. + + Server implementations SHOULD verify that any mailbox name with an + embedded "&" character, used as an argument to CREATE, is: in the + correctly modified UTF-7 syntax, has no superfluous shifts, and + has no encoding in modified BASE64 of any printing US-ASCII + character which can represent itself. However, client + implementations MUST NOT depend upon the server doing this, and + SHOULD NOT attempt to create a mailbox name with an embedded "&" + character unless it complies with the modified UTF-7 syntax. + + Server implementations which export a mail store that does not + follow the modified UTF-7 convention MUST convert to modified + UTF-7 any mailbox name that contains either non-ASCII characters + or the "&" character. + + + + +Crispin Standards Track [Page 20] + +RFC 3501 IMAPv4 March 2003 + + + For example, here is a mailbox name which mixes English, + Chinese, and Japanese text: + ~peter/mail/&U,BTFw-/&ZeVnLIqe- + + For example, the string "&Jjo!" is not a valid mailbox + name because it does not contain a shift to US-ASCII + before the "!". The correct form is "&Jjo-!". The + string "&U,BTFw-&ZeVnLIqe-" is not permitted because it + contains a superfluous shift. The correct form is + "&U,BTF2XlZyyKng-". + +5.2. Mailbox Size and Message Status Updates + + At any time, a server can send data that the client did not request. + Sometimes, such behavior is REQUIRED. For example, agents other than + the server MAY add messages to the mailbox (e.g., new message + delivery), change the flags of the messages in the mailbox (e.g., + simultaneous access to the same mailbox by multiple agents), or even + remove messages from the mailbox. A server MUST send mailbox size + updates automatically if a mailbox size change is observed during the + processing of a command. A server SHOULD send message flag updates + automatically, without requiring the client to request such updates + explicitly. + + Special rules exist for server notification of a client about the + removal of messages to prevent synchronization errors; see the + description of the EXPUNGE response for more detail. In particular, + it is NOT permitted to send an EXISTS response that would reduce the + number of messages in the mailbox; only the EXPUNGE response can do + this. + + Regardless of what implementation decisions a client makes on + remembering data from the server, a client implementation MUST record + mailbox size updates. It MUST NOT assume that any command after the + initial mailbox selection will return the size of the mailbox. + +5.3. Response when no Command in Progress + + Server implementations are permitted to send an untagged response + (except for EXPUNGE) while there is no command in progress. Server + implementations that send such responses MUST deal with flow control + considerations. Specifically, they MUST either (1) verify that the + size of the data does not exceed the underlying transport's available + window size, or (2) use non-blocking writes. + + + + + + + +Crispin Standards Track [Page 21] + +RFC 3501 IMAPv4 March 2003 + + +5.4. Autologout Timer + + If a server has an inactivity autologout timer, the duration of that + timer MUST be at least 30 minutes. The receipt of ANY command from + the client during that interval SHOULD suffice to reset the + autologout timer. + +5.5. Multiple Commands in Progress + + The client MAY send another command without waiting for the + completion result response of a command, subject to ambiguity rules + (see below) and flow control constraints on the underlying data + stream. Similarly, a server MAY begin processing another command + before processing the current command to completion, subject to + ambiguity rules. However, any command continuation request responses + and command continuations MUST be negotiated before any subsequent + command is initiated. + + The exception is if an ambiguity would result because of a command + that would affect the results of other commands. Clients MUST NOT + send multiple commands without waiting if an ambiguity would result. + If the server detects a possible ambiguity, it MUST execute commands + to completion in the order given by the client. + + The most obvious example of ambiguity is when a command would affect + the results of another command, e.g., a FETCH of a message's flags + and a STORE of that same message's flags. + + A non-obvious ambiguity occurs with commands that permit an untagged + EXPUNGE response (commands other than FETCH, STORE, and SEARCH), + since an untagged EXPUNGE response can invalidate sequence numbers in + a subsequent command. This is not a problem for FETCH, STORE, or + SEARCH commands because servers are prohibited from sending EXPUNGE + responses while any of those commands are in progress. Therefore, if + the client sends any command other than FETCH, STORE, or SEARCH, it + MUST wait for the completion result response before sending a command + with message sequence numbers. + + Note: UID FETCH, UID STORE, and UID SEARCH are different + commands from FETCH, STORE, and SEARCH. If the client + sends a UID command, it must wait for a completion result + response before sending a command with message sequence + numbers. + + + + + + + + +Crispin Standards Track [Page 22] + +RFC 3501 IMAPv4 March 2003 + + + For example, the following non-waiting command sequences are invalid: + + FETCH + NOOP + STORE + STORE + COPY + FETCH + COPY + COPY + CHECK + FETCH + + The following are examples of valid non-waiting command sequences: + + FETCH + STORE + SEARCH + CHECK + STORE + COPY + EXPUNGE + + UID SEARCH + UID SEARCH may be valid or invalid as a non-waiting + command sequence, depending upon whether or not the second UID + SEARCH contains message sequence numbers. + +6. Client Commands + + IMAP4rev1 commands are described in this section. Commands are + organized by the state in which the command is permitted. Commands + which are permitted in multiple states are listed in the minimum + permitted state (for example, commands valid in authenticated and + selected state are listed in the authenticated state commands). + + Command arguments, identified by "Arguments:" in the command + descriptions below, are described by function, not by syntax. The + precise syntax of command arguments is described in the Formal Syntax + section. + + Some commands cause specific server responses to be returned; these + are identified by "Responses:" in the command descriptions below. + See the response descriptions in the Responses section for + information on these responses, and the Formal Syntax section for the + precise syntax of these responses. It is possible for server data to + be transmitted as a result of any command. Thus, commands that do + not specifically require server data specify "no specific responses + for this command" instead of "none". + + The "Result:" in the command description refers to the possible + tagged status responses to a command, and any special interpretation + of these status responses. + + The state of a connection is only changed by successful commands + which are documented as changing state. A rejected command (BAD + response) never changes the state of the connection or of the + selected mailbox. A failed command (NO response) generally does not + change the state of the connection or of the selected mailbox; the + exception being the SELECT and EXAMINE commands. + + + +Crispin Standards Track [Page 23] + +RFC 3501 IMAPv4 March 2003 + + +6.1. Client Commands - Any State + + The following commands are valid in any state: CAPABILITY, NOOP, and + LOGOUT. + +6.1.1. CAPABILITY Command + + Arguments: none + + Responses: REQUIRED untagged response: CAPABILITY + + Result: OK - capability completed + BAD - command unknown or arguments invalid + + The CAPABILITY command requests a listing of capabilities that the + server supports. The server MUST send a single untagged + CAPABILITY response with "IMAP4rev1" as one of the listed + capabilities before the (tagged) OK response. + + A capability name which begins with "AUTH=" indicates that the + server supports that particular authentication mechanism. All + such names are, by definition, part of this specification. For + example, the authorization capability for an experimental + "blurdybloop" authenticator would be "AUTH=XBLURDYBLOOP" and not + "XAUTH=BLURDYBLOOP" or "XAUTH=XBLURDYBLOOP". + + Other capability names refer to extensions, revisions, or + amendments to this specification. See the documentation of the + CAPABILITY response for additional information. No capabilities, + beyond the base IMAP4rev1 set defined in this specification, are + enabled without explicit client action to invoke the capability. + + Client and server implementations MUST implement the STARTTLS, + LOGINDISABLED, and AUTH=PLAIN (described in [IMAP-TLS]) + capabilities. See the Security Considerations section for + important information. + + See the section entitled "Client Commands - + Experimental/Expansion" for information about the form of site or + implementation-specific capabilities. + + + + + + + + + + + +Crispin Standards Track [Page 24] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: abcd CAPABILITY + S: * CAPABILITY IMAP4rev1 STARTTLS AUTH=GSSAPI + LOGINDISABLED + S: abcd OK CAPABILITY completed + C: efgh STARTTLS + S: efgh OK STARTLS completed + + C: ijkl CAPABILITY + S: * CAPABILITY IMAP4rev1 AUTH=GSSAPI AUTH=PLAIN + S: ijkl OK CAPABILITY completed + + +6.1.2. NOOP Command + + Arguments: none + + Responses: no specific responses for this command (but see below) + + Result: OK - noop completed + BAD - command unknown or arguments invalid + + The NOOP command always succeeds. It does nothing. + + Since any command can return a status update as untagged data, the + NOOP command can be used as a periodic poll for new messages or + message status updates during a period of inactivity (this is the + preferred method to do this). The NOOP command can also be used + to reset any inactivity autologout timer on the server. + + Example: C: a002 NOOP + S: a002 OK NOOP completed + . . . + C: a047 NOOP + S: * 22 EXPUNGE + S: * 23 EXISTS + S: * 3 RECENT + S: * 14 FETCH (FLAGS (\Seen \Deleted)) + S: a047 OK NOOP completed + + + + + + + + + + + + + +Crispin Standards Track [Page 25] + +RFC 3501 IMAPv4 March 2003 + + +6.1.3. LOGOUT Command + + Arguments: none + + Responses: REQUIRED untagged response: BYE + + Result: OK - logout completed + BAD - command unknown or arguments invalid + + The LOGOUT command informs the server that the client is done with + the connection. The server MUST send a BYE untagged response + before the (tagged) OK response, and then close the network + connection. + + Example: C: A023 LOGOUT + S: * BYE IMAP4rev1 Server logging out + S: A023 OK LOGOUT completed + (Server and client then close the connection) + +6.2. Client Commands - Not Authenticated State + + In the not authenticated state, the AUTHENTICATE or LOGIN command + establishes authentication and enters the authenticated state. The + AUTHENTICATE command provides a general mechanism for a variety of + authentication techniques, privacy protection, and integrity + checking; whereas the LOGIN command uses a traditional user name and + plaintext password pair and has no means of establishing privacy + protection or integrity checking. + + The STARTTLS command is an alternate form of establishing session + privacy protection and integrity checking, but does not establish + authentication or enter the authenticated state. + + Server implementations MAY allow access to certain mailboxes without + establishing authentication. This can be done by means of the + ANONYMOUS [SASL] authenticator described in [ANONYMOUS]. An older + convention is a LOGIN command using the userid "anonymous"; in this + case, a password is required although the server may choose to accept + any password. The restrictions placed on anonymous users are + implementation-dependent. + + Once authenticated (including as anonymous), it is not possible to + re-enter not authenticated state. + + + + + + + + +Crispin Standards Track [Page 26] + +RFC 3501 IMAPv4 March 2003 + + + In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), + the following commands are valid in the not authenticated state: + STARTTLS, AUTHENTICATE and LOGIN. See the Security Considerations + section for important information about these commands. + +6.2.1. STARTTLS Command + + Arguments: none + + Responses: no specific response for this command + + Result: OK - starttls completed, begin TLS negotiation + BAD - command unknown or arguments invalid + + A [TLS] negotiation begins immediately after the CRLF at the end + of the tagged OK response from the server. Once a client issues a + STARTTLS command, it MUST NOT issue further commands until a + server response is seen and the [TLS] negotiation is complete. + + The server remains in the non-authenticated state, even if client + credentials are supplied during the [TLS] negotiation. This does + not preclude an authentication mechanism such as EXTERNAL (defined + in [SASL]) from using client identity determined by the [TLS] + negotiation. + + Once [TLS] has been started, the client MUST discard cached + information about server capabilities and SHOULD re-issue the + CAPABILITY command. This is necessary to protect against man-in- + the-middle attacks which alter the capabilities list prior to + STARTTLS. The server MAY advertise different capabilities after + STARTTLS. + + Example: C: a001 CAPABILITY + S: * CAPABILITY IMAP4rev1 STARTTLS LOGINDISABLED + S: a001 OK CAPABILITY completed + C: a002 STARTTLS + S: a002 OK Begin TLS negotiation now + + C: a003 CAPABILITY + S: * CAPABILITY IMAP4rev1 AUTH=PLAIN + S: a003 OK CAPABILITY completed + C: a004 LOGIN joe password + S: a004 OK LOGIN completed + + + + + + + + +Crispin Standards Track [Page 27] + +RFC 3501 IMAPv4 March 2003 + + +6.2.2. AUTHENTICATE Command + + Arguments: authentication mechanism name + + Responses: continuation data can be requested + + Result: OK - authenticate completed, now in authenticated state + NO - authenticate failure: unsupported authentication + mechanism, credentials rejected + BAD - command unknown or arguments invalid, + authentication exchange cancelled + + The AUTHENTICATE command indicates a [SASL] authentication + mechanism to the server. If the server supports the requested + authentication mechanism, it performs an authentication protocol + exchange to authenticate and identify the client. It MAY also + negotiate an OPTIONAL security layer for subsequent protocol + interactions. If the requested authentication mechanism is not + supported, the server SHOULD reject the AUTHENTICATE command by + sending a tagged NO response. + + The AUTHENTICATE command does not support the optional "initial + response" feature of [SASL]. Section 5.1 of [SASL] specifies how + to handle an authentication mechanism which uses an initial + response. + + The service name specified by this protocol's profile of [SASL] is + "imap". + + The authentication protocol exchange consists of a series of + server challenges and client responses that are specific to the + authentication mechanism. A server challenge consists of a + command continuation request response with the "+" token followed + by a BASE64 encoded string. The client response consists of a + single line consisting of a BASE64 encoded string. If the client + wishes to cancel an authentication exchange, it issues a line + consisting of a single "*". If the server receives such a + response, it MUST reject the AUTHENTICATE command by sending a + tagged BAD response. + + If a security layer is negotiated through the [SASL] + authentication exchange, it takes effect immediately following the + CRLF that concludes the authentication exchange for the client, + and the CRLF of the tagged OK response for the server. + + While client and server implementations MUST implement the + AUTHENTICATE command itself, it is not required to implement any + authentication mechanisms other than the PLAIN mechanism described + + + +Crispin Standards Track [Page 28] + +RFC 3501 IMAPv4 March 2003 + + + in [IMAP-TLS]. Also, an authentication mechanism is not required + to support any security layers. + + Note: a server implementation MUST implement a + configuration in which it does NOT permit any plaintext + password mechanisms, unless either the STARTTLS command + has been negotiated or some other mechanism that + protects the session from password snooping has been + provided. Server sites SHOULD NOT use any configuration + which permits a plaintext password mechanism without + such a protection mechanism against password snooping. + Client and server implementations SHOULD implement + additional [SASL] mechanisms that do not use plaintext + passwords, such the GSSAPI mechanism described in [SASL] + and/or the [DIGEST-MD5] mechanism. + + Servers and clients can support multiple authentication + mechanisms. The server SHOULD list its supported authentication + mechanisms in the response to the CAPABILITY command so that the + client knows which authentication mechanisms to use. + + A server MAY include a CAPABILITY response code in the tagged OK + response of a successful AUTHENTICATE command in order to send + capabilities automatically. It is unnecessary for a client to + send a separate CAPABILITY command if it recognizes these + automatic capabilities. This should only be done if a security + layer was not negotiated by the AUTHENTICATE command, because the + tagged OK response as part of an AUTHENTICATE command is not + protected by encryption/integrity checking. [SASL] requires the + client to re-issue a CAPABILITY command in this case. + + If an AUTHENTICATE command fails with a NO response, the client + MAY try another authentication mechanism by issuing another + AUTHENTICATE command. It MAY also attempt to authenticate by + using the LOGIN command (see section 6.2.3 for more detail). In + other words, the client MAY request authentication types in + decreasing order of preference, with the LOGIN command as a last + resort. + + The authorization identity passed from the client to the server + during the authentication exchange is interpreted by the server as + the user name whose privileges the client is requesting. + + + + + + + + + +Crispin Standards Track [Page 29] + +RFC 3501 IMAPv4 March 2003 + + + Example: S: * OK IMAP4rev1 Server + C: A001 AUTHENTICATE GSSAPI + S: + + C: YIIB+wYJKoZIhvcSAQICAQBuggHqMIIB5qADAgEFoQMCAQ6iBw + MFACAAAACjggEmYYIBIjCCAR6gAwIBBaESGxB1Lndhc2hpbmd0 + b24uZWR1oi0wK6ADAgEDoSQwIhsEaW1hcBsac2hpdmFtcy5jYW + Mud2FzaGluZ3Rvbi5lZHWjgdMwgdCgAwIBAaEDAgEDooHDBIHA + cS1GSa5b+fXnPZNmXB9SjL8Ollj2SKyb+3S0iXMljen/jNkpJX + AleKTz6BQPzj8duz8EtoOuNfKgweViyn/9B9bccy1uuAE2HI0y + C/PHXNNU9ZrBziJ8Lm0tTNc98kUpjXnHZhsMcz5Mx2GR6dGknb + I0iaGcRerMUsWOuBmKKKRmVMMdR9T3EZdpqsBd7jZCNMWotjhi + vd5zovQlFqQ2Wjc2+y46vKP/iXxWIuQJuDiisyXF0Y8+5GTpAL + pHDc1/pIGmMIGjoAMCAQGigZsEgZg2on5mSuxoDHEA1w9bcW9n + FdFxDKpdrQhVGVRDIzcCMCTzvUboqb5KjY1NJKJsfjRQiBYBdE + NKfzK+g5DlV8nrw81uOcP8NOQCLR5XkoMHC0Dr/80ziQzbNqhx + O6652Npft0LQwJvenwDI13YxpwOdMXzkWZN/XrEqOWp6GCgXTB + vCyLWLlWnbaUkZdEYbKHBPjd8t/1x5Yg== + S: + YGgGCSqGSIb3EgECAgIAb1kwV6ADAgEFoQMCAQ+iSzBJoAMC + AQGiQgRAtHTEuOP2BXb9sBYFR4SJlDZxmg39IxmRBOhXRKdDA0 + uHTCOT9Bq3OsUTXUlk0CsFLoa8j+gvGDlgHuqzWHPSQg== + C: + S: + YDMGCSqGSIb3EgECAgIBAAD/////6jcyG4GE3KkTzBeBiVHe + ceP2CWY0SR0fAQAgAAQEBAQ= + C: YDMGCSqGSIb3EgECAgIBAAD/////3LQBHXTpFfZgrejpLlLImP + wkhbfa2QteAQAgAG1yYwE= + S: A001 OK GSSAPI authentication successful + + Note: The line breaks within server challenges and client + responses are for editorial clarity and are not in real + authenticators. + + +6.2.3. LOGIN Command + + Arguments: user name + password + + Responses: no specific responses for this command + + Result: OK - login completed, now in authenticated state + NO - login failure: user name or password rejected + BAD - command unknown or arguments invalid + + The LOGIN command identifies the client to the server and carries + the plaintext password authenticating this user. + + + + + + +Crispin Standards Track [Page 30] + +RFC 3501 IMAPv4 March 2003 + + + A server MAY include a CAPABILITY response code in the tagged OK + response to a successful LOGIN command in order to send + capabilities automatically. It is unnecessary for a client to + send a separate CAPABILITY command if it recognizes these + automatic capabilities. + + Example: C: a001 LOGIN SMITH SESAME + S: a001 OK LOGIN completed + + Note: Use of the LOGIN command over an insecure network + (such as the Internet) is a security risk, because anyone + monitoring network traffic can obtain plaintext passwords. + The LOGIN command SHOULD NOT be used except as a last + resort, and it is recommended that client implementations + have a means to disable any automatic use of the LOGIN + command. + + Unless either the STARTTLS command has been negotiated or + some other mechanism that protects the session from + password snooping has been provided, a server + implementation MUST implement a configuration in which it + advertises the LOGINDISABLED capability and does NOT permit + the LOGIN command. Server sites SHOULD NOT use any + configuration which permits the LOGIN command without such + a protection mechanism against password snooping. A client + implementation MUST NOT send a LOGIN command if the + LOGINDISABLED capability is advertised. + +6.3. Client Commands - Authenticated State + + In the authenticated state, commands that manipulate mailboxes as + atomic entities are permitted. Of these commands, the SELECT and + EXAMINE commands will select a mailbox for access and enter the + selected state. + + In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), + the following commands are valid in the authenticated state: SELECT, + EXAMINE, CREATE, DELETE, RENAME, SUBSCRIBE, UNSUBSCRIBE, LIST, LSUB, + STATUS, and APPEND. + + + + + + + + + + + + +Crispin Standards Track [Page 31] + +RFC 3501 IMAPv4 March 2003 + + +6.3.1. SELECT Command + + Arguments: mailbox name + + Responses: REQUIRED untagged responses: FLAGS, EXISTS, RECENT + REQUIRED OK untagged responses: UNSEEN, PERMANENTFLAGS, + UIDNEXT, UIDVALIDITY + + Result: OK - select completed, now in selected state + NO - select failure, now in authenticated state: no + such mailbox, can't access mailbox + BAD - command unknown or arguments invalid + + The SELECT command selects a mailbox so that messages in the + mailbox can be accessed. Before returning an OK to the client, + the server MUST send the following untagged data to the client. + Note that earlier versions of this protocol only required the + FLAGS, EXISTS, and RECENT untagged data; consequently, client + implementations SHOULD implement default behavior for missing data + as discussed with the individual item. + + FLAGS Defined flags in the mailbox. See the description + of the FLAGS response for more detail. + + EXISTS The number of messages in the mailbox. See the + description of the EXISTS response for more detail. + + RECENT The number of messages with the \Recent flag set. + See the description of the RECENT response for more + detail. + + OK [UNSEEN ] + The message sequence number of the first unseen + message in the mailbox. If this is missing, the + client can not make any assumptions about the first + unseen message in the mailbox, and needs to issue a + SEARCH command if it wants to find it. + + OK [PERMANENTFLAGS ()] + A list of message flags that the client can change + permanently. If this is missing, the client should + assume that all flags can be changed permanently. + + OK [UIDNEXT ] + The next unique identifier value. Refer to section + 2.3.1.1 for more information. If this is missing, + the client can not make any assumptions about the + next unique identifier value. + + + +Crispin Standards Track [Page 32] + +RFC 3501 IMAPv4 March 2003 + + + OK [UIDVALIDITY ] + The unique identifier validity value. Refer to + section 2.3.1.1 for more information. If this is + missing, the server does not support unique + identifiers. + + Only one mailbox can be selected at a time in a connection; + simultaneous access to multiple mailboxes requires multiple + connections. The SELECT command automatically deselects any + currently selected mailbox before attempting the new selection. + Consequently, if a mailbox is selected and a SELECT command that + fails is attempted, no mailbox is selected. + + If the client is permitted to modify the mailbox, the server + SHOULD prefix the text of the tagged OK response with the + "[READ-WRITE]" response code. + + If the client is not permitted to modify the mailbox but is + permitted read access, the mailbox is selected as read-only, and + the server MUST prefix the text of the tagged OK response to + SELECT with the "[READ-ONLY]" response code. Read-only access + through SELECT differs from the EXAMINE command in that certain + read-only mailboxes MAY permit the change of permanent state on a + per-user (as opposed to global) basis. Netnews messages marked in + a server-based .newsrc file are an example of such per-user + permanent state that can be modified with read-only mailboxes. + + Example: C: A142 SELECT INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: A142 OK [READ-WRITE] SELECT completed + + + + + + + + + + + + + + + +Crispin Standards Track [Page 33] + +RFC 3501 IMAPv4 March 2003 + + +6.3.2. EXAMINE Command + + Arguments: mailbox name + + Responses: REQUIRED untagged responses: FLAGS, EXISTS, RECENT + REQUIRED OK untagged responses: UNSEEN, PERMANENTFLAGS, + UIDNEXT, UIDVALIDITY + + Result: OK - examine completed, now in selected state + NO - examine failure, now in authenticated state: no + such mailbox, can't access mailbox + BAD - command unknown or arguments invalid + + The EXAMINE command is identical to SELECT and returns the same + output; however, the selected mailbox is identified as read-only. + No changes to the permanent state of the mailbox, including + per-user state, are permitted; in particular, EXAMINE MUST NOT + cause messages to lose the \Recent flag. + + The text of the tagged OK response to the EXAMINE command MUST + begin with the "[READ-ONLY]" response code. + + Example: C: A932 EXAMINE blurdybloop + S: * 17 EXISTS + S: * 2 RECENT + S: * OK [UNSEEN 8] Message 8 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS ()] No permanent flags permitted + S: A932 OK [READ-ONLY] EXAMINE completed + + +6.3.3. CREATE Command + + Arguments: mailbox name + + Responses: no specific responses for this command + + Result: OK - create completed + NO - create failure: can't create mailbox with that name + BAD - command unknown or arguments invalid + + The CREATE command creates a mailbox with the given name. An OK + response is returned only if a new mailbox with that name has been + created. It is an error to attempt to create INBOX or a mailbox + with a name that refers to an extant mailbox. Any error in + creation will return a tagged NO response. + + + +Crispin Standards Track [Page 34] + +RFC 3501 IMAPv4 March 2003 + + + If the mailbox name is suffixed with the server's hierarchy + separator character (as returned from the server by a LIST + command), this is a declaration that the client intends to create + mailbox names under this name in the hierarchy. Server + implementations that do not require this declaration MUST ignore + the declaration. In any case, the name created is without the + trailing hierarchy delimiter. + + If the server's hierarchy separator character appears elsewhere in + the name, the server SHOULD create any superior hierarchical names + that are needed for the CREATE command to be successfully + completed. In other words, an attempt to create "foo/bar/zap" on + a server in which "/" is the hierarchy separator character SHOULD + create foo/ and foo/bar/ if they do not already exist. + + If a new mailbox is created with the same name as a mailbox which + was deleted, its unique identifiers MUST be greater than any + unique identifiers used in the previous incarnation of the mailbox + UNLESS the new incarnation has a different unique identifier + validity value. See the description of the UID command for more + detail. + + Example: C: A003 CREATE owatagusiam/ + S: A003 OK CREATE completed + C: A004 CREATE owatagusiam/blurdybloop + S: A004 OK CREATE completed + + Note: The interpretation of this example depends on whether + "/" was returned as the hierarchy separator from LIST. If + "/" is the hierarchy separator, a new level of hierarchy + named "owatagusiam" with a member called "blurdybloop" is + created. Otherwise, two mailboxes at the same hierarchy + level are created. + + +6.3.4. DELETE Command + + Arguments: mailbox name + + Responses: no specific responses for this command + + Result: OK - delete completed + NO - delete failure: can't delete mailbox with that name + BAD - command unknown or arguments invalid + + + + + + + +Crispin Standards Track [Page 35] + +RFC 3501 IMAPv4 March 2003 + + + The DELETE command permanently removes the mailbox with the given + name. A tagged OK response is returned only if the mailbox has + been deleted. It is an error to attempt to delete INBOX or a + mailbox name that does not exist. + + The DELETE command MUST NOT remove inferior hierarchical names. + For example, if a mailbox "foo" has an inferior "foo.bar" + (assuming "." is the hierarchy delimiter character), removing + "foo" MUST NOT remove "foo.bar". It is an error to attempt to + delete a name that has inferior hierarchical names and also has + the \Noselect mailbox name attribute (see the description of the + LIST response for more details). + + It is permitted to delete a name that has inferior hierarchical + names and does not have the \Noselect mailbox name attribute. In + this case, all messages in that mailbox are removed, and the name + will acquire the \Noselect mailbox name attribute. + + The value of the highest-used unique identifier of the deleted + mailbox MUST be preserved so that a new mailbox created with the + same name will not reuse the identifiers of the former + incarnation, UNLESS the new incarnation has a different unique + identifier validity value. See the description of the UID command + for more detail. + + Examples: C: A682 LIST "" * + S: * LIST () "/" blurdybloop + S: * LIST (\Noselect) "/" foo + S: * LIST () "/" foo/bar + S: A682 OK LIST completed + C: A683 DELETE blurdybloop + S: A683 OK DELETE completed + C: A684 DELETE foo + S: A684 NO Name "foo" has inferior hierarchical names + C: A685 DELETE foo/bar + S: A685 OK DELETE Completed + C: A686 LIST "" * + S: * LIST (\Noselect) "/" foo + S: A686 OK LIST completed + C: A687 DELETE foo + S: A687 OK DELETE Completed + + + + + + + + + + +Crispin Standards Track [Page 36] + +RFC 3501 IMAPv4 March 2003 + + + C: A82 LIST "" * + S: * LIST () "." blurdybloop + S: * LIST () "." foo + S: * LIST () "." foo.bar + S: A82 OK LIST completed + C: A83 DELETE blurdybloop + S: A83 OK DELETE completed + C: A84 DELETE foo + S: A84 OK DELETE Completed + C: A85 LIST "" * + S: * LIST () "." foo.bar + S: A85 OK LIST completed + C: A86 LIST "" % + S: * LIST (\Noselect) "." foo + S: A86 OK LIST completed + + +6.3.5. RENAME Command + + Arguments: existing mailbox name + new mailbox name + + Responses: no specific responses for this command + + Result: OK - rename completed + NO - rename failure: can't rename mailbox with that name, + can't rename to mailbox with that name + BAD - command unknown or arguments invalid + + The RENAME command changes the name of a mailbox. A tagged OK + response is returned only if the mailbox has been renamed. It is + an error to attempt to rename from a mailbox name that does not + exist or to a mailbox name that already exists. Any error in + renaming will return a tagged NO response. + + If the name has inferior hierarchical names, then the inferior + hierarchical names MUST also be renamed. For example, a rename of + "foo" to "zap" will rename "foo/bar" (assuming "/" is the + hierarchy delimiter character) to "zap/bar". + + If the server's hierarchy separator character appears in the name, + the server SHOULD create any superior hierarchical names that are + needed for the RENAME command to complete successfully. In other + words, an attempt to rename "foo/bar/zap" to baz/rag/zowie on a + server in which "/" is the hierarchy separator character SHOULD + create baz/ and baz/rag/ if they do not already exist. + + + + + +Crispin Standards Track [Page 37] + +RFC 3501 IMAPv4 March 2003 + + + The value of the highest-used unique identifier of the old mailbox + name MUST be preserved so that a new mailbox created with the same + name will not reuse the identifiers of the former incarnation, + UNLESS the new incarnation has a different unique identifier + validity value. See the description of the UID command for more + detail. + + Renaming INBOX is permitted, and has special behavior. It moves + all messages in INBOX to a new mailbox with the given name, + leaving INBOX empty. If the server implementation supports + inferior hierarchical names of INBOX, these are unaffected by a + rename of INBOX. + + Examples: C: A682 LIST "" * + S: * LIST () "/" blurdybloop + S: * LIST (\Noselect) "/" foo + S: * LIST () "/" foo/bar + S: A682 OK LIST completed + C: A683 RENAME blurdybloop sarasoop + S: A683 OK RENAME completed + C: A684 RENAME foo zowie + S: A684 OK RENAME Completed + C: A685 LIST "" * + S: * LIST () "/" sarasoop + S: * LIST (\Noselect) "/" zowie + S: * LIST () "/" zowie/bar + S: A685 OK LIST completed + + C: Z432 LIST "" * + S: * LIST () "." INBOX + S: * LIST () "." INBOX.bar + S: Z432 OK LIST completed + C: Z433 RENAME INBOX old-mail + S: Z433 OK RENAME completed + C: Z434 LIST "" * + S: * LIST () "." INBOX + S: * LIST () "." INBOX.bar + S: * LIST () "." old-mail + S: Z434 OK LIST completed + + + + + + + + + + + + +Crispin Standards Track [Page 38] + +RFC 3501 IMAPv4 March 2003 + + +6.3.6. SUBSCRIBE Command + + Arguments: mailbox + + Responses: no specific responses for this command + + Result: OK - subscribe completed + NO - subscribe failure: can't subscribe to that name + BAD - command unknown or arguments invalid + + The SUBSCRIBE command adds the specified mailbox name to the + server's set of "active" or "subscribed" mailboxes as returned by + the LSUB command. This command returns a tagged OK response only + if the subscription is successful. + + A server MAY validate the mailbox argument to SUBSCRIBE to verify + that it exists. However, it MUST NOT unilaterally remove an + existing mailbox name from the subscription list even if a mailbox + by that name no longer exists. + + Note: This requirement is because a server site can + choose to routinely remove a mailbox with a well-known + name (e.g., "system-alerts") after its contents expire, + with the intention of recreating it when new contents + are appropriate. + + + Example: C: A002 SUBSCRIBE #news.comp.mail.mime + S: A002 OK SUBSCRIBE completed + + +6.3.7. UNSUBSCRIBE Command + + Arguments: mailbox name + + Responses: no specific responses for this command + + Result: OK - unsubscribe completed + NO - unsubscribe failure: can't unsubscribe that name + BAD - command unknown or arguments invalid + + The UNSUBSCRIBE command removes the specified mailbox name from + the server's set of "active" or "subscribed" mailboxes as returned + by the LSUB command. This command returns a tagged OK response + only if the unsubscription is successful. + + Example: C: A002 UNSUBSCRIBE #news.comp.mail.mime + S: A002 OK UNSUBSCRIBE completed + + + +Crispin Standards Track [Page 39] + +RFC 3501 IMAPv4 March 2003 + + +6.3.8. LIST Command + + Arguments: reference name + mailbox name with possible wildcards + + Responses: untagged responses: LIST + + Result: OK - list completed + NO - list failure: can't list that reference or name + BAD - command unknown or arguments invalid + + The LIST command returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name; see the description of the LIST reply for + more detail. + + The LIST command SHOULD return its data quickly, without undue + delay. For example, it SHOULD NOT go to excess trouble to + calculate the \Marked or \Unmarked status or perform other + processing; if each name requires 1 second of processing, then a + list of 1200 names would take 20 minutes! + + An empty ("" string) reference name argument indicates that the + mailbox name is interpreted as by SELECT. The returned mailbox + names MUST match the supplied mailbox name pattern. A non-empty + reference name argument is the name of a mailbox or a level of + mailbox hierarchy, and indicates the context in which the mailbox + name is interpreted. + + An empty ("" string) mailbox name argument is a special request to + return the hierarchy delimiter and the root name of the name given + in the reference. The value returned as the root MAY be the empty + string if the reference is non-rooted or is an empty string. In + all cases, a hierarchy delimiter (or NIL if there is no hierarchy) + is returned. This permits a client to get the hierarchy delimiter + (or find out that the mailbox names are flat) even when no + mailboxes by that name currently exist. + + The reference and mailbox name arguments are interpreted into a + canonical form that represents an unambiguous left-to-right + hierarchy. The returned mailbox names will be in the interpreted + form. + + + + + + + + +Crispin Standards Track [Page 40] + +RFC 3501 IMAPv4 March 2003 + + + Note: The interpretation of the reference argument is + implementation-defined. It depends upon whether the + server implementation has a concept of the "current + working directory" and leading "break out characters", + which override the current working directory. + + For example, on a server which exports a UNIX or NT + filesystem, the reference argument contains the current + working directory, and the mailbox name argument would + contain the name as interpreted in the current working + directory. + + If a server implementation has no concept of break out + characters, the canonical form is normally the reference + name appended with the mailbox name. Note that if the + server implements the namespace convention (section + 5.1.2), "#" is a break out character and must be treated + as such. + + If the reference argument is not a level of mailbox + hierarchy (that is, it is a \NoInferiors name), and/or + the reference argument does not end with the hierarchy + delimiter, it is implementation-dependent how this is + interpreted. For example, a reference of "foo/bar" and + mailbox name of "rag/baz" could be interpreted as + "foo/bar/rag/baz", "foo/barrag/baz", or "foo/rag/baz". + A client SHOULD NOT use such a reference argument except + at the explicit request of the user. A hierarchical + browser MUST NOT make any assumptions about server + interpretation of the reference unless the reference is + a level of mailbox hierarchy AND ends with the hierarchy + delimiter. + + Any part of the reference argument that is included in the + interpreted form SHOULD prefix the interpreted form. It SHOULD + also be in the same form as the reference name argument. This + rule permits the client to determine if the returned mailbox name + is in the context of the reference argument, or if something about + the mailbox argument overrode the reference argument. Without + this rule, the client would have to have knowledge of the server's + naming semantics including what characters are "breakouts" that + override a naming context. + + + + + + + + + +Crispin Standards Track [Page 41] + +RFC 3501 IMAPv4 March 2003 + + + For example, here are some examples of how references + and mailbox names might be interpreted on a UNIX-based + server: + + Reference Mailbox Name Interpretation + ------------ ------------ -------------- + ~smith/Mail/ foo.* ~smith/Mail/foo.* + archive/ % archive/% + #news. comp.mail.* #news.comp.mail.* + ~smith/Mail/ /usr/doc/foo /usr/doc/foo + archive/ ~fred/Mail/* ~fred/Mail/* + + The first three examples demonstrate interpretations in + the context of the reference argument. Note that + "~smith/Mail" SHOULD NOT be transformed into something + like "/u2/users/smith/Mail", or it would be impossible + for the client to determine that the interpretation was + in the context of the reference. + + The character "*" is a wildcard, and matches zero or more + characters at this position. The character "%" is similar to "*", + but it does not match a hierarchy delimiter. If the "%" wildcard + is the last character of a mailbox name argument, matching levels + of hierarchy are also returned. If these levels of hierarchy are + not also selectable mailboxes, they are returned with the + \Noselect mailbox name attribute (see the description of the LIST + response for more details). + + Server implementations are permitted to "hide" otherwise + accessible mailboxes from the wildcard characters, by preventing + certain characters or names from matching a wildcard in certain + situations. For example, a UNIX-based server might restrict the + interpretation of "*" so that an initial "/" character does not + match. + + The special name INBOX is included in the output from LIST, if + INBOX is supported by this server for this user and if the + uppercase string "INBOX" matches the interpreted reference and + mailbox name arguments with wildcards as described above. The + criteria for omitting INBOX is whether SELECT INBOX will return + failure; it is not relevant whether the user's real INBOX resides + on this or some other server. + + + + + + + + + +Crispin Standards Track [Page 42] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: A101 LIST "" "" + S: * LIST (\Noselect) "/" "" + S: A101 OK LIST Completed + C: A102 LIST #news.comp.mail.misc "" + S: * LIST (\Noselect) "." #news. + S: A102 OK LIST Completed + C: A103 LIST /usr/staff/jones "" + S: * LIST (\Noselect) "/" / + S: A103 OK LIST Completed + C: A202 LIST ~/Mail/ % + S: * LIST (\Noselect) "/" ~/Mail/foo + S: * LIST () "/" ~/Mail/meetings + S: A202 OK LIST completed + + +6.3.9. LSUB Command + + Arguments: reference name + mailbox name with possible wildcards + + Responses: untagged responses: LSUB + + Result: OK - lsub completed + NO - lsub failure: can't list that reference or name + BAD - command unknown or arguments invalid + + The LSUB command returns a subset of names from the set of names + that the user has declared as being "active" or "subscribed". + Zero or more untagged LSUB replies are returned. The arguments to + LSUB are in the same form as those for LIST. + + The returned untagged LSUB response MAY contain different mailbox + flags from a LIST untagged response. If this should happen, the + flags in the untagged LIST are considered more authoritative. + + A special situation occurs when using LSUB with the % wildcard. + Consider what happens if "foo/bar" (with a hierarchy delimiter of + "/") is subscribed but "foo" is not. A "%" wildcard to LSUB must + return foo, not foo/bar, in the LSUB response, and it MUST be + flagged with the \Noselect attribute. + + The server MUST NOT unilaterally remove an existing mailbox name + from the subscription list even if a mailbox by that name no + longer exists. + + + + + + + +Crispin Standards Track [Page 43] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: A002 LSUB "#news." "comp.mail.*" + S: * LSUB () "." #news.comp.mail.mime + S: * LSUB () "." #news.comp.mail.misc + S: A002 OK LSUB completed + C: A003 LSUB "#news." "comp.%" + S: * LSUB (\NoSelect) "." #news.comp.mail + S: A003 OK LSUB completed + + +6.3.10. STATUS Command + + Arguments: mailbox name + status data item names + + Responses: untagged responses: STATUS + + Result: OK - status completed + NO - status failure: no status for that name + BAD - command unknown or arguments invalid + + The STATUS command requests the status of the indicated mailbox. + It does not change the currently selected mailbox, nor does it + affect the state of any messages in the queried mailbox (in + particular, STATUS MUST NOT cause messages to lose the \Recent + flag). + + The STATUS command provides an alternative to opening a second + IMAP4rev1 connection and doing an EXAMINE command on a mailbox to + query that mailbox's status without deselecting the current + mailbox in the first IMAP4rev1 connection. + + Unlike the LIST command, the STATUS command is not guaranteed to + be fast in its response. Under certain circumstances, it can be + quite slow. In some implementations, the server is obliged to + open the mailbox read-only internally to obtain certain status + information. Also unlike the LIST command, the STATUS command + does not accept wildcards. + + Note: The STATUS command is intended to access the + status of mailboxes other than the currently selected + mailbox. Because the STATUS command can cause the + mailbox to be opened internally, and because this + information is available by other means on the selected + mailbox, the STATUS command SHOULD NOT be used on the + currently selected mailbox. + + + + + + +Crispin Standards Track [Page 44] + +RFC 3501 IMAPv4 March 2003 + + + The STATUS command MUST NOT be used as a "check for new + messages in the selected mailbox" operation (refer to + sections 7, 7.3.1, and 7.3.2 for more information about + the proper method for new message checking). + + Because the STATUS command is not guaranteed to be fast + in its results, clients SHOULD NOT expect to be able to + issue many consecutive STATUS commands and obtain + reasonable performance. + + The currently defined status data items that can be requested are: + + MESSAGES + The number of messages in the mailbox. + + RECENT + The number of messages with the \Recent flag set. + + UIDNEXT + The next unique identifier value of the mailbox. Refer to + section 2.3.1.1 for more information. + + UIDVALIDITY + The unique identifier validity value of the mailbox. Refer to + section 2.3.1.1 for more information. + + UNSEEN + The number of messages which do not have the \Seen flag set. + + + Example: C: A042 STATUS blurdybloop (UIDNEXT MESSAGES) + S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292) + S: A042 OK STATUS completed + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 45] + +RFC 3501 IMAPv4 March 2003 + + +6.3.11. APPEND Command + + Arguments: mailbox name + OPTIONAL flag parenthesized list + OPTIONAL date/time string + message literal + + Responses: no specific responses for this command + + Result: OK - append completed + NO - append error: can't append to that mailbox, error + in flags or date/time or message text + BAD - command unknown or arguments invalid + + The APPEND command appends the literal argument as a new message + to the end of the specified destination mailbox. This argument + SHOULD be in the format of an [RFC-2822] message. 8-bit + characters are permitted in the message. A server implementation + that is unable to preserve 8-bit data properly MUST be able to + reversibly convert 8-bit APPEND data to 7-bit using a [MIME-IMB] + content transfer encoding. + + Note: There MAY be exceptions, e.g., draft messages, in + which required [RFC-2822] header lines are omitted in + the message literal argument to APPEND. The full + implications of doing so MUST be understood and + carefully weighed. + + If a flag parenthesized list is specified, the flags SHOULD be set + in the resulting message; otherwise, the flag list of the + resulting message is set to empty by default. In either case, the + Recent flag is also set. + + If a date-time is specified, the internal date SHOULD be set in + the resulting message; otherwise, the internal date of the + resulting message is set to the current date and time by default. + + If the append is unsuccessful for any reason, the mailbox MUST be + restored to its state before the APPEND attempt; no partial + appending is permitted. + + If the destination mailbox does not exist, a server MUST return an + error, and MUST NOT automatically create the mailbox. Unless it + is certain that the destination mailbox can not be created, the + server MUST send the response code "[TRYCREATE]" as the prefix of + the text of the tagged NO response. This gives a hint to the + client that it can attempt a CREATE command and retry the APPEND + if the CREATE is successful. + + + +Crispin Standards Track [Page 46] + +RFC 3501 IMAPv4 March 2003 + + + If the mailbox is currently selected, the normal new message + actions SHOULD occur. Specifically, the server SHOULD notify the + client immediately via an untagged EXISTS response. If the server + does not do so, the client MAY issue a NOOP command (or failing + that, a CHECK command) after one or more APPEND commands. + + Example: C: A003 APPEND saved-messages (\Seen) {310} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK APPEND completed + + Note: The APPEND command is not used for message delivery, + because it does not provide a mechanism to transfer [SMTP] + envelope information. + +6.4. Client Commands - Selected State + + In the selected state, commands that manipulate messages in a mailbox + are permitted. + + In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), + and the authenticated state commands (SELECT, EXAMINE, CREATE, + DELETE, RENAME, SUBSCRIBE, UNSUBSCRIBE, LIST, LSUB, STATUS, and + APPEND), the following commands are valid in the selected state: + CHECK, CLOSE, EXPUNGE, SEARCH, FETCH, STORE, COPY, and UID. + +6.4.1. CHECK Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - check completed + BAD - command unknown or arguments invalid + + The CHECK command requests a checkpoint of the currently selected + mailbox. A checkpoint refers to any implementation-dependent + housekeeping associated with the mailbox (e.g., resolving the + server's in-memory state of the mailbox with the state on its + + + +Crispin Standards Track [Page 47] + +RFC 3501 IMAPv4 March 2003 + + + disk) that is not normally executed as part of each command. A + checkpoint MAY take a non-instantaneous amount of real time to + complete. If a server implementation has no such housekeeping + considerations, CHECK is equivalent to NOOP. + + There is no guarantee that an EXISTS untagged response will happen + as a result of CHECK. NOOP, not CHECK, SHOULD be used for new + message polling. + + Example: C: FXXZ CHECK + S: FXXZ OK CHECK Completed + + +6.4.2. CLOSE Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - close completed, now in authenticated state + BAD - command unknown or arguments invalid + + The CLOSE command permanently removes all messages that have the + \Deleted flag set from the currently selected mailbox, and returns + to the authenticated state from the selected state. No untagged + EXPUNGE responses are sent. + + No messages are removed, and no error is given, if the mailbox is + selected by an EXAMINE command or is otherwise selected read-only. + + Even if a mailbox is selected, a SELECT, EXAMINE, or LOGOUT + command MAY be issued without previously issuing a CLOSE command. + The SELECT, EXAMINE, and LOGOUT commands implicitly close the + currently selected mailbox without doing an expunge. However, + when many messages are deleted, a CLOSE-LOGOUT or CLOSE-SELECT + sequence is considerably faster than an EXPUNGE-LOGOUT or + EXPUNGE-SELECT because no untagged EXPUNGE responses (which the + client would probably ignore) are sent. + + Example: C: A341 CLOSE + S: A341 OK CLOSE completed + + + + + + + + + + +Crispin Standards Track [Page 48] + +RFC 3501 IMAPv4 March 2003 + + +6.4.3. EXPUNGE Command + + Arguments: none + + Responses: untagged responses: EXPUNGE + + Result: OK - expunge completed + NO - expunge failure: can't expunge (e.g., permission + denied) + BAD - command unknown or arguments invalid + + The EXPUNGE command permanently removes all messages that have the + \Deleted flag set from the currently selected mailbox. Before + returning an OK to the client, an untagged EXPUNGE response is + sent for each message that is removed. + + Example: C: A202 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 5 EXPUNGE + S: * 8 EXPUNGE + S: A202 OK EXPUNGE completed + + Note: In this example, messages 3, 4, 7, and 11 had the + \Deleted flag set. See the description of the EXPUNGE + response for further explanation. + + +6.4.4. SEARCH Command + + Arguments: OPTIONAL [CHARSET] specification + searching criteria (one or more) + + Responses: REQUIRED untagged response: SEARCH + + Result: OK - search completed + NO - search error: can't search that [CHARSET] or + criteria + BAD - command unknown or arguments invalid + + The SEARCH command searches the mailbox for messages that match + the given searching criteria. Searching criteria consist of one + or more search keys. The untagged SEARCH response from the server + contains a listing of message sequence numbers corresponding to + those messages that match the searching criteria. + + + + + + +Crispin Standards Track [Page 49] + +RFC 3501 IMAPv4 March 2003 + + + When multiple keys are specified, the result is the intersection + (AND function) of all the messages that match those keys. For + example, the criteria DELETED FROM "SMITH" SINCE 1-Feb-1994 refers + to all deleted messages from Smith that were placed in the mailbox + since February 1, 1994. A search key can also be a parenthesized + list of one or more search keys (e.g., for use with the OR and NOT + keys). + + Server implementations MAY exclude [MIME-IMB] body parts with + terminal content media types other than TEXT and MESSAGE from + consideration in SEARCH matching. + + The OPTIONAL [CHARSET] specification consists of the word + "CHARSET" followed by a registered [CHARSET]. It indicates the + [CHARSET] of the strings that appear in the search criteria. + [MIME-IMB] content transfer encodings, and [MIME-HDRS] strings in + [RFC-2822]/[MIME-IMB] headers, MUST be decoded before comparing + text in a [CHARSET] other than US-ASCII. US-ASCII MUST be + supported; other [CHARSET]s MAY be supported. + + If the server does not support the specified [CHARSET], it MUST + return a tagged NO response (not a BAD). This response SHOULD + contain the BADCHARSET response code, which MAY list the + [CHARSET]s supported by the server. + + In all search keys that use strings, a message matches the key if + the string is a substring of the field. The matching is + case-insensitive. + + The defined search keys are as follows. Refer to the Formal + Syntax section for the precise syntactic definitions of the + arguments. + + + Messages with message sequence numbers corresponding to the + specified message sequence number set. + + ALL + All messages in the mailbox; the default initial key for + ANDing. + + ANSWERED + Messages with the \Answered flag set. + + + + + + + + +Crispin Standards Track [Page 50] + +RFC 3501 IMAPv4 March 2003 + + + BCC + Messages that contain the specified string in the envelope + structure's BCC field. + + BEFORE + Messages whose internal date (disregarding time and timezone) + is earlier than the specified date. + + BODY + Messages that contain the specified string in the body of the + message. + + CC + Messages that contain the specified string in the envelope + structure's CC field. + + DELETED + Messages with the \Deleted flag set. + + DRAFT + Messages with the \Draft flag set. + + FLAGGED + Messages with the \Flagged flag set. + + FROM + Messages that contain the specified string in the envelope + structure's FROM field. + + HEADER + Messages that have a header with the specified field-name (as + defined in [RFC-2822]) and that contains the specified string + in the text of the header (what comes after the colon). If the + string to search is zero-length, this matches all messages that + have a header line with the specified field-name regardless of + the contents. + + KEYWORD + Messages with the specified keyword flag set. + + LARGER + Messages with an [RFC-2822] size larger than the specified + number of octets. + + NEW + Messages that have the \Recent flag set but not the \Seen flag. + This is functionally equivalent to "(RECENT UNSEEN)". + + + + +Crispin Standards Track [Page 51] + +RFC 3501 IMAPv4 March 2003 + + + NOT + Messages that do not match the specified search key. + + OLD + Messages that do not have the \Recent flag set. This is + functionally equivalent to "NOT RECENT" (as opposed to "NOT + NEW"). + + ON + Messages whose internal date (disregarding time and timezone) + is within the specified date. + + OR + Messages that match either search key. + + RECENT + Messages that have the \Recent flag set. + + SEEN + Messages that have the \Seen flag set. + + SENTBEFORE + Messages whose [RFC-2822] Date: header (disregarding time and + timezone) is earlier than the specified date. + + SENTON + Messages whose [RFC-2822] Date: header (disregarding time and + timezone) is within the specified date. + + SENTSINCE + Messages whose [RFC-2822] Date: header (disregarding time and + timezone) is within or later than the specified date. + + SINCE + Messages whose internal date (disregarding time and timezone) + is within or later than the specified date. + + SMALLER + Messages with an [RFC-2822] size smaller than the specified + number of octets. + + + + + + + + + + + +Crispin Standards Track [Page 52] + +RFC 3501 IMAPv4 March 2003 + + + SUBJECT + Messages that contain the specified string in the envelope + structure's SUBJECT field. + + TEXT + Messages that contain the specified string in the header or + body of the message. + + TO + Messages that contain the specified string in the envelope + structure's TO field. + + UID + Messages with unique identifiers corresponding to the specified + unique identifier set. Sequence set ranges are permitted. + + UNANSWERED + Messages that do not have the \Answered flag set. + + UNDELETED + Messages that do not have the \Deleted flag set. + + UNDRAFT + Messages that do not have the \Draft flag set. + + UNFLAGGED + Messages that do not have the \Flagged flag set. + + UNKEYWORD + Messages that do not have the specified keyword flag set. + + UNSEEN + Messages that do not have the \Seen flag set. + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 53] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: A282 SEARCH FLAGGED SINCE 1-Feb-1994 NOT FROM "Smith" + S: * SEARCH 2 84 882 + S: A282 OK SEARCH completed + C: A283 SEARCH TEXT "string not in mailbox" + S: * SEARCH + S: A283 OK SEARCH completed + C: A284 SEARCH CHARSET UTF-8 TEXT {6} + C: XXXXXX + S: * SEARCH 43 + S: A284 OK SEARCH completed + + Note: Since this document is restricted to 7-bit ASCII + text, it is not possible to show actual UTF-8 data. The + "XXXXXX" is a placeholder for what would be 6 octets of + 8-bit data in an actual transaction. + + +6.4.5. FETCH Command + + Arguments: sequence set + message data item names or macro + + Responses: untagged responses: FETCH + + Result: OK - fetch completed + NO - fetch error: can't fetch that data + BAD - command unknown or arguments invalid + + The FETCH command retrieves data associated with a message in the + mailbox. The data items to be fetched can be either a single atom + or a parenthesized list. + + Most data items, identified in the formal syntax under the + msg-att-static rule, are static and MUST NOT change for any + particular message. Other data items, identified in the formal + syntax under the msg-att-dynamic rule, MAY change, either as a + result of a STORE command or due to external events. + + For example, if a client receives an ENVELOPE for a + message when it already knows the envelope, it can + safely ignore the newly transmitted envelope. + + There are three macros which specify commonly-used sets of data + items, and can be used instead of data items. A macro must be + used by itself, and not in conjunction with other macros or data + items. + + + + + +Crispin Standards Track [Page 54] + +RFC 3501 IMAPv4 March 2003 + + + ALL + Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE ENVELOPE) + + FAST + Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE) + + FULL + Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE ENVELOPE + BODY) + + The currently defined data items that can be fetched are: + + BODY + Non-extensible form of BODYSTRUCTURE. + + BODY[
]<> + The text of a particular body section. The section + specification is a set of zero or more part specifiers + delimited by periods. A part specifier is either a part number + or one of the following: HEADER, HEADER.FIELDS, + HEADER.FIELDS.NOT, MIME, and TEXT. An empty section + specification refers to the entire message, including the + header. + + Every message has at least one part number. Non-[MIME-IMB] + messages, and non-multipart [MIME-IMB] messages with no + encapsulated message, only have a part 1. + + Multipart messages are assigned consecutive part numbers, as + they occur in the message. If a particular part is of type + message or multipart, its parts MUST be indicated by a period + followed by the part number within that nested multipart part. + + A part of type MESSAGE/RFC822 also has nested part numbers, + referring to parts of the MESSAGE part's body. + + The HEADER, HEADER.FIELDS, HEADER.FIELDS.NOT, and TEXT part + specifiers can be the sole part specifier or can be prefixed by + one or more numeric part specifiers, provided that the numeric + part specifier refers to a part of type MESSAGE/RFC822. The + MIME part specifier MUST be prefixed by one or more numeric + part specifiers. + + The HEADER, HEADER.FIELDS, and HEADER.FIELDS.NOT part + specifiers refer to the [RFC-2822] header of the message or of + an encapsulated [MIME-IMT] MESSAGE/RFC822 message. + HEADER.FIELDS and HEADER.FIELDS.NOT are followed by a list of + field-name (as defined in [RFC-2822]) names, and return a + + + +Crispin Standards Track [Page 55] + +RFC 3501 IMAPv4 March 2003 + + + subset of the header. The subset returned by HEADER.FIELDS + contains only those header fields with a field-name that + matches one of the names in the list; similarly, the subset + returned by HEADER.FIELDS.NOT contains only the header fields + with a non-matching field-name. The field-matching is + case-insensitive but otherwise exact. Subsetting does not + exclude the [RFC-2822] delimiting blank line between the header + and the body; the blank line is included in all header fetches, + except in the case of a message which has no body and no blank + line. + + The MIME part specifier refers to the [MIME-IMB] header for + this part. + + The TEXT part specifier refers to the text body of the message, + omitting the [RFC-2822] header. + + Here is an example of a complex message with some of its + part specifiers: + + HEADER ([RFC-2822] header of the message) + TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED + 1 TEXT/PLAIN + 2 APPLICATION/OCTET-STREAM + 3 MESSAGE/RFC822 + 3.HEADER ([RFC-2822] header of the message) + 3.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED + 3.1 TEXT/PLAIN + 3.2 APPLICATION/OCTET-STREAM + 4 MULTIPART/MIXED + 4.1 IMAGE/GIF + 4.1.MIME ([MIME-IMB] header for the IMAGE/GIF) + 4.2 MESSAGE/RFC822 + 4.2.HEADER ([RFC-2822] header of the message) + 4.2.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED + 4.2.1 TEXT/PLAIN + 4.2.2 MULTIPART/ALTERNATIVE + 4.2.2.1 TEXT/PLAIN + 4.2.2.2 TEXT/RICHTEXT + + + It is possible to fetch a substring of the designated text. + This is done by appending an open angle bracket ("<"), the + octet position of the first desired octet, a period, the + maximum number of octets desired, and a close angle bracket + (">") to the part specifier. If the starting octet is beyond + the end of the text, an empty string is returned. + + + + +Crispin Standards Track [Page 56] + +RFC 3501 IMAPv4 March 2003 + + + Any partial fetch that attempts to read beyond the end of the + text is truncated as appropriate. A partial fetch that starts + at octet 0 is returned as a partial fetch, even if this + truncation happened. + + Note: This means that BODY[]<0.2048> of a 1500-octet message + will return BODY[]<0> with a literal of size 1500, not + BODY[]. + + Note: A substring fetch of a HEADER.FIELDS or + HEADER.FIELDS.NOT part specifier is calculated after + subsetting the header. + + The \Seen flag is implicitly set; if this causes the flags to + change, they SHOULD be included as part of the FETCH responses. + + BODY.PEEK[
]<> + An alternate form of BODY[
] that does not implicitly + set the \Seen flag. + + BODYSTRUCTURE + The [MIME-IMB] body structure of the message. This is computed + by the server by parsing the [MIME-IMB] header fields in the + [RFC-2822] header and [MIME-IMB] headers. + + ENVELOPE + The envelope structure of the message. This is computed by the + server by parsing the [RFC-2822] header into the component + parts, defaulting various fields as necessary. + + FLAGS + The flags that are set for this message. + + INTERNALDATE + The internal date of the message. + + RFC822 + Functionally equivalent to BODY[], differing in the syntax of + the resulting untagged FETCH data (RFC822 is returned). + + RFC822.HEADER + Functionally equivalent to BODY.PEEK[HEADER], differing in the + syntax of the resulting untagged FETCH data (RFC822.HEADER is + returned). + + RFC822.SIZE + The [RFC-2822] size of the message. + + + + +Crispin Standards Track [Page 57] + +RFC 3501 IMAPv4 March 2003 + + + RFC822.TEXT + Functionally equivalent to BODY[TEXT], differing in the syntax + of the resulting untagged FETCH data (RFC822.TEXT is returned). + + UID + The unique identifier for the message. + + + Example: C: A654 FETCH 2:4 (FLAGS BODY[HEADER.FIELDS (DATE FROM)]) + S: * 2 FETCH .... + S: * 3 FETCH .... + S: * 4 FETCH .... + S: A654 OK FETCH completed + + +6.4.6. STORE Command + + Arguments: sequence set + message data item name + value for message data item + + Responses: untagged responses: FETCH + + Result: OK - store completed + NO - store error: can't store that data + BAD - command unknown or arguments invalid + + The STORE command alters data associated with a message in the + mailbox. Normally, STORE will return the updated value of the + data with an untagged FETCH response. A suffix of ".SILENT" in + the data item name prevents the untagged FETCH, and the server + SHOULD assume that the client has determined the updated value + itself or does not care about the updated value. + + Note: Regardless of whether or not the ".SILENT" suffix + was used, the server SHOULD send an untagged FETCH + response if a change to a message's flags from an + external source is observed. The intent is that the + status of the flags is determinate without a race + condition. + + + + + + + + + + + +Crispin Standards Track [Page 58] + +RFC 3501 IMAPv4 March 2003 + + + The currently defined data items that can be stored are: + + FLAGS + Replace the flags for the message (other than \Recent) with the + argument. The new value of the flags is returned as if a FETCH + of those flags was done. + + FLAGS.SILENT + Equivalent to FLAGS, but without returning a new value. + + +FLAGS + Add the argument to the flags for the message. The new value + of the flags is returned as if a FETCH of those flags was done. + + +FLAGS.SILENT + Equivalent to +FLAGS, but without returning a new value. + + -FLAGS + Remove the argument from the flags for the message. The new + value of the flags is returned as if a FETCH of those flags was + done. + + -FLAGS.SILENT + Equivalent to -FLAGS, but without returning a new value. + + + Example: C: A003 STORE 2:4 +FLAGS (\Deleted) + S: * 2 FETCH (FLAGS (\Deleted \Seen)) + S: * 3 FETCH (FLAGS (\Deleted)) + S: * 4 FETCH (FLAGS (\Deleted \Flagged \Seen)) + S: A003 OK STORE completed + + +6.4.7. COPY Command + + Arguments: sequence set + mailbox name + + Responses: no specific responses for this command + + Result: OK - copy completed + NO - copy error: can't copy those messages or to that + name + BAD - command unknown or arguments invalid + + + + + + + +Crispin Standards Track [Page 59] + +RFC 3501 IMAPv4 March 2003 + + + The COPY command copies the specified message(s) to the end of the + specified destination mailbox. The flags and internal date of the + message(s) SHOULD be preserved, and the Recent flag SHOULD be set, + in the copy. + + If the destination mailbox does not exist, a server SHOULD return + an error. It SHOULD NOT automatically create the mailbox. Unless + it is certain that the destination mailbox can not be created, the + server MUST send the response code "[TRYCREATE]" as the prefix of + the text of the tagged NO response. This gives a hint to the + client that it can attempt a CREATE command and retry the COPY if + the CREATE is successful. + + If the COPY command is unsuccessful for any reason, server + implementations MUST restore the destination mailbox to its state + before the COPY attempt. + + Example: C: A003 COPY 2:4 MEETING + S: A003 OK COPY completed + + +6.4.8. UID Command + + Arguments: command name + command arguments + + Responses: untagged responses: FETCH, SEARCH + + Result: OK - UID command completed + NO - UID command error + BAD - command unknown or arguments invalid + + The UID command has two forms. In the first form, it takes as its + arguments a COPY, FETCH, or STORE command with arguments + appropriate for the associated command. However, the numbers in + the sequence set argument are unique identifiers instead of + message sequence numbers. Sequence set ranges are permitted, but + there is no guarantee that unique identifiers will be contiguous. + + A non-existent unique identifier is ignored without any error + message generated. Thus, it is possible for a UID FETCH command + to return an OK without any data or a UID COPY or UID STORE to + return an OK without performing any operations. + + In the second form, the UID command takes a SEARCH command with + SEARCH command arguments. The interpretation of the arguments is + the same as with SEARCH; however, the numbers returned in a SEARCH + response for a UID SEARCH command are unique identifiers instead + + + +Crispin Standards Track [Page 60] + +RFC 3501 IMAPv4 March 2003 + + + of message sequence numbers. For example, the command UID SEARCH + 1:100 UID 443:557 returns the unique identifiers corresponding to + the intersection of two sequence sets, the message sequence number + range 1:100 and the UID range 443:557. + + Note: in the above example, the UID range 443:557 + appears. The same comment about a non-existent unique + identifier being ignored without any error message also + applies here. Hence, even if neither UID 443 or 557 + exist, this range is valid and would include an existing + UID 495. + + Also note that a UID range of 559:* always includes the + UID of the last message in the mailbox, even if 559 is + higher than any assigned UID value. This is because the + contents of a range are independent of the order of the + range endpoints. Thus, any UID range with * as one of + the endpoints indicates at least one message (the + message with the highest numbered UID), unless the + mailbox is empty. + + The number after the "*" in an untagged FETCH response is always a + message sequence number, not a unique identifier, even for a UID + command response. However, server implementations MUST implicitly + include the UID message data item as part of any FETCH response + caused by a UID command, regardless of whether a UID was specified + as a message data item to the FETCH. + + + Note: The rule about including the UID message data item as part + of a FETCH response primarily applies to the UID FETCH and UID + STORE commands, including a UID FETCH command that does not + include UID as a message data item. Although it is unlikely that + the other UID commands will cause an untagged FETCH, this rule + applies to these commands as well. + + Example: C: A999 UID FETCH 4827313:4828442 FLAGS + S: * 23 FETCH (FLAGS (\Seen) UID 4827313) + S: * 24 FETCH (FLAGS (\Seen) UID 4827943) + S: * 25 FETCH (FLAGS (\Seen) UID 4828442) + S: A999 OK UID FETCH completed + + + + + + + + + + +Crispin Standards Track [Page 61] + +RFC 3501 IMAPv4 March 2003 + + +6.5. Client Commands - Experimental/Expansion + + +6.5.1. X Command + + Arguments: implementation defined + + Responses: implementation defined + + Result: OK - command completed + NO - failure + BAD - command unknown or arguments invalid + + Any command prefixed with an X is an experimental command. + Commands which are not part of this specification, a standard or + standards-track revision of this specification, or an + IESG-approved experimental protocol, MUST use the X prefix. + + Any added untagged responses issued by an experimental command + MUST also be prefixed with an X. Server implementations MUST NOT + send any such untagged responses, unless the client requested it + by issuing the associated experimental command. + + Example: C: a441 CAPABILITY + S: * CAPABILITY IMAP4rev1 XPIG-LATIN + S: a441 OK CAPABILITY completed + C: A442 XPIG-LATIN + S: * XPIG-LATIN ow-nay eaking-spay ig-pay atin-lay + S: A442 OK XPIG-LATIN ompleted-cay + +7. Server Responses + + Server responses are in three forms: status responses, server data, + and command continuation request. The information contained in a + server response, identified by "Contents:" in the response + descriptions below, is described by function, not by syntax. The + precise syntax of server responses is described in the Formal Syntax + section. + + The client MUST be prepared to accept any response at all times. + + Status responses can be tagged or untagged. Tagged status responses + indicate the completion result (OK, NO, or BAD status) of a client + command, and have a tag matching the command. + + Some status responses, and all server data, are untagged. An + untagged response is indicated by the token "*" instead of a tag. + Untagged status responses indicate server greeting, or server status + + + +Crispin Standards Track [Page 62] + +RFC 3501 IMAPv4 March 2003 + + + that does not indicate the completion of a command (for example, an + impending system shutdown alert). For historical reasons, untagged + server data responses are also called "unsolicited data", although + strictly speaking, only unilateral server data is truly + "unsolicited". + + Certain server data MUST be recorded by the client when it is + received; this is noted in the description of that data. Such data + conveys critical information which affects the interpretation of all + subsequent commands and responses (e.g., updates reflecting the + creation or destruction of messages). + + Other server data SHOULD be recorded for later reference; if the + client does not need to record the data, or if recording the data has + no obvious purpose (e.g., a SEARCH response when no SEARCH command is + in progress), the data SHOULD be ignored. + + An example of unilateral untagged server data occurs when the IMAP + connection is in the selected state. In the selected state, the + server checks the mailbox for new messages as part of command + execution. Normally, this is part of the execution of every command; + hence, a NOOP command suffices to check for new messages. If new + messages are found, the server sends untagged EXISTS and RECENT + responses reflecting the new size of the mailbox. Server + implementations that offer multiple simultaneous access to the same + mailbox SHOULD also send appropriate unilateral untagged FETCH and + EXPUNGE responses if another agent changes the state of any message + flags or expunges any messages. + + Command continuation request responses use the token "+" instead of a + tag. These responses are sent by the server to indicate acceptance + of an incomplete client command and readiness for the remainder of + the command. + +7.1. Server Responses - Status Responses + + Status responses are OK, NO, BAD, PREAUTH and BYE. OK, NO, and BAD + can be tagged or untagged. PREAUTH and BYE are always untagged. + + Status responses MAY include an OPTIONAL "response code". A response + code consists of data inside square brackets in the form of an atom, + possibly followed by a space and arguments. The response code + contains additional information or status codes for client software + beyond the OK/NO/BAD condition, and are defined when there is a + specific action that a client can take based upon the additional + information. + + + + + +Crispin Standards Track [Page 63] + +RFC 3501 IMAPv4 March 2003 + + + The currently defined response codes are: + + ALERT + + The human-readable text contains a special alert that MUST be + presented to the user in a fashion that calls the user's + attention to the message. + + BADCHARSET + + Optionally followed by a parenthesized list of charsets. A + SEARCH failed because the given charset is not supported by + this implementation. If the optional list of charsets is + given, this lists the charsets that are supported by this + implementation. + + CAPABILITY + + Followed by a list of capabilities. This can appear in the + initial OK or PREAUTH response to transmit an initial + capabilities list. This makes it unnecessary for a client to + send a separate CAPABILITY command if it recognizes this + response. + + PARSE + + The human-readable text represents an error in parsing the + [RFC-2822] header or [MIME-IMB] headers of a message in the + mailbox. + + PERMANENTFLAGS + + Followed by a parenthesized list of flags, indicates which of + the known flags the client can change permanently. Any flags + that are in the FLAGS untagged response, but not the + PERMANENTFLAGS list, can not be set permanently. If the client + attempts to STORE a flag that is not in the PERMANENTFLAGS + list, the server will either ignore the change or store the + state change for the remainder of the current session only. + The PERMANENTFLAGS list can also include the special flag \*, + which indicates that it is possible to create new keywords by + attempting to store those flags in the mailbox. + + + + + + + + + +Crispin Standards Track [Page 64] + +RFC 3501 IMAPv4 March 2003 + + + READ-ONLY + + The mailbox is selected read-only, or its access while selected + has changed from read-write to read-only. + + READ-WRITE + + The mailbox is selected read-write, or its access while + selected has changed from read-only to read-write. + + TRYCREATE + + An APPEND or COPY attempt is failing because the target mailbox + does not exist (as opposed to some other reason). This is a + hint to the client that the operation can succeed if the + mailbox is first created by the CREATE command. + + UIDNEXT + + Followed by a decimal number, indicates the next unique + identifier value. Refer to section 2.3.1.1 for more + information. + + UIDVALIDITY + + Followed by a decimal number, indicates the unique identifier + validity value. Refer to section 2.3.1.1 for more information. + + UNSEEN + + Followed by a decimal number, indicates the number of the first + message without the \Seen flag set. + + Additional response codes defined by particular client or server + implementations SHOULD be prefixed with an "X" until they are + added to a revision of this protocol. Client implementations + SHOULD ignore response codes that they do not recognize. + +7.1.1. OK Response + + Contents: OPTIONAL response code + human-readable text + + The OK response indicates an information message from the server. + When tagged, it indicates successful completion of the associated + command. The human-readable text MAY be presented to the user as + an information message. The untagged form indicates an + + + + +Crispin Standards Track [Page 65] + +RFC 3501 IMAPv4 March 2003 + + + information-only message; the nature of the information MAY be + indicated by a response code. + + The untagged form is also used as one of three possible greetings + at connection startup. It indicates that the connection is not + yet authenticated and that a LOGIN command is needed. + + Example: S: * OK IMAP4rev1 server ready + C: A001 LOGIN fred blurdybloop + S: * OK [ALERT] System shutdown in 10 minutes + S: A001 OK LOGIN Completed + + +7.1.2. NO Response + + Contents: OPTIONAL response code + human-readable text + + The NO response indicates an operational error message from the + server. When tagged, it indicates unsuccessful completion of the + associated command. The untagged form indicates a warning; the + command can still complete successfully. The human-readable text + describes the condition. + + Example: C: A222 COPY 1:2 owatagusiam + S: * NO Disk is 98% full, please delete unnecessary data + S: A222 OK COPY completed + C: A223 COPY 3:200 blurdybloop + S: * NO Disk is 98% full, please delete unnecessary data + S: * NO Disk is 99% full, please delete unnecessary data + S: A223 NO COPY failed: disk is full + + +7.1.3. BAD Response + + Contents: OPTIONAL response code + human-readable text + + The BAD response indicates an error message from the server. When + tagged, it reports a protocol-level error in the client's command; + the tag indicates the command that caused the error. The untagged + form indicates a protocol-level error for which the associated + command can not be determined; it can also indicate an internal + server failure. The human-readable text describes the condition. + + + + + + + +Crispin Standards Track [Page 66] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: ...very long command line... + S: * BAD Command line too long + C: ...empty line... + S: * BAD Empty command line + C: A443 EXPUNGE + S: * BAD Disk crash, attempting salvage to a new disk! + S: * OK Salvage successful, no data lost + S: A443 OK Expunge completed + + +7.1.4. PREAUTH Response + + Contents: OPTIONAL response code + human-readable text + + The PREAUTH response is always untagged, and is one of three + possible greetings at connection startup. It indicates that the + connection has already been authenticated by external means; thus + no LOGIN command is needed. + + Example: S: * PREAUTH IMAP4rev1 server logged in as Smith + + +7.1.5. BYE Response + + Contents: OPTIONAL response code + human-readable text + + The BYE response is always untagged, and indicates that the server + is about to close the connection. The human-readable text MAY be + displayed to the user in a status report by the client. The BYE + response is sent under one of four conditions: + + 1) as part of a normal logout sequence. The server will close + the connection after sending the tagged OK response to the + LOGOUT command. + + 2) as a panic shutdown announcement. The server closes the + connection immediately. + + 3) as an announcement of an inactivity autologout. The server + closes the connection immediately. + + 4) as one of three possible greetings at connection startup, + indicating that the server is not willing to accept a + connection from this client. The server closes the + connection immediately. + + + + +Crispin Standards Track [Page 67] + +RFC 3501 IMAPv4 March 2003 + + + The difference between a BYE that occurs as part of a normal + LOGOUT sequence (the first case) and a BYE that occurs because of + a failure (the other three cases) is that the connection closes + immediately in the failure case. In all cases the client SHOULD + continue to read response data from the server until the + connection is closed; this will ensure that any pending untagged + or completion responses are read and processed. + + Example: S: * BYE Autologout; idle for too long + +7.2. Server Responses - Server and Mailbox Status + + These responses are always untagged. This is how server and mailbox + status data are transmitted from the server to the client. Many of + these responses typically result from a command with the same name. + +7.2.1. CAPABILITY Response + + Contents: capability listing + + The CAPABILITY response occurs as a result of a CAPABILITY + command. The capability listing contains a space-separated + listing of capability names that the server supports. The + capability listing MUST include the atom "IMAP4rev1". + + In addition, client and server implementations MUST implement the + STARTTLS, LOGINDISABLED, and AUTH=PLAIN (described in [IMAP-TLS]) + capabilities. See the Security Considerations section for + important information. + + A capability name which begins with "AUTH=" indicates that the + server supports that particular authentication mechanism. + + The LOGINDISABLED capability indicates that the LOGIN command is + disabled, and that the server will respond with a tagged NO + response to any attempt to use the LOGIN command even if the user + name and password are valid. An IMAP client MUST NOT issue the + LOGIN command if the server advertises the LOGINDISABLED + capability. + + Other capability names indicate that the server supports an + extension, revision, or amendment to the IMAP4rev1 protocol. + Server responses MUST conform to this document until the client + issues a command that uses the associated capability. + + Capability names MUST either begin with "X" or be standard or + standards-track IMAP4rev1 extensions, revisions, or amendments + registered with IANA. A server MUST NOT offer unregistered or + + + +Crispin Standards Track [Page 68] + +RFC 3501 IMAPv4 March 2003 + + + non-standard capability names, unless such names are prefixed with + an "X". + + Client implementations SHOULD NOT require any capability name + other than "IMAP4rev1", and MUST ignore any unknown capability + names. + + A server MAY send capabilities automatically, by using the + CAPABILITY response code in the initial PREAUTH or OK responses, + and by sending an updated CAPABILITY response code in the tagged + OK response as part of a successful authentication. It is + unnecessary for a client to send a separate CAPABILITY command if + it recognizes these automatic capabilities. + + Example: S: * CAPABILITY IMAP4rev1 STARTTLS AUTH=GSSAPI XPIG-LATIN + + +7.2.2. LIST Response + + Contents: name attributes + hierarchy delimiter + name + + The LIST response occurs as a result of a LIST command. It + returns a single name that matches the LIST specification. There + can be multiple LIST responses for a single LIST command. + + Four name attributes are defined: + + \Noinferiors + It is not possible for any child levels of hierarchy to exist + under this name; no child levels exist now and none can be + created in the future. + + \Noselect + It is not possible to use this name as a selectable mailbox. + + \Marked + The mailbox has been marked "interesting" by the server; the + mailbox probably contains messages that have been added since + the last time the mailbox was selected. + + \Unmarked + The mailbox does not contain any additional messages since the + last time the mailbox was selected. + + + + + + +Crispin Standards Track [Page 69] + +RFC 3501 IMAPv4 March 2003 + + + If it is not feasible for the server to determine whether or not + the mailbox is "interesting", or if the name is a \Noselect name, + the server SHOULD NOT send either \Marked or \Unmarked. + + The hierarchy delimiter is a character used to delimit levels of + hierarchy in a mailbox name. A client can use it to create child + mailboxes, and to search higher or lower levels of naming + hierarchy. All children of a top-level hierarchy node MUST use + the same separator character. A NIL hierarchy delimiter means + that no hierarchy exists; the name is a "flat" name. + + The name represents an unambiguous left-to-right hierarchy, and + MUST be valid for use as a reference in LIST and LSUB commands. + Unless \Noselect is indicated, the name MUST also be valid as an + argument for commands, such as SELECT, that accept mailbox names. + + Example: S: * LIST (\Noselect) "/" ~/Mail/foo + + +7.2.3. LSUB Response + + Contents: name attributes + hierarchy delimiter + name + + The LSUB response occurs as a result of an LSUB command. It + returns a single name that matches the LSUB specification. There + can be multiple LSUB responses for a single LSUB command. The + data is identical in format to the LIST response. + + Example: S: * LSUB () "." #news.comp.mail.misc + + +7.2.4 STATUS Response + + Contents: name + status parenthesized list + + The STATUS response occurs as a result of an STATUS command. It + returns the mailbox name that matches the STATUS specification and + the requested mailbox status information. + + Example: S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292) + + + + + + + + +Crispin Standards Track [Page 70] + +RFC 3501 IMAPv4 March 2003 + + +7.2.5. SEARCH Response + + Contents: zero or more numbers + + The SEARCH response occurs as a result of a SEARCH or UID SEARCH + command. The number(s) refer to those messages that match the + search criteria. For SEARCH, these are message sequence numbers; + for UID SEARCH, these are unique identifiers. Each number is + delimited by a space. + + Example: S: * SEARCH 2 3 6 + + +7.2.6. FLAGS Response + + Contents: flag parenthesized list + + The FLAGS response occurs as a result of a SELECT or EXAMINE + command. The flag parenthesized list identifies the flags (at a + minimum, the system-defined flags) that are applicable for this + mailbox. Flags other than the system flags can also exist, + depending on server implementation. + + The update from the FLAGS response MUST be recorded by the client. + + Example: S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + + +7.3. Server Responses - Mailbox Size + + These responses are always untagged. This is how changes in the size + of the mailbox are transmitted from the server to the client. + Immediately following the "*" token is a number that represents a + message count. + +7.3.1. EXISTS Response + + Contents: none + + The EXISTS response reports the number of messages in the mailbox. + This response occurs as a result of a SELECT or EXAMINE command, + and if the size of the mailbox changes (e.g., new messages). + + The update from the EXISTS response MUST be recorded by the + client. + + Example: S: * 23 EXISTS + + + + +Crispin Standards Track [Page 71] + +RFC 3501 IMAPv4 March 2003 + + +7.3.2. RECENT Response + + Contents: none + + The RECENT response reports the number of messages with the + \Recent flag set. This response occurs as a result of a SELECT or + EXAMINE command, and if the size of the mailbox changes (e.g., new + messages). + + Note: It is not guaranteed that the message sequence + numbers of recent messages will be a contiguous range of + the highest n messages in the mailbox (where n is the + value reported by the RECENT response). Examples of + situations in which this is not the case are: multiple + clients having the same mailbox open (the first session + to be notified will see it as recent, others will + probably see it as non-recent), and when the mailbox is + re-ordered by a non-IMAP agent. + + The only reliable way to identify recent messages is to + look at message flags to see which have the \Recent flag + set, or to do a SEARCH RECENT. + + The update from the RECENT response MUST be recorded by the + client. + + Example: S: * 5 RECENT + + +7.4. Server Responses - Message Status + + These responses are always untagged. This is how message data are + transmitted from the server to the client, often as a result of a + command with the same name. Immediately following the "*" token is a + number that represents a message sequence number. + +7.4.1. EXPUNGE Response + + Contents: none + + The EXPUNGE response reports that the specified message sequence + number has been permanently removed from the mailbox. The message + sequence number for each successive message in the mailbox is + immediately decremented by 1, and this decrement is reflected in + message sequence numbers in subsequent responses (including other + untagged EXPUNGE responses). + + + + + +Crispin Standards Track [Page 72] + +RFC 3501 IMAPv4 March 2003 + + + The EXPUNGE response also decrements the number of messages in the + mailbox; it is not necessary to send an EXISTS response with the + new value. + + As a result of the immediate decrement rule, message sequence + numbers that appear in a set of successive EXPUNGE responses + depend upon whether the messages are removed starting from lower + numbers to higher numbers, or from higher numbers to lower + numbers. For example, if the last 5 messages in a 9-message + mailbox are expunged, a "lower to higher" server will send five + untagged EXPUNGE responses for message sequence number 5, whereas + a "higher to lower server" will send successive untagged EXPUNGE + responses for message sequence numbers 9, 8, 7, 6, and 5. + + An EXPUNGE response MUST NOT be sent when no command is in + progress, nor while responding to a FETCH, STORE, or SEARCH + command. This rule is necessary to prevent a loss of + synchronization of message sequence numbers between client and + server. A command is not "in progress" until the complete command + has been received; in particular, a command is not "in progress" + during the negotiation of command continuation. + + Note: UID FETCH, UID STORE, and UID SEARCH are different + commands from FETCH, STORE, and SEARCH. An EXPUNGE + response MAY be sent during a UID command. + + The update from the EXPUNGE response MUST be recorded by the + client. + + Example: S: * 44 EXPUNGE + + +7.4.2. FETCH Response + + Contents: message data + + The FETCH response returns data about a message to the client. + The data are pairs of data item names and their values in + parentheses. This response occurs as the result of a FETCH or + STORE command, as well as by unilateral server decision (e.g., + flag updates). + + The current data items are: + + BODY + A form of BODYSTRUCTURE without extension data. + + + + + +Crispin Standards Track [Page 73] + +RFC 3501 IMAPv4 March 2003 + + + BODY[
]<> + A string expressing the body contents of the specified section. + The string SHOULD be interpreted by the client according to the + content transfer encoding, body type, and subtype. + + If the origin octet is specified, this string is a substring of + the entire body contents, starting at that origin octet. This + means that BODY[]<0> MAY be truncated, but BODY[] is NEVER + truncated. + + Note: The origin octet facility MUST NOT be used by a server + in a FETCH response unless the client specifically requested + it by means of a FETCH of a BODY[
]<> data + item. + + 8-bit textual data is permitted if a [CHARSET] identifier is + part of the body parameter parenthesized list for this section. + Note that headers (part specifiers HEADER or MIME, or the + header portion of a MESSAGE/RFC822 part), MUST be 7-bit; 8-bit + characters are not permitted in headers. Note also that the + [RFC-2822] delimiting blank line between the header and the + body is not affected by header line subsetting; the blank line + is always included as part of header data, except in the case + of a message which has no body and no blank line. + + Non-textual data such as binary data MUST be transfer encoded + into a textual form, such as BASE64, prior to being sent to the + client. To derive the original binary data, the client MUST + decode the transfer encoded string. + + BODYSTRUCTURE + A parenthesized list that describes the [MIME-IMB] body + structure of a message. This is computed by the server by + parsing the [MIME-IMB] header fields, defaulting various fields + as necessary. + + For example, a simple text message of 48 lines and 2279 octets + can have a body structure of: ("TEXT" "PLAIN" ("CHARSET" + "US-ASCII") NIL NIL "7BIT" 2279 48) + + Multiple parts are indicated by parenthesis nesting. Instead + of a body type as the first element of the parenthesized list, + there is a sequence of one or more nested body structures. The + second element of the parenthesized list is the multipart + subtype (mixed, digest, parallel, alternative, etc.). + + + + + + +Crispin Standards Track [Page 74] + +RFC 3501 IMAPv4 March 2003 + + + For example, a two part message consisting of a text and a + BASE64-encoded text attachment can have a body structure of: + (("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 1152 + 23)("TEXT" "PLAIN" ("CHARSET" "US-ASCII" "NAME" "cc.diff") + "<960723163407.20117h@cac.washington.edu>" "Compiler diff" + "BASE64" 4554 73) "MIXED") + + Extension data follows the multipart subtype. Extension data + is never returned with the BODY fetch, but can be returned with + a BODYSTRUCTURE fetch. Extension data, if present, MUST be in + the defined order. The extension data of a multipart body part + are in the following order: + + body parameter parenthesized list + A parenthesized list of attribute/value pairs [e.g., ("foo" + "bar" "baz" "rag") where "bar" is the value of "foo", and + "rag" is the value of "baz"] as defined in [MIME-IMB]. + + body disposition + A parenthesized list, consisting of a disposition type + string, followed by a parenthesized list of disposition + attribute/value pairs as defined in [DISPOSITION]. + + body language + A string or parenthesized list giving the body language + value as defined in [LANGUAGE-TAGS]. + + body location + A string list giving the body content URI as defined in + [LOCATION]. + + Any following extension data are not yet defined in this + version of the protocol. Such extension data can consist of + zero or more NILs, strings, numbers, or potentially nested + parenthesized lists of such data. Client implementations that + do a BODYSTRUCTURE fetch MUST be prepared to accept such + extension data. Server implementations MUST NOT send such + extension data until it has been defined by a revision of this + protocol. + + The basic fields of a non-multipart body part are in the + following order: + + body type + A string giving the content media type name as defined in + [MIME-IMB]. + + + + + +Crispin Standards Track [Page 75] + +RFC 3501 IMAPv4 March 2003 + + + body subtype + A string giving the content subtype name as defined in + [MIME-IMB]. + + body parameter parenthesized list + A parenthesized list of attribute/value pairs [e.g., ("foo" + "bar" "baz" "rag") where "bar" is the value of "foo" and + "rag" is the value of "baz"] as defined in [MIME-IMB]. + + body id + A string giving the content id as defined in [MIME-IMB]. + + body description + A string giving the content description as defined in + [MIME-IMB]. + + body encoding + A string giving the content transfer encoding as defined in + [MIME-IMB]. + + body size + A number giving the size of the body in octets. Note that + this size is the size in its transfer encoding and not the + resulting size after any decoding. + + A body type of type MESSAGE and subtype RFC822 contains, + immediately after the basic fields, the envelope structure, + body structure, and size in text lines of the encapsulated + message. + + A body type of type TEXT contains, immediately after the basic + fields, the size of the body in text lines. Note that this + size is the size in its content transfer encoding and not the + resulting size after any decoding. + + Extension data follows the basic fields and the type-specific + fields listed above. Extension data is never returned with the + BODY fetch, but can be returned with a BODYSTRUCTURE fetch. + Extension data, if present, MUST be in the defined order. + + The extension data of a non-multipart body part are in the + following order: + + body MD5 + A string giving the body MD5 value as defined in [MD5]. + + + + + + +Crispin Standards Track [Page 76] + +RFC 3501 IMAPv4 March 2003 + + + body disposition + A parenthesized list with the same content and function as + the body disposition for a multipart body part. + + body language + A string or parenthesized list giving the body language + value as defined in [LANGUAGE-TAGS]. + + body location + A string list giving the body content URI as defined in + [LOCATION]. + + Any following extension data are not yet defined in this + version of the protocol, and would be as described above under + multipart extension data. + + ENVELOPE + A parenthesized list that describes the envelope structure of a + message. This is computed by the server by parsing the + [RFC-2822] header into the component parts, defaulting various + fields as necessary. + + The fields of the envelope structure are in the following + order: date, subject, from, sender, reply-to, to, cc, bcc, + in-reply-to, and message-id. The date, subject, in-reply-to, + and message-id fields are strings. The from, sender, reply-to, + to, cc, and bcc fields are parenthesized lists of address + structures. + + An address structure is a parenthesized list that describes an + electronic mail address. The fields of an address structure + are in the following order: personal name, [SMTP] + at-domain-list (source route), mailbox name, and host name. + + [RFC-2822] group syntax is indicated by a special form of + address structure in which the host name field is NIL. If the + mailbox name field is also NIL, this is an end of group marker + (semi-colon in RFC 822 syntax). If the mailbox name field is + non-NIL, this is a start of group marker, and the mailbox name + field holds the group name phrase. + + If the Date, Subject, In-Reply-To, and Message-ID header lines + are absent in the [RFC-2822] header, the corresponding member + of the envelope is NIL; if these header lines are present but + empty the corresponding member of the envelope is the empty + string. + + + + + +Crispin Standards Track [Page 77] + +RFC 3501 IMAPv4 March 2003 + + + Note: some servers may return a NIL envelope member in the + "present but empty" case. Clients SHOULD treat NIL and + empty string as identical. + + Note: [RFC-2822] requires that all messages have a valid + Date header. Therefore, the date member in the envelope can + not be NIL or the empty string. + + Note: [RFC-2822] requires that the In-Reply-To and + Message-ID headers, if present, have non-empty content. + Therefore, the in-reply-to and message-id members in the + envelope can not be the empty string. + + If the From, To, cc, and bcc header lines are absent in the + [RFC-2822] header, or are present but empty, the corresponding + member of the envelope is NIL. + + If the Sender or Reply-To lines are absent in the [RFC-2822] + header, or are present but empty, the server sets the + corresponding member of the envelope to be the same value as + the from member (the client is not expected to know to do + this). + + Note: [RFC-2822] requires that all messages have a valid + From header. Therefore, the from, sender, and reply-to + members in the envelope can not be NIL. + + FLAGS + A parenthesized list of flags that are set for this message. + + INTERNALDATE + A string representing the internal date of the message. + + RFC822 + Equivalent to BODY[]. + + RFC822.HEADER + Equivalent to BODY[HEADER]. Note that this did not result in + \Seen being set, because RFC822.HEADER response data occurs as + a result of a FETCH of RFC822.HEADER. BODY[HEADER] response + data occurs as a result of a FETCH of BODY[HEADER] (which sets + \Seen) or BODY.PEEK[HEADER] (which does not set \Seen). + + RFC822.SIZE + A number expressing the [RFC-2822] size of the message. + + + + + + +Crispin Standards Track [Page 78] + +RFC 3501 IMAPv4 March 2003 + + + RFC822.TEXT + Equivalent to BODY[TEXT]. + + UID + A number expressing the unique identifier of the message. + + + Example: S: * 23 FETCH (FLAGS (\Seen) RFC822.SIZE 44827) + + +7.5. Server Responses - Command Continuation Request + + The command continuation request response is indicated by a "+" token + instead of a tag. This form of response indicates that the server is + ready to accept the continuation of a command from the client. The + remainder of this response is a line of text. + + This response is used in the AUTHENTICATE command to transmit server + data to the client, and request additional client data. This + response is also used if an argument to any command is a literal. + + The client is not permitted to send the octets of the literal unless + the server indicates that it is expected. This permits the server to + process commands and reject errors on a line-by-line basis. The + remainder of the command, including the CRLF that terminates a + command, follows the octets of the literal. If there are any + additional command arguments, the literal octets are followed by a + space and those arguments. + + Example: C: A001 LOGIN {11} + S: + Ready for additional command text + C: FRED FOOBAR {7} + S: + Ready for additional command text + C: fat man + S: A001 OK LOGIN completed + C: A044 BLURDYBLOOP {102856} + S: A044 BAD No such command as "BLURDYBLOOP" + + + + + + + + + + + + + + +Crispin Standards Track [Page 79] + +RFC 3501 IMAPv4 March 2003 + + +8. Sample IMAP4rev1 connection + + The following is a transcript of an IMAP4rev1 connection. A long + line in this sample is broken for editorial clarity. + +S: * OK IMAP4rev1 Service Ready +C: a001 login mrc secret +S: a001 OK LOGIN completed +C: a002 select inbox +S: * 18 EXISTS +S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) +S: * 2 RECENT +S: * OK [UNSEEN 17] Message 17 is the first unseen message +S: * OK [UIDVALIDITY 3857529045] UIDs valid +S: a002 OK [READ-WRITE] SELECT completed +C: a003 fetch 12 full +S: * 12 FETCH (FLAGS (\Seen) INTERNALDATE "17-Jul-1996 02:44:25 -0700" + RFC822.SIZE 4286 ENVELOPE ("Wed, 17 Jul 1996 02:23:25 -0700 (PDT)" + "IMAP4rev1 WG mtg summary and minutes" + (("Terry Gray" NIL "gray" "cac.washington.edu")) + (("Terry Gray" NIL "gray" "cac.washington.edu")) + (("Terry Gray" NIL "gray" "cac.washington.edu")) + ((NIL NIL "imap" "cac.washington.edu")) + ((NIL NIL "minutes" "CNRI.Reston.VA.US") + ("John Klensin" NIL "KLENSIN" "MIT.EDU")) NIL NIL + "") + BODY ("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 3028 + 92)) +S: a003 OK FETCH completed +C: a004 fetch 12 body[header] +S: * 12 FETCH (BODY[HEADER] {342} +S: Date: Wed, 17 Jul 1996 02:23:25 -0700 (PDT) +S: From: Terry Gray +S: Subject: IMAP4rev1 WG mtg summary and minutes +S: To: imap@cac.washington.edu +S: cc: minutes@CNRI.Reston.VA.US, John Klensin +S: Message-Id: +S: MIME-Version: 1.0 +S: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII +S: +S: ) +S: a004 OK FETCH completed +C: a005 store 12 +flags \deleted +S: * 12 FETCH (FLAGS (\Seen \Deleted)) +S: a005 OK +FLAGS completed +C: a006 logout +S: * BYE IMAP4rev1 server terminating connection +S: a006 OK LOGOUT completed + + + +Crispin Standards Track [Page 80] + +RFC 3501 IMAPv4 March 2003 + + +9. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + In the case of alternative or optional rules in which a later rule + overlaps an earlier rule, the rule which is listed earlier MUST take + priority. For example, "\Seen" when parsed as a flag is the \Seen + flag name and not a flag-extension, even though "\Seen" can be parsed + as a flag-extension. Some, but not all, instances of this rule are + noted below. + + Note: [ABNF] rules MUST be followed strictly; in + particular: + + (1) Except as noted otherwise, all alphabetic characters + are case-insensitive. The use of upper or lower case + characters to define token strings is for editorial clarity + only. Implementations MUST accept these strings in a + case-insensitive fashion. + + (2) In all cases, SP refers to exactly one space. It is + NOT permitted to substitute TAB, insert additional spaces, + or otherwise treat SP as being equivalent to LWSP. + + (3) The ASCII NUL character, %x00, MUST NOT be used at any + time. + +address = "(" addr-name SP addr-adl SP addr-mailbox SP + addr-host ")" + +addr-adl = nstring + ; Holds route from [RFC-2822] route-addr if + ; non-NIL + +addr-host = nstring + ; NIL indicates [RFC-2822] group syntax. + ; Otherwise, holds [RFC-2822] domain name + +addr-mailbox = nstring + ; NIL indicates end of [RFC-2822] group; if + ; non-NIL and addr-host is NIL, holds + ; [RFC-2822] group name. + ; Otherwise, holds [RFC-2822] local-part + ; after removing [RFC-2822] quoting + + + + + + +Crispin Standards Track [Page 81] + +RFC 3501 IMAPv4 March 2003 + + +addr-name = nstring + ; If non-NIL, holds phrase from [RFC-2822] + ; mailbox after removing [RFC-2822] quoting + +append = "APPEND" SP mailbox [SP flag-list] [SP date-time] SP + literal + +astring = 1*ASTRING-CHAR / string + +ASTRING-CHAR = ATOM-CHAR / resp-specials + +atom = 1*ATOM-CHAR + +ATOM-CHAR = + +atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / + quoted-specials / resp-specials + +authenticate = "AUTHENTICATE" SP auth-type *(CRLF base64) + +auth-type = atom + ; Defined by [SASL] + +base64 = *(4base64-char) [base64-terminal] + +base64-char = ALPHA / DIGIT / "+" / "/" + ; Case-sensitive + +base64-terminal = (2base64-char "==") / (3base64-char "=") + +body = "(" (body-type-1part / body-type-mpart) ")" + +body-extension = nstring / number / + "(" body-extension *(SP body-extension) ")" + ; Future expansion. Client implementations + ; MUST accept body-extension fields. Server + ; implementations MUST NOT generate + ; body-extension fields except as defined by + ; future standard or standards-track + ; revisions of this specification. + +body-ext-1part = body-fld-md5 [SP body-fld-dsp [SP body-fld-lang + [SP body-fld-loc *(SP body-extension)]]] + ; MUST NOT be returned on non-extensible + ; "BODY" fetch + + + + + + +Crispin Standards Track [Page 82] + +RFC 3501 IMAPv4 March 2003 + + +body-ext-mpart = body-fld-param [SP body-fld-dsp [SP body-fld-lang + [SP body-fld-loc *(SP body-extension)]]] + ; MUST NOT be returned on non-extensible + ; "BODY" fetch + +body-fields = body-fld-param SP body-fld-id SP body-fld-desc SP + body-fld-enc SP body-fld-octets + +body-fld-desc = nstring + +body-fld-dsp = "(" string SP body-fld-param ")" / nil + +body-fld-enc = (DQUOTE ("7BIT" / "8BIT" / "BINARY" / "BASE64"/ + "QUOTED-PRINTABLE") DQUOTE) / string + +body-fld-id = nstring + +body-fld-lang = nstring / "(" string *(SP string) ")" + +body-fld-loc = nstring + +body-fld-lines = number + +body-fld-md5 = nstring + +body-fld-octets = number + +body-fld-param = "(" string SP string *(SP string SP string) ")" / nil + +body-type-1part = (body-type-basic / body-type-msg / body-type-text) + [SP body-ext-1part] + +body-type-basic = media-basic SP body-fields + ; MESSAGE subtype MUST NOT be "RFC822" + +body-type-mpart = 1*body SP media-subtype + [SP body-ext-mpart] + +body-type-msg = media-message SP body-fields SP envelope + SP body SP body-fld-lines + +body-type-text = media-text SP body-fields SP body-fld-lines + +capability = ("AUTH=" auth-type) / atom + ; New capabilities MUST begin with "X" or be + ; registered with IANA as standard or + ; standards-track + + + + +Crispin Standards Track [Page 83] + +RFC 3501 IMAPv4 March 2003 + + +capability-data = "CAPABILITY" *(SP capability) SP "IMAP4rev1" + *(SP capability) + ; Servers MUST implement the STARTTLS, AUTH=PLAIN, + ; and LOGINDISABLED capabilities + ; Servers which offer RFC 1730 compatibility MUST + ; list "IMAP4" as the first capability. + +CHAR8 = %x01-ff + ; any OCTET except NUL, %x00 + +command = tag SP (command-any / command-auth / command-nonauth / + command-select) CRLF + ; Modal based on state + +command-any = "CAPABILITY" / "LOGOUT" / "NOOP" / x-command + ; Valid in all states + +command-auth = append / create / delete / examine / list / lsub / + rename / select / status / subscribe / unsubscribe + ; Valid only in Authenticated or Selected state + +command-nonauth = login / authenticate / "STARTTLS" + ; Valid only when in Not Authenticated state + +command-select = "CHECK" / "CLOSE" / "EXPUNGE" / copy / fetch / store / + uid / search + ; Valid only when in Selected state + +continue-req = "+" SP (resp-text / base64) CRLF + +copy = "COPY" SP sequence-set SP mailbox + +create = "CREATE" SP mailbox + ; Use of INBOX gives a NO error + +date = date-text / DQUOTE date-text DQUOTE + +date-day = 1*2DIGIT + ; Day of month + +date-day-fixed = (SP DIGIT) / 2DIGIT + ; Fixed-format version of date-day + +date-month = "Jan" / "Feb" / "Mar" / "Apr" / "May" / "Jun" / + "Jul" / "Aug" / "Sep" / "Oct" / "Nov" / "Dec" + +date-text = date-day "-" date-month "-" date-year + + + + +Crispin Standards Track [Page 84] + +RFC 3501 IMAPv4 March 2003 + + +date-year = 4DIGIT + +date-time = DQUOTE date-day-fixed "-" date-month "-" date-year + SP time SP zone DQUOTE + +delete = "DELETE" SP mailbox + ; Use of INBOX gives a NO error + +digit-nz = %x31-39 + ; 1-9 + +envelope = "(" env-date SP env-subject SP env-from SP + env-sender SP env-reply-to SP env-to SP env-cc SP + env-bcc SP env-in-reply-to SP env-message-id ")" + +env-bcc = "(" 1*address ")" / nil + +env-cc = "(" 1*address ")" / nil + +env-date = nstring + +env-from = "(" 1*address ")" / nil + +env-in-reply-to = nstring + +env-message-id = nstring + +env-reply-to = "(" 1*address ")" / nil + +env-sender = "(" 1*address ")" / nil + +env-subject = nstring + +env-to = "(" 1*address ")" / nil + +examine = "EXAMINE" SP mailbox + +fetch = "FETCH" SP sequence-set SP ("ALL" / "FULL" / "FAST" / + fetch-att / "(" fetch-att *(SP fetch-att) ")") + +fetch-att = "ENVELOPE" / "FLAGS" / "INTERNALDATE" / + "RFC822" [".HEADER" / ".SIZE" / ".TEXT"] / + "BODY" ["STRUCTURE"] / "UID" / + "BODY" section ["<" number "." nz-number ">"] / + "BODY.PEEK" section ["<" number "." nz-number ">"] + + + + + + +Crispin Standards Track [Page 85] + +RFC 3501 IMAPv4 March 2003 + + +flag = "\Answered" / "\Flagged" / "\Deleted" / + "\Seen" / "\Draft" / flag-keyword / flag-extension + ; Does not include "\Recent" + +flag-extension = "\" atom + ; Future expansion. Client implementations + ; MUST accept flag-extension flags. Server + ; implementations MUST NOT generate + ; flag-extension flags except as defined by + ; future standard or standards-track + ; revisions of this specification. + +flag-fetch = flag / "\Recent" + +flag-keyword = atom + +flag-list = "(" [flag *(SP flag)] ")" + +flag-perm = flag / "\*" + +greeting = "*" SP (resp-cond-auth / resp-cond-bye) CRLF + +header-fld-name = astring + +header-list = "(" header-fld-name *(SP header-fld-name) ")" + +list = "LIST" SP mailbox SP list-mailbox + +list-mailbox = 1*list-char / string + +list-char = ATOM-CHAR / list-wildcards / resp-specials + +list-wildcards = "%" / "*" + +literal = "{" number "}" CRLF *CHAR8 + ; Number represents the number of CHAR8s + +login = "LOGIN" SP userid SP password + +lsub = "LSUB" SP mailbox SP list-mailbox + + + + + + + + + + + +Crispin Standards Track [Page 86] + +RFC 3501 IMAPv4 March 2003 + + +mailbox = "INBOX" / astring + ; INBOX is case-insensitive. All case variants of + ; INBOX (e.g., "iNbOx") MUST be interpreted as INBOX + ; not as an astring. An astring which consists of + ; the case-insensitive sequence "I" "N" "B" "O" "X" + ; is considered to be INBOX and not an astring. + ; Refer to section 5.1 for further + ; semantic details of mailbox names. + +mailbox-data = "FLAGS" SP flag-list / "LIST" SP mailbox-list / + "LSUB" SP mailbox-list / "SEARCH" *(SP nz-number) / + "STATUS" SP mailbox SP "(" [status-att-list] ")" / + number SP "EXISTS" / number SP "RECENT" + +mailbox-list = "(" [mbx-list-flags] ")" SP + (DQUOTE QUOTED-CHAR DQUOTE / nil) SP mailbox + +mbx-list-flags = *(mbx-list-oflag SP) mbx-list-sflag + *(SP mbx-list-oflag) / + mbx-list-oflag *(SP mbx-list-oflag) + +mbx-list-oflag = "\Noinferiors" / flag-extension + ; Other flags; multiple possible per LIST response + +mbx-list-sflag = "\Noselect" / "\Marked" / "\Unmarked" + ; Selectability flags; only one per LIST response + +media-basic = ((DQUOTE ("APPLICATION" / "AUDIO" / "IMAGE" / + "MESSAGE" / "VIDEO") DQUOTE) / string) SP + media-subtype + ; Defined in [MIME-IMT] + +media-message = DQUOTE "MESSAGE" DQUOTE SP DQUOTE "RFC822" DQUOTE + ; Defined in [MIME-IMT] + +media-subtype = string + ; Defined in [MIME-IMT] + +media-text = DQUOTE "TEXT" DQUOTE SP media-subtype + ; Defined in [MIME-IMT] + +message-data = nz-number SP ("EXPUNGE" / ("FETCH" SP msg-att)) + +msg-att = "(" (msg-att-dynamic / msg-att-static) + *(SP (msg-att-dynamic / msg-att-static)) ")" + +msg-att-dynamic = "FLAGS" SP "(" [flag-fetch *(SP flag-fetch)] ")" + ; MAY change for a message + + + +Crispin Standards Track [Page 87] + +RFC 3501 IMAPv4 March 2003 + + +msg-att-static = "ENVELOPE" SP envelope / "INTERNALDATE" SP date-time / + "RFC822" [".HEADER" / ".TEXT"] SP nstring / + "RFC822.SIZE" SP number / + "BODY" ["STRUCTURE"] SP body / + "BODY" section ["<" number ">"] SP nstring / + "UID" SP uniqueid + ; MUST NOT change for a message + +nil = "NIL" + +nstring = string / nil + +number = 1*DIGIT + ; Unsigned 32-bit integer + ; (0 <= n < 4,294,967,296) + +nz-number = digit-nz *DIGIT + ; Non-zero unsigned 32-bit integer + ; (0 < n < 4,294,967,296) + +password = astring + +quoted = DQUOTE *QUOTED-CHAR DQUOTE + +QUOTED-CHAR = / + "\" quoted-specials + +quoted-specials = DQUOTE / "\" + +rename = "RENAME" SP mailbox SP mailbox + ; Use of INBOX as a destination gives a NO error + +response = *(continue-req / response-data) response-done + +response-data = "*" SP (resp-cond-state / resp-cond-bye / + mailbox-data / message-data / capability-data) CRLF + +response-done = response-tagged / response-fatal + +response-fatal = "*" SP resp-cond-bye CRLF + ; Server closes connection immediately + +response-tagged = tag SP resp-cond-state CRLF + +resp-cond-auth = ("OK" / "PREAUTH") SP resp-text + ; Authentication condition + + + + + +Crispin Standards Track [Page 88] + +RFC 3501 IMAPv4 March 2003 + + +resp-cond-bye = "BYE" SP resp-text + +resp-cond-state = ("OK" / "NO" / "BAD") SP resp-text + ; Status condition + +resp-specials = "]" + +resp-text = ["[" resp-text-code "]" SP] text + +resp-text-code = "ALERT" / + "BADCHARSET" [SP "(" astring *(SP astring) ")" ] / + capability-data / "PARSE" / + "PERMANENTFLAGS" SP "(" + [flag-perm *(SP flag-perm)] ")" / + "READ-ONLY" / "READ-WRITE" / "TRYCREATE" / + "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number / + "UNSEEN" SP nz-number / + atom [SP 1*] + +search = "SEARCH" [SP "CHARSET" SP astring] 1*(SP search-key) + ; CHARSET argument to MUST be registered with IANA + +search-key = "ALL" / "ANSWERED" / "BCC" SP astring / + "BEFORE" SP date / "BODY" SP astring / + "CC" SP astring / "DELETED" / "FLAGGED" / + "FROM" SP astring / "KEYWORD" SP flag-keyword / + "NEW" / "OLD" / "ON" SP date / "RECENT" / "SEEN" / + "SINCE" SP date / "SUBJECT" SP astring / + "TEXT" SP astring / "TO" SP astring / + "UNANSWERED" / "UNDELETED" / "UNFLAGGED" / + "UNKEYWORD" SP flag-keyword / "UNSEEN" / + ; Above this line were in [IMAP2] + "DRAFT" / "HEADER" SP header-fld-name SP astring / + "LARGER" SP number / "NOT" SP search-key / + "OR" SP search-key SP search-key / + "SENTBEFORE" SP date / "SENTON" SP date / + "SENTSINCE" SP date / "SMALLER" SP number / + "UID" SP sequence-set / "UNDRAFT" / sequence-set / + "(" search-key *(SP search-key) ")" + +section = "[" [section-spec] "]" + +section-msgtext = "HEADER" / "HEADER.FIELDS" [".NOT"] SP header-list / + "TEXT" + ; top-level or MESSAGE/RFC822 part + +section-part = nz-number *("." nz-number) + ; body part nesting + + + +Crispin Standards Track [Page 89] + +RFC 3501 IMAPv4 March 2003 + + +section-spec = section-msgtext / (section-part ["." section-text]) + +section-text = section-msgtext / "MIME" + ; text other than actual body part (headers, etc.) + +select = "SELECT" SP mailbox + +seq-number = nz-number / "*" + ; message sequence number (COPY, FETCH, STORE + ; commands) or unique identifier (UID COPY, + ; UID FETCH, UID STORE commands). + ; * represents the largest number in use. In + ; the case of message sequence numbers, it is + ; the number of messages in a non-empty mailbox. + ; In the case of unique identifiers, it is the + ; unique identifier of the last message in the + ; mailbox or, if the mailbox is empty, the + ; mailbox's current UIDNEXT value. + ; The server should respond with a tagged BAD + ; response to a command that uses a message + ; sequence number greater than the number of + ; messages in the selected mailbox. This + ; includes "*" if the selected mailbox is empty. + +seq-range = seq-number ":" seq-number + ; two seq-number values and all values between + ; these two regardless of order. + ; Example: 2:4 and 4:2 are equivalent and indicate + ; values 2, 3, and 4. + ; Example: a unique identifier sequence range of + ; 3291:* includes the UID of the last message in + ; the mailbox, even if that value is less than 3291. + +sequence-set = (seq-number / seq-range) *("," sequence-set) + ; set of seq-number values, regardless of order. + ; Servers MAY coalesce overlaps and/or execute the + ; sequence in any order. + ; Example: a message sequence number set of + ; 2,4:7,9,12:* for a mailbox with 15 messages is + ; equivalent to 2,4,5,6,7,9,12,13,14,15 + ; Example: a message sequence number set of *:4,5:7 + ; for a mailbox with 10 messages is equivalent to + ; 10,9,8,7,6,5,4,5,6,7 and MAY be reordered and + ; overlap coalesced to be 4,5,6,7,8,9,10. + +status = "STATUS" SP mailbox SP + "(" status-att *(SP status-att) ")" + + + + +Crispin Standards Track [Page 90] + +RFC 3501 IMAPv4 March 2003 + + +status-att = "MESSAGES" / "RECENT" / "UIDNEXT" / "UIDVALIDITY" / + "UNSEEN" + +status-att-list = status-att SP number *(SP status-att SP number) + +store = "STORE" SP sequence-set SP store-att-flags + +store-att-flags = (["+" / "-"] "FLAGS" [".SILENT"]) SP + (flag-list / (flag *(SP flag))) + +string = quoted / literal + +subscribe = "SUBSCRIBE" SP mailbox + +tag = 1* + +text = 1*TEXT-CHAR + +TEXT-CHAR = + +time = 2DIGIT ":" 2DIGIT ":" 2DIGIT + ; Hours minutes seconds + +uid = "UID" SP (copy / fetch / search / store) + ; Unique identifiers used instead of message + ; sequence numbers + +uniqueid = nz-number + ; Strictly ascending + +unsubscribe = "UNSUBSCRIBE" SP mailbox + +userid = astring + +x-command = "X" atom + +zone = ("+" / "-") 4DIGIT + ; Signed four-digit value of hhmm representing + ; hours and minutes east of Greenwich (that is, + ; the amount that the given time differs from + ; Universal Time). Subtracting the timezone + ; from the given time will give the UT form. + ; The Universal Time zone is "+0000". + + + + + + + + +Crispin Standards Track [Page 91] + +RFC 3501 IMAPv4 March 2003 + + +10. Author's Note + + This document is a revision or rewrite of earlier documents, and + supercedes the protocol specification in those documents: RFC 2060, + RFC 1730, unpublished IMAP2bis.TXT document, RFC 1176, and RFC 1064. + +11. Security Considerations + + IMAP4rev1 protocol transactions, including electronic mail data, are + sent in the clear over the network unless protection from snooping is + negotiated. This can be accomplished either by the use of STARTTLS, + negotiated privacy protection in the AUTHENTICATE command, or some + other protection mechanism. + +11.1. STARTTLS Security Considerations + + The specification of the STARTTLS command and LOGINDISABLED + capability in this document replaces that in [IMAP-TLS]. [IMAP-TLS] + remains normative for the PLAIN [SASL] authenticator. + + IMAP client and server implementations MUST implement the + TLS_RSA_WITH_RC4_128_MD5 [TLS] cipher suite, and SHOULD implement the + TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA [TLS] cipher suite. This is + important as it assures that any two compliant implementations can be + configured to interoperate. All other cipher suites are OPTIONAL. + Note that this is a change from section 2.1 of [IMAP-TLS]. + + During the [TLS] negotiation, the client MUST check its understanding + of the server hostname against the server's identity as presented in + the server Certificate message, in order to prevent man-in-the-middle + attacks. If the match fails, the client SHOULD either ask for + explicit user confirmation, or terminate the connection and indicate + that the server's identity is suspect. Matching is performed + according to these rules: + + The client MUST use the server hostname it used to open the + connection as the value to compare against the server name + as expressed in the server certificate. The client MUST + NOT use any form of the server hostname derived from an + insecure remote source (e.g., insecure DNS lookup). CNAME + canonicalization is not done. + + If a subjectAltName extension of type dNSName is present in + the certificate, it SHOULD be used as the source of the + server's identity. + + Matching is case-insensitive. + + + + +Crispin Standards Track [Page 92] + +RFC 3501 IMAPv4 March 2003 + + + A "*" wildcard character MAY be used as the left-most name + component in the certificate. For example, *.example.com + would match a.example.com, foo.example.com, etc. but would + not match example.com. + + If the certificate contains multiple names (e.g., more than + one dNSName field), then a match with any one of the fields + is considered acceptable. + + Both the client and server MUST check the result of the STARTTLS + command and subsequent [TLS] negotiation to see whether acceptable + authentication or privacy was achieved. + +11.2. Other Security Considerations + + A server error message for an AUTHENTICATE command which fails due to + invalid credentials SHOULD NOT detail why the credentials are + invalid. + + Use of the LOGIN command sends passwords in the clear. This can be + avoided by using the AUTHENTICATE command with a [SASL] mechanism + that does not use plaintext passwords, by first negotiating + encryption via STARTTLS or some other protection mechanism. + + A server implementation MUST implement a configuration that, at the + time of authentication, requires: + (1) The STARTTLS command has been negotiated. + OR + (2) Some other mechanism that protects the session from password + snooping has been provided. + OR + (3) The following measures are in place: + (a) The LOGINDISABLED capability is advertised, and [SASL] + mechanisms (such as PLAIN) using plaintext passwords are NOT + advertised in the CAPABILITY list. + AND + (b) The LOGIN command returns an error even if the password is + correct. + AND + (c) The AUTHENTICATE command returns an error with all [SASL] + mechanisms that use plaintext passwords, even if the password + is correct. + + A server error message for a failing LOGIN command SHOULD NOT specify + that the user name, as opposed to the password, is invalid. + + A server SHOULD have mechanisms in place to limit or delay failed + AUTHENTICATE/LOGIN attempts. + + + +Crispin Standards Track [Page 93] + +RFC 3501 IMAPv4 March 2003 + + + Additional security considerations are discussed in the section + discussing the AUTHENTICATE and LOGIN commands. + +12. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + As this specification revises the STARTTLS and LOGINDISABLED + extensions previously defined in [IMAP-TLS], the registry will be + updated accordingly. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 94] + +RFC 3501 IMAPv4 March 2003 + + +Appendices + +A. Normative References + + The following documents contain definitions or specifications that + are necessary to understand this document properly: + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 2234, + November 1997. + + [ANONYMOUS] Newman, C., "Anonymous SASL Mechanism", RFC + 2245, November 1997. + + [CHARSET] Freed, N. and J. Postel, "IANA Character Set + Registration Procedures", RFC 2978, October + 2000. + + [DIGEST-MD5] Leach, P. and C. Newman, "Using Digest + Authentication as a SASL Mechanism", RFC 2831, + May 2000. + + [DISPOSITION] Troost, R., Dorner, S. and K. Moore, + "Communicating Presentation Information in + Internet Messages: The Content-Disposition + Header", RFC 2183, August 1997. + + [IMAP-TLS] Newman, C., "Using TLS with IMAP, POP3 and + ACAP", RFC 2595, June 1999. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to + Indicate Requirement Levels", BCP 14, RFC 2119, + March 1997. + + [LANGUAGE-TAGS] Alvestrand, H., "Tags for the Identification of + Languages", BCP 47, RFC 3066, January 2001. + + [LOCATION] Palme, J., Hopmann, A. and N. Shelness, "MIME + Encapsulation of Aggregate Documents, such as + HTML (MHTML)", RFC 2557, March 1999. + + [MD5] Myers, J. and M. Rose, "The Content-MD5 Header + Field", RFC 1864, October 1995. + + + + + + + + + +Crispin Standards Track [Page 95] + +RFC 3501 IMAPv4 March 2003 + + + [MIME-HDRS] Moore, K., "MIME (Multipurpose Internet Mail + Extensions) Part Three: Message Header + Extensions for Non-ASCII Text", RFC 2047, + November 1996. + + [MIME-IMB] Freed, N. and N. Borenstein, "MIME + (Multipurpose Internet Mail Extensions) Part + One: Format of Internet Message Bodies", RFC + 2045, November 1996. + + [MIME-IMT] Freed, N. and N. Borenstein, "MIME + (Multipurpose Internet Mail Extensions) Part + Two: Media Types", RFC 2046, November 1996. + + [RFC-2822] Resnick, P., "Internet Message Format", RFC + 2822, April 2001. + + [SASL] Myers, J., "Simple Authentication and Security + Layer (SASL)", RFC 2222, October 1997. + + [TLS] Dierks, T. and C. Allen, "The TLS Protocol + Version 1.0", RFC 2246, January 1999. + + [UTF-7] Goldsmith, D. and M. Davis, "UTF-7: A Mail-Safe + Transformation Format of Unicode", RFC 2152, + May 1997. + + The following documents describe quality-of-implementation issues + that should be carefully considered when implementing this protocol: + + [IMAP-IMPLEMENTATION] Leiba, B., "IMAP Implementation + Recommendations", RFC 2683, September 1999. + + [IMAP-MULTIACCESS] Gahrns, M., "IMAP4 Multi-Accessed Mailbox + Practice", RFC 2180, July 1997. + +A.1 Informative References + + The following documents describe related protocols: + + [IMAP-DISC] Austein, R., "Synchronization Operations for + Disconnected IMAP4 Clients", Work in Progress. + + [IMAP-MODEL] Crispin, M., "Distributed Electronic Mail + Models in IMAP4", RFC 1733, December 1994. + + + + + + +Crispin Standards Track [Page 96] + +RFC 3501 IMAPv4 March 2003 + + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, + November 1997. + + [SMTP] Klensin, J., "Simple Mail Transfer Protocol", + STD 10, RFC 2821, April 2001. + + The following documents are historical or describe historical aspects + of this protocol: + + [IMAP-COMPAT] Crispin, M., "IMAP4 Compatibility with + IMAP2bis", RFC 2061, December 1996. + + [IMAP-HISTORICAL] Crispin, M., "IMAP4 Compatibility with IMAP2 + and IMAP2bis", RFC 1732, December 1994. + + [IMAP-OBSOLETE] Crispin, M., "Internet Message Access Protocol + - Obsolete Syntax", RFC 2062, December 1996. + + [IMAP2] Crispin, M., "Interactive Mail Access Protocol + - Version 2", RFC 1176, August 1990. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA + Internet Text Messages", STD 11, RFC 822, + August 1982. + + [RFC-821] Postel, J., "Simple Mail Transfer Protocol", + STD 10, RFC 821, August 1982. + +B. Changes from RFC 2060 + + 1) Clarify description of unique identifiers and their semantics. + + 2) Fix the SELECT description to clarify that UIDVALIDITY is required + in the SELECT and EXAMINE responses. + + 3) Added an example of a failing search. + + 4) Correct store-att-flags: "#flag" should be "1#flag". + + 5) Made search and section rules clearer. + + 6) Correct the STORE example. + + 7) Correct "BASE645" misspelling. + + 8) Remove extraneous close parenthesis in example of two-part message + with text and BASE64 attachment. + + + +Crispin Standards Track [Page 97] + +RFC 3501 IMAPv4 March 2003 + + + 9) Remove obsolete "MAILBOX" response from mailbox-data. + + 10) A spurious "<" in the rule for mailbox-data was removed. + + 11) Add CRLF to continue-req. + + 12) Specifically exclude "]" from the atom in resp-text-code. + + 13) Clarify that clients and servers should adhere strictly to the + protocol syntax. + + 14) Emphasize in 5.2 that EXISTS can not be used to shrink a mailbox. + + 15) Add NEWNAME to resp-text-code. + + 16) Clarify that the empty string, not NIL, is used as arguments to + LIST. + + 17) Clarify that NIL can be returned as a hierarchy delimiter for the + empty string mailbox name argument if the mailbox namespace is flat. + + 18) Clarify that addr-mailbox and addr-name have RFC-2822 quoting + removed. + + 19) Update UTF-7 reference. + + 20) Fix example in 6.3.11. + + 21) Clarify that non-existent UIDs are ignored. + + 22) Update DISPOSITION reference. + + 23) Expand state diagram. + + 24) Clarify that partial fetch responses are only returned in + response to a partial fetch command. + + 25) Add UIDNEXT response code. Correct UIDVALIDITY definition + reference. + + 26) Further clarification of "can" vs. "MAY". + + 27) Reference RFC-2119. + + 28) Clarify that superfluous shifts are not permitted in modified + UTF-7. + + 29) Clarify that there are no implicit shifts in modified UTF-7. + + + +Crispin Standards Track [Page 98] + +RFC 3501 IMAPv4 March 2003 + + + 30) Clarify that "INBOX" in a mailbox name is always INBOX, even if + it is given as a string. + + 31) Add missing open parenthesis in media-basic grammar rule. + + 32) Correct attribute syntax in mailbox-data. + + 33) Add UIDNEXT to EXAMINE responses. + + 34) Clarify UNSEEN, PERMANENTFLAGS, UIDVALIDITY, and UIDNEXT + responses in SELECT and EXAMINE. They are required now, but weren't + in older versions. + + 35) Update references with RFC numbers. + + 36) Flush text-mime2. + + 37) Clarify that modified UTF-7 names must be case-sensitive and that + violating the convention should be avoided. + + 38) Correct UID FETCH example. + + 39) Clarify UID FETCH, UID STORE, and UID SEARCH vs. untagged EXPUNGE + responses. + + 40) Clarify the use of the word "convention". + + 41) Clarify that a command is not "in progress" until it has been + fully received (specifically, that a command is not "in progress" + during command continuation negotiation). + + 42) Clarify envelope defaulting. + + 43) Clarify that SP means one and only one space character. + + 44) Forbid silly states in LIST response. + + 45) Clarify that the ENVELOPE, INTERNALDATE, RFC822*, BODY*, and UID + for a message is static. + + 46) Add BADCHARSET response code. + + 47) Update formal syntax to [ABNF] conventions. + + 48) Clarify trailing hierarchy delimiter in CREATE semantics. + + 49) Clarify that the "blank line" is the [RFC-2822] delimiting blank + line. + + + +Crispin Standards Track [Page 99] + +RFC 3501 IMAPv4 March 2003 + + + 50) Clarify that RENAME should also create hierarchy as needed for + the command to complete. + + 51) Fix body-ext-mpart to not require language if disposition + present. + + 52) Clarify the RFC822.HEADER response. + + 53) Correct missing space after charset astring in search. + + 54) Correct missing quote for BADCHARSET in resp-text-code. + + 55) Clarify that ALL, FAST, and FULL preclude any other data items + appearing. + + 56) Clarify semantics of reference argument in LIST. + + 57) Clarify that a null string for SEARCH HEADER X-FOO means any + message with a header line with a field-name of X-FOO regardless of + the text of the header. + + 58) Specifically reserve 8-bit mailbox names for future use as UTF-8. + + 59) It is not an error for the client to store a flag that is not in + the PERMANENTFLAGS list; however, the server will either ignore the + change or make the change in the session only. + + 60) Correct/clarify the text regarding superfluous shifts. + + 61) Correct typographic errors in the "Changes" section. + + 62) Clarify that STATUS must not be used to check for new messages in + the selected mailbox + + 63) Clarify LSUB behavior with "%" wildcard. + + 64) Change AUTHORIZATION to AUTHENTICATE in section 7.5. + + 65) Clarify description of multipart body type. + + 66) Clarify that STORE FLAGS does not affect \Recent. + + 67) Change "west" to "east" in description of timezone. + + 68) Clarify that commands which break command pipelining must wait + for a completion result response. + + 69) Clarify that EXAMINE does not affect \Recent. + + + +Crispin Standards Track [Page 100] + +RFC 3501 IMAPv4 March 2003 + + + 70) Make description of MIME structure consistent. + + 71) Clarify that date searches disregard the time and timezone of the + INTERNALDATE or Date: header. In other words, "ON 13-APR-2000" means + messages with an INTERNALDATE text which starts with "13-APR-2000", + even if timezone differential from the local timezone is sufficient + to move that INTERNALDATE into the previous or next day. + + 72) Clarify that the header fetches don't add a blank line if one + isn't in the [RFC-2822] message. + + 73) Clarify (in discussion of UIDs) that messages are immutable. + + 74) Add an example of CHARSET searching. + + 75) Clarify in SEARCH that keywords are a type of flag. + + 76) Clarify the mandatory nature of the SELECT data responses. + + 77) Add optional CAPABILITY response code in the initial OK or + PREAUTH. + + 78) Add note that server can send an untagged CAPABILITY command as + part of the responses to AUTHENTICATE and LOGIN. + + 79) Remove statement about it being unnecessary to issue a CAPABILITY + command more than once in a connection. That statement is no longer + true. + + 80) Clarify that untagged EXPUNGE decrements the number of messages + in the mailbox. + + 81) Fix definition of "body" (concatenation has tighter binding than + alternation). + + 82) Add a new "Special Notes to Implementors" section with reference + to [IMAP-IMPLEMENTATION]. + + 83) Clarify that an untagged CAPABILITY response to an AUTHENTICATE + command should only be done if a security layer was not negotiated. + + 84) Change the definition of atom to exclude "]". Update astring to + include "]" for compatibility with the past. Remove resp-text-atom. + + 85) Remove NEWNAME. It can't work because mailbox names can be + literals and can include "]". Functionality can be addressed via + referrals. + + + + +Crispin Standards Track [Page 101] + +RFC 3501 IMAPv4 March 2003 + + + 86) Move modified UTF-7 rationale in order to have more logical + paragraph flow. + + 87) Clarify UID uniqueness guarantees with the use of MUST. + + 88) Note that clients should read response data until the connection + is closed instead of immediately closing on a BYE. + + 89) Change RFC-822 references to RFC-2822. + + 90) Clarify that RFC-2822 should be followed instead of RFC-822. + + 91) Change recommendation of optional automatic capabilities in LOGIN + and AUTHENTICATE to use the CAPABILITY response code in the tagged + OK. This is more interoperable than an unsolicited untagged + CAPABILITY response. + + 92) STARTTLS and AUTH=PLAIN are mandatory to implement; add + recommendations for other [SASL] mechanisms. + + 93) Clarify that a "connection" (as opposed to "server" or "command") + is in one of the four states. + + 94) Clarify that a failed or rejected command does not change state. + + 95) Split references between normative and informative. + + 96) Discuss authentication failure issues in security section. + + 97) Clarify that a data item is not necessarily of only one data + type. + + 98) Clarify that sequence ranges are independent of order. + + 99) Change an example to clarify that superfluous shifts in + Modified-UTF7 can not be fixed just by omitting the shift. The + entire string must be recalculated. + + 100) Change Envelope Structure definition since [RFC-2822] uses + "envelope" to refer to the [SMTP] envelope and not the envelope data + that appears in the [RFC-2822] header. + + 101) Expand on RFC822.HEADER response data vs. BODY[HEADER]. + + 102) Clarify Logout state semantics, change ASCII art. + + 103) Security changes to comply with IESG requirements. + + + + +Crispin Standards Track [Page 102] + +RFC 3501 IMAPv4 March 2003 + + + 104) Add definition for body URI. + + 105) Break sequence range definition into three rules, with rewritten + descriptions for each. + + 106) Move STARTTLS and LOGINDISABLED here from [IMAP-TLS]. + + 107) Add IANA Considerations section. + + 108) Clarify valid client assumptions for new message UIDs vs. + UIDNEXT. + + 109) Clarify that changes to permanentflags affect concurrent + sessions as well as subsequent sessions. + + 110) Clarify that authenticated state can be entered by the CLOSE + command. + + 111) Emphasize that SELECT and EXAMINE are the exceptions to the rule + that a failing command does not change state. + + 112) Clarify that newly-appended messages have the Recent flag set. + + 113) Clarify that newly-copied messages SHOULD have the Recent flag + set. + + 114) Clarify that UID commands always return the UID in FETCH + responses. + +C. Key Word Index + + +FLAGS (store command data item) ............... 59 + +FLAGS.SILENT (store command data item) ........ 59 + -FLAGS (store command data item) ............... 59 + -FLAGS.SILENT (store command data item) ........ 59 + ALERT (response code) ...................................... 64 + ALL (fetch item) ........................................... 55 + ALL (search key) ........................................... 50 + ANSWERED (search key) ...................................... 50 + APPEND (command) ........................................... 45 + AUTHENTICATE (command) ..................................... 27 + BAD (response) ............................................. 66 + BADCHARSET (response code) ................................. 64 + BCC (search key) .................................. 51 + BEFORE (search key) ................................. 51 + BODY (fetch item) .......................................... 55 + BODY (fetch result) ........................................ 73 + BODY (search key) ................................. 51 + + + +Crispin Standards Track [Page 103] + +RFC 3501 IMAPv4 March 2003 + + + BODY.PEEK[
]<> (fetch item) ............... 57 + BODYSTRUCTURE (fetch item) ................................. 57 + BODYSTRUCTURE (fetch result) ............................... 74 + BODY[
]<> (fetch result) ............. 74 + BODY[
]<> (fetch item) .................... 55 + BYE (response) ............................................. 67 + Body Structure (message attribute) ......................... 12 + CAPABILITY (command) ....................................... 24 + CAPABILITY (response code) ................................. 64 + CAPABILITY (response) ...................................... 68 + CC (search key) ................................... 51 + CHECK (command) ............................................ 47 + CLOSE (command) ............................................ 48 + COPY (command) ............................................. 59 + CREATE (command) ........................................... 34 + DELETE (command) ........................................... 35 + DELETED (search key) ....................................... 51 + DRAFT (search key) ......................................... 51 + ENVELOPE (fetch item) ...................................... 57 + ENVELOPE (fetch result) .................................... 77 + EXAMINE (command) .......................................... 33 + EXISTS (response) .......................................... 71 + EXPUNGE (command) .......................................... 48 + EXPUNGE (response) ......................................... 72 + Envelope Structure (message attribute) ..................... 12 + FAST (fetch item) .......................................... 55 + FETCH (command) ............................................ 54 + FETCH (response) ........................................... 73 + FLAGGED (search key) ....................................... 51 + FLAGS (fetch item) ......................................... 57 + FLAGS (fetch result) ....................................... 78 + FLAGS (response) ........................................... 71 + FLAGS (store command data item) ................ 59 + FLAGS.SILENT (store command data item) ......... 59 + FROM (search key) ................................. 51 + FULL (fetch item) .......................................... 55 + Flags (message attribute) .................................. 11 + HEADER (part specifier) .................................... 55 + HEADER (search key) .................. 51 + HEADER.FIELDS (part specifier) ............... 55 + HEADER.FIELDS.NOT (part specifier) ........... 55 + INTERNALDATE (fetch item) .................................. 57 + INTERNALDATE (fetch result) ................................ 78 + Internal Date (message attribute) .......................... 12 + KEYWORD (search key) ................................ 51 + Keyword (type of flag) ..................................... 11 + LARGER (search key) .................................... 51 + LIST (command) ............................................. 40 + + + +Crispin Standards Track [Page 104] + +RFC 3501 IMAPv4 March 2003 + + + LIST (response) ............................................ 69 + LOGIN (command) ............................................ 30 + LOGOUT (command) ........................................... 25 + LSUB (command) ............................................. 43 + LSUB (response) ............................................ 70 + MAY (specification requirement term) ....................... 4 + MESSAGES (status item) ..................................... 45 + MIME (part specifier) ...................................... 56 + MUST (specification requirement term) ...................... 4 + MUST NOT (specification requirement term) .................. 4 + Message Sequence Number (message attribute) ................ 10 + NEW (search key) ........................................... 51 + NO (response) .............................................. 66 + NOOP (command) ............................................. 25 + NOT (search key) .............................. 52 + OK (response) .............................................. 65 + OLD (search key) ........................................... 52 + ON (search key) ..................................... 52 + OPTIONAL (specification requirement term) .................. 4 + OR (search key) ................ 52 + PARSE (response code) ...................................... 64 + PERMANENTFLAGS (response code) ............................. 64 + PREAUTH (response) ......................................... 67 + Permanent Flag (class of flag) ............................. 12 + READ-ONLY (response code) .................................. 65 + READ-WRITE (response code) ................................. 65 + RECENT (response) .......................................... 72 + RECENT (search key) ........................................ 52 + RECENT (status item) ....................................... 45 + RENAME (command) ........................................... 37 + REQUIRED (specification requirement term) .................. 4 + RFC822 (fetch item) ........................................ 57 + RFC822 (fetch result) ...................................... 78 + RFC822.HEADER (fetch item) ................................. 57 + RFC822.HEADER (fetch result) ............................... 78 + RFC822.SIZE (fetch item) ................................... 57 + RFC822.SIZE (fetch result) ................................. 78 + RFC822.TEXT (fetch item) ................................... 58 + RFC822.TEXT (fetch result) ................................. 79 + SEARCH (command) ........................................... 49 + SEARCH (response) .......................................... 71 + SEEN (search key) .......................................... 52 + SELECT (command) ........................................... 31 + SENTBEFORE (search key) ............................. 52 + SENTON (search key) ................................. 52 + SENTSINCE (search key) .............................. 52 + SHOULD (specification requirement term) .................... 4 + SHOULD NOT (specification requirement term) ................ 4 + + + +Crispin Standards Track [Page 105] + +RFC 3501 IMAPv4 March 2003 + + + SINCE (search key) .................................. 52 + SMALLER (search key) ................................... 52 + STARTTLS (command) ......................................... 27 + STATUS (command) ........................................... 44 + STATUS (response) .......................................... 70 + STORE (command) ............................................ 58 + SUBJECT (search key) .............................. 53 + SUBSCRIBE (command) ........................................ 38 + Session Flag (class of flag) ............................... 12 + System Flag (type of flag) ................................. 11 + TEXT (part specifier) ...................................... 56 + TEXT (search key) ................................. 53 + TO (search key) ................................... 53 + TRYCREATE (response code) .................................. 65 + UID (command) .............................................. 60 + UID (fetch item) ........................................... 58 + UID (fetch result) ......................................... 79 + UID (search key) ............................ 53 + UIDNEXT (response code) .................................... 65 + UIDNEXT (status item) ...................................... 45 + UIDVALIDITY (response code) ................................ 65 + UIDVALIDITY (status item) .................................. 45 + UNANSWERED (search key) .................................... 53 + UNDELETED (search key) ..................................... 53 + UNDRAFT (search key) ....................................... 53 + UNFLAGGED (search key) ..................................... 53 + UNKEYWORD (search key) .............................. 53 + UNSEEN (response code) ..................................... 65 + UNSEEN (search key) ........................................ 53 + UNSEEN (status item) ....................................... 45 + UNSUBSCRIBE (command) ...................................... 39 + Unique Identifier (UID) (message attribute) ................ 8 + X (command) .......................................... 62 + [RFC-2822] Size (message attribute) ........................ 12 + \Answered (system flag) .................................... 11 + \Deleted (system flag) ..................................... 11 + \Draft (system flag) ....................................... 11 + \Flagged (system flag) ..................................... 11 + \Marked (mailbox name attribute) ........................... 69 + \Noinferiors (mailbox name attribute) ...................... 69 + \Noselect (mailbox name attribute) ......................... 69 + \Recent (system flag) ...................................... 11 + \Seen (system flag) ........................................ 11 + \Unmarked (mailbox name attribute) ......................... 69 + + + + + + + +Crispin Standards Track [Page 106] + +RFC 3501 IMAPv4 March 2003 + + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Avenue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 107] + +RFC 3501 IMAPv4 March 2003 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. v This + document and the information contained herein is provided on an "AS + IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK + FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT + LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL + NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY + OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 108] + diff --git a/docs/rfcs/rfc3502.txt b/docs/rfcs/rfc3502.txt new file mode 100644 index 0000000..f6b61a4 --- /dev/null +++ b/docs/rfcs/rfc3502.txt @@ -0,0 +1,395 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 3502 University of Washington +Category: Standards Track March 2003 + + + Internet Message Access Protocol (IMAP) - MULTIAPPEND Extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + This document describes the multiappending extension to the Internet + Message Access Protocol (IMAP) (RFC 3501). This extension provides + substantial performance improvements for IMAP clients which upload + multiple messages at a time to a mailbox on the server. + + A server which supports this extension indicates this with a + capability name of "MULTIAPPEND". + +Terminology + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to + be interpreted as described in [KEYWORDS]. + +Introduction + + The MULTIAPPEND extension permits uploading of multiple messages with + a single command. When used in conjunction with the [LITERAL+] + extension, the entire upload is accomplished in a single + command/response round trip. + + A MULTIAPPEND APPEND operation is atomic; either all messages are + successfully appended, or no messages are appended. + + In the base IMAP specification, each message must be appended in a + separate command, and there is no mechanism to "unappend" messages if + an error occurs while appending. Also, some mail stores may require + + + +Crispin Standards Track [Page 1] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + + an expensive "open/lock + sync/unlock/close" operation as part of + appending; this can be quite expensive if it must be done on a + per-message basis. + + If the server supports both LITERAL+ and pipelining but not + MULTIAPPEND, it may be possible to get some of the performance + advantages of MULTIAPPEND by doing a pipelined "batch" append. + However, it will not work as well as MULTIAPPEND for the following + reasons: + + 1) Multiple APPEND commands, even as part of a pipelined batch, + are non-atomic by definition. There is no way to revert the + mailbox to the state before the batch append in the event of an + error. + + 2) It may not be feasible for the server to coalesce pipelined + APPEND operations so as to avoid the "open/lock + + sync/unlock/close" overhead described above. In any case, such + coalescing would be timing dependent and thus potentially + unreliable. In particular, with traditional UNIX mailbox files, + it is assumed that a lock is held only for a single atomic + operation, and many applications disregard any lock that is + older than 5 minutes. + + 3) If an error occurs, depending upon the nature of the error, + it is possible for additional messages to be appended after the + error. For example, the user wants to append 5 messages, but a + disk quota error occurs with the third message because of its + size. However, the fourth and fifth messages have already been + sent in the pipeline, so the mailbox ends up with the first, + second, fourth, and fifth messages of the batch appended. + +6.3.11. APPEND Command + + Arguments: mailbox name + one or more messages to upload, specified as: + OPTIONAL flag parenthesized list + OPTIONAL date/time string + message literal + + Data: no specific responses for this command + + Result: OK - append completed + NO - append error: can't append to that mailbox, error + in flags or date/time or message text, + append cancelled + BAD - command unknown or arguments invalid + + + + +Crispin Standards Track [Page 2] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + + The APPEND command appends the literal arguments as new messages + to the end of the specified destination mailbox. This argument + SHOULD be in the format of an [RFC-2822] message. 8-bit + characters are permitted in the message. A server implementation + that is unable to preserve 8-bit data properly MUST be able to + reversibly convert 8-bit APPEND data to 7-bit using a [MIME-IMB] + content transfer encoding. + + Note: There MAY be exceptions, e.g., draft messages, in + which required [RFC-2822] header lines are omitted in the + message literal argument to APPEND. The full implications + of doing so MUST be understood and carefully weighed. + + If a flag parenthesized list is specified, the flags SHOULD be set + in the resulting message; otherwise, the flag list of the + resulting message is set empty by default. + + If a date-time is specified, the internal date SHOULD be set in + the resulting message; otherwise, the internal date of the + resulting message is set to the current date and time by default. + + A zero-length message literal argument is an error, and MUST + return a NO. This can be used to cancel the append. + + If the append is unsuccessful for any reason (including being + cancelled), the mailbox MUST be restored to its state before the + APPEND attempt; no partial appending is permitted. The server MAY + return an error before processing all the message arguments. + + If the destination mailbox does not exist, a server MUST return an + error, and MUST NOT automatically create the mailbox. Unless it + is certain that the destination mailbox can not be created, the + server MUST send the response code "[TRYCREATE]" as the prefix of + the text of the tagged NO response. This gives a hint to the + client that it can attempt a CREATE command and retry the APPEND + if the CREATE is successful. + + If the mailbox is currently selected, the normal new message + actions SHOULD occur. Specifically, the server SHOULD notify the + client immediately via an untagged EXISTS response. If the server + does not do so, the client MAY issue a NOOP command (or failing + that, a CHECK command) after one or more APPEND commands. + + + + + + + + + +Crispin Standards Track [Page 3] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + + Example: C: A003 APPEND saved-messages (\Seen) {329} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.example.net + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: (\Seen) " 7-Feb-1994 22:43:04 -0800" {295} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) + C: From: Joe Mooch + C: Subject: Re: afternoon meeting + C: To: foobar@blurdybloop.example.com + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: 3:30 is fine with me. + C: + S: A003 OK APPEND completed + C: A004 APPEND bogusname (\Flagged) {1023} + S: A004 NO [TRYCREATE] No such mailbox as bogusname + C: A005 APPEND test (\Flagged) {99} + S: + Ready for literal data + C: Date: Mon, 7 Feb 2000 22:43:04 -0800 (PST) + C: From: Fred Foobar + C: Subject: hmm... + C: {35403} + S: A005 NO APPEND failed: Disk quota exceeded + + Note: The APPEND command is not used for message delivery, + because it does not provide a mechanism to transfer [SMTP] + envelope information. + +Modification to IMAP4rev1 Base Protocol Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + append = "APPEND" SP mailbox 1*append-message + + append-message = [SP flag-list] [SP date-time] SP literal + + + + + +Crispin Standards Track [Page 4] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + +MULTIAPPEND Interaction with UIDPLUS Extension + + Servers which support both MULTIAPPEND and [UIDPLUS] will have the + "resp-code-apnd" rule modified as follows: + + resp-code-apnd = "APPENDUID" SP nz-number SP set + + That is, the APPENDUID response code returns as many UIDs as there + were messages appended in the multiple append. The UIDs returned + should be in the order the articles where appended. The message set + may not contain extraneous UIDs or the symbol "*". + +Security Considerations + + The MULTIAPPEND extension does not raise any security considerations + that are not present in the base [IMAP] protocol, and these issues + are discussed in [IMAP]. Nevertheless, it is important to remember + that IMAP4rev1 protocol transactions, including electronic mail data, + are sent in the clear over the network unless protection from + snooping is negotiated, either by the use of STARTTLS, privacy + protection is negotiated in the AUTHENTICATE command, or some other + protection mechanism is in effect. + +Normative References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MIME-IMB] Freed, N. and N. Borenstein, "MIME (Multipurpose Internet + Mail Extensions) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [RFC-2822] Resnick, P., "Internet Message Format", RFC 2822, April + 2001. + + + + + + + + + + + +Crispin Standards Track [Page 5] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + +Informative References + + [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC 2088, + January 1997. + + [UIDPLUS] Myers, J., "IMAP4 UIDPLUS extension", RFC 2359, June 1988. + + [SMTP] Klensin, J., Editor, "Simple Mail Transfer Protocol", RFC + 2821, April 2001. + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Avenue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 6] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 7] + diff --git a/docs/rfcs/rfc3503.txt b/docs/rfcs/rfc3503.txt new file mode 100644 index 0000000..5b82fb0 --- /dev/null +++ b/docs/rfcs/rfc3503.txt @@ -0,0 +1,507 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 3503 ACI Worldwide/MessagingDirect +Category: Standards Track March 2003 + + + Message Disposition Notification (MDN) profile for + Internet Message Access Protocol (IMAP) + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + The Message Disposition Notification (MDN) facility defined in RFC + 2298 provides a means by which a message can request that message + processing by the recipient be acknowledged as well as a format to be + used for such acknowledgements. However, it doesn't describe how + multiple Mail User Agents (MUAs) should handle the generation of MDNs + in an Internet Message Access Protocol (IMAP4) environment. + + This document describes how to handle MDNs in such an environment and + provides guidelines for implementers of IMAP4 that want to add MDN + support to their products. + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 1] + +RFC 3503 MDN profile for IMAP March 2003 + + +Table of Contents + + 1. Conventions Used in this Document............................. 2 + 2. Introduction and Overview..................................... 2 + 3. Client behavior............................................... 3 + 3.1. Client behavior when receiving a message................. 5 + 3.2. Client behavior when copying a message................... 5 + 3.3. Client behavior when sending a message................... 5 + 3.4. Client behavior when saving a temporary message.......... 5 + 4. Server behavior............................................... 5 + 4.1. Server that supports arbitrary keywords.................. 5 + 4.2. Server that supports only $MDNSent keyword............... 5 + 4.3. Interaction with IMAP ACL extension...................... 6 + 5. Examples...................................................... 6 + 6. Security Considerations....................................... 7 + 7. Formal Syntax................................................. 7 + 8. Acknowledgments............................................... 7 + 9. Normative References.......................................... 8 + 10. Author's Address.............................................. 8 + 11. Full Copyright Statement...................................... 9 + +1. Conventions Used in this Document + + "C:" and "S:" in examples show lines sent by the client and server + respectively. + + The keywords "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in + this document when typed in uppercase are to be interpreted as + defined in "Key words for use in RFCs to Indicate Requirement Levels" + [KEYWORDS]. + +2. Introduction and Overview + + This memo defines an additional [IMAP4] mailbox keyword that allows + multiple Mail User Agents (MUAs) to know if a requested receipt + notification was sent. + + Message Disposition Notification [MDN] does not require any special + support of IMAP in the case where a user has access to the mailstore + from only one computer and is using a single MUA. In this case, the + MUA behaves as described in [MDN], i.e., the MUA performs automatic + processing and generates corresponding MDNs, it performs requested + action and, with the user's permission, sends appropriate MDNs. The + MUA will not send MDN twice because the MUA keeps track of sent + notifications in a local configuration. However, that does not work + when IMAP is used to access the same mailstore from different + locations or is using different MUAs. + + + + +Melnikov Standards Track [Page 2] + +RFC 3503 MDN profile for IMAP March 2003 + + + This document defines a new special purpose mailbox keyword $MDNSent + that must be used by MUAs. It does not define any new command or + response for IMAP, but describes a technique that MUAs should use to + achieve interoperability. + + When a client opens a mailbox for the first time, it verifies that + the server is capable of storing the $MDNSent keyword by examining + the PERMANENTFLAGS response code. In order to support MDN in IMAP, a + server MUST support either the $MDNSent keyword, or arbitrary message + keywords. + +3. Client behavior + + The use of IMAP requires few additional steps in mail processing on + the client side. The following timeline modifies the timeline found + in Section 4 of [MDN]. + + -- User composes message. + + -- User tells MUA to send message. + + -- MUA passes message to MSA (original recipient information passed + along). MUA [optionally] saves message to a folder for sent mail + with $MDNSent flag set. + + -- MSA sends message to MTA. + + -- Final MTA receives message. + + -- Final MTA delivers message to MUA (possibly generating DSN). + + -- MUA logs into IMAP server, opens mailbox, verifies if mailbox can + store $MDNSent keyword by examining PERMANENTFLAGS response. + + -- MUA performs automatic processing and generates corresponding MDNs + ("dispatched", "processed", "deleted", "denied" or "failed" + disposition type with "automatic-action" and "MDN-sent- + automatically" disposition modes) for messages that do not have + $MDNSent keyword, or \Draft flag set. (*) + + -- MUA sets the $MDNSent keyword for every message that required an + automatic MDN to be sent, whether or not the MDN was sent. + + -- MUA displays a list of messages to user. + + -- User selects a message and requests that some action be performed + on it. + + + + +Melnikov Standards Track [Page 3] + +RFC 3503 MDN profile for IMAP March 2003 + + + -- MUA performs requested action and, with user's permission, sends + appropriate MDN ("displayed", "dispatched", "processed", + "deleted", "denied" or "failed" disposition type with "manual- + action" and "MDN-sent-manually" or "MDN-sent-automatically" + disposition mode). If the generated MDN is saved to a mailbox + with the APPEND command, the client MUST specify the $MDNSent + keyword in the APPEND. + + -- MUA sets the $MDNSent keyword for all messages for which the user + confirmed the dispatching of disposition (or was explicitly + prohibited to do so). + + -- User possibly performs other actions on message, but no further + MDNs are generated. + + (*) Note: MUA MUST NOT use \Recent flag as an indicator that it + should send MDN, because according to [IMAP4], "If multiple + connections have the same mailbox selected simultaneously, it is + undefined which of these connections will see newly-arrived + messages with \Recent set and which will see it without \Recent + set". Thus, using \Recent as an indicator will cause + unpredictable client behavior with different IMAP4 servers. + However, the client MAY use \Seen flag as one of the indicators + that MDN must not be sent. The client MUST NOT use any other + standard flags, like \Draft or \Answered, to indicate that MDN + was previously sent, because they have different well known + meaning. In any case, in the presence of the $MDNSent keyword, + the client MUST ignore all other flags or keywords for the + purpose of generating an MDN and MUST NOT send the MDN. + + When the client opens a mailbox for the first time, it must verify + that the server supports the $MDNSent keyword, or arbitrary message + keywords by examining PERMANENTFLAGS response code. + + The client MUST NOT try to set the $MDNSent keyword if the server is + incapable of storing it permanently. + + The client MUST be prepared to receive NO from the server as the + result of STORE $MDNSent when the server advertises the support of + storing arbitrary keywords, because the server may limit the number + of message keywords it can store in a particular mailbox. A client + SHOULD NOT send MDN if it fails to store the $MDNSent keyword. + + Once the $MDNSent keyword is set, it MUST NOT be unset by a client. + The client MAY set the $MDNSent keyword when a user denies sending + the notification. This prohibits all other MUAs from sending MDN for + this message. + + + + +Melnikov Standards Track [Page 4] + +RFC 3503 MDN profile for IMAP March 2003 + + +3.1. Client behavior when receiving a message + + The client MUST NOT send MDN if a message has the $MDNSent keyword + set. It also MUST NOT send MDN if a message has \Draft flag, because + some clients use this flag to mark a message as incomplete. + + See the timeline in section 3 for details on client behavior when + receiving a message. + +3.2. Client behavior when copying a message + + The client SHOULD verify that $MDNSent is preserved on a COPY + operation. Furthermore, when a message is copied between servers + with the APPEND command, the client MUST set the $MDNSent keyword + correctly. + +3.3. Client behavior when sending a message + + When saving a sent message to any folder, the client MUST set the + $MDNSent keyword to prevent another client from sending MDN for the + message. + +3.4. Client behavior when saving a temporary message + + When saving an unfinished message to any folder client MUST set + $MDNSent keyword to prevent another client from sending MDN for the + message. + +4. Server behavior + + Server implementors that want to follow this specification must + insure that their server complies with either section 4.1 or section + 4.2. If the server also supports the IMAP [ACL] extension, it MUST + also comply with the section 4.3. + +4.1. Server that supports arbitrary keywords + + No changes are required from the server to make it compatible with + the extension described in this document if it supports arbitrary + keywords. + +4.2. Server that supports only $MDNSent keyword + + Servers that support only the $MDNSent keyword MUST preserve it on + the COPY operation. It is also expected that a server that supports + SEARCH will also support the SEARCH KEYWORD $MDNSent. + + + + + +Melnikov Standards Track [Page 5] + +RFC 3503 MDN profile for IMAP March 2003 + + +4.3. Interaction with IMAP ACL extension + + Any server that conforms to either 4.1 or 4.2 and also supports the + IMAP [ACL] extension, SHOULD preserve the $MDNSent keyword on COPY + even if the client does not have 'w' right. This will prevent the + generation of a duplicated MDN for the same message. Note that the + server MUST still check if the client has rights to perform the COPY + operation on a message according to [ACL]. + +5. Examples + + 1) MUA opens mailbox for the first time. + + a) The server supports storing of arbitrary keywords + + C: a100 select INBOX + S: * FLAGS (\Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Flagged \Draft \Deleted \Seen \*)] + S: * 5 EXISTS + S: * 3 RECENT + S: * OK [UIDVALIDITY 894294713] + S: a100 OK [READ-WRITE] Completed + + b) The server supports storing of the $MDNSent keyword + + C: a100 select INBOX + S: * FLAGS (\Flagged \Draft \Deleted \Seen $MDNSent) + S: * OK [PERMANENTFLAGS (\Flagged \Draft \Deleted \Seen $MDNSent)] + S: * 5 EXISTS + S: * 3 RECENT + S: * OK [UIDVALIDITY 894294713] + S: a100 OK [READ-WRITE] Completed + + 2) The MUA successfully sets the $MDNSent keyword + + C: a200 STORE 4 +FLAGS ($MDNSent) + S: * 4 FETCH (FLAGS (\Flagged \Seen $MDNSent)) + S: * FLAGS ($MDNSent \Flagged \Deleted \Draft \Seen) + S: * OK [PERMANENTFLAGS ($MDNSent \Flagged \Deleted \Draft \Seen \*)] + S: a200 OK STORE completed + + 3) The server refuses to store the $MDNSent keyword + + C: a200 STORE 4 +FLAGS ($MDNSent) + S: a200 NO STORE failed : no space left to store $MDNSent keyword + + + + + + +Melnikov Standards Track [Page 6] + +RFC 3503 MDN profile for IMAP March 2003 + + + 4) All clients and servers MUST treat the $MDNSent keyword as case + insensitive in all operations, as stated in [IMAP]. + + C: a300 FETCH 1:* FLAGS + S: * 1 FETCH (FLAGS (\Seen)) + S: * 2 FETCH (FLAGS (\Answered \Seen $MdnSENt)) + S: * 3 FETCH (FLAGS ()) + S: * 4 FETCH (FLAGS (\Flagged \Seen $MdnSENT)) + S: * 5 FETCH (FLAGS ($MDNSent)) + S: * 6 FETCH (FLAGS (\Recent)) + S: a300 OK FETCH completed + C: a400 SEARCH KEYWORDS $mdnsent + S: * SEARCH 2 4 5 + S: a400 OK SEARCH completed + +6. Security Considerations + + There are no known security issues with this extension, not found in + [MDN] and/or [IMAP4]. + + Section 4.3 changes ACL checking requirements on an IMAP server that + implements IMAP [ACL] extension. + +7. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822], as modified by + [IMAP4]. Non-terminals referenced, but not defined below, are as + defined by [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + flag_keyword ::= "$MDNSent" / other_keywords + + other_keywords ::= atom + +8. Acknowledgments + + This document is the product of discussions that took place on the + IMAP mailing list. Special gratitude to Cyrus Daboo and Randall + Gellens for reviewing the document. + + Thank you to my father who as he has helped to make me what I am. I + miss you terribly. + + + + +Melnikov Standards Track [Page 7] + +RFC 3503 MDN profile for IMAP March 2003 + + +9. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MDN] Fajman, R., "An Extensible Message Format for Message + Disposition Notifications", RFC 2298, March 1998. + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [ACL] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. + +10. Author's Address + + Alexey Melnikov + ACI Worldwide/MessagingDirect + 59 Clarendon Road + Watford, Hertfordshire + United Kingdom, WD17 1FQ + + Phone: +44 1923 81 2877 + EMail: mel@messagingdirect.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 8] + +RFC 3503 MDN profile for IMAP March 2003 + + +11. Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 9] + diff --git a/docs/rfcs/rfc3516.txt b/docs/rfcs/rfc3516.txt new file mode 100644 index 0000000..4d02197 --- /dev/null +++ b/docs/rfcs/rfc3516.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group L. Nerenberg +Request for Comments: 3516 Orthanc Systems +Category: Standards Track April 2003 + + + IMAP4 Binary Content Extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + This memo defines the Binary extension to the Internet Message Access + Protocol (IMAP4). It provides a mechanism for IMAP4 clients and + servers to exchange message body data without using a MIME content- + transfer-encoding. + +1. Conventions Used in this Document + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as described in [KEYWORD]. + + The abbreviation "CTE" means content-transfer-encoding. + +2. Introduction + + The MIME extensions to Internet messaging allow for the transmission + of non-textual (binary) message content [MIME-IMB]. Since the + traditional transports for messaging are not always capable of + passing binary data transparently, MIME provides encoding schemes + that allow binary content to be transmitted over transports that are + not otherwise able to do so. + + The overhead of MIME-encoding this content can be considerable in + some contexts (e.g., slow radio links, streaming multimedia). + Reducing the overhead associated with CTE schemes such as base64 + + + + + + +Nerenberg Standards Track [Page 1] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + + can give a noticeable reduction in resource consumption. The Binary + extension lets the server perform CTE decoding prior to transmitting + message data to the client. + +3. Content-Transfer-Encoding Considerations + + Every IMAP4 body section has a MIME content-transfer-encoding. + (Those without an explicit Content-Transfer-Encoding header are + implicitly labeled as "7bit" content.) In the terminology of [MIME- + IMB], the CTE specifies both a decoding algorithm and the domain of + the decoded data. In this memo, "decoding" refers to the CTE + decoding step described in [MIME-IMB]. + + Certain CTEs use an identity encoding transformation. For these CTEs + there is no decoding required, however the domain of the underlying + data may not be expressible in the IMAP4 protocol (e.g., MIME + "binary" content containing NUL octets). To accommodate these cases + the Binary extension introduces a new type of literal protocol + element that is fully eight bit transparent. + + Thus, server processing of the FETCH BINARY command involves two + logical steps: + + 1) perform any CTE-related decoding + + 2) determine the domain of the decoded data + + Step 2 is necessary to determine which protocol element should be + used to transmit the decoded data. (See FETCH Response Extensions + for further details.) + +4. Framework for the IMAP4 Binary Extension + + This memo defines the following extensions to [IMAP4rev1]. + +4.1. CAPABILITY Identification + + IMAP4 servers that support this extension MUST include "BINARY" in + the response list to the CAPABILITY command. + +4.2. FETCH Command Extensions + + This extension defines three new FETCH command data items. + + BINARY[] + + Requests that the specified section be transmitted after + performing CTE-related decoding. + + + +Nerenberg Standards Track [Page 2] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + + The argument, if present, requests that a subset of + the data be returned. The semantics of a partial FETCH BINARY + command are the same as for a partial FETCH BODY command, with + the exception that the arguments refer to the DECODED + section data. + + BINARY.PEEK[] + + An alternate form of FETCH BINARY that does not implicitly set + the \Seen flag. + + BINARY.SIZE + + Requests the decoded size of the section (i.e., the size to + expect in response to the corresponding FETCH BINARY request). + + Note: client authors are cautioned that this might be an + expensive operation for some server implementations. + Needlessly issuing this request could result in degraded + performance due to servers having to calculate the value every + time the request is issued. + +4.3. FETCH Response Extensions + + This extension defines two new FETCH response data items. + + BINARY[<>] + + An or expressing the content of the + specified section after removing any CTE-related encoding. If + is present it refers to the offset within the DECODED + section data. + + If the domain of the decoded data is "8bit" and the data does + not contain the NUL octet, the server SHOULD return the data in + a instead of a ; this allows the client to + determine if the "8bit" data contains the NUL octet without + having to explicitly scan the data stream for for NULs. + + If the server does not know how to decode the section's CTE, it + MUST fail the request and issue a "NO" response that contains + the "UNKNOWN-CTE" extended response code. + + + + + + + + + +Nerenberg Standards Track [Page 3] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + + BINARY.SIZE + + The size of the section after removing any CTE-related + encoding. The value returned MUST match the size of the + or that will be returned by the + corresponding FETCH BINARY request. + + If the server does not know how to decode the section's CTE, it + MUST fail the request and issue a "NO" response that contains + the "UNKNOWN-CTE" extended response code. + +4.4. APPEND Command Extensions + + The APPEND command is extended to allow the client to append data + containing NULs by using the syntax. The server MAY + modify the CTE of the appended data, however any such transformation + MUST NOT result in a loss of data. + + If the destination mailbox does not support the storage of binary + content, the server MUST fail the request and issue a "NO" response + that contains the "UNKNOWN-CTE" extended response code. + +5. MIME Encoded Headers + + [MIME-MHE] defines an encoding that allows for non-US-ASCII text in + message headers. This encoding is not the same as the content- + transfer-encoding applied to message bodies, and the decoding + transformations described in this memo do not apply to [MIME-MHE] + encoded header text. A server MUST NOT perform any conversion of + [MIME-MHE] encoded header text in response to any binary FETCH or + APPEND request. + +6. Implementation Considerations + + Messaging clients and servers have been notoriously lax in their + adherence to the Internet CRLF convention for terminating lines of + textual data in Internet protocols. When sending data using the + Binary extension, servers MUST ensure that textual line-oriented + sections are always transmitted using the IMAP4 CRLF line termination + syntax, regardless of the underlying storage representation of the + data on the server. + + A server may choose to store message body binary content in a non- + encoded format. Regardless of the internal storage representation + used, the server MUST issue BODYSTRUCTURE responses that describe the + message as though the binary-encoded sections are encoded in a CTE + + + + + +Nerenberg Standards Track [Page 4] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + + acceptable to the IMAP4 base specification. Furthermore, the results + of a FETCH BODY MUST return the message body content in the format + described by the corresponding FETCH BODYSTRUCTURE response. + + While the server is allowed to modify the CTE of APPENDed + data, this should only be done when it is absolutely necessary. + Gratuitous encoding changes will render useless most cryptographic + operations that have been performed on the message. + + This extension provides an optimization that is useful in certain + specific situations. It does not absolve clients from providing + basic functionality (content transfer decoding) that should be + available in all messaging clients. Clients supporting this + extension SHOULD be prepared to perform their own CTE decoding + operations. + +7. Formal Protocol Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (ABNF) notation as used in [ABNF], and incorporates by reference + the Core Rules defined in that document. + + This syntax augments the grammar specified in [IMAP4rev1]. + + append =/ "APPEND" SP mailbox [SP flag-list] + [SP date-time] SP literal8 + + fetch-att =/ "BINARY" [".PEEK"] section-binary [partial] + / "BINARY.SIZE" section-binary + + literal8 = "~{" number "}" CRLF *OCTET + ; represents the number of OCTETs + ; in the response string. + + msg-att-static =/ "BINARY" section-binary SP (nstring / literal8) + / "BINARY.SIZE" section-binary SP number + + partial = "<" number "." nz-number ">" + + resp-text-code =/ "UNKNOWN-CTE" + + section-binary = "[" [section-part] "]" + + + + + + + + + +Nerenberg Standards Track [Page 5] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + +8. Normative References + + [ABNF] Crocker, D., Editor, and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 2234, November 1997. + + [IMAP4rev1] Crispin, M., "Internet Message Access Protocol Version + 4rev1", RFC 3501, March 2003. + + [KEYWORD] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MIME-IMB] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [MIME-MHE] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII + Text", RFC 2047, November 1996. + +9. Security Considerations + + There are no known additional security issues with this extension + beyond those described in the base protocol described in [IMAP4rev1]. + +10. Intellectual Property + + The IETF takes no position regarding the validity or scope of any + intellectual property or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; neither does it represent that it + has made any effort to identify any such rights. Information on the + IETF's procedures with respect to rights in standards-track and + standards-related documentation can be found in BCP-11. Copies of + claims of rights made available for publication and any assurances of + licenses to be made available, or the result of an attempt made to + obtain a general license or permission for the use of such + proprietary rights by implementors or users of this specification can + be obtained from the IETF Secretariat. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights which may cover technology that may be required to practice + this standard. Please address the information to the IETF Executive + Director. + + + + + + +Nerenberg Standards Track [Page 6] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + +11. Author's Address + + Lyndon Nerenberg + Orthanc Systems + 1606 - 10770 Winterburn Road + Edmonton, Alberta + Canada T5S 1T6 + + EMail: lyndon@orthanc.ab.ca + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Nerenberg Standards Track [Page 7] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + +12. Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Nerenberg Standards Track [Page 8] + diff --git a/docs/rfcs/rfc3656.txt b/docs/rfcs/rfc3656.txt new file mode 100644 index 0000000..6c0ab5b --- /dev/null +++ b/docs/rfcs/rfc3656.txt @@ -0,0 +1,1067 @@ + + + + + + +Network Working Group R. Siemborski +Request for Comments: 3656 Carnegie Mellon University +Category: Experimental December 2003 + + + The Mailbox Update (MUPDATE) + Distributed Mailbox Database Protocol + +Status of this Memo + + This memo defines an Experimental Protocol for the Internet + community. It does not specify an Internet standard of any kind. + Discussion and suggestions for improvement are requested. + Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + As the demand for high-performance mail delivery agents increases, it + becomes apparent that single-machine solutions are inadequate to the + task, both because of capacity limits and that the failure of the + single machine means a loss of mail delivery for all users. It is + preferable to allow many machines to share the responsibility of mail + delivery. + + The Mailbox Update (MUPDATE) protocol allows a group of Internet + Message Access Protocol (IMAP) or Post Office Protocol - Version 3 + (POP3) servers to function with a unified mailbox namespace. This + document is intended to serve as a reference guide to that protocol. + + + + + + + + + + + + + + + + + + + +Siemborski Experimental [Page 1] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Protocol Overview . . . . . . . . . . . . . . . . . . . . . . 3 + 2.1. Atoms . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 2.2. Strings . . . . . . . . . . . . . . . . . . . . . . . . 4 + 3. Server Responses . . . . . . . . . . . . . . . . . . . . . . 4 + 3.1. Response: OK . . . . . . . . . . . . . . . . . . . . . 5 + 3.2. Response: NO . . . . . . . . . . . . . . . . . . . . . 5 + 3.3. Response: BAD . . . . . . . . . . . . . . . . . . . . . 5 + 3.4. Response: BYE . . . . . . . . . . . . . . . . . . . . . 6 + 3.5. Response: RESERVE . . . . . . . . . . . . . . . . . . . 6 + 3.6. Response: MAILBOX . . . . . . . . . . . . . . . . . . . 6 + 3.7. Response: DELETE . . . . . . . . . . . . . . . . . . . 7 + 3.8. Server Capability Response. . . . . . . . . . . . . . . 7 + 4. Client Commands . . . . . . . . . . . . . . . . . . . . . . . 8 + 4.1. Command: ACTIVATE . . . . . . . . . . . . . . . . . . . 8 + 4.2. Command: AUTHENTICATE . . . . . . . . . . . . . . . . . 8 + 4.3. Command: DEACTIVATE . . . . . . . . . . . . . . . . . . 9 + 4.4. Command: DELETE . . . . . . . . . . . . . . . . . . . . 9 + 4.5. Command: FIND . . . . . . . . . . . . . . . . . . . . . 9 + 4.6. Command: LIST . . . . . . . . . . . . . . . . . . . . . 10 + 4.7. Command: LOGOUT . . . . . . . . . . . . . . . . . . . . 10 + 4.8. Command: NOOP . . . . . . . . . . . . . . . . . . . . . 10 + 4.9. Command: RESERVE. . . . . . . . . . . . . . . . . . . . 10 + 4.10. Command: STARTTLS . . . . . . . . . . . . . . . . . . . 11 + 4.11. Command: UPDATE . . . . . . . . . . . . . . . . . . . . 12 + 5. MUPDATE Formal Syntax . . . . . . . . . . . . . . . . . . . . 12 + 6. MUPDATE URL Scheme. . . . . . . . . . . . . . . . . . . . . . 14 + 6.1. MUPDATE URL Scheme Registration Form. . . . . . . . . . 14 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 15 + 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 16 + 9. Intellectual Property Rights. . . . . . . . . . . . . . . . . 16 + 10. References. . . . . . . . . . . . . . . . . . . . . . . . . . 17 + 10.1. Normative References. . . . . . . . . . . . . . . . . . 17 + 10.2. Informative References. . . . . . . . . . . . . . . . . 17 + 11. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 18 + 12. Author's Address. . . . . . . . . . . . . . . . . . . . . . . 18 + 13. Full Copyright Statement. . . . . . . . . . . . . . . . . . . 19 + + + + + + + + + + + + +Siemborski Experimental [Page 2] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +1. Introduction + + In order to support an architecture where there are multiple [IMAP, + POP3] servers sharing a common mailbox database, it is necessary to + be able to provide atomic mailbox operations, as well as offer + sufficient guarantees about database consistency. + + The primary goal of the MUPDATE protocol is to be simple to implement + yet allow for database consistency between participants. + + The key words "MUST, "MUST NOT", "REQUIRED", "SHOULD", "SHOULD NOT", + "RECOMMENDED", and "MAY" in this document are to be interpreted as + defined in BCP 14, RFC 2119 [KEYWORDS]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + +2. Protocol Overview + + The MUPDATE protocol assumes a reliable data stream such as a TCP + network connection. IANA has registered port 3905 with a short name + of "mupdate" for this purpose. + + In the current implementation of the MUPDATE protocol there are three + types of participants: a single master server, slave (or replica) + servers, and clients. The master server maintains an authoritative + copy of the mailbox database. Slave servers connect to the MUPDATE + master server as clients, and function as replicas from the point of + view of end clients. End clients may connect to either the master or + any slave and perform searches against the database, however + operations that change the database can only be performed against the + master. For the purposes of protocol discussion we will consider a + slave's connection to the master identical to that of any other + client. + + After connection, all commands from a client to server must have an + associated unique tag which is an alphanumeric string. Commands MAY + be pipelined from the client to the server (that is, the client need + not wait for the response before sending the next command). The + server MUST execute the commands in the order they were received, + however. + + If the server supports an inactivity login timeout, it MUST be at + least 15 minutes. + + + + + + + +Siemborski Experimental [Page 3] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + + MUPDATE uses data formats similar to those used in [ACAP]. That is, + atoms and strings. All commands and tags in the protocol are + transmitted as atoms. All other data is considered to a string, and + must be quoted or transmitted as a literal. + + Outside of a literal, both clients and servers MUST support line + lengths of at least 1024 octets (including the trailing CR and LF + characters). If a line of a longer length must be transmitted, + implementations MUST make use of literals to do so. + +2.1. Atoms + + An atom consists of one or more alphanumeric characters. Atoms MUST + be less than 15 octets in length. + +2.2. Strings + + As in [ACAP], a string may be either literal or a quoted string. A + literal is a sequence of zero or more octets (including CR and LF), + prefix-quoted with an octet count in the form of an open brace ("{"), + the number of octets, an optional plus sign to indicate that the data + follows immediately (a non-synchronized literal), a close brace + ("}"), and a CRLF sequence. If the plus sign is omitted (a + synchronized literal), then the receiving side MUST send a "+ go + ahead" response, and the sending side MUST wait for this response. + Servers MUST support literals of atleast 4096 octets. + + Strings that are sent from server to client SHOULD NOT be in the + synchronized literal format. + + A quoted string is a sequence of zero or more 7-bit characters, + excluding CR, LF, and the double quote (<">), with double quote + characters at each end. + + The empty string is represented as either "" (a quoted string with + zero characters between double quotes) or as {0} followed by CRLF (a + literal with an octet count of 0). + +3. Server Responses + + Every client command in the MUPDATE protocol may receive one or more + tagged responses from the server. Each response is preceded by the + same tag as the command that elicited the response from the server. + + + + + + + + +Siemborski Experimental [Page 4] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +3.1. Response: OK + + A tagged OK response indicates that the operation completed + successfully. There is a mandatory implementation-defined string + after the OK response. This response also indicates the beginning of + the streaming update mode when given in response to an UPDATE + command. + + Example: + +C: N01 NOOP +S: N01 OK "NOOP Complete" + +3.2. Response: NO + + A tagged NO response indicates that the operation was explicitly + denied by the server or otherwise failed. There is a mandatory + implementation-defined string after the NO response that SHOULD + explain the reason for denial. + + Example: + +C: A01 AUTHENTICATE "PLAIN" +S: A01 NO "PLAIN is not a supported SASL mechanism" + +3.3. Response: BAD + + A tagged BAD response indicates that the command from the client + could not be parsed or understood. There is a mandatory + implementation-defined string after the BAD response to provide + additional information about the error. Note that untagged BAD + responses are allowed if it is unclear what the tag for a given + command is (for example, if a blank line is received by the mupdate + server, it can generate an untagged BAD response). In the case of an + untagged response, the tag should be replaced with a "*". + + Example: + +C: C01 SELECT "INBOX" +S: C01 BAD "This is not an IMAP server" +C: +S: * BAD "Need Command" + + + + + + + + + +Siemborski Experimental [Page 5] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +3.4. Response: BYE + + A tagged BYE response indicates that the server has decided to close + the connection. There is a mandatory implementation-defined string + after the BYE response that SHOULD explain the reason for closing the + connection. The server MUST close the connection immediately after + transmitting the BYE response. + + Example: + +C: L01 LOGOUT +S: L01 BYE "User Logged Out" + +3.5. Response: RESERVE + + A tagged RESERVE response may only be given in response to a FIND, + LIST, or UPDATE command. It includes two parameters: the name of the + mailbox that is being reserved (in mUTF-7 encoding, as specified in + [IMAP]) and a location string whose contents is defined by the + clients that are using the database, though it is RECOMMENDED that + the format of this string be the hostname of the server which is + storing the mailbox. + + This response indicates that the given name is no longer available in + the namespace, though it does not indicate that the given mailbox is + available to clients at the current time. + + Example: + +S: U01 RESERVE "internet.bugtraq" "mail2.example.org" + +3.6. Response: MAILBOX + + A tagged MAILBOX response may only be given in response to a FIND, + LIST, or UPDATE command. It includes three parameters: the name of + the mailbox, a location string (as with RESERVE), and a client- + defined string that specifies the IMAP ACL [IMAP-ACL] of the mailbox. + This message indicates that the given mailbox is ready to be accessed + by clients. + + Example: + +S: U01 MAILBOX "internet.bugtraq" "mail2.example.org" "anyone rls" + + + + + + + + +Siemborski Experimental [Page 6] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +3.7. Response: DELETE + + A tagged DELETE response may only be given in response to an UPDATE + command, and MUST NOT be given before the OK response to the UPDATE + command is given. It contains a single parameter, that of the + mailbox that should be deleted from the slave's database. This + response indicates that the given mailbox no longer exists in the + namespace of the database, and may be given for any mailbox name, + active, reserved, or nonexistent. (Though implementations SHOULD NOT + issue DELETE responses for nonexistent mailboxes). + + Example: + +S: U01 DELETE "user.rjs3.sent-mail-jan-2002" + +3.8. Server Capability Response + + Upon connection of the client to the server, and directly following a + successful STARTTLS command, the server MUST issue a capabilities + banner, of the following format: + + The banner MUST contain a line that begins with "* AUTH" and contain + a space-separated list of SASL mechanisms that the server will accept + for authentication. The mechanism names are transmitted as atoms. + Servers MAY advertise no available mechanisms (to indicate that + STARTTLS must be completed before authentication may occur). If + STARTTLS is not supported by the server, then the line MUST contain + at least one mechanism. + + If the banner is being issued without a TLS layer, and the server + supports the STARTTLS command, the banner MUST contain the line "* + STARTTLS". If the banner is being issued under a TLS layer (or the + server does not support STARTTLS), the banner MUST NOT contain this + line. + + The last line of the banner MUST start with "* OK MUPDATE" and be + followed by four strings: the server's hostname, an implementation- + defined string giving the name of the implementation, an + implementation-defined string giving the version of the + implementation, and a string that indicates if the server is a master + or a slave. The master/slave indication MUST be either "(master)" or + an MUPDATE URL that defines where the master can be contacted. + + Any unrecognized responses before the "* OK MUPDATE" response MUST be + ignored by the client. + + + + + + +Siemborski Experimental [Page 7] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + + Example: + +S: * AUTH KERBEROS_V4 GSSAPI +S: * STARTTLS +S: * OK MUPDATE "mupdate.example.org" "Cyrus" "v2.1.2" "(master)" + +4. Client Commands + + The following are valid commands that a client may send to the + MUPDATE server: AUTHENTICATE, ACTIVATE, DEACTIVATE, DELETE, FIND, + LIST, LOGOUT, NOOP, RESERVE, STARTTLS, and UPDATE. + + Before a successful AUTHENTICATE command has occurred, the server + MUST NOT accept any commands except for AUTHENTICATE, STARTTLS, and + LOGOUT (and SHOULD reply with a NO response for all other commands). + +4.1. Command: ACTIVATE + + The ACTIVATE command has 3 parameters: the mailbox name, its + location, and its ACL. This command MUST NOT not be issued to a + slave server. + + This command can also be used to update the ACL or location + information of a mailbox. Note that it is not a requirement for a + mailbox to be reserved (or even exist in the database) for an + ACTIVATE command to succeed, implementations MUST allow this behavior + as it facilitates synchronization of the database with the current + state of the mailboxes. + +4.2. Command: AUTHENTICATE + + The AUTHENTICATE command initiates a [SASL] negotiation session + between the client and the server. It has two parameters. The first + parameter is mandatory, and is a string indicating the desired [SASL] + mechanism. The second is a string containing an optional BASE64 + encoded (as defined in section 6.8 of [MIME]) client first send. + + All of the remaining SASL blobs that are sent MUST be sent across the + wire must be in BASE64 encoded format, and followed by a CR and LF + combination. They MUST NOT be encoded as strings. + + Clients may cancel authentication by sending a * followed by a CR and + LF. + + The [SASL] service name for the MUPDATE protocol is "mupdate". + Implementations are REQUIRED to implement the GSSAPI [SASL] + mechanism, though they SHOULD implement as many mechanisms as + possible. + + + +Siemborski Experimental [Page 8] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + + If a security layer is negotiated, it should be used directly + following the CR and LF combination at the end of the server's OK + response (i.e., beginning with the client's next command) Only one + successful AUTHENTICATE command may be issued per session. + +4.3. Command: DEACTIVATE + + The DEACTIVATE command takes two parameters, the mailbox name and + location data. The mailbox MUST already exist and be activated on + the MUPDATE server. If the server responds OK, then the mailbox name + has been moved to the RESERVE state. If the server responds NO, then + the mailbox name has not been moved (for example, the mailbox was not + already active). Any ACL information that is known about the mailbox + MAY be lost when a DEACTIVATE succeeds. This command MUST NOT be + issued to a slave. + + Example: + +C: A01 DEACTIVATE "user.rjs3.new" "mail3.example.org!u4" +S: A01 OK "Mailbox Reserved." + +4.4. Command: DELETE + + The DELETE command takes only a single parameter, the mailbox name to + be removed from the database's namespace. The server SHOULD give a + NO response if the mailbox does not exist. This command MUST NOT be + issued to a slave server. + +4.5. Command: FIND + + The FIND command takes a single parameter, a mailbox name. The + server then responds with the current record for the given mailbox, + if any, and an OK response. + + Example (mailbox does not exist): + +C: F01 FIND "user.rjs3.xyzzy" +S: F01 OK "Search Complete" + + Example (mailbox is reserved): + +C: F01 FIND "user.rjs3" +S: F01 RESERVE "user.rjs3" "mail4.example.org" +S: F01 OK "Search Complete" + + + + + + + +Siemborski Experimental [Page 9] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +4.6. Command: LIST + + The LIST command is similar to running FIND across the entire + database. The LIST command takes a single optional parameter, which + is a prefix to try to match against the location field of the + records. Without the parameter, LIST returns every record in the + database. + + For each mailbox that matches, either a MAILBOX or a RESERVE response + (as applicable) is sent to the client. When all responses are + complete, an OK response is issued. + + Example: + +C: L01 LIST +S: L01 RESERVE "user.rjs3" "mail4.example.org!u2" +S: L01 MAILBOX "user.leg" "mail2.example.org!u1" "leg lrswipcda" +S: L01 OK "List Complete" +C: L02 LIST "mail4.example.org!" +S: L02 RESERVE "user.rjs3" "mail4.example.org!u2" +S: L02 OK "List Complete" + +4.7. Command: LOGOUT + + The LOGOUT command tells the server to close the connection. Its + only valid response is the BYE response. The LOGOUT command takes no + parameters. + +4.8. Command: NOOP + + The NOOP command takes no parameters. Provided the client is + authenticated, its only acceptable response is an OK. Any idle + timeouts that the server may have on the connection SHOULD be reset + upon receipt of this command. + + If this command is issued after an UPDATE command has been issued, + then the OK response also indicates that all pending database updates + have been sent to the client. That is, the slave can guarantee that + its local database is up to date as of a certain time by issuing a + NOOP and waiting for the OK. The OK MUST NOT return until all + updates that were pending at the time of the NOOP have been sent. + +4.9. Command: RESERVE + + The RESERVE command takes two parameters (just like the RESERVE + response), the mailbox name to reserve and location data. If the + server responds OK, then the mailbox name has been reserved. If the + server responds NO, then the mailbox name has not been reserved (for + + + +Siemborski Experimental [Page 10] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + + example, another server has reserved it already). This command MUST + NOT be issued to a slave. + + The typical sequence for mailbox creation is: + +C: R01 RESERVE "user.rjs3.new" "mail3.example.org!u4" +S: R01 OK "Mailbox Reserved." + +C: A01 ACTIVATE "user.rjs3.new" "mail3.example.org!u4" "rjs3 lrswipcda" +S: A01 OK "Mailbox Activated." + +4.10. Command: STARTTLS + + The STARTTLS command requests the commencement of a [TLS] + negotiation. The negotiation begins immediately after the CRLF in + the OK response. After a client issues a STARTTLS command, it MUST + NOT issue further commands until a server response is seen and the + [TLS] negotiation is complete. + + The STARTTLS command is only valid in non-authenticated state. The + server remains in non-authenticated state, even if client credentials + are supplied during the [TLS] negotiation. The [SASL] EXTERNAL + mechanism MAY be used to authenticate once [TLS] client credentials + are successfully exchanged. Note that servers are not required to + support the EXTERNAL mechanism. + + After the [TLS] layer is established, the server MUST re-issue the + initial response banner (see Section 3.8). This is necessary to + protect against man-in-the-middle attacks which alter the + capabilities list prior to STARTTLS, as well as to advertise any new + SASL mechanisms (or other capabilities) that may be available under + the layer. The client MUST discard cached capability information and + replace it with the new information. + + After the a successful STARTTLS command, the server SHOULD return a + NO response to additional STARTTLS commands. + + Servers MAY choose to not implement STARTTLS. In this case, they + MUST NOT advertise STARTTLS in their capabilities banner, and SHOULD + return a BAD response to the STARTTLS command, if it is issued. + + Example: + +C: S01 STARTTLS +S: S01 OK "Begin TLS negotiation now" + +S: * AUTH KERBEROS_V4 GSSAPI PLAIN +S: * OK MUPDATE "mupdate.example.org" "Cyrus" "v2.1.2" "(master)" + + + +Siemborski Experimental [Page 11] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +4.11. Command: UPDATE + + The UPDATE command is how a slave initializes an update stream from + the master (though it is also valid to issue this command to a + slave). In response to the command, the server returns a list of all + mailboxes in its database (the same results as a parameterless LIST + command) followed by an OK response. From this point forward, + whenever an update occurs to the master database, it MUST stream the + update to the slave within 30 seconds. That is, it will send + RESERVE, MAILBOX, or DELETE responses as they are applicable. + + After a client has issued an UPDATE command, it may only issue NOOP + and LOGOUT commands for the remainder of the session. + + Example: + +C: U01 UPDATE +S: U01 MAILBOX "user.leg" "mail2.example.org!u1" "leg lrswipcda" +S: U01 MAILBOX "user.rjs3" "mail3.example.org!u4" "rjs3 lrswipcda" +S: U01 RESERVE "internet.bugtraq" "mail1.example.org!u5" "anyone lrs" +S: U01 OK "Streaming Begins" + +S: U01 RESERVE "user.leg.new" "mail2.example.org!u1" + +S: U01 MAILBOX "user.leg.new" "mail2.example.org!u1" "leg lrswipcda" + +C: N01 NOOP +S: U01 DELETE "user.leg.new" +S: N01 OK "NOOP Complete" + +5. MUPDATE Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. This uses the ABNF core + rules as specified in Appendix A of [ABNF]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + Note that this specification also uses some terminals from section 8 + of [ACAP]. + + cmd-activate = "ACTIVATE" SP string SP string SP string + + cmd-authenticate = "AUTHENTICATE" SP sasl-mech [ SP string ] + + + +Siemborski Experimental [Page 12] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + + cmd-delete = "DELETE" SP string + + cmd-find = "FIND" SP string + + cmd-list = "LIST" [ SP string ] + + cmd-logout = "LOGOUT" + + cmd-noop = "NOOP" + + cmd-reserve = "RESERVE" SP string SP string + + cmd-starttls = "STARTTLS" + + cmd-update = "UPDATE" + + command = tag SP command-type CRLF + + command-type = cmd-activate / cmd-authenticate / cmd-delete / + cmd-find / cmd-list / cmd-logout / cmd-noop / + cmd-reserve / cmd-starttls / cmd-update + + response = tag SP response-type CRLF + + response-type = rsp-ok / rsp-no / rsp-bad / rsp-bye / rsp-mailbox / + rsp-reserve / rsp-delete + + rsp-bad = "BAD" SP string + + rsp-bye = "BYE" SP string + + rsp-mailbox = "MAILBOX" SP string SP string SP string + + rsp-no = "NO" SP string + + rsp-ok = "OK" SP string + + rsp-reserve = "RESERVE" SP string SP string + + rsp-delete = "DELETE" SP string + + sasl-mech = 1*ATOM-CHAR + ; ATOM-CHAR is defined in [ACAP] + + string = quoted / literal + ; quoted and literal are defined in [ACAP] + + + + + +Siemborski Experimental [Page 13] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + + tag = 1*ATOM-CHAR + ; ATOM-CHAR is defined in [ACAP] + +6. MUPDATE URL Scheme + + This document defines the a URL scheme for the purposes of + referencing MUPDATE resources, according to the requirements in + [RFC2717]. This includes both MUPDATE servers as a whole, along with + individual mailbox entries on a given MUPDATE server. + + There is no MIME type associated with these resources. It is + intended that a URL consumer would either retrieve the MUPDATE record + in question, or simply connect to the MUPDATE server running on the + specified host. Note that the consumer will need to have + authentication credentials for the specified host. + + The MUPDATE URL scheme is similar to the IMAP URL scheme [IMAP-URL]. + However, it only takes one of two possible forms: + + mupdate:/// + mupdate:/// + + The first form refers to a MUPDATE server as a whole, the second form + indicates both the server and a mailbox to run a FIND against once + authenticated to the server. Note that part of may include + username and authentication information along with a hostname and + port. + +6.1. MUPDATE URL Scheme Registration Form + + URL scheme name: "mupdate" + + URL scheme syntax: + + This defines the MUPDATE URL Scheme in [ABNF]. Terminals from the + BNF of IMAP URLs [IMAP-URL] are also used. + + mupdateurl = "mupdate://" iserver "/" [ enc_mailbox ] + ; iserver and enc_mailbox are as defined in [IMAP-URL] + + Character encoding considerations: + + Identical to those described in [IMAP-URL] for the appropriate + terminals. + + + + + + + +Siemborski Experimental [Page 14] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + + Intended Usage: + + The form of the URL without an associated mailbox is intended to + designate a MUPDATE server only. If a mailbox name is included in + the URL, then the consumer is expected to execute a FIND command + for that mailbox on the specified server. + + Applications and/or protocols which use this URL scheme name: + + The protocol described in this document. + + Interoperability Considerations: + + None. + + Security Considerations: + + Users of the MUPDATE URL Scheme should review the security + considerations that are discussed in [IMAP-URL]. In particular, + the consequences of including authentication mechanism information + in a URL should be reviewed. + + Relevant Publications: + + This document and [IMAP-URL]. + + Author, Change Controller, and Contact for Further Information: + + Author of this document. + +7. Security Considerations + + While no unauthenticated users may make modifications or even perform + searches on the database, it is important to note that this + specification assumes no protections of any type for authenticated + users. + + All authenticated users have complete access to the database. For + this reason it is important to ensure that accounts that are making + use of the database are well secured. + + A more secure deployment might have all read only access go through a + slave, and only have accounts which need write access use the master. + This has the disadvantage of a marginally longer time for updates to + reach the clients. + + + + + + +Siemborski Experimental [Page 15] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + + The protocol assumes that all authenticated users are cooperating to + maintain atomic operations. Therefore, all new mailboxes SHOULD be + RESERVEd before they are ACTIVATEd, despite the fact that the + protocol does not require this, and it is therefore possible for a + set of participants which do not obey the provided locking to create + an inconsistent database. RESERVEing the mailbox first is not + required to perform an activate because this behavior simplifies + synchronization with the actual location of the mailboxes. + +8. IANA Considerations + + The IANA has assigned TCP port number 3905 to "mupdate". + + The IANA has registered a URL scheme for the MUPDATE protocol, as + defined in section 6.1 of this document. + + IANA has registered a GSSAPI service name of "mupdate" for the + MUPDATE protocol in the registry maintained at: + + http://www.iana.org/assignments/gssapi-service-names + +9. Intellectual Property Rights + + The IETF takes no position regarding the validity or scope of any + intellectual property or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; neither does it represent that it + has made any effort to identify any such rights. Information on the + IETF's procedures with respect to rights in standards-track and + standards-related documentation can be found in BCP-11. Copies of + claims of rights made available for publication and any assurances of + licenses to be made available, or the result of an attempt made to + obtain a general license or permission for the use of such + proprietary rights by implementors or users of this specification can + be obtained from the IETF Secretariat. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights which may cover technology that may be required to practice + this standard. Please address the information to the IETF Executive + Director. + + + + + + + + + +Siemborski Experimental [Page 16] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +10. References + +10.1. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4", RFC 3501, March 2003. + + [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 2234, November 1997. + + [MIME] Freed, N. and N. Bornstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [IMAP-ACL] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. + + [SASL] Myers, J., "Simple Authentication and Security Layer + (SASL)", RFC 2222, October 1997. + + [IMAP-URL] Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [TLS] Dierks, T. and C. Allen, "The TLS Protocol Version 1.0", + RFC 2246, January 1999. + +10.2. Informative References + + [POP3] Myers, J. and M. Rose, "Post Office Protocol - Version + 3", STD 53, RFC 1939, May 1996. + + [RFC2717] Petke, R. and I. King, "Registration Procedures for URL + Scheme Names", BCP 35, RFC 2717, November 1999. + + + + + + + + + + + + + + +Siemborski Experimental [Page 17] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +11. Acknowledgments + + Lawrence Greenfield and Ken Murchison, for a great deal of input on + both the protocol and the text of the documents. + +12. Author's Address + + Robert Siemborski + Carnegie Mellon, Andrew Systems Group + Cyert Hall 207 + 5000 Forbes Avenue + Pittsburgh, PA 15213 + + Phone: (412) 268-7456 + EMail: rjs3+@andrew.cmu.edu + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Siemborski Experimental [Page 18] + +RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 + + +13. Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assignees. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Siemborski Experimental [Page 19] + diff --git a/docs/rfcs/rfc3691.txt b/docs/rfcs/rfc3691.txt new file mode 100644 index 0000000..2f4e9b4 --- /dev/null +++ b/docs/rfcs/rfc3691.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 3691 Isode Ltd. +Category: Standards Track February 2004 + + + Internet Message Access Protocol (IMAP) UNSELECT command + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2004). All Rights Reserved. + +Abstract + + This document defines an UNSELECT command that can be used to close + the current mailbox in an Internet Message Access Protocol - version + 4 (IMAP4) session without expunging it. Certain types of IMAP + clients need to release resources associated with the selected + mailbox without selecting a different mailbox. While IMAP4 provides + this functionality (via a SELECT command with a nonexistent mailbox + name or reselecting the same mailbox with EXAMINE command), a more + clean solution is desirable. + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 2. UNSELECT command . . . . . . . . . . . . . . . . . . . . . . . 2 + 3. Security Considerations. . . . . . . . . . . . . . . . . . . . 3 + 4. Formal Syntax. . . . . . . . . . . . . . . . . . . . . . . . . 3 + 5. IANA Considerations. . . . . . . . . . . . . . . . . . . . . . 3 + 6. Acknowledgments. . . . . . . . . . . . . . . . . . . . . . . . 3 + 7. Normative References . . . . . . . . . . . . . . . . . . . . . 4 + 8. Author's Address . . . . . . . . . . . . . . . . . . . . . . . 4 + 9. Full Copyright Statement . . . . . . . . . . . . . . . . . . . 5 + + + + + + + + + + +Melnikov Standards Track [Page 1] + +RFC 3691 IMAP UNSELECT command February 2004 + + +1. Introduction + + Certain types of IMAP clients need to release resources associated + with the selected mailbox without selecting a different mailbox. + While [IMAP4] provides this functionality (via a SELECT command with + a nonexistent mailbox name or reselecting the same mailbox with + EXAMINE command), a more clean solution is desirable. + + [IMAP4] defines the CLOSE command that closes the selected mailbox as + well as permanently removes all messages with the \Deleted flag set. + + However [IMAP4] lacks a command that simply closes the mailbox + without expunging it. This document defines the UNSELECT command for + this purpose. + + A server which supports this extension indicates this with a + capability name of "UNSELECT". + + "C:" and "S:" in examples show lines sent by the client and server + respectively. + + The keywords "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in + this document when typed in uppercase are to be interpreted as + defined in "Key words for use in RFCs to Indicate Requirement Levels" + [KEYWORDS]. + +2. UNSELECT Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - unselect completed, now in authenticated state + BAD - no mailbox selected, or argument supplied but + none permitted + + The UNSELECT command frees server's resources associated with the + selected mailbox and returns the server to the authenticated + state. This command performs the same actions as CLOSE, except + that no messages are permanently removed from the currently + selected mailbox. + + Example: C: A341 UNSELECT + S: A341 OK Unselect completed + + + + + + + +Melnikov Standards Track [Page 2] + +RFC 3691 IMAP UNSELECT command February 2004 + + +3. Security Considerations + + It is believed that this extension doesn't raise any additional + security concerns not already discussed in [IMAP4]. + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. Non-terminals + referenced but not defined below are as defined by [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + command-select /= "UNSELECT" + +5. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + This document defines the UNSELECT IMAP capabilities. IANA has added + this capability to the registry. + +6. Acknowledgments + + UNSELECT command was originally implemented by Tim Showalter in Cyrus + IMAP server. + + Also, the author of the document would like to thank Vladimir Butenko + and Mark Crispin for reminding that UNSELECT has to be documented. + Also thanks to Simon Josefsson for pointing out that there are + multiple ways to implement UNSELECT. + + + + + + + + + + + + + +Melnikov Standards Track [Page 3] + +RFC 3691 IMAP UNSELECT command February 2004 + + +7. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + +8. Author's Address + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + Hampton, Middlesex TW12 2BX + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 4] + +RFC 3691 IMAP UNSELECT command February 2004 + + +9. Full Copyright Statement + + Copyright (C) The Internet Society (2004). This document is subject + to the rights, licenses and restrictions contained in BCP 78 and + except as set forth therein, the authors retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE + REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE + INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed + to pertain to the implementation or use of the technology + described in this document or the extent to which any license + under such rights might or might not be available; nor does it + represent that it has made any independent effort to identify any + such rights. Information on the procedures with respect to + rights in RFC documents can be found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use + of such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository + at http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention + any copyrights, patents or patent applications, or other + proprietary rights that may cover technology that may be required + to implement this standard. Please address the information to the + IETF at ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + +Melnikov Standards Track [Page 5] + diff --git a/docs/rfcs/rfc4314.txt b/docs/rfcs/rfc4314.txt new file mode 100644 index 0000000..e73a56f --- /dev/null +++ b/docs/rfcs/rfc4314.txt @@ -0,0 +1,1515 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 4314 Isode Ltd. +Obsoletes: 2086 December 2005 +Category: Standards Track + + + IMAP4 Access Control List (ACL) Extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2005). + +Abstract + + The Access Control List (ACL) extension (RFC 2086) of the Internet + Message Access Protocol (IMAP) permits mailbox access control lists + to be retrieved and manipulated through the IMAP protocol. + + This document is a revision of RFC 2086. It defines several new + access control rights and clarifies which rights are required for + different IMAP commands. + + + + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 1] + +RFC 4314 IMAP ACL December 2005 + + +Table of Contents + + 1. Introduction and Overview .......................................3 + 1.1. Conventions Used in This Document ..........................3 + 2. Access Control ..................................................3 + 2.1. Standard Rights ............................................5 + 2.1.1. Obsolete Rights .....................................5 + 2.2. Rights Defined in RFC 2086 .................................8 + 3. Access control management commands and responses ................8 + 3.1. SETACL Command .............................................8 + 3.2. DELETEACL Command ..........................................9 + 3.3. GETACL Command ............................................10 + 3.4. LISTRIGHTS Command ........................................10 + 3.5. MYRIGHTS Command ..........................................11 + 3.6. ACL Response ..............................................11 + 3.7. LISTRIGHTS Response .......................................12 + 3.8. MYRIGHTS Response .........................................12 + 4. Rights Required to Perform Different IMAP4rev1 Commands ........12 + 5. Other Considerations ...........................................17 + 5.1. Additional Requirements and Implementation Notes ..........17 + 5.1.1. Servers ............................................17 + 5.1.2. Clients ............................................18 + 5.2. Mapping of ACL Rights to READ-WRITE and READ-ONLY + Response Codes ............................................19 + 6. Security Considerations ........................................20 + 7. Formal Syntax ..................................................21 + 8. IANA Considerations ............................................22 + 9. Internationalization Considerations ............................22 + Appendix A. Changes since RFC 2086 ................................23 + Appendix B. Compatibility with RFC 2086 ...........................24 + Appendix C. Known Deficiencies ....................................24 + Appendix D. Acknowledgements ......................................25 + Normative References ..............................................25 + Informative References ............................................25 + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 2] + +RFC 4314 IMAP ACL December 2005 + + +1. Introduction and Overview + + The ACL (Access Control List) extension of the Internet Message + Access Protocol [IMAP4] permits mailbox access control lists to be + retrieved and manipulated through the IMAP protocol. + + This document is a revision of RFC 2086 [RFC2086]. It tries to + clarify different ambiguities in RFC 2086, in particular, the use of + UTF-8 [UTF-8] in access identifiers, which rights are required for + different IMAP4 commands, and how READ-WRITE/READ-ONLY response codes + are related to ACL. + +1.1. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + In all examples "/" character is used as hierarchy separator. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + + The phrase "ACL server" is just a shortcut for saying "IMAP server + that supports ACL extension as defined in this document". + +2. Access Control + + The ACL extension is present in any IMAP4 implementation that returns + "ACL" as one of the supported capabilities to the CAPABILITY command. + + A server implementation conformant to this document MUST also return + rights (see below) not defined in Section 2.2 in the "RIGHTS=" + capability. + + An access control list is a set of pairs. + An ACL applies to a mailbox name. + + Access identifier (or just "identifier") is a UTF-8 [UTF-8] string. + The identifier "anyone" is reserved to refer to the universal + identity (all authentications, including anonymous). All user name + strings accepted by the LOGIN or AUTHENTICATE commands to + authenticate to the IMAP server are reserved as identifiers for the + corresponding users. Identifiers starting with a dash ("-") are + reserved for "negative rights", described below. All other + identifier strings are interpreted in an implementation-defined + manner. + + + + +Melnikov Standards Track [Page 3] + +RFC 4314 IMAP ACL December 2005 + + + Rights is a string listing a (possibly empty) set of alphanumeric + characters, each character listing a set of operations that is being + controlled. Lowercase letters are reserved for "standard" rights, + listed in Section 2.1. (Note that for compatibility with deployed + clients and servers uppercase rights are not allowed.) The set of + standard rights can only be extended by a standards-track document. + Digits are reserved for implementation- or site-defined rights. + + An implementation MAY tie rights together or MAY force rights to + always or never be granted to particular identifiers. For example, + in an implementation that uses UNIX mode bits, the rights "swite" are + tied, the "a" right is always granted to the owner of a mailbox and + is never granted to another user. If rights are tied in an + implementation, the implementation must be conservative in granting + rights in response to SETACL commands--unless all rights in a tied + set are specified, none of that set should be included in the ACL + entry for that identifier. A client can discover the set of rights + that may be granted to a given identifier in the ACL for a given + mailbox name by using the LISTRIGHTS command. + + It is possible for multiple identifiers in an access control list to + apply to a given user. For example, an ACL may include rights to be + granted to the identifier matching the user, one or more + implementation-defined identifiers matching groups that include the + user, and/or the identifier "anyone". How these rights are combined + to determine the user's access is implementation defined. An + implementation may choose, for example, to use the union of the + rights granted to the applicable identifiers. An implementation may + instead choose, for example, to use only those rights granted to the + most specific identifier present in the ACL. A client can determine + the set of rights granted to the logged-in user for a given mailbox + name by using the MYRIGHTS command. + + When an identifier in an ACL starts with a dash ("-"), that indicates + that associated rights are to be removed from the identifier prefixed + by the dash. This is referred to as a "negative right". This + differs from DELETEACL in that a negative right is added to the ACL + and is a part of the calculation of the rights. + + Let's assume that an identifier "fred" refers to a user with login + "fred". If the identifier "-fred" is granted the "w" right, that + indicates that the "w" right is to be removed from users matching the + identifier "fred", even though the user "fred" might have the "w" + right as a consequence of some other identifier in the ACL. A + DELETEACL of "fred" simply deletes the identifier "fred" from the + ACL; it does not affect any rights that the user "fred" may get from + another entry in the ACL, in particular it doesn't affect rights + granted to the identifier "-fred". + + + +Melnikov Standards Track [Page 4] + +RFC 4314 IMAP ACL December 2005 + + + Server implementations are not required to support "negative right" + identifiers. + +2.1. Standard Rights + + The currently defined standard rights are (note that the list below + doesn't list all commands that use a particular right): + + l - lookup (mailbox is visible to LIST/LSUB commands, SUBSCRIBE + mailbox) + r - read (SELECT the mailbox, perform STATUS) + s - keep seen/unseen information across sessions (set or clear + \SEEN flag via STORE, also set \SEEN during APPEND/COPY/ + FETCH BODY[...]) + w - write (set or clear flags other than \SEEN and \DELETED via + STORE, also set them during APPEND/COPY) + i - insert (perform APPEND, COPY into mailbox) + p - post (send mail to submission address for mailbox, + not enforced by IMAP4 itself) + k - create mailboxes (CREATE new sub-mailboxes in any + implementation-defined hierarchy, parent mailbox for the new + mailbox name in RENAME) + x - delete mailbox (DELETE mailbox, old mailbox name in RENAME) + t - delete messages (set or clear \DELETED flag via STORE, set + \DELETED flag during APPEND/COPY) + e - perform EXPUNGE and expunge as a part of CLOSE + a - administer (perform SETACL/DELETEACL/GETACL/LISTRIGHTS) + +2.1.1. Obsolete Rights + + Due to ambiguity in RFC 2086, some existing RFC 2086 server + implementations use the "c" right to control the DELETE command. + Others chose to use the "d" right to control the DELETE command. For + the former group, let's define the "create" right as union of the "k" + and "x" rights, and the "delete" right as union of the "e" and "t" + rights. For the latter group, let's define the "create" rights as a + synonym to the "k" right, and the "delete" right as union of the "e", + "t", and "x" rights. + + For compatibility with RFC 2086, this section defines two virtual + rights "d" and "c". + + If a client includes the "d" right in a rights list, then it MUST be + treated as if the client had included every member of the "delete" + right. (It is not an error for a client to specify both the "d" + right and one or more members of the "delete" right, but the effect + is no different than if just the "d" right or all members of the + "delete" right had been specified.) + + + +Melnikov Standards Track [Page 5] + +RFC 4314 IMAP ACL December 2005 + + + When any of the "delete" member rights is set in a list of rights, + the server MUST also include the "d" right when returning the list in + a MYRIGHTS or ACL response. This is to enable older clients + conforming to RFC 2086 to work with newer servers. (*) + + Example: C: A001 SeTacl INBOX/Drafts David lrswida + S: A001 OK Setacl complete + + The client has specified the "d" right in the SETACL command above + and it expands to "et" on the server: + + C: A002 getacl INBOX/Drafts + S: * ACL INBOX Fred rwipslxcetda David lrswideta + S: A002 OK Getacl complete + + If the identifier specified in the LISTRIGHTS command can be granted + any of the "delete" member rights on a mailbox, then the server MUST + include the "d" right in the corresponding LISTRIGHTS response. (*) + If the member rights aren't tied to non-member rights, then the "d" + right is returned by itself in the LISTRIGHTS response. If any of + the member rights needs to be tied to one (or more) non-member right, + then the "d" right and all of the member rights need to be tied to + the same non-member right(s) (**). + + If a client includes the "c" right in a rights list, then it MUST be + treated as if the client had included every member of the "create" + right. (It is not an error for a client to specify both the "c" + right and one or more members of the "create" right, but the effect + is no different than if just the "c" right or all members of the + "create" right had been specified.) + + When any of the "create" member rights is set in a list of rights, + the server MUST also include the "c" right when returning the list in + a MYRIGHTS or ACL response. This is to enable older clients + conforming to RFC 2086 to work with newer servers. (*) + + Example: C: A003 Setacl INBOX/Drafts Byron lrswikda + S: A001 OK Setacl complete + C: A002 getAcl INBOX/Drafts + S: * ACL INBOX Fred rwipslxcetda Byron lrswikcdeta + S: A002 OK Getacl complete + + The client has specified the "d" right in the SETACL command above + and it expands to "et" on the server: As the client has specified the + "k" right (which is a member of the "c" right), the server also + returns the "c" right. + + + + + +Melnikov Standards Track [Page 6] + +RFC 4314 IMAP ACL December 2005 + + + If the identifier specified in the LISTRIGHTS command can be granted + any of the "create" member rights on a mailbox, then the server MUST + include the "c" right in the corresponding LISTRIGHTS response. (*) + If the member rights aren't tied to non-member rights, then the "c" + right is returned by itself in the LISTRIGHTS response. If any of + the member rights needs to be tied to one (or more) non-member right, + then the "c" right and all of the member rights need to be tied to + the same non-member right(s) (**). + + Example: The server that ties the rights as follows: + + lr s w i p k x t + + and c=k + + will return: + + S: * LISTRIGHTS archive/imap anyone "" + lr s w i p k x t c d + + Example: The server that ties the rights as follows: + + lr s w i p k xte + + and c=k + + will return: + + S: * LISTRIGHTS archive/imap anyone "" + lr s w i p k xte c d + + Example: The server that ties the rights as follows: + + lr s w i p k x te + + and c=k + + will return: + + S: * LISTRIGHTS archive/imap anyone "" + lr s w i p k c x te d + + Example: The server that ties the rights as follows: + + lr swte i p k x + + and c=kx + + + + +Melnikov Standards Track [Page 7] + +RFC 4314 IMAP ACL December 2005 + + + will return: + + S: * LISTRIGHTS archive/imap anyone "" + lr swted i p k x c + + (*) Clients conforming to this document MUST ignore the virtual "d" + and "c" rights in MYRIGHTS, ACL, and LISTRIGHTS responses. + + (**) The IMAPEXT Working Group has debated this issue in great length + and after reviewing existing ACL implementations concluded that + this is a reasonable restriction. + +2.2. Rights Defined in RFC 2086 + + The "RIGHTS=" capability MUST NOT include any of the rights defined + in RFC 2086: "l", "r", "s", "w", "i", "p", "a", "c", "d", and the + digits ("0" .. "9"). + +3. Access control management commands and responses + + Servers, when processing a command that has an identifier as a + parameter (i.e., any of SETACL, DELETEACL, and LISTRIGHTS commands), + SHOULD first prepare the received identifier using "SASLprep" profile + [SASLprep] of the "stringprep" algorithm [Stringprep]. If the + preparation of the identifier fails or results in an empty string, + the server MUST refuse to perform the command with a BAD response. + Note that Section 6 recommends additional identifier's verification + steps. + +3.1. SETACL Command + + Arguments: mailbox name + identifier + access right modification + + Data: no specific data for this command + + Result: OK - setacl completed + NO - setacl failure: can't set acl + BAD - arguments invalid + + The SETACL command changes the access control list on the specified + mailbox so that the specified identifier is granted permissions as + specified in the third argument. + + The third argument is a string containing an optional plus ("+") or + minus ("-") prefix, followed by zero or more rights characters. If + the string starts with a plus, the following rights are added to any + + + +Melnikov Standards Track [Page 8] + +RFC 4314 IMAP ACL December 2005 + + + existing rights for the identifier. If the string starts with a + minus, the following rights are removed from any existing rights for + the identifier. If the string does not start with a plus or minus, + the rights replace any existing rights for the identifier. + + Note that an unrecognized right MUST cause the command to return the + BAD response. In particular, the server MUST NOT silently ignore + unrecognized rights. + + Example: C: A001 GETACL INBOX/Drafts + S: * ACL INBOX/Drafts Fred rwipslxetad Chris lrswi + S: A001 OK Getacl complete + C: A002 SETACL INBOX/Drafts Chris +cda + S: A002 OK Setacl complete + C: A003 GETACL INBOX/Drafts + S: * ACL INBOX/Drafts Fred rwipslxetad Chris lrswicdakxet + S: A003 OK Getacl complete + + + C: A035 SETACL INBOX/Drafts John lrQswicda + S: A035 BAD Uppercase rights are not allowed + + + C: A036 SETACL INBOX/Drafts John lrqswicda + S: A036 BAD The q right is not supported + +3.2. DELETEACL Command + + Arguments: mailbox name + identifier + + Data: no specific data for this command + + Result: OK - deleteacl completed + NO - deleteacl failure: can't delete acl + BAD - arguments invalid + + The DELETEACL command removes any pair for the + specified identifier from the access control list for the specified + mailbox. + + Example: C: B001 getacl INBOX + S: * ACL INBOX Fred rwipslxetad -Fred wetd $team w + S: B001 OK Getacl complete + C: B002 DeleteAcl INBOX Fred + S: B002 OK Deleteacl complete + + + + + +Melnikov Standards Track [Page 9] + +RFC 4314 IMAP ACL December 2005 + + + C: B003 GETACL INBOX + S: * ACL INBOX -Fred wetd $team w + S: B003 OK Getacl complete + +3.3. GETACL Command + + Arguments: mailbox name + + Data: untagged responses: ACL + + Result: OK - getacl completed + NO - getacl failure: can't get acl + BAD - arguments invalid + + The GETACL command returns the access control list for mailbox in an + untagged ACL response. + + Some implementations MAY permit multiple forms of an identifier to + reference the same IMAP account. Usually, such implementations will + have a canonical form that is stored internally. An ACL response + caused by a GETACL command MAY include a canonicalized form of the + identifier that might be different from the one used in the + corresponding SETACL command. + + Example: C: A002 GETACL INBOX + S: * ACL INBOX Fred rwipsldexta + S: A002 OK Getacl complete + +3.4. LISTRIGHTS Command + + Arguments: mailbox name + identifier + + Data: untagged responses: LISTRIGHTS + + Result: OK - listrights completed + NO - listrights failure: can't get rights list + BAD - arguments invalid + + The LISTRIGHTS command takes a mailbox name and an identifier and + returns information about what rights can be granted to the + identifier in the ACL for the mailbox. + + Some implementations MAY permit multiple forms of an identifier to + reference the same IMAP account. Usually, such implementations will + have a canonical form that is stored internally. A LISTRIGHTS + + + + + +Melnikov Standards Track [Page 10] + +RFC 4314 IMAP ACL December 2005 + + + response caused by a LISTRIGHTS command MUST always return the same + form of an identifier as specified by the client. This is to allow + the client to correlate the response with the command. + + Example: C: a001 LISTRIGHTS ~/Mail/saved smith + S: * LISTRIGHTS ~/Mail/saved smith la r swicdkxte + S: a001 OK Listrights completed + + Example: C: a005 listrights archive/imap anyone + S: * LISTRIGHTS archive.imap anyone "" + l r s w i p k x t e c d a 0 1 2 3 4 5 6 7 8 9 + S: a005 Listrights successful + +3.5. MYRIGHTS Command + + Arguments: mailbox name + + Data: untagged responses: MYRIGHTS + + Result: OK - myrights completed + NO - myrights failure: can't get rights + BAD - arguments invalid + + The MYRIGHTS command returns the set of rights that the user has to + mailbox in an untagged MYRIGHTS reply. + + Example: C: A003 MYRIGHTS INBOX + S: * MYRIGHTS INBOX rwiptsldaex + S: A003 OK Myrights complete + +3.6. ACL Response + + Data: mailbox name + zero or more identifier rights pairs + + The ACL response occurs as a result of a GETACL command. The first + string is the mailbox name for which this ACL applies. This is + followed by zero or more pairs of strings; each pair contains the + identifier for which the entry applies followed by the set of rights + that the identifier has. + + Section 2.1.1 details additional server requirements related to + handling of the virtual "d" and "c" rights. + + + + + + + + +Melnikov Standards Track [Page 11] + +RFC 4314 IMAP ACL December 2005 + + +3.7. LISTRIGHTS Response + + Data: mailbox name + identifier + required rights + list of optional rights + + The LISTRIGHTS response occurs as a result of a LISTRIGHTS command. + The first two strings are the mailbox name and identifier for which + this rights list applies. Following the identifier is a string + containing the (possibly empty) set of rights the identifier will + always be granted in the mailbox. + + Following this are zero or more strings each containing a set of + rights the identifier can be granted in the mailbox. Rights + mentioned in the same string are tied together. The server MUST + either grant all tied rights to the identifier in the mailbox or + grant none. Section 2.1.1 details additional server requirements + related to handling of the virtual "d" and "c" rights. + + The same right MUST NOT be listed more than once in the LISTRIGHTS + command. + +3.8. MYRIGHTS Response + + Data: mailbox name + rights + + The MYRIGHTS response occurs as a result of a MYRIGHTS command. The + first string is the mailbox name for which these rights apply. The + second string is the set of rights that the client has. + + Section 2.1.1 details additional server requirements related to + handling of the virtual "d" and "c" rights. + +4. Rights Required to Perform Different IMAP4rev1 Commands + + Before executing a command, an ACL-compliant server MUST check which + rights are required to perform it. This section groups command by + functions they perform and list the rights required. It also gives + the detailed description of any special processing required. + + For the purpose of this section the UID counterpart of a command is + considered to be the same command, e.g., both UID COPY and COPY + commands require the same set of rights. + + + + + + +Melnikov Standards Track [Page 12] + +RFC 4314 IMAP ACL December 2005 + + + The table below summarizes different rights or their combinations + that are required in order to perform different IMAP operations. As + it is not always possible to express complex right checking and + interactions, the description after the table should be used as the + primary reference. + + +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ + |Operations\Rights | l | r | s | w | i | k | x | t | e | a |Any|Non| + +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ + | commands in authenticated state | + +-------------------------------------------------------------------+ + | LIST | + | | | | | | | | | | | | + | SUBSCRIBE | * | | | | | | | | | | | * | + | UNSUBSCRIBE | | | | | | | | | | | | + | + | LSUB | * | | | | | | | | | | | * | + |CREATE (for parent)| | | | | | + | | | | | | | + | DELETE | | ? | | | | | + | ? | ? | | | | + | RENAME | | | | | | + | + | | | | | | + | SELECT/EXAMINE | | + | | | | | | | | | | | + | STATUS | | + | | | | | | | | | | | + | SETACL/DELETEACL | | | | | | | | | | + | | | + | GETACL/LISTRIGHTS | | | | | | | | | | + | | | + | MYRIGHTS | | | | | | | | | | | + | | + | APPEND | | | ? | ? | + | | | ? | | | | | + +-------------------------------------------------------------------+ + | commands in selected state | + +-------------------------------------------------------------------+ + | COPY | | | ? | ? | + | | | ? | | | | | + | EXPUNGE | | | | | | | | | + | | | | + | CLOSE | | | | | | | | | ? | | | | + | FETCH | | | ? | | | | | | | | | | + | STORE flags | | | ? | ? | | | | ? | | | | | + +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ + + Note: for all commands in the selected state, the "r" is implied, + because it is required to SELECT/EXAMINE a mailbox. Servers are not + required to check presence of the "r" right once a mailbox is + successfully selected. + + Legend: + + - The right is required + * - Only one of the rights marked with * is required + (see description below) + ? - The right is OPTIONAL (see description below) + "Any" - at least one of the "l", "r", "i", "k", "x", "a" rights is + required + "Non" - No rights required to perform the command + + + + +Melnikov Standards Track [Page 13] + +RFC 4314 IMAP ACL December 2005 + + + Listing and subscribing/unsubscribing mailboxes: + LIST - "l" right is required. However, unlike other commands + (e.g., SELECT) the server MUST NOT return a NO response if it + can't list a mailbox. + Note that if the user has "l" right to a mailbox "A/B", but not to + its parent mailbox "A", the LIST command should behave as if the + mailbox "A" doesn't exist, for example: + + C: A777 LIST "" * + S: * LIST (\NoInferiors) "/" "A/B" + S: * LIST () "/" "C" + S: * LIST (\NoInferiors) "/" "C/D" + S: A777 OK LIST completed + + + SUBSCRIBE - "l" right is required only if the server checks for + mailbox existence when performing SUBSCRIBE. + + UNSUBSCRIBE - no rights required to perform this operation. + + LSUB - "l" right is required only if the server checks for mailbox + existence when performing SUBSCRIBE. However, unlike other + commands (e.g., SELECT) the server MUST NOT return a NO response + if it can't list a subscribed mailbox. + + Mailbox management: + CREATE - "k" right on a nearest existing parent mailbox. When a + new mailbox is created, it SHOULD inherit the ACL from the parent + mailbox (if one exists) in the defined hierarchy. + + DELETE - "x" right on the mailbox. Note that some servers don't + allow to delete a non-empty mailbox. If this is the case, the + user would also need "r", "e", and "t" rights, in order to open + the mailbox and empty it. + + The DELETE command MUST delete the ACL associated with the deleted + mailbox. + + RENAME - Moving a mailbox from one parent to another requires the + "x" right on the mailbox itself and the "k" right for the new + parent. For example, if the user wants to rename the mailbox + named "A/B/C" to "D/E", the user must have the "x" right for the + mailbox "A/B/C" and the "k" right for the mailbox "D". + The RENAME command SHOULD NOT change the ACLs on the renamed + mailbox and submailboxes. + + + + + + +Melnikov Standards Track [Page 14] + +RFC 4314 IMAP ACL December 2005 + + + Copying or appending messages: + Before performing a COPY/APPEND command, the server MUST check if + the user has "i" right for the target mailbox. If the user + doesn't have "i" right, the operation fails. Otherwise for each + copied/appended message the server MUST check if the user has + "t" right - when the message has \Deleted flag set + "s" right - when the message has \Seen flag set + "w" right - for all other message flags. + Only when the user has a particular right are the corresponding + flags stored for the newly created message. The server MUST NOT + fail a COPY/APPEND if the user has no rights to set a particular + flag. + + Example: C: A003 MYRIGHTS TargetMailbox + S: * MYRIGHTS TargetMailbox rwis + S: A003 OK Myrights complete + + C: A004 FETCH 1:3 (FLAGS) + S: * 1 FETCH (FLAGS (\Draft \Deleted) + S: * 2 FETCH (FLAGS (\Answered) + S: * 3 FETCH (FLAGS ($Forwarded \Seen) + S: A004 OK Fetch Completed + + C: A005 COPY 1:3 TargetMailbox + S: A005 OK Copy completed + + C: A006 SELECT TargetMailbox + ... + S: A006 Select Completed + + Let's assume that the copied messages received message numbers + 77:79. + + C: A007 FETCH 77:79 (FLAGS) + S: * 77 FETCH (FLAGS (\Draft)) + S: * 78 FETCH (FLAGS (\Answered)) + S: * 79 FETCH (FLAGS ($Forwarded \Seen)) + S: A007 OK Fetch Completed + + \Deleted flag was lost on COPY, as the user has no "t" right in + the target mailbox. + If the MYRIGHTS command with the tag A003 would have returned: + + S: * MYRIGHTS TargetMailbox rsti + + the response from the FETCH with the tag A007 would have been: + + C: A007 FETCH 77:79 (FLAGS) + + + +Melnikov Standards Track [Page 15] + +RFC 4314 IMAP ACL December 2005 + + + S: * 77 FETCH (FLAGS (\Deleted)) + S: * 78 FETCH (FLAGS ()) + S: * 79 FETCH (FLAGS (\Seen)) + S: A007 OK Fetch Completed + + In the latter case, \Answered, $Forwarded, and \Draft flags were + lost on COPY, as the user has no "w" right in the target mailbox. + + Expunging the selected mailbox: + EXPUNGE - "e" right on the selected mailbox. + + CLOSE - "e" right on the selected mailbox. If the server is + unable to expunge the mailbox because the user doesn't have the + "e" right, the server MUST ignore the expunge request, close the + mailbox, and return the tagged OK response. + + Fetch information about a mailbox and its messages: + SELECT/EXAMINE/STATUS - "r" right on the mailbox. + + FETCH - A FETCH request that implies setting \Seen flag MUST NOT + set it, if the current user doesn't have "s" right. + + Changing flags: + STORE - the server MUST check if the user has + "t" right - when the user modifies \Deleted flag + "s" right - when the user modifies \Seen flag + "w" right - for all other message flags. + STORE operation SHOULD NOT fail if the user has rights to modify + at least one flag specified in the STORE, as the tagged NO + response to a STORE command is not handled very well by deployed + clients. + + Changing ACLs: + SETACL/DELETEACL - "a" right on the mailbox. + + Reading ACLs: + GETACL - "a" right on the mailbox. + + MYRIGHTS - any of the following rights is required to perform the + operation: "l", "r", "i", "k", "x", "a". + + LISTRIGHTS - "a" right on the mailbox. + + + + + + + + + +Melnikov Standards Track [Page 16] + +RFC 4314 IMAP ACL December 2005 + + +5. Other Considerations + +5.1. Additional Requirements and Implementation Notes + +5.1.1. Servers + + This document defines an additional capability that is used to + announce the list of extra rights (excluding the ones defined in RFC + 2086) supported by the server. The set of rights MUST include "t", + "e", "x", and "k". Note that the extra rights can appear in any + order. + + Example: C: 1 capability + S: * CAPABILITY IMAP4REV1 STARTTLS LITERAL+ + ACL RIGHTS=texk + S: 1 OK completed + + Any server implementing an ACL extension MUST accurately reflect the + current user's rights in FLAGS and PERMANENTFLAGS responses. + + Example: C: A142 SELECT INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Seen \Answered \Flagged \*)] L + S: A142 OK [READ-WRITE] SELECT completed + C: A143 MYRIGHTS INBOX + S: * MYRIGHTS INBOX lrwis + S: A143 OK completed + + Note that in order to get better performance the client MAY pipeline + SELECT and MYRIGHTS commands: + + C: A142 SELECT INBOX + C: A143 MYRIGHTS INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Seen \Answered \Flagged \*)] L + S: A142 OK [READ-WRITE] SELECT completed + S: * MYRIGHTS INBOX lrwis + S: A143 OK completed + + + +Melnikov Standards Track [Page 17] + +RFC 4314 IMAP ACL December 2005 + + + Servers MAY cache the rights a user has on a mailbox when the mailbox + is selected, so that if a client's rights on a mailbox are changed + with SETACL or DELETEACL, commands specific to the selected state + (e.g., STORE, EXPUNGE) might not reflect the changed rights until the + mailbox is re-selected. If the server checks the rights on each + command, then it SHOULD send FLAGS and PERMANENTFLAGS responses if + they have changed. If such server detects that the user no longer + has read access to the mailbox, it MAY send an untagged BYE response + and close connection. It MAY also refuse to execute all commands + specific to the selected state until the mailbox is closed; however, + server implementors should note that most clients don't handle NO + responses very well. + + An ACL server MAY modify one or more ACLs for one or more identifiers + as a side effect of modifying the ACL specified in a + SETACL/DELETEACL. If the server does that, it MUST send untagged ACL + response(s) to notify the client about the changes made. + + An ACL server implementation MUST treat received ACL modification + commands as a possible ambiguity with respect to subsequent commands + affected by the ACL, as described in Section 5.5 of [IMAP4]. Hence a + pipeline SETACL + MYRIGHTS is an ambiguity with respect to the + server, meaning that the server must execute the SETACL command to + completion before the MYRIGHTS. However, clients are permitted to + send such a pipeline. + +5.1.2. Clients + + The following requirement is put on clients in order to allow for + future extensibility. A client implementation that allows a user to + read and update ACLs MUST preserve unrecognized rights that it + doesn't allow the user to change. That is, if the client + + 1) can read ACLs + and + 2) can update ACLs + but + 3) doesn't allow the user to change the rights the client doesn't + recognize, then it MUST preserve unrecognized rights. + + Otherwise the client could risk unintentionally removing permissions + it doesn't understand. + + + + + + + + + +Melnikov Standards Track [Page 18] + +RFC 4314 IMAP ACL December 2005 + + +5.2. Mapping of ACL Rights to READ-WRITE and READ-ONLY Response Codes + + A particular ACL server implementation MAY allow "shared multiuser + access" to some mailboxes. "Shared multiuser access" to a mailbox + means that multiple different users are able to access the same + mailbox, if they have proper access rights. "Shared multiuser + access" to the mailbox doesn't mean that the ACL for the mailbox is + currently set to allow access by multiple users. Let's denote a + "shared multiuser write access" as a "shared multiuser access" when a + user can be granted flag modification rights (any of "w", "s", or + "t"). + + Section 4 describes which rights are required for modifying different + flags. + + If the ACL server implements some flags as shared for a mailbox + (i.e., the ACL for the mailbox MAY be set up so that changes to those + flags are visible to another user), let's call the set of rights + associated with these flags (as described in Section 4) for that + mailbox collectively as "shared flag rights". Note that the "shared + flag rights" set MAY be different for different mailboxes. + + If the server doesn't support "shared multiuser write access" to a + mailbox or doesn't implement shared flags on the mailbox, "shared + flag rights" for the mailbox is defined to be the empty set. + + Example 1: Mailbox "banan" allows "shared multiuser write access" and + implements flags \Deleted, \Answered, and $MDNSent as + shared flags. "Shared flag rights" for the mailbox "banan" + is a set containing flags "t" (because system flag + \Deleted requires "t" right) and "w" (because both + \Answered and $MDNSent require "w" right). + + Example 2: Mailbox "apple" allows "shared multiuser write access" and + implements \Seen system flag as shared flag. "Shared flag + rights" for the mailbox "apple" contains "s" right + because system flag \Seen requires "s" right. + + Example 3: Mailbox "pear" allows "shared multiuser write access" and + implements flags \Seen, \Draft as shared flags. "Shared + flag rights" for the mailbox "apple" is a set containing + flags "s" (because system flag \Seen requires "s" right) + and "w" (because system flag \Draft requires "w" right). + + The server MUST include a READ-ONLY response code in the tagged OK + response to a SELECT command if none of the following rights is + granted to the current user: + + + + +Melnikov Standards Track [Page 19] + +RFC 4314 IMAP ACL December 2005 + + + "i", "e", and "shared flag rights"(***). + + The server SHOULD include a READ-WRITE response code in the tagged OK + response if at least one of the "i", "e", or "shared flag + rights"(***) is granted to the current user. + + (***) Note that a future extension to this document can extend the + list of rights that causes the server to return the READ-WRITE + response code. + + Example 1 (continued): The user that has "lrs" rights for the mailbox + "banan". The server returns READ-ONLY + response code on SELECT, as none of "iewt" + rights is granted to the user. + + Example 2 (continued): The user that has "rit" rights for the mailbox + "apple". The server returns READ-WRITE + response code on SELECT, as the user has "i" + right. + + Example 3 (continued): The user that has "rset" rights for the + mailbox "pear". The server returns READ-WRITE + response code on SELECT, as the user has "e" + and "s" rights. + +6. Security Considerations + + An implementation MUST make sure the ACL commands themselves do not + give information about mailboxes with appropriately restricted ACLs. + For example, when a user agent executes a GETACL command on a mailbox + that the user has no permission to LIST, the server would respond to + that request with the same error that would be used if the mailbox + did not exist, thus revealing no existence information, much less the + mailbox's ACL. + + IMAP clients implementing ACL that are able to modify ACLs SHOULD + warn a user that wants to give full access (or even just the "a" + right) to the special identifier "anyone". + + This document relies on [SASLprep] to describe steps required to + perform identifier canonicalization (preparation). The preparation + algorithm in SASLprep was specifically designed such that its output + is canonical, and it is well-formed. However, due to an anomaly + [PR29] in the specification of Unicode normalization, canonical + equivalence is not guaranteed for a select few character sequences. + Identifiers prepared with SASLprep can be stored and returned by an + ACL server. The anomaly affects ACL manipulation and evaluation of + identifiers containing the selected character sequences. These + + + +Melnikov Standards Track [Page 20] + +RFC 4314 IMAP ACL December 2005 + + + sequences, however, do not appear in well-formed text. In order to + address this problem, an ACL server MAY reject identifiers containing + sequences described in [PR29] by sending the tagged BAD response. + This is in addition to the requirement to reject identifiers that + fail SASLprep preparation as described in Section 3. + + Other security considerations described in [IMAP4] are relevant to + this document. In particular, ACL information is sent in the clear + over the network unless confidentiality protection is negotiated. + + This can be accomplished either by the use of STARTTLS, negotiated + privacy protection in the AUTHENTICATE command, or some other + protection mechanism. + +7. Formal Syntax + + Formal syntax is defined using ABNF [ABNF], extending the ABNF rules + in Section 9 of [IMAP4]. Elements not defined here can be found in + [ABNF] and [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case + insensitive. The use of uppercase or lowercase characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + LOWER-ALPHA = %x61-7A ;; a-z + + acl-data = "ACL" SP mailbox *(SP identifier SP + rights) + + capability =/ rights-capa + ;;capability is defined in [IMAP4] + + command-auth =/ setacl / deleteacl / getacl / + listrights / myrights + ;;command-auth is defined in [IMAP4] + + deleteacl = "DELETEACL" SP mailbox SP identifier + + getacl = "GETACL" SP mailbox + + identifier = astring + + listrights = "LISTRIGHTS" SP mailbox SP identifier + + listrights-data = "LISTRIGHTS" SP mailbox SP identifier + SP rights *(SP rights) + + + + +Melnikov Standards Track [Page 21] + +RFC 4314 IMAP ACL December 2005 + + + mailbox-data =/ acl-data / listrights-data / myrights-data + ;;mailbox-data is defined in [IMAP4] + + mod-rights = astring + ;; +rights to add, -rights to remove + ;; rights to replace + + myrights = "MYRIGHTS" SP mailbox + + myrights-data = "MYRIGHTS" SP mailbox SP rights + + new-rights = 1*LOWER-ALPHA + ;; MUST include "t", "e", "x", and "k". + ;; MUST NOT include standard rights listed + ;; in section 2.2 + + rights = astring + ;; only lowercase ASCII letters and digits + ;; are allowed. + + rights-capa = "RIGHTS=" new-rights + ;; RIGHTS=... capability + + setacl = "SETACL" SP mailbox SP identifier + SP mod-rights + +8. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards-track or + IESG-approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + This document defines the RIGHTS= IMAP capability. IANA has added + this capability to the registry. + +9. Internationalization Considerations + + Section 3 states requirements on servers regarding + internationalization of identifiers. + + + + + + + + + + +Melnikov Standards Track [Page 22] + +RFC 4314 IMAP ACL December 2005 + + +Appendix A. Changes since RFC 2086 + + 1. Changed the charset of "identifier" from US-ASCII to UTF-8. + 2. Specified that mailbox deletion is controlled by the "x" right + and EXPUNGE is controlled by the "e" right. + 3. Added the "t" right that controls STORE \Deleted. Redefined the + "d" right to be a macro for "e", "t", and possibly "x". + 4. Added the "k" right that controls CREATE. Redefined the "c" + right to be a macro for "k" and possibly "x". + 5. Specified that the "a" right also controls DELETEACL. + 6. Specified that the "r" right also controls STATUS. + 7. Removed the requirement to check the "r" right for CHECK, SEARCH + and FETCH, as this is required for SELECT/EXAMINE to be + successful. + 8. LISTRIGHTS requires the "a" right on the mailbox (same as + SETACL). + 9. Deleted "PARTIAL", this is a deprecated feature of RFC 1730. + 10. Specified that the "w" right controls setting flags other than + \Seen and \Deleted on APPEND. Also specified that the "s" right + controls the \Seen flag and that the "t" right controls the + \Deleted flag. + 11. Specified that SUBSCRIBE is NOT allowed with the "r" right. + 12. Specified that the "l" right controls SUBSCRIBE. + 13. GETACL is NOT allowed with the "r" right, even though there are + several implementations that allows that. If a user only has + "r" right, GETACL can disclose information about identifiers + existing on the mail system. + 14. Clarified that RENAME requires the "k" right for the new parent + and the "x" right for the old name. + 15. Added new section that describes which rights are required + and/or checked when performing various IMAP commands. + 16. Added mail client security considerations when dealing with + special identifier "anyone". + 17. Clarified that negative rights are not the same as DELETEACL. + 18. Added "Compatibility with RFC 2086" section. + 19. Added section about mapping of ACL rights to READ-WRITE and + READ-ONLY response codes. + 20. Changed BNF to ABNF. + 21. Added "Implementation Notes" section. + 22. Updated "References" section. + 23. Added more examples. + 24. Clarified when the virtual "c" and "d" rights are returned in + ACL, MYRIGHTS, and LISTRIGHTS responses. + + + + + + + + +Melnikov Standards Track [Page 23] + +RFC 4314 IMAP ACL December 2005 + + +Appendix B. Compatibility with RFC 2086 + + This non-normative section gives guidelines as to how an existing RFC + 2086 server implementation may be updated to comply with this + document. + + This document splits the "d" right into several new different rights: + "t", "e", and possibly "x" (see Section 2.1.1 for more details). The + "d" right remains for backward-compatibility, but it is a virtual + right. There are two approaches for RFC 2086 server implementors to + handle the "d" right and the new rights that have replaced it: + + a. Tie "t", "e" (and possibly "x) together - almost no changes. + b. Implement separate "x", "t" and "e". Return the "d" right in a + MYRIGHTS response or an ACL response containing ACL information + when any of the "t", "e" (and "x") is granted. + + In a similar manner this document splits the "c" right into several + new different rights: "k" and possibly "x" (see Section 2.1.1 for + more details). The "c" right remains for backwards-compatibility but + it is a virtual right. Again, RFC 2086 server implementors can + choose to tie rights or to implement separate rights, as described + above. + + Also check Sections 5.1.1 and 5.1.2, as well as Appendix A, to see + other changes required. Server implementors should check which + rights are required to invoke different IMAP4 commands as described + in Section 4. + +Appendix C. Known Deficiencies + + This specification has some known deficiencies including: + + 1. This is inadequate to provide complete read-write access to + mailboxes protected by Unix-style rights bits because there is no + equivalent to "chown" and "chgrp" commands nor is there a good + way to discover such limitations are present. + 2. Because this extension leaves the specific semantics of how + rights are combined by the server as implementation defined, the + ability to build a user-friendly interface is limited. + 3. Users, groups, and special identifiers (e.g., anyone) exist in + the same namespace. + + The work-in-progress "ACL2" extension is intended to redesign this + extension to address these deficiencies without the constraint of + backward-compatibility and may eventually supercede this facility. + + + + + +Melnikov Standards Track [Page 24] + +RFC 4314 IMAP ACL December 2005 + + + However, RFC 2086 is deployed in multiple implementations so this + intermediate step, which fixes the straightforward deficiencies in a + backward-compatible fashion, is considered worthwhile. + +Appendix D. Acknowledgements + + This document is a revision of RFC 2086 written by John G. Myers. + + Editor appreciates comments received from Mark Crispin, Chris Newman, + Cyrus Daboo, John G. Myers, Dave Cridland, Ken Murchison, Steve Hole, + Vladimir Butenko, Larry Greenfield, Robert Siemborski, Harrie + Hazewinkel, Philip Guenther, Brian Candler, Curtis King, Lyndon + Nerenberg, Lisa Dusseault, Arnt Gulbrandsen, and other participants + of the IMAPEXT working group. + +Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [Stringprep] Hoffman, P. and M. Blanchet, "Preparation of + Internationalized Strings ("stringprep")", RFC 3454, + December 2002. + + [SASLprep] Zeilenga, K., "SASLprep: Stringprep Profile for User + Names and Passwords", RFC 4013, February 2005. + +Informative References + + [RFC2086] Myers, J., "IMAP4 ACL extension", RFC 2086, + January 1997. + + [PR29] "Public Review Issue #29: Normalization Issue", + February 2004, + . + + + + + + + +Melnikov Standards Track [Page 25] + +RFC 4314 IMAP ACL December 2005 + + +Author's Address + + Alexey Melnikov + Isode Ltd. + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + GB + + EMail: alexey.melnikov@isode.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 26] + +RFC 4314 IMAP ACL December 2005 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2005). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at ietf- + ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + +Melnikov Standards Track [Page 27] + diff --git a/docs/rfcs/rfc4315.txt b/docs/rfcs/rfc4315.txt new file mode 100644 index 0000000..c026f42 --- /dev/null +++ b/docs/rfcs/rfc4315.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 4315 December 2005 +Obsoletes: 2359 +Category: Standards Track + + + Internet Message Access Protocol (IMAP) - UIDPLUS extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2005). + +Abstract + + The UIDPLUS extension of the Internet Message Access Protocol (IMAP) + provides a set of features intended to reduce the amount of time and + resources used by some client operations. The features in UIDPLUS + are primarily intended for disconnected-use clients. + +1. Introduction and Overview + + The UIDPLUS extension is present in any IMAP server implementation + that returns "UIDPLUS" as one of the supported capabilities to the + CAPABILITY command. + + The UIDPLUS extension defines an additional command. In addition, + this document recommends new status response codes in IMAP that + SHOULD be returned by all server implementations, regardless of + whether or not the UIDPLUS extension is implemented. + + The added facilities of the features in UIDPLUS are optimizations; + clients can provide equivalent functionality, albeit less + efficiently, by using facilities in the base protocol. + +1.1. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + + + + +Crispin Standards Track [Page 1] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to + be interpreted as described in [KEYWORDS]. + + A "UID set" is similar to the [IMAP] sequence set; however, the "*" + value for a sequence number is not permitted. + +2. Additional Commands + + The following command definition is an extension to [IMAP] section + 6.4. + +2.1. UID EXPUNGE Command + + Arguments: sequence set + + Data: untagged responses: EXPUNGE + + Result: OK - expunge completed + NO - expunge failure (e.g., permission denied) + BAD - command unknown or arguments invalid + + The UID EXPUNGE command permanently removes all messages that both + have the \Deleted flag set and have a UID that is included in the + specified sequence set from the currently selected mailbox. If a + message either does not have the \Deleted flag set or has a UID + that is not included in the specified sequence set, it is not + affected. + + This command is particularly useful for disconnected use clients. + By using UID EXPUNGE instead of EXPUNGE when resynchronizing with + the server, the client can ensure that it does not inadvertantly + remove any messages that have been marked as \Deleted by other + clients between the time that the client was last connected and + the time the client resynchronizes. + + If the server does not support the UIDPLUS capability, the client + should fall back to using the STORE command to temporarily remove + the \Deleted flag from messages it does not want to remove, then + issuing the EXPUNGE command. Finally, the client should use the + STORE command to restore the \Deleted flag on the messages in + which it was temporarily removed. + + Alternatively, the client may fall back to using just the EXPUNGE + command, risking the unintended removal of some messages. + + + + + + +Crispin Standards Track [Page 2] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + Example: C: A003 UID EXPUNGE 3000:3002 + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: A003 OK UID EXPUNGE completed + +3. Additional Response Codes + + The following response codes are extensions to the response codes + defined in [IMAP] section 7.1. With limited exceptions, discussed + below, server implementations that advertise the UIDPLUS extension + SHOULD return these response codes. + + In the case of a mailbox that has permissions set so that the client + can COPY or APPEND to the mailbox, but not SELECT or EXAMINE it, the + server SHOULD NOT send an APPENDUID or COPYUID response code as it + would disclose information about the mailbox. + + In the case of a mailbox that has UIDNOTSTICKY status (as defined + below), the server MAY omit the APPENDUID or COPYUID response code as + it is not meaningful. + + If the server does not return the APPENDUID or COPYUID response + codes, the client can discover this information by selecting the + destination mailbox. The location of messages placed in the + destination mailbox by COPY or APPEND can be determined by using + FETCH and/or SEARCH commands (e.g., for Message-ID or some unique + marker placed in the message in an APPEND). + + APPENDUID + + Followed by the UIDVALIDITY of the destination mailbox and the UID + assigned to the appended message in the destination mailbox, + indicates that the message has been appended to the destination + mailbox with that UID. + + If the server also supports the [MULTIAPPEND] extension, and if + multiple messages were appended in the APPEND command, then the + second value is a UID set containing the UIDs assigned to the + appended messages, in the order they were transmitted in the + APPEND command. This UID set may not contain extraneous UIDs or + the symbol "*". + + Note: the UID set form of the APPENDUID response code MUST NOT + be used if only a single message was appended. In particular, + a server MUST NOT send a range such as 123:123. This is + because a client that does not support [MULTIAPPEND] expects + only a single UID and not a UID set. + + + +Crispin Standards Track [Page 3] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + UIDs are assigned in strictly ascending order in the mailbox + (refer to [IMAP], section 2.3.1.1) and UID ranges are as in + [IMAP]; in particular, note that a range of 12:10 is exactly + equivalent to 10:12 and refers to the sequence 10,11,12. + + This response code is returned in a tagged OK response to the + APPEND command. + + COPYUID + + Followed by the UIDVALIDITY of the destination mailbox, a UID set + containing the UIDs of the message(s) in the source mailbox that + were copied to the destination mailbox and containing the UIDs + assigned to the copied message(s) in the destination mailbox, + indicates that the message(s) have been copied to the destination + mailbox with the stated UID(s). + + The source UID set is in the order the message(s) were copied; the + destination UID set corresponds to the source UID set and is in + the same order. Neither of the UID sets may contain extraneous + UIDs or the symbol "*". + + UIDs are assigned in strictly ascending order in the mailbox + (refer to [IMAP], section 2.3.1.1) and UID ranges are as in + [IMAP]; in particular, note that a range of 12:10 is exactly + equivalent to 10:12 and refers to the sequence 10,11,12. + + This response code is returned in a tagged OK response to the COPY + command. + + UIDNOTSTICKY + + The selected mailbox is supported by a mail store that does not + support persistent UIDs; that is, UIDVALIDITY will be different + each time the mailbox is selected. Consequently, APPEND or COPY + to this mailbox will not return an APPENDUID or COPYUID response + code. + + This response code is returned in an untagged NO response to the + SELECT command. + + Note: servers SHOULD NOT have any UIDNOTSTICKY mail stores. + This facility exists to support legacy mail stores in which it + is technically infeasible to support persistent UIDs. This + should be avoided when designing new mail stores. + + + + + + +Crispin Standards Track [Page 4] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + Example: C: A003 APPEND saved-messages (\Seen) {297} + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@example.com + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 38505 3955] APPEND completed + C: A004 COPY 2:4 meeting + S: A004 OK [COPYUID 38505 304,319:320 3956:3958] Done + C: A005 UID COPY 305:310 meeting + S: A005 OK No matching messages, so nothing copied + C: A006 COPY 2 funny + S: A006 OK Done + C: A007 SELECT funny + S: * 1 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 1] Message 1 is first unseen + S: * OK [UIDVALIDITY 3857529045] Validity session-only + S: * OK [UIDNEXT 2] Predicted next UID + S: * NO [UIDNOTSTICKY] Non-persistent UIDs + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen)] Limited + S: A007 OK [READ-WRITE] SELECT completed + + In this example, A003 and A004 demonstrate successful appending and + copying to a mailbox that returns the UIDs assigned to the messages. + A005 is an example in which no messages were copied; this is because + in A003, we see that message 2 had UID 304, and message 3 had UID + 319; therefore, UIDs 305 through 310 do not exist (refer to section + 2.3.1.1 of [IMAP] for further explanation). A006 is an example of a + message being copied that did not return a COPYUID; and, as expected, + A007 shows that the mail store containing that mailbox does not + support persistent UIDs. + +4. Formal Syntax + + Formal syntax is defined using ABNF [ABNF], which extends the ABNF + rules defined in [IMAP]. The IMAP4 ABNF should be imported before + attempting to validate these rules. + + append-uid = uniqueid + + capability =/ "UIDPLUS" + + + +Crispin Standards Track [Page 5] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + command-select =/ uid-expunge + + resp-code-apnd = "APPENDUID" SP nz-number SP append-uid + + resp-code-copy = "COPYUID" SP nz-number SP uid-set SP uid-set + + resp-text-code =/ resp-code-apnd / resp-code-copy / "UIDNOTSTICKY" + ; incorporated before the expansion rule of + ; atom [SP 1*] + ; that appears in [IMAP] + + uid-expunge = "UID" SP "EXPUNGE" SP sequence-set + + uid-set = (uniqueid / uid-range) *("," uid-set) + + uid-range = (uniqueid ":" uniqueid) + ; two uniqueid values and all values + ; between these two regards of order. + ; Example: 2:4 and 4:2 are equivalent. + + Servers that support [MULTIAPPEND] will have the following extension + to the above rules: + + append-uid =/ uid-set + ; only permitted if client uses [MULTIAPPEND] + ; to append multiple messages. + +5. Security Considerations + + The COPYUID and APPENDUID response codes return information about the + mailbox, which may be considered sensitive if the mailbox has + permissions set that permit the client to COPY or APPEND to the + mailbox, but not SELECT or EXAMINE it. + + Consequently, these response codes SHOULD NOT be issued if the client + does not have access to SELECT or EXAMINE the mailbox. + +6. IANA Considerations + + This document constitutes registration of the UIDPLUS capability in + the imap4-capabilities registry, replacing [RFC2359]. + +7. Normative References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + + + + +Crispin Standards Track [Page 6] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + [IMAP] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - + VERSION 4rev1", RFC 3501, March 2003. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + +8. Informative References + + [RFC2359] Myers, J., "IMAP4 UIDPLUS extension", RFC 2359, June + 1998. + +9. Changes from RFC 2359 + + This document obsoletes [RFC2359]. However, it is based upon that + document, and takes substantial text from it (albeit with numerous + clarifications in wording). + + [RFC2359] implied that a server must always return COPYUID/APPENDUID + data; thus suggesting that in such cases the server should return + arbitrary data if the destination mailbox did not support persistent + UIDs. This document adds the UIDNOTSTICKY response code to indicate + that a mailbox does not support persistent UIDs, and stipulates that + a UIDPLUS server does not return COPYUID/APPENDUID data when the COPY + (or APPEND) destination mailbox has UIDNOTSTICKY status. + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Avenue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + +Crispin Standards Track [Page 7] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2005). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at ietf- + ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + +Crispin Standards Track [Page 8] + diff --git a/docs/rfcs/rfc4466.txt b/docs/rfcs/rfc4466.txt new file mode 100644 index 0000000..dfde168 --- /dev/null +++ b/docs/rfcs/rfc4466.txt @@ -0,0 +1,955 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 4466 Isode Ltd. +Updates: 2088, 2342, 3501, 3502, 3516 C. Daboo +Category: Standards Track April 2006 + + + Collected Extensions to IMAP4 ABNF + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + Over the years, many documents from IMAPEXT and LEMONADE working + groups, as well as many individual documents, have added syntactic + extensions to many base IMAP commands described in RFC 3501. For + ease of reference, this document collects most of such ABNF changes + in one place. + + This document also suggests a set of standard patterns for adding + options and extensions to several existing IMAP commands defined in + RFC 3501. The patterns provide for compatibility between existing + and future extensions. + + This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. + It also includes part of the errata to RFC 3501. This document + doesn't specify any semantic changes to the listed RFCs. + + + + + + + + + + + + + + + +Melnikov & Daboo Standards Track [Page 1] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +Table of Contents + + 1. Introduction ....................................................2 + 1.1. Purpose of This Document ...................................2 + 1.2. Conventions Used in This Document ..........................3 + 2. IMAP ABNF Extensions ............................................3 + 2.1. Optional Parameters with the SELECT/EXAMINE Commands .......3 + 2.2. Extended CREATE Command ....................................4 + 2.3. Extended RENAME Command ....................................5 + 2.4. Extensions to FETCH and UID FETCH Commands .................6 + 2.5. Extensions to STORE and UID STORE Commands .................6 + 2.6. Extensions to SEARCH Command ...............................7 + 2.6.1. Extended SEARCH Command .............................7 + 2.6.2. ESEARCH untagged response ...........................8 + 2.7. Extensions to APPEND Command ...............................8 + 3. Formal Syntax ...................................................9 + 4. Security Considerations ........................................14 + 5. Normative References ...........................................15 + 6. Acknowledgements ...............................................15 + +1. Introduction + +1.1. Purpose of This Document + + This document serves several purposes: + + 1. rationalize and generalize ABNF for some existing IMAP + extensions; + 2. collect the ABNF in one place in order to minimize cross + references between documents; + 3. define building blocks for future extensions so that they can + be used together in a compatible way. + + It is expected that a future revision of this document will be + incorporated into a revision of RFC 3501. + + This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. + It also includes part of the errata to RFC 3501. This document + doesn't specify any semantic changes to the listed RFCs. + + The ABNF in section 6 of RFC 2342 got rewritten to conform to the + ABNF syntax as defined in RFC 4234 and to reference new non-terminals + from RFC 3501. It was also restructured to allow for better + readability. There were no changes "on the wire". + + Section 2 extends ABNF for SELECT, EXAMINE, CREATE, RENAME, FETCH/UID + FETCH, STORE/UID STORE, SEARCH, and APPEND commands in a consistent + manner. Extensions to all the commands but APPEND have the same + + + +Melnikov & Daboo Standards Track [Page 2] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + structure. Extensibility for the APPEND command was done slightly + differently in order to preserve backward compatibility with existing + extensions. + + Section 2 also defines a new ESEARCH response, whose purpose is to + define a better version of the SEARCH response defined in RFC 3501. + + Section 3 defines the collected ABNF that replaces pieces of ABNF in + the aforementioned RFCs. The collected ABNF got generalized to allow + for easier future extensibility. + +1.2. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in "Key words for + use in RFCs to Indicate Requirement Levels" [KEYWORDS]. + +2. IMAP ABNF Extensions + + This section is not normative. It provides some background on the + intended use of different extensions and it gives some guidance about + how future extensions should extend the described commands. + +2.1. Optional Parameters with the SELECT/EXAMINE Commands + + This document adds the ability to include one or more parameters with + the IMAP SELECT (section 6.3.1 of [IMAP4]) or EXAMINE (section 6.3.2 + of [IMAP4]) commands, to turn on or off certain standard behaviors, + or to add new optional behaviors required for a particular extension. + + There are two possible modes of operation: + + o A global state change where a single use of the optional parameter + will affect the session state from that time on, irrespective of + subsequent SELECT/EXAMINE commands. + + o A per-mailbox state change that will affect the session only for + the duration of the new selected state. A subsequent + SELECT/EXAMINE without the optional parameter will cancel its + effect for the newly selected mailbox. + + Optional parameters to the SELECT or EXAMINE commands are added as a + parenthesized list of attribute/value pairs, and appear after the + mailbox name in the standard SELECT or EXAMINE command. The order of + individual parameters is arbitrary. A parameter value is optional + + + +Melnikov & Daboo Standards Track [Page 3] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + and may consist of atoms, strings, or lists in a specific order. If + the parameter value is present, it always appears in parentheses (*). + Any parameter not defined by extensions that the server supports must + be rejected with a BAD response. + + Example: + + C: a SELECT INBOX (ANNOTATE) + S: ... + S: a OK SELECT complete + + In the above example, a single parameter is used with the SELECT + command. + + Example: + + C: a EXAMINE INBOX (ANNOTATE RESPONSES ("UID Responses") + CONDSTORE) + S: ... + S: a OK EXAMINE complete + + In the above example, three parameters are used with the EXAMINE + command. The second parameter consists of two items: an atom + "RESPONSES" followed by a quoted string. + + Example: + + C: a SELECT INBOX (BLURDYBLOOP) + S: a BAD Unknown parameter in SELECT command + + In the above example, a parameter not supported by the server is + used. This results in the BAD response from the server. + + (*) - if a parameter has a mandatory value, which can always be + represented as a number or a sequence-set, the parameter value does + not need the enclosing (). See ABNF for more details. + +2.2. Extended CREATE Command + + Arguments: mailbox name + OPTIONAL list of CREATE parameters + + Responses: no specific responses for this command + + Result: OK - create completed + NO - create failure: cannot create mailbox with + that name + BAD - argument(s) invalid + + + +Melnikov & Daboo Standards Track [Page 4] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + This document adds the ability to include one or more parameters with + the IMAP CREATE command (see section 6.3.3 of [IMAP4]), to turn on or + off certain standard behaviors, or to add new optional behaviors + required for a particular extension. No CREATE parameters are + defined in this document. + + Optional parameters to the CREATE command are added as a + parenthesized list of attribute/value pairs after the mailbox name. + The order of individual parameters is arbitrary. A parameter value + is optional and may consist of atoms, strings, or lists in a specific + order. If the parameter value is present, it always appears in + parentheses (*). Any parameter not defined by extensions that the + server supports must be rejected with a BAD response. + + (*) - if a parameter has a mandatory value, which can always be + represented as a number or a sequence-set, the parameter value does + not need the enclosing (). See ABNF for more details. + +2.3. Extended RENAME Command + + Arguments: existing mailbox name + new mailbox name + OPTIONAL list of RENAME parameters + + Responses: no specific responses for this command + + Result: OK - rename completed + NO - rename failure: cannot rename mailbox with + that name, cannot rename to mailbox with + that name, etc. + BAD - argument(s) invalid + + This document adds the ability to include one or more parameters with + the IMAP RENAME command (see section 6.3.5 of [IMAP4]), to turn on or + off certain standard behaviors, or to add new optional behaviors + required for a particular extension. No RENAME parameters are + defined in this document. + + Optional parameters to the RENAME command are added as a + parenthesized list of attribute/value pairs after the new mailbox + name. The order of individual parameters is arbitrary. A parameter + value is optional and may consist of atoms, strings, or lists in a + specific order. If the parameter value is present, it always appears + in parentheses (*). Any parameter not defined by extensions that the + server supports must be rejected with a BAD response. + + + + + + +Melnikov & Daboo Standards Track [Page 5] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + (*) - if a parameter has a mandatory value, which can always be + represented as a number or a sequence-set, the parameter value does + not need the enclosing (). See ABNF for more details. + +2.4. Extensions to FETCH and UID FETCH Commands + + Arguments: sequence set + message data item names or macro + OPTIONAL fetch modifiers + + Responses: untagged responses: FETCH + + Result: OK - fetch completed + NO - fetch error: cannot fetch that data + BAD - command unknown or arguments invalid + + This document extends the syntax of the FETCH and UID FETCH commands + (see section 6.4.5 of [IMAP4]) to include optional FETCH modifiers. + No fetch modifiers are defined in this document. + + The order of individual modifiers is arbitrary. Each modifier is an + attribute/value pair. A modifier value is optional and may consist + of atoms and/or strings and/or lists in a specific order. If the + modifier value is present, it always appears in parentheses (*). Any + modifiers not defined by extensions that the server supports must be + rejected with a BAD response. + + (*) - if a modifier has a mandatory value, which can always be + represented as a number or a sequence-set, the modifier value does + not need the enclosing (). See ABNF for more details. + +2.5. Extensions to STORE and UID STORE Commands + + Arguments: message set + OPTIONAL store modifiers + message data item name + value for message data item + + Responses: untagged responses: FETCH + + Result: OK - store completed + NO - store error: cannot store that data + BAD - command unknown or arguments invalid + + This document extends the syntax of the STORE and UID STORE commands + (see section 6.4.6 of [IMAP4]) to include optional STORE modifiers. + No store modifiers are defined in this document. + + + + +Melnikov & Daboo Standards Track [Page 6] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + The order of individual modifiers is arbitrary. Each modifier is an + attribute/value pair. A modifier value is optional and may consist + of atoms and/or strings and/or lists in a specific order. If the + modifier value is present, it always appears in parentheses (*). Any + modifiers not defined by extensions that the server supports must be + rejected with a BAD response. + + (*) - if a modifier has a mandatory value, which can always be + represented as a number or a sequence-set, the modifier value does + not need the enclosing (). See ABNF for more details. + +2.6. Extensions to SEARCH Command + +2.6.1. Extended SEARCH Command + + Arguments: OPTIONAL result specifier + OPTIONAL [CHARSET] specification + searching criteria (one or more) + + Responses: REQUIRED untagged response: SEARCH (*) + + Result: OK - search completed + NO - search error: cannot search that [CHARSET] or + criteria + BAD - command unknown or arguments invalid + + This section updates definition of the SEARCH command described in + section 6.4.4 of [IMAP4]. + + The SEARCH command is extended to allow for result options. This + document does not define any result options. + + The order of individual options is arbitrary. Individual options may + contain parameters enclosed in parentheses (**). If an option has + parameters, they consist of atoms and/or strings and/or lists in a + specific order. Any options not defined by extensions that the + server supports must be rejected with a BAD response. + + (*) - An extension to the SEARCH command may require another untagged + response, or no untagged response to be returned. Section 2.6.2 + defines a new ESEARCH untagged response that replaces the SEARCH + untagged response. Note that for a given extended SEARCH command the + SEARCH and ESEARCH responses SHOULD be mutually exclusive, i.e., only + one of them should be returned. + + (**) - if an option has a mandatory parameter, which can always be + represented as a number or a sequence-set, the option parameter does + not need the enclosing (). See ABNF for more details. + + + +Melnikov & Daboo Standards Track [Page 7] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +2.6.2. ESEARCH untagged response + + Contents: one or more search-return-data pairs + + The ESEARCH response SHOULD be sent as a result of an extended SEARCH + or UID SEARCH command specified in section 2.6.1. + + The ESEARCH response starts with an optional search correlator. If + it is missing, then the response was not caused by a particular IMAP + command, whereas if it is present, it contains the tag of the command + that caused the response to be returned. + + The search correlator is followed by an optional UID indicator. If + this indicator is present, all data in the ESEARCH response refers to + UIDs, otherwise all returned data refers to message numbers. + + The rest of the ESEARCH response contains one or more search data + pairs. Each pair starts with unique return item name, followed by a + space and the corresponding data. Search data pairs may be returned + in any order. Unless specified otherwise by an extension, any return + item name SHOULD appear only once in an ESEARCH response. + + Example: S: * ESEARCH UID COUNT 5 ALL 4:19,21,28 + + Example: S: * ESEARCH (TAG "a567") UID COUNT 5 ALL 4:19,21,28 + + Example: S: * ESEARCH COUNT 5 ALL 1:17,21 + +2.7. Extensions to APPEND Command + + The IMAP BINARY extension [BINARY] extends the APPEND command to + allow a client to append data containing NULs by using the + syntax. The ABNF was rewritten to allow for easier extensibility by + IMAP extensions. This document hasn't specified any semantical + changes to the [BINARY] extension. + + In addition, the non-terminal "literal8" defined in [BINARY] got + extended to allow for non-synchronizing literals if both [BINARY] and + [LITERAL+] extensions are supported by the server. + + The IMAP MULTIAPPEND extension [MULTIAPPEND] extends the APPEND + command to allow a client to append multiple messages atomically. + This document defines a common syntax for the APPEND command that + takes into consideration syntactic extensions defined by both + [BINARY] and [MULTIAPPEND] extensions. + + + + + + +Melnikov & Daboo Standards Track [Page 8] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +3. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of uppercase or lowercase characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + append = "APPEND" SP mailbox 1*append-message + ;; only a single append-message may appear + ;; if MULTIAPPEND [MULTIAPPEND] capability + ;; is not present + + append-message = append-opts SP append-data + + append-ext = append-ext-name SP append-ext-value + ;; This non-terminal define extensions to + ;; to message metadata. + + append-ext-name = tagged-ext-label + + append-ext-value= tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + append-data = literal / literal8 / append-data-ext + + append-data-ext = tagged-ext + ;; This non-terminal shows recommended syntax + ;; for future extensions, + ;; i.e., a mandatory label followed + ;; by parameters. + + append-opts = [SP flag-list] [SP date-time] *(SP append-ext) + ;; message metadata + + charset = atom / quoted + ;; Exact syntax is defined in [CHARSET]. + + create = "CREATE" SP mailbox + [create-params] + ;; Use of INBOX gives a NO error. + + + +Melnikov & Daboo Standards Track [Page 9] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + create-params = SP "(" create-param *( SP create-param) ")" + + create-param-name = tagged-ext-label + + create-param = create-param-name [SP create-param-value] + + create-param-value= tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + esearch-response = "ESEARCH" [search-correlator] [SP "UID"] + *(SP search-return-data) + ;; Note that SEARCH and ESEARCH responses + ;; SHOULD be mutually exclusive, + ;; i.e., only one of the response types + ;; should be + ;; returned as a result of a command. + + + examine = "EXAMINE" SP mailbox [select-params] + ;; modifies the original IMAP EXAMINE command + ;; to accept optional parameters + + fetch = "FETCH" SP sequence-set SP ("ALL" / "FULL" / + "FAST" / fetch-att / + "(" fetch-att *(SP fetch-att) ")") + [fetch-modifiers] + ;; modifies the original IMAP4 FETCH command to + ;; accept optional modifiers + + fetch-modifiers = SP "(" fetch-modifier *(SP fetch-modifier) ")" + + fetch-modifier = fetch-modifier-name [ SP fetch-modif-params ] + + fetch-modif-params = tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + fetch-modifier-name = tagged-ext-label + + literal8 = "~{" number ["+"] "}" CRLF *OCTET + ;; A string that might contain NULs. + ;; represents the number of OCTETs + ;; in the response string. + ;; The "+" is only allowed when both LITERAL+ and + ;; BINARY extensions are supported by the server. + + + + +Melnikov & Daboo Standards Track [Page 10] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + mailbox-data =/ Namespace-Response / + esearch-response + + Namespace = nil / "(" 1*Namespace-Descr ")" + + Namespace-Command = "NAMESPACE" + + Namespace-Descr = "(" string SP + (DQUOTE QUOTED-CHAR DQUOTE / nil) + *(Namespace-Response-Extension) ")" + + Namespace-Response-Extension = SP string SP + "(" string *(SP string) ")" + + Namespace-Response = "NAMESPACE" SP Namespace + SP Namespace SP Namespace + ;; This response is currently only allowed + ;; if the IMAP server supports [NAMESPACE]. + ;; The first Namespace is the Personal Namespace(s) + ;; The second Namespace is the Other Users' Namespace(s) + ;; The third Namespace is the Shared Namespace(s) + + rename = "RENAME" SP mailbox SP mailbox + [rename-params] + ;; Use of INBOX as a destination gives + ;; a NO error, unless rename-params + ;; is not empty. + + rename-params = SP "(" rename-param *( SP rename-param) ")" + + rename-param = rename-param-name [SP rename-param-value] + + rename-param-name = tagged-ext-label + + rename-param-value= tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + response-data = "*" SP response-payload CRLF + + response-payload= resp-cond-state / resp-cond-bye / + mailbox-data / message-data / capability-data + + search = "SEARCH" [search-return-opts] + SP search-program + + search-correlator = SP "(" "TAG" SP tag-string ")" + + + +Melnikov & Daboo Standards Track [Page 11] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + search-program = ["CHARSET" SP charset SP] + search-key *(SP search-key) + ;; CHARSET argument to SEARCH MUST be + ;; registered with IANA. + + search-return-data = search-modifier-name SP search-return-value + ;; Note that not every SEARCH return option + ;; is required to have the corresponding + ;; ESEARCH return data. + + search-return-opts = SP "RETURN" SP "(" [search-return-opt + *(SP search-return-opt)] ")" + + search-return-opt = search-modifier-name [SP search-mod-params] + + search-return-value = tagged-ext-val + ;; Data for the returned search option. + ;; A single "nz-number"/"number" value + ;; can be returned as an atom (i.e., without + ;; quoting). A sequence-set can be returned + ;; as an atom as well. + + search-modifier-name = tagged-ext-label + + search-mod-params = tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + select = "SELECT" SP mailbox [select-params] + ;; modifies the original IMAP SELECT command to + ;; accept optional parameters + + select-params = SP "(" select-param *(SP select-param) ")" + + select-param = select-param-name [SP select-param-value] + ;; a parameter to SELECT may contain one or + ;; more atoms and/or strings and/or lists. + + select-param-name= tagged-ext-label + + select-param-value= tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + status-att-list = status-att-val *(SP status-att-val) + ;; Redefines status-att-list from RFC 3501. + + + +Melnikov & Daboo Standards Track [Page 12] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + ;; status-att-val is defined in RFC 3501 errata + + status-att-val = ("MESSAGES" SP number) / + ("RECENT" SP number) / + ("UIDNEXT" SP nz-number) / + ("UIDVALIDITY" SP nz-number) / + ("UNSEEN" SP number) + ;; Extensions to the STATUS responses + ;; should extend this production. + ;; Extensions should use the generic + ;; syntax defined by tagged-ext. + + store = "STORE" SP sequence-set [store-modifiers] + SP store-att-flags + ;; extend [IMAP4] STORE command syntax + ;; to allow for optional store-modifiers + + store-modifiers = SP "(" store-modifier *(SP store-modifier) + ")" + + store-modifier = store-modifier-name [SP store-modif-params] + + store-modif-params = tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + store-modifier-name = tagged-ext-label + + tag-string = string + ;; tag of the command that caused + ;; the ESEARCH response, sent as + ;; a string. + + tagged-ext = tagged-ext-label SP tagged-ext-val + ;; recommended overarching syntax for + ;; extensions + + tagged-ext-label = tagged-label-fchar *tagged-label-char + ;; Is a valid RFC 3501 "atom". + + tagged-label-fchar = ALPHA / "-" / "_" / "." + + tagged-label-char = tagged-label-fchar / DIGIT / ":" + + + + + + + + +Melnikov & Daboo Standards Track [Page 13] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + tagged-ext-comp = astring / + tagged-ext-comp *(SP tagged-ext-comp) / + "(" tagged-ext-comp ")" + ;; Extensions that follow this general + ;; syntax should use nstring instead of + ;; astring when appropriate in the context + ;; of the extension. + ;; Note that a message set or a "number" + ;; can always be represented as an "atom". + ;; An URL should be represented as + ;; a "quoted" string. + + tagged-ext-simple = sequence-set / number + + tagged-ext-val = tagged-ext-simple / + "(" [tagged-ext-comp] ")" + +4. Security Considerations + + This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. + The updated documents must be consulted for security considerations + for the extensions that they define. + + As a protocol gets more complex, parser bugs become more common + including buffer overflow, denial of service, and other common + security coding errors. To the extent that this document makes the + parser more complex, it makes this situation worse. To the extent + that this document makes the parser more consistent and thus simpler, + the situation is improved. The impact will depend on how many + deployed IMAP extensions are consistent with this document. + Implementers are encouraged to take care of these issues when + extending existing implementations. Future IMAP extensions should + strive for consistency and simplicity to the greatest extent + possible. + + Extensions to IMAP commands that are permitted in NOT AUTHENTICATED + state are more sensitive to these security issues due to the larger + possible attacker community prior to authentication, and the fact + that some IMAP servers run with elevated privileges in that state. + This document does not extend any commands permitted in NOT + AUTHENTICATED state. Future IMAP extensions to commands permitted in + NOT AUTHENTICATED state should favor simplicity over consistency or + extensibility. + + + + + + + + +Melnikov & Daboo Standards Track [Page 14] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +5. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - + VERSION 4rev1", RFC 3501, March 2003. + + [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 4234, October 2005. + + [CHARSET] Freed, N. and J. Postel, "IANA Charset Registration + Procedures", BCP 19, RFC 2978, October 2000. + + [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + + [NAMESPACE] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, + May 1998. + + [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC + 2088, January 1997. + + [BINARY] Nerenberg, L., "IMAP4 Binary Content Extension", RFC + 3516, April 2003. + +6. Acknowledgements + + This documents is based on ideas proposed by Pete Resnick, Mark + Crispin, Ken Murchison, Philip Guenther, Randall Gellens, and Lyndon + Nerenberg. + + However, all errors and omissions must be attributed to the authors + of the document. + + Thanks to Philip Guenther, Dave Cridland, Mark Crispin, Chris Newman, + Elwyn Davies, and Barry Leiba for comments and corrections. + + literal8 syntax was taken from RFC 3516. + + + + + + + + + + + + +Melnikov & Daboo Standards Track [Page 15] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex, TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + Cyrus Daboo + + EMail: cyrus@daboo.name + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Daboo Standards Track [Page 16] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Melnikov & Daboo Standards Track [Page 17] + diff --git a/docs/rfcs/rfc4467.txt b/docs/rfcs/rfc4467.txt new file mode 100644 index 0000000..83b6516 --- /dev/null +++ b/docs/rfcs/rfc4467.txt @@ -0,0 +1,1011 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 4467 University of Washington +Updates: 3501 May 2006 +Category: Standards Track + + + Internet Message Access Protocol (IMAP) - URLAUTH Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + This document describes the URLAUTH extension to the Internet Message + Access Protocol (IMAP) (RFC 3501) and the IMAP URL Scheme (IMAPURL) + (RFC 2192). This extension provides a means by which an IMAP client + can use URLs carrying authorization to access limited message data on + the IMAP server. + + An IMAP server that supports this extension indicates this with a + capability name of "URLAUTH". + +1. Introduction + + In [IMAPURL], a URL of the form imap://fred@example.com/INBOX/;uid=20 + requires authorization as userid "fred". However, [IMAPURL] implies + that it only supports authentication and confuses the concepts of + authentication and authorization. + + The URLAUTH extension defines an authorization mechanism for IMAP + URLs to replace [IMAPURL]'s authentication-only mechanism. URLAUTH + conveys authorization in the URL string itself and reuses a portion + of the syntax of the [IMAPURL] authentication mechanism to convey the + authorization identity (which also defines the default namespace in + [IMAP]). + + The URLAUTH extension provides a means by which an authorized user of + an IMAP server can create URLAUTH-authorized IMAP URLs. A URLAUTH- + authorized URL conveys authorization (not authentication) to the data + + + +Crispin Standards Track [Page 1] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + addressed by that URL. This URL can be used in another IMAP session + to access specific content on the IMAP server, without otherwise + providing authorization to any other data (such as other data in the + mailbox specified in the URL) owned by the authorizing user. + + Conceptually, a URLAUTH-authorized URL can be thought of as a "pawn + ticket" that carries no authentication information and can be + redeemed by whomever presents it. However, unlike a pawn ticket, + URLAUTH has optional mechanisms to restrict the usage of a URLAUTH- + authorized URL. Using these mechanisms, URLAUTH-authorized URLs can + be usable by: + + . anonymous (the "pawn ticket" model) + . authenticated users only + . a specific authenticated user only + . message submission acting on behalf of a specific user only + + There is also a mechanism for expiration. + + A URLAUTH-authorized URL can be used in the argument to the BURL + command in message composition, as described in [BURL], for such + purposes as allowing a client (with limited memory or other + resources) to submit a message forward or to resend from an IMAP + mailbox without requiring the client to fetch that message data. + + The URLAUTH is generated using an authorization mechanism name and an + authorization token, which is generated using a secret mailbox access + key. An IMAP client can request that the server generate and assign + a new mailbox access key (thus effectively revoking all current URLs + using URLAUTH with the old mailbox access key) but cannot set the + mailbox access key to a key of its own choosing. + +1.1. Conventions Used in this Document + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in [KEYWORDS]. + + The formal syntax uses the Augmented Backus-Naur Form (ABNF) notation + including the core rules defined in Appendix A of [ABNF]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. If a single "C:" or "S:" label applies to + multiple lines, then the line breaks between those lines are for + editorial clarity only and are not part of the actual protocol + exchange. + + + + + + +Crispin Standards Track [Page 2] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +2. Concepts + +2.1. URLAUTH + + The URLAUTH is a component, appended at the end of a URL, that + conveys authorization to access the data addressed by that URL. It + contains an authorized access identifier, an authorization mechanism + name, and an authorization token. The authorization token is + generated from the URL, the authorized access identifier, the + authorization mechanism name, and a mailbox access key. + +2.2. Mailbox Access Key + + The mailbox access key is a random string with at least 128 bits of + entropy. It is generated by software (not by the human user) and + MUST be unpredictable. + + Each user has a table of mailboxes and an associated mailbox access + key for each mailbox. Consequently, the mailbox access key is per- + user and per-mailbox. In other words, two users sharing the same + mailbox each have a different mailbox access key for that mailbox, + and each mailbox accessed by a single user also has a different + mailbox access key. + +2.3. Authorized Access Identifier + + The authorized access identifier restricts use of the URLAUTH + authorized URL to certain users authorized on the server, as + described in section 3. + +2.4. Authorization Mechanism + + The authorization mechanism is the algorithm by which the URLAUTH is + generated and subsequently verified, using the mailbox access key. + +2.4.1. INTERNAL Authorization Mechanism + + This specification defines the INTERNAL mechanism, which uses a token + generation algorithm of the server's choosing and does not involve + disclosure of the mailbox access key to the client. + + Note: The token generation algorithm chosen by the server + implementation should be modern and reasonably secure. At the + time of the writing of this document, an [HMAC] such as HMAC-SHA1 + is recommended. + + + + + + +Crispin Standards Track [Page 3] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + If it becomes necessary to change the token generation algorithm + of the INTERNAL mechanism (e.g., because an attack against the + current algorithm has been discovered), all currently existing + URLAUTH-authorized URLs are invalidated by the change in + algorithm. Since this would be an unpleasant surprise to + applications that depend upon the validity of a URLAUTH-authorized + URL, and there is no good way to do a bulk update of existing + deployed URLs, it is best to avoid this situation by using a + secure algorithm as opposed to one that is "good enough". + + Server implementations SHOULD consider the possibility of changing + the algorithm. In some cases, it may be desirable to implement + the change of algorithm in a way that newly-generated tokens use + the new algorithm, but that for a limited period of time tokens + using either the new or old algorithm can be validated. + Consequently, the server SHOULD incorporate some means of + identifying the token generation algorithm within the token. + + Although this specification is extensible for other mechanisms, none + are defined in this document. In addition to the mechanism name + itself, other mechanisms may have mechanism-specific data, which is + to be interpreted according to the definition of that mechanism. + +2.5. Authorization Token + + The authorization token is a deterministic string of at least 128 + bits that an entity with knowledge of the secret mailbox access key + and URL authorization mechanism can use to verify the URL. + +3. IMAP URL Extensions + + [IMAPURL] is extended by allowing the addition of + ";EXPIRE=" and ";URLAUTH=::" to IMAP + URLs that refer to a specific message or message parts. + + The URLAUTH is comprised of ";URLAUTH=::" and + MUST be at the end of the URL. + + URLAUTH does not apply to, and MUST NOT be used with, any IMAP URL + that refers to an entire IMAP server, a list of mailboxes, an entire + IMAP mailbox, or IMAP search results. + + When ";EXPIRE=" is used, this indicates the latest date and + time that the URL is valid. After that date and time, the URL has + expired, and server implementations MUST reject the URL. If + ";EXPIRE=" is not used, the URL has no expiration, but + still can be revoked as discussed below. + + + + +Crispin Standards Track [Page 4] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + The URLAUTH takes the form ";URLAUTH=::". It is + composed of three parts. The portion provides the + authorized access identifiers, which may constrain the operations and + users that are permitted to use this URL. The portion + provides the authorization mechanism used by the IMAP server to + generate the authorization token that follows. The portion + provides the authorization token. + + The "submit+" access identifier prefix, followed by a userid, + indicates that only a userid authorized as a message submission + entity on behalf of the specified userid is permitted to use this + URL. The IMAP server does not validate the specified userid but does + validate that the IMAP session has an authorization identity that is + authorized as a message submission entity. The authorized message + submission entity MUST validate the userid prior to contacting the + IMAP server. + + The "user+" access identifier prefix, followed by a userid, indicates + that use of this URL is limited to IMAP sessions that are logged in + as the specified userid (that is, have authorization identity as that + userid). + + Note: If a SASL mechanism that provides both authorization and + authentication identifiers is used to authenticate to the IMAP + server, the "user+" access identifier MUST match the authorization + identifier. + + The "authuser" access identifier indicates that use of this URL is + limited to IMAP sessions that are logged in as an authorized user + (that is, have authorization identity as an authorized user) of that + IMAP server. Use of this URL is prohibited to anonymous IMAP + sessions. + + The "anonymous" access identifier indicates that use of this URL is + not restricted by session authorization identity; that is, any IMAP + session in authenticated or selected state (as defined in [IMAP]), + including anonymous sessions, may issue a URLFETCH using this URL. + + The authorization token is represented as an ASCII-encoded + hexadecimal string, which is used to authorize the URL. The length + and the calculation of the authorization token depends upon the + mechanism used; but, in all cases, the authorization token is at + least 128 bits (and therefore at least 32 hexadecimal digits). + + + + + + + + +Crispin Standards Track [Page 5] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +4. Discussion of URLAUTH Authorization Issues + + In [IMAPURL], the userid before the "@" in the URL has two purposes: + + 1) It provides context for user-specific mailbox paths such as + "INBOX". + + 2) It specifies that resolution of the URL requires logging in as + that user and limits use of that URL to only that user. + + An obvious limitation of using the same field for both purposes is + that the URL can only be resolved by the mailbox owner. + + URLAUTH overrides the second purpose of the userid in the IMAP URL + and by default permits the URL to be resolved by any user permitted + by the access identifier. + + The "user+" access identifier limits resolution of that URL + to a particular userid, whereas the "submit+" access + identifier is more general and simply requires that the session be + authorized by a user that has been granted a "submit" role within the + authentication system. Use of either of these access identifiers + makes it impossible for an attacker, spying on the session, to use + the same URL, either directly or by submission to a message + submission entity. + + The "authuser" and "anonymous" access identifiers do not have this + level of protection and should be used with caution. These access + identifiers are primarily useful for public export of data from an + IMAP server, without requiring that it be copied to a web or + anonymous FTP server. Refer to the Security Considerations for more + details. + +5. Generation of URLAUTH-Authorized URLs + + A URLAUTH-authorized URL is generated from an initial URL as follows: + + An initial URL is built, ending with ";URLAUTH=" but without + the "::" components. An authorization mechanism is + selected and used to calculate the authorization token, with the + initial URL as the data and a secret known to the IMAP server as the + key. The URLAUTH-authorized URL is generated by taking the initial + URL and appending ":", the URL authorization mechanism name, ":", and + the ASCII-encoded hexadecimal representation of the authorization + token. + + + + + + +Crispin Standards Track [Page 6] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + Note: ASCII-encoded hexadecimal is used instead of BASE64 because + a BASE64 representation may have "=" padding characters, which + would be problematic in a URL. + + In the INTERNAL mechanism, the mailbox access key for that mailbox is + the secret known to the IMAP server, and a server-selected algorithm + is used as described in section 2.4.1. + +6. Validation of URLAUTH-authorized URLs + + A URLAUTH-authorized URL is validated as follows: + + The URL is split at the ":" that separates "" from + ":" in the ";URLAUTH=::" portion of + the URL. The ":" portion is first parsed and saved as + the authorization mechanism and the authorization token. The URL is + truncated, discarding the ":" described above, to create a "rump URL" + (the URL minus the ":" and the ":" portion). The rump + URL is then analyzed to identify the mailbox. + + If the mailbox cannot be identified, an authorization token is + calculated on the rump URL, using random "plausible" keys (selected + by the server) as needed, before returning a validation failure. + This prevents timing attacks aimed at identifying mailbox names. + + If the mailbox can be identified, the authorization token is + calculated on the rump URL and a secret known to the IMAP server + using the given URL authorization mechanism. Validation is + successful if, and only if, the calculated authorization token for + that mechanism matches the authorization token supplied in + ";URLAUTH=::". + + Removal of the "::" portion of the URL MUST be the only + operation applied to the URLAUTH-authorized URL to get the rump URL. + In particular, URL percent escape decoding and case-folding + (including to the domain part of the URL) MUST NOT occur. + + In the INTERNAL mechanism, the mailbox access key for that mailbox is + used as the secret known to the IMAP server, and the same server- + selected algorithm used for generating URLs is used to calculate the + authorization token for verification. + + + + + + + + + + +Crispin Standards Track [Page 7] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +7. Additional Commands + + These commands are extensions to the [IMAP] base protocol. + + The section headings of these commands are intended to correspond + with where they would be located in the base protocol document if + they were part of that document. + +BASE.6.3.RESETKEY. RESETKEY Command + + Arguments: optional mailbox name + optional mechanism name(s) + + Responses: none other than in result + + Result: OK - RESETKEY completed, URLMECH containing new data + NO - RESETKEY error: can't change key of that mailbox + BAD - command unknown or arguments invalid + + The RESETKEY command has two forms. + + The first form accepts a mailbox name as an argument and generates a + new mailbox access key for the given mailbox in the user's mailbox + access key table, replacing any previous mailbox access key (and + revoking any URLs that were authorized with a URLAUTH using that key) + in that table. By default, the mailbox access key is generated for + the INTERNAL mechanism; other mechanisms can be specified with the + optional mechanism argument. + + The second form, with no arguments, removes all mailbox access keys + in the user's mailbox access key table, revoking all URLs currently + authorized using URLAUTH by the user. + + Any current IMAP session logged in as the user that has the mailbox + selected will receive an untagged OK response with the URLMECH status + response code (see section BASE.7.1.URLMECH for more details about + the URLMECH status response code). + + Example: + + C: a31 RESETKEY + S: a31 OK All keys removed + C: a32 RESETKEY INBOX + S: a32 OK [URLMECH INTERNAL] mechs + C: a33 RESETKEY INBOX XSAMPLE + S: a33 OK [URLMECH INTERNAL XSAMPLE=P34OKhO7VEkCbsiYY8rGEg==] done + + + + + +Crispin Standards Track [Page 8] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +BASE.6.3.GENURLAUTH. GENURLAUTH Command + + Argument: one or more URL/mechanism pairs + + Response: untagged response: GENURLAUTH + + Result: OK - GENURLAUTH completed + NO - GENURLAUTH error: can't generate a URLAUTH + BAD - command unknown or arguments invalid + + The GENURLAUTH command requests that the server generate a URLAUTH- + authorized URL for each of the given URLs using the given URL + authorization mechanism. + + The server MUST validate each supplied URL as follows: + + (1) The mailbox component of the URL MUST refer to an existing + mailbox. + + (2) The server component of the URL MUST contain a valid userid + that identifies the owner of the mailbox access key table that + will be used to generate the URLAUTH-authorized URL. As a + consequence, the iserver rule of [IMAPURL] is modified so that + iuserauth is mandatory. + + Note: the server component of the URL is generally the + logged in userid and server. If not, then the logged in + userid and server MUST have owner-type access to the + mailbox access key table owned by the userid and server + indicated by the server component of the URL. + + (3) There is a valid access identifier that, in the case of + "submit+" and "user+", will contain a valid userid. This + userid is not necessarily the same as the owner userid + described in (2). + + (4) The server MAY also verify that the iuid and/or isection + components (if present) are valid. + + If any of the above checks fail, the server MUST return a tagged BAD + response with the following exception. If an invalid userid is + supplied as the mailbox access key owner and/or as part of the access + identifier, the server MAY issue a tagged OK response with a + generated mailbox key that always fails validation when used with a + URLFETCH command. This exception prevents an attacker from + validating userids. + + + + + +Crispin Standards Track [Page 9] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + If there is currently no mailbox access key for the given mailbox in + the owner's mailbox access key table, one is automatically generated. + That is, it is not necessary to use RESETKEY prior to first-time use + of GENURLAUTH. + + If the command is successful, a GENURLAUTH response code is returned + listing the requested URLs as URLAUTH-authorized URLs. + + Examples: + + C: a775 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2" INTERNAL + S: a775 BAD missing access identifier in supplied URL + C: a776 GENURLAUTH "imap://example.com/Shared/;uid=20/ + ;section=1.2;urlauth=submit+fred" INTERNAL + S: a776 BAD missing owner username in supplied URL + C: a777 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2;urlauth=submit+fred" INTERNAL + S: * GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 + ;urlauth=submit+fred:internal:91354a473744909de610943775f92038" + S: a777 OK GENURLAUTH completed + +BASE.6.3.URLFETCH. URLFETCH Command + + Argument: one or more URLs + + Response: untagged response: URLFETCH + + Result: OK - urlfetch completed + NO - urlfetch failed due to server internal error + BAD - command unknown or arguments invalid + + The URLFETCH command requests that the server return the text data + associated with the specified IMAP URLs, as described in [IMAPURL] + and extended by this document. The data is returned for all + validated URLs, regardless of whether or not the session would + otherwise be able to access the mailbox containing that data via + SELECT or EXAMINE. + + Note: This command does not require that the URL refer to the + selected mailbox; nor does it require that any mailbox be + selected. It also does not in any way interfere with any selected + mailbox. + + + + + + + + +Crispin Standards Track [Page 10] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + The URLFETCH command effectively executes with the access of the + userid in the server component of the URL (which is generally the + userid that issued the GENURLAUTH). By itself, the URLAUTH does NOT + grant access to the data; once validated, it grants whatever access + to the data is held by the userid in the server component of the URL. + That access may have changed since the GENURLAUTH was done. + + The URLFETCH command MUST return an untagged URLFETCH response and a + tagged OK response to any URLFETCH command that is syntactically + valid. A NO response indicates a server internal failure that may be + resolved on later retry. + + Note: The possibility of a NO response is to accommodate + implementations that would otherwise have to issue an untagged BYE + with a fatal error due to an inability to respond to a valid + request. In an ideal world, a server SHOULD NOT issue a NO + response. + + The server MUST return NIL for any IMAP URL that references an entire + IMAP server, a list of mailboxes, an entire IMAP mailbox, or IMAP + search results. + + Example: + + Note: For clarity, this example uses the LOGIN command, which + SHOULD NOT be used over a non-encrypted communication path. + + This example is of a submit server, obtaining a message segment + for a message that it has already validated was submitted by + "fred". + + S: * OK [CAPABILITY IMAP4REV1 URLAUTH] example.com IMAP server + C: a001 LOGIN submitserver secret + S: a001 OK submitserver logged in + C: a002 URLFETCH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2;urlauth=submit+fred:internal + :91354a473744909de610943775f92038" + S: * URLFETCH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 + ;urlauth=submit+fred:internal + :91354a473744909de610943775f92038" {28} + S: Si vis pacem, para bellum. + S: + S: a002 OK URLFETCH completed + + + + + + + + +Crispin Standards Track [Page 11] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +8. Additional Responses + + These responses are extensions to the [IMAP] base protocol. + + The section headings of these responses are intended to correspond + with where they would be located in the base protocol document if + they were part of that document. + +BASE.7.1.URLMECH. URLMECH Status Response Code + + The URLMECH status response code is followed by a list of URL + authorization mechanism names. Mechanism names other than INTERNAL + may be appended with an "=" and BASE64-encoded form of mechanism- + specific data. + + This status response code is returned in an untagged OK response in + response to a RESETKEY, SELECT, or EXAMINE command. In the case of + the RESETKEY command, this status response code can be sent in the + tagged OK response instead of requiring a separate untagged OK + response. + + Example: + + C: a33 RESETKEY INBOX XSAMPLE + S: a33 OK [URLMECH INTERNAL XSAMPLE=P34OKhO7VEkCbsiYY8rGEg==] done + + In this example, the server supports the INTERNAL mechanism and an + experimental mechanism called XSAMPLE, which also holds some + mechanism-specific data (the name "XSAMPLE" is for illustrative + purposes only). + +BASE.7.4.GENURLAUTH. GENURLAUTH Response + + Contents: One or more URLs + + The GENURLAUTH response returns the URLAUTH-authorized URL(s) + requested by a GENURLAUTH command. + + Example: + + C: a777 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2;urlauth=submit+fred" INTERNAL + S: * GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 + ;urlauth=submit+fred:internal:91354a473744909de610943775f92038" + S: a777 OK GENURLAUTH completed + + + + + + +Crispin Standards Track [Page 12] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +BASE.7.4.URLFETCH. URLFETCH Response + + Contents: One or more URL/nstring pairs + + The URLFETCH response returns the message text data associated with + one or more IMAP URLs, as described in [IMAPURL] and extended by this + document. This response occurs as the result of a URLFETCH command. + + The returned data string is NIL if the URL is invalid for any reason + (including validation failure). If the URL is valid, but the IMAP + fetch of the body part returned NIL (this should not happen), the + returned data string should be the empty string ("") and not NIL. + + Note: This command does not require that the URL refer to the + selected mailbox; nor does it require that any mailbox be + selected. It also does not in any way interfere with any selected + mailbox. + + Example: + + C: a002 URLFETCH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2;urlauth=submit+fred:internal + :91354a473744909de610943775f92038" + S: * URLFETCH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 + ;urlauth=submit+fred:internal + :91354a473744909de610943775f92038" {28} + S: Si vis pacem, para bellum. + S: + S: a002 OK URLFETCH completed + +9. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + The following modifications are made to the Formal Syntax in [IMAP]: + +resetkey = "RESETKEY" [SP mailbox *(SP mechanism)] + +capability =/ "URLAUTH" + +command-auth =/ resetkey / genurlauth / urlfetch + +resp-text-code =/ "URLMECH" SP "INTERNAL" *(SP mechanism ["=" base64]) + +genurlauth = "GENURLAUTH" 1*(SP url-rump SP mechanism) + +genurlauth-data = "*" SP "GENURLAUTH" 1*(SP url-full) + + + +Crispin Standards Track [Page 13] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +url-full = astring + ; contains authimapurlfull as defined below + +url-rump = astring + ; contains authimapurlrump as defined below + +urlfetch = "URLFETCH" 1*(SP url-full) + +urlfetch-data = "*" SP "URLFETCH" 1*(SP url-full SP nstring) + + The following extensions are made to the Formal Syntax in [IMAPURL]: + +authimapurl = "imap://" enc-user [iauth] "@" hostport "/" + imessagepart + ; replaces "imapurl" and "iserver" rules for + ; URLAUTH authorized URLs + +authimapurlfull = authimapurl iurlauth + +authimapurlrump = authimapurl iurlauth-rump + +enc-urlauth = 32*HEXDIG + +enc-user = 1*achar + ; same as "enc_user" in RFC 2192 + +iurlauth = iurlauth-rump ":" mechanism ":" enc-urlauth + +iurlauth-rump = [expire] ";URLAUTH=" access + +access = ("submit+" enc-user) / ("user+" enc-user) / + "authuser" / "anonymous" + +expire = ";EXPIRE=" date-time + ; date-time defined in [DATETIME] + +mechanism = "INTERNAL" / 1*(ALPHA / DIGIT / "-" / ".") + ; case-insensitive + ; new mechanisms MUST be registered with IANA + + + + + + + + + + + + +Crispin Standards Track [Page 14] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +10. Security Considerations + + Security considerations are discussed throughout this memo. + + The mailbox access key SHOULD have at least 128 bits of entropy + (refer to [RANDOM] for more details) and MUST be unpredictable. + + The server implementation of the INTERNAL mechanism SHOULD consider + the possibility of needing to change the token generation algorithm, + and SHOULD incorporate some means of identifying the token generation + algorithm within the token. + + The URLMECH status response code may expose sensitive data in the + mechanism-specific data for mechanisms other than INTERNAL. A server + implementation MUST implement a configuration that will not return a + URLMECH status response code unless some mechanism is provided that + protects the session from snooping, such as a TLS or SASL security + layer that provides confidentiality protection. + + The calculation of an authorization token with a "plausible" key if + the mailbox can not be identified is necessary to avoid attacks in + which the server is probed to see if a particular mailbox exists on + the server by measuring the amount of time taken to reject a known + bad name versus some other name. + + To protect against a computational denial-of-service attack, a server + MAY impose progressively longer delays on multiple URL requests that + fail validation. + + The decision to use the "authuser" access identifier should be made + with caution. An "authuser" access identifier can be used by any + authorized user of the IMAP server; therefore, use of this access + identifier should be limited to content that may be disclosed to any + authorized user of the IMAP server. + + The decision to use the "anonymous" access identifier should be made + with extreme caution. An "anonymous" access identifier can be used + by anyone; therefore, use of this access identifier should be limited + to content that may be disclosed to anyone. Many IMAP servers do not + permit anonymous access; in this case, the "anonymous" access + identifier is equivalent to "authuser", but this MUST NOT be relied + upon. + + Although this specification does not prohibit the theoretical + capability to generate a URL with a server component other than the + logged in userid and server, this capability should only be provided + + + + + +Crispin Standards Track [Page 15] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + when the logged in userid/server has been authorized as equivalent to + the server component userid/server, or otherwise has access to that + userid/server mailbox access key table. + +11. IANA Considerations + + This document constitutes registration of the URLAUTH capability in + the imap4-capabilities registry. + + URLAUTH authorization mechanisms are registered by publishing a + standards track or IESG-approved experimental RFC. The registry is + currently located at: + +http://www.iana.org/assignments/urlauth-authorization-mechanism-registry + + This registry is case-insensitive. + + This document constitutes registration of the INTERNAL URLAUTH + authorization mechanism. + + IMAP URLAUTH Authorization Mechanism Registry + + Mechanism Name Reference + -------------- --------- + INTERNAL [RFC4467] + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 16] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +12. Normative References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + [BURL] Newman, C., "Message Submission BURL Extension", RFC 4468, + May 2006. + + [DATETIME] Klyne, G. and C. Newman, "Date and Time on the Internet: + Timestamps", RFC 3339, July 2002. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [IMAPURL] Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + +13. Informative References + + [HMAC] Krawczyk, H., Bellare, M., and R. Canetti, "HMAC: Keyed- + Hashing for Message Authentication", RFC 2104, February + 1997. + + [RANDOM] Eastlake, D., 3rd, Schiller, J., and S. Crocker, + "Randomness Requirements for Security", BCP 106, RFC 4086, + June 2005. + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Avenue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + +Crispin Standards Track [Page 17] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Crispin Standards Track [Page 18] + diff --git a/docs/rfcs/rfc4469.txt b/docs/rfcs/rfc4469.txt new file mode 100644 index 0000000..da36551 --- /dev/null +++ b/docs/rfcs/rfc4469.txt @@ -0,0 +1,731 @@ + + + + + + +Network Working Group P. Resnick +Request for Comments: 4469 QUALCOMM Incorporated +Updates: 3501, 3502 April 2006 +Category: Standards Track + + + Internet Message Access Protocol (IMAP) CATENATE Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + The CATENATE extension to the Internet Message Access Protocol (IMAP) + extends the APPEND command to allow clients to create messages on the + IMAP server that may contain a combination of new data along with + parts of (or entire) messages already on the server. Using this + extension, the client can catenate parts of an already existing + message onto a new message without having to first download the data + and then upload it back to the server. + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 1] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +1. Introduction + + The CATENATE extension to the Internet Message Access Protocol (IMAP) + [1] allows the client to create a message on the server that can + include the text of messages (or parts of messages) that already + exist on the server without having to FETCH them and APPEND them back + to the server. The CATENATE extension extends the APPEND command so + that, instead of a single message literal, the command can take as + arguments any combination of message literals (as described in IMAP + [1]) and message URLs (as described in the IMAP URL Scheme [2] + specification). The server takes all the pieces and catenates them + into the output message. The CATENATE extension can also coexist + with the MULTIAPPEND extension [3] to APPEND multiple messages in a + single command. + + There are some obvious uses for the CATENATE extension. The + motivating use case was to provide a way for a resource-constrained + client to compose a message for subsequent submission that contains + data that already exists in that client's IMAP store. Because the + client does not have to download and re-upload potentially large + message parts, bandwidth and processing limitations do not have as + much impact. In addition, since the client can create a message in + its own IMAP store, the command also addresses the desire of the + client to archive a copy of a sent message without having to upload + the message twice. (Mechanisms for sending the message are outside + the scope of this document.) + + The extended APPEND command can also be used to copy parts of a + message to another mailbox for archival purposes while getting rid of + undesired parts. In environments where server storage is limited, a + client could get rid of large message parts by copying over only the + necessary parts and then deleting the original message. The + mechanism could also be used to add data to a message (such as + prepending message header fields) or to include other data by making + a copy of the original and catenating the new data. + +2. The CATENATE Capability + + A server that supports this extension returns "CATENATE" as one of + the responses to the CAPABILITY command. + + + + + + + + + + + +Resnick Standards Track [Page 2] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +3. The APPEND Command + + Arguments: mailbox name + (The following can be repeated in the presence of the + MULTIAPPEND extension [3]) + OPTIONAL flag parenthesized list + OPTIONAL date/time string + a single message literal or one or more message parts to + catenate, specified as: + message literal + or + message (or message part) URL + + Responses: OPTIONAL NO responses: BADURL, TOOBIG + + Result: OK - append completed + NO - append error: can't append to that mailbox, error + in flags or date/time or message text, or can't + fetch that data + BAD - command unknown or arguments invalid + + The APPEND command concatenates all the message parts and appends + them as a new message to the end of the specified mailbox. The + parenthesized flag list and date/time string set the flags and the + internal date, respectively, as described in IMAP [1]. The + subsequent command parameters specify the message parts that are + appended sequentially to the output message. + + If the original form of APPEND is used, a message literal follows the + optional flag list and date/time string, which is appended as + described in IMAP [1]. If the extended form is used, "CATENATE" and + a parenthesized list of message literals and message URLs follows, + each of which is appended to the new message. If a message literal + is specified (indicated by "TEXT"), the octets following the count + are appended. If a message URL is specified (indicated by "URL"), + the octets of the body part pointed to by that URL are appended, as + if the literal returned in a FETCH BODY response were put in place of + the message part specifier. The APPEND command does not cause the + \Seen flag to be set for any catenated body part. The APPEND command + does not change the selected mailbox. + + In the extended APPEND command, the string following "URL" is an IMAP + URL [2] and is interpreted according to the rules of [2]. The + present document only describes the behavior of the command using + IMAP URLs that refer to specific messages or message parts on the + current IMAP server from the current authenticated IMAP session. + Because of that, only relative IMAP message or message part URLs + (i.e., those having no scheme or ) are used. The base URL + + + +Resnick Standards Track [Page 3] + +RFC 4469 IMAP CATENATE Extension April 2006 + + + for evaluating the relative URL is considered "imap://user@server/", + where "user" is the user name of the currently authenticated user and + "server" is the domain name of the current server. When in the + selected state, the base URL is considered + "imap://user@server/mailbox", where "mailbox" is the encoded name of + the currently selected mailbox. Additionally, since the APPEND + command is valid in the authenticated state of an IMAP session, no + further LOGIN or AUTHENTICATE command is performed for URLs specified + in the extended APPEND command. + + Note: Use of an absolute IMAP URL or any URL that refers to + anything other than a message or message part from the current + authenticated IMAP session is outside the scope of this document + and would require an extension to this specification, and a server + implementing only this specification would return NO to such a + request. + + The client is responsible for making sure that the catenated message + is in the format of an Internet Message Format (RFC 2822) [4] or + Multipurpose Internet Mail Extension (MIME) [5] message. In + particular, when a URL is catenated, the server copies octets, + unchanged, from the indicated message or message part to the + catenated message. It does no data conversion (e.g., MIME transfer + encodings) nor any verification that the data is appropriate for the + MIME part of the message into which it is inserted. The client is + also responsible for inserting appropriate MIME boundaries between + body parts, and writing MIME Content-Type and Content-Transfer- + Encoding lines as needed in the appropriate places. + + Responses behave just as the original APPEND command described in + IMAP [1]. If the server implements the IMAP UIDPLUS extension [6], + it will also return an APPENDUID response code in the tagged OK + response. Two response codes are provided in Section 4 that can be + used in the tagged NO response if the APPEND command fails. + +4. Response Codes + + When a APPEND command fails, it may return a response code that + describes a reason for the failure. + +4.1. BADURL Response + + The BADURL response code is returned if the APPEND fails to process + one of the specified URLs. Possible reasons for this are bad URL + syntax, unrecognized URL schema, invalid message UID, or invalid body + part. The BADURL response code contains the first URL specified as a + parameter to the APPEND command that has caused the operation to + fail. + + + +Resnick Standards Track [Page 4] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +4.2. TOOBIG Response + + The TOOBIG response code is returned if the resulting message will + exceed the 4-GB IMAP message limit. This might happen, for example, + if the client specifies 3 URLs for 2-GB messages. Note that even if + the server doesn't return TOOBIG, it still has to be defensive + against misbehaving or malicious clients that try to construct a + message over the 4-GB limit. The server may also wish to return the + TOOBIG response code if the resulting message exceeds a server- + specific message size limit. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) [7] notation. Elements not defined here can be found in + the formal syntax of the ABNF [7], IMAP [1], and IMAP ABNF extensions + [8] specifications. Note that capability and resp-text-code are + extended from the IMAP [1] specification and append-data is extended + from the IMAP ABNF extensions [8] specification. + + append-data =/ "CATENATE" SP "(" cat-part *(SP cat-part) ")" + + cat-part = text-literal / url + + text-literal = "TEXT" SP literal + + url = "URL" SP astring + + resp-text-code =/ toobig-response-code / badurl-response-code + + toobig-response-code = "TOOBIG" + + badurl-response-code = "BADURL" SP url-resp-text + + url-resp-text = 1*(%x01-09 / + %x0B-0C / + %x0E-5B / + %x5D-FE) ; Any TEXT-CHAR except "]" + + capability =/ "CATENATE" + + The astring in the definition of url and the url-resp-text in the + definition of badurl-response-code each contain an imapurl as defined + by [2]. + + + + + + + +Resnick Standards Track [Page 5] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +6. Acknowledgements + + Thanks to the members of the LEMONADE working group for their input. + Special thanks to Alexey Melnikov for the examples. + +7. Security Considerations + + The CATENATE extension does not raise any security considerations + that are not present for the base protocol or in the use of IMAP + URLs, and these issues are discussed in the IMAP [1] and IMAP URL [2] + documents. + +8. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located at + . This document + defines the CATENATE IMAP capability. The IANA has added this + capability to the registry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 6] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +Appendix A. Examples + + Lines not starting with "C: " or "S: " are continuations of the + previous lines. + + The original message in examples 1 and 2 below (UID = 20) has the + following structure: + + + multipart/mixed MIME message with two body parts: + + 1. text/plain + + 2. application/x-zip-compressed + + Example 1: The following example demonstrates how a CATENATE client + can replace an attachment in a draft message, without the need to + download it to the client and upload it back. + + C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE + (URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=HEADER" + TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050907 + C: URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=1.MIME" + URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=1" TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050907 + C: URL "/Drafts;UIDVALIDITY=385759045/;UID=30" TEXT {44} + S: + Ready for literal data + C: + C: --------------030308070208000400050907-- + C: ) + S: A003 OK catenate append completed + + + + + + + + + + + + + + + +Resnick Standards Track [Page 7] + +RFC 4469 IMAP CATENATE Extension April 2006 + + + Example 2: The following example demonstrates how the CATENATE + extension can be used to replace edited text in a draft message, as + well as header fields for the top level message part (e.g., Subject + has changed). The previous version of the draft is marked as + \Deleted. Note that the server also supports the UIDPLUS extension, + so the APPENDUID response code is returned in the successful OK + response to the APPEND command. + + C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE (TEXT {738} + S: + Ready for literal data + C: Return-Path: + C: Received: from [127.0.0.2] + C: by rufus.example.org via TCP (internal) with ESMTPA; + C: Thu, 11 Nov 2004 16:57:07 +0000 + C: Message-ID: <419399E1.6000505@example.org> + C: Date: Thu, 12 Nov 2004 16:57:05 +0000 + C: From: Bob Ar + C: X-Accept-Language: en-us, en + C: MIME-Version: 1.0 + C: To: foo@example.net + C: Subject: About our holiday trip + C: Content-Type: multipart/mixed; + C: boundary="------------030308070208000400050907" + C: + C: --------------030308070208000400050907 + C: Content-Type: text/plain; charset=us-ascii; format=flowed + C: Content-Transfer-Encoding: 7bit + C: + C: Our travel agent has sent the updated schedule. + C: + C: Cheers, + C: Bob + C: --------------030308070208000400050907 + C: URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;Section=2.MIME" + URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;Section=2" TEXT {44} + S: + Ready for literal data + C: + C: --------------030308070208000400050907-- + C: ) + S: A003 OK [APPENDUID 385759045 45] append Completed + C: A004 UID STORE 20 +FLAGS.SILENT (\Deleted) + S: A004 OK STORE completed + + + + + + + + + +Resnick Standards Track [Page 8] + +RFC 4469 IMAP CATENATE Extension April 2006 + + + Example 3: The following example demonstrates how the CATENATE + extension can be used to strip attachments. Below, a PowerPoint + attachment was replaced by a small text part explaining that the + attachment was stripped. + + C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE + (URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=HEADER" + TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050903 + C: URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=1.MIME" + URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=1" TEXT {255} + S: + Ready for literal data + C: + C: --------------030308070208000400050903 + C: Content-type: text/plain; charset="us-ascii" + C: Content-transfer-encoding: 7bit + C: + C: This body part contained a Power Point presentation that was + C: deleted upon your request. + C: --------------030308070208000400050903-- + C: ) + S: A003 OK append Completed + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 9] + +RFC 4469 IMAP CATENATE Extension April 2006 + + + Example 4: The following example demonstrates a failed APPEND + command. The server returns the BADURL response code to indicate + that one of the provided URLs is invalid. This example also + demonstrates how the CATENATE extension can be used to construct a + digest of several messages. + + C: A003 APPEND Sent (\Seen $MDNSent) CATENATE (TEXT {541} + S: + Ready for literal data + C: Return-Path: + C: Received: from [127.0.0.2] + C: by rufus.example.org via TCP (internal) with ESMTPA; + C: Thu, 11 Nov 2004 16:57:07 +0000 + C: Message-ID: <419399E1.6000505@example.org> + C: Date: Thu, 21 Nov 2004 16:57:05 +0000 + C: From: Farren Oo + C: X-Accept-Language: en-us, en + C: MIME-Version: 1.0 + C: To: bar@example.org + C: Subject: Digest of the mailing list for today + C: Content-Type: multipart/digest; + C: boundary="------------030308070208000400050904" + C: + C: --------------030308070208000400050904 + C: URL "/INBOX;UIDVALIDITY=785799047/;UID=11467" TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050904 + C: URL "/INBOX;UIDVALIDITY=785799047/;UID=113330/;section=1.5.9" + TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050904 + C: URL "/INBOX;UIDVALIDITY=785799047/;UID=11916" TEXT {44} + S: + Ready for literal data + C: + C: --------------030308070208000400050904-- + C: ) + S: A003 NO [BADURL "/INBOX;UIDVALIDITY=785799047/;UID=113330; + section=1.5.9"] CATENATE append has failed, one message expunged + + Note that the server could have validated the URLs as they were + received and therefore could have returned the tagged NO response + with BADURL response-code in place of any continuation request after + the URL was received. + + + + + + + +Resnick Standards Track [Page 10] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +9. Normative References + + [1] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", + RFC 3501, March 2003. + + [2] Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. + + [3] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + + [4] Resnick, P., "Internet Message Format", RFC 2822, April 2001. + + [5] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message Bodies", + RFC 2045, November 1996. + + [6] Crispin, M., "Internet Message Access Protocol (IMAP) - UIDPLUS + extension", RFC 4315, December 2005. + + [7] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + [8] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", + RFC 4466, April 2006. + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 11] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +Author's Address + + Peter W. Resnick + QUALCOMM Incorporated + 5775 Morehouse Drive + San Diego, CA 92121-1714 + US + + Phone: +1 858 651 4478 + EMail: presnick@qualcomm.com + URI: http://www.qualcomm.com/~presnick/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 12] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Resnick Standards Track [Page 13] + diff --git a/docs/rfcs/rfc4549.txt b/docs/rfcs/rfc4549.txt new file mode 100644 index 0000000..8430ee1 --- /dev/null +++ b/docs/rfcs/rfc4549.txt @@ -0,0 +1,1963 @@ + + + + + + +Network Working Group A. Melnikov, Ed. +Request for Comments: 4549 Isode Ltd. +Category: Informational June 2006 + + + Synchronization Operations for Disconnected IMAP4 Clients + +Status of This Memo + + This memo provides information for the Internet community. It does + not specify an Internet standard of any kind. Distribution of this + memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + This document attempts to address some of the issues involved in + building a disconnected IMAP4 client. In particular, it deals with + the issues of what might be called the "driver" portion of the + synchronization tool: the portion of the code responsible for issuing + the correct set of IMAP4 commands to synchronize the disconnected + client in the way that is most likely to make the human who uses the + disconnected client happy. + + This note describes different strategies that can be used by + disconnected clients and shows how to use IMAP protocol in order to + minimize the time of the synchronization process. + + This note also lists IMAP extensions that a server should implement + in order to provide better synchronization facilities to disconnected + clients. + + + + + + + + + + + + + + + + + +Melnikov Informational [Page 1] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +Table of Contents + + 1. Introduction ....................................................3 + 1.1. Conventions Used in This Document ..........................3 + 2. Design Principles ...............................................3 + 3. Overall Picture of Synchronization ..............................4 + 4. Mailbox Synchronization Steps and Strategies ....................7 + 4.1. Checking UID Validity ......................................7 + 4.2. Synchronizing Local Changes with the Server ................8 + 4.2.1. Uploading Messages to the Mailbox ...................8 + 4.2.2. Optimizing "move" and "copy" Operations .............9 + 4.2.3. Replaying Local Flag Changes .......................14 + 4.2.4. Processing Mailbox Compression (EXPUNGE) Requests ..15 + 4.2.5. Closing a Mailbox ..................................17 + 4.3. Details of "Normal" Synchronization of a Single Mailbox ...18 + 4.3.1. Discovering New Messages and Changes to Old + Messages ...........................................18 + 4.3.2. Searching for "Interesting" Messages. ..............20 + 4.3.3. Populating Cache with "Interesting" Messages. ......21 + 4.3.4. User-Initiated Synchronization .....................22 + 4.4. Special Case: Descriptor-Only Synchronization .............22 + 4.5. Special Case: Fast New-Only Synchronization ...............23 + 4.6. Special Case: Blind FETCH .................................23 + 5. Implementation Considerations ..................................24 + 5.1. Error Recovery during Playback ............................26 + 5.2. Quality of Implementation Issues ..........................28 + 5.3. Optimizations .............................................28 + 6. IMAP Extensions That May Help ..................................30 + 6.1. CONDSTORE Extension .......................................30 + 7. Security Considerations ........................................33 + 8. References .....................................................33 + 8.1. Normative References ......................................33 + 8.2. Informative References ....................................34 + 9. Acknowledgements ...............................................34 + + + + + + + + + + + + + + + + + +Melnikov Informational [Page 2] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +1. Introduction + + Several recommendations presented in this document are generally + applicable to all types of IMAP clients. However, this document + tries to concentrate on disconnected mail clients [IMAP-MODEL]. It + also suggests some IMAP extensions* that should be implemented by + IMAP servers in order to make the life of disconnected clients + easier. In particular, the [UIDPLUS] extension was specifically + designed to streamline certain disconnected operations, like + expunging, uploading, and copying messages (see Sections 4.2.1, + 4.2.2.1, and 4.2.4). + + Readers of this document are also strongly advised to read RFC 2683 + [RFC2683]. + + * Note that the functionality provided by the base IMAP protocol + [IMAP4] is sufficient to perform basic synchronization. + +1.1. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. Long lines in examples are broken for + editorial clarity. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + + Let's call an IMAP command idempotent if the result of executing the + command twice sequentially is the same as the result of executing the + command just once. + +2. Design Principles + + All mailbox state or content information stored on the disconnected + client should be viewed strictly as a cache of the state of the + server. The "master" state remains on the server, just as it would + with an interactive IMAP4 client. The one exception to this rule is + that information about the state of the disconnected client's cache + (the state includes flag changes while offline and during scheduled + message uploads) remains on the disconnected client: that is, the + IMAP4 server is not responsible for remembering the state of the + disconnected IMAP4 client. + + We assume that a disconnected client is a client that, for whatever + reason, wants to minimize the length of time that it is "on the + phone" to the IMAP4 server. Often this will be because the client is + using a dialup connection, possibly with very low bandwidth, but + + + +Melnikov Informational [Page 3] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + sometimes it might just be that the human is in a hurry to catch an + airplane, or some other event beyond our control. Whatever the + reason, we assume that we must make efficient use of the network + connection, both in the usual sense (not generating spurious traffic) + and in the sense that we would prefer not to have the connection + sitting idle while the client and/or the server is performing + strictly local computation or I/O. Another, perhaps simpler way of + stating this is that we assume that network connections are + "expensive". + + Practical experience with disconnected mail systems has shown that + there is no single synchronization strategy that is appropriate for + all cases. Different humans have different preferences, and the same + human's preference will vary depending both on external circumstance + (how much of a hurry the human is in today) and on the value that the + human places on the messages being transferred. The point here is + that there is no way that the synchronization program can guess + exactly what the human wants to do, so the human will have to provide + some guidance. + + Taken together, the preceding two principles lead to the conclusion + that the synchronization program must make its decisions based on + some kind of guidance provided by the human, by selecting the + appropriate options in the user interface or through some sort of + configuration file. Almost certainly, it should not pause for I/O + with the human in the middle of the synchronization process. The + human will almost certainly have several different configurations for + the synchronization program, for different circumstances. + + Since a disconnected client has no way of knowing what changes might + have occurred to the mailbox while it was disconnected, message + numbers are not useful to a disconnected client. All disconnected + client operations should be performed using UIDs, so that the client + can be sure that it and the server are talking about the same + messages during the synchronization process. + +3. Overall Picture of Synchronization + + The basic strategy for synchronization is outlined below. Note that + the real strategy may vary from one application to another or may + depend on a synchronization mode. + + a) Process any "actions" that were pending on the client that were + not associated with any mailbox. (In particular sending messages + composed offline with SMTP. This is not part of IMAP + synchronization, but it is mentioned here for completeness.) + + + + + +Melnikov Informational [Page 4] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + b) Fetch the current list of "interesting" mailboxes. (The + disconnected client should allow the user to skip this step + completely.) + + c) "Client-to-server synchronization": for each IMAP "action" that + was pending on the client, do the following: + + 1) If the action implies opening a new mailbox (any operation that + operates on messages), open the mailbox. Check its UID + validity value (see Section 4.1 for more details) returned in + the UIDVALIDITY response code. If the UIDVALIDITY value + returned by the server differs, the client MUST empty the local + cache of the mailbox and remove any pending "actions" that + refer to UIDs in that mailbox (and consider them failed). Note + that this doesn't affect actions performed on client-generated + fake UIDs (see Section 5). + + 2) Perform the action. If the action is to delete a mailbox + (DELETE), make sure that the mailbox is closed first (see also + Section 3.4.12 of [RFC2683]). + + d) "Server-to-client synchronization": for each mailbox that requires + synchronization, do the following: + + 1) Check the mailbox UIDVALIDITY (see Section 4.1 for more + details) with SELECT/EXAMINE/STATUS. + + If UIDVALIDITY value returned by the server differs, the client + MUST + + * empty the local cache of that mailbox; + * remove any pending "actions" that refer to UIDs in that + mailbox and consider them failed; and + * skip step 2-II. + + 2) Fetch the current "descriptors"; + + I) Discover new messages. + + II) Discover changes to old messages. + + 3) Fetch the bodies of any "interesting" messages that the client + doesn't already have. + + e) Close all open mailboxes not required for further operations (if + staying online) or disconnect all open connections (if going + offline). + + + + +Melnikov Informational [Page 5] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Terms used: + + "Actions" are queued requests that were made by the human to the + client's Mail User Agent (MUA) software while the client was + disconnected. + + We define "descriptors" as a set of IMAP4 FETCH data items. + Conceptually, a message's descriptor is that set of information that + allows the synchronization program to decide what protocol actions + are necessary to bring the local cache to the desired state for this + message; since this decision is really up to the human, this + information probably includes at least a few header fields intended + for human consumption. Exactly what will constitute a descriptor + depends on the client implementation. At a minimum, the descriptor + contains the message's UID and FLAGS. Other likely candidates are + the RFC822.SIZE, RFC822.HEADER, BODYSTRUCTURE, or ENVELOPE data + items. + + Comments: + + 1) The list of actions should be ordered. For example, if the human + deletes message A1 in mailbox A, then expunges mailbox A, and then + deletes message A2 in mailbox A, the human will expect that + message A1 is gone and that message A2 is still present but is now + deleted. + + By processing all the actions before proceeding with + synchronization, we avoid having to compensate for the local MUA's + changes to the server's state. That is, once we have processed + all the pending actions, the steps that the client must take to + synchronize itself will be the same no matter where the changes to + the server's state originated. + + 2) Steps a and b can be performed in parallel. Alternatively, step a + can be performed after d. + + 3) On step b, the set of "interesting" mailboxes pretty much has to + be determined by the human. What mailboxes belong to this set may + vary between different IMAP4 sessions with the same server, + client, and human. An interesting mailbox can be a mailbox + returned by LSUB command (see Section 6.3.9 of [IMAP4]). The + special mailbox "INBOX" SHOULD be in the default set of mailboxes + that the client considers interesting. However, providing the + ability to ignore INBOX for a particular session or client may be + valuable for some mail filtering strategies. + + + + + + +Melnikov Informational [Page 6] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 4) On step d-2-II, the client also finds out about changes to the + flags of messages that the client already has in its local cache, + and about messages in the local cache that no longer exist on the + server (i.e., messages that have been expunged). + + 5) "Interesting" messages are those messages that the synchronization + program thinks the human wants to have cached locally, based on + the configuration and the data retrieved in step b. + + 6) A disconnected IMAP client is a special case of an IMAP client, so + it MUST be able to handle any "unexpected" unsolicited responses, + like EXISTS and EXPUNGE, at any time. The disconnected client MAY + ignore EXPUNGE response during "client-to-server" synchronization + phase (step c). + + The rest of this discussion will focus primarily on the + synchronization issues for a single mailbox. + +4. Mailbox Synchronization Steps and Strategies + +4.1. Checking UID Validity + + The "UID validity" of a mailbox is a number returned in an + UIDVALIDITY response code in an OK untagged response at mailbox + selection time. The UID validity value changes between sessions when + UIDs fail to persist between sessions. + + Whenever the client selects a mailbox, the client must compare the + returned UID validity value with the value stored in the local cache. + If the UID validity values differ, the UIDs in the client's cache are + no longer valid. The client MUST then empty the local cache of that + mailbox and remove any pending "actions" that refer to UIDs in that + mailbox. The client MAY also issue a warning to the human. The + client MUST NOT cancel any scheduled uploads (i.e., APPENDs) for the + mailbox. + + Note that UIDVALIDITY is not only returned on a mailbox selection. + The COPYUID and APPENDUID response codes defined in the [UIDPLUS] + extension (see also 4.2.2) and the UIDVALIDITY STATUS response data + item also contain a UIDVALIDITY value for some other mailbox. The + client SHOULD behave as described in the previous paragraph (but it + should act on the other mailbox's cache), no matter how it obtained + the UIDVALIDITY value. + + + + + + + + +Melnikov Informational [Page 7] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +4.2. Synchronizing Local Changes with the Server + +4.2.1. Uploading Messages to the Mailbox + + Two of the most common examples of operations resulting in message + uploads are: + + 1) Saving a draft message + + 2) Copying a message between remote mailboxes on two different IMAP + servers or a local mailbox and a remote mailbox. + + Message upload is performed with the APPEND command. A message + scheduled to be uploaded has no UID associated with it, as all UIDs + are assigned by the server. The APPEND command will effectively + associate a UID with the uploaded message that can be stored in the + local cache for future reference. However, [IMAP4] doesn't describe + a simple mechanism to discover the message UID by just performing the + APPEND command. In order to discover the UID, the client can do one + of the following: + + 1) Remove the uploaded message from cache. Then, use the mechanism + described in 4.3 to fetch the information about the uploaded + message as if it had been uploaded by some other client. + + 2) Try to fetch header information as described in 4.2.2 in order to + find a message that corresponds to the uploaded message. One + strategy for doing this is described in 4.2.2. + + Case 1 describes a not particularly smart client. + + C: A003 APPEND Drafts (\Seen $MDNSent) {310} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK APPEND Completed + + Fortunately, there is a simpler way to discover the message UID in + the presence of the [UIDPLUS] extension: + + + + +Melnikov Informational [Page 8] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + C: A003 APPEND Drafts (\Seen $MDNSent) {310} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 1022843275 77712] APPEND completed + + The UID of the appended message is the second parameter of APPENDUID + response code. + +4.2.2. Optimizing "move" and "copy" Operations + + Practical experience with IMAP and other mailbox access protocols + that support multiple mailboxes suggests that moving a message from + one mailbox to another is an extremely common operation. + +4.2.2.1. Moving a Message between Two Mailboxes on the Same Server + + In IMAP4, a "move" operation between two mailboxes on the same server + is really a combination of a COPY operation and a STORE +FLAGS + (\Deleted) operation. This makes good protocol sense for IMAP, but + it leaves a simple-minded disconnected client in the silly position + of deleting and possibly expunging its cached copy of a message, then + fetching an identical copy via the network. + + However, the presence of the UIDPLUS extension in the server can + help: + + C: A001 UID COPY 567,414 "Interesting Messages" + S: A001 OK [COPYUID 1022843275 414,567 5:6] Completed + + This tells the client that the message with UID 414 in the current + mailbox was successfully copied to the mailbox "Interesting Messages" + and was given the UID 5, and that the message with UID 567 was given + the UID 6. + + In the absence of UIDPLUS extension support in the server, the + following trick can be used. By including the Message-ID: header and + the INTERNALDATE data item as part of the descriptor, the client can + check the descriptor of a "new" message against messages that are + already in its cache and avoid fetching the extra copy. Of course, + + + +Melnikov Informational [Page 9] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + it's possible that the cost of checking to see if the message is + already in the local cache may exceed the cost of just fetching it, + so this technique should not be used blindly. If the MUA implements + a "move" command, it makes special provisions to use this technique + when it knows that a copy/delete sequence is the result of a "move" + command. + + Note that servers are not required (although they are strongly + encouraged with "SHOULD language") to preserve INTERNALDATE when + copying messages. + + Also note that since it's theoretically possible for this algorithm + to find the wrong message (given sufficiently malignant Message-ID + headers), implementers should provide a way to disable this + optimization, both permanently and on a message-by-message basis. + + Example 1: Copying a message in the absence of UIDPLUS extension. + + At some point in time the client has fetched the source message and + some information was cached: + + C: C021 UID FETCH (BODY.PEEK[] INTERNALDATE FLAGS) + ... + S: * 27 FETCH (UID 123 INTERNALDATE "31-May-2002 05:26:59 -0600" + FLAGS (\Draft $MDNSent) BODY[] {1036} + S: ... + S: Message-Id: <20040903110856.22a127cd@chardonnay> + S: ... + S: ...message body... + S: ) + ... + S: C021 OK fetch completed + + Later on, the client decides to copy the message: + + C: C035 UID COPY 123 "Interesting Messages" + S: C035 OK Completed + + As the server hasn't provided the COPYUID response code, the client + tries the optimization described above: + + C: C036 SELECT "Interesting Messages" + ... + C: C037 UID SEARCH ON 31-May-2002 HEADER + "Message-Id" "20040903110856.22a127cd@chardonnay" + S: SEARCH 12368 + S: C037 OK completed + + + + +Melnikov Informational [Page 10] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Note that if the server has returned multiple UIDs in the SEARCH + response, the client MUST NOT use any of the returned UID. + +4.2.2.2. Moving a Message from a Remote Mailbox to a Local + + Moving a message from a remote mailbox to a local is done with FETCH + (that includes FLAGS and INTERNALDATE) followed by UID STORE + +FLAGS.SILENT (\Deleted): + + C: A003 UID FETCH 123 (BODY.PEEK[] INTERNALDATE FLAGS) + S: * 27 FETCH (UID 123 INTERNALDATE "31-May-2002 05:26:59 -0600" + FLAGS (\Seen $MDNSent) BODY[] + S: ...message body... + S: ) + S: A003 OK UID FETCH completed + C: A004 UID STORE +FLAGS.SILENT (\Deleted) + S: A004 STORE completed + + Note that there is no reason to fetch the message during + synchronization if it's already in the client's cache. Also, the + client SHOULD preserve delivery date in the local cache. + +4.2.2.3. Moving a Message from a Local Mailbox to a Remote + + Moving a message from a local mailbox to a remote is done with + APPEND: + + C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" + {310} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 1022843275 77712] completed + + The client SHOULD specify the delivery date from the local cache in + the APPEND. + + If the [LITERAL+] extension is available, the client can save a + round-trip*: + + + + +Melnikov Informational [Page 11] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" + {310+} + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 1022843275 77712] completed + + * Note that there is a risk that the server will reject the message + due to its size. If this happens, the client will waste bandwidth + transferring the whole message. If the client wouldn't have used + the LITERAL+, this could have been avoided: + + C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2004 05:26:59 -0600" + {16777215} + S: A003 NO Sorry, message is too big + +4.2.2.4. Moving a Message between Two Mailboxes on Different Servers + + Moving a message between two mailbox on two different servers is a + combination of the operations described in 4.2.2.2 followed by the + operations described in 4.2.2.3. + +4.2.2.5. Uploading Multiple Messages to a Remote Mailbox with + MULTIAPPEND + + When there is a need to upload multiple messages to a remote mailbox + (e.g., as per 4.2.2.3), the presence of certain IMAP extensions may + significantly improve performance. One of them is [MULTIAPPEND]. + + For some mail stores, opening a mailbox for appending might be + expensive. [MULTIAPPEND] tells the server to open the mailbox once + (instead of opening and closing it "n" times per "n" messages to be + uploaded) and to keep it open while a group of messages is being + uploaded to the server. + + Also, if the server supports both [MULTIAPPEND] and [LITERAL+] + extensions, the entire upload is accomplished in a single + command/response round-trip. + + + + + + +Melnikov Informational [Page 12] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Note: Client implementers should be aware that [MULTIAPPEND] performs + append of multiple messages atomically. This means, for example, if + there is not enough space to save "n"-th message (or the message has + invalid structure and is rejected by the server) after successful + upload of "n-1" messages, the whole upload operation fails, and no + message will be saved in the mailbox. Although this behavior might + be desirable in certain situations, it might not be what you want. + Otherwise, the client should use the regular APPEND command (Section + 4.2.2.3), possibly utilizing the [LITERAL+] extension. See also + Section 5.1 for discussions about error recovery. + + Note: MULTIAPPEND can be used together with the UIDPLUS extension in + a way similar to what was described in Section 4.2.1. [MULTIAPPEND] + extends the syntax of the APPENDUID response code to allow for + multiple message UIDs in the second parameter. + + Example 2: + + This example demonstrates the use of MULTIAPPEND together with + UIDPLUS (synchronization points where the client waits for + confirmations from the server are marked with "<--->"): + + C: A003 APPEND Jan-2002 (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" + {310} + <---> + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: (\Seen) " 1-Jun-2002 22:43:04 -0800" {286} + <---> + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) + C: From: Joe Mooch + C: Subject: Re: afternoon meeting + C: To: foobar@blt.example.com + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: 3:30 is fine with me. + C: + + + +Melnikov Informational [Page 13] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + S: A003 OK [APPENDUID 1022843275 77712,77713] completed + + The upload takes 3 round-trips. + + Example 3: + + In this example, Example 2 was modified for the case when the server + supports MULTIAPPEND, LITERAL+, and UIDPLUS. The upload takes only 1 + round-trip. + + C: A003 APPEND Jan-2002 (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" + {310+} + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: (\Seen) " 1-Jun-2002 22:43:04 -0800" {286+} + C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) + C: From: Joe Mooch + C: Subject: Re: afternoon meeting + C: To: foobar@blt.example.com + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: 3:30 is fine with me. + C: + S: A003 OK [APPENDUID 1022843275 77712,77713] completed + +4.2.3. Replaying Local Flag Changes + + The disconnected client uses the STORE command to synchronize local + flag state with the server. The disconnected client SHOULD use + +FLAGS.SILENT or -FLAGS.SILENT in order to set or unset flags + modified by the user while offline. The FLAGS form MUST NOT be used, + as there is a risk that this will overwrite flags on the server that + have been changed by some other client. + + Example 4: + + For the message with UID 15, the disconnected client stores the + following flags \Seen and $Highest. The flags were modified on the + server by some other client: \Seen, \Answered, and $Highest. While + + + +Melnikov Informational [Page 14] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + offline, the user requested that the $Highest flags be removed and + that the \Deleted flag be added. The flag synchronization sequence + for the message should look like: + + C: A001 UID STORE 15 +FLAGS.SILENT (\Deleted) + S: A001 STORE completed + C: A002 UID STORE 15 -FLAGS.SILENT ($Highest) + S: A002 STORE completed + + If the disconnected client is able to store an additional binary + state information (or a piece of information that can take a value + from a predefined set of values) in the local cache of an IMAP + mailbox or in a local mailbox (e.g., message priority), and if the + server supports storing of arbitrary keywords, the client MUST use + keywords to store this state on the server. + + Example 5: + + Imagine a speculative mail client that can mark a message as one of + work-related ($Work), personal ($Personal), or spam ($Spam). In + order to mark a message as personal, the client issues: + + C: A001 UID STORE 15 +FLAGS.SILENT ($Personal) + S: A001 STORE completed + C: A002 UID STORE 15 -FLAGS.SILENT ($Work $Spam) + S: A002 STORE completed + + In order to mark the message as not work, not personal and not spam, + the client issues: + + C: A003 UID STORE 15 -FLAGS.SILENT ($Personal $Work $Spam) + S: A003 STORE completed + +4.2.4. Processing Mailbox Compression (EXPUNGE) Requests + + A naive disconnected client implementation that supports compressing + a mailbox while offline may decide to issue an EXPUNGE command to the + server in order to expunge messages marked \Deleted. The problem + with this command during synchronization is that it permanently + erases all messages with the \Deleted flag set, i.e., even those + messages that were marked as \Deleted on the server while the user + was offline. Doing this might result in an unpleasant surprise for + the user. + + Fortunately the [UIDPLUS] extension can help in this case as well. + The extension introduces UID EXPUNGE command, that, unlike EXPUNGE, + takes a UID set parameter, that lists UIDs of all messages that can + be expunged. When processing this command the server erases only + + + +Melnikov Informational [Page 15] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + messages with \Deleted flag listed in the UID list. Thus, messages + not listed in the UID set will not be expunged even if they have the + \Deleted flag set. + + Example 6: + + While the user was offline, 3 messages with UIDs 7, 27, and 65 were + marked \Deleted when the user requested to compress the open mailbox. + Another client marked a message \Deleted on the server (UID 34). + During synchronization, the disconnected client issues: + + C: A001 UID EXPUNGE 7,27,65 + S: * ... EXPUNGE + S: * ... EXPUNGE + S: * ... EXPUNGE + S: A001 UID EXPUNGE completed + + If another client issues UID SEARCH DELETED command (to find all + messages with the \Deleted flag) before and after the UID EXPUNGE, it + will get: + + Before: + + C: B001 UID SEARCH DELETED + S: * SEARCH 65 34 27 7 + S: B001 UID SEARCH completed + + After: + + C: B002 UID SEARCH DELETED + S: * SEARCH 34 + S: B002 UID SEARCH completed + + In the absence of the [UIDPLUS] extension, the following sequence of + commands can be used as an approximation. Note: It's possible for + another client to mark additional messages as deleted while this + sequence is being performed. In this case, these additional messages + will be expunged as well. + + 1) Find all messages marked \Deleted on the server. + + C: A001 UID SEARCH DELETED + S: * SEARCH 65 34 27 7 + S: A001 UID SEARCH completed + + 2) Find all messages that must not be erased (for the previous + example the list will consist of the message with UID 34). + + + + +Melnikov Informational [Page 16] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 3) Temporarily remove \Deleted flag on all messages found in step 2. + + C: A002 UID STORE 34 -FLAGS.SILENT (\Deleted) + S: A002 UID STORE completed + + 4) Expunge the mailbox. + + C: A003 EXPUNGE + S: * 20 EXPUNGE + S: * 7 EXPUNGE + S: * 1 EXPUNGE + S: A003 EXPUNGE completed + + Here, the message with UID 7 has message number 1, with UID 27 has + message number 7, and with UID 65 has message number 20. + + 5) Restore \Deleted flag on all messages found when performing step + 2. + + C: A004 UID STORE 34 +FLAGS.SILENT (\Deleted) + S: A004 UID STORE completed + +4.2.5. Closing a Mailbox + + When the disconnected client has to close a mailbox, it should not + use the CLOSE command, because CLOSE does a silent EXPUNGE. (Section + 4.2.4 explains why EXPUNGE should not be used by a disconnected + client.) It is safe to use CLOSE only if the mailbox was opened with + EXAMINE. + + If the mailbox was opened with SELECT, the client can use one of the + following commands to implicitly close the mailbox and prevent the + silent expunge: + + 1) UNSELECT - This is a command described in [UNSELECT] that works as + CLOSE, but doesn't cause the silent EXPUNGE. This command is + supported by the server if it reports UNSELECT in its CAPABILITY + list. + + 2) SELECT - SELECT causes implicit CLOSE without + EXPUNGE. + + 3) If the client intends to issue LOGOUT after closing the mailbox, + it may just issue LOGOUT, because LOGOUT causes implicit CLOSE + without EXPUNGE as well. + + 4) SELECT - If the client knows a mailbox that + doesn't exist or can't be selected, it MAY SELECT it. + + + +Melnikov Informational [Page 17] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + If the client opened the mailbox with SELECT and just wants to avoid + implicit EXPUNGE without closing the mailbox, it may also use the + following: + + 5) EXAMINE - Reselect the same mailbox in read-only mode. + +4.3. Details of "Normal" Synchronization of a Single Mailbox + + The most common form of synchronization is where the human trusts the + integrity of the client's copy of the state of a particular mailbox + and simply wants to bring the client's cache up to date so that it + accurately reflects the mailbox's current state on the server. + +4.3.1. Discovering New Messages and Changes to Old Messages + + Let represent the highest UID that the client knows + about in this mailbox. Since UIDs are allocated in strictly + ascending order, this is simply the UID of the last message in the + mailbox that the client knows about. Let represent + 's UID plus one. Let represent a list + consisting of all the FETCH data item items that the implementation + considers part of the descriptor; at a minimum this is just the FLAGS + data item, but it usually also includes BODYSTRUCTURE and + RFC822.SIZE. At this step, SHOULD NOT include RFC822. + + With no further information, the client can issue the following two + commands: + + tag1 UID FETCH :* + tag2 UID FETCH 1: FLAGS + + The first command will request some information about "new" messages + (i.e., messages received by the server since the last + synchronization). It will also allow the client to build a message + number to UID map (only for new messages). The second command allows + the client to + + 1) update cached flags for old messages; + + 2) find out which old messages got expunged; and + + 3) build a mapping between message numbers and UIDs (for old + messages). + + The order here is significant. We want the server to start returning + the list of new message descriptors as fast as it can, so that the + client can start issuing more FETCH commands, so we start out by + asking for the descriptors of all the messages we know the client + + + +Melnikov Informational [Page 18] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + cannot possibly have cached yet. The second command fetches the + information we need to determine what changes may have occurred to + messages that the client already has cached. Note that the former + command should only be issued if the UIDNEXT value cached by the + client differs from the one returned by the server. Once the client + has issued these two commands, there's nothing more the client can do + with this mailbox until the responses to the first command start + arriving. A clever synchronization program might use this time to + fetch its local cache state from disk or to start the process of + synchronizing another mailbox. + + The following is an example of the first FETCH: + + C: A011 UID fetch 131:* (FLAGS BODYSTRUCTURE INTERNALDATE + RFC822.SIZE) + + Note 1: The first FETCH may result in the server's sending a huge + volume of data. A smart disconnected client should use message + ranges (see also Section 3.2.1.2 of [RFC2683]), so that the user is + able to execute a different operation between fetching information + for a group of new messages. + + Example 7: + + Knowing the new UIDNEXT returned by the server on SELECT or EXAMINE + (), the client can split the UID range + : into groups, e.g., 100 messages. After + that, the client can issue: + + C: A011 UID fetch : + (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) + ... + C: A012 UID fetch : + (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) + ... + ... + C: A0FF UID fetch : + (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) + + Note that unless a SEARCH command is issued, it is impossible to + determine how many messages will fall into a subrange, as UIDs are + not necessarily contiguous. + + Note 2: The client SHOULD ignore any unsolicited EXPUNGE responses + received during the first FETCH command. EXPUNGE responses contain + message numbers that are useless to a client that doesn't have the + message-number-to-UID translation table. + + + + +Melnikov Informational [Page 19] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + The second FETCH command will result in zero or more untagged fetch + responses. Each response will have a corresponding UID FETCH data + item. All messages that didn't have a matching untagged FETCH + response MUST be removed from the local cache. + + For example, if the had a value 15000 and the local + cache contained 3 messages with the UIDs 12, 777, and 14999, + respectively, then after receiving the following responses from the + server, the client must remove the message with UID 14999 from its + local cache. + + S: * 1 FETCH (UID 12 FLAGS (\Seen)) + S: * 2 FETCH (UID 777 FLAGS (\Answered \Deleted)) + + Note 3: If the client is not interested in flag changes (i.e., the + client only wants to know which old messages are still on the + server), the second FETCH command can be substituted with: + + tag2 UID SEARCH UID 1: + + This command will generate less traffic. However, an implementor + should be aware that in order to build the mapping table from message + numbers to UIDs, the output of the SEARCH command MUST be sorted + first, because there is no requirement for a server to return UIDs in + SEARCH response in any particular order. + +4.3.2. Searching for "Interesting" Messages. + + This step is performed entirely on the client (from the information + received in the step described in 4.3.1), entirely on the server, or + on some combination of both. The decision on what is an + "interesting" message is up to the client software and the human. + One easy criterion that should probably be implemented in any client + is whether the message is "too big" for automatic retrieval, where + "too big" is a parameter defined in the client's configuration. + + Another commonly used criterion is the age of a message. For + example, the client may choose to download only messages received in + the last week (in this case, would be today's date minus 7 + days): + + tag3 UID SEARCH UID SINCE + + Keep in mind that a date search disregards time and time zone. The + client can avoid doing this search if it specified INTERNALDATE in + on the step described in 4.3.1. If the client did, it + can perform the local search on its message cache. + + + + +Melnikov Informational [Page 20] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + At this step, the client also decides what kind of information about + a particular message to fetch from the server. In particular, even + for a message that is considered "too big", the client MAY choose to + fetch some part(s) of it. For example, if the message is a + multipart/mixed containing a text part and a MPEG attachment, there + is no reason for the client not to fetch the text part. The decision + of which part should or should not be fetched can be based on the + information received in the BODYSTRUCTURE FETCH response data item + (i.e., if BODYSTRUCTURE was included in on the step + described in 4.3.1). + +4.3.3. Populating Cache with "Interesting" Messages. + + Once the client has found out which messages are "interesting", it + can start issuing appropriate FETCH commands for "interesting" + messages or parts thereof. + + Note that fetching a message into the disconnected client's local + cache does NOT imply that the human has (or even will) read the + message. Thus, the synchronization program for a disconnected client + should always be careful to use the .PEEK variants of the FETCH data + items that implicitly set the \Seen flag. + + Once the last descriptor has arrived and the last FETCH command has + been issued, the client simply needs to process the incoming fetch + items and use them to update the local message cache. + + In order to avoid deadlock problems, the client must give processing + of received messages priority over issuing new FETCH commands during + this synchronization process. This may necessitate temporary local + queuing of FETCH requests that cannot be issued without causing a + deadlock. In order to achieve the best use of the "expensive" + network connection, the client will almost certainly need to pay + careful attention to any flow-control information that it can obtain + from the underlying transport connection (usually a TCP connection). + + Note: The requirement stated in the previous paragraph might result + in an unpleasant user experience, if followed blindly. For example, + the user might be unwilling to wait for the client to finish + synchronization before starting to process the user's requests. A + smart disconnected client should allow the user to perform requested + operations in between IMAP commands that are part of the + synchronization process. See also Note 1 in Section 4.3.1. + + + + + + + + +Melnikov Informational [Page 21] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Example 8: + + After fetching a message BODYSTRUCTURE, the client discovers a + complex MIME message. Then, it decides to fetch MIME headers of the + nested MIME messages and some body parts. + + C: A011 UID fetch 11 (BODYSTRUCTURE) + S: ... + C: A012 UID fetch 11 (BODY[HEADER] BODY[1.MIME] BODY[1.1.MIME] + BODY[1.2.MIME] BODY[2.MIME] BODY[3.MIME] BODY[4.MIME] + BODY[5.MIME] BODY[6.MIME] BODY[7.MIME] BODY[8.MIME] BODY[9.MIME] + BODY[10.MIME] BODY[11.MIME] BODY[12.MIME] BODY[13.MIME] + BODY[14.MIME] BODY[15.MIME] BODY[16.MIME] BODY[17.MIME] + BODY[18.MIME] BODY[19.MIME] BODY[20.MIME] BODY[21.MIME]) + S: ... + C: A013 UID fetch 11 (BODY[1.1] BODY[1.2]) + S: ... + C: A014 UID fetch 11 (BODY[3] BODY[4] BODY[5] BODY[6] BODY[7] BODY[8] + BODY[9] BODY[10] BODY[11] BODY[13] BODY[14] BODY[15] BODY[16] + BODY[21]) + S: ... + +4.3.4. User-Initiated Synchronization + + After the client has finished the main synchronization process as + described in Sections 4.3.1-4.3.3, the user may optionally request + additional synchronization steps while the client is still online. + This is not any different from the process described in Sections + 4.3.2 and 4.3.3. + + Typical examples are: + + 1) fetch all messages selected in UI. + 2) fetch all messages marked as \Flagged on the server. + +4.4. Special Case: Descriptor-Only Synchronization + + For some mailboxes, fetching the descriptors might be the entire + synchronization step. Practical experience with IMAP has shown that + a certain class of mailboxes (e.g., "archival" mailboxes) are used + primarily for long-term storage of important messages that the human + wants to have instantly available on demand but does not want + cluttering up the disconnected client's cache at any other time. + Messages in this kind of mailbox would be fetched exclusively by + explicit actions queued by the local MUA. Thus, the only + synchronization desirable on this kind of mailbox is fetching enough + descriptor information for the user to be able to identify messages + for subsequent download. + + + +Melnikov Informational [Page 22] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Special mailboxes that receive messages from a high volume, low + priority mailing list might also be in this category, at least when + the human is in a hurry. + +4.5. Special Case: Fast New-Only Synchronization + + In some cases, the human might be in such a hurry that he or she + doesn't care about changes to old messages, just about new messages. + In this case, the client can skip the UID FETCH command that obtains + the flags and UIDs for old messages (1:). + +4.6. Special Case: Blind FETCH + + In some cases, the human may know (for whatever reason) that he or + she always wants to fetch any new messages in a particular mailbox, + unconditionally. In this case, the client can just fetch the + messages themselves, rather than just the descriptors, by using a + command like: + + tag1 UID FETCH :* (FLAGS BODY.PEEK[]) + + Note that this example ignores the fact that the messages can be + arbitrary long. The disconnected client MUST always check for + message size before downloading, unless explicitly told otherwise. A + well-behaved client should instead use something like the following: + + 1) Issue "tag1 UID FETCH :* (FLAGS RFC822.SIZE)". + + 2) From the message sizes returned in step 1, construct UID set + . + + 3) Issue "tag2 UID FETCH (BODY.PEEK[])". + + or + + 1) Issue "tag1 UID FETCH :* (FLAGS)". + + 2) Construct UID set from the responses of step 1. + + 3) Issue "tag2 SEARCH UID SMALLER ". + Construct UID set from the result of the + SEARCH command. + + 4) Issue "tag3 UID FETCH (BODY.PEEK[])". + + + + + + + +Melnikov Informational [Page 23] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + or + + 1) Issue "tag1 UID FETCH :* (FLAGS + BODY.PEEK[]<0.>)", where should be replaced with + the maximal message size the client is willing to download. + + Note: In response to such a command, the server will only return + partial data if the message is longer than . It will + return the full message data for any message whose size is smaller + than or equal to . In the former case, the client will + not be able to extract the full MIME structure of the message from + the truncated data, so the client should include BODYSTRUCTURE in + the UID FETCH command as well. + +5. Implementation Considerations + + Below are listed some common implementation pitfalls that should be + considered when implementing a disconnected client. + + 1) Implementing fake UIDs on the client. + + A message scheduled to be uploaded has no UID, as UIDs are + selected by the server. The client may implement fake UIDs + internally in order to reference not-yet-uploaded messages in + further operations. (For example, a message could be scheduled to + be uploaded, but subsequently marked as deleted or copied to + another mailbox). Here, the client MUST NOT under any + circumstances send these fake UIDs to the server. Also, client + implementers should be reminded that according to [IMAP4] a UID is + a 32-bit unsigned integer excluding 0. So, both 4294967295 and + 2147483648 are valid UIDs, and 0 and -1 are both invalid. Some + disconnected mail clients have been known to send negative numbers + (e.g., "-1") as message UIDs to servers during synchronization. + + Situation 1: The user starts composing a new message, edits it, + saves it, continues to edit it, and saves it again. + + A disconnected client may record in its replay log (log of + operations to be replayed on the server during synchronization) + the sequence of operations as shown below. For the purpose of + this situation, we assume that all draft messages are stored in + the mailbox called Drafts on an IMAP server. We will also use the + following conventions: is the UID of the intermediate + version of the draft when it was saved for the first time. This + is a fake UID generated on the client. is the UID of + the final version of the draft. This is another fake UID + generated on the client. + + + + +Melnikov Informational [Page 24] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 1) APPEND Drafts (\Seen $MDNSent \Drafts) {} + ...first version of the message follows... + + 2) APPEND Drafts (\Seen $MDNSent \Drafts) {} + ...final version of the message follows... + + 3) STORE +FLAGS (\Deleted) + + Step 1 corresponds to the first attempt to save the draft message, + step 2 corresponds to the second attempt to save the draft + message, and step 3 deletes the first version of the draft message + saved in step 1. + + A naive disconnected client may send the command in step 3 without + replacing the fake client generated with the value + returned by the server in step 1. A server will probably reject + this command, which will make the client believe that the + synchronization sequence has failed. + + 2) Section 5.1 discusses common implementation errors related to + error recovery during playback. + + 3) Don't assume that the disconnected client is the only client used + by the user. + + Situation 2: Some clients may use the \Deleted flag as an + indicator that the message should not appear in the user's view. + Usage of the \Deleted flag for this purpose is not safe, as other + clients (e.g., online clients) might EXPUNGE the mailbox at any + time. + + 4) Beware of data dependencies between synchronization operations. + + It might be very tempting for a client writer to perform some + optimizations on the playback log. Such optimizations might + include removing redundant operations (for example, see + optimization 2 in Section 5.3), or their reordering. + + It is not always safe to reorder or remove redundant operations + during synchronization because some operations may have + dependencies (as Situation 3 demonstrates). So, if in doubt, + don't do this. + + Situation 3: The user copies a message out of a mailbox and then + deletes the mailbox. + + + + + + +Melnikov Informational [Page 25] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + C: A001 SELECT Old-Mail + S: ... + C: A002 UID COPY 111 ToDo + S: A002 OK [COPYUID 1022843345 111 94] Copy completed + ... + C: A015 CLOSE + S: A015 OK Completed + C: A016 DELETE Old-Mail + S: A016 OK Mailbox deletion completed successfully + + If the client performs DELETE (tag A016) first and COPY (tag A002) + second, then the COPY fails. Also, the message that the user so + carefully copied into another mailbox has been lost. + +5.1. Error Recovery during Playback + + Error recovery during synchronization is one of the trickiest parts + to get right. Below, we will discuss certain error conditions and + suggest possible choices for handling them. + + 1) Lost connection to the server. + + The client MUST remember the current position in the playback + (replay) log and replay it starting from the interrupted operation + (the last command issued by the client, but not acknowledged by + the server) the next time it successfully connects to the same + server. If the connection was lost while executing a non- + idempotent IMAP command (see the definition in Section 1), then + when the client is reconnected, it MUST make sure that the + interrupted command was indeed not executed. If it wasn't + executed, the client must restart playback from the interrupted + command, otherwise from the following command. + + Upon reconnect, care must be taken in order to properly reapply + logical operations that are represented by multiple IMAP commands, + e.g., UID EXPUNGE emulation when UID EXPUNGE is not supported by + the server (see Section 4.2.4). + + Once the client detects that the connection to the server was + lost, it MUST stop replaying its log. There are existing + disconnected clients that, to the great annoyance of users, pop up + an error dialog for each and every playback operation that fails. + + 2) Copying/appending messages to a mailbox that doesn't exist. (The + server advertises this condition by sending the TRYCREATE response + code in the tagged NO response to the APPEND or COPY command.) + + + + + +Melnikov Informational [Page 26] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + The user should be advised about the situation and be given one of + the following choices: + + a) Try to recreate a mailbox. + b) Copy/upload messages to another mailbox. + c) Skip copy/upload. + d) Abort replay. + + 3) Copying messages from a mailbox that doesn't exist, or renaming or + getting/changing ACLs [ACL] on a mailbox that doesn't exist: + + a) Skip operation. + b) Abort replay. + + 4) Deleting mailboxes or deleting/expunging messages that no longer + exist. + + This is actually is not an error and should be ignored by the + client. + + 5) Performing operations on messages that no longer exist. + + a) Skip operation. + b) Abort replay. + + In the case of changing flags on an expunged message, the client + should silently ignore the error. + + Note 1: Several synchronization operations map to multiple IMAP + commands (for example, "move" described in 4.2.2). The client must + guarantee atomicity of each such multistep operation. For example, + when performing a "move" between two mailboxes on the same server, if + the server is unable to copy messages, the client MUST NOT attempt to + set the \Deleted flag on the messages being copied, let alone expunge + them. However, the client MAY consider that move operation to have + succeeded even if the server was unable to set the \Deleted flag on + copied messages. + + Note 2: Many synchronization operations have data dependencies. A + failed operation must cause all dependent operations to fail as well. + The client should check this and MUST NOT try to perform all + dependent operations blindly (unless the user corrected the original + problem). For example, a message may be scheduled to be appended to + a mailbox on the server and later on the appended message may be + copied to another mailbox. If the APPEND operation fails, the client + must not attempt to COPY the failed message later on. (See also + Section 5, Situation 3). + + + + +Melnikov Informational [Page 27] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +5.2. Quality of Implementation Issues + + Below, some quality of implementation issues are listed for + disconnected clients. They will help to write a disconnected client + that works correctly, performs synchronization as quickly as possible + (and thus can make the user happier as well as save her some money), + and minimizes the server load: + + 1) Don't lose information. + + No matter how smart your client is in other areas, if it loses + information, users will get very upset. + + 2) Don't do work unless explicitly asked. Be flexible. Ask all + questions BEFORE starting synchronization, if possible. + + 3) Minimize traffic. + + The client MUST NOT issue a command if the client already received + the required information from the server. + + The client MUST make use of UIDPLUS extension if it is supported + by the server. + + See also optimization 1 in Section 5.3. + + 4) Minimize the number of round-trips. + + Round-trips kill performance, especially on links with high + latency. Sections 4.2.2.5 and 5.2 give some advice on how to + minimize the number of round-trips. + + See also optimization 1 in Section 5.3. + +5.3. Optimizations + + Some useful optimizations are described in this section. A + disconnected client that supports the recommendations listed below + will give the user a more pleasant experience. + + 1) The initial OK or PREAUTH responses may contain the CAPABILITY + response code as described in Section 7.1 of [IMAP4]. This + response code gives the same information as returned by the + CAPABILITY command*. A disconnected client that pays attention to + this response code can avoid sending CAPABILITY command and will + save a round-trip. + + + + + +Melnikov Informational [Page 28] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + * Note: Some servers report in the CAPABILITY response code + extensions that are only relevant in unauthenticated state or in + all states. Such servers usually send another CAPABILITY + response code upon successful authentication using LOGIN or + AUTHENTICATE command (that negotiates no security layer; see + Section 6.2.2 of [IMAP4]). The CAPABILITY response code sent + upon successful LOGIN/AUTHENTICATE might be different from the + CAPABILITY response code in the initial OK response, as + extensions only relevant for unauthenticated state will not be + advertised, and some additional extensions available only in + authenticated and/or selected state will be. + + Example 9: + + S: * OK [CAPABILITY IMAP4REV1 LOGIN-REFERRALS STARTTLS + AUTH=DIGEST-MD5 AUTH=SRP] imap.example.com ready + C: 2 authenticate DIGEST-MD5 + S: 2 OK [CAPABILITY IMAP4REV1 IDLE NAMESPACE MAILBOX-REFERRALS SCAN + SORT THREAD=REFERENCES THREAD=ORDEREDSUBJECT MULTIAPPEND] + User authenticated (no layer) + + 2) An advanced disconnected client may choose to optimize its replay + log. For example, there might be some operations that are + redundant (the list is not complete): + + a) an EXPUNGE followed by another EXPUNGE or CLOSE; + b) changing flags (other than the \Deleted flag) on a message that + gets immediately expunged; + c) opening and closing the same mailbox. + + When optimizing, be careful about data dependencies between commands. + For example, if the client is wishing to optimize (see case b, above) + + tag1 UID STORE +FLAGS (\Deleted) + ... + tag2 UID STORE +FLAGS (\Flagged) + ... + tag3 UID COPY "Backup" + ... + tag4 UID EXPUNGE + + it can't remove the second UID STORE command because the message is + being copied before it gets expunged. + + In general, it might be a good idea to keep mailboxes open during + synchronization (see case c above), if possible. This can be more + easily achieved in conjunction with optimization 3 described below. + + + + +Melnikov Informational [Page 29] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 3) Perform some synchronization steps in parallel, if possible. + + Several synchronization steps don't depend on each other and thus + can be performed in parallel. Because the server machine is + usually more powerful than the client machine and can perform some + operations in parallel, this may speed up the total time of + synchronization. + + In order to achieve such parallelization, the client will have to + open more than one connection to the same server. Client writers + should not forget about non-trivial cost associated with + establishing a TCP connection and performing an authentication. + The disconnected client MUST NOT use one connection per mailbox. + In most cases, it is sufficient to have two connections. The + disconnected client SHOULD avoid selecting the same mailbox in + more than one connection; see Section 3.1.1 of [RFC2683] for more + details. + + Any mailbox synchronization MUST start with checking the + UIDVALIDITY as described in Section 4.1 of this document. The + client MAY use STATUS command to check UID Validity of a non- + selected mailbox. This is preferable to opening many connections + to the same server to perform synchronization of multiple + mailboxes simultaneously. As described in Section 5.3.10 of + [IMAP4], this SHOULD NOT be used on the selected mailbox. + +6. IMAP Extensions That May Help + + The following extensions can save traffic and/or the number of + round-trips: + + 1) The use of [UIDPLUS] is discussed in Sections 4.1, 4.2.1, 4.2.2.1 + and 4.2.4. + + 2) The use of the MULTIAPPEND and LITERAL+ extensions for uploading + messages is discussed in Section 4.2.2.5. + + 3) Use the CONDSTORE extension (see Section 6.1) for quick flag + resynchronization. + +6.1. CONDSTORE Extension + + An advanced disconnected mail client should use the [CONDSTORE] + extension when it is supported by the server. The client must cache + the value from HIGHESTMODSEQ OK response code received on mailbox + opening and update it whenever the server sends MODSEQ FETCH data + items. + + + + +Melnikov Informational [Page 30] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + If the client receives NOMODSEQ OK untagged response instead of + HIGHESTMODSEQ, it MUST remove the last known HIGHESTMODSEQ value from + its cache and follow the more general instructions in Section 3. + + When the client opens the mailbox for synchronization, it first + compares UIDVALIDITY as described in step d-1 in Section 3. If the + cached UIDVALIDITY value matches the one returned by the server, the + client MUST compare the cached value of HIGHESTMODSEQ with the one + returned by the server. If the cached HIGHESTMODSEQ value also + matches the one returned by the server, then the client MUST NOT + fetch flags for cached messages, as they hasn't changed. If the + value on the server is higher than the cached one, the client MAY use + "SEARCH MODSEQ " to find all messages with flags + changed since the last time the client was online and had the mailbox + opened. Alternatively, the client MAY use "FETCH 1:* (FLAGS) + (CHANGEDSINCE )". The latter operation combines + searching for changed messages and fetching new information. + + In all cases, the client still needs to fetch information about new + messages (if requested by the user) as well as discover which + messages have been expunged. + + Step d ("Server-to-client synchronization") in Section 4 in the + presence of the CONDSTORE extension is amended as follows: + + d) "Server-to-client synchronization" - For each mailbox that + requires synchronization, do the following: + + 1a) Check the mailbox UIDVALIDITY (see section 4.1 for more + details) with SELECT/EXAMINE/STATUS. + + If the UIDVALIDITY value returned by the server differs, the + client MUST + + * empty the local cache of that mailbox; + * "forget" the cached HIGHESTMODSEQ value for the mailbox; + * remove any pending "actions" that refer to UIDs in that + mailbox (note that this doesn't affect actions performed on + client-generated fake UIDs; see Section 5); and + * skip steps 1b and 2-II; + + 1b) Check the mailbox HIGHESTMODSEQ. If the cached value is the + same as the one returned by the server, skip fetching message + flags on step 2-II, i.e., the client only has to find out + which messages got expunged. + + + + + + +Melnikov Informational [Page 31] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 2) Fetch the current "descriptors". + + I) Discover new messages. + + II) Discover changes to old messages and flags for new messages + using + "FETCH 1:* (FLAGS) (CHANGEDSINCE )" or + "SEARCH MODSEQ ". + + Discover expunged messages; for example, using + "UID SEARCH 1:". (All messages not returned + in this command are expunged.) + + 3) Fetch the bodies of any "interesting" messages that the client + doesn't already have. + + Example 10: + + The UIDVALIDITY value is the same, but the HIGHESTMODSEQ value + has changed on the server while the client was offline. + + C: A142 SELECT INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 201] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [HIGHESTMODSEQ 20010715194045007] + S: A142 OK [READ-WRITE] SELECT completed + + After that, either: + + C: A143 UID FETCH 1:* (FLAGS) (CHANGEDSINCE 20010715194032001) + S: * 2 FETCH (UID 6 MODSEQ (20010715205008000) FLAGS (\Deleted)) + S: * 5 FETCH (UID 9 MODSEQ (20010715195517000) FLAGS ($NoJunk + $AutoJunk $MDNSent)) + ... + S: A143 OK FETCH completed + + or: + + + + + + + + + +Melnikov Informational [Page 32] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + C: A143 UID SEARCH MODSEQ 20010715194032001 UID 1:20 + S: * SEARCH 6 9 11 12 18 19 20 23 (MODSEQ 20010917162500) + S: A143 OK Search complete + C: A144 UID SEARCH 1:20 + S: * SEARCH 6 9 ... + S: A144 OK FETCH completed + +7. Security Considerations + + It is believed that this document does not raise any new security + concerns that are not already present in the base [IMAP4] protocol, + and these issues are discussed in [IMAP4]. Additional security + considerations may be found in different extensions mentioned in this + document; in particular, in [UIDPLUS], [LITERAL+], [CONDSTORE], + [MULTIAPPEND], and [UNSELECT]. + + Implementers are also reminded about the importance of thorough + testing. + +8. References + +8.1. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - + VERSION 4rev1", RFC 3501, March 2003. + + [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - + UIDPLUS extension", RFC 4315, December 2005. + + [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC + 2088, January 1997. + + [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for + Conditional STORE Operation or Quick Flag Changes + Resynchronization", RFC 4551, June 2006. + + [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + + [UNSELECT] Melnikov, A., "Internet Message Access Protocol (IMAP) + UNSELECT command", RFC 3691, February 2004. + + [RFC2683] Leiba, B., "IMAP4 Implementation Recommendations", RFC + 2683, September 1999. + + + + +Melnikov Informational [Page 33] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +8.2. Informative References + + [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) + Extension", RFC 4314, December 2005. + + [IMAP-MODEL] Crispin, M., "Distributed Electronic Mail Models in + IMAP4", RFC 1733, December 1994. + +9. Acknowledgements + + This document is based on version 01 of the text written by Rob + Austein in November 1994. + + The editor appreciates comments posted by Mark Crispin to the IMAP + mailing list and the comments/corrections/ideas received from Grant + Baillie, Cyrus Daboo, John G. Myers, Chris Newman, and Timo Sirainen. + + The editor would also like to thank the developers of Netscape + Messenger and Mozilla mail clients for providing examples of + disconnected mail clients that served as a base for many + recommendations in this document. + +Editor's Address + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex + TW12 2BX + United Kingdom + + Phone: +44 77 53759732 + EMail: alexey.melnikov@isode.com + + + + + + + + + + + + + + + + + +Melnikov Informational [Page 34] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Melnikov Informational [Page 35] + diff --git a/docs/rfcs/rfc4551.txt b/docs/rfcs/rfc4551.txt new file mode 100644 index 0000000..894b510 --- /dev/null +++ b/docs/rfcs/rfc4551.txt @@ -0,0 +1,1403 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 4551 Isode Ltd. +Updates: 3501 S. Hole +Category: Standards Track ACI WorldWide/MessagingDirect + June 2006 + + + IMAP Extension for Conditional STORE Operation + or Quick Flag Changes Resynchronization + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + Often, multiple IMAP (RFC 3501) clients need to coordinate changes to + a common IMAP mailbox. Examples include different clients working on + behalf of the same user, and multiple users accessing shared + mailboxes. These clients need a mechanism to synchronize state + changes for messages within the mailbox. They must be able to + guarantee that only one client can change message state (e.g., + message flags) at any time. An example of such an application is use + of an IMAP mailbox as a message queue with multiple dequeueing + clients. + + The Conditional Store facility provides a protected update mechanism + for message state information that can detect and resolve conflicts + between multiple writing mail clients. + + The Conditional Store facility also allows a client to quickly + resynchronize mailbox flag changes. + + This document defines an extension to IMAP (RFC 3501). + + + + + + + + + +Melnikov & Hole Standards Track [Page 1] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +Table of Contents + + 1. Introduction and Overview ................................. 3 + 2. Conventions Used in This Document ......................... 5 + 3. IMAP Protocol Changes ..................................... 6 + 3.1. New OK untagged responses for SELECT and EXAMINE ......... 6 + 3.1.1. HIGHESTMODSEQ response code ............................ 6 + 3.1.2. NOMODSEQ response code ................................. 7 + 3.2. STORE and UID STORE Commands ............................. 7 + 3.3 FETCH and UID FETCH Commands ..............................13 + 3.3.1. CHANGEDSINCE FETCH modifier ............................13 + 3.3.2. MODSEQ message data item in FETCH Command ..............14 + 3.4. MODSEQ search criterion in SEARCH ........................16 + 3.5. Modified SEARCH untagged response ........................17 + 3.6. HIGHESTMODSEQ status data items ..........................17 + 3.7. CONDSTORE parameter to SELECT and EXAMINE ................18 + 3.8. Additional quality of implementation issues ..............18 + 4. Formal Syntax .............................................19 + 5. Server implementation considerations ......................21 + 6. Security Considerations ...................................22 + 7. IANA Considerations .......................................22 + 8. References ................................................23 + 8.1. Normative References .....................................23 + 8.2. Informative References ...................................23 + 9. Acknowledgements ..........................................23 + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Hole Standards Track [Page 2] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +1. Introduction and Overview + + The Conditional STORE extension is present in any IMAP4 + implementation that returns "CONDSTORE" as one of the supported + capabilities in the CAPABILITY command response. + + An IMAP server that supports this extension MUST associate a positive + unsigned 64-bit value called a modification sequence (mod-sequence) + with every IMAP message. This is an opaque value updated by the + server whenever a metadata item is modified. The server MUST + guarantee that each STORE command performed on the same mailbox + (including simultaneous stores to different metadata items from + different connections) will get a different mod-sequence value. + Also, for any two successful STORE operations performed in the same + session on the same mailbox, the mod-sequence of the second completed + operation MUST be greater than the mod-sequence of the first + completed. Note that the latter rule disallows the use of the system + clock as a mod-sequence, because if system time changes (e.g., an NTP + [NTP] client adjusting the time), the next generated value might be + less than the previous one. + + Mod-sequences allow a client that supports the CONDSTORE extension to + determine if a message metadata has changed since some known moment. + Whenever the state of a flag changes (i.e., the flag is added where + previously it wasn't set, or the flag is removed and before it was + set) the value of the modification sequence for the message MUST be + updated. Adding the flag when it is already present or removing when + it is not present SHOULD NOT change the mod-sequence. + + When a message is appended to a mailbox (via the IMAP APPEND command, + COPY to the mailbox, or using an external mechanism) the server + generates a new modification sequence that is higher than the highest + modification sequence of all messages in the mailbox and assigns it + to the appended message. + + The server MAY store separate (per-message) modification sequence + values for different metadata items. If the server does so, per- + message mod-sequence is the highest mod-sequence of all metadata + items for the specified message. + + The server that supports this extension is not required to be able to + store mod-sequences for every available mailbox. Section 3.1.2 + describes how the server may act if a particular mailbox doesn't + support the persistent storage of mod-sequences. + + + + + + + +Melnikov & Hole Standards Track [Page 3] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + This extension makes the following changes to the IMAP4 protocol: + + a) adds UNCHANGEDSINCE STORE modifier. + + b) adds the MODIFIED response code which should be used with an OK + response to the STORE command. (It can also be used in a NO + response.) + + c) adds a new MODSEQ message data item for use with the FETCH + command. + + d) adds CHANGEDSINCE FETCH modifier. + + e) adds a new MODSEQ search criterion. + + f) extends the syntax of untagged SEARCH responses to include + mod-sequence. + + g) adds new OK untagged responses for the SELECT and EXAMINE + commands. + + h) defines an additional parameter to SELECT/EXAMINE commands. + + i) adds the HIGHESTMODSEQ status data item to the STATUS command. + + A client supporting CONDSTORE extension indicates its willingness to + receive mod-sequence updates in all untagged FETCH responses by + issuing: + + - a SELECT or EXAMINE command with the CONDSTORE parameter, + - a STATUS (HIGHESTMODSEQ) command, + - a FETCH or SEARCH command that includes the MODSEQ message data + item, + - a FETCH command with the CHANGEDSINCE modifier, or + - a STORE command with the UNCHANGEDSINCE modifier. + + The server MUST include mod-sequence data in all subsequent untagged + FETCH responses (until the connection is closed), whether they were + caused by a regular STORE, a STORE with UNCHANGEDSINCE modifier, or + an external agent. + + This document uses the term "CONDSTORE-aware client" to refer to a + client that announces its willingness to receive mod-sequence updates + as described above. The term "CONDSTORE enabling command" will refer + any of the commands listed above. A future extension to this + document may extend the list of CONDSTORE enabling commands. A first + CONDSTORE enabling command executed in the session MUST cause the + + + + +Melnikov & Hole Standards Track [Page 4] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + server to return HIGHESTMODSEQ (Section 3.1.1) unless the server has + sent NOMODSEQ (Section 3.1.2) response code when the currently + selected mailbox was selected. + + The rest of this document describes the protocol changes more + rigorously. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + + In examples, lines beginning with "S:" are sent by the IMAP server, + and lines beginning with "C:" are sent by the client. Line breaks + may appear in example commands solely for editorial clarity; when + present in the actual message, they are represented by "CRLF". + + Formal syntax is defined using ABNF [ABNF]. + + The term "metadata" or "metadata item" is used throughout this + document. It refers to any system or user-defined keyword. Future + documents may extend "metadata" to include other dynamic message + data. + + Some IMAP mailboxes are private, accessible only to the owning user. + Other mailboxes are not, either because the owner has set an Access + Control List [ACL] that permits access by other users, or because it + is a shared mailbox. Let's call a metadata item "shared" for the + mailbox if any changes to the metadata items are persistent and + visible to all other users accessing the mailbox. Otherwise, the + metadata item is called "private". Note that private metadata items + are still visible to all sessions accessing the mailbox as the same + user. Also note that different mailboxes may have different metadata + items as shared. + + See Section 1 for the definition of a "CONDSTORE-aware client" and a + "CONDSTORE enabling command". + + + + + + + + + + + + + +Melnikov & Hole Standards Track [Page 5] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +3. IMAP Protocol Changes + +3.1. New OK Untagged Responses for SELECT and EXAMINE + + This document adds two new response codes, HIGHESTMODSEQ and + NOMODSEQ. One of those response codes MUST be returned in the OK + untagged response for a successful SELECT/EXAMINE command. + + When opening a mailbox, the server must check if the mailbox supports + the persistent storage of mod-sequences. If the mailbox supports the + persistent storage of mod-sequences and the mailbox open operation + succeeds, the server MUST send the OK untagged response including + HIGHESTMODSEQ response code. If the persistent storage for the + mailbox is not supported, the server MUST send the OK untagged + response including NOMODSEQ response code instead. + +3.1.1. HIGHESTMODSEQ Response Code + + This document adds a new response code that is returned in the OK + untagged response for the SELECT and EXAMINE commands. A server + supporting the persistent storage of mod-sequences for the mailbox + MUST send the OK untagged response including HIGHESTMODSEQ response + code with every successful SELECT or EXAMINE command: + + OK [HIGHESTMODSEQ ] + + where is the highest mod-sequence value of + all messages in the mailbox. When the server changes UIDVALIDITY + for a mailbox, it doesn't have to keep the same HIGHESTMODSEQ for + the mailbox. + + A disconnected client can use the value of HIGHESTMODSEQ to check if + it has to refetch metadata from the server. If the UIDVALIDITY value + has changed for the selected mailbox, the client MUST delete the + cached value of HIGHESTMODSEQ. If UIDVALIDITY for the mailbox is the + same, and if the HIGHESTMODSEQ value stored in the client's cache is + less than the value returned by the server, then some metadata items + on the server have changed since the last synchronization, and the + client needs to update its cache. The client MAY use SEARCH MODSEQ + (Section 3.4) to find out exactly which metadata items have changed. + Alternatively, the client MAY issue FETCH with the CHANGEDSINCE + modifier (Section 3.3.1) in order to fetch data for all messages that + have metadata items changed since some known modification sequence. + + Example 1: + + C: A142 SELECT INBOX + S: * 172 EXISTS + + + +Melnikov & Hole Standards Track [Page 6] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [HIGHESTMODSEQ 715194045007] + S: A142 OK [READ-WRITE] SELECT completed + +3.1.2. NOMODSEQ Response Code + + A server that doesn't support the persistent storage of mod-sequences + for the mailbox MUST send the OK untagged response including NOMODSEQ + response code with every successful SELECT or EXAMINE command. A + server that returned NOMODSEQ response code for a mailbox, which + subsequently receives one of the following commands while the mailbox + is selected: + + - a FETCH command with the CHANGEDSINCE modifier, + - a FETCH or SEARCH command that includes the MODSEQ message data + item, or + - a STORE command with the UNCHANGEDSINCE modifier + + MUST reject any such command with the tagged BAD response. + + Example 2: + + C: A142 SELECT INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [NOMODSEQ] Sorry, this mailbox format doesn't support + modsequences + S: A142 OK [READ-WRITE] SELECT completed + +3.2. STORE and UID STORE Commands + + This document defines the following STORE modifier (see Section 2.5 + of [IMAPABNF]): + + UNCHANGEDSINCE + + For each message specified in the message set, the server performs + the following. If the mod-sequence of any metadata item of the + + + +Melnikov & Hole Standards Track [Page 7] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + message is equal or less than the specified UNCHANGEDSINCE value, + then the requested operation (as described by the message data + item) is performed. If the operation is successful, the server + MUST update the mod-sequence attribute of the message. An + untagged FETCH response MUST be sent, even if the .SILENT suffix + is specified, and the response MUST include the MODSEQ message + data item. This is required to update the client's cache with the + correct mod-sequence values. See Section 3.3.2 for more details. + + However, if the mod-sequence of any metadata item of the message + is greater than the specified UNCHANGEDSINCE value, then the + requested operation MUST NOT be performed. In this case, the + mod-sequence attribute of the message is not updated, and the + message number (or unique identifier in the case of the UID STORE + command) is added to the list of messages that failed the + UNCHANGESINCE test. + + When the server finished performing the operation on all the + messages in the message set, it checks for a non-empty list of + messages that failed the UNCHANGESINCE test. If this list is + non-empty, the server MUST return in the tagged response a + MODIFIED response code. The MODIFIED response code includes the + message set (for STORE) or set of UIDs (for UID STORE) of all + messages that failed the UNCHANGESINCE test. + + Example 3: + + All messages pass the UNCHANGESINCE test. + + C: a103 UID STORE 6,4,8 (UNCHANGEDSINCE 12121230045) + +FLAGS.SILENT (\Deleted) + S: * 1 FETCH (UID 4 MODSEQ (12121231000)) + S: * 2 FETCH (UID 6 MODSEQ (12121230852)) + S: * 4 FETCH (UID 8 MODSEQ (12121130956)) + S: a103 OK Conditional Store completed + + Example 4: + + C: a104 STORE * (UNCHANGEDSINCE 12121230045) +FLAGS.SILENT + (\Deleted $Processed) + S: * 50 FETCH (MODSEQ (12111230047)) + S: a104 OK Store (conditional) completed + + Example 5: + + C: c101 STORE 1 (UNCHANGEDSINCE 12121230045) -FLAGS.SILENT + (\Deleted) + S: * OK [HIGHESTMODSEQ 12111230047] + + + +Melnikov & Hole Standards Track [Page 8] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + S: * 50 FETCH (MODSEQ (12111230048)) + S: c101 OK Store (conditional) completed + + HIGHESTMODSEQ response code was sent by the server presumably + because this was the first CONDSTORE enabling command. + + Example 6: + + In spite of the failure of the conditional STORE operation for + message 7, the server continues to process the conditional STORE + in order to find all messages that fail the test. + + C: d105 STORE 7,5,9 (UNCHANGEDSINCE 320162338) + +FLAGS.SILENT (\Deleted) + S: * 5 FETCH (MODSEQ (320162350)) + S: d105 OK [MODIFIED 7,9] Conditional STORE failed + + Example 7: + + Same as above, but the server follows the SHOULD recommendation in + Section 6.4.6 of [IMAP4]. + + C: d105 STORE 7,5,9 (UNCHANGEDSINCE 320162338) + +FLAGS.SILENT (\Deleted) + S: * 7 FETCH (MODSEQ (320162342) FLAGS (\Seen \Deleted)) + S: * 5 FETCH (MODSEQ (320162350)) + S: * 9 FETCH (MODSEQ (320162349) FLAGS (\Answered)) + S: d105 OK [MODIFIED 7,9] Conditional STORE failed + + Use of UNCHANGEDSINCE with a modification sequence of 0 always + fails if the metadata item exists. A system flag MUST always be + considered existent, whether it was set or not. + + Example 8: + + C: a102 STORE 12 (UNCHANGEDSINCE 0) + +FLAGS.SILENT ($MDNSent) + S: a102 OK [MODIFIED 12] Conditional STORE failed + + The client has tested the presence of the $MDNSent user-defined + keyword. + + Note: A client trying to make an atomic change to the state of a + particular metadata item (or a set of metadata items) should be + prepared to deal with the case when the server returns the MODIFIED + response code if the state of the metadata item being watched hasn't + changed (but the state of some other metadata item has). This is + necessary, because some servers don't store separate mod-sequences + + + +Melnikov & Hole Standards Track [Page 9] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + for different metadata items. However, a server implementation + SHOULD avoid generating spurious MODIFIED responses for +FLAGS/-FLAGS + STORE operations, even when the server stores a single mod-sequence + per message. Section 5 describes how this can be achieved. + + Unless the server has included an unsolicited FETCH to update + client's knowledge about messages that have failed the UNCHANGEDSINCE + test, upon receipt of the MODIFIED response code, the client SHOULD + try to figure out if the required metadata items have indeed changed + by issuing FETCH or NOOP command. It is RECOMMENDED that the server + avoids the need for the client to do that by sending an unsolicited + FETCH response (Examples 9 and 10). + + If the required metadata items haven't changed, the client SHOULD + retry the command with the new mod-sequence. The client SHOULD allow + for a configurable but reasonable number of retries (at least 2). + + Example 9: + + In the example below, the server returns the MODIFIED response + code without sending information describing why the STORE + UNCHANGEDSINCE operation has failed. + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + S: * 100 FETCH (MODSEQ (303181230852)) + S: * 102 FETCH (MODSEQ (303181230852)) + ... + S: * 150 FETCH (MODSEQ (303181230852)) + S: a106 OK [MODIFIED 101] Conditional STORE failed + + The flag $Processed was set on the message 101... + + C: a107 NOOP + S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed)) + S: a107 OK + + Or the flag hasn't changed, but another has (note that this server + behaviour is discouraged. Server implementers should also see + Section 5)... + + C: b107 NOOP + S: * 101 FETCH (MODSEQ (303011130956) FLAGS (\Deleted \Answered)) + S: b107 OK + + ...and the client retries the operation for the message 101 with + the updated UNCHANGEDSINCE value + + + + +Melnikov & Hole Standards Track [Page 10] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + C: b108 STORE 101 (UNCHANGEDSINCE 303011130956) + +FLAGS.SILENT ($Processed) + S: * 101 FETCH (MODSEQ (303181230852)) + S: b108 OK Conditional Store completed + + Example 10: + + Same as above, but the server avoids the need for the client to + poll for changes. + + The flag $Processed was set on the message 101 by another + client... + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + S: * 100 FETCH (MODSEQ (303181230852)) + S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed)) + S: * 102 FETCH (MODSEQ (303181230852)) + ... + S: * 150 FETCH (MODSEQ (303181230852)) + S: a106 OK [MODIFIED 101] Conditional STORE failed + + Or the flag hasn't changed, but another has (note that this server + behaviour is discouraged. Server implementers should also see + Section 5)... + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + S: * 100 FETCH (MODSEQ (303181230852)) + S: * 101 FETCH (MODSEQ (303011130956) FLAGS (\Deleted \Answered)) + S: * 102 FETCH (MODSEQ (303181230852)) + ... + S: * 150 FETCH (MODSEQ (303181230852)) + S: a106 OK [MODIFIED 101] Conditional STORE failed + + ...and the client retries the operation for the message 101 with + the updated UNCHANGEDSINCE value + + C: b108 STORE 101 (UNCHANGEDSINCE 303011130956) + +FLAGS.SILENT ($Processed) + S: * 101 FETCH (MODSEQ (303181230852)) + S: b108 OK Conditional Store completed + + Or the flag hasn't changed, but another has (nice server + behaviour. Server implementers should also see Section 5)... + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + + + +Melnikov & Hole Standards Track [Page 11] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + S: * 100 FETCH (MODSEQ (303181230852)) + S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed \Deleted + \Answered)) + S: * 102 FETCH (MODSEQ (303181230852)) + ... + S: * 150 FETCH (MODSEQ (303181230852)) + S: a106 OK Conditional STORE completed + + Example 11: + + The following example is based on the example from the Section + 4.2.3 of [RFC-2180] and demonstrates that the MODIFIED response + code may be also returned in the tagged NO response. + + Client tries to conditionally STORE flags on a mixture of expunged + and non-expunged messages; one message fails the UNCHANGEDSINCE + test. + + C: B001 STORE 1:7 (UNCHANGEDSINCE 320172338) +FLAGS (\SEEN) + S: * 1 FETCH (MODSEQ (320172342) FLAGS (\SEEN)) + S: * 3 FETCH (MODSEQ (320172342) FLAGS (\SEEN)) + S: B001 NO [MODIFIED 2] Some of the messages no longer exist. + + C: B002 NOOP + S: * 4 EXPUNGE + S: * 4 EXPUNGE + S: * 4 EXPUNGE + S: * 4 EXPUNGE + S: * 2 FETCH (MODSEQ (320172340) FLAGS (\Deleted \Answered)) + S: B002 OK NOOP Completed. + + By receiving FETCH responses for messages 1 and 3, and EXPUNGE + responses that indicate that messages 4 through 7 have been + expunged, the client retries the operation only for the message 2. + The updated UNCHANGEDSINCE value is used. + + C: b003 STORE 2 (UNCHANGEDSINCE 320172340) +FLAGS (\Seen) + S: * 2 FETCH (MODSEQ (320180050)) + S: b003 OK Conditional Store completed + + Note: If a message is specified multiple times in the message set, + and the server doesn't internally eliminate duplicates from the + message set, it MUST NOT fail the conditional STORE operation for the + second (or subsequent) occurrence of the message if the operation + completed successfully for the first occurrence. For example, if the + client specifies: + + + + + +Melnikov & Hole Standards Track [Page 12] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + e105 STORE 7,3:9 (UNCHANGEDSINCE 12121230045) + +FLAGS.SILENT (\Deleted) + + the server must not fail the operation for message 7 as part of + processing "3:9" if it succeeded when message 7 was processed the + first time. + + Once the client specified the UNCHANGEDSINCE modifier in a STORE + command, the server MUST include the MODSEQ fetch response data items + in all subsequent unsolicited FETCH responses. + + This document also changes the behaviour of the server when it has + performed a STORE or UID STORE command and the UNCHANGEDSINCE + modifier is not specified. If the operation is successful for a + message, the server MUST update the mod-sequence attribute of the + message. The server is REQUIRED to include the mod-sequence value + whenever it decides to send the unsolicited FETCH response to all + CONDSTORE-aware clients that have opened the mailbox containing the + message. + + Server implementers should also see Section 3.8 for additional + quality of implementation issues related to the STORE command. + +3.3. FETCH and UID FETCH Commands + +3.3.1. CHANGEDSINCE FETCH Modifier + + This document defines the following FETCH modifier (see Section 2.4 + of [IMAPABNF]): + + CHANGEDSINCE + + CHANGEDSINCE FETCH modifier allows to create a further subset of + the list of messages described by sequence set. The information + described by message data items is only returned for messages that + have mod-sequence bigger than . + + When CHANGEDSINCE FETCH modifier is specified, it implicitly adds + MODSEQ FETCH message data item (Section 3.3.2). + + Example 12: + + C: s100 UID FETCH 1:* (FLAGS) (CHANGEDSINCE 12345) + S: * 1 FETCH (UID 4 MODSEQ (65402) FLAGS (\Seen)) + S: * 2 FETCH (UID 6 MODSEQ (75403) FLAGS (\Deleted)) + S: * 4 FETCH (UID 8 MODSEQ (29738) FLAGS ($NoJunk $AutoJunk + $MDNSent)) + S: s100 OK FETCH completed + + + +Melnikov & Hole Standards Track [Page 13] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +3.3.2. MODSEQ Message Data Item in FETCH Command + + This extension adds a MODSEQ message data item to the FETCH command. + The MODSEQ message data item allows clients to retrieve mod-sequence + values for a range of messages in the currently selected mailbox. + + Once the client specified the MODSEQ message data item in a FETCH + request, the server MUST include the MODSEQ fetch response data items + in all subsequent unsolicited FETCH responses. + + Syntax: MODSEQ + + The MODSEQ message data item causes the server to return MODSEQ + fetch response data items. + + Syntax: MODSEQ ( ) + + MODSEQ response data items contain per-message mod-sequences. + + The MODSEQ response data item is returned if the client issued + FETCH with MODSEQ message data item. It also allows the server to + notify the client about mod-sequence changes caused by conditional + STOREs (Section 3.2) and/or changes caused by external sources. + + Example 13: + + C: a FETCH 1:3 (MODSEQ) + S: * 1 FETCH (MODSEQ (624140003)) + S: * 2 FETCH (MODSEQ (624140007)) + S: * 3 FETCH (MODSEQ (624140005)) + S: a OK Fetch complete + + In this example, the client requests per-message mod-sequences for + a set of messages. + + When a flag for a message is modified in a different session, the + server sends an unsolicited FETCH response containing the mod- + sequence for the message. + + Example 14: + + (Session 1, authenticated as a user "alex"). The user adds a + shared flag \Deleted: + + C: A142 SELECT INBOX + ... + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Answered \Deleted \Seen \*)] Limited + + + +Melnikov & Hole Standards Track [Page 14] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + ... + + C: A160 STORE 7 +FLAGS.SILENT (\Deleted) + S: * 7 FETCH (MODSEQ (2121231000)) + S: A160 OK Store completed + + (Session 2, also authenticated as the user "alex"). Any changes + to flags are always reported to all sessions authenticated as the + same user as in the session 1. + + C: C180 NOOP + S: * 7 FETCH (FLAGS (\Deleted \Answered) MODSEQ (12121231000)) + S: C180 OK Noop completed + + (Session 3, authenticated as a user "andrew"). As \Deleted is a + shared flag, changes in session 1 are also reported in session 3: + + C: D210 NOOP + S: * 7 FETCH (FLAGS (\Deleted \Answered) MODSEQ (12121231000)) + S: D210 OK Noop completed + + The user modifies a private flag \Seen in session 1... + + C: A240 STORE 7 +FLAGS.SILENT (\Seen) + S: * 7 FETCH (MODSEQ (12121231777)) + S: A240 OK Store completed + + ...which is only reported in session 2... + + C: C270 NOOP + S: * 7 FETCH (FLAGS (\Deleted \Answered \Seen) MODSEQ + (12121231777)) + S: C270 OK Noop completed + + ...but not in session 3. + + C: D300 NOOP + S: D300 OK Noop completed + + And finally, the user removes flags \Answered (shared) and \Seen + (private) in session 1. + + C: A330 STORE 7 -FLAGS.SILENT (\Answered \Seen) + S: * 7 FETCH (MODSEQ (12121245160)) + S: A330 OK Store completed + + + + + + +Melnikov & Hole Standards Track [Page 15] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + Both changes are reported in the session 2... + + C: C360 NOOP + S: * 7 FETCH (FLAGS (\Deleted) MODSEQ (12121245160)) + S: C360 OK Noop completed + + ...and only changes to shared flags are reported in session 3. + + C: D390 NOOP + S: * 7 FETCH (FLAGS (\Deleted) MODSEQ (12121245160)) + S: D390 OK Noop completed + + Server implementers should also see Section 3.8 for additional + quality of implementation issues related to the FETCH command. + +3.4. MODSEQ Search Criterion in SEARCH + + The MODSEQ criterion for the SEARCH command allows a client to search + for the metadata items that were modified since a specified moment. + + Syntax: MODSEQ [ ] + + Messages that have modification values that are equal to or + greater than . This allows a client, for + example, to find out which messages contain metadata items that + have changed since the last time it updated its disconnected + cache. The client may also specify (name of metadata + item) and (type of metadata item) before + . can be one of "shared", + "priv" (private), or "all". The latter means that the server + should use the biggest value among "priv" and "shared" mod- + sequences for the metadata item. If the server doesn't store + internally separate mod-sequences for different metadata items, it + MUST ignore and . Otherwise, the + server should use them to narrow down the search. + + For a flag , the corresponding has a form + "/flags/" as defined in [IMAPABNF]. Note that the + leading "\" character that denotes a system flag has to be escaped + as per Section 4.3 of [IMAP4], as the uses syntax for + quoted strings. + + If client specifies a MODSEQ criterion in a SEARCH command and the + server returns a non-empty SEARCH result, the server MUST also append + (to the end of the untagged SEARCH response) the highest mod-sequence + for all messages being returned. See also Section 3.5. + + + + + +Melnikov & Hole Standards Track [Page 16] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + Example 15: + + C: a SEARCH MODSEQ "/flags/\\draft" all 620162338 + S: * SEARCH 2 5 6 7 11 12 18 19 20 23 (MODSEQ 917162500) + S: a OK Search complete + + In the above example, the message numbers of any messages + containing the string "IMAP4" in the "value" attribute of the + "/comment" entry and having a mod-sequence equal to or greater + than 620162338 for the "\Draft" flag are returned in the search + results. + + Example 16: + + C: t SEARCH OR NOT MODSEQ 720162338 LARGER 50000 + S: * SEARCH + S: t OK Search complete, nothing found + +3.5. Modified SEARCH Untagged Response + + Data: zero or more numbers + mod-sequence value (omitted if no match) + + This document extends syntax of the untagged SEARCH response to + include the highest mod-sequence for all messages being returned. + + If a client specifies a MODSEQ criterion in a SEARCH (or UID SEARCH) + command and the server returns a non-empty SEARCH result, the server + MUST also append (to the end of the untagged SEARCH response) the + highest mod-sequence for all messages being returned. See Section + 3.4 for examples. + +3.6. HIGHESTMODSEQ Status Data Items + + This document defines a new status data item: + + HIGHESTMODSEQ + + The highest mod-sequence value of all messages in the mailbox. + This is the same value that is returned by the server in the + HIGHESTMODSEQ response code in an OK untagged response (see + Section 3.1.1). If the server doesn't support the persistent + storage of mod-sequences for the mailbox (see Section 3.1.2), the + server MUST return 0 as the value of HIGHESTMODSEQ status data + item. + + + + + + +Melnikov & Hole Standards Track [Page 17] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + Example 17: + + C: A042 STATUS blurdybloop (UIDNEXT MESSAGES HIGHESTMODSEQ) + S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292 + HIGHESTMODSEQ 7011231777) + S: A042 OK STATUS completed + +3.7. CONDSTORE Parameter to SELECT and EXAMINE + + The CONDSTORE extension defines a single optional select parameter, + "CONDSTORE", which tells the server that it MUST include the MODSEQ + fetch response data items in all subsequent unsolicited FETCH + responses. + + The CONDSTORE parameter to SELECT/EXAMINE helps avoid a race + condition that might arise when one or more metadata items are + modified in another session after the server has sent the + HIGHESTMODSEQ response code and before the client was able to issue a + CONDSTORE enabling command. + + Example 18: + + C: A142 SELECT INBOX (CONDSTORE) + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [HIGHESTMODSEQ 715194045007] + S: A142 OK [READ-WRITE] SELECT completed, CONDSTORE is now enabled + +3.8. Additional Quality-of-Implementation Issues + + Server implementations should follow the following rule, which + applies to any successfully completed STORE/UID STORE (with and + without UNCHANGEDSINCE modifier), as well as to a FETCH command that + implicitly sets \Seen flag: + + Adding the flag when it is already present or removing when it is + not present SHOULD NOT change the mod-sequence. + + This will prevent spurious client synchronization requests. + + + + + + + +Melnikov & Hole Standards Track [Page 18] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + However, note that client implementers MUST NOT rely on this server + behavior. A client can't distinguish between the case when a server + has violated the SHOULD mentioned above, and that when one or more + clients set and unset (or unset and set) the flag in another session. + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) [ABNF] notation. Elements not defined here can be found + in the formal syntax of the ABNF [ABNF], IMAP [IMAP4], and IMAP ABNF + extensions [IMAPABNF] specifications. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper- or lowercase characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + capability =/ "CONDSTORE" + + status-att =/ "HIGHESTMODSEQ" + ;; extends non-terminal defined in RFC 3501. + + status-att-val =/ "HIGHESTMODSEQ" SP mod-sequence-valzer + ;; extends non-terminal defined in [IMAPABNF]. + ;; Value 0 denotes that the mailbox doesn't + ;; support persistent mod-sequences + ;; as described in Section 3.1.2 + + store-modifier =/ "UNCHANGEDSINCE" SP mod-sequence-valzer + ;; Only a single "UNCHANGEDSINCE" may be + ;; specified in a STORE operation + + fetch-modifier =/ chgsince-fetch-mod + ;; conforms to the generic "fetch-modifier" + ;; syntax defined in [IMAPABNF]. + + chgsince-fetch-mod = "CHANGEDSINCE" SP mod-sequence-value + ;; CHANGEDSINCE FETCH modifier conforms to + ;; the fetch-modifier syntax + + fetch-att =/ fetch-mod-sequence + ;; modifies original IMAP4 fetch-att + + fetch-mod-sequence = "MODSEQ" + + fetch-mod-resp = "MODSEQ" SP "(" permsg-modsequence ")" + + msg-att-dynamic =/ fetch-mod-resp + + + +Melnikov & Hole Standards Track [Page 19] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + search-key =/ search-modsequence + ;; modifies original IMAP4 search-key + ;; + ;; This change applies to all commands + ;; referencing this non-terminal, in + ;; particular SEARCH. + + search-modsequence = "MODSEQ" [search-modseq-ext] SP + mod-sequence-valzer + + search-modseq-ext = SP entry-name SP entry-type-req + + resp-text-code =/ "HIGHESTMODSEQ" SP mod-sequence-value / + "NOMODSEQ" / + "MODIFIED" SP set + + entry-name = entry-flag-name + + entry-flag-name = DQUOTE "/flags/" attr-flag DQUOTE + ;; each system or user defined flag + ;; is mapped to "/flags/". + ;; + ;; follows the escape rules + ;; used by "quoted" string as described in + ;; Section 4.3 of [IMAP4], e.g., for the flag + ;; \Seen the corresponding is + ;; "/flags/\\seen", and for the flag + ;; $MDNSent, the corresponding + ;; is "/flags/$mdnsent". + + entry-type-resp = "priv" / "shared" + ;; metadata item type + + entry-type-req = entry-type-resp / "all" + ;; perform SEARCH operation on private + ;; metadata item, shared metadata item or both + + permsg-modsequence = mod-sequence-value + ;; per message mod-sequence + + mod-sequence-value = 1*DIGIT + ;; Positive unsigned 64-bit integer + ;; (mod-sequence) + ;; (1 <= n < 18,446,744,073,709,551,615) + + mod-sequence-valzer = "0" / mod-sequence-value + + search-sort-mod-seq = "(" "MODSEQ" SP mod-sequence-value ")" + + + +Melnikov & Hole Standards Track [Page 20] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + select-param =/ condstore-param + ;; conforms to the generic "select-param" + ;; non-terminal syntax defined in [IMAPABNF]. + + condstore-param = "CONDSTORE" + + mailbox-data =/ "SEARCH" [1*(SP nz-number) SP + search-sort-mod-seq] + + attr-flag = "\\Answered" / "\\Flagged" / "\\Deleted" / + "\\Seen" / "\\Draft" / attr-flag-keyword / + attr-flag-extension + ;; Does not include "\\Recent" + + attr-flag-extension = "\\" atom + ;; Future expansion. Client implementations + ;; MUST accept flag-extension flags. Server + ;; implementations MUST NOT generate + ;; flag-extension flags except as defined by + ;; future standard or standards-track + ;; revisions of [IMAP4]. + + attr-flag-keyword = atom + +5. Server Implementation Considerations + + This section describes how a server implementation that doesn't store + separate per-metadata mod-sequences for different metadata items can + avoid sending the MODIFIED response to any of the following + conditional STORE operations: + + +FLAGS + -FLAGS + +FLAGS.SILENT + -FLAGS.SILENT + + Note that the optimization described in this section can't be + performed in case of a conditional STORE FLAGS operation. + + Let's use the following example. The client has issued + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + + When the server receives the command and parses it successfully, it + iterates through the message set and tries to execute the conditional + STORE command for each message. + + + + +Melnikov & Hole Standards Track [Page 21] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + Each server internally works as a client, i.e., it has to cache the + current state of all IMAP flags as it is known to the client. In + order to report flag changes to the client, the server compares the + cached values with the values in its database for IMAP flags. + + Imagine that another client has changed the state of a flag \Deleted + on the message 101 and that the change updated the mod-sequence for + the message. The server knows that the mod-sequence for the mailbox + has changed; however, it also knows that: + + a) the client is not interested in \Deleted flag, as it hasn't + included it in +FLAGS.SILENT operation; and + + b) the state of the flag $Processed hasn't changed (the server can + determine this by comparing cached flag state with the state of + the flag in the database). + + Therefore, the server doesn't have to report MODIFIED to the client. + Instead, the server may set $Processed flag, update the mod-sequence + for the message 101 once again and send an untagged FETCH response + with new mod-sequence and flags: + + S: * 101 FETCH (MODSEQ (303011130956) + FLAGS ($Processed \Deleted \Answered)) + + See also Section 3.8 for additional quality-of-implementation issues. + +6. Security Considerations + + It is believed that the Conditional STORE extension doesn't raise any + new security concerns that are not already discussed in [IMAP4]. + However, the availability of this extension may make it possible for + IMAP4 to be used in critical applications it could not be used for + previously, making correct IMAP server implementation and operation + even more important. + +7. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + This document defines the CONDSTORE IMAP capability. IANA has added + it to the registry accordingly. + + + + + +Melnikov & Hole Standards Track [Page 22] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +8. References + +8.1. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + +8.2. Informative References + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [ANN] Daboo, C. and R. Gellens, "IMAP ANNOTATE Extension", Work + in Progress, March 2006. + + [NTP] Mills, D., "Network Time Protocol (Version 3) + Specification, Implementation and Analysis", RFC 1305, + March 1992. + + [RFC-2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC + 2180, July 1997. + +9. Acknowledgements + + Some text was borrowed from "IMAP ANNOTATE Extension" [ANN] by + Randall Gellens and Cyrus Daboo and from "ACAP -- Application + Configuration Access Protocol" [ACAP] by Chris Newman and John Myers. + + Many thanks to Randall Gellens for his thorough review of the + document. + + The authors also acknowledge the feedback provided by Cyrus Daboo, + Larry Greenfield, Chris Newman, Harrie Hazewinkel, Arnt Gulbrandsen, + Timo Sirainen, Mark Crispin, Ned Freed, Ken Murchison, and Dave + Cridland. + + + + +Melnikov & Hole Standards Track [Page 23] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex + TW12 2BX, + United Kingdom + + EMail: Alexey.Melnikov@isode.com + + + Steve Hole + ACI WorldWide/MessagingDirect + #1807, 10088 102 Ave + Edmonton, AB + T5J 2Z1 + Canada + + EMail: Steve.Hole@messagingdirect.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Hole Standards Track [Page 24] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Melnikov & Hole Standards Track [Page 25] + From 2377353cae49b249402b23769a29c3d5439afe8a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 6 Mar 2015 21:27:52 +0100 Subject: [PATCH 727/817] rfcs: update RFCs and provide better filenames Signed-off-by: Nicolas Sebrecht --- docs/rfcs/README.md | 4 +- .../{rfc1731.txt => rfc1731.IMAP4_auth.txt} | 0 ...> rfc1732.compatibiliy_IMAP2-IMAP2bis.txt} | 0 ...fc1733.txt => rfc1733.models_in_IMAP4.txt} | 0 docs/rfcs/rfc1734.POP3_AUTHentication | 439 ++ docs/rfcs/rfc1939.txt | 1291 ----- ... rfc2061.compatibility_IMAP4-IMAP2bis.txt} | 0 docs/rfcs/rfc2062.txt | 451 -- ...86.txt => rfc2086.IMAP4_ACL_extension.txt} | 0 ....txt => rfc2087.IMAP4_QUOTA_extension.txt} | 0 ...2088.IMAP4_non_synchronizing_literals.txt} | 0 ... rfc2095.IMAP-POP_AUTHorize_extension.txt} | 0 ...177.txt => rfc2177.IMAP4_IDLE_command.txt} | 0 ...IMAP4_multi-accessed_Mailbox_practice.txt} | 0 ...fc2192.txt => rfc2192.IMAP_URL_scheme.txt} | 0 ...xt => rfc2193.IMAP4_Mailbox_referrals.txt} | 0 ... rfc2195.IMAP-POP_AUTHorize_extension.txt} | 0 ....txt => rfc2221.IMAP4_Login_referrals.txt} | 0 docs/rfcs/{rfc2244.txt => rfc2244.ACAP.txt} | 0 ...fc2342.txt => rfc2342.IMAP4_Namespace.txt} | 0 ...xt => rfc2359.IMAP4_UIDPLUS_extension.txt} | 0 ...> rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt} | 0 ....IMAP4_Implementation_recommendations.txt} | 0 docs/rfcs/rfc2821.txt | 4427 ----------------- ...e_Digest_AUTHentication_as_a_SASL_mech.txt | 1515 ++++++ ...971.txt => rfc2971.IMAP4_ID_extension.txt} | 0 ...rfc3028.Sieve_Mail_filtering_language.txt} | 0 ...rfc3348.IMAP4_Child_Mailbox_extension.txt} | 0 .../{rfc3501.txt => rfc3501.IMAP4rev1.txt} | 0 ....txt => rfc3502.MULTIAPPEND_extension.txt} | 0 ...3503.Message_Disposition_Notification.txt} | 0 ...fc3516.IMAP4_Binary_content_extension.txt} | 0 docs/rfcs/rfc3656.txt | 1067 ---- ....txt => rfc3691.IMAP_UNSELECT_command.txt} | 0 ...14.txt => rfc4314.IMAP4_ACL_extension.txt} | 0 ...txt => rfc4315.IMAP_UIDPLUS_extension.txt} | 0 ...66.Collected_extensions_to_IMAP4_ABNF.txt} | 0 ...txt => rfc4467.IMAP_URLAUTH_extension.txt} | 0 ...xt => rfc4469.IMAP_CATENATE_extension.txt} | 0 ...ations_for_disconnected_IMAP4_Clients.txt} | 0 ...al_STORE_or_Quick_flag_changes_resync.txt} | 0 ...4731.IMAP4_Extension_to_SEARCH_command.txt | 451 ++ docs/rfcs/rfc4978.IMAP_Compress_extension.txt | 507 ++ .../rfc5032.IMAP_WITHIN_Search_extension.txt | 283 ++ docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt | 395 ++ ...P4_Extensions_for_Quick_Mailbox_resync.txt | 1291 +++++ ...5182.IMAP_extension_last_SEARCH_result.txt | 731 +++ docs/rfcs/rfc5182.Sieve_and_extensions.txt | 2355 +++++++++ docs/rfcs/rfc5255.IMAP_i18n.txt | 1123 +++++ docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt | 1739 +++++++ .../rfc5258.IMAP4_LIST_command_extension.txt | 1739 +++++++ docs/rfcs/rfc5423.IM_Store_Events.txt | 955 ++++ docs/rfcs/rfc5464.IMAP_METADATA_extension.txt | 1177 +++++ docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt | 1235 +++++ docs/rfcs/rfc5530.IMAP_Response_codes.txt | 507 ++ docs/rfcs/rfc5738.IMAP_UTF8.txt | 843 ++++ docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt | 619 +++ ...xtension_Returning_STATUS_info_in_LIST.txt | 339 ++ docs/rfcs/rfc5957.IMAP4_SORT_extension.txt | 283 ++ ...fc6154.IMAP_LIST_Special-use_Mailboxes.txt | 675 +++ .../rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt | 395 ++ ...37.IMAP4_Multimailbox_SEARCH_extension.txt | 563 +++ .../rfc6331.Moving_Digest-MD5_to_Historic | 339 ++ 63 files changed, 20499 insertions(+), 7239 deletions(-) rename docs/rfcs/{rfc1731.txt => rfc1731.IMAP4_auth.txt} (100%) rename docs/rfcs/{rfc1732.txt => rfc1732.compatibiliy_IMAP2-IMAP2bis.txt} (100%) rename docs/rfcs/{rfc1733.txt => rfc1733.models_in_IMAP4.txt} (100%) create mode 100644 docs/rfcs/rfc1734.POP3_AUTHentication delete mode 100644 docs/rfcs/rfc1939.txt rename docs/rfcs/{rfc2061.txt => rfc2061.compatibility_IMAP4-IMAP2bis.txt} (100%) delete mode 100644 docs/rfcs/rfc2062.txt rename docs/rfcs/{rfc2086.txt => rfc2086.IMAP4_ACL_extension.txt} (100%) rename docs/rfcs/{rfc2087.txt => rfc2087.IMAP4_QUOTA_extension.txt} (100%) rename docs/rfcs/{rfc2088.txt => rfc2088.IMAP4_non_synchronizing_literals.txt} (100%) rename docs/rfcs/{rfc2095.txt => rfc2095.IMAP-POP_AUTHorize_extension.txt} (100%) rename docs/rfcs/{rfc2177.txt => rfc2177.IMAP4_IDLE_command.txt} (100%) rename docs/rfcs/{rfc2180.txt => rfc2180.IMAP4_multi-accessed_Mailbox_practice.txt} (100%) rename docs/rfcs/{rfc2192.txt => rfc2192.IMAP_URL_scheme.txt} (100%) rename docs/rfcs/{rfc2193.txt => rfc2193.IMAP4_Mailbox_referrals.txt} (100%) rename docs/rfcs/{rfc2195.txt => rfc2195.IMAP-POP_AUTHorize_extension.txt} (100%) rename docs/rfcs/{rfc2221.txt => rfc2221.IMAP4_Login_referrals.txt} (100%) rename docs/rfcs/{rfc2244.txt => rfc2244.ACAP.txt} (100%) rename docs/rfcs/{rfc2342.txt => rfc2342.IMAP4_Namespace.txt} (100%) rename docs/rfcs/{rfc2359.txt => rfc2359.IMAP4_UIDPLUS_extension.txt} (100%) rename docs/rfcs/{rfc2595.txt => rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt} (100%) rename docs/rfcs/{rfc2683.txt => rfc2683.IMAP4_Implementation_recommendations.txt} (100%) delete mode 100644 docs/rfcs/rfc2821.txt create mode 100644 docs/rfcs/rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt rename docs/rfcs/{rfc2971.txt => rfc2971.IMAP4_ID_extension.txt} (100%) rename docs/rfcs/{rfc3028.txt => rfc3028.Sieve_Mail_filtering_language.txt} (100%) rename docs/rfcs/{rfc3348.txt => rfc3348.IMAP4_Child_Mailbox_extension.txt} (100%) rename docs/rfcs/{rfc3501.txt => rfc3501.IMAP4rev1.txt} (100%) rename docs/rfcs/{rfc3502.txt => rfc3502.MULTIAPPEND_extension.txt} (100%) rename docs/rfcs/{rfc3503.txt => rfc3503.Message_Disposition_Notification.txt} (100%) rename docs/rfcs/{rfc3516.txt => rfc3516.IMAP4_Binary_content_extension.txt} (100%) delete mode 100644 docs/rfcs/rfc3656.txt rename docs/rfcs/{rfc3691.txt => rfc3691.IMAP_UNSELECT_command.txt} (100%) rename docs/rfcs/{rfc4314.txt => rfc4314.IMAP4_ACL_extension.txt} (100%) rename docs/rfcs/{rfc4315.txt => rfc4315.IMAP_UIDPLUS_extension.txt} (100%) rename docs/rfcs/{rfc4466.txt => rfc4466.Collected_extensions_to_IMAP4_ABNF.txt} (100%) rename docs/rfcs/{rfc4467.txt => rfc4467.IMAP_URLAUTH_extension.txt} (100%) rename docs/rfcs/{rfc4469.txt => rfc4469.IMAP_CATENATE_extension.txt} (100%) rename docs/rfcs/{rfc4549.txt => rfc4549.Sync_operations_for_disconnected_IMAP4_Clients.txt} (100%) rename docs/rfcs/{rfc4551.txt => rfc4551.IMAP_Conditional_STORE_or_Quick_flag_changes_resync.txt} (100%) create mode 100644 docs/rfcs/rfc4731.IMAP4_Extension_to_SEARCH_command.txt create mode 100644 docs/rfcs/rfc4978.IMAP_Compress_extension.txt create mode 100644 docs/rfcs/rfc5032.IMAP_WITHIN_Search_extension.txt create mode 100644 docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt create mode 100644 docs/rfcs/rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt create mode 100644 docs/rfcs/rfc5182.IMAP_extension_last_SEARCH_result.txt create mode 100644 docs/rfcs/rfc5182.Sieve_and_extensions.txt create mode 100644 docs/rfcs/rfc5255.IMAP_i18n.txt create mode 100644 docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt create mode 100644 docs/rfcs/rfc5258.IMAP4_LIST_command_extension.txt create mode 100644 docs/rfcs/rfc5423.IM_Store_Events.txt create mode 100644 docs/rfcs/rfc5464.IMAP_METADATA_extension.txt create mode 100644 docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt create mode 100644 docs/rfcs/rfc5530.IMAP_Response_codes.txt create mode 100644 docs/rfcs/rfc5738.IMAP_UTF8.txt create mode 100644 docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt create mode 100644 docs/rfcs/rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt create mode 100644 docs/rfcs/rfc5957.IMAP4_SORT_extension.txt create mode 100644 docs/rfcs/rfc6154.IMAP_LIST_Special-use_Mailboxes.txt create mode 100644 docs/rfcs/rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt create mode 100644 docs/rfcs/rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt create mode 100644 docs/rfcs/rfc6331.Moving_Digest-MD5_to_Historic diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index 70264ee..e011dbc 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -1,6 +1,4 @@ All RFCs related to IMAP. -TODO: -- rename the files to know what they are talking about. -- Add a brief introduction here to introduce the most important RFCs. +TODO: Add a brief introduction here to introduce the most important RFCs. diff --git a/docs/rfcs/rfc1731.txt b/docs/rfcs/rfc1731.IMAP4_auth.txt similarity index 100% rename from docs/rfcs/rfc1731.txt rename to docs/rfcs/rfc1731.IMAP4_auth.txt diff --git a/docs/rfcs/rfc1732.txt b/docs/rfcs/rfc1732.compatibiliy_IMAP2-IMAP2bis.txt similarity index 100% rename from docs/rfcs/rfc1732.txt rename to docs/rfcs/rfc1732.compatibiliy_IMAP2-IMAP2bis.txt diff --git a/docs/rfcs/rfc1733.txt b/docs/rfcs/rfc1733.models_in_IMAP4.txt similarity index 100% rename from docs/rfcs/rfc1733.txt rename to docs/rfcs/rfc1733.models_in_IMAP4.txt diff --git a/docs/rfcs/rfc1734.POP3_AUTHentication b/docs/rfcs/rfc1734.POP3_AUTHentication new file mode 100644 index 0000000..2e6c431 --- /dev/null +++ b/docs/rfcs/rfc1734.POP3_AUTHentication @@ -0,0 +1,439 @@ + + + + + + + + + + + + + + + + RFC 1734 - POP3 AUTHentication command + + + + + + + + +
+
+ +
+[Docs] [txt|pdf] [draft-myers-pop3-...] [Diff1] [Diff2]
+
+Obsoleted by: 5034 PROPOSED STANDARD
+
+
+Network Working Group                                           J. Myers
+Request for Comments: 1734                               Carnegie Mellon
+Category: Standards Track                                  December 1994
+
+
+                      POP3 AUTHentication command
+
+Status of this Memo
+
+   This document specifies an Internet standards track protocol for the
+   Internet community, and requests discussion and suggestions for
+   improvements.  Please refer to the current edition of the "Internet
+   Official Protocol Standards" (STD 1) for the standardization state
+   and status of this protocol.  Distribution of this memo is unlimited.
+
+
+1. Introduction
+
+   This document describes the optional AUTH command, for indicating an
+   authentication mechanism to the server, performing an authentication
+   protocol exchange, and optionally negotiating a protection mechanism
+   for subsequent protocol interactions.  The authentication and
+   protection mechanisms used by the POP3 AUTH command are those used by
+   IMAP4.
+
+
+2. The AUTH command
+
+   AUTH mechanism
+
+         Arguments:
+             a string identifying an IMAP4 authentication mechanism,
+             such as defined by [IMAP4-AUTH].  Any use of the string
+             "imap" used in a server authentication identity in the
+             definition of an authentication mechanism is replaced with
+             the string "pop".
+
+         Restrictions:
+             may only be given in the AUTHORIZATION state
+
+         Discussion:
+             The AUTH command indicates an authentication mechanism to
+             the server.  If the server supports the requested
+             authentication mechanism, it performs an authentication
+             protocol exchange to authenticate and identify the user.
+             Optionally, it also negotiates a protection mechanism for
+             subsequent protocol interactions.  If the requested
+             authentication mechanism is not supported, the server
+
+
+
+Myers                                                           [Page 1]
+

+RFC 1734                       POP3 AUTH                   December 1994
+
+
+             should reject the AUTH command by sending a negative
+             response.
+
+             The authentication protocol exchange consists of a series
+             of server challenges and client answers that are specific
+             to the authentication mechanism.  A server challenge,
+             otherwise known as a ready response, is a line consisting
+             of a "+" character followed by a single space and a BASE64
+             encoded string.  The client answer consists of a line
+             containing a BASE64 encoded string.  If the client wishes
+             to cancel an authentication exchange, it should issue a
+             line with a single "*".  If the server receives such an
+             answer, it must reject the AUTH command by sending a
+             negative response.
+
+             A protection mechanism provides integrity and privacy
+             protection to the protocol session.  If a protection
+             mechanism is negotiated, it is applied to all subsequent
+             data sent over the connection.  The protection mechanism
+             takes effect immediately following the CRLF that concludes
+             the authentication exchange for the client, and the CRLF of
+             the positive response for the server.  Once the protection
+             mechanism is in effect, the stream of command and response
+             octets is processed into buffers of ciphertext.  Each
+             buffer is transferred over the connection as a stream of
+             octets prepended with a four octet field in network byte
+             order that represents the length of the following data.
+             The maximum ciphertext buffer length is defined by the
+             protection mechanism.
+
+             The server is not required to support any particular
+             authentication mechanism, nor are authentication mechanisms
+             required to support any protection mechanisms.  If an AUTH
+             command fails with a negative response, the session remains
+             in the AUTHORIZATION state and client may try another
+             authentication mechanism by issuing another AUTH command,
+             or may attempt to authenticate by using the USER/PASS or
+             APOP commands.  In other words, the client may request
+             authentication types in decreasing order of preference,
+             with the USER/PASS or APOP command as a last resort.
+
+             Should the client successfully complete the authentication
+             exchange, the POP3 server issues a positive response and
+             the POP3 session enters the TRANSACTION state.
+
+         Possible Responses:
+             +OK maildrop locked and ready
+             -ERR authentication exchange failed
+
+
+
+Myers                                                           [Page 2]
+

+RFC 1734                       POP3 AUTH                   December 1994
+
+
+
+         Examples:
+             S: +OK POP3 server ready
+             C: AUTH KERBEROS_V4
+             S: + AmFYig==
+             C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT
+                +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd
+                WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh
+             S: + or//EoAADZI=
+             C: DiAF5A4gA+oOIALuBkAAmw==
+             S: +OK Kerberos V4 authentication successful
+                ...
+             C: AUTH FOOBAR
+             S: -ERR Unrecognized authentication type
+
+              Note: the line breaks in the first client answer  are
+              for editorial clarity and are not in real authentica-
+              tors.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Myers                                                           [Page 3]
+

+RFC 1734                       POP3 AUTH                   December 1994
+
+
+3. Formal Syntax
+
+   The following syntax specification uses the augmented Backus-Naur
+   Form (BNF) notation as specified in RFC 822.
+
+   Except as noted otherwise, all alphabetic characters are case-
+   insensitive.  The use of upper or lower case characters to define
+   token strings is for editorial clarity only.  Implementations MUST
+   accept these strings in a case-insensitive fashion.
+
+   ATOM_CHAR       ::= <any CHAR except atom_specials>
+
+   atom_specials   ::= "(" / ")" / "{" / SPACE / CTLs / "%" / "*" /
+                       <"> / "\"
+
+   auth            ::= "AUTH" 1*(SPACE / TAB) auth_type *(CRLF base64)
+                       CRLF
+
+   auth_type       ::= 1*ATOM_CHAR
+
+   base64          ::= *(4base64_CHAR) [base64_terminal]
+
+   base64_char     ::= "A" / "B" / "C" / "D" / "E" / "F" / "G" / "H" /
+           "I" / "J" / "K" / "L" / "M" / "N" / "O" / "P" /
+                       "Q" / "R" / "S" / "T" / "U" / "V" / "W" / "X" /
+                       "Y" / "Z" /
+                       "a" / "b" / "c" / "d" / "e" / "f" / "g" / "h" /
+                       "i" / "j" / "k" / "l" / "m" / "n" / "o" / "p" /
+                       "q" / "r" / "s" / "t" / "u" / "v" / "w" / "x" /
+                       "y" / "z" /
+                       "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7" /
+                       "8" / "9" / "+" / "/"
+                       ;; Case-sensitive
+
+   base64_terminal ::= (2base64_char "==") / (3base64_char "=")
+
+   CHAR            ::= <any 7-bit US-ASCII character except NUL,
+                        0x01 - 0x7f>
+
+   continue_req    ::= "+" SPACE base64 CRLF
+
+   CR              ::= <ASCII CR, carriage return, 0x0C>
+
+   CRLF            ::= CR LF
+
+   CTL             ::= <any ASCII control character and DEL,
+                        0x00 - 0x1f, 0x7f>
+
+
+
+
+Myers                                                           [Page 4]
+

+RFC 1734                       POP3 AUTH                   December 1994
+
+
+   LF              ::= <ASCII LF, line feed, 0x0A>
+
+   SPACE           ::= <ASCII SP, space, 0x20>
+
+   TAB             ::= <ASCII HT, tab, 0x09>
+
+
+
+4. References
+
+   [IMAP4-AUTH]  Myers, J., "IMAP4 Authentication Mechanisms", RFC 1731,
+   Carnegie Mellon, December 1994.
+
+
+
+5. Security Considerations
+
+   Security issues are discussed throughout this memo.
+
+
+
+6. Author's Address
+
+   John G. Myers
+   Carnegie-Mellon University
+   5000 Forbes Ave
+   Pittsburgh, PA 15213
+
+   EMail: jgm+@cmu.edu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Myers                                                           [Page 5]
+
+

+Html markup produced by rfcmarkup 1.111, available from +https://tools.ietf.org/tools/rfcmarkup/ + + diff --git a/docs/rfcs/rfc1939.txt b/docs/rfcs/rfc1939.txt deleted file mode 100644 index b11350e..0000000 --- a/docs/rfcs/rfc1939.txt +++ /dev/null @@ -1,1291 +0,0 @@ - - - - - - -Network Working Group J. Myers -Request for Comments: 1939 Carnegie Mellon -STD: 53 M. Rose -Obsoletes: 1725 Dover Beach Consulting, Inc. -Category: Standards Track May 1996 - - - Post Office Protocol - Version 3 - -Status of this Memo - - This document specifies an Internet standards track protocol for the - Internet community, and requests discussion and suggestions for - improvements. Please refer to the current edition of the "Internet - Official Protocol Standards" (STD 1) for the standardization state - and status of this protocol. Distribution of this memo is unlimited. - -Table of Contents - - 1. Introduction ................................................ 2 - 2. A Short Digression .......................................... 2 - 3. Basic Operation ............................................. 3 - 4. The AUTHORIZATION State ..................................... 4 - QUIT Command ................................................ 5 - 5. The TRANSACTION State ....................................... 5 - STAT Command ................................................ 6 - LIST Command ................................................ 6 - RETR Command ................................................ 8 - DELE Command ................................................ 8 - NOOP Command ................................................ 9 - RSET Command ................................................ 9 - 6. The UPDATE State ............................................ 10 - QUIT Command ................................................ 10 - 7. Optional POP3 Commands ...................................... 11 - TOP Command ................................................. 11 - UIDL Command ................................................ 12 - USER Command ................................................ 13 - PASS Command ................................................ 14 - APOP Command ................................................ 15 - 8. Scaling and Operational Considerations ...................... 16 - 9. POP3 Command Summary ........................................ 18 - 10. Example POP3 Session ....................................... 19 - 11. Message Format ............................................. 19 - 12. References ................................................. 20 - 13. Security Considerations .................................... 20 - 14. Acknowledgements ........................................... 20 - 15. Authors' Addresses ......................................... 21 - Appendix A. Differences from RFC 1725 .......................... 22 - - - -Myers & Rose Standards Track [Page 1] - -RFC 1939 POP3 May 1996 - - - Appendix B. Command Index ...................................... 23 - -1. Introduction - - On certain types of smaller nodes in the Internet it is often - impractical to maintain a message transport system (MTS). For - example, a workstation may not have sufficient resources (cycles, - disk space) in order to permit a SMTP server [RFC821] and associated - local mail delivery system to be kept resident and continuously - running. Similarly, it may be expensive (or impossible) to keep a - personal computer interconnected to an IP-style network for long - amounts of time (the node is lacking the resource known as - "connectivity"). - - Despite this, it is often very useful to be able to manage mail on - these smaller nodes, and they often support a user agent (UA) to aid - the tasks of mail handling. To solve this problem, a node which can - support an MTS entity offers a maildrop service to these less endowed - nodes. The Post Office Protocol - Version 3 (POP3) is intended to - permit a workstation to dynamically access a maildrop on a server - host in a useful fashion. Usually, this means that the POP3 protocol - is used to allow a workstation to retrieve mail that the server is - holding for it. - - POP3 is not intended to provide extensive manipulation operations of - mail on the server; normally, mail is downloaded and then deleted. A - more advanced (and complex) protocol, IMAP4, is discussed in - [RFC1730]. - - For the remainder of this memo, the term "client host" refers to a - host making use of the POP3 service, while the term "server host" - refers to a host which offers the POP3 service. - -2. A Short Digression - - This memo does not specify how a client host enters mail into the - transport system, although a method consistent with the philosophy of - this memo is presented here: - - When the user agent on a client host wishes to enter a message - into the transport system, it establishes an SMTP connection to - its relay host and sends all mail to it. This relay host could - be, but need not be, the POP3 server host for the client host. Of - course, the relay host must accept mail for delivery to arbitrary - recipient addresses, that functionality is not required of all - SMTP servers. - - - - - -Myers & Rose Standards Track [Page 2] - -RFC 1939 POP3 May 1996 - - -3. Basic Operation - - Initially, the server host starts the POP3 service by listening on - TCP port 110. When a client host wishes to make use of the service, - it establishes a TCP connection with the server host. When the - connection is established, the POP3 server sends a greeting. The - client and POP3 server then exchange commands and responses - (respectively) until the connection is closed or aborted. - - Commands in the POP3 consist of a case-insensitive keyword, possibly - followed by one or more arguments. All commands are terminated by a - CRLF pair. Keywords and arguments consist of printable ASCII - characters. Keywords and arguments are each separated by a single - SPACE character. Keywords are three or four characters long. Each - argument may be up to 40 characters long. - - Responses in the POP3 consist of a status indicator and a keyword - possibly followed by additional information. All responses are - terminated by a CRLF pair. Responses may be up to 512 characters - long, including the terminating CRLF. There are currently two status - indicators: positive ("+OK") and negative ("-ERR"). Servers MUST - send the "+OK" and "-ERR" in upper case. - - Responses to certain commands are multi-line. In these cases, which - are clearly indicated below, after sending the first line of the - response and a CRLF, any additional lines are sent, each terminated - by a CRLF pair. When all lines of the response have been sent, a - final line is sent, consisting of a termination octet (decimal code - 046, ".") and a CRLF pair. If any line of the multi-line response - begins with the termination octet, the line is "byte-stuffed" by - pre-pending the termination octet to that line of the response. - Hence a multi-line response is terminated with the five octets - "CRLF.CRLF". When examining a multi-line response, the client checks - to see if the line begins with the termination octet. If so and if - octets other than CRLF follow, the first octet of the line (the - termination octet) is stripped away. If so and if CRLF immediately - follows the termination character, then the response from the POP - server is ended and the line containing ".CRLF" is not considered - part of the multi-line response. - - A POP3 session progresses through a number of states during its - lifetime. Once the TCP connection has been opened and the POP3 - server has sent the greeting, the session enters the AUTHORIZATION - state. In this state, the client must identify itself to the POP3 - server. Once the client has successfully done this, the server - acquires resources associated with the client's maildrop, and the - session enters the TRANSACTION state. In this state, the client - requests actions on the part of the POP3 server. When the client has - - - -Myers & Rose Standards Track [Page 3] - -RFC 1939 POP3 May 1996 - - - issued the QUIT command, the session enters the UPDATE state. In - this state, the POP3 server releases any resources acquired during - the TRANSACTION state and says goodbye. The TCP connection is then - closed. - - A server MUST respond to an unrecognized, unimplemented, or - syntactically invalid command by responding with a negative status - indicator. A server MUST respond to a command issued when the - session is in an incorrect state by responding with a negative status - indicator. There is no general method for a client to distinguish - between a server which does not implement an optional command and a - server which is unwilling or unable to process the command. - - A POP3 server MAY have an inactivity autologout timer. Such a timer - MUST be of at least 10 minutes' duration. The receipt of any command - from the client during that interval should suffice to reset the - autologout timer. When the timer expires, the session does NOT enter - the UPDATE state--the server should close the TCP connection without - removing any messages or sending any response to the client. - -4. The AUTHORIZATION State - - Once the TCP connection has been opened by a POP3 client, the POP3 - server issues a one line greeting. This can be any positive - response. An example might be: - - S: +OK POP3 server ready - - The POP3 session is now in the AUTHORIZATION state. The client must - now identify and authenticate itself to the POP3 server. Two - possible mechanisms for doing this are described in this document, - the USER and PASS command combination and the APOP command. Both - mechanisms are described later in this document. Additional - authentication mechanisms are described in [RFC1734]. While there is - no single authentication mechanism that is required of all POP3 - servers, a POP3 server must of course support at least one - authentication mechanism. - - Once the POP3 server has determined through the use of any - authentication command that the client should be given access to the - appropriate maildrop, the POP3 server then acquires an exclusive- - access lock on the maildrop, as necessary to prevent messages from - being modified or removed before the session enters the UPDATE state. - If the lock is successfully acquired, the POP3 server responds with a - positive status indicator. The POP3 session now enters the - TRANSACTION state, with no messages marked as deleted. If the - maildrop cannot be opened for some reason (for example, a lock can - not be acquired, the client is denied access to the appropriate - - - -Myers & Rose Standards Track [Page 4] - -RFC 1939 POP3 May 1996 - - - maildrop, or the maildrop cannot be parsed), the POP3 server responds - with a negative status indicator. (If a lock was acquired but the - POP3 server intends to respond with a negative status indicator, the - POP3 server must release the lock prior to rejecting the command.) - After returning a negative status indicator, the server may close the - connection. If the server does not close the connection, the client - may either issue a new authentication command and start again, or the - client may issue the QUIT command. - - After the POP3 server has opened the maildrop, it assigns a message- - number to each message, and notes the size of each message in octets. - The first message in the maildrop is assigned a message-number of - "1", the second is assigned "2", and so on, so that the nth message - in a maildrop is assigned a message-number of "n". In POP3 commands - and responses, all message-numbers and message sizes are expressed in - base-10 (i.e., decimal). - - Here is the summary for the QUIT command when used in the - AUTHORIZATION state: - - QUIT - - Arguments: none - - Restrictions: none - - Possible Responses: - +OK - - Examples: - C: QUIT - S: +OK dewey POP3 server signing off - -5. The TRANSACTION State - - Once the client has successfully identified itself to the POP3 server - and the POP3 server has locked and opened the appropriate maildrop, - the POP3 session is now in the TRANSACTION state. The client may now - issue any of the following POP3 commands repeatedly. After each - command, the POP3 server issues a response. Eventually, the client - issues the QUIT command and the POP3 session enters the UPDATE state. - - - - - - - - - - -Myers & Rose Standards Track [Page 5] - -RFC 1939 POP3 May 1996 - - - Here are the POP3 commands valid in the TRANSACTION state: - - STAT - - Arguments: none - - Restrictions: - may only be given in the TRANSACTION state - - Discussion: - The POP3 server issues a positive response with a line - containing information for the maildrop. This line is - called a "drop listing" for that maildrop. - - In order to simplify parsing, all POP3 servers are - required to use a certain format for drop listings. The - positive response consists of "+OK" followed by a single - space, the number of messages in the maildrop, a single - space, and the size of the maildrop in octets. This memo - makes no requirement on what follows the maildrop size. - Minimal implementations should just end that line of the - response with a CRLF pair. More advanced implementations - may include other information. - - NOTE: This memo STRONGLY discourages implementations - from supplying additional information in the drop - listing. Other, optional, facilities are discussed - later on which permit the client to parse the messages - in the maildrop. - - Note that messages marked as deleted are not counted in - either total. - - Possible Responses: - +OK nn mm - - Examples: - C: STAT - S: +OK 2 320 - - - LIST [msg] - - Arguments: - a message-number (optional), which, if present, may NOT - refer to a message marked as deleted - - - - - -Myers & Rose Standards Track [Page 6] - -RFC 1939 POP3 May 1996 - - - Restrictions: - may only be given in the TRANSACTION state - - Discussion: - If an argument was given and the POP3 server issues a - positive response with a line containing information for - that message. This line is called a "scan listing" for - that message. - - If no argument was given and the POP3 server issues a - positive response, then the response given is multi-line. - After the initial +OK, for each message in the maildrop, - the POP3 server responds with a line containing - information for that message. This line is also called a - "scan listing" for that message. If there are no - messages in the maildrop, then the POP3 server responds - with no scan listings--it issues a positive response - followed by a line containing a termination octet and a - CRLF pair. - - In order to simplify parsing, all POP3 servers are - required to use a certain format for scan listings. A - scan listing consists of the message-number of the - message, followed by a single space and the exact size of - the message in octets. Methods for calculating the exact - size of the message are described in the "Message Format" - section below. This memo makes no requirement on what - follows the message size in the scan listing. Minimal - implementations should just end that line of the response - with a CRLF pair. More advanced implementations may - include other information, as parsed from the message. - - NOTE: This memo STRONGLY discourages implementations - from supplying additional information in the scan - listing. Other, optional, facilities are discussed - later on which permit the client to parse the messages - in the maildrop. - - Note that messages marked as deleted are not listed. - - Possible Responses: - +OK scan listing follows - -ERR no such message - - Examples: - C: LIST - S: +OK 2 messages (320 octets) - S: 1 120 - - - -Myers & Rose Standards Track [Page 7] - -RFC 1939 POP3 May 1996 - - - S: 2 200 - S: . - ... - C: LIST 2 - S: +OK 2 200 - ... - C: LIST 3 - S: -ERR no such message, only 2 messages in maildrop - - - RETR msg - - Arguments: - a message-number (required) which may NOT refer to a - message marked as deleted - - Restrictions: - may only be given in the TRANSACTION state - - Discussion: - If the POP3 server issues a positive response, then the - response given is multi-line. After the initial +OK, the - POP3 server sends the message corresponding to the given - message-number, being careful to byte-stuff the termination - character (as with all multi-line responses). - - Possible Responses: - +OK message follows - -ERR no such message - - Examples: - C: RETR 1 - S: +OK 120 octets - S: - S: . - - - DELE msg - - Arguments: - a message-number (required) which may NOT refer to a - message marked as deleted - - Restrictions: - may only be given in the TRANSACTION state - - - - - - -Myers & Rose Standards Track [Page 8] - -RFC 1939 POP3 May 1996 - - - Discussion: - The POP3 server marks the message as deleted. Any future - reference to the message-number associated with the message - in a POP3 command generates an error. The POP3 server does - not actually delete the message until the POP3 session - enters the UPDATE state. - - Possible Responses: - +OK message deleted - -ERR no such message - - Examples: - C: DELE 1 - S: +OK message 1 deleted - ... - C: DELE 2 - S: -ERR message 2 already deleted - - - NOOP - - Arguments: none - - Restrictions: - may only be given in the TRANSACTION state - - Discussion: - The POP3 server does nothing, it merely replies with a - positive response. - - Possible Responses: - +OK - - Examples: - C: NOOP - S: +OK - - - RSET - - Arguments: none - - Restrictions: - may only be given in the TRANSACTION state - - Discussion: - If any messages have been marked as deleted by the POP3 - server, they are unmarked. The POP3 server then replies - - - -Myers & Rose Standards Track [Page 9] - -RFC 1939 POP3 May 1996 - - - with a positive response. - - Possible Responses: - +OK - - Examples: - C: RSET - S: +OK maildrop has 2 messages (320 octets) - -6. The UPDATE State - - When the client issues the QUIT command from the TRANSACTION state, - the POP3 session enters the UPDATE state. (Note that if the client - issues the QUIT command from the AUTHORIZATION state, the POP3 - session terminates but does NOT enter the UPDATE state.) - - If a session terminates for some reason other than a client-issued - QUIT command, the POP3 session does NOT enter the UPDATE state and - MUST not remove any messages from the maildrop. - - QUIT - - Arguments: none - - Restrictions: none - - Discussion: - The POP3 server removes all messages marked as deleted - from the maildrop and replies as to the status of this - operation. If there is an error, such as a resource - shortage, encountered while removing messages, the - maildrop may result in having some or none of the messages - marked as deleted be removed. In no case may the server - remove any messages not marked as deleted. - - Whether the removal was successful or not, the server - then releases any exclusive-access lock on the maildrop - and closes the TCP connection. - - Possible Responses: - +OK - -ERR some deleted messages not removed - - Examples: - C: QUIT - S: +OK dewey POP3 server signing off (maildrop empty) - ... - C: QUIT - - - -Myers & Rose Standards Track [Page 10] - -RFC 1939 POP3 May 1996 - - - S: +OK dewey POP3 server signing off (2 messages left) - ... - -7. Optional POP3 Commands - - The POP3 commands discussed above must be supported by all minimal - implementations of POP3 servers. - - The optional POP3 commands described below permit a POP3 client - greater freedom in message handling, while preserving a simple POP3 - server implementation. - - NOTE: This memo STRONGLY encourages implementations to support - these commands in lieu of developing augmented drop and scan - listings. In short, the philosophy of this memo is to put - intelligence in the part of the POP3 client and not the POP3 - server. - - TOP msg n - - Arguments: - a message-number (required) which may NOT refer to to a - message marked as deleted, and a non-negative number - of lines (required) - - Restrictions: - may only be given in the TRANSACTION state - - Discussion: - If the POP3 server issues a positive response, then the - response given is multi-line. After the initial +OK, the - POP3 server sends the headers of the message, the blank - line separating the headers from the body, and then the - number of lines of the indicated message's body, being - careful to byte-stuff the termination character (as with - all multi-line responses). - - Note that if the number of lines requested by the POP3 - client is greater than than the number of lines in the - body, then the POP3 server sends the entire message. - - Possible Responses: - +OK top of message follows - -ERR no such message - - Examples: - C: TOP 1 10 - S: +OK - - - -Myers & Rose Standards Track [Page 11] - -RFC 1939 POP3 May 1996 - - - S: - S: . - ... - C: TOP 100 3 - S: -ERR no such message - - - UIDL [msg] - - Arguments: - a message-number (optional), which, if present, may NOT - refer to a message marked as deleted - - Restrictions: - may only be given in the TRANSACTION state. - - Discussion: - If an argument was given and the POP3 server issues a positive - response with a line containing information for that message. - This line is called a "unique-id listing" for that message. - - If no argument was given and the POP3 server issues a positive - response, then the response given is multi-line. After the - initial +OK, for each message in the maildrop, the POP3 server - responds with a line containing information for that message. - This line is called a "unique-id listing" for that message. - - In order to simplify parsing, all POP3 servers are required to - use a certain format for unique-id listings. A unique-id - listing consists of the message-number of the message, - followed by a single space and the unique-id of the message. - No information follows the unique-id in the unique-id listing. - - The unique-id of a message is an arbitrary server-determined - string, consisting of one to 70 characters in the range 0x21 - to 0x7E, which uniquely identifies a message within a - maildrop and which persists across sessions. This - persistence is required even if a session ends without - entering the UPDATE state. The server should never reuse an - unique-id in a given maildrop, for as long as the entity - using the unique-id exists. - - Note that messages marked as deleted are not listed. - - While it is generally preferable for server implementations - to store arbitrarily assigned unique-ids in the maildrop, - - - -Myers & Rose Standards Track [Page 12] - -RFC 1939 POP3 May 1996 - - - this specification is intended to permit unique-ids to be - calculated as a hash of the message. Clients should be able - to handle a situation where two identical copies of a - message in a maildrop have the same unique-id. - - Possible Responses: - +OK unique-id listing follows - -ERR no such message - - Examples: - C: UIDL - S: +OK - S: 1 whqtswO00WBw418f9t5JxYwZ - S: 2 QhdPYR:00WBw1Ph7x7 - S: . - ... - C: UIDL 2 - S: +OK 2 QhdPYR:00WBw1Ph7x7 - ... - C: UIDL 3 - S: -ERR no such message, only 2 messages in maildrop - - - USER name - - Arguments: - a string identifying a mailbox (required), which is of - significance ONLY to the server - - Restrictions: - may only be given in the AUTHORIZATION state after the POP3 - greeting or after an unsuccessful USER or PASS command - - Discussion: - To authenticate using the USER and PASS command - combination, the client must first issue the USER - command. If the POP3 server responds with a positive - status indicator ("+OK"), then the client may issue - either the PASS command to complete the authentication, - or the QUIT command to terminate the POP3 session. If - the POP3 server responds with a negative status indicator - ("-ERR") to the USER command, then the client may either - issue a new authentication command or may issue the QUIT - command. - - The server may return a positive response even though no - such mailbox exists. The server may return a negative - response if mailbox exists, but does not permit plaintext - - - -Myers & Rose Standards Track [Page 13] - -RFC 1939 POP3 May 1996 - - - password authentication. - - Possible Responses: - +OK name is a valid mailbox - -ERR never heard of mailbox name - - Examples: - C: USER frated - S: -ERR sorry, no mailbox for frated here - ... - C: USER mrose - S: +OK mrose is a real hoopy frood - - - PASS string - - Arguments: - a server/mailbox-specific password (required) - - Restrictions: - may only be given in the AUTHORIZATION state immediately - after a successful USER command - - Discussion: - When the client issues the PASS command, the POP3 server - uses the argument pair from the USER and PASS commands to - determine if the client should be given access to the - appropriate maildrop. - - Since the PASS command has exactly one argument, a POP3 - server may treat spaces in the argument as part of the - password, instead of as argument separators. - - Possible Responses: - +OK maildrop locked and ready - -ERR invalid password - -ERR unable to lock maildrop - - Examples: - C: USER mrose - S: +OK mrose is a real hoopy frood - C: PASS secret - S: -ERR maildrop already locked - ... - C: USER mrose - S: +OK mrose is a real hoopy frood - C: PASS secret - S: +OK mrose's maildrop has 2 messages (320 octets) - - - -Myers & Rose Standards Track [Page 14] - -RFC 1939 POP3 May 1996 - - - APOP name digest - - Arguments: - a string identifying a mailbox and a MD5 digest string - (both required) - - Restrictions: - may only be given in the AUTHORIZATION state after the POP3 - greeting or after an unsuccessful USER or PASS command - - Discussion: - Normally, each POP3 session starts with a USER/PASS - exchange. This results in a server/user-id specific - password being sent in the clear on the network. For - intermittent use of POP3, this may not introduce a sizable - risk. However, many POP3 client implementations connect to - the POP3 server on a regular basis -- to check for new - mail. Further the interval of session initiation may be on - the order of five minutes. Hence, the risk of password - capture is greatly enhanced. - - An alternate method of authentication is required which - provides for both origin authentication and replay - protection, but which does not involve sending a password - in the clear over the network. The APOP command provides - this functionality. - - A POP3 server which implements the APOP command will - include a timestamp in its banner greeting. The syntax of - the timestamp corresponds to the `msg-id' in [RFC822], and - MUST be different each time the POP3 server issues a banner - greeting. For example, on a UNIX implementation in which a - separate UNIX process is used for each instance of a POP3 - server, the syntax of the timestamp might be: - - - - where `process-ID' is the decimal value of the process's - PID, clock is the decimal value of the system clock, and - hostname is the fully-qualified domain-name corresponding - to the host where the POP3 server is running. - - The POP3 client makes note of this timestamp, and then - issues the APOP command. The `name' parameter has - identical semantics to the `name' parameter of the USER - command. The `digest' parameter is calculated by applying - the MD5 algorithm [RFC1321] to a string consisting of the - timestamp (including angle-brackets) followed by a shared - - - -Myers & Rose Standards Track [Page 15] - -RFC 1939 POP3 May 1996 - - - secret. This shared secret is a string known only to the - POP3 client and server. Great care should be taken to - prevent unauthorized disclosure of the secret, as knowledge - of the secret will allow any entity to successfully - masquerade as the named user. The `digest' parameter - itself is a 16-octet value which is sent in hexadecimal - format, using lower-case ASCII characters. - - When the POP3 server receives the APOP command, it verifies - the digest provided. If the digest is correct, the POP3 - server issues a positive response, and the POP3 session - enters the TRANSACTION state. Otherwise, a negative - response is issued and the POP3 session remains in the - AUTHORIZATION state. - - Note that as the length of the shared secret increases, so - does the difficulty of deriving it. As such, shared - secrets should be long strings (considerably longer than - the 8-character example shown below). - - Possible Responses: - +OK maildrop locked and ready - -ERR permission denied - - Examples: - S: +OK POP3 server ready <1896.697170952@dbc.mtview.ca.us> - C: APOP mrose c4c9334bac560ecc979e58001b3e22fb - S: +OK maildrop has 1 message (369 octets) - - In this example, the shared secret is the string `tan- - staaf'. Hence, the MD5 algorithm is applied to the string - - <1896.697170952@dbc.mtview.ca.us>tanstaaf - - which produces a digest value of - - c4c9334bac560ecc979e58001b3e22fb - -8. Scaling and Operational Considerations - - Since some of the optional features described above were added to the - POP3 protocol, experience has accumulated in using them in large- - scale commercial post office operations where most of the users are - unrelated to each other. In these situations and others, users and - vendors of POP3 clients have discovered that the combination of using - the UIDL command and not issuing the DELE command can provide a weak - version of the "maildrop as semi-permanent repository" functionality - normally associated with IMAP. Of course the other capabilities of - - - -Myers & Rose Standards Track [Page 16] - -RFC 1939 POP3 May 1996 - - - IMAP, such as polling an existing connection for newly arrived - messages and supporting multiple folders on the server, are not - present in POP3. - - When these facilities are used in this way by casual users, there has - been a tendency for already-read messages to accumulate on the server - without bound. This is clearly an undesirable behavior pattern from - the standpoint of the server operator. This situation is aggravated - by the fact that the limited capabilities of the POP3 do not permit - efficient handling of maildrops which have hundreds or thousands of - messages. - - Consequently, it is recommended that operators of large-scale multi- - user servers, especially ones in which the user's only access to the - maildrop is via POP3, consider such options as: - - * Imposing a per-user maildrop storage quota or the like. - - A disadvantage to this option is that accumulation of messages may - result in the user's inability to receive new ones into the - maildrop. Sites which choose this option should be sure to inform - users of impending or current exhaustion of quota, perhaps by - inserting an appropriate message into the user's maildrop. - - * Enforce a site policy regarding mail retention on the server. - - Sites are free to establish local policy regarding the storage and - retention of messages on the server, both read and unread. For - example, a site might delete unread messages from the server after - 60 days and delete read messages after 7 days. Such message - deletions are outside the scope of the POP3 protocol and are not - considered a protocol violation. - - Server operators enforcing message deletion policies should take - care to make all users aware of the policies in force. - - Clients must not assume that a site policy will automate message - deletions, and should continue to explicitly delete messages using - the DELE command when appropriate. - - It should be noted that enforcing site message deletion policies - may be confusing to the user community, since their POP3 client - may contain configuration options to leave mail on the server - which will not in fact be supported by the server. - - One special case of a site policy is that messages may only be - downloaded once from the server, and are deleted after this has - been accomplished. This could be implemented in POP3 server - - - -Myers & Rose Standards Track [Page 17] - -RFC 1939 POP3 May 1996 - - - software by the following mechanism: "following a POP3 login by a - client which was ended by a QUIT, delete all messages downloaded - during the session with the RETR command". It is important not to - delete messages in the event of abnormal connection termination - (ie, if no QUIT was received from the client) because the client - may not have successfully received or stored the messages. - Servers implementing a download-and-delete policy may also wish to - disable or limit the optional TOP command, since it could be used - as an alternate mechanism to download entire messages. - -9. POP3 Command Summary - - Minimal POP3 Commands: - - USER name valid in the AUTHORIZATION state - PASS string - QUIT - - STAT valid in the TRANSACTION state - LIST [msg] - RETR msg - DELE msg - NOOP - RSET - QUIT - - Optional POP3 Commands: - - APOP name digest valid in the AUTHORIZATION state - - TOP msg n valid in the TRANSACTION state - UIDL [msg] - - POP3 Replies: - - +OK - -ERR - - Note that with the exception of the STAT, LIST, and UIDL commands, - the reply given by the POP3 server to any command is significant - only to "+OK" and "-ERR". Any text occurring after this reply - may be ignored by the client. - - - - - - - - - -Myers & Rose Standards Track [Page 18] - -RFC 1939 POP3 May 1996 - - -10. Example POP3 Session - - S: - C: - S: +OK POP3 server ready <1896.697170952@dbc.mtview.ca.us> - C: APOP mrose c4c9334bac560ecc979e58001b3e22fb - S: +OK mrose's maildrop has 2 messages (320 octets) - C: STAT - S: +OK 2 320 - C: LIST - S: +OK 2 messages (320 octets) - S: 1 120 - S: 2 200 - S: . - C: RETR 1 - S: +OK 120 octets - S: - S: . - C: DELE 1 - S: +OK message 1 deleted - C: RETR 2 - S: +OK 200 octets - S: - S: . - C: DELE 2 - S: +OK message 2 deleted - C: QUIT - S: +OK dewey POP3 server signing off (maildrop empty) - C: - S: - -11. Message Format - - All messages transmitted during a POP3 session are assumed to conform - to the standard for the format of Internet text messages [RFC822]. - - It is important to note that the octet count for a message on the - server host may differ from the octet count assigned to that message - due to local conventions for designating end-of-line. Usually, - during the AUTHORIZATION state of the POP3 session, the POP3 server - can calculate the size of each message in octets when it opens the - maildrop. For example, if the POP3 server host internally represents - end-of-line as a single character, then the POP3 server simply counts - each occurrence of this character in a message as two octets. Note - that lines in the message which start with the termination octet need - not (and must not) be counted twice, since the POP3 client will - remove all byte-stuffed termination characters when it receives a - multi-line response. - - - -Myers & Rose Standards Track [Page 19] - -RFC 1939 POP3 May 1996 - - -12. References - - [RFC821] Postel, J., "Simple Mail Transfer Protocol", STD 10, RFC - 821, USC/Information Sciences Institute, August 1982. - - [RFC822] Crocker, D., "Standard for the Format of ARPA-Internet Text - Messages", STD 11, RFC 822, University of Delaware, August 1982. - - [RFC1321] Rivest, R., "The MD5 Message-Digest Algorithm", RFC 1321, - MIT Laboratory for Computer Science, April 1992. - - [RFC1730] Crispin, M., "Internet Message Access Protocol - Version - 4", RFC 1730, University of Washington, December 1994. - - [RFC1734] Myers, J., "POP3 AUTHentication command", RFC 1734, - Carnegie Mellon, December 1994. - -13. Security Considerations - - It is conjectured that use of the APOP command provides origin - identification and replay protection for a POP3 session. - Accordingly, a POP3 server which implements both the PASS and APOP - commands should not allow both methods of access for a given user; - that is, for a given mailbox name, either the USER/PASS command - sequence or the APOP command is allowed, but not both. - - Further, note that as the length of the shared secret increases, so - does the difficulty of deriving it. - - Servers that answer -ERR to the USER command are giving potential - attackers clues about which names are valid. - - Use of the PASS command sends passwords in the clear over the - network. - - Use of the RETR and TOP commands sends mail in the clear over the - network. - - Otherwise, security issues are not discussed in this memo. - -14. Acknowledgements - - The POP family has a long and checkered history. Although primarily - a minor revision to RFC 1460, POP3 is based on the ideas presented in - RFCs 918, 937, and 1081. - - In addition, Alfred Grimstad, Keith McCloghrie, and Neil Ostroff - provided significant comments on the APOP command. - - - -Myers & Rose Standards Track [Page 20] - -RFC 1939 POP3 May 1996 - - -15. Authors' Addresses - - John G. Myers - Carnegie-Mellon University - 5000 Forbes Ave - Pittsburgh, PA 15213 - - EMail: jgm+@cmu.edu - - - Marshall T. Rose - Dover Beach Consulting, Inc. - 420 Whisman Court - Mountain View, CA 94043-2186 - - EMail: mrose@dbc.mtview.ca.us - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Myers & Rose Standards Track [Page 21] - -RFC 1939 POP3 May 1996 - - -Appendix A. Differences from RFC 1725 - - This memo is a revision to RFC 1725, a Draft Standard. It makes the - following changes from that document: - - - clarifies that command keywords are case insensitive. - - - specifies that servers must send "+OK" and "-ERR" in - upper case. - - - specifies that the initial greeting is a positive response, - instead of any string which should be a positive response. - - - clarifies behavior for unimplemented commands. - - - makes the USER and PASS commands optional. - - - clarified the set of possible responses to the USER command. - - - reverses the order of the examples in the USER and PASS - commands, to reduce confusion. - - - clarifies that the PASS command may only be given immediately - after a successful USER command. - - - clarified the persistence requirements of UIDs and added some - implementation notes. - - - specifies a UID length limitation of one to 70 octets. - - - specifies a status indicator length limitation - of 512 octets, including the CRLF. - - - clarifies that LIST with no arguments on an empty mailbox - returns success. - - - adds a reference from the LIST command to the Message Format - section - - - clarifies the behavior of QUIT upon failure - - - clarifies the security section to not imply the use of the - USER command with the APOP command. - - - adds references to RFCs 1730 and 1734 - - - clarifies the method by which a UA may enter mail into the - transport system. - - - -Myers & Rose Standards Track [Page 22] - -RFC 1939 POP3 May 1996 - - - - clarifies that the second argument to the TOP command is a - number of lines. - - - changes the suggestion in the Security Considerations section - for a server to not accept both PASS and APOP for a given user - from a "must" to a "should". - - - adds a section on scaling and operational considerations - -Appendix B. Command Index - - APOP ....................................................... 15 - DELE ....................................................... 8 - LIST ....................................................... 6 - NOOP ....................................................... 9 - PASS ....................................................... 14 - QUIT ....................................................... 5 - QUIT ....................................................... 10 - RETR ....................................................... 8 - RSET ....................................................... 9 - STAT ....................................................... 6 - TOP ........................................................ 11 - UIDL ....................................................... 12 - USER ....................................................... 13 - - - - - - - - - - - - - - - - - - - - - - - - - - - -Myers & Rose Standards Track [Page 23] - diff --git a/docs/rfcs/rfc2061.txt b/docs/rfcs/rfc2061.compatibility_IMAP4-IMAP2bis.txt similarity index 100% rename from docs/rfcs/rfc2061.txt rename to docs/rfcs/rfc2061.compatibility_IMAP4-IMAP2bis.txt diff --git a/docs/rfcs/rfc2062.txt b/docs/rfcs/rfc2062.txt deleted file mode 100644 index 865d1da..0000000 --- a/docs/rfcs/rfc2062.txt +++ /dev/null @@ -1,451 +0,0 @@ - - - - - - -Network Working Group M. Crispin -Request for Comments: 2062 University of Washington -Category: Informational December 1996 - - - Internet Message Access Protocol - Obsolete Syntax - -Status of this Memo - - This memo provides information for the Internet community. This memo - does not specify an Internet standard of any kind. Distribution of - this memo is unlimited. - -Abstract - - This document describes obsolete syntax which may be encountered by - IMAP4 implementations which deal with older versions of the Internet - Mail Access Protocol. IMAP4 implementations MAY implement this - syntax in order to maximize interoperability with older - implementations. - - This document repeats information from earlier documents, most - notably RFC 1176 and RFC 1730. - -Obsolete Commands and Fetch Data Items - - The following commands are OBSOLETE. It is NOT required to support - any of these commands or fetch data items in new server - implementations. These commands are documented here for the benefit - of implementors who may wish to support them for compatibility with - old client implementations. - - The section headings of these commands are intended to correspond - with where they would be located in the main document if they were - not obsoleted. - -6.3.OBS.1. FIND ALL.MAILBOXES Command - - Arguments: mailbox name with possible wildcards - - Data: untagged responses: MAILBOX - - Result: OK - find completed - NO - find failure: can't list that name - BAD - command unknown or arguments invalid - - - - - - -Crispin Informational [Page 1] - -RFC 2062 IMAP4 Obsolete December 1996 - - - The FIND ALL.MAILBOXES command returns a subset of names from the - complete set of all names available to the user. It returns zero - or more untagged MAILBOX replies. The mailbox argument to FIND - ALL.MAILBOXES is similar to that for LIST with an empty reference, - except that the characters "%" and "?" match a single character. - - Example: C: A002 FIND ALL.MAILBOXES * - S: * MAILBOX blurdybloop - S: * MAILBOX INBOX - S: A002 OK FIND ALL.MAILBOXES completed - -6.3.OBS.2. FIND MAILBOXES Command - - Arguments: mailbox name with possible wildcards - - Data: untagged responses: MAILBOX - - Result: OK - find completed - NO - find failure: can't list that name - BAD - command unknown or arguments invalid - - The FIND MAILBOXES command returns a subset of names from the set - of names that the user has declared as being "active" or - "subscribed". It returns zero or more untagged MAILBOX replies. - The mailbox argument to FIND MAILBOXES is similar to that for LSUB - with an empty reference, except that the characters "%" and "?" - match a single character. - - Example: C: A002 FIND MAILBOXES * - S: * MAILBOX blurdybloop - S: * MAILBOX INBOX - S: A002 OK FIND MAILBOXES completed - -6.3.OBS.3. SUBSCRIBE MAILBOX Command - - Arguments: mailbox name - - Data: no specific data for this command - - Result: OK - subscribe completed - NO - subscribe failure: can't subscribe to that name - BAD - command unknown or arguments invalid - - The SUBSCRIBE MAILBOX command is identical in effect to the - SUBSCRIBE command. A server which implements this command must be - able to distinguish between a SUBSCRIBE MAILBOX command and a - SUBSCRIBE command with a mailbox name argument of "MAILBOX". - - - - -Crispin Informational [Page 2] - -RFC 2062 IMAP4 Obsolete December 1996 - - - Example: C: A002 SUBSCRIBE MAILBOX #news.comp.mail.mime - S: A002 OK SUBSCRIBE MAILBOX to #news.comp.mail.mime - completed - C: A003 SUBSCRIBE MAILBOX - S: A003 OK SUBSCRIBE to MAILBOX completed - - -6.3.OBS.4. UNSUBSCRIBE MAILBOX Command - - Arguments: mailbox name - - Data: no specific data for this command - - Result: OK - unsubscribe completed - NO - unsubscribe failure: can't unsubscribe that name - BAD - command unknown or arguments invalid - - The UNSUBSCRIBE MAILBOX command is identical in effect to the - UNSUBSCRIBE command. A server which implements this command must - be able to distinguish between a UNSUBSCRIBE MAILBOX command and - an UNSUBSCRIBE command with a mailbox name argument of "MAILBOX". - - Example: C: A002 UNSUBSCRIBE MAILBOX #news.comp.mail.mime - S: A002 OK UNSUBSCRIBE MAILBOX from #news.comp.mail.mime - completed - C: A003 UNSUBSCRIBE MAILBOX - S: A003 OK UNSUBSCRIBE from MAILBOX completed - -6.4.OBS.1 PARTIAL Command - - Arguments: message sequence number - message data item name - position of first octet - number of octets - - Data: untagged responses: FETCH - - Result: OK - partial completed - NO - partial error: can't fetch that data - BAD - command unknown or arguments invalid - - The PARTIAL command is equivalent to the associated FETCH command, - with the added functionality that only the specified number of - octets, beginning at the specified starting octet, are returned. - Only a single message can be fetched at a time. The first octet - of a message, and hence the minimum for the starting octet, is - octet 1. - - - - -Crispin Informational [Page 3] - -RFC 2062 IMAP4 Obsolete December 1996 - - - The following FETCH items are valid data for PARTIAL: RFC822, - RFC822.HEADER, RFC822.TEXT, BODY[
], as well as any .PEEK - forms of these. - - Any partial fetch that attempts to read beyond the end of the text - is truncated as appropriate. If the starting octet is beyond the - end of the text, an empty string is returned. - - The data are returned with the FETCH response. There is no - indication of the range of the partial data in this response. It - is not possible to stream multiple PARTIAL commands of the same - data item without processing and synchronizing at each step, since - streamed commands may be executed out of order. - - There is no requirement that partial fetches follow any sequence. - For example, if a partial fetch of octets 1 through 10000 breaks - in an awkward place for BASE64 decoding, it is permitted to - continue with a partial fetch of 9987 through 19987, etc. - - The handling of the \Seen flag is the same as in the associated - FETCH command. - - Example: C: A005 PARTIAL 4 RFC822 1 1024 - S: * 1 FETCH (RFC822 {1024} - S: Return-Path: - S: ... - S: ......... FLAGS (\Seen)) - S: A005 OK PARTIAL completed - -6.4.5.OBS.1 Obsolete FETCH Data Items - - The following FETCH data items are obsolete: - - BODY[<...>0] A body part number of 0 is the [RFC-822] header of - the message. BODY[0] is functionally equivalent to - BODY[HEADER], differing in the syntax of the - resulting untagged FETCH data (BODY[0] is - returned). - - RFC822.HEADER.LINES - Functionally equivalent to BODY.PEEK[HEADER.LINES - ], differing in the syntax of the - resulting untagged FETCH data (RFC822.HEADER is - returned). - - - - - - - -Crispin Informational [Page 4] - -RFC 2062 IMAP4 Obsolete December 1996 - - - RFC822.HEADER.LINES.NOT - Functionally equivalent to - BODY.PEEK[HEADER.LINES.NOT ], - differing in the syntax of the resulting untagged - FETCH data (RFC822.HEADER is returned). - - RFC822.PEEK Functionally equivalent to BODY.PEEK[], except for - the syntax of the resulting untagged FETCH data - (RFC822 is returned). - - RFC822.TEXT.PEEK - Functionally equivalent to BODY.PEEK[TEXT], except - for the syntax of the resulting untagged FETCH data - (RFC822.TEXT is returned). - -Obsolete Responses - - The following responses are OBSOLETE. Except as noted below, these - responses MUST NOT be transmitted by new server implementations. - Client implementations SHOULD accept these responses. - - The section headings of these responses are intended to correspond - with where they would be located in the main document if they were - not obsoleted. - -7.2.OBS.1. MAILBOX Response - - Data: name - - The MAILBOX response MUST NOT be transmitted by server - implementations except in response to the obsolete FIND MAILBOXES - and FIND ALL.MAILBOXES commands. Client implementations that do - not use these commands MAY ignore this response. It is documented - here for the benefit of implementors who may wish to support it - for compatibility with old client implementations. - - This response occurs as a result of the FIND MAILBOXES and FIND - ALL.MAILBOXES commands. It returns a single name that matches the - FIND specification. There are no attributes or hierarchy - delimiter. - - Example: S: * MAILBOX blurdybloop - - - - - - - - - -Crispin Informational [Page 5] - -RFC 2062 IMAP4 Obsolete December 1996 - - -7.3.OBS.1. COPY Response - - Data: none - - The COPY response MUST NOT be transmitted by new server - implementations. Client implementations MUST ignore the COPY - response. It is documented here for the benefit of client - implementors who may encounter this response from old server - implementations. - - In some experimental versions of this protocol, this response was - returned in response to a COPY command to indicate on a - per-message basis that the message was copied successfully. - - Example: S: * 44 COPY - -7.3.OBS.2. STORE Response - - Data: message data - - The STORE response MUST NOT be transmitted by new server - implementations. Client implementations MUST treat the STORE - response as equivalent to the FETCH response. It is documented - here for the benefit of client implementors who may encounter this - response from old server implementations. - - In some experimental versions of this protocol, this response was - returned instead of FETCH in response to a STORE command to report - the new value of the flags. - - Example: S: * 69 STORE (FLAGS (\Deleted)) - -Formal Syntax of Obsolete Commands and Responses - - Each obsolete syntax rule that is suffixed with "_old" is added to - the corresponding name in the formal syntax. For example, - command_auth_old adds the FIND command to command_auth. - - command_auth_old ::= find - - command_select_old - ::= partial - - date_year_old ::= 2digit - ;; (year - 1900) - - date_time_old ::= <"> date_day_fixed "-" date_month "-" date_year - SPACE time "-" zone_name <"> - - - -Crispin Informational [Page 6] - -RFC 2062 IMAP4 Obsolete December 1996 - - - find ::= "FIND" SPACE ["ALL."] "MAILBOXES" SPACE - list_mailbox - - fetch_att_old ::= "RFC822.HEADER.LINES" [".NOT"] SPACE header_list / - fetch_text_old - - fetch_text_old ::= "BODY" [".PEEK"] section_old / - "RFC822" [".HEADER" / ".TEXT" [".PEEK"]] - - msg_data_old ::= "COPY" / ("STORE" SPACE msg_att) - - partial ::= "PARTIAL" SPACE nz_number SPACE fetch_text_old SPACE - number SPACE number - - section_old ::= "[" (number ["." number]) "]" - - subscribe_old ::= "SUBSCRIBE" SPACE "MAILBOX" SPACE mailbox - - unsubscribe_old ::= "UNSUBSCRIBE" SPACE "MAILBOX" SPACE mailbox - - zone_name ::= "UT" / "GMT" / "Z" / ;; +0000 - "AST" / "EDT" / ;; -0400 - "EST" / "CDT" / ;; -0500 - "CST" / "MDT" / ;; -0600 - "MST" / "PDT" / ;; -0700 - "PST" / "YDT" / ;; -0800 - "YST" / "HDT" / ;; -0900 - "HST" / "BDT" / ;; -1000 - "BST" / ;; -1100 - "A" / "B" / "C" / "D" / "E" / "F" / ;; +1 to +6 - "G" / "H" / "I" / "K" / "L" / "M" / ;; +7 to +12 - "N" / "O" / "P" / "Q" / "R" / "S" / ;; -1 to -6 - "T" / "U" / "V" / "W" / "X" / "Y" ;; -7 to -12 - -Security Considerations - - Security issues are not discussed in this memo. - - - - - - - - - - - - - - -Crispin Informational [Page 7] - -RFC 2062 IMAP4 Obsolete December 1996 - - -Author's Address - - Mark R. Crispin - Networks and Distributed Computing - University of Washington - 4545 15th Aveneue NE - Seattle, WA 98105-4527 - - Phone: (206) 543-5762 - EMail: MRC@CAC.Washington.EDU - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Crispin Informational [Page 8] - diff --git a/docs/rfcs/rfc2086.txt b/docs/rfcs/rfc2086.IMAP4_ACL_extension.txt similarity index 100% rename from docs/rfcs/rfc2086.txt rename to docs/rfcs/rfc2086.IMAP4_ACL_extension.txt diff --git a/docs/rfcs/rfc2087.txt b/docs/rfcs/rfc2087.IMAP4_QUOTA_extension.txt similarity index 100% rename from docs/rfcs/rfc2087.txt rename to docs/rfcs/rfc2087.IMAP4_QUOTA_extension.txt diff --git a/docs/rfcs/rfc2088.txt b/docs/rfcs/rfc2088.IMAP4_non_synchronizing_literals.txt similarity index 100% rename from docs/rfcs/rfc2088.txt rename to docs/rfcs/rfc2088.IMAP4_non_synchronizing_literals.txt diff --git a/docs/rfcs/rfc2095.txt b/docs/rfcs/rfc2095.IMAP-POP_AUTHorize_extension.txt similarity index 100% rename from docs/rfcs/rfc2095.txt rename to docs/rfcs/rfc2095.IMAP-POP_AUTHorize_extension.txt diff --git a/docs/rfcs/rfc2177.txt b/docs/rfcs/rfc2177.IMAP4_IDLE_command.txt similarity index 100% rename from docs/rfcs/rfc2177.txt rename to docs/rfcs/rfc2177.IMAP4_IDLE_command.txt diff --git a/docs/rfcs/rfc2180.txt b/docs/rfcs/rfc2180.IMAP4_multi-accessed_Mailbox_practice.txt similarity index 100% rename from docs/rfcs/rfc2180.txt rename to docs/rfcs/rfc2180.IMAP4_multi-accessed_Mailbox_practice.txt diff --git a/docs/rfcs/rfc2192.txt b/docs/rfcs/rfc2192.IMAP_URL_scheme.txt similarity index 100% rename from docs/rfcs/rfc2192.txt rename to docs/rfcs/rfc2192.IMAP_URL_scheme.txt diff --git a/docs/rfcs/rfc2193.txt b/docs/rfcs/rfc2193.IMAP4_Mailbox_referrals.txt similarity index 100% rename from docs/rfcs/rfc2193.txt rename to docs/rfcs/rfc2193.IMAP4_Mailbox_referrals.txt diff --git a/docs/rfcs/rfc2195.txt b/docs/rfcs/rfc2195.IMAP-POP_AUTHorize_extension.txt similarity index 100% rename from docs/rfcs/rfc2195.txt rename to docs/rfcs/rfc2195.IMAP-POP_AUTHorize_extension.txt diff --git a/docs/rfcs/rfc2221.txt b/docs/rfcs/rfc2221.IMAP4_Login_referrals.txt similarity index 100% rename from docs/rfcs/rfc2221.txt rename to docs/rfcs/rfc2221.IMAP4_Login_referrals.txt diff --git a/docs/rfcs/rfc2244.txt b/docs/rfcs/rfc2244.ACAP.txt similarity index 100% rename from docs/rfcs/rfc2244.txt rename to docs/rfcs/rfc2244.ACAP.txt diff --git a/docs/rfcs/rfc2342.txt b/docs/rfcs/rfc2342.IMAP4_Namespace.txt similarity index 100% rename from docs/rfcs/rfc2342.txt rename to docs/rfcs/rfc2342.IMAP4_Namespace.txt diff --git a/docs/rfcs/rfc2359.txt b/docs/rfcs/rfc2359.IMAP4_UIDPLUS_extension.txt similarity index 100% rename from docs/rfcs/rfc2359.txt rename to docs/rfcs/rfc2359.IMAP4_UIDPLUS_extension.txt diff --git a/docs/rfcs/rfc2595.txt b/docs/rfcs/rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt similarity index 100% rename from docs/rfcs/rfc2595.txt rename to docs/rfcs/rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt diff --git a/docs/rfcs/rfc2683.txt b/docs/rfcs/rfc2683.IMAP4_Implementation_recommendations.txt similarity index 100% rename from docs/rfcs/rfc2683.txt rename to docs/rfcs/rfc2683.IMAP4_Implementation_recommendations.txt diff --git a/docs/rfcs/rfc2821.txt b/docs/rfcs/rfc2821.txt deleted file mode 100644 index 0eac911..0000000 --- a/docs/rfcs/rfc2821.txt +++ /dev/null @@ -1,4427 +0,0 @@ - - - - - - -Network Working Group J. Klensin, Editor -Request for Comments: 2821 AT&T Laboratories -Obsoletes: 821, 974, 1869 April 2001 -Updates: 1123 -Category: Standards Track - - - Simple Mail Transfer Protocol - -Status of this Memo - - This document specifies an Internet standards track protocol for the - Internet community, and requests discussion and suggestions for - improvements. Please refer to the current edition of the "Internet - Official Protocol Standards" (STD 1) for the standardization state - and status of this protocol. Distribution of this memo is unlimited. - -Copyright Notice - - Copyright (C) The Internet Society (2001). All Rights Reserved. - -Abstract - - This document is a self-contained specification of the basic protocol - for the Internet electronic mail transport. It consolidates, updates - and clarifies, but doesn't add new or change existing functionality - of the following: - - - the original SMTP (Simple Mail Transfer Protocol) specification of - RFC 821 [30], - - - domain name system requirements and implications for mail - transport from RFC 1035 [22] and RFC 974 [27], - - - the clarifications and applicability statements in RFC 1123 [2], - and - - - material drawn from the SMTP Extension mechanisms [19]. - - It obsoletes RFC 821, RFC 974, and updates RFC 1123 (replaces the - mail transport materials of RFC 1123). However, RFC 821 specifies - some features that were not in significant use in the Internet by the - mid-1990s and (in appendices) some additional transport models. - Those sections are omitted here in the interest of clarity and - brevity; readers needing them should refer to RFC 821. - - - - - - -Klensin Standards Track [Page 1] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - It also includes some additional material from RFC 1123 that required - amplification. This material has been identified in multiple ways, - mostly by tracking flaming on various lists and newsgroups and - problems of unusual readings or interpretations that have appeared as - the SMTP extensions have been deployed. Where this specification - moves beyond consolidation and actually differs from earlier - documents, it supersedes them technically as well as textually. - - Although SMTP was designed as a mail transport and delivery protocol, - this specification also contains information that is important to its - use as a 'mail submission' protocol, as recommended for POP [3, 26] - and IMAP [6]. Additional submission issues are discussed in RFC 2476 - [15]. - - Section 2.3 provides definitions of terms specific to this document. - Except when the historical terminology is necessary for clarity, this - document uses the current 'client' and 'server' terminology to - identify the sending and receiving SMTP processes, respectively. - - A companion document [32] discusses message headers, message bodies - and formats and structures for them, and their relationship. - -Table of Contents - - 1. Introduction .................................................. 4 - 2. The SMTP Model ................................................ 5 - 2.1 Basic Structure .............................................. 5 - 2.2 The Extension Model .......................................... 7 - 2.2.1 Background ................................................. 7 - 2.2.2 Definition and Registration of Extensions .................. 8 - 2.3 Terminology .................................................. 9 - 2.3.1 Mail Objects ............................................... 10 - 2.3.2 Senders and Receivers ...................................... 10 - 2.3.3 Mail Agents and Message Stores ............................. 10 - 2.3.4 Host ....................................................... 11 - 2.3.5 Domain ..................................................... 11 - 2.3.6 Buffer and State Table ..................................... 11 - 2.3.7 Lines ...................................................... 12 - 2.3.8 Originator, Delivery, Relay, and Gateway Systems ........... 12 - 2.3.9 Message Content and Mail Data .............................. 13 - 2.3.10 Mailbox and Address ....................................... 13 - 2.3.11 Reply ..................................................... 13 - 2.4 General Syntax Principles and Transaction Model .............. 13 - 3. The SMTP Procedures: An Overview .............................. 15 - 3.1 Session Initiation ........................................... 15 - 3.2 Client Initiation ............................................ 16 - 3.3 Mail Transactions ............................................ 16 - 3.4 Forwarding for Address Correction or Updating ................ 19 - - - -Klensin Standards Track [Page 2] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - 3.5 Commands for Debugging Addresses ............................. 20 - 3.5.1 Overview ................................................... 20 - 3.5.2 VRFY Normal Response ....................................... 22 - 3.5.3 Meaning of VRFY or EXPN Success Response ................... 22 - 3.5.4 Semantics and Applications of EXPN ......................... 23 - 3.6 Domains ...................................................... 23 - 3.7 Relaying ..................................................... 24 - 3.8 Mail Gatewaying .............................................. 25 - 3.8.1 Header Fields in Gatewaying ................................ 26 - 3.8.2 Received Lines in Gatewaying ............................... 26 - 3.8.3 Addresses in Gatewaying .................................... 26 - 3.8.4 Other Header Fields in Gatewaying .......................... 27 - 3.8.5 Envelopes in Gatewaying .................................... 27 - 3.9 Terminating Sessions and Connections ......................... 27 - 3.10 Mailing Lists and Aliases ................................... 28 - 3.10.1 Alias ..................................................... 28 - 3.10.2 List ...................................................... 28 - 4. The SMTP Specifications ....................................... 29 - 4.1 SMTP Commands ................................................ 29 - 4.1.1 Command Semantics and Syntax ............................... 29 - 4.1.1.1 Extended HELLO (EHLO) or HELLO (HELO) ................... 29 - 4.1.1.2 MAIL (MAIL) .............................................. 31 - 4.1.1.3 RECIPIENT (RCPT) ......................................... 31 - 4.1.1.4 DATA (DATA) .............................................. 33 - 4.1.1.5 RESET (RSET) ............................................. 34 - 4.1.1.6 VERIFY (VRFY) ............................................ 35 - 4.1.1.7 EXPAND (EXPN) ............................................ 35 - 4.1.1.8 HELP (HELP) .............................................. 35 - 4.1.1.9 NOOP (NOOP) .............................................. 35 - 4.1.1.10 QUIT (QUIT) ............................................. 36 - 4.1.2 Command Argument Syntax .................................... 36 - 4.1.3 Address Literals ........................................... 38 - 4.1.4 Order of Commands .......................................... 39 - 4.1.5 Private-use Commands ....................................... 40 - 4.2 SMTP Replies ................................................ 40 - 4.2.1 Reply Code Severities and Theory ........................... 42 - 4.2.2 Reply Codes by Function Groups ............................. 44 - 4.2.3 Reply Codes in Numeric Order .............................. 45 - 4.2.4 Reply Code 502 ............................................. 46 - 4.2.5 Reply Codes After DATA and the Subsequent . .... 46 - 4.3 Sequencing of Commands and Replies ........................... 47 - 4.3.1 Sequencing Overview ........................................ 47 - 4.3.2 Command-Reply Sequences .................................... 48 - 4.4 Trace Information ............................................ 49 - 4.5 Additional Implementation Issues ............................. 53 - 4.5.1 Minimum Implementation ..................................... 53 - 4.5.2 Transparency ............................................... 53 - 4.5.3 Sizes and Timeouts ......................................... 54 - - - -Klensin Standards Track [Page 3] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - 4.5.3.1 Size limits and minimums ................................. 54 - 4.5.3.2 Timeouts ................................................. 56 - 4.5.4 Retry Strategies ........................................... 57 - 4.5.4.1 Sending Strategy ......................................... 58 - 4.5.4.2 Receiving Strategy ....................................... 59 - 4.5.5 Messages with a null reverse-path .......................... 59 - 5. Address Resolution and Mail Handling .......................... 60 - 6. Problem Detection and Handling ................................ 62 - 6.1 Reliable Delivery and Replies by Email ....................... 62 - 6.2 Loop Detection ............................................... 63 - 6.3 Compensating for Irregularities .............................. 63 - 7. Security Considerations ....................................... 64 - 7.1 Mail Security and Spoofing ................................... 64 - 7.2 "Blind" Copies ............................................... 65 - 7.3 VRFY, EXPN, and Security ..................................... 65 - 7.4 Information Disclosure in Announcements ...................... 66 - 7.5 Information Disclosure in Trace Fields ....................... 66 - 7.6 Information Disclosure in Message Forwarding ................. 67 - 7.7 Scope of Operation of SMTP Servers ........................... 67 - 8. IANA Considerations ........................................... 67 - 9. References .................................................... 68 - 10. Editor's Address ............................................. 70 - 11. Acknowledgments .............................................. 70 - Appendices ....................................................... 71 - A. TCP Transport Service ......................................... 71 - B. Generating SMTP Commands from RFC 822 Headers ................. 71 - C. Source Routes ................................................. 72 - D. Scenarios ..................................................... 73 - E. Other Gateway Issues .......................................... 76 - F. Deprecated Features of RFC 821 ................................ 76 - Full Copyright Statement ......................................... 79 - -1. Introduction - - The objective of the Simple Mail Transfer Protocol (SMTP) is to - transfer mail reliably and efficiently. - - SMTP is independent of the particular transmission subsystem and - requires only a reliable ordered data stream channel. While this - document specifically discusses transport over TCP, other transports - are possible. Appendices to RFC 821 describe some of them. - - An important feature of SMTP is its capability to transport mail - across networks, usually referred to as "SMTP mail relaying" (see - section 3.8). A network consists of the mutually-TCP-accessible - hosts on the public Internet, the mutually-TCP-accessible hosts on a - firewall-isolated TCP/IP Intranet, or hosts in some other LAN or WAN - environment utilizing a non-TCP transport-level protocol. Using - - - -Klensin Standards Track [Page 4] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - SMTP, a process can transfer mail to another process on the same - network or to some other network via a relay or gateway process - accessible to both networks. - - In this way, a mail message may pass through a number of intermediate - relay or gateway hosts on its path from sender to ultimate recipient. - The Mail eXchanger mechanisms of the domain name system [22, 27] (and - section 5 of this document) are used to identify the appropriate - next-hop destination for a message being transported. - -2. The SMTP Model - -2.1 Basic Structure - - The SMTP design can be pictured as: - - +----------+ +----------+ - +------+ | | | | - | User |<-->| | SMTP | | - +------+ | Client- |Commands/Replies| Server- | - +------+ | SMTP |<-------------->| SMTP | +------+ - | File |<-->| | and Mail | |<-->| File | - |System| | | | | |System| - +------+ +----------+ +----------+ +------+ - SMTP client SMTP server - - When an SMTP client has a message to transmit, it establishes a two- - way transmission channel to an SMTP server. The responsibility of an - SMTP client is to transfer mail messages to one or more SMTP servers, - or report its failure to do so. - - The means by which a mail message is presented to an SMTP client, and - how that client determines the domain name(s) to which mail messages - are to be transferred is a local matter, and is not addressed by this - document. In some cases, the domain name(s) transferred to, or - determined by, an SMTP client will identify the final destination(s) - of the mail message. In other cases, common with SMTP clients - associated with implementations of the POP [3, 26] or IMAP [6] - protocols, or when the SMTP client is inside an isolated transport - service environment, the domain name determined will identify an - intermediate destination through which all mail messages are to be - relayed. SMTP clients that transfer all traffic, regardless of the - target domain names associated with the individual messages, or that - do not maintain queues for retrying message transmissions that - initially cannot be completed, may otherwise conform to this - specification but are not considered fully-capable. Fully-capable - SMTP implementations, including the relays used by these less capable - - - - -Klensin Standards Track [Page 5] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - ones, and their destinations, are expected to support all of the - queuing, retrying, and alternate address functions discussed in this - specification. - - The means by which an SMTP client, once it has determined a target - domain name, determines the identity of an SMTP server to which a - copy of a message is to be transferred, and then performs that - transfer, is covered by this document. To effect a mail transfer to - an SMTP server, an SMTP client establishes a two-way transmission - channel to that SMTP server. An SMTP client determines the address - of an appropriate host running an SMTP server by resolving a - destination domain name to either an intermediate Mail eXchanger host - or a final target host. - - An SMTP server may be either the ultimate destination or an - intermediate "relay" (that is, it may assume the role of an SMTP - client after receiving the message) or "gateway" (that is, it may - transport the message further using some protocol other than SMTP). - SMTP commands are generated by the SMTP client and sent to the SMTP - server. SMTP replies are sent from the SMTP server to the SMTP - client in response to the commands. - - In other words, message transfer can occur in a single connection - between the original SMTP-sender and the final SMTP-recipient, or can - occur in a series of hops through intermediary systems. In either - case, a formal handoff of responsibility for the message occurs: the - protocol requires that a server accept responsibility for either - delivering a message or properly reporting the failure to do so. - - Once the transmission channel is established and initial handshaking - completed, the SMTP client normally initiates a mail transaction. - Such a transaction consists of a series of commands to specify the - originator and destination of the mail and transmission of the - message content (including any headers or other structure) itself. - When the same message is sent to multiple recipients, this protocol - encourages the transmission of only one copy of the data for all - recipients at the same destination (or intermediate relay) host. - - The server responds to each command with a reply; replies may - indicate that the command was accepted, that additional commands are - expected, or that a temporary or permanent error condition exists. - Commands specifying the sender or recipients may include server- - permitted SMTP service extension requests as discussed in section - 2.2. The dialog is purposely lock-step, one-at-a-time, although this - can be modified by mutually-agreed extension requests such as command - pipelining [13]. - - - - - -Klensin Standards Track [Page 6] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - Once a given mail message has been transmitted, the client may either - request that the connection be shut down or may initiate other mail - transactions. In addition, an SMTP client may use a connection to an - SMTP server for ancillary services such as verification of email - addresses or retrieval of mailing list subscriber addresses. - - As suggested above, this protocol provides mechanisms for the - transmission of mail. This transmission normally occurs directly - from the sending user's host to the receiving user's host when the - two hosts are connected to the same transport service. When they are - not connected to the same transport service, transmission occurs via - one or more relay SMTP servers. An intermediate host that acts as - either an SMTP relay or as a gateway into some other transmission - environment is usually selected through the use of the domain name - service (DNS) Mail eXchanger mechanism. - - Usually, intermediate hosts are determined via the DNS MX record, not - by explicit "source" routing (see section 5 and appendices C and - F.2). - -2.2 The Extension Model - -2.2.1 Background - - In an effort that started in 1990, approximately a decade after RFC - 821 was completed, the protocol was modified with a "service - extensions" model that permits the client and server to agree to - utilize shared functionality beyond the original SMTP requirements. - The SMTP extension mechanism defines a means whereby an extended SMTP - client and server may recognize each other, and the server can inform - the client as to the service extensions that it supports. - - Contemporary SMTP implementations MUST support the basic extension - mechanisms. For instance, servers MUST support the EHLO command even - if they do not implement any specific extensions and clients SHOULD - preferentially utilize EHLO rather than HELO. (However, for - compatibility with older conforming implementations, SMTP clients and - servers MUST support the original HELO mechanisms as a fallback.) - Unless the different characteristics of HELO must be identified for - interoperability purposes, this document discusses only EHLO. - - SMTP is widely deployed and high-quality implementations have proven - to be very robust. However, the Internet community now considers - some services to be important that were not anticipated when the - protocol was first designed. If support for those services is to be - added, it must be done in a way that permits older implementations to - continue working acceptably. The extension framework consists of: - - - - -Klensin Standards Track [Page 7] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - - The SMTP command EHLO, superseding the earlier HELO, - - - a registry of SMTP service extensions, - - - additional parameters to the SMTP MAIL and RCPT commands, and - - - optional replacements for commands defined in this protocol, such - as for DATA in non-ASCII transmissions [33]. - - SMTP's strength comes primarily from its simplicity. Experience with - many protocols has shown that protocols with few options tend towards - ubiquity, whereas protocols with many options tend towards obscurity. - - Each and every extension, regardless of its benefits, must be - carefully scrutinized with respect to its implementation, deployment, - and interoperability costs. In many cases, the cost of extending the - SMTP service will likely outweigh the benefit. - -2.2.2 Definition and Registration of Extensions - - The IANA maintains a registry of SMTP service extensions. A - corresponding EHLO keyword value is associated with each extension. - Each service extension registered with the IANA must be defined in a - formal standards-track or IESG-approved experimental protocol - document. The definition must include: - - - the textual name of the SMTP service extension; - - - the EHLO keyword value associated with the extension; - - - the syntax and possible values of parameters associated with the - EHLO keyword value; - - - any additional SMTP verbs associated with the extension - (additional verbs will usually be, but are not required to be, the - same as the EHLO keyword value); - - - any new parameters the extension associates with the MAIL or RCPT - verbs; - - - a description of how support for the extension affects the - behavior of a server and client SMTP; and, - - - the increment by which the extension is increasing the maximum - length of the commands MAIL and/or RCPT, over that specified in - this standard. - - - - - -Klensin Standards Track [Page 8] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - In addition, any EHLO keyword value starting with an upper or lower - case "X" refers to a local SMTP service extension used exclusively - through bilateral agreement. Keywords beginning with "X" MUST NOT be - used in a registered service extension. Conversely, keyword values - presented in the EHLO response that do not begin with "X" MUST - correspond to a standard, standards-track, or IESG-approved - experimental SMTP service extension registered with IANA. A - conforming server MUST NOT offer non-"X"-prefixed keyword values that - are not described in a registered extension. - - Additional verbs and parameter names are bound by the same rules as - EHLO keywords; specifically, verbs beginning with "X" are local - extensions that may not be registered or standardized. Conversely, - verbs not beginning with "X" must always be registered. - -2.3 Terminology - - The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - document are to be interpreted as described below. - - 1. MUST This word, or the terms "REQUIRED" or "SHALL", mean that - the definition is an absolute requirement of the specification. - - 2. MUST NOT This phrase, or the phrase "SHALL NOT", mean that the - definition is an absolute prohibition of the specification. - - 3. SHOULD This word, or the adjective "RECOMMENDED", mean that - there may exist valid reasons in particular circumstances to - ignore a particular item, but the full implications must be - understood and carefully weighed before choosing a different - course. - - 4. SHOULD NOT This phrase, or the phrase "NOT RECOMMENDED" mean - that there may exist valid reasons in particular circumstances - when the particular behavior is acceptable or even useful, but the - full implications should be understood and the case carefully - weighed before implementing any behavior described with this - label. - - 5. MAY This word, or the adjective "OPTIONAL", mean that an item is - truly optional. One vendor may choose to include the item because - a particular marketplace requires it or because the vendor feels - that it enhances the product while another vendor may omit the - same item. An implementation which does not include a particular - option MUST be prepared to interoperate with another - implementation which does include the option, though perhaps with - reduced functionality. In the same vein an implementation which - - - -Klensin Standards Track [Page 9] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - does include a particular option MUST be prepared to interoperate - with another implementation which does not include the option - (except, of course, for the feature the option provides.) - -2.3.1 Mail Objects - - SMTP transports a mail object. A mail object contains an envelope - and content. - - The SMTP envelope is sent as a series of SMTP protocol units - (described in section 3). It consists of an originator address (to - which error reports should be directed); one or more recipient - addresses; and optional protocol extension material. Historically, - variations on the recipient address specification command (RCPT TO) - could be used to specify alternate delivery modes, such as immediate - display; those variations have now been deprecated (see appendix F, - section F.6). - - The SMTP content is sent in the SMTP DATA protocol unit and has two - parts: the headers and the body. If the content conforms to other - contemporary standards, the headers form a collection of field/value - pairs structured as in the message format specification [32]; the - body, if structured, is defined according to MIME [12]. The content - is textual in nature, expressed using the US-ASCII repertoire [1]. - Although SMTP extensions (such as "8BITMIME" [20]) may relax this - restriction for the content body, the content headers are always - encoded using the US-ASCII repertoire. A MIME extension [23] defines - an algorithm for representing header values outside the US-ASCII - repertoire, while still encoding them using the US-ASCII repertoire. - -2.3.2 Senders and Receivers - - In RFC 821, the two hosts participating in an SMTP transaction were - described as the "SMTP-sender" and "SMTP-receiver". This document - has been changed to reflect current industry terminology and hence - refers to them as the "SMTP client" (or sometimes just "the client") - and "SMTP server" (or just "the server"), respectively. Since a - given host may act both as server and client in a relay situation, - "receiver" and "sender" terminology is still used where needed for - clarity. - -2.3.3 Mail Agents and Message Stores - - Additional mail system terminology became common after RFC 821 was - published and, where convenient, is used in this specification. In - particular, SMTP servers and clients provide a mail transport service - and therefore act as "Mail Transfer Agents" (MTAs). "Mail User - Agents" (MUAs or UAs) are normally thought of as the sources and - - - -Klensin Standards Track [Page 10] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - targets of mail. At the source, an MUA might collect mail to be - transmitted from a user and hand it off to an MTA; the final - ("delivery") MTA would be thought of as handing the mail off to an - MUA (or at least transferring responsibility to it, e.g., by - depositing the message in a "message store"). However, while these - terms are used with at least the appearance of great precision in - other environments, the implied boundaries between MUAs and MTAs - often do not accurately match common, and conforming, practices with - Internet mail. Hence, the reader should be cautious about inferring - the strong relationships and responsibilities that might be implied - if these terms were used elsewhere. - -2.3.4 Host - - For the purposes of this specification, a host is a computer system - attached to the Internet (or, in some cases, to a private TCP/IP - network) and supporting the SMTP protocol. Hosts are known by names - (see "domain"); identifying them by numerical address is discouraged. - -2.3.5 Domain - - A domain (or domain name) consists of one or more dot-separated - components. These components ("labels" in DNS terminology [22]) are - restricted for SMTP purposes to consist of a sequence of letters, - digits, and hyphens drawn from the ASCII character set [1]. Domain - names are used as names of hosts and of other entities in the domain - name hierarchy. For example, a domain may refer to an alias (label - of a CNAME RR) or the label of Mail eXchanger records to be used to - deliver mail instead of representing a host name. See [22] and - section 5 of this specification. - - The domain name, as described in this document and in [22], is the - entire, fully-qualified name (often referred to as an "FQDN"). A - domain name that is not in FQDN form is no more than a local alias. - Local aliases MUST NOT appear in any SMTP transaction. - -2.3.6 Buffer and State Table - - SMTP sessions are stateful, with both parties carefully maintaining a - common view of the current state. In this document we model this - state by a virtual "buffer" and a "state table" on the server which - may be used by the client to, for example, "clear the buffer" or - "reset the state table," causing the information in the buffer to be - discarded and the state to be returned to some previous state. - - - - - - - -Klensin Standards Track [Page 11] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -2.3.7 Lines - - SMTP commands and, unless altered by a service extension, message - data, are transmitted in "lines". Lines consist of zero or more data - characters terminated by the sequence ASCII character "CR" (hex value - 0D) followed immediately by ASCII character "LF" (hex value 0A). - This termination sequence is denoted as in this document. - Conforming implementations MUST NOT recognize or generate any other - character or character sequence as a line terminator. Limits MAY be - imposed on line lengths by servers (see section 4.5.3). - - In addition, the appearance of "bare" "CR" or "LF" characters in text - (i.e., either without the other) has a long history of causing - problems in mail implementations and applications that use the mail - system as a tool. SMTP client implementations MUST NOT transmit - these characters except when they are intended as line terminators - and then MUST, as indicated above, transmit them only as a - sequence. - -2.3.8 Originator, Delivery, Relay, and Gateway Systems - - This specification makes a distinction among four types of SMTP - systems, based on the role those systems play in transmitting - electronic mail. An "originating" system (sometimes called an SMTP - originator) introduces mail into the Internet or, more generally, - into a transport service environment. A "delivery" SMTP system is - one that receives mail from a transport service environment and - passes it to a mail user agent or deposits it in a message store - which a mail user agent is expected to subsequently access. A - "relay" SMTP system (usually referred to just as a "relay") receives - mail from an SMTP client and transmits it, without modification to - the message data other than adding trace information, to another SMTP - server for further relaying or for delivery. - - A "gateway" SMTP system (usually referred to just as a "gateway") - receives mail from a client system in one transport environment and - transmits it to a server system in another transport environment. - Differences in protocols or message semantics between the transport - environments on either side of a gateway may require that the gateway - system perform transformations to the message that are not permitted - to SMTP relay systems. For the purposes of this specification, - firewalls that rewrite addresses should be considered as gateways, - even if SMTP is used on both sides of them (see [11]). - - - - - - - - -Klensin Standards Track [Page 12] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -2.3.9 Message Content and Mail Data - - The terms "message content" and "mail data" are used interchangeably - in this document to describe the material transmitted after the DATA - command is accepted and before the end of data indication is - transmitted. Message content includes message headers and the - possibly-structured message body. The MIME specification [12] - provides the standard mechanisms for structured message bodies. - -2.3.10 Mailbox and Address - - As used in this specification, an "address" is a character string - that identifies a user to whom mail will be sent or a location into - which mail will be deposited. The term "mailbox" refers to that - depository. The two terms are typically used interchangeably unless - the distinction between the location in which mail is placed (the - mailbox) and a reference to it (the address) is important. An - address normally consists of user and domain specifications. The - standard mailbox naming convention is defined to be "local- - part@domain": contemporary usage permits a much broader set of - applications than simple "user names". Consequently, and due to a - long history of problems when intermediate hosts have attempted to - optimize transport by modifying them, the local-part MUST be - interpreted and assigned semantics only by the host specified in the - domain part of the address. - -2.3.11 Reply - - An SMTP reply is an acknowledgment (positive or negative) sent from - receiver to sender via the transmission channel in response to a - command. The general form of a reply is a numeric completion code - (indicating failure or success) usually followed by a text string. - The codes are for use by programs and the text is usually intended - for human users. Recent work [34] has specified further structuring - of the reply strings, including the use of supplemental and more - specific completion codes. - -2.4 General Syntax Principles and Transaction Model - - SMTP commands and replies have a rigid syntax. All commands begin - with a command verb. All Replies begin with a three digit numeric - code. In some commands and replies, arguments MUST follow the verb - or reply code. Some commands do not accept arguments (after the - verb), and some reply codes are followed, sometimes optionally, by - free form text. In both cases, where text appears, it is separated - from the verb or reply code by a space character. Complete - definitions of commands and replies appear in section 4. - - - - -Klensin Standards Track [Page 13] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - Verbs and argument values (e.g., "TO:" or "to:" in the RCPT command - and extension name keywords) are not case sensitive, with the sole - exception in this specification of a mailbox local-part (SMTP - Extensions may explicitly specify case-sensitive elements). That is, - a command verb, an argument value other than a mailbox local-part, - and free form text MAY be encoded in upper case, lower case, or any - mixture of upper and lower case with no impact on its meaning. This - is NOT true of a mailbox local-part. The local-part of a mailbox - MUST BE treated as case sensitive. Therefore, SMTP implementations - MUST take care to preserve the case of mailbox local-parts. Mailbox - domains are not case sensitive. In particular, for some hosts the - user "smith" is different from the user "Smith". However, exploiting - the case sensitivity of mailbox local-parts impedes interoperability - and is discouraged. - - A few SMTP servers, in violation of this specification (and RFC 821) - require that command verbs be encoded by clients in upper case. - Implementations MAY wish to employ this encoding to accommodate those - servers. - - The argument field consists of a variable length character string - ending with the end of the line, i.e., with the character sequence - . The receiver will take no action until this sequence is - received. - - The syntax for each command is shown with the discussion of that - command. Common elements and parameters are shown in section 4.1.2. - - Commands and replies are composed of characters from the ASCII - character set [1]. When the transport service provides an 8-bit byte - (octet) transmission channel, each 7-bit character is transmitted - right justified in an octet with the high order bit cleared to zero. - More specifically, the unextended SMTP service provides seven bit - transport only. An originating SMTP client which has not - successfully negotiated an appropriate extension with a particular - server MUST NOT transmit messages with information in the high-order - bit of octets. If such messages are transmitted in violation of this - rule, receiving SMTP servers MAY clear the high-order bit or reject - the message as invalid. In general, a relay SMTP SHOULD assume that - the message content it has received is valid and, assuming that the - envelope permits doing so, relay it without inspecting that content. - Of course, if the content is mislabeled and the data path cannot - accept the actual content, this may result in ultimate delivery of a - severely garbled message to the recipient. Delivery SMTP systems MAY - reject ("bounce") such messages rather than deliver them. No sending - SMTP system is permitted to send envelope commands in any character - - - - - -Klensin Standards Track [Page 14] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - set other than US-ASCII; receiving systems SHOULD reject such - commands, normally using "500 syntax error - invalid character" - replies. - - Eight-bit message content transmission MAY be requested of the server - by a client using extended SMTP facilities, notably the "8BITMIME" - extension [20]. 8BITMIME SHOULD be supported by SMTP servers. - However, it MUST not be construed as authorization to transmit - unrestricted eight bit material. 8BITMIME MUST NOT be requested by - senders for material with the high bit on that is not in MIME format - with an appropriate content-transfer encoding; servers MAY reject - such messages. - - The metalinguistic notation used in this document corresponds to the - "Augmented BNF" used in other Internet mail system documents. The - reader who is not familiar with that syntax should consult the ABNF - specification [8]. Metalanguage terms used in running text are - surrounded by pointed brackets (e.g., ) for clarity. - -3. The SMTP Procedures: An Overview - - This section contains descriptions of the procedures used in SMTP: - session initiation, the mail transaction, forwarding mail, verifying - mailbox names and expanding mailing lists, and the opening and - closing exchanges. Comments on relaying, a note on mail domains, and - a discussion of changing roles are included at the end of this - section. Several complete scenarios are presented in appendix D. - -3.1 Session Initiation - - An SMTP session is initiated when a client opens a connection to a - server and the server responds with an opening message. - - SMTP server implementations MAY include identification of their - software and version information in the connection greeting reply - after the 220 code, a practice that permits more efficient isolation - and repair of any problems. Implementations MAY make provision for - SMTP servers to disable the software and version announcement where - it causes security concerns. While some systems also identify their - contact point for mail problems, this is not a substitute for - maintaining the required "postmaster" address (see section 4.5.1). - - The SMTP protocol allows a server to formally reject a transaction - while still allowing the initial connection as follows: a 554 - response MAY be given in the initial connection opening message - instead of the 220. A server taking this approach MUST still wait - for the client to send a QUIT (see section 4.1.1.10) before closing - the connection and SHOULD respond to any intervening commands with - - - -Klensin Standards Track [Page 15] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - "503 bad sequence of commands". Since an attempt to make an SMTP - connection to such a system is probably in error, a server returning - a 554 response on connection opening SHOULD provide enough - information in the reply text to facilitate debugging of the sending - system. - -3.2 Client Initiation - - Once the server has sent the welcoming message and the client has - received it, the client normally sends the EHLO command to the - server, indicating the client's identity. In addition to opening the - session, use of EHLO indicates that the client is able to process - service extensions and requests that the server provide a list of the - extensions it supports. Older SMTP systems which are unable to - support service extensions and contemporary clients which do not - require service extensions in the mail session being initiated, MAY - use HELO instead of EHLO. Servers MUST NOT return the extended - EHLO-style response to a HELO command. For a particular connection - attempt, if the server returns a "command not recognized" response to - EHLO, the client SHOULD be able to fall back and send HELO. - - In the EHLO command the host sending the command identifies itself; - the command may be interpreted as saying "Hello, I am " (and, - in the case of EHLO, "and I support service extension requests"). - -3.3 Mail Transactions - - There are three steps to SMTP mail transactions. The transaction - starts with a MAIL command which gives the sender identification. - (In general, the MAIL command may be sent only when no mail - transaction is in progress; see section 4.1.4.) A series of one or - more RCPT commands follows giving the receiver information. Then a - DATA command initiates transfer of the mail data and is terminated by - the "end of mail" data indicator, which also confirms the - transaction. - - The first step in the procedure is the MAIL command. - - MAIL FROM: [SP ] - - This command tells the SMTP-receiver that a new mail transaction is - starting and to reset all its state tables and buffers, including any - recipients or mail data. The portion of the first or - only argument contains the source mailbox (between "<" and ">" - brackets), which can be used to report errors (see section 4.2 for a - discussion of error reporting). If accepted, the SMTP server returns - a 250 OK reply. If the mailbox specification is not acceptable for - some reason, the server MUST return a reply indicating whether the - - - -Klensin Standards Track [Page 16] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - failure is permanent (i.e., will occur again if the client tries to - send the same address again) or temporary (i.e., the address might be - accepted if the client tries again later). Despite the apparent - scope of this requirement, there are circumstances in which the - acceptability of the reverse-path may not be determined until one or - more forward-paths (in RCPT commands) can be examined. In those - cases, the server MAY reasonably accept the reverse-path (with a 250 - reply) and then report problems after the forward-paths are received - and examined. Normally, failures produce 550 or 553 replies. - - Historically, the can contain more than just a - mailbox, however, contemporary systems SHOULD NOT use source routing - (see appendix C). - - The optional are associated with negotiated SMTP - service extensions (see section 2.2). - - The second step in the procedure is the RCPT command. - - RCPT TO: [ SP ] - - The first or only argument to this command includes a forward-path - (normally a mailbox and domain, always surrounded by "<" and ">" - brackets) identifying one recipient. If accepted, the SMTP server - returns a 250 OK reply and stores the forward-path. If the recipient - is known not to be a deliverable address, the SMTP server returns a - 550 reply, typically with a string such as "no such user - " and the - mailbox name (other circumstances and reply codes are possible). - This step of the procedure can be repeated any number of times. - - The can contain more than just a mailbox. - Historically, the can be a source routing list of - hosts and the destination mailbox, however, contemporary SMTP clients - SHOULD NOT utilize source routes (see appendix C). Servers MUST be - prepared to encounter a list of source routes in the forward path, - but SHOULD ignore the routes or MAY decline to support the relaying - they imply. Similarly, servers MAY decline to accept mail that is - destined for other hosts or systems. These restrictions make a - server useless as a relay for clients that do not support full SMTP - functionality. Consequently, restricted-capability clients MUST NOT - assume that any SMTP server on the Internet can be used as their mail - processing (relaying) site. If a RCPT command appears without a - previous MAIL command, the server MUST return a 503 "Bad sequence of - commands" response. The optional are associated - with negotiated SMTP service extensions (see section 2.2). - - The third step in the procedure is the DATA command (or some - alternative specified in a service extension). - - - -Klensin Standards Track [Page 17] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - DATA - - If accepted, the SMTP server returns a 354 Intermediate reply and - considers all succeeding lines up to but not including the end of - mail data indicator to be the message text. When the end of text is - successfully received and stored the SMTP-receiver sends a 250 OK - reply. - - Since the mail data is sent on the transmission channel, the end of - mail data must be indicated so that the command and reply dialog can - be resumed. SMTP indicates the end of the mail data by sending a - line containing only a "." (period or full stop). A transparency - procedure is used to prevent this from interfering with the user's - text (see section 4.5.2). - - The end of mail data indicator also confirms the mail transaction and - tells the SMTP server to now process the stored recipients and mail - data. If accepted, the SMTP server returns a 250 OK reply. The DATA - command can fail at only two points in the protocol exchange: - - - If there was no MAIL, or no RCPT, command, or all such commands - were rejected, the server MAY return a "command out of sequence" - (503) or "no valid recipients" (554) reply in response to the DATA - command. If one of those replies (or any other 5yz reply) is - received, the client MUST NOT send the message data; more - generally, message data MUST NOT be sent unless a 354 reply is - received. - - - If the verb is initially accepted and the 354 reply issued, the - DATA command should fail only if the mail transaction was - incomplete (for example, no recipients), or if resources were - unavailable (including, of course, the server unexpectedly - becoming unavailable), or if the server determines that the - message should be rejected for policy or other reasons. - - However, in practice, some servers do not perform recipient - verification until after the message text is received. These servers - SHOULD treat a failure for one or more recipients as a "subsequent - failure" and return a mail message as discussed in section 6. Using - a "550 mailbox not found" (or equivalent) reply code after the data - are accepted makes it difficult or impossible for the client to - determine which recipients failed. - - When RFC 822 format [7, 32] is being used, the mail data include the - memo header items such as Date, Subject, To, Cc, From. Server SMTP - systems SHOULD NOT reject messages based on perceived defects in the - RFC 822 or MIME [12] message header or message body. In particular, - - - - -Klensin Standards Track [Page 18] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - they MUST NOT reject messages in which the numbers of Resent-fields - do not match or Resent-to appears without Resent-from and/or Resent- - date. - - Mail transaction commands MUST be used in the order discussed above. - -3.4 Forwarding for Address Correction or Updating - - Forwarding support is most often required to consolidate and simplify - addresses within, or relative to, some enterprise and less frequently - to establish addresses to link a person's prior address with current - one. Silent forwarding of messages (without server notification to - the sender), for security or non-disclosure purposes, is common in - the contemporary Internet. - - In both the enterprise and the "new address" cases, information - hiding (and sometimes security) considerations argue against exposure - of the "final" address through the SMTP protocol as a side-effect of - the forwarding activity. This may be especially important when the - final address may not even be reachable by the sender. Consequently, - the "forwarding" mechanisms described in section 3.2 of RFC 821, and - especially the 251 (corrected destination) and 551 reply codes from - RCPT must be evaluated carefully by implementers and, when they are - available, by those configuring systems. - - In particular: - - * Servers MAY forward messages when they are aware of an address - change. When they do so, they MAY either provide address-updating - information with a 251 code, or may forward "silently" and return - a 250 code. But, if a 251 code is used, they MUST NOT assume that - the client will actually update address information or even return - that information to the user. - - Alternately, - - * Servers MAY reject or bounce messages when they are not - deliverable when addressed. When they do so, they MAY either - provide address-updating information with a 551 code, or may - reject the message as undeliverable with a 550 code and no - address-specific information. But, if a 551 code is used, they - MUST NOT assume that the client will actually update address - information or even return that information to the user. - - SMTP server implementations that support the 251 and/or 551 reply - codes are strongly encouraged to provide configuration mechanisms so - that sites which conclude that they would undesirably disclose - information can disable or restrict their use. - - - -Klensin Standards Track [Page 19] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -3.5 Commands for Debugging Addresses - -3.5.1 Overview - - SMTP provides commands to verify a user name or obtain the content of - a mailing list. This is done with the VRFY and EXPN commands, which - have character string arguments. Implementations SHOULD support VRFY - and EXPN (however, see section 3.5.2 and 7.3). - - For the VRFY command, the string is a user name or a user name and - domain (see below). If a normal (i.e., 250) response is returned, - the response MAY include the full name of the user and MUST include - the mailbox of the user. It MUST be in either of the following - forms: - - User Name - local-part@domain - - When a name that is the argument to VRFY could identify more than one - mailbox, the server MAY either note the ambiguity or identify the - alternatives. In other words, any of the following are legitimate - response to VRFY: - - 553 User ambiguous - - or - - 553- Ambiguous; Possibilities are - 553-Joe Smith - 553-Harry Smith - 553 Melvin Smith - - or - - 553-Ambiguous; Possibilities - 553- - 553- - 553 - - Under normal circumstances, a client receiving a 553 reply would be - expected to expose the result to the user. Use of exactly the forms - given, and the "user ambiguous" or "ambiguous" keywords, possibly - supplemented by extended reply codes such as those described in [34], - will facilitate automated translation into other languages as needed. - Of course, a client that was highly automated or that was operating - in another language than English, might choose to try to translate - the response, to return some other indication to the user than the - - - - -Klensin Standards Track [Page 20] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - literal text of the reply, or to take some automated action such as - consulting a directory service for additional information before - reporting to the user. - - For the EXPN command, the string identifies a mailing list, and the - successful (i.e., 250) multiline response MAY include the full name - of the users and MUST give the mailboxes on the mailing list. - - In some hosts the distinction between a mailing list and an alias for - a single mailbox is a bit fuzzy, since a common data structure may - hold both types of entries, and it is possible to have mailing lists - containing only one mailbox. If a request is made to apply VRFY to a - mailing list, a positive response MAY be given if a message so - addressed would be delivered to everyone on the list, otherwise an - error SHOULD be reported (e.g., "550 That is a mailing list, not a - user" or "252 Unable to verify members of mailing list"). If a - request is made to expand a user name, the server MAY return a - positive response consisting of a list containing one name, or an - error MAY be reported (e.g., "550 That is a user name, not a mailing - list"). - - In the case of a successful multiline reply (normal for EXPN) exactly - one mailbox is to be specified on each line of the reply. The case - of an ambiguous request is discussed above. - - "User name" is a fuzzy term and has been used deliberately. An - implementation of the VRFY or EXPN commands MUST include at least - recognition of local mailboxes as "user names". However, since - current Internet practice often results in a single host handling - mail for multiple domains, hosts, especially hosts that provide this - functionality, SHOULD accept the "local-part@domain" form as a "user - name"; hosts MAY also choose to recognize other strings as "user - names". - - The case of expanding a mailbox list requires a multiline reply, such - as: - - C: EXPN Example-People - S: 250-Jon Postel - S: 250-Fred Fonebone - S: 250 Sam Q. Smith - - or - - C: EXPN Executive-Washroom-List - S: 550 Access Denied to You. - - - - - -Klensin Standards Track [Page 21] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - The character string arguments of the VRFY and EXPN commands cannot - be further restricted due to the variety of implementations of the - user name and mailbox list concepts. On some systems it may be - appropriate for the argument of the EXPN command to be a file name - for a file containing a mailing list, but again there are a variety - of file naming conventions in the Internet. Similarly, historical - variations in what is returned by these commands are such that the - response SHOULD be interpreted very carefully, if at all, and SHOULD - generally only be used for diagnostic purposes. - -3.5.2 VRFY Normal Response - - When normal (2yz or 551) responses are returned from a VRFY or EXPN - request, the reply normally includes the mailbox name, i.e., - "", where "domain" is a fully qualified domain - name, MUST appear in the syntax. In circumstances exceptional enough - to justify violating the intent of this specification, free-form text - MAY be returned. In order to facilitate parsing by both computers - and people, addresses SHOULD appear in pointed brackets. When - addresses, rather than free-form debugging information, are returned, - EXPN and VRFY MUST return only valid domain addresses that are usable - in SMTP RCPT commands. Consequently, if an address implies delivery - to a program or other system, the mailbox name used to reach that - target MUST be given. Paths (explicit source routes) MUST NOT be - returned by VRFY or EXPN. - - Server implementations SHOULD support both VRFY and EXPN. For - security reasons, implementations MAY provide local installations a - way to disable either or both of these commands through configuration - options or the equivalent. When these commands are supported, they - are not required to work across relays when relaying is supported. - Since they were both optional in RFC 821, they MUST be listed as - service extensions in an EHLO response, if they are supported. - -3.5.3 Meaning of VRFY or EXPN Success Response - - A server MUST NOT return a 250 code in response to a VRFY or EXPN - command unless it has actually verified the address. In particular, - a server MUST NOT return 250 if all it has done is to verify that the - syntax given is valid. In that case, 502 (Command not implemented) - or 500 (Syntax error, command unrecognized) SHOULD be returned. As - stated elsewhere, implementation (in the sense of actually validating - addresses and returning information) of VRFY and EXPN are strongly - recommended. Hence, implementations that return 500 or 502 for VRFY - are not in full compliance with this specification. - - - - - - -Klensin Standards Track [Page 22] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - There may be circumstances where an address appears to be valid but - cannot reasonably be verified in real time, particularly when a - server is acting as a mail exchanger for another server or domain. - "Apparent validity" in this case would normally involve at least - syntax checking and might involve verification that any domains - specified were ones to which the host expected to be able to relay - mail. In these situations, reply code 252 SHOULD be returned. These - cases parallel the discussion of RCPT verification discussed in - section 2.1. Similarly, the discussion in section 3.4 applies to the - use of reply codes 251 and 551 with VRFY (and EXPN) to indicate - addresses that are recognized but that would be forwarded or bounced - were mail received for them. Implementations generally SHOULD be - more aggressive about address verification in the case of VRFY than - in the case of RCPT, even if it takes a little longer to do so. - -3.5.4 Semantics and Applications of EXPN - - EXPN is often very useful in debugging and understanding problems - with mailing lists and multiple-target-address aliases. Some systems - have attempted to use source expansion of mailing lists as a means of - eliminating duplicates. The propagation of aliasing systems with - mail on the Internet, for hosts (typically with MX and CNAME DNS - records), for mailboxes (various types of local host aliases), and in - various proxying arrangements, has made it nearly impossible for - these strategies to work consistently, and mail systems SHOULD NOT - attempt them. - -3.6 Domains - - Only resolvable, fully-qualified, domain names (FQDNs) are permitted - when domain names are used in SMTP. In other words, names that can - be resolved to MX RRs or A RRs (as discussed in section 5) are - permitted, as are CNAME RRs whose targets can be resolved, in turn, - to MX or A RRs. Local nicknames or unqualified names MUST NOT be - used. There are two exceptions to the rule requiring FQDNs: - - - The domain name given in the EHLO command MUST BE either a primary - host name (a domain name that resolves to an A RR) or, if the host - has no name, an address literal as described in section 4.1.1.1. - - - The reserved mailbox name "postmaster" may be used in a RCPT - command without domain qualification (see section 4.1.1.3) and - MUST be accepted if so used. - - - - - - - - -Klensin Standards Track [Page 23] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -3.7 Relaying - - In general, the availability of Mail eXchanger records in the domain - name system [22, 27] makes the use of explicit source routes in the - Internet mail system unnecessary. Many historical problems with - their interpretation have made their use undesirable. SMTP clients - SHOULD NOT generate explicit source routes except under unusual - circumstances. SMTP servers MAY decline to act as mail relays or to - accept addresses that specify source routes. When route information - is encountered, SMTP servers are also permitted to ignore the route - information and simply send to the final destination specified as the - last element in the route and SHOULD do so. There has been an - invalid practice of using names that do not appear in the DNS as - destination names, with the senders counting on the intermediate - hosts specified in source routing to resolve any problems. If source - routes are stripped, this practice will cause failures. This is one - of several reasons why SMTP clients MUST NOT generate invalid source - routes or depend on serial resolution of names. - - When source routes are not used, the process described in RFC 821 for - constructing a reverse-path from the forward-path is not applicable - and the reverse-path at the time of delivery will simply be the - address that appeared in the MAIL command. - - A relay SMTP server is usually the target of a DNS MX record that - designates it, rather than the final delivery system. The relay - server may accept or reject the task of relaying the mail in the same - way it accepts or rejects mail for a local user. If it accepts the - task, it then becomes an SMTP client, establishes a transmission - channel to the next SMTP server specified in the DNS (according to - the rules in section 5), and sends it the mail. If it declines to - relay mail to a particular address for policy reasons, a 550 response - SHOULD be returned. - - Many mail-sending clients exist, especially in conjunction with - facilities that receive mail via POP3 or IMAP, that have limited - capability to support some of the requirements of this specification, - such as the ability to queue messages for subsequent delivery - attempts. For these clients, it is common practice to make private - arrangements to send all messages to a single server for processing - and subsequent distribution. SMTP, as specified here, is not ideally - suited for this role, and work is underway on standardized mail - submission protocols that might eventually supercede the current - practices. In any event, because these arrangements are private and - fall outside the scope of this specification, they are not described - here. - - - - - -Klensin Standards Track [Page 24] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - It is important to note that MX records can point to SMTP servers - which act as gateways into other environments, not just SMTP relays - and final delivery systems; see sections 3.8 and 5. - - If an SMTP server has accepted the task of relaying the mail and - later finds that the destination is incorrect or that the mail cannot - be delivered for some other reason, then it MUST construct an - "undeliverable mail" notification message and send it to the - originator of the undeliverable mail (as indicated by the reverse- - path). Formats specified for non-delivery reports by other standards - (see, for example, [24, 25]) SHOULD be used if possible. - - This notification message must be from the SMTP server at the relay - host or the host that first determines that delivery cannot be - accomplished. Of course, SMTP servers MUST NOT send notification - messages about problems transporting notification messages. One way - to prevent loops in error reporting is to specify a null reverse-path - in the MAIL command of a notification message. When such a message - is transmitted the reverse-path MUST be set to null (see section - 4.5.5 for additional discussion). A MAIL command with a null - reverse-path appears as follows: - - MAIL FROM:<> - - As discussed in section 2.4.1, a relay SMTP has no need to inspect or - act upon the headers or body of the message data and MUST NOT do so - except to add its own "Received:" header (section 4.4) and, - optionally, to attempt to detect looping in the mail system (see - section 6.2). - -3.8 Mail Gatewaying - - While the relay function discussed above operates within the Internet - SMTP transport service environment, MX records or various forms of - explicit routing may require that an intermediate SMTP server perform - a translation function between one transport service and another. As - discussed in section 2.3.8, when such a system is at the boundary - between two transport service environments, we refer to it as a - "gateway" or "gateway SMTP". - - Gatewaying mail between different mail environments, such as - different mail formats and protocols, is complex and does not easily - yield to standardization. However, some general requirements may be - given for a gateway between the Internet and another mail - environment. - - - - - - -Klensin Standards Track [Page 25] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -3.8.1 Header Fields in Gatewaying - - Header fields MAY be rewritten when necessary as messages are - gatewayed across mail environment boundaries. This may involve - inspecting the message body or interpreting the local-part of the - destination address in spite of the prohibitions in section 2.4.1. - - Other mail systems gatewayed to the Internet often use a subset of - RFC 822 headers or provide similar functionality with a different - syntax, but some of these mail systems do not have an equivalent to - the SMTP envelope. Therefore, when a message leaves the Internet - environment, it may be necessary to fold the SMTP envelope - information into the message header. A possible solution would be to - create new header fields to carry the envelope information (e.g., - "X-SMTP-MAIL:" and "X-SMTP-RCPT:"); however, this would require - changes in mail programs in foreign environments and might risk - disclosure of private information (see section 7.2). - -3.8.2 Received Lines in Gatewaying - - When forwarding a message into or out of the Internet environment, a - gateway MUST prepend a Received: line, but it MUST NOT alter in any - way a Received: line that is already in the header. - - "Received:" fields of messages originating from other environments - may not conform exactly to this specification. However, the most - important use of Received: lines is for debugging mail faults, and - this debugging can be severely hampered by well-meaning gateways that - try to "fix" a Received: line. As another consequence of trace - fields arising in non-SMTP environments, receiving systems MUST NOT - reject mail based on the format of a trace field and SHOULD be - extremely robust in the light of unexpected information or formats in - those fields. - - The gateway SHOULD indicate the environment and protocol in the "via" - clauses of Received field(s) that it supplies. - -3.8.3 Addresses in Gatewaying - - From the Internet side, the gateway SHOULD accept all valid address - formats in SMTP commands and in RFC 822 headers, and all valid RFC - 822 messages. Addresses and headers generated by gateways MUST - conform to applicable Internet standards (including this one and RFC - 822). Gateways are, of course, subject to the same rules for - handling source routes as those described for other SMTP systems in - section 3.3. - - - - - -Klensin Standards Track [Page 26] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -3.8.4 Other Header Fields in Gatewaying - - The gateway MUST ensure that all header fields of a message that it - forwards into the Internet mail environment meet the requirements for - Internet mail. In particular, all addresses in "From:", "To:", - "Cc:", etc., fields MUST be transformed (if necessary) to satisfy RFC - 822 syntax, MUST reference only fully-qualified domain names, and - MUST be effective and useful for sending replies. The translation - algorithm used to convert mail from the Internet protocols to another - environment's protocol SHOULD ensure that error messages from the - foreign mail environment are delivered to the return path from the - SMTP envelope, not to the sender listed in the "From:" field (or - other fields) of the RFC 822 message. - -3.8.5 Envelopes in Gatewaying - - Similarly, when forwarding a message from another environment into - the Internet, the gateway SHOULD set the envelope return path in - accordance with an error message return address, if supplied by the - foreign environment. If the foreign environment has no equivalent - concept, the gateway must select and use a best approximation, with - the message originator's address as the default of last resort. - -3.9 Terminating Sessions and Connections - - An SMTP connection is terminated when the client sends a QUIT - command. The server responds with a positive reply code, after which - it closes the connection. - - An SMTP server MUST NOT intentionally close the connection except: - - - After receiving a QUIT command and responding with a 221 reply. - - - After detecting the need to shut down the SMTP service and - returning a 421 response code. This response code can be issued - after the server receives any command or, if necessary, - asynchronously from command receipt (on the assumption that the - client will receive it after the next command is issued). - - In particular, a server that closes connections in response to - commands that are not understood is in violation of this - specification. Servers are expected to be tolerant of unknown - commands, issuing a 500 reply and awaiting further instructions from - the client. - - - - - - - -Klensin Standards Track [Page 27] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - An SMTP server which is forcibly shut down via external means SHOULD - attempt to send a line containing a 421 response code to the SMTP - client before exiting. The SMTP client will normally read the 421 - response code after sending its next command. - - SMTP clients that experience a connection close, reset, or other - communications failure due to circumstances not under their control - (in violation of the intent of this specification but sometimes - unavoidable) SHOULD, to maintain the robustness of the mail system, - treat the mail transaction as if a 451 response had been received and - act accordingly. - -3.10 Mailing Lists and Aliases - - An SMTP-capable host SHOULD support both the alias and the list - models of address expansion for multiple delivery. When a message is - delivered or forwarded to each address of an expanded list form, the - return address in the envelope ("MAIL FROM:") MUST be changed to be - the address of a person or other entity who administers the list. - However, in this case, the message header [32] MUST be left - unchanged; in particular, the "From" field of the message header is - unaffected. - - An important mail facility is a mechanism for multi-destination - delivery of a single message, by transforming (or "expanding" or - "exploding") a pseudo-mailbox address into a list of destination - mailbox addresses. When a message is sent to such a pseudo-mailbox - (sometimes called an "exploder"), copies are forwarded or - redistributed to each mailbox in the expanded list. Servers SHOULD - simply utilize the addresses on the list; application of heuristics - or other matching rules to eliminate some addresses, such as that of - the originator, is strongly discouraged. We classify such a pseudo- - mailbox as an "alias" or a "list", depending upon the expansion - rules. - -3.10.1 Alias - - To expand an alias, the recipient mailer simply replaces the pseudo- - mailbox address in the envelope with each of the expanded addresses - in turn; the rest of the envelope and the message body are left - unchanged. The message is then delivered or forwarded to each - expanded address. - -3.10.2 List - - A mailing list may be said to operate by "redistribution" rather than - by "forwarding". To expand a list, the recipient mailer replaces the - pseudo-mailbox address in the envelope with all of the expanded - - - -Klensin Standards Track [Page 28] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - addresses. The return address in the envelope is changed so that all - error messages generated by the final deliveries will be returned to - a list administrator, not to the message originator, who generally - has no control over the contents of the list and will typically find - error messages annoying. - -4. The SMTP Specifications - -4.1 SMTP Commands - -4.1.1 Command Semantics and Syntax - - The SMTP commands define the mail transfer or the mail system - function requested by the user. SMTP commands are character strings - terminated by . The commands themselves are alphabetic - characters terminated by if parameters follow and - otherwise. (In the interest of improved interoperability, SMTP - receivers are encouraged to tolerate trailing white space before the - terminating .) The syntax of the local part of a mailbox must - conform to receiver site conventions and the syntax specified in - section 4.1.2. The SMTP commands are discussed below. The SMTP - replies are discussed in section 4.2. - - A mail transaction involves several data objects which are - communicated as arguments to different commands. The reverse-path is - the argument of the MAIL command, the forward-path is the argument of - the RCPT command, and the mail data is the argument of the DATA - command. These arguments or data objects must be transmitted and - held pending the confirmation communicated by the end of mail data - indication which finalizes the transaction. The model for this is - that distinct buffers are provided to hold the types of data objects, - that is, there is a reverse-path buffer, a forward-path buffer, and a - mail data buffer. Specific commands cause information to be appended - to a specific buffer, or cause one or more buffers to be cleared. - - Several commands (RSET, DATA, QUIT) are specified as not permitting - parameters. In the absence of specific extensions offered by the - server and accepted by the client, clients MUST NOT send such - parameters and servers SHOULD reject commands containing them as - having invalid syntax. - -4.1.1.1 Extended HELLO (EHLO) or HELLO (HELO) - - These commands are used to identify the SMTP client to the SMTP - server. The argument field contains the fully-qualified domain name - of the SMTP client if one is available. In situations in which the - SMTP client system does not have a meaningful domain name (e.g., when - its address is dynamically allocated and no reverse mapping record is - - - -Klensin Standards Track [Page 29] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - available), the client SHOULD send an address literal (see section - 4.1.3), optionally followed by information that will help to identify - the client system. y The SMTP server identifies itself to the SMTP - client in the connection greeting reply and in the response to this - command. - - A client SMTP SHOULD start an SMTP session by issuing the EHLO - command. If the SMTP server supports the SMTP service extensions it - will give a successful response, a failure response, or an error - response. If the SMTP server, in violation of this specification, - does not support any SMTP service extensions it will generate an - error response. Older client SMTP systems MAY, as discussed above, - use HELO (as specified in RFC 821) instead of EHLO, and servers MUST - support the HELO command and reply properly to it. In any event, a - client MUST issue HELO or EHLO before starting a mail transaction. - - These commands, and a "250 OK" reply to one of them, confirm that - both the SMTP client and the SMTP server are in the initial state, - that is, there is no transaction in progress and all state tables and - buffers are cleared. - - Syntax: - - ehlo = "EHLO" SP Domain CRLF - helo = "HELO" SP Domain CRLF - - Normally, the response to EHLO will be a multiline reply. Each line - of the response contains a keyword and, optionally, one or more - parameters. Following the normal syntax for multiline replies, these - keyworks follow the code (250) and a hyphen for all but the last - line, and the code and a space for the last line. The syntax for a - positive response, using the ABNF notation and terminal symbols of - [8], is: - - ehlo-ok-rsp = ( "250" domain [ SP ehlo-greet ] CRLF ) - / ( "250-" domain [ SP ehlo-greet ] CRLF - *( "250-" ehlo-line CRLF ) - "250" SP ehlo-line CRLF ) - - ehlo-greet = 1*(%d0-9 / %d11-12 / %d14-127) - ; string of any characters other than CR or LF - - ehlo-line = ehlo-keyword *( SP ehlo-param ) - - ehlo-keyword = (ALPHA / DIGIT) *(ALPHA / DIGIT / "-") - ; additional syntax of ehlo-params depends on - ; ehlo-keyword - - - - -Klensin Standards Track [Page 30] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - ehlo-param = 1*(%d33-127) - ; any CHAR excluding and all - ; control characters (US-ASCII 0-31 inclusive) - - Although EHLO keywords may be specified in upper, lower, or mixed - case, they MUST always be recognized and processed in a case- - insensitive manner. This is simply an extension of practices - specified in RFC 821 and section 2.4.1. - -4.1.1.2 MAIL (MAIL) - - This command is used to initiate a mail transaction in which the mail - data is delivered to an SMTP server which may, in turn, deliver it to - one or more mailboxes or pass it on to another system (possibly using - SMTP). The argument field contains a reverse-path and may contain - optional parameters. In general, the MAIL command may be sent only - when no mail transaction is in progress, see section 4.1.4. - - The reverse-path consists of the sender mailbox. Historically, that - mailbox might optionally have been preceded by a list of hosts, but - that behavior is now deprecated (see appendix C). In some types of - reporting messages for which a reply is likely to cause a mail loop - (for example, mail delivery and nondelivery notifications), the - reverse-path may be null (see section 3.7). - - This command clears the reverse-path buffer, the forward-path buffer, - and the mail data buffer; and inserts the reverse-path information - from this command into the reverse-path buffer. - - If service extensions were negotiated, the MAIL command may also - carry parameters associated with a particular service extension. - - Syntax: - - "MAIL FROM:" ("<>" / Reverse-Path) - [SP Mail-parameters] CRLF - -4.1.1.3 RECIPIENT (RCPT) - - This command is used to identify an individual recipient of the mail - data; multiple recipients are specified by multiple use of this - command. The argument field contains a forward-path and may contain - optional parameters. - - The forward-path normally consists of the required destination - mailbox. Sending systems SHOULD not generate the optional list of - hosts known as a source route. Receiving systems MUST recognize - - - - -Klensin Standards Track [Page 31] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - source route syntax but SHOULD strip off the source route - specification and utilize the domain name associated with the mailbox - as if the source route had not been provided. - - Similarly, relay hosts SHOULD strip or ignore source routes, and - names MUST NOT be copied into the reverse-path. When mail reaches - its ultimate destination (the forward-path contains only a - destination mailbox), the SMTP server inserts it into the destination - mailbox in accordance with its host mail conventions. - - For example, mail received at relay host xyz.com with envelope - commands - - MAIL FROM: - RCPT TO:<@hosta.int,@jkl.org:userc@d.bar.org> - - will normally be sent directly on to host d.bar.org with envelope - commands - - MAIL FROM: - RCPT TO: - - As provided in appendix C, xyz.com MAY also choose to relay the - message to hosta.int, using the envelope commands - - MAIL FROM: - RCPT TO:<@hosta.int,@jkl.org:userc@d.bar.org> - - or to jkl.org, using the envelope commands - - MAIL FROM: - RCPT TO:<@jkl.org:userc@d.bar.org> - - Of course, since hosts are not required to relay mail at all, xyz.com - may also reject the message entirely when the RCPT command is - received, using a 550 code (since this is a "policy reason"). - - If service extensions were negotiated, the RCPT command may also - carry parameters associated with a particular service extension - offered by the server. The client MUST NOT transmit parameters other - than those associated with a service extension offered by the server - in its EHLO response. - -Syntax: - "RCPT TO:" ("" / "" / Forward-Path) - [SP Rcpt-parameters] CRLF - - - - - -Klensin Standards Track [Page 32] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -4.1.1.4 DATA (DATA) - - The receiver normally sends a 354 response to DATA, and then treats - the lines (strings ending in sequences, as described in - section 2.3.7) following the command as mail data from the sender. - This command causes the mail data to be appended to the mail data - buffer. The mail data may contain any of the 128 ASCII character - codes, although experience has indicated that use of control - characters other than SP, HT, CR, and LF may cause problems and - SHOULD be avoided when possible. - - The mail data is terminated by a line containing only a period, that - is, the character sequence "." (see section 4.5.2). This - is the end of mail data indication. Note that the first of - this terminating sequence is also the that ends the final line - of the data (message text) or, if there was no data, ends the DATA - command itself. An extra MUST NOT be added, as that would - cause an empty line to be added to the message. The only exception - to this rule would arise if the message body were passed to the - originating SMTP-sender with a final "line" that did not end in - ; in that case, the originating SMTP system MUST either reject - the message as invalid or add in order to have the receiving - SMTP server recognize the "end of data" condition. - - The custom of accepting lines ending only in , as a concession to - non-conforming behavior on the part of some UNIX systems, has proven - to cause more interoperability problems than it solves, and SMTP - server systems MUST NOT do this, even in the name of improved - robustness. In particular, the sequence "." (bare line - feeds, without carriage returns) MUST NOT be treated as equivalent to - . as the end of mail data indication. - - Receipt of the end of mail data indication requires the server to - process the stored mail transaction information. This processing - consumes the information in the reverse-path buffer, the forward-path - buffer, and the mail data buffer, and on the completion of this - command these buffers are cleared. If the processing is successful, - the receiver MUST send an OK reply. If the processing fails the - receiver MUST send a failure reply. The SMTP model does not allow - for partial failures at this point: either the message is accepted by - the server for delivery and a positive response is returned or it is - not accepted and a failure reply is returned. In sending a positive - completion reply to the end of data indication, the receiver takes - full responsibility for the message (see section 6.1). Errors that - are diagnosed subsequently MUST be reported in a mail message, as - discussed in section 4.4. - - - - - -Klensin Standards Track [Page 33] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - When the SMTP server accepts a message either for relaying or for - final delivery, it inserts a trace record (also referred to - interchangeably as a "time stamp line" or "Received" line) at the top - of the mail data. This trace record indicates the identity of the - host that sent the message, the identity of the host that received - the message (and is inserting this time stamp), and the date and time - the message was received. Relayed messages will have multiple time - stamp lines. Details for formation of these lines, including their - syntax, is specified in section 4.4. - - Additional discussion about the operation of the DATA command appears - in section 3.3. - - Syntax: - "DATA" CRLF - -4.1.1.5 RESET (RSET) - - This command specifies that the current mail transaction will be - aborted. Any stored sender, recipients, and mail data MUST be - discarded, and all buffers and state tables cleared. The receiver - MUST send a "250 OK" reply to a RSET command with no arguments. A - reset command may be issued by the client at any time. It is - effectively equivalent to a NOOP (i.e., if has no effect) if issued - immediately after EHLO, before EHLO is issued in the session, after - an end-of-data indicator has been sent and acknowledged, or - immediately before a QUIT. An SMTP server MUST NOT close the - connection as the result of receiving a RSET; that action is reserved - for QUIT (see section 4.1.1.10). - - Since EHLO implies some additional processing and response by the - server, RSET will normally be more efficient than reissuing that - command, even though the formal semantics are the same. - - There are circumstances, contrary to the intent of this - specification, in which an SMTP server may receive an indication that - the underlying TCP connection has been closed or reset. To preserve - the robustness of the mail system, SMTP servers SHOULD be prepared - for this condition and SHOULD treat it as if a QUIT had been received - before the connection disappeared. - - Syntax: - "RSET" CRLF - - - - - - - - -Klensin Standards Track [Page 34] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -4.1.1.6 VERIFY (VRFY) - - This command asks the receiver to confirm that the argument - identifies a user or mailbox. If it is a user name, information is - returned as specified in section 3.5. - - This command has no effect on the reverse-path buffer, the forward- - path buffer, or the mail data buffer. - - Syntax: - "VRFY" SP String CRLF - -4.1.1.7 EXPAND (EXPN) - - This command asks the receiver to confirm that the argument - identifies a mailing list, and if so, to return the membership of - that list. If the command is successful, a reply is returned - containing information as described in section 3.5. This reply will - have multiple lines except in the trivial case of a one-member list. - - This command has no effect on the reverse-path buffer, the forward- - path buffer, or the mail data buffer and may be issued at any time. - - Syntax: - "EXPN" SP String CRLF - -4.1.1.8 HELP (HELP) - - This command causes the server to send helpful information to the - client. The command MAY take an argument (e.g., any command name) - and return more specific information as a response. - - This command has no effect on the reverse-path buffer, the forward- - path buffer, or the mail data buffer and may be issued at any time. - - SMTP servers SHOULD support HELP without arguments and MAY support it - with arguments. - - Syntax: - "HELP" [ SP String ] CRLF - -4.1.1.9 NOOP (NOOP) - - This command does not affect any parameters or previously entered - commands. It specifies no action other than that the receiver send - an OK reply. - - - - - -Klensin Standards Track [Page 35] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - This command has no effect on the reverse-path buffer, the forward- - path buffer, or the mail data buffer and may be issued at any time. - If a parameter string is specified, servers SHOULD ignore it. - - Syntax: - "NOOP" [ SP String ] CRLF - -4.1.1.10 QUIT (QUIT) - - This command specifies that the receiver MUST send an OK reply, and - then close the transmission channel. - - The receiver MUST NOT intentionally close the transmission channel - until it receives and replies to a QUIT command (even if there was an - error). The sender MUST NOT intentionally close the transmission - channel until it sends a QUIT command and SHOULD wait until it - receives the reply (even if there was an error response to a previous - command). If the connection is closed prematurely due to violations - of the above or system or network failure, the server MUST cancel any - pending transaction, but not undo any previously completed - transaction, and generally MUST act as if the command or transaction - in progress had received a temporary error (i.e., a 4yz response). - - The QUIT command may be issued at any time. - - Syntax: - "QUIT" CRLF - -4.1.2 Command Argument Syntax - - The syntax of the argument fields of the above commands (using the - syntax specified in [8] where applicable) is given below. Some of - the productions given below are used only in conjunction with source - routes as described in appendix C. Terminals not defined in this - document, such as ALPHA, DIGIT, SP, CR, LF, CRLF, are as defined in - the "core" syntax [8 (section 6)] or in the message format syntax - [32]. - - Reverse-path = Path - Forward-path = Path - Path = "<" [ A-d-l ":" ] Mailbox ">" - A-d-l = At-domain *( "," A-d-l ) - ; Note that this form, the so-called "source route", - ; MUST BE accepted, SHOULD NOT be generated, and SHOULD be - ; ignored. - At-domain = "@" domain - Mail-parameters = esmtp-param *(SP esmtp-param) - Rcpt-parameters = esmtp-param *(SP esmtp-param) - - - -Klensin Standards Track [Page 36] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - esmtp-param = esmtp-keyword ["=" esmtp-value] - esmtp-keyword = (ALPHA / DIGIT) *(ALPHA / DIGIT / "-") - esmtp-value = 1*(%d33-60 / %d62-127) - ; any CHAR excluding "=", SP, and control characters - Keyword = Ldh-str - Argument = Atom - Domain = (sub-domain 1*("." sub-domain)) / address-literal - sub-domain = Let-dig [Ldh-str] - - address-literal = "[" IPv4-address-literal / - IPv6-address-literal / - General-address-literal "]" - ; See section 4.1.3 - - Mailbox = Local-part "@" Domain - - Local-part = Dot-string / Quoted-string - ; MAY be case-sensitive - - Dot-string = Atom *("." Atom) - - Atom = 1*atext - - Quoted-string = DQUOTE *qcontent DQUOTE - - String = Atom / Quoted-string - - While the above definition for Local-part is relatively permissive, - for maximum interoperability, a host that expects to receive mail - SHOULD avoid defining mailboxes where the Local-part requires (or - uses) the Quoted-string form or where the Local-part is case- - sensitive. For any purposes that require generating or comparing - Local-parts (e.g., to specific mailbox names), all quoted forms MUST - be treated as equivalent and the sending system SHOULD transmit the - form that uses the minimum quoting possible. - - Systems MUST NOT define mailboxes in such a way as to require the use - in SMTP of non-ASCII characters (octets with the high order bit set - to one) or ASCII "control characters" (decimal value 0-31 and 127). - These characters MUST NOT be used in MAIL or RCPT commands or other - commands that require mailbox names. - - Note that the backslash, "\", is a quote character, which is used to - indicate that the next character is to be used literally (instead of - its normal interpretation). For example, "Joe\,Smith" indicates a - single nine character user field with the comma being the fourth - character of the field. - - - - -Klensin Standards Track [Page 37] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - To promote interoperability and consistent with long-standing - guidance about conservative use of the DNS in naming and applications - (e.g., see section 2.3.1 of the base DNS document, RFC1035 [22]), - characters outside the set of alphas, digits, and hyphen MUST NOT - appear in domain name labels for SMTP clients or servers. In - particular, the underscore character is not permitted. SMTP servers - that receive a command in which invalid character codes have been - employed, and for which there are no other reasons for rejection, - MUST reject that command with a 501 response. - -4.1.3 Address Literals - - Sometimes a host is not known to the domain name system and - communication (and, in particular, communication to report and repair - the error) is blocked. To bypass this barrier a special literal form - of the address is allowed as an alternative to a domain name. For - IPv4 addresses, this form uses four small decimal integers separated - by dots and enclosed by brackets such as [123.255.37.2], which - indicates an (IPv4) Internet Address in sequence-of-octets form. For - IPv6 and other forms of addressing that might eventually be - standardized, the form consists of a standardized "tag" that - identifies the address syntax, a colon, and the address itself, in a - format specified as part of the IPv6 standards [17]. - - Specifically: - - IPv4-address-literal = Snum 3("." Snum) - IPv6-address-literal = "IPv6:" IPv6-addr - General-address-literal = Standardized-tag ":" 1*dcontent - Standardized-tag = Ldh-str - ; MUST be specified in a standards-track RFC - ; and registered with IANA - - Snum = 1*3DIGIT ; representing a decimal integer - ; value in the range 0 through 255 - Let-dig = ALPHA / DIGIT - Ldh-str = *( ALPHA / DIGIT / "-" ) Let-dig - - IPv6-addr = IPv6-full / IPv6-comp / IPv6v4-full / IPv6v4-comp - IPv6-hex = 1*4HEXDIG - IPv6-full = IPv6-hex 7(":" IPv6-hex) - IPv6-comp = [IPv6-hex *5(":" IPv6-hex)] "::" [IPv6-hex *5(":" - IPv6-hex)] - ; The "::" represents at least 2 16-bit groups of zeros - ; No more than 6 groups in addition to the "::" may be - ; present - IPv6v4-full = IPv6-hex 5(":" IPv6-hex) ":" IPv4-address-literal - IPv6v4-comp = [IPv6-hex *3(":" IPv6-hex)] "::" - - - -Klensin Standards Track [Page 38] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - [IPv6-hex *3(":" IPv6-hex) ":"] IPv4-address-literal - ; The "::" represents at least 2 16-bit groups of zeros - ; No more than 4 groups in addition to the "::" and - ; IPv4-address-literal may be present - -4.1.4 Order of Commands - - There are restrictions on the order in which these commands may be - used. - - A session that will contain mail transactions MUST first be - initialized by the use of the EHLO command. An SMTP server SHOULD - accept commands for non-mail transactions (e.g., VRFY or EXPN) - without this initialization. - - An EHLO command MAY be issued by a client later in the session. If - it is issued after the session begins, the SMTP server MUST clear all - buffers and reset the state exactly as if a RSET command had been - issued. In other words, the sequence of RSET followed immediately by - EHLO is redundant, but not harmful other than in the performance cost - of executing unnecessary commands. - - If the EHLO command is not acceptable to the SMTP server, 501, 500, - or 502 failure replies MUST be returned as appropriate. The SMTP - server MUST stay in the same state after transmitting these replies - that it was in before the EHLO was received. - - The SMTP client MUST, if possible, ensure that the domain parameter - to the EHLO command is a valid principal host name (not a CNAME or MX - name) for its host. If this is not possible (e.g., when the client's - address is dynamically assigned and the client does not have an - obvious name), an address literal SHOULD be substituted for the - domain name and supplemental information provided that will assist in - identifying the client. - - An SMTP server MAY verify that the domain name parameter in the EHLO - command actually corresponds to the IP address of the client. - However, the server MUST NOT refuse to accept a message for this - reason if the verification fails: the information about verification - failure is for logging and tracing only. - - The NOOP, HELP, EXPN, VRFY, and RSET commands can be used at any time - during a session, or without previously initializing a session. SMTP - servers SHOULD process these normally (that is, not return a 503 - code) even if no EHLO command has yet been received; clients SHOULD - open a session with EHLO before sending these commands. - - - - - -Klensin Standards Track [Page 39] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - If these rules are followed, the example in RFC 821 that shows "550 - access denied to you" in response to an EXPN command is incorrect - unless an EHLO command precedes the EXPN or the denial of access is - based on the client's IP address or other authentication or - authorization-determining mechanisms. - - The MAIL command (or the obsolete SEND, SOML, or SAML commands) - begins a mail transaction. Once started, a mail transaction consists - of a transaction beginning command, one or more RCPT commands, and a - DATA command, in that order. A mail transaction may be aborted by - the RSET (or a new EHLO) command. There may be zero or more - transactions in a session. MAIL (or SEND, SOML, or SAML) MUST NOT be - sent if a mail transaction is already open, i.e., it should be sent - only if no mail transaction had been started in the session, or it - the previous one successfully concluded with a successful DATA - command, or if the previous one was aborted with a RSET. - - If the transaction beginning command argument is not acceptable, a - 501 failure reply MUST be returned and the SMTP server MUST stay in - the same state. If the commands in a transaction are out of order to - the degree that they cannot be processed by the server, a 503 failure - reply MUST be returned and the SMTP server MUST stay in the same - state. - - The last command in a session MUST be the QUIT command. The QUIT - command cannot be used at any other time in a session, but SHOULD be - used by the client SMTP to request connection closure, even when no - session opening command was sent and accepted. - -4.1.5 Private-use Commands - - As specified in section 2.2.2, commands starting in "X" may be used - by bilateral agreement between the client (sending) and server - (receiving) SMTP agents. An SMTP server that does not recognize such - a command is expected to reply with "500 Command not recognized". An - extended SMTP server MAY list the feature names associated with these - private commands in the response to the EHLO command. - - Commands sent or accepted by SMTP systems that do not start with "X" - MUST conform to the requirements of section 2.2.2. - -4.2 SMTP Replies - - Replies to SMTP commands serve to ensure the synchronization of - requests and actions in the process of mail transfer and to guarantee - that the SMTP client always knows the state of the SMTP server. - Every command MUST generate exactly one reply. - - - - -Klensin Standards Track [Page 40] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - The details of the command-reply sequence are described in section - 4.3. - - An SMTP reply consists of a three digit number (transmitted as three - numeric characters) followed by some text unless specified otherwise - in this document. The number is for use by automata to determine - what state to enter next; the text is for the human user. The three - digits contain enough encoded information that the SMTP client need - not examine the text and may either discard it or pass it on to the - user, as appropriate. Exceptions are as noted elsewhere in this - document. In particular, the 220, 221, 251, 421, and 551 reply codes - are associated with message text that must be parsed and interpreted - by machines. In the general case, the text may be receiver dependent - and context dependent, so there are likely to be varying texts for - each reply code. A discussion of the theory of reply codes is given - in section 4.2.1. Formally, a reply is defined to be the sequence: a - three-digit code, , one line of text, and , or a multiline - reply (as defined in section 4.2.1). Since, in violation of this - specification, the text is sometimes not sent, clients which do not - receive it SHOULD be prepared to process the code alone (with or - without a trailing space character). Only the EHLO, EXPN, and HELP - commands are expected to result in multiline replies in normal - circumstances, however, multiline replies are allowed for any - command. - - In ABNF, server responses are: - - Greeting = "220 " Domain [ SP text ] CRLF - Reply-line = Reply-code [ SP text ] CRLF - - where "Greeting" appears only in the 220 response that announces that - the server is opening its part of the connection. - - An SMTP server SHOULD send only the reply codes listed in this - document. An SMTP server SHOULD use the text shown in the examples - whenever appropriate. - - An SMTP client MUST determine its actions only by the reply code, not - by the text (except for the "change of address" 251 and 551 and, if - necessary, 220, 221, and 421 replies); in the general case, any text, - including no text at all (although senders SHOULD NOT send bare - codes), MUST be acceptable. The space (blank) following the reply - code is considered part of the text. Whenever possible, a receiver- - SMTP SHOULD test the first digit (severity indication) of the reply - code. - - - - - - -Klensin Standards Track [Page 41] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - The list of codes that appears below MUST NOT be construed as - permanent. While the addition of new codes should be a rare and - significant activity, with supplemental information in the textual - part of the response being preferred, new codes may be added as the - result of new Standards or Standards-track specifications. - Consequently, a sender-SMTP MUST be prepared to handle codes not - specified in this document and MUST do so by interpreting the first - digit only. - -4.2.1 Reply Code Severities and Theory - - The three digits of the reply each have a special significance. The - first digit denotes whether the response is good, bad or incomplete. - An unsophisticated SMTP client, or one that receives an unexpected - code, will be able to determine its next action (proceed as planned, - redo, retrench, etc.) by examining this first digit. An SMTP client - that wants to know approximately what kind of error occurred (e.g., - mail system error, command syntax error) may examine the second - digit. The third digit and any supplemental information that may be - present is reserved for the finest gradation of information. - - There are five values for the first digit of the reply code: - - 1yz Positive Preliminary reply - The command has been accepted, but the requested action is being - held in abeyance, pending confirmation of the information in this - reply. The SMTP client should send another command specifying - whether to continue or abort the action. Note: unextended SMTP - does not have any commands that allow this type of reply, and so - does not have continue or abort commands. - - 2yz Positive Completion reply - The requested action has been successfully completed. A new - request may be initiated. - - 3yz Positive Intermediate reply - The command has been accepted, but the requested action is being - held in abeyance, pending receipt of further information. The - SMTP client should send another command specifying this - information. This reply is used in command sequence groups (i.e., - in DATA). - - 4yz Transient Negative Completion reply - The command was not accepted, and the requested action did not - occur. However, the error condition is temporary and the action - may be requested again. The sender should return to the beginning - of the command sequence (if any). It is difficult to assign a - meaning to "transient" when two different sites (receiver- and - - - -Klensin Standards Track [Page 42] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - sender-SMTP agents) must agree on the interpretation. Each reply - in this category might have a different time value, but the SMTP - client is encouraged to try again. A rule of thumb to determine - whether a reply fits into the 4yz or the 5yz category (see below) - is that replies are 4yz if they can be successful if repeated - without any change in command form or in properties of the sender - or receiver (that is, the command is repeated identically and the - receiver does not put up a new implementation.) - - 5yz Permanent Negative Completion reply - The command was not accepted and the requested action did not - occur. The SMTP client is discouraged from repeating the exact - request (in the same sequence). Even some "permanent" error - conditions can be corrected, so the human user may want to direct - the SMTP client to reinitiate the command sequence by direct - action at some point in the future (e.g., after the spelling has - been changed, or the user has altered the account status). - - The second digit encodes responses in specific categories: - - x0z Syntax: These replies refer to syntax errors, syntactically - correct commands that do not fit any functional category, and - unimplemented or superfluous commands. - - x1z Information: These are replies to requests for information, - such as status or help. - - x2z Connections: These are replies referring to the transmission - channel. - - x3z Unspecified. - - x4z Unspecified. - - x5z Mail system: These replies indicate the status of the receiver - mail system vis-a-vis the requested transfer or other mail system - action. - - The third digit gives a finer gradation of meaning in each category - specified by the second digit. The list of replies illustrates this. - Each reply text is recommended rather than mandatory, and may even - change according to the command with which it is associated. On the - other hand, the reply codes must strictly follow the specifications - in this section. Receiver implementations should not invent new - codes for slightly different situations from the ones described here, - but rather adapt codes already defined. - - - - - -Klensin Standards Track [Page 43] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - For example, a command such as NOOP, whose successful execution does - not offer the SMTP client any new information, will return a 250 - reply. The reply is 502 when the command requests an unimplemented - non-site-specific action. A refinement of that is the 504 reply for - a command that is implemented, but that requests an unimplemented - parameter. - - The reply text may be longer than a single line; in these cases the - complete text must be marked so the SMTP client knows when it can - stop reading the reply. This requires a special format to indicate a - multiple line reply. - - The format for multiline replies requires that every line, except the - last, begin with the reply code, followed immediately by a hyphen, - "-" (also known as minus), followed by text. The last line will - begin with the reply code, followed immediately by , optionally - some text, and . As noted above, servers SHOULD send the - if subsequent text is not sent, but clients MUST be prepared for it - to be omitted. - - For example: - - 123-First line - 123-Second line - 123-234 text beginning with numbers - 123 The last line - - In many cases the SMTP client then simply needs to search for a line - beginning with the reply code followed by or and ignore - all preceding lines. In a few cases, there is important data for the - client in the reply "text". The client will be able to identify - these cases from the current context. - -4.2.2 Reply Codes by Function Groups - - 500 Syntax error, command unrecognized - (This may include errors such as command line too long) - 501 Syntax error in parameters or arguments - 502 Command not implemented (see section 4.2.4) - 503 Bad sequence of commands - 504 Command parameter not implemented - - 211 System status, or system help reply - 214 Help message - (Information on how to use the receiver or the meaning of a - particular non-standard command; this reply is useful only - to the human user) - - - - -Klensin Standards Track [Page 44] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - 220 Service ready - 221 Service closing transmission channel - 421 Service not available, closing transmission channel - (This may be a reply to any command if the service knows it - must shut down) - - 250 Requested mail action okay, completed - 251 User not local; will forward to - (See section 3.4) - 252 Cannot VRFY user, but will accept message and attempt - delivery - (See section 3.5.3) - 450 Requested mail action not taken: mailbox unavailable - (e.g., mailbox busy) - 550 Requested action not taken: mailbox unavailable - (e.g., mailbox not found, no access, or command rejected - for policy reasons) - 451 Requested action aborted: error in processing - 551 User not local; please try - (See section 3.4) - 452 Requested action not taken: insufficient system storage - 552 Requested mail action aborted: exceeded storage allocation - 553 Requested action not taken: mailbox name not allowed - (e.g., mailbox syntax incorrect) - 354 Start mail input; end with . - 554 Transaction failed (Or, in the case of a connection-opening - response, "No SMTP service here") - -4.2.3 Reply Codes in Numeric Order - - 211 System status, or system help reply - 214 Help message - (Information on how to use the receiver or the meaning of a - particular non-standard command; this reply is useful only - to the human user) - 220 Service ready - 221 Service closing transmission channel - 250 Requested mail action okay, completed - 251 User not local; will forward to - (See section 3.4) - 252 Cannot VRFY user, but will accept message and attempt - delivery - (See section 3.5.3) - - 354 Start mail input; end with . - - - - - - -Klensin Standards Track [Page 45] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - 421 Service not available, closing transmission channel - (This may be a reply to any command if the service knows it - must shut down) - 450 Requested mail action not taken: mailbox unavailable - (e.g., mailbox busy) - 451 Requested action aborted: local error in processing - 452 Requested action not taken: insufficient system storage - 500 Syntax error, command unrecognized - (This may include errors such as command line too long) - 501 Syntax error in parameters or arguments - 502 Command not implemented (see section 4.2.4) - 503 Bad sequence of commands - 504 Command parameter not implemented - 550 Requested action not taken: mailbox unavailable - (e.g., mailbox not found, no access, or command rejected - for policy reasons) - 551 User not local; please try - (See section 3.4) - 552 Requested mail action aborted: exceeded storage allocation - 553 Requested action not taken: mailbox name not allowed - (e.g., mailbox syntax incorrect) - 554 Transaction failed (Or, in the case of a connection-opening - response, "No SMTP service here") - -4.2.4 Reply Code 502 - - Questions have been raised as to when reply code 502 (Command not - implemented) SHOULD be returned in preference to other codes. 502 - SHOULD be used when the command is actually recognized by the SMTP - server, but not implemented. If the command is not recognized, code - 500 SHOULD be returned. Extended SMTP systems MUST NOT list - capabilities in response to EHLO for which they will return 502 (or - 500) replies. - -4.2.5 Reply Codes After DATA and the Subsequent . - - When an SMTP server returns a positive completion status (2yz code) - after the DATA command is completed with ., it accepts - responsibility for: - - - delivering the message (if the recipient mailbox exists), or - - - if attempts to deliver the message fail due to transient - conditions, retrying delivery some reasonable number of times at - intervals as specified in section 4.5.4. - - - - - - -Klensin Standards Track [Page 46] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - - if attempts to deliver the message fail due to permanent - conditions, or if repeated attempts to deliver the message fail - due to transient conditions, returning appropriate notification to - the sender of the original message (using the address in the SMTP - MAIL command). - - When an SMTP server returns a permanent error status (5yz) code after - the DATA command is completed with ., it MUST NOT make - any subsequent attempt to deliver that message. The SMTP client - retains responsibility for delivery of that message and may either - return it to the user or requeue it for a subsequent attempt (see - section 4.5.4.1). - - The user who originated the message SHOULD be able to interpret the - return of a transient failure status (by mail message or otherwise) - as a non-delivery indication, just as a permanent failure would be - interpreted. I.e., if the client SMTP successfully handles these - conditions, the user will not receive such a reply. - - When an SMTP server returns a permanent error status (5yz) code after - the DATA command is completely with ., it MUST NOT make - any subsequent attempt to deliver the message. As with temporary - error status codes, the SMTP client retains responsibility for the - message, but SHOULD not again attempt delivery to the same server - without user review and intervention of the message. - -4.3 Sequencing of Commands and Replies - -4.3.1 Sequencing Overview - - The communication between the sender and receiver is an alternating - dialogue, controlled by the sender. As such, the sender issues a - command and the receiver responds with a reply. Unless other - arrangements are negotiated through service extensions, the sender - MUST wait for this response before sending further commands. - - One important reply is the connection greeting. Normally, a receiver - will send a 220 "Service ready" reply when the connection is - completed. The sender SHOULD wait for this greeting message before - sending any commands. - - Note: all the greeting-type replies have the official name (the - fully-qualified primary domain name) of the server host as the first - word following the reply code. Sometimes the host will have no - meaningful name. See 4.1.3 for a discussion of alternatives in these - situations. - - - - - -Klensin Standards Track [Page 47] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - For example, - - 220 ISIF.USC.EDU Service ready - or - 220 mail.foo.com SuperSMTP v 6.1.2 Service ready - or - 220 [10.0.0.1] Clueless host service ready - - The table below lists alternative success and failure replies for - each command. These SHOULD be strictly adhered to: a receiver may - substitute text in the replies, but the meaning and action implied by - the code numbers and by the specific command reply sequence cannot be - altered. - -4.3.2 Command-Reply Sequences - - Each command is listed with its usual possible replies. The prefixes - used before the possible replies are "I" for intermediate, "S" for - success, and "E" for error. Since some servers may generate other - replies under special circumstances, and to allow for future - extension, SMTP clients SHOULD, when possible, interpret only the - first digit of the reply and MUST be prepared to deal with - unrecognized reply codes by interpreting the first digit only. - Unless extended using the mechanisms described in section 2.2, SMTP - servers MUST NOT transmit reply codes to an SMTP client that are - other than three digits or that do not start in a digit between 2 and - 5 inclusive. - - These sequencing rules and, in principle, the codes themselves, can - be extended or modified by SMTP extensions offered by the server and - accepted (requested) by the client. - - In addition to the codes listed below, any SMTP command can return - any of the following codes if the corresponding unusual circumstances - are encountered: - - 500 For the "command line too long" case or if the command name was - not recognized. Note that producing a "command not recognized" - error in response to the required subset of these commands is a - violation of this specification. - - 501 Syntax error in command or arguments. In order to provide for - future extensions, commands that are specified in this document as - not accepting arguments (DATA, RSET, QUIT) SHOULD return a 501 - message if arguments are supplied in the absence of EHLO- - advertised extensions. - - 421 Service shutting down and closing transmission channel - - - -Klensin Standards Track [Page 48] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - Specific sequences are: - - CONNECTION ESTABLISHMENT - S: 220 - E: 554 - EHLO or HELO - S: 250 - E: 504, 550 - MAIL - S: 250 - E: 552, 451, 452, 550, 553, 503 - RCPT - S: 250, 251 (but see section 3.4 for discussion of 251 and 551) - E: 550, 551, 552, 553, 450, 451, 452, 503, 550 - DATA - I: 354 -> data -> S: 250 - E: 552, 554, 451, 452 - E: 451, 554, 503 - RSET - S: 250 - VRFY - S: 250, 251, 252 - E: 550, 551, 553, 502, 504 - EXPN - S: 250, 252 - E: 550, 500, 502, 504 - HELP - S: 211, 214 - E: 502, 504 - NOOP - S: 250 - QUIT - S: 221 - -4.4 Trace Information - - When an SMTP server receives a message for delivery or further - processing, it MUST insert trace ("time stamp" or "Received") - information at the beginning of the message content, as discussed in - section 4.1.1.4. - - This line MUST be structured as follows: - - - The FROM field, which MUST be supplied in an SMTP environment, - SHOULD contain both (1) the name of the source host as presented - in the EHLO command and (2) an address literal containing the IP - address of the source, determined from the TCP connection. - - - - -Klensin Standards Track [Page 49] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - - The ID field MAY contain an "@" as suggested in RFC 822, but this - is not required. - - - The FOR field MAY contain a list of entries when multiple - RCPT commands have been given. This may raise some security - issues and is usually not desirable; see section 7.2. - - An Internet mail program MUST NOT change a Received: line that was - previously added to the message header. SMTP servers MUST prepend - Received lines to messages; they MUST NOT change the order of - existing lines or insert Received lines in any other location. - - As the Internet grows, comparability of Received fields is important - for detecting problems, especially slow relays. SMTP servers that - create Received fields SHOULD use explicit offsets in the dates - (e.g., -0800), rather than time zone names of any type. Local time - (with an offset) is preferred to UT when feasible. This formulation - allows slightly more information about local circumstances to be - specified. If UT is needed, the receiver need merely do some simple - arithmetic to convert the values. Use of UT loses information about - the time zone-location of the server. If it is desired to supply a - time zone name, it SHOULD be included in a comment. - - When the delivery SMTP server makes the "final delivery" of a - message, it inserts a return-path line at the beginning of the mail - data. This use of return-path is required; mail systems MUST support - it. The return-path line preserves the information in the from the MAIL command. Here, final delivery means the message - has left the SMTP environment. Normally, this would mean it had been - delivered to the destination user or an associated mail drop, but in - some cases it may be further processed and transmitted by another - mail system. - - It is possible for the mailbox in the return path to be different - from the actual sender's mailbox, for example, if error responses are - to be delivered to a special error handling mailbox rather than to - the message sender. When mailing lists are involved, this - arrangement is common and useful as a means of directing errors to - the list maintainer rather than the message originator. - - The text above implies that the final mail data will begin with a - return path line, followed by one or more time stamp lines. These - lines will be followed by the mail data headers and body [32]. - - It is sometimes difficult for an SMTP server to determine whether or - not it is making final delivery since forwarding or other operations - may occur after the message is accepted for delivery. Consequently, - - - - -Klensin Standards Track [Page 50] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - any further (forwarding, gateway, or relay) systems MAY remove the - return path and rebuild the MAIL command as needed to ensure that - exactly one such line appears in a delivered message. - - A message-originating SMTP system SHOULD NOT send a message that - already contains a Return-path header. SMTP servers performing a - relay function MUST NOT inspect the message data, and especially not - to the extent needed to determine if Return-path headers are present. - SMTP servers making final delivery MAY remove Return-path headers - before adding their own. - - The primary purpose of the Return-path is to designate the address to - which messages indicating non-delivery or other mail system failures - are to be sent. For this to be unambiguous, exactly one return path - SHOULD be present when the message is delivered. Systems using RFC - 822 syntax with non-SMTP transports SHOULD designate an unambiguous - address, associated with the transport envelope, to which error - reports (e.g., non-delivery messages) should be sent. - - Historical note: Text in RFC 822 that appears to contradict the use - of the Return-path header (or the envelope reverse path address from - the MAIL command) as the destination for error messages is not - applicable on the Internet. The reverse path address (as copied into - the Return-path) MUST be used as the target of any mail containing - delivery error messages. - - In particular: - - - a gateway from SMTP->elsewhere SHOULD insert a return-path header, - unless it is known that the "elsewhere" transport also uses - Internet domain addresses and maintains the envelope sender - address separately. - - - a gateway from elsewhere->SMTP SHOULD delete any return-path - header present in the message, and either copy that information to - the SMTP envelope or combine it with information present in the - envelope of the other transport system to construct the reverse - path argument to the MAIL command in the SMTP envelope. - - The server must give special treatment to cases in which the - processing following the end of mail data indication is only - partially successful. This could happen if, after accepting several - recipients and the mail data, the SMTP server finds that the mail - data could be successfully delivered to some, but not all, of the - recipients. In such cases, the response to the DATA command MUST be - an OK reply. However, the SMTP server MUST compose and send an - "undeliverable mail" notification message to the originator of the - message. - - - -Klensin Standards Track [Page 51] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - A single notification listing all of the failed recipients or - separate notification messages MUST be sent for each failed - recipient. For economy of processing by the sender, the former is - preferred when possible. All undeliverable mail notification - messages are sent using the MAIL command (even if they result from - processing the obsolete SEND, SOML, or SAML commands) and use a null - return path as discussed in section 3.7. - - The time stamp line and the return path line are formally defined as - follows: - -Return-path-line = "Return-Path:" FWS Reverse-path - -Time-stamp-line = "Received:" FWS Stamp - -Stamp = From-domain By-domain Opt-info ";" FWS date-time - - ; where "date-time" is as defined in [32] - ; but the "obs-" forms, especially two-digit - ; years, are prohibited in SMTP and MUST NOT be used. - -From-domain = "FROM" FWS Extended-Domain CFWS - -By-domain = "BY" FWS Extended-Domain CFWS - -Extended-Domain = Domain / - ( Domain FWS "(" TCP-info ")" ) / - ( Address-literal FWS "(" TCP-info ")" ) - -TCP-info = Address-literal / ( Domain FWS Address-literal ) - ; Information derived by server from TCP connection - ; not client EHLO. - -Opt-info = [Via] [With] [ID] [For] - -Via = "VIA" FWS Link CFWS - -With = "WITH" FWS Protocol CFWS - -ID = "ID" FWS String / msg-id CFWS - -For = "FOR" FWS 1*( Path / Mailbox ) CFWS - -Link = "TCP" / Addtl-Link -Addtl-Link = Atom - ; Additional standard names for links are registered with the - ; Internet Assigned Numbers Authority (IANA). "Via" is - ; primarily of value with non-Internet transports. SMTP - - - -Klensin Standards Track [Page 52] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - ; servers SHOULD NOT use unregistered names. -Protocol = "ESMTP" / "SMTP" / Attdl-Protocol -Attdl-Protocol = Atom - ; Additional standard names for protocols are registered with the - ; Internet Assigned Numbers Authority (IANA). SMTP servers - ; SHOULD NOT use unregistered names. - -4.5 Additional Implementation Issues - -4.5.1 Minimum Implementation - - In order to make SMTP workable, the following minimum implementation - is required for all receivers. The following commands MUST be - supported to conform to this specification: - - EHLO - HELO - MAIL - RCPT - DATA - RSET - NOOP - QUIT - VRFY - - Any system that includes an SMTP server supporting mail relaying or - delivery MUST support the reserved mailbox "postmaster" as a case- - insensitive local name. This postmaster address is not strictly - necessary if the server always returns 554 on connection opening (as - described in section 3.1). The requirement to accept mail for - postmaster implies that RCPT commands which specify a mailbox for - postmaster at any of the domains for which the SMTP server provides - mail service, as well as the special case of "RCPT TO:" - (with no domain specification), MUST be supported. - - SMTP systems are expected to make every reasonable effort to accept - mail directed to Postmaster from any other system on the Internet. - In extreme cases --such as to contain a denial of service attack or - other breach of security-- an SMTP server may block mail directed to - Postmaster. However, such arrangements SHOULD be narrowly tailored - so as to avoid blocking messages which are not part of such attacks. - -4.5.2 Transparency - - Without some provision for data transparency, the character sequence - "." ends the mail text and cannot be sent by the user. - In general, users are not aware of such "forbidden" sequences. To - - - - -Klensin Standards Track [Page 53] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - allow all user composed text to be transmitted transparently, the - following procedures are used: - - - Before sending a line of mail text, the SMTP client checks the - first character of the line. If it is a period, one additional - period is inserted at the beginning of the line. - - - When a line of mail text is received by the SMTP server, it checks - the line. If the line is composed of a single period, it is - treated as the end of mail indicator. If the first character is a - period and there are other characters on the line, the first - character is deleted. - - The mail data may contain any of the 128 ASCII characters. All - characters are to be delivered to the recipient's mailbox, including - spaces, vertical and horizontal tabs, and other control characters. - If the transmission channel provides an 8-bit byte (octet) data - stream, the 7-bit ASCII codes are transmitted right justified in the - octets, with the high order bits cleared to zero. See 3.7 for - special treatment of these conditions in SMTP systems serving a relay - function. - - In some systems it may be necessary to transform the data as it is - received and stored. This may be necessary for hosts that use a - different character set than ASCII as their local character set, that - store data in records rather than strings, or which use special - character sequences as delimiters inside mailboxes. If such - transformations are necessary, they MUST be reversible, especially if - they are applied to mail being relayed. - -4.5.3 Sizes and Timeouts - -4.5.3.1 Size limits and minimums - - There are several objects that have required minimum/maximum sizes. - Every implementation MUST be able to receive objects of at least - these sizes. Objects larger than these sizes SHOULD be avoided when - possible. However, some Internet mail constructs such as encoded - X.400 addresses [16] will often require larger objects: clients MAY - attempt to transmit these, but MUST be prepared for a server to - reject them if they cannot be handled by it. To the maximum extent - possible, implementation techniques which impose no limits on the - length of these objects should be used. - - local-part - The maximum total length of a user name or other local-part is 64 - characters. - - - - -Klensin Standards Track [Page 54] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - domain - The maximum total length of a domain name or number is 255 - characters. - - path - The maximum total length of a reverse-path or forward-path is 256 - characters (including the punctuation and element separators). - - command line - The maximum total length of a command line including the command - word and the is 512 characters. SMTP extensions may be - used to increase this limit. - - reply line - The maximum total length of a reply line including the reply code - and the is 512 characters. More information may be - conveyed through multiple-line replies. - - text line - The maximum total length of a text line including the is - 1000 characters (not counting the leading dot duplicated for - transparency). This number may be increased by the use of SMTP - Service Extensions. - - message content - The maximum total length of a message content (including any - message headers as well as the message body) MUST BE at least 64K - octets. Since the introduction of Internet standards for - multimedia mail [12], message lengths on the Internet have grown - dramatically, and message size restrictions should be avoided if - at all possible. SMTP server systems that must impose - restrictions SHOULD implement the "SIZE" service extension [18], - and SMTP client systems that will send large messages SHOULD - utilize it when possible. - - recipients buffer - The minimum total number of recipients that must be buffered is - 100 recipients. Rejection of messages (for excessive recipients) - with fewer than 100 RCPT commands is a violation of this - specification. The general principle that relaying SMTP servers - MUST NOT, and delivery SMTP servers SHOULD NOT, perform validation - tests on message headers suggests that rejecting a message based - on the total number of recipients shown in header fields is to be - discouraged. A server which imposes a limit on the number of - recipients MUST behave in an orderly fashion, such as to reject - additional addresses over its limit rather than silently - discarding addresses previously accepted. A client that needs to - - - - -Klensin Standards Track [Page 55] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - deliver a message containing over 100 RCPT commands SHOULD be - prepared to transmit in 100-recipient "chunks" if the server - declines to accept more than 100 recipients in a single message. - - Errors due to exceeding these limits may be reported by using the - reply codes. Some examples of reply codes are: - - 500 Line too long. - or - 501 Path too long - or - 452 Too many recipients (see below) - or - 552 Too much mail data. - - RFC 821 [30] incorrectly listed the error where an SMTP server - exhausts its implementation limit on the number of RCPT commands - ("too many recipients") as having reply code 552. The correct reply - code for this condition is 452. Clients SHOULD treat a 552 code in - this case as a temporary, rather than permanent, failure so the logic - below works. - - When a conforming SMTP server encounters this condition, it has at - least 100 successful RCPT commands in its recipients buffer. If the - server is able to accept the message, then at least these 100 - addresses will be removed from the SMTP client's queue. When the - client attempts retransmission of those addresses which received 452 - responses, at least 100 of these will be able to fit in the SMTP - server's recipients buffer. Each retransmission attempt which is - able to deliver anything will be able to dispose of at least 100 of - these recipients. - - If an SMTP server has an implementation limit on the number of RCPT - commands and this limit is exhausted, it MUST use a response code of - 452 (but the client SHOULD also be prepared for a 552, as noted - above). If the server has a configured site-policy limitation on the - number of RCPT commands, it MAY instead use a 5XX response code. - This would be most appropriate if the policy limitation was intended - to apply if the total recipient count for a particular message body - were enforced even if that message body was sent in multiple mail - transactions. - -4.5.3.2 Timeouts - - An SMTP client MUST provide a timeout mechanism. It MUST use per- - command timeouts rather than somehow trying to time the entire mail - transaction. Timeouts SHOULD be easily reconfigurable, preferably - without recompiling the SMTP code. To implement this, a timer is set - - - -Klensin Standards Track [Page 56] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - for each SMTP command and for each buffer of the data transfer. The - latter means that the overall timeout is inherently proportional to - the size of the message. - - Based on extensive experience with busy mail-relay hosts, the minimum - per-command timeout values SHOULD be as follows: - - Initial 220 Message: 5 minutes - An SMTP client process needs to distinguish between a failed TCP - connection and a delay in receiving the initial 220 greeting - message. Many SMTP servers accept a TCP connection but delay - delivery of the 220 message until their system load permits more - mail to be processed. - - MAIL Command: 5 minutes - - RCPT Command: 5 minutes - A longer timeout is required if processing of mailing lists and - aliases is not deferred until after the message was accepted. - - DATA Initiation: 2 minutes - This is while awaiting the "354 Start Input" reply to a DATA - command. - - Data Block: 3 minutes - This is while awaiting the completion of each TCP SEND call - transmitting a chunk of data. - - DATA Termination: 10 minutes. - This is while awaiting the "250 OK" reply. When the receiver gets - the final period terminating the message data, it typically - performs processing to deliver the message to a user mailbox. A - spurious timeout at this point would be very wasteful and would - typically result in delivery of multiple copies of the message, - since it has been successfully sent and the server has accepted - responsibility for delivery. See section 6.1 for additional - discussion. - - An SMTP server SHOULD have a timeout of at least 5 minutes while it - is awaiting the next command from the sender. - -4.5.4 Retry Strategies - - The common structure of a host SMTP implementation includes user - mailboxes, one or more areas for queuing messages in transit, and one - or more daemon processes for sending and receiving mail. The exact - structure will vary depending on the needs of the users on the host - - - - -Klensin Standards Track [Page 57] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - and the number and size of mailing lists supported by the host. We - describe several optimizations that have proved helpful, particularly - for mailers supporting high traffic levels. - - Any queuing strategy MUST include timeouts on all activities on a - per-command basis. A queuing strategy MUST NOT send error messages - in response to error messages under any circumstances. - -4.5.4.1 Sending Strategy - - The general model for an SMTP client is one or more processes that - periodically attempt to transmit outgoing mail. In a typical system, - the program that composes a message has some method for requesting - immediate attention for a new piece of outgoing mail, while mail that - cannot be transmitted immediately MUST be queued and periodically - retried by the sender. A mail queue entry will include not only the - message itself but also the envelope information. - - The sender MUST delay retrying a particular destination after one - attempt has failed. In general, the retry interval SHOULD be at - least 30 minutes; however, more sophisticated and variable strategies - will be beneficial when the SMTP client can determine the reason for - non-delivery. - - Retries continue until the message is transmitted or the sender gives - up; the give-up time generally needs to be at least 4-5 days. The - parameters to the retry algorithm MUST be configurable. - - A client SHOULD keep a list of hosts it cannot reach and - corresponding connection timeouts, rather than just retrying queued - mail items. - - Experience suggests that failures are typically transient (the target - system or its connection has crashed), favoring a policy of two - connection attempts in the first hour the message is in the queue, - and then backing off to one every two or three hours. - - The SMTP client can shorten the queuing delay in cooperation with the - SMTP server. For example, if mail is received from a particular - address, it is likely that mail queued for that host can now be sent. - Application of this principle may, in many cases, eliminate the - requirement for an explicit "send queues now" function such as ETRN - [9]. - - The strategy may be further modified as a result of multiple - addresses per host (see below) to optimize delivery time vs. resource - usage. - - - - -Klensin Standards Track [Page 58] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - An SMTP client may have a large queue of messages for each - unavailable destination host. If all of these messages were retried - in every retry cycle, there would be excessive Internet overhead and - the sending system would be blocked for a long period. Note that an - SMTP client can generally determine that a delivery attempt has - failed only after a timeout of several minutes and even a one-minute - timeout per connection will result in a very large delay if retries - are repeated for dozens, or even hundreds, of queued messages to the - same host. - - At the same time, SMTP clients SHOULD use great care in caching - negative responses from servers. In an extreme case, if EHLO is - issued multiple times during the same SMTP connection, different - answers may be returned by the server. More significantly, 5yz - responses to the MAIL command MUST NOT be cached. - - When a mail message is to be delivered to multiple recipients, and - the SMTP server to which a copy of the message is to be sent is the - same for multiple recipients, then only one copy of the message - SHOULD be transmitted. That is, the SMTP client SHOULD use the - command sequence: MAIL, RCPT, RCPT,... RCPT, DATA instead of the - sequence: MAIL, RCPT, DATA, ..., MAIL, RCPT, DATA. However, if there - are very many addresses, a limit on the number of RCPT commands per - MAIL command MAY be imposed. Implementation of this efficiency - feature is strongly encouraged. - - Similarly, to achieve timely delivery, the SMTP client MAY support - multiple concurrent outgoing mail transactions. However, some limit - may be appropriate to protect the host from devoting all its - resources to mail. - -4.5.4.2 Receiving Strategy - - The SMTP server SHOULD attempt to keep a pending listen on the SMTP - port at all times. This requires the support of multiple incoming - TCP connections for SMTP. Some limit MAY be imposed but servers that - cannot handle more than one SMTP transaction at a time are not in - conformance with the intent of this specification. - - As discussed above, when the SMTP server receives mail from a - particular host address, it could activate its own SMTP queuing - mechanisms to retry any mail pending for that host address. - -4.5.5 Messages with a null reverse-path - - There are several types of notification messages which are required - by existing and proposed standards to be sent with a null reverse - path, namely non-delivery notifications as discussed in section 3.7, - - - -Klensin Standards Track [Page 59] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - other kinds of Delivery Status Notifications (DSNs) [24], and also - Message Disposition Notifications (MDNs) [10]. All of these kinds of - messages are notifications about a previous message, and they are - sent to the reverse-path of the previous mail message. (If the - delivery of such a notification message fails, that usually indicates - a problem with the mail system of the host to which the notification - message is addressed. For this reason, at some hosts the MTA is set - up to forward such failed notification messages to someone who is - able to fix problems with the mail system, e.g., via the postmaster - alias.) - - All other types of messages (i.e., any message which is not required - by a standards-track RFC to have a null reverse-path) SHOULD be sent - with with a valid, non-null reverse-path. - - Implementors of automated email processors should be careful to make - sure that the various kinds of messages with null reverse-path are - handled correctly, in particular such systems SHOULD NOT reply to - messages with null reverse-path. - -5. Address Resolution and Mail Handling - - Once an SMTP client lexically identifies a domain to which mail will - be delivered for processing (as described in sections 3.6 and 3.7), a - DNS lookup MUST be performed to resolve the domain name [22]. The - names are expected to be fully-qualified domain names (FQDNs): - mechanisms for inferring FQDNs from partial names or local aliases - are outside of this specification and, due to a history of problems, - are generally discouraged. The lookup first attempts to locate an MX - record associated with the name. If a CNAME record is found instead, - the resulting name is processed as if it were the initial name. If - no MX records are found, but an A RR is found, the A RR is treated as - if it was associated with an implicit MX RR, with a preference of 0, - pointing to that host. If one or more MX RRs are found for a given - name, SMTP systems MUST NOT utilize any A RRs associated with that - name unless they are located using the MX RRs; the "implicit MX" rule - above applies only if there are no MX records present. If MX records - are present, but none of them are usable, this situation MUST be - reported as an error. - - When the lookup succeeds, the mapping can result in a list of - alternative delivery addresses rather than a single address, because - of multiple MX records, multihoming, or both. To provide reliable - mail transmission, the SMTP client MUST be able to try (and retry) - each of the relevant addresses in this list in order, until a - delivery attempt succeeds. However, there MAY also be a configurable - limit on the number of alternate addresses that can be tried. In any - case, the SMTP client SHOULD try at least two addresses. - - - -Klensin Standards Track [Page 60] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - Two types of information is used to rank the host addresses: multiple - MX records, and multihomed hosts. - - Multiple MX records contain a preference indication that MUST be used - in sorting (see below). Lower numbers are more preferred than higher - ones. If there are multiple destinations with the same preference - and there is no clear reason to favor one (e.g., by recognition of an - easily-reached address), then the sender-SMTP MUST randomize them to - spread the load across multiple mail exchangers for a specific - organization. - - The destination host (perhaps taken from the preferred MX record) may - be multihomed, in which case the domain name resolver will return a - list of alternative IP addresses. It is the responsibility of the - domain name resolver interface to have ordered this list by - decreasing preference if necessary, and SMTP MUST try them in the - order presented. - - Although the capability to try multiple alternative addresses is - required, specific installations may want to limit or disable the use - of alternative addresses. The question of whether a sender should - attempt retries using the different addresses of a multihomed host - has been controversial. The main argument for using the multiple - addresses is that it maximizes the probability of timely delivery, - and indeed sometimes the probability of any delivery; the counter- - argument is that it may result in unnecessary resource use. Note - that resource use is also strongly determined by the sending strategy - discussed in section 4.5.4.1. - - If an SMTP server receives a message with a destination for which it - is a designated Mail eXchanger, it MAY relay the message (potentially - after having rewritten the MAIL FROM and/or RCPT TO addresses), make - final delivery of the message, or hand it off using some mechanism - outside the SMTP-provided transport environment. Of course, neither - of the latter require that the list of MX records be examined - further. - - If it determines that it should relay the message without rewriting - the address, it MUST sort the MX records to determine candidates for - delivery. The records are first ordered by preference, with the - lowest-numbered records being most preferred. The relay host MUST - then inspect the list for any of the names or addresses by which it - might be known in mail transactions. If a matching record is found, - all records at that preference level and higher-numbered ones MUST be - discarded from consideration. If there are no records left at that - point, it is an error condition, and the message MUST be returned as - undeliverable. If records do remain, they SHOULD be tried, best - preference first, as described above. - - - -Klensin Standards Track [Page 61] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -6. Problem Detection and Handling - -6.1 Reliable Delivery and Replies by Email - - When the receiver-SMTP accepts a piece of mail (by sending a "250 OK" - message in response to DATA), it is accepting responsibility for - delivering or relaying the message. It must take this responsibility - seriously. It MUST NOT lose the message for frivolous reasons, such - as because the host later crashes or because of a predictable - resource shortage. - - If there is a delivery failure after acceptance of a message, the - receiver-SMTP MUST formulate and mail a notification message. This - notification MUST be sent using a null ("<>") reverse path in the - envelope. The recipient of this notification MUST be the address - from the envelope return path (or the Return-Path: line). However, - if this address is null ("<>"), the receiver-SMTP MUST NOT send a - notification. Obviously, nothing in this section can or should - prohibit local decisions (i.e., as part of the same system - environment as the receiver-SMTP) to log or otherwise transmit - information about null address events locally if that is desired. If - the address is an explicit source route, it MUST be stripped down to - its final hop. - - For example, suppose that an error notification must be sent for a - message that arrived with: - - MAIL FROM:<@a,@b:user@d> - - The notification message MUST be sent using: - - RCPT TO: - - Some delivery failures after the message is accepted by SMTP will be - unavoidable. For example, it may be impossible for the receiving - SMTP server to validate all the delivery addresses in RCPT command(s) - due to a "soft" domain system error, because the target is a mailing - list (see earlier discussion of RCPT), or because the server is - acting as a relay and has no immediate access to the delivering - system. - - To avoid receiving duplicate messages as the result of timeouts, a - receiver-SMTP MUST seek to minimize the time required to respond to - the final . end of data indicator. See RFC 1047 [28] for - a discussion of this problem. - - - - - - -Klensin Standards Track [Page 62] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -6.2 Loop Detection - - Simple counting of the number of "Received:" headers in a message has - proven to be an effective, although rarely optimal, method of - detecting loops in mail systems. SMTP servers using this technique - SHOULD use a large rejection threshold, normally at least 100 - Received entries. Whatever mechanisms are used, servers MUST contain - provisions for detecting and stopping trivial loops. - -6.3 Compensating for Irregularities - - Unfortunately, variations, creative interpretations, and outright - violations of Internet mail protocols do occur; some would suggest - that they occur quite frequently. The debate as to whether a well- - behaved SMTP receiver or relay should reject a malformed message, - attempt to pass it on unchanged, or attempt to repair it to increase - the odds of successful delivery (or subsequent reply) began almost - with the dawn of structured network mail and shows no signs of - abating. Advocates of rejection claim that attempted repairs are - rarely completely adequate and that rejection of bad messages is the - only way to get the offending software repaired. Advocates of - "repair" or "deliver no matter what" argue that users prefer that - mail go through it if at all possible and that there are significant - market pressures in that direction. In practice, these market - pressures may be more important to particular vendors than strict - conformance to the standards, regardless of the preference of the - actual developers. - - The problems associated with ill-formed messages were exacerbated by - the introduction of the split-UA mail reading protocols [3, 26, 5, - 21]. These protocols have encouraged the use of SMTP as a posting - protocol, and SMTP servers as relay systems for these client hosts - (which are often only intermittently connected to the Internet). - Historically, many of those client machines lacked some of the - mechanisms and information assumed by SMTP (and indeed, by the mail - format protocol [7]). Some could not keep adequate track of time; - others had no concept of time zones; still others could not identify - their own names or addresses; and, of course, none could satisfy the - assumptions that underlay RFC 822's conception of authenticated - addresses. - - In response to these weak SMTP clients, many SMTP systems now - complete messages that are delivered to them in incomplete or - incorrect form. This strategy is generally considered appropriate - when the server can identify or authenticate the client, and there - are prior agreements between them. By contrast, there is at best - great concern about fixes applied by a relay or delivery SMTP server - that has little or no knowledge of the user or client machine. - - - -Klensin Standards Track [Page 63] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - The following changes to a message being processed MAY be applied - when necessary by an originating SMTP server, or one used as the - target of SMTP as an initial posting protocol: - - - Addition of a message-id field when none appears - - - Addition of a date, time or time zone when none appears - - - Correction of addresses to proper FQDN format - - The less information the server has about the client, the less likely - these changes are to be correct and the more caution and conservatism - should be applied when considering whether or not to perform fixes - and how. These changes MUST NOT be applied by an SMTP server that - provides an intermediate relay function. - - In all cases, properly-operating clients supplying correct - information are preferred to corrections by the SMTP server. In all - cases, documentation of actions performed by the servers (in trace - fields and/or header comments) is strongly encouraged. - -7. Security Considerations - -7.1 Mail Security and Spoofing - - SMTP mail is inherently insecure in that it is feasible for even - fairly casual users to negotiate directly with receiving and relaying - SMTP servers and create messages that will trick a naive recipient - into believing that they came from somewhere else. Constructing such - a message so that the "spoofed" behavior cannot be detected by an - expert is somewhat more difficult, but not sufficiently so as to be a - deterrent to someone who is determined and knowledgeable. - Consequently, as knowledge of Internet mail increases, so does the - knowledge that SMTP mail inherently cannot be authenticated, or - integrity checks provided, at the transport level. Real mail - security lies only in end-to-end methods involving the message - bodies, such as those which use digital signatures (see [14] and, - e.g., PGP [4] or S/MIME [31]). - - Various protocol extensions and configuration options that provide - authentication at the transport level (e.g., from an SMTP client to - an SMTP server) improve somewhat on the traditional situation - described above. However, unless they are accompanied by careful - handoffs of responsibility in a carefully-designed trust environment, - they remain inherently weaker than end-to-end mechanisms which use - digitally signed messages rather than depending on the integrity of - the transport system. - - - - -Klensin Standards Track [Page 64] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - Efforts to make it more difficult for users to set envelope return - path and header "From" fields to point to valid addresses other than - their own are largely misguided: they frustrate legitimate - applications in which mail is sent by one user on behalf of another - or in which error (or normal) replies should be directed to a special - address. (Systems that provide convenient ways for users to alter - these fields on a per-message basis should attempt to establish a - primary and permanent mailbox address for the user so that Sender - fields within the message data can be generated sensibly.) - - This specification does not further address the authentication issues - associated with SMTP other than to advocate that useful functionality - not be disabled in the hope of providing some small margin of - protection against an ignorant user who is trying to fake mail. - -7.2 "Blind" Copies - - Addresses that do not appear in the message headers may appear in the - RCPT commands to an SMTP server for a number of reasons. The two - most common involve the use of a mailing address as a "list exploder" - (a single address that resolves into multiple addresses) and the - appearance of "blind copies". Especially when more than one RCPT - command is present, and in order to avoid defeating some of the - purpose of these mechanisms, SMTP clients and servers SHOULD NOT copy - the full set of RCPT command arguments into the headers, either as - part of trace headers or as informational or private-extension - headers. Since this rule is often violated in practice, and cannot - be enforced, sending SMTP systems that are aware of "bcc" use MAY - find it helpful to send each blind copy as a separate message - transaction containing only a single RCPT command. - - There is no inherent relationship between either "reverse" (from - MAIL, SAML, etc., commands) or "forward" (RCPT) addresses in the SMTP - transaction ("envelope") and the addresses in the headers. Receiving - systems SHOULD NOT attempt to deduce such relationships and use them - to alter the headers of the message for delivery. The popular - "Apparently-to" header is a violation of this principle as well as a - common source of unintended information disclosure and SHOULD NOT be - used. - -7.3 VRFY, EXPN, and Security - - As discussed in section 3.5, individual sites may want to disable - either or both of VRFY or EXPN for security reasons. As a corollary - to the above, implementations that permit this MUST NOT appear to - have verified addresses that are not, in fact, verified. If a site - - - - - -Klensin Standards Track [Page 65] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - disables these commands for security reasons, the SMTP server MUST - return a 252 response, rather than a code that could be confused with - successful or unsuccessful verification. - - Returning a 250 reply code with the address listed in the VRFY - command after having checked it only for syntax violates this rule. - Of course, an implementation that "supports" VRFY by always returning - 550 whether or not the address is valid is equally not in - conformance. - - Within the last few years, the contents of mailing lists have become - popular as an address information source for so-called "spammers." - The use of EXPN to "harvest" addresses has increased as list - administrators have installed protections against inappropriate uses - of the lists themselves. Implementations SHOULD still provide - support for EXPN, but sites SHOULD carefully evaluate the tradeoffs. - As authentication mechanisms are introduced into SMTP, some sites may - choose to make EXPN available only to authenticated requestors. - -7.4 Information Disclosure in Announcements - - There has been an ongoing debate about the tradeoffs between the - debugging advantages of announcing server type and version (and, - sometimes, even server domain name) in the greeting response or in - response to the HELP command and the disadvantages of exposing - information that might be useful in a potential hostile attack. The - utility of the debugging information is beyond doubt. Those who - argue for making it available point out that it is far better to - actually secure an SMTP server rather than hope that trying to - conceal known vulnerabilities by hiding the server's precise identity - will provide more protection. Sites are encouraged to evaluate the - tradeoff with that issue in mind; implementations are strongly - encouraged to minimally provide for making type and version - information available in some way to other network hosts. - -7.5 Information Disclosure in Trace Fields - - In some circumstances, such as when mail originates from within a LAN - whose hosts are not directly on the public Internet, trace - ("Received") fields produced in conformance with this specification - may disclose host names and similar information that would not - normally be available. This ordinarily does not pose a problem, but - sites with special concerns about name disclosure should be aware of - it. Also, the optional FOR clause should be supplied with caution or - not at all when multiple recipients are involved lest it - inadvertently disclose the identities of "blind copy" recipients to - others. - - - - -Klensin Standards Track [Page 66] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -7.6 Information Disclosure in Message Forwarding - - As discussed in section 3.4, use of the 251 or 551 reply codes to - identify the replacement address associated with a mailbox may - inadvertently disclose sensitive information. Sites that are - concerned about those issues should ensure that they select and - configure servers appropriately. - -7.7 Scope of Operation of SMTP Servers - - It is a well-established principle that an SMTP server may refuse to - accept mail for any operational or technical reason that makes sense - to the site providing the server. However, cooperation among sites - and installations makes the Internet possible. If sites take - excessive advantage of the right to reject traffic, the ubiquity of - email availability (one of the strengths of the Internet) will be - threatened; considerable care should be taken and balance maintained - if a site decides to be selective about the traffic it will accept - and process. - - In recent years, use of the relay function through arbitrary sites - has been used as part of hostile efforts to hide the actual origins - of mail. Some sites have decided to limit the use of the relay - function to known or identifiable sources, and implementations SHOULD - provide the capability to perform this type of filtering. When mail - is rejected for these or other policy reasons, a 550 code SHOULD be - used in response to EHLO, MAIL, or RCPT as appropriate. - -8. IANA Considerations - - IANA will maintain three registries in support of this specification. - The first consists of SMTP service extensions with the associated - keywords, and, as needed, parameters and verbs. As specified in - section 2.2.2, no entry may be made in this registry that starts in - an "X". Entries may be made only for service extensions (and - associated keywords, parameters, or verbs) that are defined in - standards-track or experimental RFCs specifically approved by the - IESG for this purpose. - - The second registry consists of "tags" that identify forms of domain - literals other than those for IPv4 addresses (specified in RFC 821 - and in this document) and IPv6 addresses (specified in this - document). Additional literal types require standardization before - being used; none are anticipated at this time. - - The third, established by RFC 821 and renewed by this specification, - is a registry of link and protocol identifiers to be used with the - "via" and "with" subclauses of the time stamp ("Received: header") - - - -Klensin Standards Track [Page 67] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - described in section 4.4. Link and protocol identifiers in addition - to those specified in this document may be registered only by - standardization or by way of an RFC-documented, IESG-approved, - Experimental protocol extension. - -9. References - - [1] American National Standards Institute (formerly United States of - America Standards Institute), X3.4, 1968, "USA Code for - Information Interchange". ANSI X3.4-1968 has been replaced by - newer versions with slight modifications, but the 1968 version - remains definitive for the Internet. - - [2] Braden, R., "Requirements for Internet hosts - application and - support", STD 3, RFC 1123, October 1989. - - [3] Butler, M., Chase, D., Goldberger, J., Postel, J. and J. - Reynolds, "Post Office Protocol - version 2", RFC 937, February - 1985. - - [4] Callas, J., Donnerhacke, L., Finney, H. and R. Thayer, "OpenPGP - Message Format", RFC 2440, November 1998. - - [5] Crispin, M., "Interactive Mail Access Protocol - Version 2", RFC - 1176, August 1990. - - [6] Crispin, M., "Internet Message Access Protocol - Version 4", RFC - 2060, December 1996. - - [7] Crocker, D., "Standard for the Format of ARPA Internet Text - Messages", RFC 822, August 1982. - - [8] Crocker, D. and P. Overell, Eds., "Augmented BNF for Syntax - Specifications: ABNF", RFC 2234, November 1997. - - [9] De Winter, J., "SMTP Service Extension for Remote Message Queue - Starting", RFC 1985, August 1996. - - [10] Fajman, R., "An Extensible Message Format for Message - Disposition Notifications", RFC 2298, March 1998. - - [11] Freed, N, "Behavior of and Requirements for Internet Firewalls", - RFC 2979, October 2000. - - [12] Freed, N. and N. Borenstein, "Multipurpose Internet Mail - Extensions (MIME) Part One: Format of Internet Message Bodies", - RFC 2045, December 1996. - - - - -Klensin Standards Track [Page 68] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - [13] Freed, N., "SMTP Service Extension for Command Pipelining", RFC - 2920, September 2000. - - [14] Galvin, J., Murphy, S., Crocker, S. and N. Freed, "Security - Multiparts for MIME: Multipart/Signed and Multipart/Encrypted", - RFC 1847, October 1995. - - [15] Gellens, R. and J. Klensin, "Message Submission", RFC 2476, - December 1998. - - [16] Kille, S., "Mapping between X.400 and RFC822/MIME", RFC 2156, - January 1998. - - [17] Hinden, R and S. Deering, Eds. "IP Version 6 Addressing - Architecture", RFC 2373, July 1998. - - [18] Klensin, J., Freed, N. and K. Moore, "SMTP Service Extension for - Message Size Declaration", STD 10, RFC 1870, November 1995. - - [19] Klensin, J., Freed, N., Rose, M., Stefferud, E. and D. Crocker, - "SMTP Service Extensions", STD 10, RFC 1869, November 1995. - - [20] Klensin, J., Freed, N., Rose, M., Stefferud, E. and D. Crocker, - "SMTP Service Extension for 8bit-MIMEtransport", RFC 1652, July - 1994. - - [21] Lambert, M., "PCMAIL: A distributed mail system for personal - computers", RFC 1056, July 1988. - - [22] Mockapetris, P., "Domain names - implementation and - specification", STD 13, RFC 1035, November 1987. - - Mockapetris, P., "Domain names - concepts and facilities", STD - 13, RFC 1034, November 1987. - - [23] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part - Three: Message Header Extensions for Non-ASCII Text", RFC 2047, - December 1996. - - [24] Moore, K., "SMTP Service Extension for Delivery Status - Notifications", RFC 1891, January 1996. - - [25] Moore, K., and G. Vaudreuil, "An Extensible Message Format for - Delivery Status Notifications", RFC 1894, January 1996. - - [26] Myers, J. and M. Rose, "Post Office Protocol - Version 3", STD - 53, RFC 1939, May 1996. - - - - -Klensin Standards Track [Page 69] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - [27] Partridge, C., "Mail routing and the domain system", RFC 974, - January 1986. - - [28] Partridge, C., "Duplicate messages and SMTP", RFC 1047, February - 1988. - - [29] Postel, J., ed., "Transmission Control Protocol - DARPA Internet - Program Protocol Specification", STD 7, RFC 793, September 1981. - - [30] Postel, J., "Simple Mail Transfer Protocol", RFC 821, August - 1982. - - [31] Ramsdell, B., Ed., "S/MIME Version 3 Message Specification", RFC - 2633, June 1999. - - [32] Resnick, P., Ed., "Internet Message Format", RFC 2822, April - 2001. - - [33] Vaudreuil, G., "SMTP Service Extensions for Transmission of - Large and Binary MIME Messages", RFC 1830, August 1995. - - [34] Vaudreuil, G., "Enhanced Mail System Status Codes", RFC 1893, - January 1996. - -10. Editor's Address - - John C. Klensin - AT&T Laboratories - 99 Bedford St - Boston, MA 02111 USA - - Phone: 617-574-3076 - EMail: klensin@research.att.com - -11. Acknowledgments - - Many people worked long and hard on the many iterations of this - document. There was wide-ranging debate in the IETF DRUMS Working - Group, both on its mailing list and in face to face discussions, - about many technical issues and the role of a revised standard for - Internet mail transport, and many contributors helped form the - wording in this specification. The hundreds of participants in the - many discussions since RFC 821 was produced are too numerous to - mention, but they all helped this document become what it is. - - - - - - - -Klensin Standards Track [Page 70] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -APPENDICES - -A. TCP Transport Service - - The TCP connection supports the transmission of 8-bit bytes. The - SMTP data is 7-bit ASCII characters. Each character is transmitted - as an 8-bit byte with the high-order bit cleared to zero. Service - extensions may modify this rule to permit transmission of full 8-bit - data bytes as part of the message body, but not in SMTP commands or - responses. - -B. Generating SMTP Commands from RFC 822 Headers - - Some systems use RFC 822 headers (only) in a mail submission - protocol, or otherwise generate SMTP commands from RFC 822 headers - when such a message is handed to an MTA from a UA. While the MTA-UA - protocol is a private matter, not covered by any Internet Standard, - there are problems with this approach. For example, there have been - repeated problems with proper handling of "bcc" copies and - redistribution lists when information that conceptually belongs to a - mail envelopes is not separated early in processing from header - information (and kept separate). - - It is recommended that the UA provide its initial ("submission - client") MTA with an envelope separate from the message itself. - However, if the envelope is not supplied, SMTP commands SHOULD be - generated as follows: - - 1. Each recipient address from a TO, CC, or BCC header field SHOULD - be copied to a RCPT command (generating multiple message copies if - that is required for queuing or delivery). This includes any - addresses listed in a RFC 822 "group". Any BCC fields SHOULD then - be removed from the headers. Once this process is completed, the - remaining headers SHOULD be checked to verify that at least one - To:, Cc:, or Bcc: header remains. If none do, then a bcc: header - with no additional information SHOULD be inserted as specified in - [32]. - - 2. The return address in the MAIL command SHOULD, if possible, be - derived from the system's identity for the submitting (local) - user, and the "From:" header field otherwise. If there is a - system identity available, it SHOULD also be copied to the Sender - header field if it is different from the address in the From - header field. (Any Sender field that was already there SHOULD be - removed.) Systems may provide a way for submitters to override - the envelope return address, but may want to restrict its use to - privileged users. This will not prevent mail forgery, but may - lessen its incidence; see section 7.1. - - - -Klensin Standards Track [Page 71] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - When an MTA is being used in this way, it bears responsibility for - ensuring that the message being transmitted is valid. The mechanisms - for checking that validity, and for handling (or returning) messages - that are not valid at the time of arrival, are part of the MUA-MTA - interface and not covered by this specification. - - A submission protocol based on Standard RFC 822 information alone - MUST NOT be used to gateway a message from a foreign (non-SMTP) mail - system into an SMTP environment. Additional information to construct - an envelope must come from some source in the other environment, - whether supplemental headers or the foreign system's envelope. - - Attempts to gateway messages using only their header "to" and "cc" - fields have repeatedly caused mail loops and other behavior adverse - to the proper functioning of the Internet mail environment. These - problems have been especially common when the message originates from - an Internet mailing list and is distributed into the foreign - environment using envelope information. When these messages are then - processed by a header-only remailer, loops back to the Internet - environment (and the mailing list) are almost inevitable. - -C. Source Routes - - Historically, the was a reverse source routing list of - hosts and a source mailbox. The first host in the - SHOULD be the host sending the MAIL command. Similarly, the - may be a source routing lists of hosts and a - destination mailbox. However, in general, the SHOULD - contain only a mailbox and domain name, relying on the domain name - system to supply routing information if required. The use of source - routes is deprecated; while servers MUST be prepared to receive and - handle them as discussed in section 3.3 and F.2, clients SHOULD NOT - transmit them and this section was included only to provide context. - - For relay purposes, the forward-path may be a source route of the - form "@ONE,@TWO:JOE@THREE", where ONE, TWO, and THREE MUST BE fully- - qualified domain names. This form is used to emphasize the - distinction between an address and a route. The mailbox is an - absolute address, and the route is information about how to get - there. The two concepts should not be confused. - - If source routes are used, RFC 821 and the text below should be - consulted for the mechanisms for constructing and updating the - forward- and reverse-paths. - - - - - - - -Klensin Standards Track [Page 72] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - The SMTP server transforms the command arguments by moving its own - identifier (its domain name or that of any domain for which it is - acting as a mail exchanger), if it appears, from the forward-path to - the beginning of the reverse-path. - - Notice that the forward-path and reverse-path appear in the SMTP - commands and replies, but not necessarily in the message. That is, - there is no need for these paths and especially this syntax to appear - in the "To:" , "From:", "CC:", etc. fields of the message header. - Conversely, SMTP servers MUST NOT derive final message delivery - information from message header fields. - - When the list of hosts is present, it is a "reverse" source route and - indicates that the mail was relayed through each host on the list - (the first host in the list was the most recent relay). This list is - used as a source route to return non-delivery notices to the sender. - As each relay host adds itself to the beginning of the list, it MUST - use its name as known in the transport environment to which it is - relaying the mail rather than that of the transport environment from - which the mail came (if they are different). - -D. Scenarios - - This section presents complete scenarios of several types of SMTP - sessions. In the examples, "C:" indicates what is said by the SMTP - client, and "S:" indicates what is said by the SMTP server. - -D.1 A Typical SMTP Transaction Scenario - - This SMTP example shows mail sent by Smith at host bar.com, to Jones, - Green, and Brown at host foo.com. Here we assume that host bar.com - contacts host foo.com directly. The mail is accepted for Jones and - Brown. Green does not have a mailbox at host foo.com. - - S: 220 foo.com Simple Mail Transfer Service Ready - C: EHLO bar.com - S: 250-foo.com greets bar.com - S: 250-8BITMIME - S: 250-SIZE - S: 250-DSN - S: 250 HELP - C: MAIL FROM: - S: 250 OK - C: RCPT TO: - S: 250 OK - C: RCPT TO: - S: 550 No such user here - C: RCPT TO: - - - -Klensin Standards Track [Page 73] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - S: 250 OK - C: DATA - S: 354 Start mail input; end with . - C: Blah blah blah... - C: ...etc. etc. etc. - C: . - S: 250 OK - C: QUIT - S: 221 foo.com Service closing transmission channel - -D.2 Aborted SMTP Transaction Scenario - - S: 220 foo.com Simple Mail Transfer Service Ready - C: EHLO bar.com - S: 250-foo.com greets bar.com - S: 250-8BITMIME - S: 250-SIZE - S: 250-DSN - S: 250 HELP - C: MAIL FROM: - S: 250 OK - C: RCPT TO: - S: 250 OK - C: RCPT TO: - S: 550 No such user here - C: RSET - S: 250 OK - C: QUIT - S: 221 foo.com Service closing transmission channel - -D.3 Relayed Mail Scenario - - Step 1 -- Source Host to Relay Host - - S: 220 foo.com Simple Mail Transfer Service Ready - C: EHLO bar.com - S: 250-foo.com greets bar.com - S: 250-8BITMIME - S: 250-SIZE - S: 250-DSN - S: 250 HELP - C: MAIL FROM: - S: 250 OK - C: RCPT TO:<@foo.com:Jones@XYZ.COM> - S: 250 OK - C: DATA - S: 354 Start mail input; end with . - C: Date: Thu, 21 May 1998 05:33:29 -0700 - - - -Klensin Standards Track [Page 74] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - C: From: John Q. Public - C: Subject: The Next Meeting of the Board - C: To: Jones@xyz.com - C: - C: Bill: - C: The next meeting of the board of directors will be - C: on Tuesday. - C: John. - C: . - S: 250 OK - C: QUIT - S: 221 foo.com Service closing transmission channel - - Step 2 -- Relay Host to Destination Host - - S: 220 xyz.com Simple Mail Transfer Service Ready - C: EHLO foo.com - S: 250 xyz.com is on the air - C: MAIL FROM:<@foo.com:JQP@bar.com> - S: 250 OK - C: RCPT TO: - S: 250 OK - C: DATA - S: 354 Start mail input; end with . - C: Received: from bar.com by foo.com ; Thu, 21 May 1998 - C: 05:33:29 -0700 - C: Date: Thu, 21 May 1998 05:33:22 -0700 - C: From: John Q. Public - C: Subject: The Next Meeting of the Board - C: To: Jones@xyz.com - C: - C: Bill: - C: The next meeting of the board of directors will be - C: on Tuesday. - C: John. - C: . - S: 250 OK - C: QUIT - S: 221 foo.com Service closing transmission channel - -D.4 Verifying and Sending Scenario - - S: 220 foo.com Simple Mail Transfer Service Ready - C: EHLO bar.com - S: 250-foo.com greets bar.com - S: 250-8BITMIME - S: 250-SIZE - S: 250-DSN - - - -Klensin Standards Track [Page 75] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - - S: 250-VRFY - S: 250 HELP - C: VRFY Crispin - S: 250 Mark Crispin - C: SEND FROM: - S: 250 OK - C: RCPT TO: - S: 250 OK - C: DATA - S: 354 Start mail input; end with . - C: Blah blah blah... - C: ...etc. etc. etc. - C: . - S: 250 OK - C: QUIT - S: 221 foo.com Service closing transmission channel - -E. Other Gateway Issues - - In general, gateways between the Internet and other mail systems - SHOULD attempt to preserve any layering semantics across the - boundaries between the two mail systems involved. Gateway- - translation approaches that attempt to take shortcuts by mapping, - (such as envelope information from one system to the message headers - or body of another) have generally proven to be inadequate in - important ways. Systems translating between environments that do not - support both envelopes and headers and Internet mail must be written - with the understanding that some information loss is almost - inevitable. - -F. Deprecated Features of RFC 821 - - A few features of RFC 821 have proven to be problematic and SHOULD - NOT be used in Internet mail. - -F.1 TURN - - This command, described in RFC 821, raises important security issues - since, in the absence of strong authentication of the host requesting - that the client and server switch roles, it can easily be used to - divert mail from its correct destination. Its use is deprecated; - SMTP systems SHOULD NOT use it unless the server can authenticate the - client. - - - - - - - - -Klensin Standards Track [Page 76] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -F.2 Source Routing - - RFC 821 utilized the concept of explicit source routing to get mail - from one host to another via a series of relays. The requirement to - utilize source routes in regular mail traffic was eliminated by the - introduction of the domain name system "MX" record and the last - significant justification for them was eliminated by the - introduction, in RFC 1123, of a clear requirement that addresses - following an "@" must all be fully-qualified domain names. - Consequently, the only remaining justifications for the use of source - routes are support for very old SMTP clients or MUAs and in mail - system debugging. They can, however, still be useful in the latter - circumstance and for routing mail around serious, but temporary, - problems such as problems with the relevant DNS records. - - SMTP servers MUST continue to accept source route syntax as specified - in the main body of this document and in RFC 1123. They MAY, if - necessary, ignore the routes and utilize only the target domain in - the address. If they do utilize the source route, the message MUST - be sent to the first domain shown in the address. In particular, a - server MUST NOT guess at shortcuts within the source route. - - Clients SHOULD NOT utilize explicit source routing except under - unusual circumstances, such as debugging or potentially relaying - around firewall or mail system configuration errors. - -F.3 HELO - - As discussed in sections 3.1 and 4.1.1, EHLO is strongly preferred to - HELO when the server will accept the former. Servers must continue - to accept and process HELO in order to support older clients. - -F.4 #-literals - - RFC 821 provided for specifying an Internet address as a decimal - integer host number prefixed by a pound sign, "#". In practice, that - form has been obsolete since the introduction of TCP/IP. It is - deprecated and MUST NOT be used. - -F.5 Dates and Years - - When dates are inserted into messages by SMTP clients or servers - (e.g., in trace fields), four-digit years MUST BE used. Two-digit - years are deprecated; three-digit years were never permitted in the - Internet mail system. - - - - - - -Klensin Standards Track [Page 77] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -F.6 Sending versus Mailing - - In addition to specifying a mechanism for delivering messages to - user's mailboxes, RFC 821 provided additional, optional, commands to - deliver messages directly to the user's terminal screen. These - commands (SEND, SAML, SOML) were rarely implemented, and changes in - workstation technology and the introduction of other protocols may - have rendered them obsolete even where they are implemented. - - Clients SHOULD NOT provide SEND, SAML, or SOML as services. Servers - MAY implement them. If they are implemented by servers, the - implementation model specified in RFC 821 MUST be used and the - command names MUST be published in the response to the EHLO command. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Klensin Standards Track [Page 78] - -RFC 2821 Simple Mail Transfer Protocol April 2001 - - -Full Copyright Statement - - Copyright (C) The Internet Society (2001). All Rights Reserved. - - This document and translations of it may be copied and furnished to - others, and derivative works that comment on or otherwise explain it - or assist in its implementation may be prepared, copied, published - and distributed, in whole or in part, without restriction of any - kind, provided that the above copyright notice and this paragraph are - included on all such copies and derivative works. However, this - document itself may not be modified in any way, such as by removing - the copyright notice or references to the Internet Society or other - Internet organizations, except as needed for the purpose of - developing Internet standards in which case the procedures for - copyrights defined in the Internet Standards process must be - followed, or as required to translate it into languages other than - English. - - The limited permissions granted above are perpetual and will not be - revoked by the Internet Society or its successors or assigns. - - This document and the information contained herein is provided on an - "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING - TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING - BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION - HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF - MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. - -Acknowledgement - - Funding for the RFC Editor function is currently provided by the - Internet Society. - - - - - - - - - - - - - - - - - - - -Klensin Standards Track [Page 79] - diff --git a/docs/rfcs/rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt b/docs/rfcs/rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt new file mode 100644 index 0000000..c1a54c4 --- /dev/null +++ b/docs/rfcs/rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt @@ -0,0 +1,1515 @@ + + + + + + +Network Working Group P. Leach +Request for Comments: 2831 Microsoft +Category: Standards Track C. Newman + Innosoft + May 2000 + + + Using Digest Authentication as a SASL Mechanism + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2000). All Rights Reserved. + +Abstract + + This specification defines how HTTP Digest Authentication [Digest] + can be used as a SASL [RFC 2222] mechanism for any protocol that has + a SASL profile. It is intended both as an improvement over CRAM-MD5 + [RFC 2195] and as a convenient way to support a single authentication + mechanism for web, mail, LDAP, and other protocols. + +Table of Contents + + 1 INTRODUCTION.....................................................2 + 1.1 CONVENTIONS AND NOTATION......................................2 + 1.2 REQUIREMENTS..................................................3 + 2 AUTHENTICATION...................................................3 + 2.1 INITIAL AUTHENTICATION........................................3 + 2.1.1 Step One...................................................3 + 2.1.2 Step Two...................................................6 + 2.1.3 Step Three................................................12 + 2.2 SUBSEQUENT AUTHENTICATION....................................12 + 2.2.1 Step one..................................................13 + 2.2.2 Step Two..................................................13 + 2.3 INTEGRITY PROTECTION.........................................13 + 2.4 CONFIDENTIALITY PROTECTION...................................14 + 3 SECURITY CONSIDERATIONS.........................................15 + 3.1 AUTHENTICATION OF CLIENTS USING DIGEST AUTHENTICATION........15 + 3.2 COMPARISON OF DIGEST WITH PLAINTEXT PASSWORDS................16 + 3.3 REPLAY ATTACKS...............................................16 + + + +Leach & Newman Standards Track [Page 1] + +RFC 2831 Digest SASL Mechanism May 2000 + + + 3.4 ONLINE DICTIONARY ATTACKS....................................16 + 3.5 OFFLINE DICTIONARY ATTACKS...................................16 + 3.6 MAN IN THE MIDDLE............................................17 + 3.7 CHOSEN PLAINTEXT ATTACKS.....................................17 + 3.8 SPOOFING BY COUNTERFEIT SERVERS..............................17 + 3.9 STORING PASSWORDS............................................17 + 3.10 MULTIPLE REALMS.............................................18 + 3.11 SUMMARY.....................................................18 + 4 EXAMPLE.........................................................18 + 5 REFERENCES......................................................20 + 6 AUTHORS' ADDRESSES..............................................21 + 7 ABNF............................................................21 + 7.1 AUGMENTED BNF................................................21 + 7.2 BASIC RULES..................................................23 + 8 SAMPLE CODE.....................................................25 + 9 FULL COPYRIGHT STATEMENT........................................27 + +1 Introduction + + This specification describes the use of HTTP Digest Access + Authentication as a SASL mechanism. The authentication type + associated with the Digest SASL mechanism is "DIGEST-MD5". + + This specification is intended to be upward compatible with the + "md5-sess" algorithm of HTTP/1.1 Digest Access Authentication + specified in [Digest]. The only difference in the "md5-sess" + algorithm is that some directives not needed in a SASL mechanism have + had their values defaulted. + + There is one new feature for use as a SASL mechanism: integrity + protection on application protocol messages after an authentication + exchange. + + Also, compared to CRAM-MD5, DIGEST-MD5 prevents chosen plaintext + attacks, and permits the use of third party authentication servers, + mutual authentication, and optimized reauthentication if a client has + recently authenticated to a server. + +1.1 Conventions and Notation + + This specification uses the same ABNF notation and lexical + conventions as HTTP/1.1 specification; see appendix A. + + Let { a, b, ... } be the concatenation of the octet strings a, b, ... + + Let H(s) be the 16 octet MD5 hash [RFC 1321] of the octet string s. + + + + + +Leach & Newman Standards Track [Page 2] + +RFC 2831 Digest SASL Mechanism May 2000 + + + Let KD(k, s) be H({k, ":", s}), i.e., the 16 octet hash of the string + k, a colon and the string s. + + Let HEX(n) be the representation of the 16 octet MD5 hash n as a + string of 32 hex digits (with alphabetic characters always in lower + case, since MD5 is case sensitive). + + Let HMAC(k, s) be the 16 octet HMAC-MD5 [RFC 2104] of the octet + string s using the octet string k as a key. + + The value of a quoted string constant as an octet string does not + include any terminating null character. + +1.2 Requirements + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC 2119]. + + An implementation is not compliant if it fails to satisfy one or more + of the MUST level requirements for the protocols it implements. An + implementation that satisfies all the MUST level and all the SHOULD + level requirements for its protocols is said to be "unconditionally + compliant"; one that satisfies all the MUST level requirements but + not all the SHOULD level requirements for its protocols is said to be + "conditionally compliant." + +2 Authentication + + The following sections describe how to use Digest as a SASL + authentication mechanism. + +2.1 Initial Authentication + + If the client has not recently authenticated to the server, then it + must perform "initial authentication", as defined in this section. If + it has recently authenticated, then a more efficient form is + available, defined in the next section. + +2.1.1 Step One + + The server starts by sending a challenge. The data encoded in the + challenge contains a string formatted according to the rules for a + "digest-challenge" defined as follows: + + + + + + + +Leach & Newman Standards Track [Page 3] + +RFC 2831 Digest SASL Mechanism May 2000 + + + digest-challenge = + 1#( realm | nonce | qop-options | stale | maxbuf | charset + algorithm | cipher-opts | auth-param ) + + realm = "realm" "=" <"> realm-value <"> + realm-value = qdstr-val + nonce = "nonce" "=" <"> nonce-value <"> + nonce-value = qdstr-val + qop-options = "qop" "=" <"> qop-list <"> + qop-list = 1#qop-value + qop-value = "auth" | "auth-int" | "auth-conf" | + token + stale = "stale" "=" "true" + maxbuf = "maxbuf" "=" maxbuf-value + maxbuf-value = 1*DIGIT + charset = "charset" "=" "utf-8" + algorithm = "algorithm" "=" "md5-sess" + cipher-opts = "cipher" "=" <"> 1#cipher-value <"> + cipher-value = "3des" | "des" | "rc4-40" | "rc4" | + "rc4-56" | token + auth-param = token "=" ( token | quoted-string ) + + The meanings of the values of the directives used above are as + follows: + + realm + Mechanistically, a string which can enable users to know which + username and password to use, in case they might have different + ones for different servers. Conceptually, it is the name of a + collection of accounts that might include the user's account. This + string should contain at least the name of the host performing the + authentication and might additionally indicate the collection of + users who might have access. An example might be + "registered_users@gotham.news.example.com". This directive is + optional; if not present, the client SHOULD solicit it from the + user or be able to compute a default; a plausible default might be + the realm supplied by the user when they logged in to the client + system. Multiple realm directives are allowed, in which case the + user or client must choose one as the realm for which to supply to + username and password. + + nonce + A server-specified data string which MUST be different each time a + digest-challenge is sent as part of initial authentication. It is + recommended that this string be base64 or hexadecimal data. Note + that since the string is passed as a quoted string, the + double-quote character is not allowed unless escaped (see section + 7.2). The contents of the nonce are implementation dependent. The + + + +Leach & Newman Standards Track [Page 4] + +RFC 2831 Digest SASL Mechanism May 2000 + + + security of the implementation depends on a good choice. It is + RECOMMENDED that it contain at least 64 bits of entropy. The nonce + is opaque to the client. This directive is required and MUST + appear exactly once; if not present, or if multiple instances are + present, the client should abort the authentication exchange. + + qop-options + A quoted string of one or more tokens indicating the "quality of + protection" values supported by the server. The value "auth" + indicates authentication; the value "auth-int" indicates + authentication with integrity protection; the value "auth-conf" + indicates authentication with integrity protection and encryption. + This directive is optional; if not present it defaults to "auth". + The client MUST ignore unrecognized options; if the client + recognizes no option, it should abort the authentication exchange. + + stale + The "stale" directive is not used in initial authentication. See + the next section for its use in subsequent authentications. This + directive may appear at most once; if multiple instances are + present, the client should abort the authentication exchange. + + maxbuf + A number indicating the size of the largest buffer the server is + able to receive when using "auth-int" or "auth-conf". If this + directive is missing, the default value is 65536. This directive + may appear at most once; if multiple instances are present, the + client should abort the authentication exchange. + + charset + This directive, if present, specifies that the server supports + UTF-8 encoding for the username and password. If not present, the + username and password must be encoded in ISO 8859-1 (of which + US-ASCII is a subset). The directive is needed for backwards + compatibility with HTTP Digest, which only supports ISO 8859-1. + This directive may appear at most once; if multiple instances are + present, the client should abort the authentication exchange. + + algorithm + This directive is required for backwards compatibility with HTTP + Digest., which supports other algorithms. . This directive is + required and MUST appear exactly once; if not present, or if + multiple instances are present, the client should abort the + authentication exchange. + + + + + + + +Leach & Newman Standards Track [Page 5] + +RFC 2831 Digest SASL Mechanism May 2000 + + + cipher-opts + A list of ciphers that the server supports. This directive must be + present exactly once if "auth-conf" is offered in the + "qop-options" directive, in which case the "3des" and "des" modes + are mandatory-to-implement. The client MUST ignore unrecognized + options; if the client recognizes no option, it should abort the + authentication exchange. + + des + the Data Encryption Standard (DES) cipher [FIPS] in cipher + block chaining (CBC) mode with a 56 bit key. + + 3des + the "triple DES" cipher in CBC mode with EDE with the same key + for each E stage (aka "two keys mode") for a total key length + of 112 bits. + + rc4, rc4-40, rc4-56 + the RC4 cipher with a 128 bit, 40 bit, and 56 bit key, + respectively. + + auth-param This construct allows for future extensions; it may appear + more than once. The client MUST ignore any unrecognized + directives. + + For use as a SASL mechanism, note that the following changes are made + to "digest-challenge" from HTTP: the following Digest options (called + "directives" in HTTP terminology) are unused (i.e., MUST NOT be sent, + and MUST be ignored if received): + + opaque + domain + + The size of a digest-challenge MUST be less than 2048 bytes. + +2.1.2 Step Two + + The client makes note of the "digest-challenge" and then responds + with a string formatted and computed according to the rules for a + "digest-response" defined as follows: + + + + + + + + + + + +Leach & Newman Standards Track [Page 6] + +RFC 2831 Digest SASL Mechanism May 2000 + + + digest-response = 1#( username | realm | nonce | cnonce | + nonce-count | qop | digest-uri | response | + maxbuf | charset | cipher | authzid | + auth-param ) + + username = "username" "=" <"> username-value <"> + username-value = qdstr-val + cnonce = "cnonce" "=" <"> cnonce-value <"> + cnonce-value = qdstr-val + nonce-count = "nc" "=" nc-value + nc-value = 8LHEX + qop = "qop" "=" qop-value + digest-uri = "digest-uri" "=" <"> digest-uri-value <"> + digest-uri-value = serv-type "/" host [ "/" serv-name ] + serv-type = 1*ALPHA + host = 1*( ALPHA | DIGIT | "-" | "." ) + serv-name = host + response = "response" "=" response-value + response-value = 32LHEX + LHEX = "0" | "1" | "2" | "3" | + "4" | "5" | "6" | "7" | + "8" | "9" | "a" | "b" | + "c" | "d" | "e" | "f" + cipher = "cipher" "=" cipher-value + authzid = "authzid" "=" <"> authzid-value <"> + authzid-value = qdstr-val + + + username + The user's name in the specified realm, encoded according to the + value of the "charset" directive. This directive is required and + MUST be present exactly once; otherwise, authentication fails. + + realm + The realm containing the user's account. This directive is + required if the server provided any realms in the + "digest-challenge", in which case it may appear exactly once and + its value SHOULD be one of those realms. If the directive is + missing, "realm-value" will set to the empty string when computing + A1 (see below for details). + + nonce + The server-specified data string received in the preceding + digest-challenge. This directive is required and MUST be present + exactly once; otherwise, authentication fails. + + + + + + +Leach & Newman Standards Track [Page 7] + +RFC 2831 Digest SASL Mechanism May 2000 + + + cnonce + A client-specified data string which MUST be different each time a + digest-response is sent as part of initial authentication. The + cnonce-value is an opaque quoted string value provided by the + client and used by both client and server to avoid chosen + plaintext attacks, and to provide mutual authentication. The + security of the implementation depends on a good choice. It is + RECOMMENDED that it contain at least 64 bits of entropy. This + directive is required and MUST be present exactly once; otherwise, + authentication fails. + + nonce-count + The nc-value is the hexadecimal count of the number of requests + (including the current request) that the client has sent with the + nonce value in this request. For example, in the first request + sent in response to a given nonce value, the client sends + "nc=00000001". The purpose of this directive is to allow the + server to detect request replays by maintaining its own copy of + this count - if the same nc-value is seen twice, then the request + is a replay. See the description below of the construction of + the response value. This directive may appear at most once; if + multiple instances are present, the client should abort the + authentication exchange. + + qop + Indicates what "quality of protection" the client accepted. If + present, it may appear exactly once and its value MUST be one of + the alternatives in qop-options. If not present, it defaults to + "auth". These values affect the computation of the response. Note + that this is a single token, not a quoted list of alternatives. + + serv-type + Indicates the type of service, such as "www" for web service, + "ftp" for FTP service, "smtp" for mail delivery service, etc. The + service name as defined in the SASL profile for the protocol see + section 4 of [RFC 2222], registered in the IANA registry of + "service" elements for the GSSAPI host-based service name form + [RFC 2078]. + + host + The DNS host name or IP address for the service requested. The + DNS host name must be the fully-qualified canonical name of the + host. The DNS host name is the preferred form; see notes on server + processing of the digest-uri. + + + + + + + +Leach & Newman Standards Track [Page 8] + +RFC 2831 Digest SASL Mechanism May 2000 + + + serv-name + Indicates the name of the service if it is replicated. The service + is considered to be replicated if the client's service-location + process involves resolution using standard DNS lookup operations, + and if these operations involve DNS records (such as SRV, or MX) + which resolve one DNS name into a set of other DNS names. In this + case, the initial name used by the client is the "serv-name", and + the final name is the "host" component. For example, the incoming + mail service for "example.com" may be replicated through the use + of MX records stored in the DNS, one of which points at an SMTP + server called "mail3.example.com"; it's "serv-name" would be + "example.com", it's "host" would be "mail3.example.com". If the + service is not replicated, or the serv-name is identical to the + host, then the serv-name component MUST be omitted. + + digest-uri + Indicates the principal name of the service with which the client + wishes to connect, formed from the serv-type, host, and serv-name. + For example, the FTP service on "ftp.example.com" would have a + "digest-uri" value of "ftp/ftp.example.com"; the SMTP server from + the example above would have a "digest-uri" value of + "smtp/mail3.example.com/example.com". + + Servers SHOULD check that the supplied value is correct. This will + detect accidental connection to the incorrect server. It is also so + that clients will be trained to provide values that will work with + implementations that use a shared back-end authentication service + that can provide server authentication. + + The serv-type component should match the service being offered. The + host component should match one of the host names of the host on + which the service is running, or it's IP address. Servers SHOULD NOT + normally support the IP address form, because server authentication + by IP address is not very useful; they should only do so if the DNS + is unavailable or unreliable. The serv-name component should match + one of the service's configured service names. + + This directive may appear at most once; if multiple instances are + present, the client should abort the authentication exchange. + + Note: In the HTTP use of Digest authentication, the digest-uri is the + URI (usually a URL) of the resource requested -- hence the name of + the directive. + + response + A string of 32 hex digits computed as defined below, which proves + that the user knows a password. This directive is required and + MUST be present exactly once; otherwise, authentication fails. + + + +Leach & Newman Standards Track [Page 9] + +RFC 2831 Digest SASL Mechanism May 2000 + + + maxbuf + A number indicating the size of the largest buffer the client is + able to receive. If this directive is missing, the default value + is 65536. This directive may appear at most once; if multiple + instances are present, the server should abort the authentication + exchange. + + charset + This directive, if present, specifies that the client has used + UTF-8 encoding for the username and password. If not present, the + username and password must be encoded in ISO 8859-1 (of which + US-ASCII is a subset). The client should send this directive only + if the server has indicated it supports UTF-8. The directive is + needed for backwards compatibility with HTTP Digest, which only + supports ISO 8859-1. + + LHEX + 32 hex digits, where the alphabetic characters MUST be lower case, + because MD5 is not case insensitive. + + cipher + The cipher chosen by the client. This directive MUST appear + exactly once if "auth-conf" is negotiated; if required and not + present, authentication fails. + + authzid + The "authorization ID" as per RFC 2222, encoded in UTF-8. This + directive is optional. If present, and the authenticating user has + sufficient privilege, and the server supports it, then after + authentication the server will use this identity for making all + accesses and access checks. If the client specifies it, and the + server does not support it, then the response-value will be + incorrect, and authentication will fail. + + The size of a digest-response MUST be less than 4096 bytes. + +2.1.2.1 Response-value + + The definition of "response-value" above indicates the encoding for + its value -- 32 lower case hex characters. The following definitions + show how the value is computed. + + Although qop-value and components of digest-uri-value may be + case-insensitive, the case which the client supplies in step two is + preserved for the purpose of computing and verifying the + response-value. + + response-value = + + + +Leach & Newman Standards Track [Page 10] + +RFC 2831 Digest SASL Mechanism May 2000 + + + HEX( KD ( HEX(H(A1)), + { nonce-value, ":" nc-value, ":", + cnonce-value, ":", qop-value, ":", HEX(H(A2)) })) + + If authzid is specified, then A1 is + + + A1 = { H( { username-value, ":", realm-value, ":", passwd } ), + ":", nonce-value, ":", cnonce-value, ":", authzid-value } + + If authzid is not specified, then A1 is + + + A1 = { H( { username-value, ":", realm-value, ":", passwd } ), + ":", nonce-value, ":", cnonce-value } + + where + + passwd = *OCTET + + The "username-value", "realm-value" and "passwd" are encoded + according to the value of the "charset" directive. If "charset=UTF-8" + is present, and all the characters of either "username-value" or + "passwd" are in the ISO 8859-1 character set, then it must be + converted to ISO 8859-1 before being hashed. This is so that + authentication databases that store the hashed username, realm and + password (which is common) can be shared compatibly with HTTP, which + specifies ISO 8859-1. A sample implementation of this conversion is + in section 8. + + If the "qop" directive's value is "auth", then A2 is: + + A2 = { "AUTHENTICATE:", digest-uri-value } + + If the "qop" value is "auth-int" or "auth-conf" then A2 is: + + A2 = { "AUTHENTICATE:", digest-uri-value, + ":00000000000000000000000000000000" } + + Note that "AUTHENTICATE:" must be in upper case, and the second + string constant is a string with a colon followed by 32 zeros. + + These apparently strange values of A2 are for compatibility with + HTTP; they were arrived at by setting "Method" to "AUTHENTICATE" and + the hash of the entity body to zero in the HTTP digest calculation of + A2. + + Also, in the HTTP usage of Digest, several directives in the + + + +Leach & Newman Standards Track [Page 11] + +RFC 2831 Digest SASL Mechanism May 2000 + + + "digest-challenge" sent by the server have to be returned by the + client in the "digest-response". These are: + + opaque + algorithm + + These directives are not needed when Digest is used as a SASL + mechanism (i.e., MUST NOT be sent, and MUST be ignored if received). + +2.1.3 Step Three + + The server receives and validates the "digest-response". The server + checks that the nonce-count is "00000001". If it supports subsequent + authentication (see section 2.2), it saves the value of the nonce and + the nonce-count. It sends a message formatted as follows: + + response-auth = "rspauth" "=" response-value + + where response-value is calculated as above, using the values sent in + step two, except that if qop is "auth", then A2 is + + A2 = { ":", digest-uri-value } + + And if qop is "auth-int" or "auth-conf" then A2 is + + A2 = { ":", digest-uri-value, ":00000000000000000000000000000000" } + + Compared to its use in HTTP, the following Digest directives in the + "digest-response" are unused: + + nextnonce + qop + cnonce + nonce-count + +2.2 Subsequent Authentication + + If the client has previously authenticated to the server, and + remembers the values of username, realm, nonce, nonce-count, cnonce, + and qop that it used in that authentication, and the SASL profile for + a protocol permits an initial client response, then it MAY perform + "subsequent authentication", as defined in this section. + + + + + + + + + +Leach & Newman Standards Track [Page 12] + +RFC 2831 Digest SASL Mechanism May 2000 + + +2.2.1 Step one + + The client uses the values from the previous authentication and sends + an initial response with a string formatted and computed according to + the rules for a "digest-response", as defined above, but with a + nonce-count one greater than used in the last "digest-response". + +2.2.2 Step Two + + The server receives the "digest-response". If the server does not + support subsequent authentication, then it sends a + "digest-challenge", and authentication proceeds as in initial + authentication. If the server has no saved nonce and nonce-count from + a previous authentication, then it sends a "digest-challenge", and + authentication proceeds as in initial authentication. Otherwise, the + server validates the "digest-response", checks that the nonce-count + is one greater than that used in the previous authentication using + that nonce, and saves the new value of nonce-count. + + If the response is invalid, then the server sends a + "digest-challenge", and authentication proceeds as in initial + authentication (and should be configurable to log an authentication + failure in some sort of security audit log, since the failure may be + a symptom of an attack). The nonce-count MUST NOT be incremented in + this case: to do so would allow a denial of service attack by sending + an out-of-order nonce-count. + + If the response is valid, the server MAY choose to deem that + authentication has succeeded. However, if it has been too long since + the previous authentication, or for any other reason, the server MAY + send a new "digest-challenge" with a new value for nonce. The + challenge MAY contain a "stale" directive with value "true", which + says that the client may respond to the challenge using the password + it used in the previous response; otherwise, the client must solicit + the password anew from the user. This permits the server to make sure + that the user has presented their password recently. (The directive + name refers to the previous nonce being stale, not to the last use of + the password.) Except for the handling of "stale", after sending the + "digest-challenge" authentication proceeds as in the case of initial + authentication. + +2.3 Integrity Protection + + If the server offered "qop=auth-int" and the client responded + "qop=auth-int", then subsequent messages, up to but not including the + next subsequent authentication, between the client and the server + + + + + +Leach & Newman Standards Track [Page 13] + +RFC 2831 Digest SASL Mechanism May 2000 + + + MUST be integrity protected. Using as a base session key the value of + H(A1) as defined above the client and server calculate a pair of + message integrity keys as follows. + + The key for integrity protecting messages from client to server is: + + Kic = MD5({H(A1), + "Digest session key to client-to-server signing key magic constant"}) + + The key for integrity protecting messages from server to client is: + + Kis = MD5({H(A1), + "Digest session key to server-to-client signing key magic constant"}) + + where MD5 is as specified in [RFC 1321]. If message integrity is + negotiated, a MAC block for each message is appended to the message. + The MAC block is 16 bytes: the first 10 bytes of the HMAC-MD5 [RFC + 2104] of the message, a 2-byte message type number in network byte + order with value 1, and the 4-byte sequence number in network byte + order. The message type is to allow for future extensions such as + rekeying. + + MAC(Ki, SeqNum, msg) = (HMAC(Ki, {SeqNum, msg})[0..9], 0x0001, + SeqNum) + + where Ki is Kic for messages sent by the client and Kis for those + sent by the server. The sequence number is initialized to zero, and + incremented by one for each message sent. + + Upon receipt, MAC(Ki, SeqNum, msg) is computed and compared with the + received value; the message is discarded if they differ. + +2.4 Confidentiality Protection + + If the server sent a "cipher-opts" directive and the client responded + with a "cipher" directive, then subsequent messages between the + client and the server MUST be confidentiality protected. Using as a + base session key the value of H(A1) as defined above the client and + server calculate a pair of message integrity keys as follows. + + The key for confidentiality protecting messages from client to server + is: + + Kcc = MD5({H(A1)[0..n], + "Digest H(A1) to client-to-server sealing key magic constant"}) + + The key for confidentiality protecting messages from server to client + is: + + + +Leach & Newman Standards Track [Page 14] + +RFC 2831 Digest SASL Mechanism May 2000 + + + Kcs = MD5({H(A1)[0..n], + "Digest H(A1) to server-to-client sealing key magic constant"}) + + where MD5 is as specified in [RFC 1321]. For cipher "rc4-40" n is 5; + for "rc4-56" n is 7; for the rest n is 16. The key for the "rc-*" + ciphers is all 16 bytes of Kcc or Kcs; the key for "des" is the first + 7 bytes; the key for "3des" is the first 14 bytes. The IV for "des" + and "3des" is the last 8 bytes of Kcc or Kcs. + + If message confidentiality is negotiated, each message is encrypted + with the chosen cipher and a MAC block is appended to the message. + + The MAC block is a variable length padding prefix followed by 16 + bytes formatted as follows: the first 10 bytes of the HMAC-MD5 [RFC + 2104] of the message, a 2-byte message type number in network byte + order with value 1, and the 4-byte sequence number in network byte + order. If the blocksize of the chosen cipher is not 1 byte, the + padding prefix is one or more octets each containing the number of + padding bytes, such that total length of the encrypted part of the + message is a multiple of the blocksize. The padding and first 10 + bytes of the MAC block are encrypted along with the message. + + SEAL(Ki, Kc, SeqNum, msg) = + {CIPHER(Kc, {msg, pad, HMAC(Ki, {SeqNum, msg})[0..9])}), 0x0001, + SeqNum} + + where CIPHER is the chosen cipher, Ki and Kc are Kic and Kcc for + messages sent by the client and Kis and Kcs for those sent by the + server. The sequence number is initialized to zero, and incremented + by one for each message sent. + + Upon receipt, the message is decrypted, HMAC(Ki, {SeqNum, msg}) is + computed and compared with the received value; the message is + discarded if they differ. + +3 Security Considerations + +3.1 Authentication of Clients using Digest Authentication + + Digest Authentication does not provide a strong authentication + mechanism, when compared to public key based mechanisms, for example. + However, since it prevents chosen plaintext attacks, it is stronger + than (e.g.) CRAM-MD5, which has been proposed for use with LDAP [10], + POP and IMAP (see RFC 2195 [9]). It is intended to replace the much + weaker and even more dangerous use of plaintext passwords; however, + since it is still a password based mechanism it avoids some of the + potential deployabilty issues with public-key, OTP or similar + mechanisms. + + + +Leach & Newman Standards Track [Page 15] + +RFC 2831 Digest SASL Mechanism May 2000 + + + Digest Authentication offers no confidentiality protection beyond + protecting the actual password. All of the rest of the challenge and + response are available to an eavesdropper, including the user's name + and authentication realm. + +3.2 Comparison of Digest with Plaintext Passwords + + The greatest threat to the type of transactions for which these + protocols are used is network snooping. This kind of transaction + might involve, for example, online access to a mail service whose use + is restricted to paying subscribers. With plaintext password + authentication an eavesdropper can obtain the password of the user. + This not only permits him to access anything in the database, but, + often worse, will permit access to anything else the user protects + with the same password. + +3.3 Replay Attacks + + Replay attacks are defeated if the client or the server chooses a + fresh nonce for each authentication, as this specification requires. + +3.4 Online dictionary attacks + + If the attacker can eavesdrop, then it can test any overheard + nonce/response pairs against a (potentially very large) list of + common words. Such a list is usually much smaller than the total + number of possible passwords. The cost of computing the response for + each password on the list is paid once for each challenge. + + The server can mitigate this attack by not allowing users to select + passwords that are in a dictionary. + +3.5 Offline dictionary attacks + + If the attacker can choose the challenge, then it can precompute the + possible responses to that challenge for a list of common words. Such + a list is usually much smaller than the total number of possible + passwords. The cost of computing the response for each password on + the list is paid just once. + + Offline dictionary attacks are defeated if the client chooses a fresh + nonce for each authentication, as this specification requires. + + + + + + + + + +Leach & Newman Standards Track [Page 16] + +RFC 2831 Digest SASL Mechanism May 2000 + + +3.6 Man in the Middle + + Digest authentication is vulnerable to "man in the middle" (MITM) + attacks. Clearly, a MITM would present all the problems of + eavesdropping. But it also offers some additional opportunities to + the attacker. + + A possible man-in-the-middle attack would be to substitute a weaker + qop scheme for the one(s) sent by the server; the server will not be + able to detect this attack. For this reason, the client should always + use the strongest scheme that it understands from the choices + offered, and should never choose a scheme that does not meet its + minimum requirements. + +3.7 Chosen plaintext attacks + + A chosen plaintext attack is where a MITM or a malicious server can + arbitrarily choose the challenge that the client will use to compute + the response. The ability to choose the challenge is known to make + cryptanalysis much easier [8]. + + However, Digest does not permit the attack to choose the challenge as + long as the client chooses a fresh nonce for each authentication, as + this specification requires. + +3.8 Spoofing by Counterfeit Servers + + If a user can be led to believe that she is connecting to a host + containing information protected by a password she knows, when in + fact she is connecting to a hostile server, then the hostile server + can obtain challenge/response pairs where it was able to partly + choose the challenge. There is no known way that this can be + exploited. + +3.9 Storing passwords + + Digest authentication requires that the authenticating agent (usually + the server) store some data derived from the user's name and password + in a "password file" associated with a given realm. Normally this + might contain pairs consisting of username and H({ username-value, + ":", realm-value, ":", passwd }), which is adequate to compute H(A1) + as described above without directly exposing the user's password. + + The security implications of this are that if this password file is + compromised, then an attacker gains immediate access to documents on + the server using this realm. Unlike, say a standard UNIX password + file, this information need not be decrypted in order to access + documents in the server realm associated with this file. On the other + + + +Leach & Newman Standards Track [Page 17] + +RFC 2831 Digest SASL Mechanism May 2000 + + + hand, decryption, or more likely a brute force attack, would be + necessary to obtain the user's password. This is the reason that the + realm is part of the digested data stored in the password file. It + means that if one Digest authentication password file is compromised, + it does not automatically compromise others with the same username + and password (though it does expose them to brute force attack). + + There are two important security consequences of this. First the + password file must be protected as if it contained plaintext + passwords, because for the purpose of accessing documents in its + realm, it effectively does. + + A second consequence of this is that the realm string should be + unique among all realms that any single user is likely to use. In + particular a realm string should include the name of the host doing + the authentication. + +3.10 Multiple realms + + Use of multiple realms may mean both that compromise of a the + security database for a single realm does not compromise all + security, and that there are more things to protect in order to keep + the whole system secure. + +3.11 Summary + + By modern cryptographic standards Digest Authentication is weak, + compared to (say) public key based mechanisms. But for a large range + of purposes it is valuable as a replacement for plaintext passwords. + Its strength may vary depending on the implementation. + +4 Example + + This example shows the use of the Digest SASL mechanism with the + IMAP4 AUTHENTICATE command [RFC 2060]. + + In this example, "C:" and "S:" represent a line sent by the client or + server respectively including a CRLF at the end. Linebreaks and + indentation within a "C:" or "S:" are editorial and not part of the + protocol. The password in this example was "secret". Note that the + base64 encoding of the challenges and responses is part of the IMAP4 + AUTHENTICATE command, not part of the Digest specification itself. + + S: * OK elwood.innosoft.com PMDF IMAP4rev1 V6.0-9 + C: c CAPABILITY + S: * CAPABILITY IMAP4 IMAP4rev1 ACL LITERAL+ NAMESPACE QUOTA + UIDPLUS AUTH=CRAM-MD5 AUTH=DIGEST-MD5 AUTH=PLAIN + S: c OK Completed + + + +Leach & Newman Standards Track [Page 18] + +RFC 2831 Digest SASL Mechanism May 2000 + + + C: a AUTHENTICATE DIGEST-MD5 + S: + cmVhbG09ImVsd29vZC5pbm5vc29mdC5jb20iLG5vbmNlPSJPQTZNRzl0 + RVFHbTJoaCIscW9wPSJhdXRoIixhbGdvcml0aG09bWQ1LXNlc3MsY2hh + cnNldD11dGYtOA== + C: Y2hhcnNldD11dGYtOCx1c2VybmFtZT0iY2hyaXMiLHJlYWxtPSJlbHdvb2 + QuaW5ub3NvZnQuY29tIixub25jZT0iT0E2TUc5dEVRR20yaGgiLG5jPTAw + MDAwMDAxLGNub25jZT0iT0E2TUhYaDZWcVRyUmsiLGRpZ2VzdC11cmk9Im + ltYXAvZWx3b29kLmlubm9zb2Z0LmNvbSIscmVzcG9uc2U9ZDM4OGRhZDkw + ZDRiYmQ3NjBhMTUyMzIxZjIxNDNhZjcscW9wPWF1dGg= + S: + cnNwYXV0aD1lYTQwZjYwMzM1YzQyN2I1NTI3Yjg0ZGJhYmNkZmZmZA== + C: + S: a OK User logged in + --- + + The base64-decoded version of the SASL exchange is: + + S: realm="elwood.innosoft.com",nonce="OA6MG9tEQGm2hh",qop="auth", + algorithm=md5-sess,charset=utf-8 + C: charset=utf-8,username="chris",realm="elwood.innosoft.com", + nonce="OA6MG9tEQGm2hh",nc=00000001,cnonce="OA6MHXh6VqTrRk", + digest-uri="imap/elwood.innosoft.com", + response=d388dad90d4bbd760a152321f2143af7,qop=auth + S: rspauth=ea40f60335c427b5527b84dbabcdfffd + + The password in this example was "secret". + + This example shows the use of the Digest SASL mechanism with the + ACAP, using the same notational conventions and password as in the + previous example. Note that ACAP does not base64 encode and uses + fewer round trips that IMAP4. + + S: * ACAP (IMPLEMENTATION "Test ACAP server") (SASL "CRAM-MD5" + "DIGEST-MD5" "PLAIN") + C: a AUTHENTICATE "DIGEST-MD5" + S: + {94} + S: realm="elwood.innosoft.com",nonce="OA9BSXrbuRhWay",qop="auth", + algorithm=md5-sess,charset=utf-8 + C: {206} + C: charset=utf-8,username="chris",realm="elwood.innosoft.com", + nonce="OA9BSXrbuRhWay",nc=00000001,cnonce="OA9BSuZWMSpW8m", + digest-uri="acap/elwood.innosoft.com", + response=6084c6db3fede7352c551284490fd0fc,qop=auth + S: a OK (SASL {40} + S: rspauth=2f0b3d7c3c2e486600ef710726aa2eae) "AUTHENTICATE + Completed" + --- + + + + + +Leach & Newman Standards Track [Page 19] + +RFC 2831 Digest SASL Mechanism May 2000 + + + The server uses the values of all the directives, plus knowledge of + the users password (or the hash of the user's name, server's realm + and the user's password) to verify the computations above. If they + check, then the user has authenticated. + +5 References + + [Digest] Franks, J., et al., "HTTP Authentication: Basic and Digest + Access Authentication", RFC 2617, June 1999. + + [ISO-8859] ISO-8859. International Standard--Information Processing-- + 8-bit Single-Byte Coded Graphic Character Sets -- + Part 1: Latin alphabet No. 1, ISO-8859-1:1987. + Part 2: Latin alphabet No. 2, ISO-8859-2, 1987. + Part 3: Latin alphabet No. 3, ISO-8859-3, 1988. + Part 4: Latin alphabet No. 4, ISO-8859-4, 1988. + Part 5: Latin/Cyrillic alphabet, ISO-8859-5, 1988. + Part 6: Latin/Arabic alphabet, ISO-8859-6, 1987. + Part 7: Latin/Greek alphabet, ISO-8859-7, 1987. + Part 8: Latin/Hebrew alphabet, ISO-8859-8, 1988. + Part 9: Latin alphabet No. 5, ISO-8859-9, 1990. + + [RFC 822] Crocker, D., "Standard for The Format of ARPA Internet + Text Messages," STD 11, RFC 822, August 1982. + + [RFC 1321] Rivest, R., "The MD5 Message-Digest Algorithm", RFC 1321, + April 1992. + + [RFC 2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII Text", + RFC 2047, November 1996. + + [RFC 2052] Gulbrandsen, A. and P. Vixie, "A DNS RR for specifying the + location of services (DNS SRV)", RFC 2052, October 1996. + + [RFC 2060] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [RFC 2104] Krawczyk, H., Bellare, M. and R. Canetti, "HMAC: Keyed- + Hashing for Message Authentication", RFC 2104, February + 1997. + + [RFC 2195] Klensin, J., Catoe, R. and P. Krumviede, "IMAP/POP + AUTHorize Extension for Simple Challenge/Response", RFC + 2195, September 1997. + + + + + + +Leach & Newman Standards Track [Page 20] + +RFC 2831 Digest SASL Mechanism May 2000 + + + [RFC 2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC 2222] Myers, J., "Simple Authentication and Security Layer + (SASL)", RFC 2222, October 1997. + + [USASCII] US-ASCII. Coded Character Set - 7-Bit American Standard + Code for Information Interchange. Standard ANSI X3.4-1986, + ANSI, 1986. + +6 Authors' Addresses + + Paul Leach + Microsoft + 1 Microsoft Way + Redmond, WA 98052 + + EMail: paulle@microsoft.com + + + Chris Newman + Innosoft International, Inc. + 1050 Lakes Drive + West Covina, CA 91790 USA + + EMail: chris.newman@innosoft.com + +7 ABNF + + What follows is the definition of the notation as is used in the + HTTP/1.1 specification (RFC 2616) and the HTTP authentication + specification (RFC 2617); it is reproduced here for ease of + reference. Since it is intended that a single Digest implementation + can support both HTTP and SASL-based protocols, the same notation is + used in both to facilitate comparison and prevention of unwanted + differences. Since it is cut-and-paste from the HTTP specifications, + not all productions may be used in this specification. It is also not + quite legal ABNF; again, the errors were copied from the HTTP + specifications. + +7.1 Augmented BNF + + All of the mechanisms specified in this document are described in + both prose and an augmented Backus-Naur Form (BNF) similar to that + used by RFC 822 [RFC 822]. Implementers will need to be familiar with + the notation in order to understand this specification. + + + + + +Leach & Newman Standards Track [Page 21] + +RFC 2831 Digest SASL Mechanism May 2000 + + + The augmented BNF includes the following constructs: + + name = definition + The name of a rule is simply the name itself (without any + enclosing "<" and ">") and is separated from its definition by the + equal "=" character. White space is only significant in that + indentation of continuation lines is used to indicate a rule + definition that spans more than one line. Certain basic rules are + in uppercase, such as SP, LWS, HT, CRLF, DIGIT, ALPHA, etc. Angle + brackets are used within definitions whenever their presence will + facilitate discerning the use of rule names. + + "literal" + Quotation marks surround literal text. Unless stated otherwise, + the text is case-insensitive. + + rule1 | rule2 + Elements separated by a bar ("|") are alternatives, e.g., "yes | + no" will accept yes or no. + + (rule1 rule2) + Elements enclosed in parentheses are treated as a single element. + Thus, "(elem (foo | bar) elem)" allows the token sequences + "elem foo elem" and "elem bar elem". + + *rule + The character "*" preceding an element indicates repetition. The + full form is "*element" indicating at least and at most + occurrences of element. Default values are 0 and infinity so + that "*(element)" allows any number, including zero; "1*element" + requires at least one; and "1*2element" allows one or two. + + [rule] + Square brackets enclose optional elements; "[foo bar]" is + equivalent to "*1(foo bar)". + + N rule + Specific repetition: "(element)" is equivalent to + "*(element)"; that is, exactly occurrences of (element). + Thus 2DIGIT is a 2-digit number, and 3ALPHA is a string of three + alphabetic characters. + + #rule + A construct "#" is defined, similar to "*", for defining lists of + elements. The full form is "#element" indicating at least + and at most elements, each separated by one or more commas + (",") and OPTIONAL linear white space (LWS). This makes the usual + form of lists very easy; a rule such as + + + +Leach & Newman Standards Track [Page 22] + +RFC 2831 Digest SASL Mechanism May 2000 + + + ( *LWS element *( *LWS "," *LWS element )) + can be shown as + 1#element + Wherever this construct is used, null elements are allowed, but do + not contribute to the count of elements present. That is, + "(element), , (element) " is permitted, but counts as only two + elements. Therefore, where at least one element is required, at + least one non-null element MUST be present. Default values are 0 + and infinity so that "#element" allows any number, including zero; + "1#element" requires at least one; and "1#2element" allows one or + two. + + ; comment + A semi-colon, set off some distance to the right of rule text, + starts a comment that continues to the end of line. This is a + simple way of including useful notes in parallel with the + specifications. + + implied *LWS + The grammar described by this specification is word-based. Except + where noted otherwise, linear white space (LWS) can be included + between any two adjacent words (token or quoted-string), and + between adjacent words and separators, without changing the + interpretation of a field. At least one delimiter (LWS and/or + separators) MUST exist between any two tokens (for the definition + of "token" below), since they would otherwise be interpreted as a + single token. + +7.2 Basic Rules + + The following rules are used throughout this specification to + describe basic parsing constructs. The US-ASCII coded character set + is defined by ANSI X3.4-1986 [USASCII]. + + OCTET = + CHAR = + UPALPHA = + LOALPHA = + ALPHA = UPALPHA | LOALPHA + DIGIT = + CTL = + CR = + LF = + SP = + HT = + <"> = + CRLF = CR LF + + + +Leach & Newman Standards Track [Page 23] + +RFC 2831 Digest SASL Mechanism May 2000 + + + + All linear white space, including folding, has the same semantics as + SP. A recipient MAY replace any linear white space with a single SP + before interpreting the field value or forwarding the message + downstream. + + LWS = [CRLF] 1*( SP | HT ) + + The TEXT rule is only used for descriptive field contents and values + that are not intended to be interpreted by the message parser. Words + of *TEXT MAY contain characters from character sets other than + ISO-8859-1 [ISO 8859] only when encoded according to the rules of RFC + 2047 [RFC 2047]. + + TEXT = + + A CRLF is allowed in the definition of TEXT only as part of a header + field continuation. It is expected that the folding LWS will be + replaced with a single SP before interpretation of the TEXT value. + + Hexadecimal numeric characters are used in several protocol elements. + + HEX = "A" | "B" | "C" | "D" | "E" | "F" + | "a" | "b" | "c" | "d" | "e" | "f" | DIGIT + + Many HTTP/1.1 header field values consist of words separated by LWS + or special characters. These special characters MUST be in a quoted + string to be used within a parameter value. + + token = 1* + separators = "(" | ")" | "<" | ">" | "@" + | "," | ";" | ":" | "\" | <"> + | "/" | "[" | "]" | "?" | "=" + | "{" | "}" | SP | HT + + A string of text is parsed as a single word if it is quoted using + double-quote marks. + + quoted-string = ( <"> qdstr-val <"> ) + qdstr-val = *( qdtext | quoted-pair ) + qdtext = > + + Note that LWS is NOT implicit between the double-quote marks (<">) + surrounding a qdstr-val and the qdstr-val; any LWS will be considered + part of the qdstr-val. This is also the case for quotation marks + surrounding any other construct. + + + + +Leach & Newman Standards Track [Page 24] + +RFC 2831 Digest SASL Mechanism May 2000 + + + The backslash character ("\") MAY be used as a single-character + quoting mechanism only within qdstr-val and comment constructs. + + quoted-pair = "\" CHAR + + The value of this construct is CHAR. Note that an effect of this rule + is that backslash must be quoted. + +8 Sample Code + + The sample implementation in [Digest] also applies to DIGEST-MD5. + + The following code implements the conversion from UTF-8 to 8859-1 if + necessary. + + /* if the string is entirely in the 8859-1 subset of UTF-8, then + * translate to 8859-1 prior to MD5 + */ + void MD5_UTF8_8859_1(MD5_CTX *ctx, const unsigned char *base, + int len) + { + const unsigned char *scan, *end; + unsigned char cbuf; + + end = base + len; + for (scan = base; scan < end; ++scan) { + if (*scan > 0xC3) break; /* abort if outside 8859-1 */ + if (*scan >= 0xC0 && *scan <= 0xC3) { + if (++scan == end || *scan < 0x80 || *scan > 0xBF) + break; + } + } + /* if we found a character outside 8859-1, don't alter string + */ + if (scan < end) { + MD5Update(ctx, base, len); + return; + } + + /* convert to 8859-1 prior to applying hash + */ + do { + for (scan = base; scan < end && *scan < 0xC0; ++scan) + ; + if (scan != base) MD5Update(ctx, base, scan - base); + if (scan + 1 >= end) break; + cbuf = ((scan[0] & 0x3) << 6) | (scan[1] & 0x3f); + MD5Update(ctx, &cbuf, 1); + + + +Leach & Newman Standards Track [Page 25] + +RFC 2831 Digest SASL Mechanism May 2000 + + + base = scan + 2; + } while (base < end); + } + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Leach & Newman Standards Track [Page 26] + +RFC 2831 Digest SASL Mechanism May 2000 + + +9 Full Copyright Statement + + Copyright (C) The Internet Society (2000). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Leach & Newman Standards Track [Page 27] + diff --git a/docs/rfcs/rfc2971.txt b/docs/rfcs/rfc2971.IMAP4_ID_extension.txt similarity index 100% rename from docs/rfcs/rfc2971.txt rename to docs/rfcs/rfc2971.IMAP4_ID_extension.txt diff --git a/docs/rfcs/rfc3028.txt b/docs/rfcs/rfc3028.Sieve_Mail_filtering_language.txt similarity index 100% rename from docs/rfcs/rfc3028.txt rename to docs/rfcs/rfc3028.Sieve_Mail_filtering_language.txt diff --git a/docs/rfcs/rfc3348.txt b/docs/rfcs/rfc3348.IMAP4_Child_Mailbox_extension.txt similarity index 100% rename from docs/rfcs/rfc3348.txt rename to docs/rfcs/rfc3348.IMAP4_Child_Mailbox_extension.txt diff --git a/docs/rfcs/rfc3501.txt b/docs/rfcs/rfc3501.IMAP4rev1.txt similarity index 100% rename from docs/rfcs/rfc3501.txt rename to docs/rfcs/rfc3501.IMAP4rev1.txt diff --git a/docs/rfcs/rfc3502.txt b/docs/rfcs/rfc3502.MULTIAPPEND_extension.txt similarity index 100% rename from docs/rfcs/rfc3502.txt rename to docs/rfcs/rfc3502.MULTIAPPEND_extension.txt diff --git a/docs/rfcs/rfc3503.txt b/docs/rfcs/rfc3503.Message_Disposition_Notification.txt similarity index 100% rename from docs/rfcs/rfc3503.txt rename to docs/rfcs/rfc3503.Message_Disposition_Notification.txt diff --git a/docs/rfcs/rfc3516.txt b/docs/rfcs/rfc3516.IMAP4_Binary_content_extension.txt similarity index 100% rename from docs/rfcs/rfc3516.txt rename to docs/rfcs/rfc3516.IMAP4_Binary_content_extension.txt diff --git a/docs/rfcs/rfc3656.txt b/docs/rfcs/rfc3656.txt deleted file mode 100644 index 6c0ab5b..0000000 --- a/docs/rfcs/rfc3656.txt +++ /dev/null @@ -1,1067 +0,0 @@ - - - - - - -Network Working Group R. Siemborski -Request for Comments: 3656 Carnegie Mellon University -Category: Experimental December 2003 - - - The Mailbox Update (MUPDATE) - Distributed Mailbox Database Protocol - -Status of this Memo - - This memo defines an Experimental Protocol for the Internet - community. It does not specify an Internet standard of any kind. - Discussion and suggestions for improvement are requested. - Distribution of this memo is unlimited. - -Copyright Notice - - Copyright (C) The Internet Society (2003). All Rights Reserved. - -Abstract - - As the demand for high-performance mail delivery agents increases, it - becomes apparent that single-machine solutions are inadequate to the - task, both because of capacity limits and that the failure of the - single machine means a loss of mail delivery for all users. It is - preferable to allow many machines to share the responsibility of mail - delivery. - - The Mailbox Update (MUPDATE) protocol allows a group of Internet - Message Access Protocol (IMAP) or Post Office Protocol - Version 3 - (POP3) servers to function with a unified mailbox namespace. This - document is intended to serve as a reference guide to that protocol. - - - - - - - - - - - - - - - - - - - -Siemborski Experimental [Page 1] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -Table of Contents - - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 3 - 2. Protocol Overview . . . . . . . . . . . . . . . . . . . . . . 3 - 2.1. Atoms . . . . . . . . . . . . . . . . . . . . . . . . . 4 - 2.2. Strings . . . . . . . . . . . . . . . . . . . . . . . . 4 - 3. Server Responses . . . . . . . . . . . . . . . . . . . . . . 4 - 3.1. Response: OK . . . . . . . . . . . . . . . . . . . . . 5 - 3.2. Response: NO . . . . . . . . . . . . . . . . . . . . . 5 - 3.3. Response: BAD . . . . . . . . . . . . . . . . . . . . . 5 - 3.4. Response: BYE . . . . . . . . . . . . . . . . . . . . . 6 - 3.5. Response: RESERVE . . . . . . . . . . . . . . . . . . . 6 - 3.6. Response: MAILBOX . . . . . . . . . . . . . . . . . . . 6 - 3.7. Response: DELETE . . . . . . . . . . . . . . . . . . . 7 - 3.8. Server Capability Response. . . . . . . . . . . . . . . 7 - 4. Client Commands . . . . . . . . . . . . . . . . . . . . . . . 8 - 4.1. Command: ACTIVATE . . . . . . . . . . . . . . . . . . . 8 - 4.2. Command: AUTHENTICATE . . . . . . . . . . . . . . . . . 8 - 4.3. Command: DEACTIVATE . . . . . . . . . . . . . . . . . . 9 - 4.4. Command: DELETE . . . . . . . . . . . . . . . . . . . . 9 - 4.5. Command: FIND . . . . . . . . . . . . . . . . . . . . . 9 - 4.6. Command: LIST . . . . . . . . . . . . . . . . . . . . . 10 - 4.7. Command: LOGOUT . . . . . . . . . . . . . . . . . . . . 10 - 4.8. Command: NOOP . . . . . . . . . . . . . . . . . . . . . 10 - 4.9. Command: RESERVE. . . . . . . . . . . . . . . . . . . . 10 - 4.10. Command: STARTTLS . . . . . . . . . . . . . . . . . . . 11 - 4.11. Command: UPDATE . . . . . . . . . . . . . . . . . . . . 12 - 5. MUPDATE Formal Syntax . . . . . . . . . . . . . . . . . . . . 12 - 6. MUPDATE URL Scheme. . . . . . . . . . . . . . . . . . . . . . 14 - 6.1. MUPDATE URL Scheme Registration Form. . . . . . . . . . 14 - 7. Security Considerations . . . . . . . . . . . . . . . . . . . 15 - 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 16 - 9. Intellectual Property Rights. . . . . . . . . . . . . . . . . 16 - 10. References. . . . . . . . . . . . . . . . . . . . . . . . . . 17 - 10.1. Normative References. . . . . . . . . . . . . . . . . . 17 - 10.2. Informative References. . . . . . . . . . . . . . . . . 17 - 11. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 18 - 12. Author's Address. . . . . . . . . . . . . . . . . . . . . . . 18 - 13. Full Copyright Statement. . . . . . . . . . . . . . . . . . . 19 - - - - - - - - - - - - -Siemborski Experimental [Page 2] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -1. Introduction - - In order to support an architecture where there are multiple [IMAP, - POP3] servers sharing a common mailbox database, it is necessary to - be able to provide atomic mailbox operations, as well as offer - sufficient guarantees about database consistency. - - The primary goal of the MUPDATE protocol is to be simple to implement - yet allow for database consistency between participants. - - The key words "MUST, "MUST NOT", "REQUIRED", "SHOULD", "SHOULD NOT", - "RECOMMENDED", and "MAY" in this document are to be interpreted as - defined in BCP 14, RFC 2119 [KEYWORDS]. - - In examples, "C:" and "S:" indicate lines sent by the client and - server respectively. - -2. Protocol Overview - - The MUPDATE protocol assumes a reliable data stream such as a TCP - network connection. IANA has registered port 3905 with a short name - of "mupdate" for this purpose. - - In the current implementation of the MUPDATE protocol there are three - types of participants: a single master server, slave (or replica) - servers, and clients. The master server maintains an authoritative - copy of the mailbox database. Slave servers connect to the MUPDATE - master server as clients, and function as replicas from the point of - view of end clients. End clients may connect to either the master or - any slave and perform searches against the database, however - operations that change the database can only be performed against the - master. For the purposes of protocol discussion we will consider a - slave's connection to the master identical to that of any other - client. - - After connection, all commands from a client to server must have an - associated unique tag which is an alphanumeric string. Commands MAY - be pipelined from the client to the server (that is, the client need - not wait for the response before sending the next command). The - server MUST execute the commands in the order they were received, - however. - - If the server supports an inactivity login timeout, it MUST be at - least 15 minutes. - - - - - - - -Siemborski Experimental [Page 3] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - - MUPDATE uses data formats similar to those used in [ACAP]. That is, - atoms and strings. All commands and tags in the protocol are - transmitted as atoms. All other data is considered to a string, and - must be quoted or transmitted as a literal. - - Outside of a literal, both clients and servers MUST support line - lengths of at least 1024 octets (including the trailing CR and LF - characters). If a line of a longer length must be transmitted, - implementations MUST make use of literals to do so. - -2.1. Atoms - - An atom consists of one or more alphanumeric characters. Atoms MUST - be less than 15 octets in length. - -2.2. Strings - - As in [ACAP], a string may be either literal or a quoted string. A - literal is a sequence of zero or more octets (including CR and LF), - prefix-quoted with an octet count in the form of an open brace ("{"), - the number of octets, an optional plus sign to indicate that the data - follows immediately (a non-synchronized literal), a close brace - ("}"), and a CRLF sequence. If the plus sign is omitted (a - synchronized literal), then the receiving side MUST send a "+ go - ahead" response, and the sending side MUST wait for this response. - Servers MUST support literals of atleast 4096 octets. - - Strings that are sent from server to client SHOULD NOT be in the - synchronized literal format. - - A quoted string is a sequence of zero or more 7-bit characters, - excluding CR, LF, and the double quote (<">), with double quote - characters at each end. - - The empty string is represented as either "" (a quoted string with - zero characters between double quotes) or as {0} followed by CRLF (a - literal with an octet count of 0). - -3. Server Responses - - Every client command in the MUPDATE protocol may receive one or more - tagged responses from the server. Each response is preceded by the - same tag as the command that elicited the response from the server. - - - - - - - - -Siemborski Experimental [Page 4] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -3.1. Response: OK - - A tagged OK response indicates that the operation completed - successfully. There is a mandatory implementation-defined string - after the OK response. This response also indicates the beginning of - the streaming update mode when given in response to an UPDATE - command. - - Example: - -C: N01 NOOP -S: N01 OK "NOOP Complete" - -3.2. Response: NO - - A tagged NO response indicates that the operation was explicitly - denied by the server or otherwise failed. There is a mandatory - implementation-defined string after the NO response that SHOULD - explain the reason for denial. - - Example: - -C: A01 AUTHENTICATE "PLAIN" -S: A01 NO "PLAIN is not a supported SASL mechanism" - -3.3. Response: BAD - - A tagged BAD response indicates that the command from the client - could not be parsed or understood. There is a mandatory - implementation-defined string after the BAD response to provide - additional information about the error. Note that untagged BAD - responses are allowed if it is unclear what the tag for a given - command is (for example, if a blank line is received by the mupdate - server, it can generate an untagged BAD response). In the case of an - untagged response, the tag should be replaced with a "*". - - Example: - -C: C01 SELECT "INBOX" -S: C01 BAD "This is not an IMAP server" -C: -S: * BAD "Need Command" - - - - - - - - - -Siemborski Experimental [Page 5] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -3.4. Response: BYE - - A tagged BYE response indicates that the server has decided to close - the connection. There is a mandatory implementation-defined string - after the BYE response that SHOULD explain the reason for closing the - connection. The server MUST close the connection immediately after - transmitting the BYE response. - - Example: - -C: L01 LOGOUT -S: L01 BYE "User Logged Out" - -3.5. Response: RESERVE - - A tagged RESERVE response may only be given in response to a FIND, - LIST, or UPDATE command. It includes two parameters: the name of the - mailbox that is being reserved (in mUTF-7 encoding, as specified in - [IMAP]) and a location string whose contents is defined by the - clients that are using the database, though it is RECOMMENDED that - the format of this string be the hostname of the server which is - storing the mailbox. - - This response indicates that the given name is no longer available in - the namespace, though it does not indicate that the given mailbox is - available to clients at the current time. - - Example: - -S: U01 RESERVE "internet.bugtraq" "mail2.example.org" - -3.6. Response: MAILBOX - - A tagged MAILBOX response may only be given in response to a FIND, - LIST, or UPDATE command. It includes three parameters: the name of - the mailbox, a location string (as with RESERVE), and a client- - defined string that specifies the IMAP ACL [IMAP-ACL] of the mailbox. - This message indicates that the given mailbox is ready to be accessed - by clients. - - Example: - -S: U01 MAILBOX "internet.bugtraq" "mail2.example.org" "anyone rls" - - - - - - - - -Siemborski Experimental [Page 6] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -3.7. Response: DELETE - - A tagged DELETE response may only be given in response to an UPDATE - command, and MUST NOT be given before the OK response to the UPDATE - command is given. It contains a single parameter, that of the - mailbox that should be deleted from the slave's database. This - response indicates that the given mailbox no longer exists in the - namespace of the database, and may be given for any mailbox name, - active, reserved, or nonexistent. (Though implementations SHOULD NOT - issue DELETE responses for nonexistent mailboxes). - - Example: - -S: U01 DELETE "user.rjs3.sent-mail-jan-2002" - -3.8. Server Capability Response - - Upon connection of the client to the server, and directly following a - successful STARTTLS command, the server MUST issue a capabilities - banner, of the following format: - - The banner MUST contain a line that begins with "* AUTH" and contain - a space-separated list of SASL mechanisms that the server will accept - for authentication. The mechanism names are transmitted as atoms. - Servers MAY advertise no available mechanisms (to indicate that - STARTTLS must be completed before authentication may occur). If - STARTTLS is not supported by the server, then the line MUST contain - at least one mechanism. - - If the banner is being issued without a TLS layer, and the server - supports the STARTTLS command, the banner MUST contain the line "* - STARTTLS". If the banner is being issued under a TLS layer (or the - server does not support STARTTLS), the banner MUST NOT contain this - line. - - The last line of the banner MUST start with "* OK MUPDATE" and be - followed by four strings: the server's hostname, an implementation- - defined string giving the name of the implementation, an - implementation-defined string giving the version of the - implementation, and a string that indicates if the server is a master - or a slave. The master/slave indication MUST be either "(master)" or - an MUPDATE URL that defines where the master can be contacted. - - Any unrecognized responses before the "* OK MUPDATE" response MUST be - ignored by the client. - - - - - - -Siemborski Experimental [Page 7] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - - Example: - -S: * AUTH KERBEROS_V4 GSSAPI -S: * STARTTLS -S: * OK MUPDATE "mupdate.example.org" "Cyrus" "v2.1.2" "(master)" - -4. Client Commands - - The following are valid commands that a client may send to the - MUPDATE server: AUTHENTICATE, ACTIVATE, DEACTIVATE, DELETE, FIND, - LIST, LOGOUT, NOOP, RESERVE, STARTTLS, and UPDATE. - - Before a successful AUTHENTICATE command has occurred, the server - MUST NOT accept any commands except for AUTHENTICATE, STARTTLS, and - LOGOUT (and SHOULD reply with a NO response for all other commands). - -4.1. Command: ACTIVATE - - The ACTIVATE command has 3 parameters: the mailbox name, its - location, and its ACL. This command MUST NOT not be issued to a - slave server. - - This command can also be used to update the ACL or location - information of a mailbox. Note that it is not a requirement for a - mailbox to be reserved (or even exist in the database) for an - ACTIVATE command to succeed, implementations MUST allow this behavior - as it facilitates synchronization of the database with the current - state of the mailboxes. - -4.2. Command: AUTHENTICATE - - The AUTHENTICATE command initiates a [SASL] negotiation session - between the client and the server. It has two parameters. The first - parameter is mandatory, and is a string indicating the desired [SASL] - mechanism. The second is a string containing an optional BASE64 - encoded (as defined in section 6.8 of [MIME]) client first send. - - All of the remaining SASL blobs that are sent MUST be sent across the - wire must be in BASE64 encoded format, and followed by a CR and LF - combination. They MUST NOT be encoded as strings. - - Clients may cancel authentication by sending a * followed by a CR and - LF. - - The [SASL] service name for the MUPDATE protocol is "mupdate". - Implementations are REQUIRED to implement the GSSAPI [SASL] - mechanism, though they SHOULD implement as many mechanisms as - possible. - - - -Siemborski Experimental [Page 8] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - - If a security layer is negotiated, it should be used directly - following the CR and LF combination at the end of the server's OK - response (i.e., beginning with the client's next command) Only one - successful AUTHENTICATE command may be issued per session. - -4.3. Command: DEACTIVATE - - The DEACTIVATE command takes two parameters, the mailbox name and - location data. The mailbox MUST already exist and be activated on - the MUPDATE server. If the server responds OK, then the mailbox name - has been moved to the RESERVE state. If the server responds NO, then - the mailbox name has not been moved (for example, the mailbox was not - already active). Any ACL information that is known about the mailbox - MAY be lost when a DEACTIVATE succeeds. This command MUST NOT be - issued to a slave. - - Example: - -C: A01 DEACTIVATE "user.rjs3.new" "mail3.example.org!u4" -S: A01 OK "Mailbox Reserved." - -4.4. Command: DELETE - - The DELETE command takes only a single parameter, the mailbox name to - be removed from the database's namespace. The server SHOULD give a - NO response if the mailbox does not exist. This command MUST NOT be - issued to a slave server. - -4.5. Command: FIND - - The FIND command takes a single parameter, a mailbox name. The - server then responds with the current record for the given mailbox, - if any, and an OK response. - - Example (mailbox does not exist): - -C: F01 FIND "user.rjs3.xyzzy" -S: F01 OK "Search Complete" - - Example (mailbox is reserved): - -C: F01 FIND "user.rjs3" -S: F01 RESERVE "user.rjs3" "mail4.example.org" -S: F01 OK "Search Complete" - - - - - - - -Siemborski Experimental [Page 9] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -4.6. Command: LIST - - The LIST command is similar to running FIND across the entire - database. The LIST command takes a single optional parameter, which - is a prefix to try to match against the location field of the - records. Without the parameter, LIST returns every record in the - database. - - For each mailbox that matches, either a MAILBOX or a RESERVE response - (as applicable) is sent to the client. When all responses are - complete, an OK response is issued. - - Example: - -C: L01 LIST -S: L01 RESERVE "user.rjs3" "mail4.example.org!u2" -S: L01 MAILBOX "user.leg" "mail2.example.org!u1" "leg lrswipcda" -S: L01 OK "List Complete" -C: L02 LIST "mail4.example.org!" -S: L02 RESERVE "user.rjs3" "mail4.example.org!u2" -S: L02 OK "List Complete" - -4.7. Command: LOGOUT - - The LOGOUT command tells the server to close the connection. Its - only valid response is the BYE response. The LOGOUT command takes no - parameters. - -4.8. Command: NOOP - - The NOOP command takes no parameters. Provided the client is - authenticated, its only acceptable response is an OK. Any idle - timeouts that the server may have on the connection SHOULD be reset - upon receipt of this command. - - If this command is issued after an UPDATE command has been issued, - then the OK response also indicates that all pending database updates - have been sent to the client. That is, the slave can guarantee that - its local database is up to date as of a certain time by issuing a - NOOP and waiting for the OK. The OK MUST NOT return until all - updates that were pending at the time of the NOOP have been sent. - -4.9. Command: RESERVE - - The RESERVE command takes two parameters (just like the RESERVE - response), the mailbox name to reserve and location data. If the - server responds OK, then the mailbox name has been reserved. If the - server responds NO, then the mailbox name has not been reserved (for - - - -Siemborski Experimental [Page 10] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - - example, another server has reserved it already). This command MUST - NOT be issued to a slave. - - The typical sequence for mailbox creation is: - -C: R01 RESERVE "user.rjs3.new" "mail3.example.org!u4" -S: R01 OK "Mailbox Reserved." - -C: A01 ACTIVATE "user.rjs3.new" "mail3.example.org!u4" "rjs3 lrswipcda" -S: A01 OK "Mailbox Activated." - -4.10. Command: STARTTLS - - The STARTTLS command requests the commencement of a [TLS] - negotiation. The negotiation begins immediately after the CRLF in - the OK response. After a client issues a STARTTLS command, it MUST - NOT issue further commands until a server response is seen and the - [TLS] negotiation is complete. - - The STARTTLS command is only valid in non-authenticated state. The - server remains in non-authenticated state, even if client credentials - are supplied during the [TLS] negotiation. The [SASL] EXTERNAL - mechanism MAY be used to authenticate once [TLS] client credentials - are successfully exchanged. Note that servers are not required to - support the EXTERNAL mechanism. - - After the [TLS] layer is established, the server MUST re-issue the - initial response banner (see Section 3.8). This is necessary to - protect against man-in-the-middle attacks which alter the - capabilities list prior to STARTTLS, as well as to advertise any new - SASL mechanisms (or other capabilities) that may be available under - the layer. The client MUST discard cached capability information and - replace it with the new information. - - After the a successful STARTTLS command, the server SHOULD return a - NO response to additional STARTTLS commands. - - Servers MAY choose to not implement STARTTLS. In this case, they - MUST NOT advertise STARTTLS in their capabilities banner, and SHOULD - return a BAD response to the STARTTLS command, if it is issued. - - Example: - -C: S01 STARTTLS -S: S01 OK "Begin TLS negotiation now" - -S: * AUTH KERBEROS_V4 GSSAPI PLAIN -S: * OK MUPDATE "mupdate.example.org" "Cyrus" "v2.1.2" "(master)" - - - -Siemborski Experimental [Page 11] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -4.11. Command: UPDATE - - The UPDATE command is how a slave initializes an update stream from - the master (though it is also valid to issue this command to a - slave). In response to the command, the server returns a list of all - mailboxes in its database (the same results as a parameterless LIST - command) followed by an OK response. From this point forward, - whenever an update occurs to the master database, it MUST stream the - update to the slave within 30 seconds. That is, it will send - RESERVE, MAILBOX, or DELETE responses as they are applicable. - - After a client has issued an UPDATE command, it may only issue NOOP - and LOGOUT commands for the remainder of the session. - - Example: - -C: U01 UPDATE -S: U01 MAILBOX "user.leg" "mail2.example.org!u1" "leg lrswipcda" -S: U01 MAILBOX "user.rjs3" "mail3.example.org!u4" "rjs3 lrswipcda" -S: U01 RESERVE "internet.bugtraq" "mail1.example.org!u5" "anyone lrs" -S: U01 OK "Streaming Begins" - -S: U01 RESERVE "user.leg.new" "mail2.example.org!u1" - -S: U01 MAILBOX "user.leg.new" "mail2.example.org!u1" "leg lrswipcda" - -C: N01 NOOP -S: U01 DELETE "user.leg.new" -S: N01 OK "NOOP Complete" - -5. MUPDATE Formal Syntax - - The following syntax specification uses the Augmented Backus-Naur - Form (ABNF) notation as specified in [ABNF]. This uses the ABNF core - rules as specified in Appendix A of [ABNF]. - - Except as noted otherwise, all alphabetic characters are case- - insensitive. The use of upper or lower case characters to define - token strings is for editorial clarity only. Implementations MUST - accept these strings in a case-insensitive fashion. - - Note that this specification also uses some terminals from section 8 - of [ACAP]. - - cmd-activate = "ACTIVATE" SP string SP string SP string - - cmd-authenticate = "AUTHENTICATE" SP sasl-mech [ SP string ] - - - -Siemborski Experimental [Page 12] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - - cmd-delete = "DELETE" SP string - - cmd-find = "FIND" SP string - - cmd-list = "LIST" [ SP string ] - - cmd-logout = "LOGOUT" - - cmd-noop = "NOOP" - - cmd-reserve = "RESERVE" SP string SP string - - cmd-starttls = "STARTTLS" - - cmd-update = "UPDATE" - - command = tag SP command-type CRLF - - command-type = cmd-activate / cmd-authenticate / cmd-delete / - cmd-find / cmd-list / cmd-logout / cmd-noop / - cmd-reserve / cmd-starttls / cmd-update - - response = tag SP response-type CRLF - - response-type = rsp-ok / rsp-no / rsp-bad / rsp-bye / rsp-mailbox / - rsp-reserve / rsp-delete - - rsp-bad = "BAD" SP string - - rsp-bye = "BYE" SP string - - rsp-mailbox = "MAILBOX" SP string SP string SP string - - rsp-no = "NO" SP string - - rsp-ok = "OK" SP string - - rsp-reserve = "RESERVE" SP string SP string - - rsp-delete = "DELETE" SP string - - sasl-mech = 1*ATOM-CHAR - ; ATOM-CHAR is defined in [ACAP] - - string = quoted / literal - ; quoted and literal are defined in [ACAP] - - - - - -Siemborski Experimental [Page 13] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - - tag = 1*ATOM-CHAR - ; ATOM-CHAR is defined in [ACAP] - -6. MUPDATE URL Scheme - - This document defines the a URL scheme for the purposes of - referencing MUPDATE resources, according to the requirements in - [RFC2717]. This includes both MUPDATE servers as a whole, along with - individual mailbox entries on a given MUPDATE server. - - There is no MIME type associated with these resources. It is - intended that a URL consumer would either retrieve the MUPDATE record - in question, or simply connect to the MUPDATE server running on the - specified host. Note that the consumer will need to have - authentication credentials for the specified host. - - The MUPDATE URL scheme is similar to the IMAP URL scheme [IMAP-URL]. - However, it only takes one of two possible forms: - - mupdate:/// - mupdate:/// - - The first form refers to a MUPDATE server as a whole, the second form - indicates both the server and a mailbox to run a FIND against once - authenticated to the server. Note that part of may include - username and authentication information along with a hostname and - port. - -6.1. MUPDATE URL Scheme Registration Form - - URL scheme name: "mupdate" - - URL scheme syntax: - - This defines the MUPDATE URL Scheme in [ABNF]. Terminals from the - BNF of IMAP URLs [IMAP-URL] are also used. - - mupdateurl = "mupdate://" iserver "/" [ enc_mailbox ] - ; iserver and enc_mailbox are as defined in [IMAP-URL] - - Character encoding considerations: - - Identical to those described in [IMAP-URL] for the appropriate - terminals. - - - - - - - -Siemborski Experimental [Page 14] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - - Intended Usage: - - The form of the URL without an associated mailbox is intended to - designate a MUPDATE server only. If a mailbox name is included in - the URL, then the consumer is expected to execute a FIND command - for that mailbox on the specified server. - - Applications and/or protocols which use this URL scheme name: - - The protocol described in this document. - - Interoperability Considerations: - - None. - - Security Considerations: - - Users of the MUPDATE URL Scheme should review the security - considerations that are discussed in [IMAP-URL]. In particular, - the consequences of including authentication mechanism information - in a URL should be reviewed. - - Relevant Publications: - - This document and [IMAP-URL]. - - Author, Change Controller, and Contact for Further Information: - - Author of this document. - -7. Security Considerations - - While no unauthenticated users may make modifications or even perform - searches on the database, it is important to note that this - specification assumes no protections of any type for authenticated - users. - - All authenticated users have complete access to the database. For - this reason it is important to ensure that accounts that are making - use of the database are well secured. - - A more secure deployment might have all read only access go through a - slave, and only have accounts which need write access use the master. - This has the disadvantage of a marginally longer time for updates to - reach the clients. - - - - - - -Siemborski Experimental [Page 15] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - - The protocol assumes that all authenticated users are cooperating to - maintain atomic operations. Therefore, all new mailboxes SHOULD be - RESERVEd before they are ACTIVATEd, despite the fact that the - protocol does not require this, and it is therefore possible for a - set of participants which do not obey the provided locking to create - an inconsistent database. RESERVEing the mailbox first is not - required to perform an activate because this behavior simplifies - synchronization with the actual location of the mailboxes. - -8. IANA Considerations - - The IANA has assigned TCP port number 3905 to "mupdate". - - The IANA has registered a URL scheme for the MUPDATE protocol, as - defined in section 6.1 of this document. - - IANA has registered a GSSAPI service name of "mupdate" for the - MUPDATE protocol in the registry maintained at: - - http://www.iana.org/assignments/gssapi-service-names - -9. Intellectual Property Rights - - The IETF takes no position regarding the validity or scope of any - intellectual property or other rights that might be claimed to - pertain to the implementation or use of the technology described in - this document or the extent to which any license under such rights - might or might not be available; neither does it represent that it - has made any effort to identify any such rights. Information on the - IETF's procedures with respect to rights in standards-track and - standards-related documentation can be found in BCP-11. Copies of - claims of rights made available for publication and any assurances of - licenses to be made available, or the result of an attempt made to - obtain a general license or permission for the use of such - proprietary rights by implementors or users of this specification can - be obtained from the IETF Secretariat. - - The IETF invites any interested party to bring to its attention any - copyrights, patents or patent applications, or other proprietary - rights which may cover technology that may be required to practice - this standard. Please address the information to the IETF Executive - Director. - - - - - - - - - -Siemborski Experimental [Page 16] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -10. References - -10.1. Normative References - - [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate - Requirement Levels", BCP 14, RFC 2119, March 1997. - - [IMAP] Crispin, M., "Internet Message Access Protocol - Version - 4", RFC 3501, March 2003. - - [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for - Syntax Specifications: ABNF", RFC 2234, November 1997. - - [MIME] Freed, N. and N. Bornstein, "Multipurpose Internet Mail - Extensions (MIME) Part One: Format of Internet Message - Bodies", RFC 2045, November 1996. - - [IMAP-ACL] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. - - [SASL] Myers, J., "Simple Authentication and Security Layer - (SASL)", RFC 2222, October 1997. - - [IMAP-URL] Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. - - [ACAP] Newman, C. and J. Myers, "ACAP -- Application - Configuration Access Protocol", RFC 2244, November 1997. - - [TLS] Dierks, T. and C. Allen, "The TLS Protocol Version 1.0", - RFC 2246, January 1999. - -10.2. Informative References - - [POP3] Myers, J. and M. Rose, "Post Office Protocol - Version - 3", STD 53, RFC 1939, May 1996. - - [RFC2717] Petke, R. and I. King, "Registration Procedures for URL - Scheme Names", BCP 35, RFC 2717, November 1999. - - - - - - - - - - - - - - -Siemborski Experimental [Page 17] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -11. Acknowledgments - - Lawrence Greenfield and Ken Murchison, for a great deal of input on - both the protocol and the text of the documents. - -12. Author's Address - - Robert Siemborski - Carnegie Mellon, Andrew Systems Group - Cyert Hall 207 - 5000 Forbes Avenue - Pittsburgh, PA 15213 - - Phone: (412) 268-7456 - EMail: rjs3+@andrew.cmu.edu - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Siemborski Experimental [Page 18] - -RFC 3656 MUPDATE Distributed Mailbox Database Protocol December 2003 - - -13. Full Copyright Statement - - Copyright (C) The Internet Society (2003). All Rights Reserved. - - This document and translations of it may be copied and furnished to - others, and derivative works that comment on or otherwise explain it - or assist in its implementation may be prepared, copied, published - and distributed, in whole or in part, without restriction of any - kind, provided that the above copyright notice and this paragraph are - included on all such copies and derivative works. However, this - document itself may not be modified in any way, such as by removing - the copyright notice or references to the Internet Society or other - Internet organizations, except as needed for the purpose of - developing Internet standards in which case the procedures for - copyrights defined in the Internet Standards process must be - followed, or as required to translate it into languages other than - English. - - The limited permissions granted above are perpetual and will not be - revoked by the Internet Society or its successors or assignees. - - This document and the information contained herein is provided on an - "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING - TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING - BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION - HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF - MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. - -Acknowledgement - - Funding for the RFC Editor function is currently provided by the - Internet Society. - - - - - - - - - - - - - - - - - - - -Siemborski Experimental [Page 19] - diff --git a/docs/rfcs/rfc3691.txt b/docs/rfcs/rfc3691.IMAP_UNSELECT_command.txt similarity index 100% rename from docs/rfcs/rfc3691.txt rename to docs/rfcs/rfc3691.IMAP_UNSELECT_command.txt diff --git a/docs/rfcs/rfc4314.txt b/docs/rfcs/rfc4314.IMAP4_ACL_extension.txt similarity index 100% rename from docs/rfcs/rfc4314.txt rename to docs/rfcs/rfc4314.IMAP4_ACL_extension.txt diff --git a/docs/rfcs/rfc4315.txt b/docs/rfcs/rfc4315.IMAP_UIDPLUS_extension.txt similarity index 100% rename from docs/rfcs/rfc4315.txt rename to docs/rfcs/rfc4315.IMAP_UIDPLUS_extension.txt diff --git a/docs/rfcs/rfc4466.txt b/docs/rfcs/rfc4466.Collected_extensions_to_IMAP4_ABNF.txt similarity index 100% rename from docs/rfcs/rfc4466.txt rename to docs/rfcs/rfc4466.Collected_extensions_to_IMAP4_ABNF.txt diff --git a/docs/rfcs/rfc4467.txt b/docs/rfcs/rfc4467.IMAP_URLAUTH_extension.txt similarity index 100% rename from docs/rfcs/rfc4467.txt rename to docs/rfcs/rfc4467.IMAP_URLAUTH_extension.txt diff --git a/docs/rfcs/rfc4469.txt b/docs/rfcs/rfc4469.IMAP_CATENATE_extension.txt similarity index 100% rename from docs/rfcs/rfc4469.txt rename to docs/rfcs/rfc4469.IMAP_CATENATE_extension.txt diff --git a/docs/rfcs/rfc4549.txt b/docs/rfcs/rfc4549.Sync_operations_for_disconnected_IMAP4_Clients.txt similarity index 100% rename from docs/rfcs/rfc4549.txt rename to docs/rfcs/rfc4549.Sync_operations_for_disconnected_IMAP4_Clients.txt diff --git a/docs/rfcs/rfc4551.txt b/docs/rfcs/rfc4551.IMAP_Conditional_STORE_or_Quick_flag_changes_resync.txt similarity index 100% rename from docs/rfcs/rfc4551.txt rename to docs/rfcs/rfc4551.IMAP_Conditional_STORE_or_Quick_flag_changes_resync.txt diff --git a/docs/rfcs/rfc4731.IMAP4_Extension_to_SEARCH_command.txt b/docs/rfcs/rfc4731.IMAP4_Extension_to_SEARCH_command.txt new file mode 100644 index 0000000..8c4869a --- /dev/null +++ b/docs/rfcs/rfc4731.IMAP4_Extension_to_SEARCH_command.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 4731 Isode Ltd +Category: Standards Track D. Cridland + Inventure Systems Ltd + November 2006 + + + IMAP4 Extension to SEARCH Command for Controlling + What Kind of Information Is Returned + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The IETF Trust (2006). + +Abstract + + This document extends IMAP (RFC 3501) SEARCH and UID SEARCH commands + with several result options, which can control what kind of + information is returned. The following result options are defined: + minimal value, maximal value, all found messages, and number of found + messages. + +Table of Contents + + 1. Introduction ....................................................2 + 2. Conventions Used in This Document ...............................2 + 3. IMAP Protocol Changes ...........................................2 + 3.1. New SEARCH/UID SEARCH Result Options .......................2 + 3.2. Interaction with CONDSTORE extension .......................4 + 4. Formal Syntax ...................................................5 + 5. Security Considerations .........................................6 + 6. IANA Considerations .............................................6 + 7. Normative References ............................................6 + 8. Acknowledgments .................................................6 + + + + + + + + + +Melnikov & Cridland Standards Track [Page 1] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + +1. Introduction + + [IMAPABNF] extended SEARCH and UID SEARCH commands with result + specifiers (also known as result options), which can control what + kind of information is returned. + + A server advertising the ESEARCH capability supports the following + result options: minimal value, maximal value, all found messages, + and number of found messages. These result options allow clients to + get SEARCH results in more convenient forms, while also saving + bandwidth required to transport the results, for example, by finding + the first unseen message or returning the number of unseen or deleted + messages. Also, when a single MIN or a single MAX result option is + specified, servers can optimize execution of SEARCHes. + +2. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + +3. IMAP Protocol Changes + +3.1. New SEARCH/UID SEARCH Result Options + + The SEARCH/UID SEARCH commands are extended to allow for the + following result options: + + MIN + Return the lowest message number/UID that satisfies the SEARCH + criteria. + + If the SEARCH results in no matches, the server MUST NOT + include the MIN result option in the ESEARCH response; however, + it still MUST send the ESEARCH response. + + MAX + Return the highest message number/UID that satisfies the SEARCH + criteria. + + If the SEARCH results in no matches, the server MUST NOT + include the MAX result option in the ESEARCH response; however, + it still MUST send the ESEARCH response. + + + + + +Melnikov & Cridland Standards Track [Page 2] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + + ALL + Return all message numbers/UIDs that satisfy the SEARCH + criteria. Unlike regular (unextended) SEARCH, the messages are + always returned using the sequence-set syntax. A sequence-set + representation may be more compact and can be used as is in a + subsequent command that accepts sequence-set. Note, the client + MUST NOT assume that messages/UIDs will be listed in any + particular order. + + If the SEARCH results in no matches, the server MUST NOT + include the ALL result option in the ESEARCH response; however, + it still MUST send the ESEARCH response. + + COUNT + Return number of the messages that satisfy the SEARCH criteria. + This result option MUST always be included in the ESEARCH + response. + + If one or more result options described above are specified, the + extended SEARCH command MUST return a single ESEARCH response + [IMAPABNF], instead of the SEARCH response. + + An extended UID SEARCH command MUST cause an ESEARCH response with + the UID indicator present. + + Note that future extensions to this document can allow servers to + return multiple ESEARCH responses for a single extended SEARCH + command. These extensions will have to describe how results from + multiple ESEARCH responses are to be amalgamated. + + If the list of result options is empty, that requests the server to + return an ESEARCH response instead of the SEARCH response. This is + equivalent to "(ALL)". + + Example: C: A282 SEARCH RETURN (MIN COUNT) FLAGGED + SINCE 1-Feb-1994 NOT FROM "Smith" + S: * ESEARCH (TAG "A282") MIN 2 COUNT 3 + S: A282 OK SEARCH completed + + Example: C: A283 SEARCH RETURN () FLAGGED + SINCE 1-Feb-1994 NOT FROM "Smith" + S: * ESEARCH (TAG "A283") ALL 2,10:11 + S: A283 OK SEARCH completed + + The following example demonstrates finding the first unseen message + as returned in the UNSEEN response code on a successful SELECT + command: + + + + +Melnikov & Cridland Standards Track [Page 3] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + + Example: C: A284 SEARCH RETURN (MIN) UNSEEN + S: * ESEARCH (TAG "A284") MIN 4 + S: A284 OK SEARCH completed + + The following example demonstrates that if the ESEARCH UID indicator + is present, all data in the ESEARCH response is referring to UIDs; + for example, the MIN result specifier will be followed by a UID. + + Example: C: A285 UID SEARCH RETURN (MIN MAX) 1:5000 + S: * ESEARCH (TAG "A285") UID MIN 7 MAX 3800 + S: A285 OK SEARCH completed + + The following example demonstrates returning the number of deleted + messages: + + Example: C: A286 SEARCH RETURN (COUNT) DELETED + S: * ESEARCH (TAG "A286") COUNT 15 + S: A286 OK SEARCH completed + +3.2. Interaction with CONDSTORE extension + + When the server supports both the ESEARCH and the CONDSTORE + [CONDSTORE] extension, and the client requests one or more result + option described in section 3.1 together with the MODSEQ search + criterion in the same SEARCH/UID SEARCH command, then the server MUST + return the ESEARCH response containing the MODSEQ result option + (described in the following paragraph) instead of the extended SEARCH + response described in section 3.5 of [CONDSTORE]. + + If the SEARCH/UID SEARCH command contained a single MIN or MAX result + option, the MODSEQ result option contains the mod-sequence for the + found message. If the SEARCH/UID SEARCH command contained both MIN + and MAX result options and no ALL/COUNT option, the MODSEQ result + option contains the highest mod-sequence for the two returned + messages. Otherwise the MODSEQ result option contains the highest + mod-sequence for all messages being returned. + + Example: The following example demonstrates how Example 15 from + [CONDSTORE] would look in the presence of one or more result option: + + C: a1 SEARCH RETURN (MIN) MODSEQ "/flags/\\draft" + all 620162338 + S: * ESEARCH (TAG "a1") MIN 2 MODSEQ 917162488 + S: a1 OK Search complete + + C: a2 SEARCH RETURN (MAX) MODSEQ "/flags/\\draft" + all 620162338 + S: * ESEARCH (TAG "a2") MAX 23 MODSEQ 907162321 + + + +Melnikov & Cridland Standards Track [Page 4] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + + S: a2 OK Search complete + + C: a3 SEARCH RETURN (MIN MAX) MODSEQ "/flags/\\draft" + all 620162338 + S: * ESEARCH (TAG "a3") MIN 2 MAX 23 MODSEQ 917162488 + S: a3 OK Search complete + + C: a4 SEARCH RETURN (MIN COUNT) MODSEQ "/flags/\\draft" + all 620162338 + S: * ESEARCH (TAG "a4") MIN 2 COUNT 10 MODSEQ 917162500 + S: a4 OK Search complete + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + Non-terminals referenced but not defined below are as defined by + [IMAP4], [CONDSTORE], or [IMAPABNF]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lowercase characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + capability =/ "ESEARCH" + + search-return-data = "MIN" SP nz-number / + "MAX" SP nz-number / + "ALL" SP sequence-set / + "COUNT" SP number + ;; conforms to the generic + ;; search-return-data syntax defined + ;; in [IMAPABNF] + + search-return-opt = "MIN" / "MAX" / "ALL" / "COUNT" + ;; conforms to generic search-return-opt + ;; syntax defined in [IMAPABNF] + + When the CONDSTORE [CONDSTORE] IMAP extension is also supported, + the ABNF is updated as follows: + + search-return-data =/ "MODSEQ" SP mod-sequence-value + ;; mod-sequence-value is defined + ;; in [CONDSTORE] + + + + + + +Melnikov & Cridland Standards Track [Page 5] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + +5. Security Considerations + + In the general case, the IMAP SEARCH/UID SEARCH commands can be CPU + and/or IO intensive, and are seen by some as a potential attack point + for denial of service attacks, so some sites/implementations even + disable them entirely. This is quite unfortunate, as SEARCH command + is one of the best examples demonstrating IMAP advantage over POP3. + + The ALL and COUNT return options don't change how SEARCH is working + internally; they only change how information about found messages is + returned. MIN and MAX SEARCH result options described in this + document can lighten the load on IMAP servers that choose to optimize + SEARCHes containing only one or both of them. + + It is believed that this extension doesn't raise any additional + security concerns not already discussed in [IMAP4]. + +6. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track RFC + or an IESG-approved experimental RFC. The registry is currently + located at . + + This document defines the ESEARCH IMAP capability, which IANA added + to the registry. + +7. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [ABNF] Crocker, D. (Ed.) and P. Overell , "Augmented BNF for + Syntax Specifications: ABNF", RFC 4234, October 2005. + + [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006.. + + [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for Conditional + STORE", RFC 4551, June 2006. + +8. Acknowledgments + + Thanks to Michael Wener, Arnt Gulbrandsen, Cyrus Daboo, Mark Crispin, + and Pete Maclean for comments and corrections. + + + + +Melnikov & Cridland Standards Track [Page 6] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex, TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + Dave A. Cridland + Inventure Systems Limited + + EMail: dave.cridland@inventuresystems.co.uk + URL: http://invsys.co.uk/dave/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Cridland Standards Track [Page 7] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST, + AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT + THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY + IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR + PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + +Melnikov & Cridland Standards Track [Page 8] + diff --git a/docs/rfcs/rfc4978.IMAP_Compress_extension.txt b/docs/rfcs/rfc4978.IMAP_Compress_extension.txt new file mode 100644 index 0000000..14b56b6 --- /dev/null +++ b/docs/rfcs/rfc4978.IMAP_Compress_extension.txt @@ -0,0 +1,507 @@ + + + + + + +Network Working Group A. Gulbrandsen +Request for Comments: 4978 Oryx Mail Systems GmbH +Category: Standards Track August 2007 + + + The IMAP COMPRESS Extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + The COMPRESS extension allows an IMAP connection to be effectively + and efficiently compressed. + + Table of Contents + + 1. Introduction and Overview .......................................2 + 2. Conventions Used in This Document ...............................2 + 3. The COMPRESS Command ............................................3 + 4. Compression Efficiency ..........................................4 + 5. Formal Syntax ...................................................6 + 6. Security Considerations .........................................6 + 7. IANA Considerations .............................................6 + 8. Acknowledgements ................................................7 + 9. References ......................................................7 + 9.1. Normative References .......................................7 + 9.2. Informative References .....................................7 + + + + + + + + + + + + + + + + + + +Gulbrandsen Standards Track [Page 1] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + +1. Introduction and Overview + + A server which supports the COMPRESS extension indicates this with + one or more capability names consisting of "COMPRESS=" followed by a + supported compression algorithm name as described in this document. + + The goal of COMPRESS is to reduce the bandwidth usage of IMAP. + + Compared to PPP compression (see [RFC1962]) and modem-based + compression (see [MNP] and [V42BIS]), COMPRESS offers much better + compression efficiency. COMPRESS can be used together with Transport + Security Layer (TLS) [RFC4346], Simple Authentication and Security + layer (SASL) encryption, Virtual Private Networks (VPNs), etc. + Compared to TLS compression [RFC3749], COMPRESS has the following + (dis)advantages: + + - COMPRESS can be implemented easily both by IMAP servers and + clients. + + - IMAP COMPRESS benefits from an intimate knowledge of the IMAP + protocol's state machine, allowing for dynamic and aggressive + optimization of the underlying compression algorithm's parameters. + + - When the TLS layer implements compression, any protocol using that + layer can transparently benefit from that compression (e.g., SMTP + and IMAP). COMPRESS is specific to IMAP. + + In order to increase interoperation, it is desirable to have as few + different compression algorithms as possible, so this document + specifies only one. The DEFLATE algorithm (defined in [RFC1951]) is + standard, widely available and fairly efficient, so it is the only + algorithm defined by this document. + + In order to increase interoperation, IMAP servers that advertise this + extension SHOULD also advertise the TLS DEFLATE compression mechanism + as defined in [RFC3749]. IMAP clients MAY use either COMPRESS or TLS + compression, however, if the client and server support both, it is + RECOMMENDED that the client choose TLS compression. + + The extension adds one new command (COMPRESS) and no new responses. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + Formal syntax is defined by [RFC4234] as modified by [RFC3501]. + + + +Gulbrandsen Standards Track [Page 2] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + In the examples, "C:" and "S:" indicate lines sent by the client and + server respectively. "[...]" denotes elision. + +3. The COMPRESS Command + + Arguments: Name of compression mechanism: "DEFLATE". + + Responses: None + + Result: OK The server will compress its responses and expects the + client to compress its commands. + NO Compression is already active via another layer. + BAD Command unknown, invalid or unknown argument, or COMPRESS + already active. + + The COMPRESS command instructs the server to use the named + compression mechanism ("DEFLATE" is the only one defined) for all + commands and/or responses after COMPRESS. + + The client MUST NOT send any further commands until it has seen the + result of COMPRESS. If the response was OK, the client MUST compress + starting with the first command after COMPRESS. If the server + response was BAD or NO, the client MUST NOT turn on compression. + + If the server responds NO because it knows that the same mechanism is + active already (e.g., because TLS has negotiated the same mechanism), + it MUST send COMPRESSIONACTIVE as resp-text-code (see [RFC3501], + Section 7.1), and the resp-text SHOULD say which layer compresses. + + If the server issues an OK response, the server MUST compress + starting immediately after the CRLF which ends the tagged OK + response. (Responses issued by the server before the OK response + will, of course, still be uncompressed.) If the server issues a BAD + or NO response, the server MUST NOT turn on compression. + + For DEFLATE (as for many other compression mechanisms), the + compressor can trade speed against quality. When decompressing there + isn't much of a tradeoff. Consequently, the client and server are + both free to pick the best reasonable rate of compression for the + data they send. + + When COMPRESS is combined with TLS (see [RFC4346]) or SASL (see + [RFC4422]) security layers, the sending order of the three extensions + MUST be first COMPRESS, then SASL, and finally TLS. That is, before + data is transmitted it is first compressed. Second, if a SASL + security layer has been negotiated, the compressed data is then + signed and/or encrypted accordingly. Third, if a TLS security layer + has been negotiated, the data from the previous step is signed and/or + + + +Gulbrandsen Standards Track [Page 3] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + encrypted accordingly. When receiving data, the processing order + MUST be reversed. This ensures that before sending, data is + compressed before it is encrypted, independent of the order in which + the client issues COMPRESS, AUTHENTICATE, and STARTTLS. + + The following example illustrates how commands and responses are + compressed during a simple login sequence: + + S: * OK [CAPABILITY IMAP4REV1 STARTTLS COMPRESS=DEFLATE] + C: a starttls + S: a OK TLS active + + From this point on, everything is encrypted. + + C: b login arnt tnra + S: b OK Logged in as arnt + C: c compress deflate + S: d OK DEFLATE active + + From this point on, everything is compressed before being + encrypted. + + The following example demonstrates how a server may refuse to + compress twice: + + S: * OK [CAPABILITY IMAP4REV1 STARTTLS COMPRESS=DEFLATE] + [...] + C: c compress deflate + S: c NO [COMPRESSIONACTIVE] DEFLATE active via TLS + +4. Compression Efficiency + + This section is informative, not normative. + + IMAP poses some unusual problems for a compression layer. + + Upstream is fairly simple. Most IMAP clients send the same few + commands again and again, so any compression algorithm that can + exploit repetition works efficiently. The APPEND command is an + exception; clients that send many APPEND commands may want to + surround large literals with flushes in the same way as is + recommended for servers later in this section. + + Downstream has the unusual property that several kinds of data are + sent, confusing all dictionary-based compression algorithms. + + + + + + +Gulbrandsen Standards Track [Page 4] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + One type is IMAP responses. These are highly compressible; zlib + using its least CPU-intensive setting compresses typical responses to + 25-40% of their original size. + + Another type is email headers. These are equally compressible, and + benefit from using the same dictionary as the IMAP responses. + + A third type is email body text. Text is usually fairly short and + includes much ASCII, so the same compression dictionary will do a + good job here, too. When multiple messages in the same thread are + read at the same time, quoted lines etc. can often be compressed + almost to zero. + + Finally, attachments (non-text email bodies) are transmitted, either + in binary form or encoded with base-64. + + When attachments are retrieved in binary form, DEFLATE may be able to + compress them, but the format of the attachment is usually not IMAP- + like, so the dictionary built while compressing IMAP does not help. + The compressor has to adapt its dictionary from IMAP to the + attachment's format, and then back. A few file formats aren't + compressible at all using deflate, e.g., .gz, .zip, and .jpg files. + + When attachments are retrieved in base-64 form, the same problems + apply, but the base-64 encoding adds another problem. 8-bit + compression algorithms such as deflate work well on 8-bit file + formats, however base-64 turns a file into something resembling 6-bit + bytes, hiding most of the 8-bit file format from the compressor. + + When using the zlib library (see [RFC1951]), the functions + deflateInit2(), deflate(), inflateInit2(), and inflate() suffice to + implement this extension. The windowBits value must be in the range + -8 to -15, or else deflateInit2() uses the wrong format. + deflateParams() can be used to improve compression rate and resource + use. The Z_FULL_FLUSH argument to deflate() can be used to clear the + dictionary (the receiving peer does not need to do anything). + + A client can improve downstream compression by implementing BINARY + (defined in [RFC3516]) and using FETCH BINARY instead of FETCH BODY. + In the author's experience, the improvement ranges from 5% to 40% + depending on the attachment being downloaded. + + A server can improve downstream compression if it hints to the + compressor that the data type is about to change strongly, e.g., by + sending a Z_FULL_FLUSH at the start and end of large non-text + literals (before and after '*CHAR8' in the definition of literal in + RFC 3501, page 86). Small literals are best left alone. A possible + boundary is 5k. + + + +Gulbrandsen Standards Track [Page 5] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + A server can improve the CPU efficiency both of the server and the + client if it adjusts the compression level (e.g., using the + deflateParams() function in zlib) at these points, to avoid trying to + compress incompressible attachments. A very simple strategy is to + change the level to 0 at the start of a literal provided the first + two bytes are either 0x1F 0x8B (as in deflate-compressed files) or + 0xFF 0xD8 (JPEG), and to keep it at 1-5 the rest of the time. More + complex strategies are possible. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC4234]. This syntax augments + the grammar specified in [RFC3501]. [RFC4234] defines SP and + [RFC3501] defines command-auth, capability, and resp-text-code. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + command-auth =/ compress + + compress = "COMPRESS" SP algorithm + + capability =/ "COMPRESS=" algorithm + ;; multiple COMPRESS capabilities allowed + + algorithm = "DEFLATE" + + resp-text-code =/ "COMPRESSIONACTIVE" + + Note that due the syntax of capability names, future algorithm names + must be atoms. + +6. Security Considerations + + As for TLS compression [RFC3749]. + +7. IANA Considerations + + The IANA has added COMPRESS=DEFLATE to the list of IMAP capabilities. + + + + + + + + + +Gulbrandsen Standards Track [Page 6] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + +8. Acknowledgements + + Eric Burger, Dave Cridland, Tony Finch, Ned Freed, Philip Guenther, + Randall Gellens, Tony Hansen, Cullen Jennings, Stephane Maes, Alexey + Melnikov, Lyndon Nerenberg, and Zoltan Ordogh have all helped with + this document. + + The author would also like to thank various people in the rooms at + meetings, whose help is real, but not reflected in the author's + mailbox. + +9. References + +9.1. Normative References + + [RFC1951] Deutsch, P., "DEFLATE Compressed Data Format Specification + version 1.3", RFC 1951, May 1996. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + +9.2. Informative References + + [RFC1962] Rand, D., "The PPP Compression Control Protocol (CCP)", + RFC 1962, June 1996. + + [RFC3516] Nerenberg, L., "IMAP4 Binary Content Extension", RFC 3516, + April 2003. + + [RFC3749] Hollenbeck, S., "Transport Layer Security Protocol + Compression Methods", RFC 3749, May 2004. + + [RFC4346] Dierks, T. and E. Rescorla, "The Transport Layer Security + (TLS) Protocol Version 1.1", RFC 4346, April 2006. + + [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication and + Security Layer (SASL)", RFC 4422, June 2006. + + [V42BIS] ITU, "V.42bis: Data compression procedures for data + circuit-terminating equipment (DCE) using error correction + procedures", http://www.itu.int/rec/T-REC-V.42bis, January + 1990. + + + +Gulbrandsen Standards Track [Page 7] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + [MNP] Gilbert Held, "The Complete Modem Reference", Second + Edition, Wiley Professional Computing, ISBN 0-471-00852-4, + May 1994. + +Author's Address + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + Fax: +49 89 4502 9758 + EMail: arnt@oryx.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gulbrandsen Standards Track [Page 8] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2007). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Gulbrandsen Standards Track [Page 9] + diff --git a/docs/rfcs/rfc5032.IMAP_WITHIN_Search_extension.txt b/docs/rfcs/rfc5032.IMAP_WITHIN_Search_extension.txt new file mode 100644 index 0000000..f8e4895 --- /dev/null +++ b/docs/rfcs/rfc5032.IMAP_WITHIN_Search_extension.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group E. Burger, Ed. +Request for Comments: 5032 BEA Systems, Inc. +Updates: 3501 September 2007 +Category: Standards Track + + + WITHIN Search Extension to the IMAP Protocol + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + This document describes the WITHIN extension to IMAP SEARCH. IMAP + SEARCH returns messages whose internal date is within or outside a + specified interval. The mechanism described here, OLDER and YOUNGER, + differs from BEFORE and SINCE in that the client specifies an + interval, rather than a date. WITHIN is useful for persistent + searches where either the device does not have the capacity to + perform the search at regular intervals or the network is of limited + bandwidth and thus there is a desire to reduce network traffic from + sending repeated requests and redundant responses. + +1. Introduction + + This extension exposes two new search keys, OLDER and YOUNGER, each + of which takes a non-zero integer argument corresponding to a time + interval in seconds. The server calculates the time of interest by + subtracting the time interval the client presents from the current + date and time of the server. The server then either returns messages + older or younger than the resultant time and date, depending on the + search key used. + +1.1. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + + + + + +Burger Standards Track [Page 1] + +RFC 5032 Search Within September 2007 + + + When describing the general syntax, we omit some definitions, as RFC + 3501 [RFC3501] defines them. + +2. Protocol Operation + + An IMAP4 server that supports the capability described here MUST + return "WITHIN" as one of the server supported capabilities in the + CAPABILITY command. + + For both the OLDER and YOUNGER search keys, the server calculates a + target date and time by subtracting the interval, specified in + seconds, from the current date and time of the server. The server + then compares the target time with the INTERNALDATE of the message, + as specified in IMAP [RFC3501]. For OLDER, messages match if the + INTERNALDATE is less recent than or equal to the target time. For + YOUNGER, messages match if the INTERNALDATE is more recent than or + equal to the target time. + + Both OLDER and YOUNGER searches always result in exact matching, to + the resolution of a second. However, if one is doing a dynamic + evaluation, for example, in a context [CONTEXT], one needs to be + aware that the server might perform the evaluation periodically. + Thus, the server may delay the updates. Clients MUST be aware that + dynamic search results may not reflect the current state of the + mailbox. If the client needs a search result that reflects the + current state of the mailbox, we RECOMMEND that the client issue a + new search. + +3. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation. Elements not defined here can be found in the + formal syntax of ABNF [RFC4234] and IMAP [RFC3501]. + + This document extends RFC 3501 [RFC3501] with two new search keys: + OLDER and YOUNGER . + + search-key =/ ( "OLDER" / "YOUNGER" ) SP nz-number + ; search-key defined in RFC 3501 + +4. Example + + C: a1 SEARCH UNSEEN YOUNGER 259200 + S: a1 * SEARCH 4 8 15 16 23 42 + + Search for all unseen messages within the past 3 days, or 259200 + seconds, according to the server's current time. + + + + +Burger Standards Track [Page 2] + +RFC 5032 Search Within September 2007 + + +5. Security Considerations + + The WITHIN extension does not raise any security considerations that + are not present in the base protocol. Considerations are the same as + for IMAP [RFC3501]. + +6. IANA Considerations + + Per the IMAP RFC [RFC3501], registration of a new IMAP capability in + the IMAP Capability registry requires the publication of a standards- + track RFC or an IESG approved experimental RFC. The registry is + currently located at + . This + standards-track document defines the WITHIN IMAP capability. IANA + has added this extension to the IANA IMAP Capability registry. + +7. References + +7.1. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, BCP 14, March 1997. + + [RFC3501] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [RFC4234] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + +7.2. Informative References + + [CONTEXT] Melnikov, D. and C. King, "Contexts for IMAP4", Work + in Progress, May 2006. + + + + + + + + + + + + + + + + + + +Burger Standards Track [Page 3] + +RFC 5032 Search Within September 2007 + + +Appendix A. Contributors + + Stephane Maes and Ray Cromwell wrote the original version of this + document as part of P-IMAP, as well as the first versions for the + IETF. From an attribution perspective, they are clearly authors. + +Appendix B. Acknowledgements + + The authors want to thank all who have contributed key insight and + who have extensively reviewed and discussed the concepts of LPSEARCH. + They also thank the authors of its early introduction in P-IMAP. + + We also want to give a special thanks to Arnt Gilbrandsen, Ken + Murchison, Zoltan Ordogh, and most especially Dave Cridland for their + review and suggestions. A special thank you goes to Alexey Melnikov + for his choice submission of text. + +Author's Address + + Eric W. Burger (editor) + BEA Systems, Inc. + USA + + EMail: eric.burger@bea.com + URI: http://www.standardstrack.com + + + + + + + + + + + + + + + + + + + + + + + + + + +Burger Standards Track [Page 4] + +RFC 5032 Search Within September 2007 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2007). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Burger Standards Track [Page 5] + diff --git a/docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt b/docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt new file mode 100644 index 0000000..13bbbf7 --- /dev/null +++ b/docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt @@ -0,0 +1,395 @@ + + + + + + +Network Working Group A. Gulbrandsen, Ed. +Request for Comments: 5161 Oryx Mail Systems GmbH +Category: Standards Track A. Melnikov, Ed. + Isode Limited + March 2008 + + + The IMAP ENABLE Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + Most IMAP extensions are used by the client when it wants to and the + server supports it. However, a few extensions require the server to + know whether a client supports that extension. The ENABLE extension + allows an IMAP client to say which extensions it supports. + +1. Overview + + Several IMAP extensions allow the server to return unsolicited + responses specific to these extensions in certain circumstances. + However, servers cannot send those unsolicited responses until they + know that the clients support such extensions and thus won't choke on + the extension response data. + + Up until now, extensions have typically stated that a server cannot + send the unsolicited responses until after the client has used a + command with the extension data (i.e., at that point the server knows + the client is aware of the extension). CONDSTORE ([RFC4551]), + ANNOTATE ([ANNOTATE]), and some extensions under consideration at the + moment use various commands to enable server extensions. For + example, CONDSTORE uses a SELECT or FETCH parameter, and ANNOTATE + uses a side effect of FETCH. + + The ENABLE extension provides an explicit indication from the client + that it supports particular extensions. This is done using a new + ENABLE command. + + An IMAP server that supports ENABLE advertises this by including the + word ENABLE in its capability list. + + + + +Gulbrandsen & Melnikov Standards Track [Page 1] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + + Most IMAP extensions do not require the client to enable the + extension in any way. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + Formal syntax is defined by [RFC5234] and [RFC3501]. + + Example lines prefaced by "C:" are sent by the client and ones + prefaced by "S:" by the server. The five characters [...] means that + something has been elided. + +3. Protocol Changes + +3.1. The ENABLE Command + + Arguments: capability names + + Result: OK: Relevant capabilities enabled + BAD: No arguments, or syntax error in an argument + + The ENABLE command takes a list of capability names, and requests the + server to enable the named extensions. Once enabled using ENABLE, + each extension remains active until the IMAP connection is closed. + For each argument, the server does the following: + + - If the argument is not an extension known to the server, the server + MUST ignore the argument. + + - If the argument is an extension known to the server, and it is not + specifically permitted to be enabled using ENABLE, the server MUST + ignore the argument. (Note that knowing about an extension doesn't + necessarily imply supporting that extension.) + + - If the argument is an extension that is supported by the server and + that needs to be enabled, the server MUST enable the extension for + the duration of the connection. At present, this applies only to + CONDSTORE ([RFC4551]). Note that once an extension is enabled, + there is no way to disable it. + + If the ENABLE command is successful, the server MUST send an untagged + ENABLED response (see Section 3.2). + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 2] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + + Clients SHOULD only include extensions that need to be enabled by the + server. At the time of publication, CONDSTORE is the only such + extension (i.e., ENABLE CONDSTORE is an additional "CONDSTORE + enabling command" as defined in [RFC4551]). Future RFCs may add to + this list. + + The ENABLE command is only valid in the authenticated state (see + [RFC3501]), before any mailbox is selected. Clients MUST NOT issue + ENABLE once they SELECT/EXAMINE a mailbox; however, server + implementations don't have to check that no mailbox is selected or + was previously selected during the duration of a connection. + + The ENABLE command can be issued multiple times in a session. It is + additive; i.e., "ENABLE a b", followed by "ENABLE c" is the same as a + single command "ENABLE a b c". When multiple ENABLE commands are + issued, each corresponding ENABLED response SHOULD only contain + extensions enabled by the corresponding ENABLE command. + + There are no limitations on pipelining ENABLE. For example, it is + possible to send ENABLE and then immediately SELECT, or a LOGIN + immediately followed by ENABLE. + + The server MUST NOT change the CAPABILITY list as a result of + executing ENABLE; i.e., a CAPABILITY command issued right after an + ENABLE command MUST list the same capabilities as a CAPABILITY + command issued before the ENABLE command. This is demonstrated in + the following example: + + C: t1 CAPABILITY + S: * CAPABILITY IMAP4rev1 ID LITERAL+ ENABLE X-GOOD-IDEA + S: t1 OK foo + C: t2 ENABLE CONDSTORE X-GOOD-IDEA + S: * ENABLED X-GOOD-IDEA + S: t2 OK foo + C: t3 CAPABILITY + S: * CAPABILITY IMAP4rev1 ID LITERAL+ ENABLE X-GOOD-IDEA + S: t3 OK foo again + + In the following example, the client enables CONDSTORE: + + C: a1 ENABLE CONDSTORE + S: * ENABLED CONDSTORE + S: a1 OK Conditional Store enabled + + + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 3] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + +3.2. The ENABLED Response + + Contents: capability listing + + The ENABLED response occurs as a result of an ENABLE command. The + capability listing contains a space-separated listing of capability + names that the server supports and that were successfully enabled. + The ENABLED response may contain no capabilities, which means that no + extensions listed by the client were successfully enabled. + +3.3. Note to Designers of Extensions That May Use the ENABLE Command + + Designers of IMAP extensions are discouraged from creating extensions + that require ENABLE unless there is no good alternative design. + Specifically, extensions that cause potentially incompatible behavior + changes to deployed server responses (and thus benefit from ENABLE) + have a higher complexity cost than extensions that do not. + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234] including the core + rules in Appendix B.1. [RFC3501] defines the non-terminals + "capability" and "command-any". + + Except as noted otherwise, all alphabetic characters are + case-insensitive. The use of upper or lower case characters to + define token strings is for editorial clarity only. Implementations + MUST accept these strings in a case-insensitive fashion. + + capability =/ "ENABLE" + + command-any =/ "ENABLE" 1*(SP capability) + + response-data =/ "*" SP enable-data CRLF + + enable-data = "ENABLED" *(SP capability) + +5. Security Considerations + + It is believed that this extension doesn't add any security + considerations that are not already present in the base IMAP protocol + [RFC3501]. + +6. IANA Considerations + + The IANA has added ENABLE to the IMAP4 Capabilities Registry. + + + + +Gulbrandsen & Melnikov Standards Track [Page 4] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + +7. Acknowledgments + + The editors would like to thank Randy Gellens, Chris Newman, Peter + Coates, Dave Cridland, Mark Crispin, Ned Freed, Dan Karp, Cyrus + Daboo, Ken Murchison, and Eric Burger for comments and corrections. + However, this doesn't necessarily mean that they endorse this + extension, agree with all details, or are responsible for errors + introduced by the editors. + +8. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional + STORE Operation or Quick Flag Changes Resynchronization", + RFC 4551, June 2006. + +9. Informative References + + [ANNOTATE] Daboo, C. and R. Gellens, "IMAP ANNOTATE Extension", Work + in Progress, August 2006. + + + + + + + + + + + + + + + + + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 5] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + +Editors' Addresses + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + Fax: +49 89 4502 9758 + EMail: arnt@oryx.com + + + Alexey Melnikov + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 6] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 7] + diff --git a/docs/rfcs/rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt b/docs/rfcs/rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt new file mode 100644 index 0000000..305c54f --- /dev/null +++ b/docs/rfcs/rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt @@ -0,0 +1,1291 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 5162 D. Cridland +Category: Standards Track Isode Ltd + C. Wilson + Nokia + March 2008 + + + IMAP4 Extensions for Quick Mailbox Resynchronization + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + This document defines an IMAP4 extension, which gives an IMAP client + the ability to quickly resynchronize any previously opened mailbox as + part of the SELECT command, without the need for server-side state or + additional client round-trips. This extension also introduces a new + response that allows for a more compact representation of a list of + expunged messages (and always includes the Unique Identifiers (UIDs) + expunged). + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov, et al. Standards Track [Page 1] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +Table of Contents + + 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 2 + 2. Requirements Notation . . . . . . . . . . . . . . . . . . . . 4 + 3. IMAP Protocol Changes . . . . . . . . . . . . . . . . . . . . 4 + 3.1. QRESYNC Parameter to SELECT/EXAMINE . . . . . . . . . . . 4 + 3.2. VANISHED UID FETCH Modifier . . . . . . . . . . . . . . . 8 + 3.3. EXPUNGE Command . . . . . . . . . . . . . . . . . . . . . 10 + 3.4. CLOSE Command . . . . . . . . . . . . . . . . . . . . . . 11 + 3.5. UID EXPUNGE Command . . . . . . . . . . . . . . . . . . . 11 + 3.6. VANISHED Response . . . . . . . . . . . . . . . . . . . . 12 + 3.7. CLOSED Response Code . . . . . . . . . . . . . . . . . . . 15 + 4. Server Implementation Considerations . . . . . . . . . . . . . 15 + 4.1. Server Implementations That Don't Store Extra State . . . 15 + 4.2. Server Implementations Storing Minimal State . . . . . . . 16 + 4.3. Additional State Required on the Server . . . . . . . . . 16 + 5. Updated Synchronization Sequence . . . . . . . . . . . . . . . 17 + 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 19 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 20 + 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 21 + 9. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 21 + 10. References . . . . . . . . . . . . . . . . . . . . . . . . . . 21 + 10.1. Normative References . . . . . . . . . . . . . . . . . . . 21 + 10.2. Informative References . . . . . . . . . . . . . . . . . . 22 + +1. Introduction and Overview + + The [CONDSTORE] extension gives a disconnected client the ability to + quickly resynchronize IMAP flag changes for previously seen messages. + This can be done using the CHANGEDSINCE FETCH modifier once a mailbox + is opened. In order for the client to discover which messages have + been expunged, the client still has to issue a UID FETCH or a UID + SEARCH command. This document defines an extension to [CONDSTORE] + that allows a reconnecting client to perform full resynchronization, + including discovery of expunged messages, in a single round-trip. + This extension also introduces a new response, VANISHED, that allows + for a more compact representation of a list of expunged messages. + + This extension can be useful for mobile clients that can experience + frequent disconnects caused by environmental factors (battery life, + signal strength, etc.). Such clients need a way to quickly reconnect + to the IMAP server, while minimizing delay experienced by the user as + well as the amount of traffic (and hence the expense) generated by + resynchronization. + + + + + + + +Melnikov, et al. Standards Track [Page 2] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + By extending the SELECT command to perform the additional + resynchronization, this also allows clients to reduce concurrent + connections to the IMAP server held purely for the sake of avoiding + the resynchronization. + + The quick resync IMAP extension is present if an IMAP4 server returns + "QRESYNC" as one of the supported capabilities to the CAPABILITY + command. + + Servers supporting this extension MUST implement and advertise + support for the [ENABLE] IMAP extension. Also, the presence of the + "QRESYNC" capability implies support for the [CONDSTORE] IMAP + extension even if the CONDSTORE capability isn't advertised. A + server compliant with this specification is REQUIREd to support + "ENABLE QRESYNC" and "ENABLE QRESYNC CONDSTORE" (which are "CONDSTORE + enabling commands", as defined in [CONDSTORE], and have identical + results), but there is no requirement for a compliant server to + support "ENABLE CONDSTORE" by itself. The "ENABLE QRESYNC"/"ENABLE + QRESYNC CONDSTORE" command also tells the server that it SHOULD start + sending VANISHED responses (see Section 3.6) instead of EXPUNGE + responses. This change remains in effect until the connection is + closed. + + For compatibility with clients that only support the [CONDSTORE] IMAP + extension, servers SHOULD advertise CONDSTORE in the CAPABILITY + response as well. + + A client making use of this extension MUST issue "ENABLE QRESYNC" + once it is authenticated. A server MUST respond with a tagged BAD + response if the QRESYNC parameter to the SELECT/EXAMINE command or + the VANISHED UID FETCH modifier is specified and the client hasn't + issued "ENABLE QRESYNC" in the current connection. + + This document puts additional requirements on a server implementing + the [CONDSTORE] extension. Each mailbox that supports persistent + storage of mod-sequences, i.e., for which the server has sent a + HIGHESTMODSEQ untagged OK response code on a successful SELECT/ + EXAMINE, MUST increment the per-mailbox mod-sequence when one or more + messages are expunged due to EXPUNGE, UID EXPUNGE or CLOSE; the + server MUST associate the incremented mod-sequence with the UIDs of + the expunged messages. + + A client that supports CONDSTORE but not this extension might + resynchronize a mailbox and discover that its HIGHESTMODSEQ has + increased from the value cached by the client. If the increase is + only due to messages having been expunged since the client last + synchronized, the client is likely to send a FETCH ... CHANGEDSINCE + command that returns no data. Thus, a client that supports CONDSTORE + + + +Melnikov, et al. Standards Track [Page 3] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + but not this extension might incur a penalty of an unneeded round- + trip when resynchronizing some mailboxes (those that have had + messages expunged but no flag changes since the last + synchronization). + + This extra round-trip is only incurred by clients that support + CONDSTORE but not this extension, and only when a mailbox has had + messages expunged but no flag changes to non-expunged messages. + Since CONDSTORE is a relatively new extension, it is thought likely + that clients that support it will also support this extension. + +2. Requirements Notation + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If a single "C:" or "S:" label applies to + multiple lines, then the line breaks between those lines are for + editorial clarity only and are not part of the actual protocol + exchange. The five characters [...] means that something has been + elided. + + Understanding of the IMAP message sequence numbers and UIDs and the + EXPUNGE response [RFC3501] is essential when reading this document. + +3. IMAP Protocol Changes + +3.1. QRESYNC Parameter to SELECT/EXAMINE + + The Quick Resynchronization parameter to SELECT/EXAMINE commands has + four arguments: + + o the last known UIDVALIDITY, + + o the last known modification sequence, + + o the optional set of known UIDs, and + + o an optional parenthesized list of known sequence ranges and their + corresponding UIDs. + + A server MUST respond with a tagged BAD response if the Quick + Resynchronization parameter to SELECT/EXAMINE command is specified + and the client hasn't issued "ENABLE QRESYNC" in the current + connection. + + + + +Melnikov, et al. Standards Track [Page 4] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Before opening the specified mailbox, the server verifies all + arguments for syntactic validity. If any parameter is not + syntactically valid, the server returns the tagged BAD response, and + the mailbox remains unselected. Once the check is done, the server + opens the mailbox as if no SELECT/EXAMINE parameters are specified + (this is subject to processing of other parameters as defined in + other extensions). In particular this means that the server MUST + send all untagged responses as specified in Sections 6.3.1 and 6.3.2 + of [RFC3501]. + + After that, the server checks the UIDVALIDITY value provided by the + client. If the provided UIDVALIDITY doesn't match the UIDVALIDITY + for the mailbox being opened, then the server MUST ignore the + remaining parameters and behave as if no dynamic message data + changed. The client can discover this situation by comparing the + UIDVALIDITY value returned by the server. This behavior allows the + client not to synchronize the mailbox or decide on the best + synchronization strategy. + + Example: Attempting to resynchronize INBOX, but the provided + UIDVALIDITY parameter doesn't match the current UIDVALIDITY + value. + + C: A02 SELECT INBOX (QRESYNC (67890007 20050715194045000 + 41,43:211,214:541)) + S: * 464 EXISTS + S: * 3 RECENT + S: * OK [UIDVALIDITY 3857529045] UIDVALIDITY + S: * OK [UIDNEXT 550] Predicted next UID + S: * OK [HIGHESTMODSEQ 90060128194045007] + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] Permanent flags + S: A02 OK [READ-WRITE] Sorry, UIDVALIDITY mismatch + + Modification Sequence and UID Parameters: + + A server that doesn't support the persistent storage of mod-sequences + for the mailbox MUST send the OK untagged response including the + NOMODSEQ response code with every successful SELECT or EXAMINE + command, as described in [CONDSTORE]. Such a server doesn't need to + remember mod-sequences for expunged messages in the mailbox. It MUST + ignore the remaining parameters and behave as if no dynamic message + data changed. + + If the provided UIDVALIDITY matches that of the selected mailbox, the + server then checks the last known modification sequence. + + + +Melnikov, et al. Standards Track [Page 5] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + The server sends the client any pending flag changes (using FETCH + responses that MUST contain UIDs) and expunges those that have + occurred in this mailbox since the provided modification sequence. + + If the list of known UIDs was also provided, the server should only + report flag changes and expunges for the specified messages. If the + client did not provide the list of UIDs, the server acts as if the + client has specified "1:", where is the mailbox's + UIDNEXT value minus 1. If the mailbox is empty and never had any + messages in it, then lack of the list of UIDs is interpreted as an + empty set of UIDs. + + Thus, the client can process just these pending events and need not + perform a full resynchronization. Without the message sequence + number matching information, the result of this step is semantically + equivalent to the client issuing: + tag1 UID FETCH "known-uids" (FLAGS) (CHANGEDSINCE + "mod-sequence-value" VANISHED) + + Example: + C: A03 SELECT INBOX (QRESYNC (67890007 + 90060115194045000 41,43:211,214:541)) + S: * OK [CLOSED] + S: * 314 EXISTS + S: * 15 RECENT + S: * OK [UIDVALIDITY 67890007] UIDVALIDITY + S: * OK [UIDNEXT 567] Predicted next UID + S: * OK [HIGHESTMODSEQ 90060115205545359] + S: * OK [UNSEEN 7] There are some unseen messages in the mailbox + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] Permanent flags + S: * VANISHED (EARLIER) 41,43:116,118,120:211,214:540 + S: * 49 FETCH (UID 117 FLAGS (\Seen \Answered) MODSEQ + (90060115194045001)) + S: * 50 FETCH (UID 119 FLAGS (\Draft $MDNSent) MODSEQ + (90060115194045308)) + S: ... + S: * 100 FETCH (UID 541 FLAGS (\Seen $Forwarded) MODSEQ + (90060115194045001)) + S: A03 OK [READ-WRITE] mailbox selected + + Message sequence match data: + + A client MAY provide a parenthesized list of a message sequence set + and the corresponding UID sets. Both MUST be provided in ascending + order. The server uses this data to restrict the range for which it + provides expunged message information. + + + +Melnikov, et al. Standards Track [Page 6] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Conceptually, the client provides a small sample of sequence numbers + for which it knows the corresponding UIDs. The server then compares + each sequence number and UID pair the client provides with the + current state of the mailbox. If a pair matches, then the client + knows of any expunges up to, and including, the message, and thus + will not include that range in the VANISHED response, even if the + "mod-sequence-value" provided by the client is too old for the server + to have data of when those messages were expunged. + + Thus, if the Nth message number in the first set in the list is 4, + and the Nth UID in the second set in the list is 8, and the mailbox's + fourth message has UID 8, then no UIDs equal to or less than 8 are + present in the VANISHED response. If the (N+1)th message number is + 12, and the (N+1)th UID is 24, and the (N+1)th message in the mailbox + has UID 25, then the lowest UID included in the VANISHED response + would be 9. + + In the following two examples, the server is unable to remember + expunges at all, and only UIDs with messages divisible by three are + present in the mailbox. In the first example, the client does not + use the fourth parameter; in the second, it provides it. This + example is somewhat extreme, but shows that judicious usage of the + sequence match data can save a substantial amount of bandwidth. + + Example: + C: A04 SELECT INBOX (QRESYNC (67890007 + 90060115194045000 1:29997)) + S: * 10003 EXISTS + S: * 5 RECENT + S: * OK [UIDVALIDITY 67890007] UIDVALIDITY + S: * OK [UIDNEXT 30013] Predicted next UID + S: * OK [HIGHESTMODSEQ 90060115205545359] + S: * OK [UNSEEN 7] There are some unseen messages in the mailbox + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] Permanent flags + S: * VANISHED (EARLIER) 1:2,4:5,7:8,10:11,13:14 [...] + 29998:29999,30001:30002,30004:30005,30007:30008 + S: * 9889 FETCH (UID 29667 FLAGS (\Seen \Answered) MODSEQ + (90060115194045027)) + S: * 9890 FETCH (UID 29670 FLAGS (\Draft $MDNSent) MODSEQ + (90060115194045028)) + S: ... + S: * 9999 FETCH (UID 29997 FLAGS (\Seen $Forwarded) MODSEQ + (90060115194045031)) + S: A04 OK [READ-WRITE] mailbox selected + + + + + +Melnikov, et al. Standards Track [Page 7] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Example: + C: B04 SELECT INBOX (QRESYNC (67890007 + 90060115194045000 1:29997 (5000,7500,9000,9990:9999 15000, + 22500,27000,29970,29973,29976,29979,29982,29985,29988,29991, + 29994,29997))) + S: * 10003 EXISTS + S: * 5 RECENT + S: * OK [UIDVALIDITY 67890007] UIDVALIDITY + S: * OK [UIDNEXT 30013] Predicted next UID + S: * OK [HIGHESTMODSEQ 90060115205545359] + S: * OK [UNSEEN 7] There are some unseen messages in the mailbox + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] Permanent flags + S: * VANISHED (EARLIER) 29998:29999,30001:30002,30004:30005,30007: + 30008 + S: * 9889 FETCH (UID 29667 FLAGS (\Seen \Answered) MODSEQ + (90060115194045027)) + S: * 9890 FETCH (UID 29670 FLAGS (\Draft $MDNSent) MODSEQ + (90060115194045028)) + S: ... + S: * 9999 FETCH (UID 29997 FLAGS (\Seen $Forwarded) MODSEQ + (90060115194045031)) + S: B04 OK [READ-WRITE] mailbox selected + +3.2. VANISHED UID FETCH Modifier + + [IMAPABNF] has extended the syntax of the FETCH and UID FETCH + commands to include an optional FETCH modifier. This document + defines a new UID FETCH modifier: VANISHED. + + Note, that the VANISHED UID FETCH modifier is NOT allowed with a + FETCH command. The server MUST return a tagged BAD response if this + response is specified as a modifier to the FETCH command. + + A server MUST respond with a tagged BAD response if the VANISHED UID + FETCH modifier is specified and the client hasn't issued "ENABLE + QRESYNC" in the current connection. + + The VANISHED UID FETCH modifier MUST only be specified together with + the CHANGEDSINCE UID FETCH modifier. + + The VANISHED UID FETCH modifier instructs the server to report those + messages from the UID set parameter that have been expunged and whose + associated mod-sequence is larger than the specified mod-sequence. + That is, the client requests to be informed of messages from the + specified set that were expunged since the specified mod-sequence. + Note that the mod-sequence(s) associated with these messages were + + + +Melnikov, et al. Standards Track [Page 8] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + updated when the messages were expunged (as described above). The + expunged messages are reported using the VANISHED response as + described in Section 3.6, which MUST contain the EARLIER tag. Any + VANISHED (EARLIER) responses MUST be returned before any FETCH + responses, as otherwise the client might get confused about how + message numbers map to UIDs. + + Note: A server that receives a mod-sequence smaller than , + where is the value of the smallest expunged mod-sequence + it remembers minus one, MUST behave as if it was requested to report + all expunged messages from the provided UID set parameter. + + Example 1: Without the VANISHED UID FETCH modifier, a CONDSTORE-aware + client [CONDSTORE] needs to issue separate commands to learn of flag + changes and expunged messages since the last synchronization: + + C: s100 UID FETCH 300:500 (FLAGS) (CHANGEDSINCE 12345) + S: * 1 FETCH (UID 404 MODSEQ (65402) FLAGS (\Seen)) + S: * 2 FETCH (UID 406 MODSEQ (75403) FLAGS (\Deleted)) + S: * 4 FETCH (UID 408 MODSEQ (29738) FLAGS ($NoJunk + $AutoJunk $MDNSent)) + S: s100 OK FETCH completed + C: s101 UID SEARCH 300:500 + S: * SEARCH 404 406 407 408 410 412 + S: s101 OK search completed + + Where 300 and 500 are the lowest and highest UIDs from client's + cache. The second SEARCH response tells the client that the messages + with UIDs 407, 410, and 412 are still present, but their flags + haven't changed since the specified modification sequence. + + Using the VANISHED UID FETCH modifier, it is sufficient to issue only + a single command: + + C: s100 UID FETCH 300:500 (FLAGS) (CHANGEDSINCE 12345 + VANISHED) + S: * VANISHED (EARLIER) 300:310,405,411 + S: * 1 FETCH (UID 404 MODSEQ (65402) FLAGS (\Seen)) + S: * 2 FETCH (UID 406 MODSEQ (75403) FLAGS (\Deleted)) + S: * 4 FETCH (UID 408 MODSEQ (29738) FLAGS ($NoJunk + $AutoJunk $MDNSent)) + S: s100 OK FETCH completed + + + + + + + + + +Melnikov, et al. Standards Track [Page 9] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +3.3. EXPUNGE Command + + Arguments: none + + Responses: untagged responses: EXPUNGE or VANISHED + + Result: OK - expunge completed + NO - expunge failure: can't expunge (e.g., permission denied) + BAD - command unknown or arguments invalid + + This section updates the definition of the EXPUNGE command described + in Section 6.4.3 of [RFC3501]. + + The EXPUNGE command permanently removes all messages that have the + \Deleted flag set from the currently selected mailbox. Before + returning an OK to the client, those messages that are removed are + reported using a VANISHED response or EXPUNGE responses. + + If the server is capable of storing modification sequences for the + selected mailbox, it MUST increment the per-mailbox mod-sequence if + at least one message was permanently removed due to the execution of + the EXPUNGE command. For each permanently removed message, the + server MUST remember the incremented mod-sequence and corresponding + UID. If at least one message got expunged, the server MUST send the + updated per-mailbox modification sequence using the HIGHESTMODSEQ + response code (defined in [CONDSTORE]) in the tagged OK response. + + Example: C: A202 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 5 EXPUNGE + S: * 8 EXPUNGE + S: A202 OK [HIGHESTMODSEQ 20010715194045319] expunged + + Note: In this example, messages 3, 4, 7, and 11 had the \Deleted flag + set. The first "* 3 EXPUNGE" reports message # 3 as expunged. The + second "* 3 EXPUNGE" reports message # 4 as expunged (the message + number got decremented due to the previous EXPUNGE response). See + the description of the EXPUNGE response in [RFC3501] for further + explanation. + + Note that if the server chooses to always send VANISHED responses + instead of EXPUNGE responses, the previous example might look like + this: + + Example: C: B202 EXPUNGE + S: * VANISHED 405,407,410,425 + S: B202 OK [HIGHESTMODSEQ 20010715194045319] expunged + + + +Melnikov, et al. Standards Track [Page 10] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Here messages with message numbers 3, 4, 7, and 11 have respective + UIDs 405, 407, 410, and 425. + +3.4. CLOSE Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - close completed, now in authenticated state + BAD - command unknown or arguments invalid + + This section updates the definition of the CLOSE command described in + Section 6.4.2 of [RFC3501]. + + The CLOSE command permanently removes all messages that have the + \Deleted flag set from the currently selected mailbox, and returns to + the authenticated state from the selected state. No untagged EXPUNGE + (or VANISHED) responses are sent. + + If the server is capable of storing modification sequences for the + selected mailbox, it MUST increment the per-mailbox mod-sequence if + at least one message was permanently removed due to the execution of + the CLOSE command. For each permanently removed message, the server + MUST remember the incremented mod-sequence and corresponding UID. If + at least one message got expunged, the server MUST send the updated + per-mailbox modification sequence using the HIGHESTMODSEQ response + code (defined in [CONDSTORE]) in the tagged OK response. + + Example: C: A202 CLOSE + S: A202 OK [HIGHESTMODSEQ 20010715194045319] done + +3.5. UID EXPUNGE Command + + Arguments: message set + + Responses: untagged responses: EXPUNGE or VANISHED + + Result: OK - expunge completed + NO - expunge failure: can't expunge (e.g., permission denied) + BAD - command unknown or arguments invalid + + This section updates the definition of the UID EXPUNGE command + described in Section 2.1 of [UIDPLUS]. Servers that implement both + [UIDPLUS] and QRESYNC extensions must implement UID EXPUNGE as + described in this section. + + + + + +Melnikov, et al. Standards Track [Page 11] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + The UID EXPUNGE command permanently removes from the currently + selected mailbox all messages that both have the \Deleted flag set + and have a UID that is included in the specified message set. If a + message either does not have the \Deleted flag set or has a UID that + is not included in the specified message set, it is not affected. + + This command is particularly useful for disconnected mode clients. + By using UID EXPUNGE instead of EXPUNGE when resynchronizing with the + server, the client can avoid inadvertently removing any messages that + have been marked as \Deleted by other clients between the time that + the client was last connected and the time the client resynchronizes. + + Before returning an OK to the client, those messages that are removed + are reported using a VANISHED response or EXPUNGE responses. + + If the server is capable of storing modification sequences for the + selected mailbox, it MUST increment the per-mailbox mod-sequence if + at least one message was permanently removed due to the execution of + the UID EXPUNGE command. For each permanently removed message, the + server MUST remember the incremented mod-sequence and corresponding + UID. If at least one message got expunged, the server MUST send the + updated per-mailbox modification sequence using the HIGHESTMODSEQ + response code (defined in [CONDSTORE]) in the tagged OK response. + + Example: C: . UID EXPUNGE 3000:3002 + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: . OK [HIGHESTMODSEQ 20010715194045319] Ok + + Note: In this example, at least messages with message numbers 3, 4, + and 5 (UIDs 3000 to 3002) had the \Deleted flag set. The first "* 3 + EXPUNGE" reports message # 3 as expunged. The second "* 3 EXPUNGE" + reports message # 4 as expunged (the message number got decremented + due to the previous EXPUNGE response). See the description of the + EXPUNGE response in [RFC3501] for further explanation. + +3.6. VANISHED Response + + Contents: an optional EARLIER tag + + list of UIDs + + The VANISHED response reports that the specified UIDs have been + permanently removed from the mailbox. This response is similar to + the EXPUNGE response [RFC3501]; however, it can return information + about multiple messages, and it returns UIDs instead of message + + + + +Melnikov, et al. Standards Track [Page 12] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + numbers. The first benefit saves bandwidth, while the second is more + convenient for clients that only use UIDs to access the IMAP server. + + The VANISHED response has the same restrictions on when it can be + sent as does the EXPUNGE response (see below). + + The VANISHED response has two forms. The first form contains the + EARLIER tag, which signifies that the response was caused by a UID + FETCH (VANISHED) or a SELECT/EXAMINE (QRESYNC) command. This + response is sent if the UID set parameter to the UID FETCH (VANISHED) + command includes UIDs of messages that are no longer in the mailbox. + When the client sees a VANISHED EARLIER response, it MUST NOT + decrement message sequence numbers for each successive message in the + mailbox. + + The second form doesn't contain the EARLIER tag and is described + below. Once a client has issued "ENABLE QRESYNC", the server SHOULD + use the VANISHED response without the EARLIER tag instead of the + EXPUNGE response. The server SHOULD continue using VANISHED in lieu + of EXPUNGE for the duration of the connection. In particular, this + affects the EXPUNGE [RFC3501] and UID EXPUNGE [UIDPLUS] commands, as + well as messages expunged in other connections. Such a VANISHED + response MUST NOT contain the EARLIER tag. + + A VANISHED response sent because of an EXPUNGE or UID EXPUNGE command + or because messages were expunged in other connections (i.e., the + VANISHED response without the EARLIER tag) also decrements the number + of messages in the mailbox; it is not necessary for the server to + send an EXISTS response with the new value. It also decrements + message sequence numbers for each successive message in the mailbox + (see the example at the end of this section). Note that a VANISHED + response caused by EXPUNGE, UID EXPUNGE, or messages expunged in + other connections SHOULD only contain UIDs for messages expunged + since the last VANISHED/EXPUNGE response sent for the currently + opened mailbox or since the mailbox was opened. That is, servers + SHOULD NOT send UIDs for previously expunged messages, unless + explicitly requested to do so by the UID FETCH (VANISHED) command. + + Note that client implementors must take care to properly decrement + the number of messages in the mailbox even if a server violates this + last SHOULD or repeats the same UID multiple times in the returned + UID set. In general, this means that a client using this extension + should either avoid using message numbers entirely, or have a + complete mapping of UIDs to message sequence numbers for the selected + mailbox. + + + + + + +Melnikov, et al. Standards Track [Page 13] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Because clients handle the two different forms of the VANISHED + response differently, servers MUST NOT report UIDs resulting from a + UID FETCH (VANISHED) or a SELECT/EXAMINE (QRESYNC) in the same + VANISHED response as UIDs of messages expunged now (i.e., messages + expunged in other connections). Instead, the server MUST send + separate VANISHED responses: one with the EARLIER tag and one + without. + + A VANISHED response MUST NOT be sent when no command is in progress, + nor while responding to a FETCH, STORE, or SEARCH command. This rule + is necessary to prevent a loss of synchronization of message sequence + numbers between client and server. A command is not "in progress" + until the complete command has been received; in particular, a + command is not "in progress" during the negotiation of command + continuation. + + Note: UID FETCH, UID STORE, and UID SEARCH are different commands + from FETCH, STORE, and SEARCH. A VANISHED response MAY be sent + during a UID command. However, the VANISHED response MUST NOT be + sent during a UID SEARCH command that contains message numbers in the + search criteria. + + The update from the VANISHED response MUST be recorded by the client. + + Example: Let's assume that there is the following mapping between + message numbers and UIDs in the currently selected mailbox (here "X" + marks messages with the \Deleted flag set, and "x" represents UIDs + which are not relevant for the example): + + Message numbers: 1 2 3 4 5 6 7 8 9 10 11 + UIDs: x 504 505 507 508 x 510 x x x 625 + \Deleted messages: X X X X + + In the presence of the extension defined in this document: + + C: A202 EXPUNGE + S: * VANISHED 505,507,510,625 + S: A202 OK EXPUNGE completed + + Without the QRESYNC extension, the same example might look like: + + C: A202 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 5 EXPUNGE + S: * 8 EXPUNGE + S: A202 OK EXPUNGE completed + + + + +Melnikov, et al. Standards Track [Page 14] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + (Continuing previous example) If subsequently messages with UIDs 504 + and 508 got marked as \Deleted: + + C: A210 EXPUNGE + S: * VANISHED 504,508 + S: A210 OK EXPUNGE completed + + i.e., the last VANISHED response only contains UIDs of messages + expunged since the previous VANISHED response. + +3.7. CLOSED Response Code + + The CLOSED response code has no parameters. A server implementing + the extension defined in this document MUST return the CLOSED + response code when the currently selected mailbox is closed + implicitly using the SELECT/EXAMINE command on another mailbox. The + CLOSED response code serves as a boundary between responses for the + previously opened mailbox (which was closed) and the newly selected + mailbox: all responses before the CLOSED response code relate to the + mailbox that was closed, and all subsequent responses relate to the + newly opened mailbox. + + There is no need to return the CLOSED response code on completion of + the CLOSE or the UNSELECT [UNSELECT] command (or similar) whose + purpose is to close the currently selected mailbox without opening a + new one. + +4. Server Implementation Considerations + + This section describes a minimalist implementation, a moderate + implementation, and an example of a full implementation. + +4.1. Server Implementations That Don't Store Extra State + + Strictly speaking, a server implementation that doesn't remember mod- + sequences associated with expunged messages can be considered + compliant with this specification. Such implementations return all + expunged messages specified in the UID set of the UID FETCH + (VANISHED) command every time, without paying attention to the + specified CHANGEDSINCE mod-sequence. Such implementations are + discouraged, as they can end up returning VANISHED responses that are + bigger than the result of a UID SEARCH command for the same UID set. + + Clients that use the message sequence match data can reduce the scope + of this VANISHED response substantially in the typical case where + expunges have not happened, or happen only toward the end of the + mailbox. + + + + +Melnikov, et al. Standards Track [Page 15] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +4.2. Server Implementations Storing Minimal State + + A server that stores the HIGHESTMODSEQ value at the time of the last + EXPUNGE can omit the VANISHED response when a client provides a + MODSEQ value that is equal to, or higher than, the current value of + this datum, that is, when there have been no EXPUNGEs. + + A client providing message sequence match data can reduce the scope + as above. In the case where there have been no expunges, the server + can ignore this data. + +4.3. Additional State Required on the Server + + When compared to the [CONDSTORE] extension, this extension requires + servers to store additional state associated with expunged messages. + Note that implementations are not required to store this state in + persistent storage; however, use of persistent storage is advisable. + + One possible way to correctly implement the extension described in + this document is to store a queue of pairs. + can be represented as a sequence of + pairs. + + When messages are expunged, one or more entries are added to the + queue tail. + + When the server receives a request to return messages expunged since + a given mod-sequence, it will search the queue from the tail (i.e., + going from the highest expunged mod-sequence to the lowest) until it + sees the first record with a mod-sequence less than or equal to the + given mod-sequence or it reaches the head of the queue. + + Note that indefinitely storing information about expunged messages + can cause storage and related problems for an implementation. In the + worst case, this could result in almost 64Gb of storage for each IMAP + mailbox. For example, consider an implementation that stores triples for each range of messages + expunged at the same time. Each triple requires 16 octets: 4 octets + for each of the two UIDs, and 8 octets for the mod-sequence. Assume + that there is a mailbox containing a single message with a UID of + 2**32-1 (the maximum possible UID value), where messages had + previously existed with UIDs starting at 1, and have been expunged + one at a time. For this mailbox alone, storage is required for the + triples <1, 1, modseq1>, <2, 2, modseq2>, ..., <2**32-2, 2**32-2, + modseq4294967294>. + + + + + + +Melnikov, et al. Standards Track [Page 16] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Hence, implementations are encouraged to adopt strategies to protect + against such storage problems, such as limiting the size of the queue + used to store mod-sequences for expunged messages and "expiring" + older records when this limit is reached. When the selected + implementation-specific queue limit is reached, the oldest record(s) + are deleted from the queue (note that such records are located at the + queue head). For all such "expired" records, the server needs to + store a single mod-sequence, which is the highest mod-sequence for + all "expired" expunged messages. + + Note that if the client provides the message sequence match data, + this can heavily reduce the data cost of sending a complete set of + missing UIDs; thus, reducing the problems for clients if a server is + unable to persist much of this queue. If the queue contains data + back to the requested mod-sequence, this data can be ignored. + + Also, note that if the UIDVALIDITY of the mailbox changes or if the + mailbox is deleted, then any state associated with expunged messages + doesn't need to be preserved and SHOULD be deleted. + +5. Updated Synchronization Sequence + + This section updates the description of optimized synchronization in + Section 6.1 of the [IMAP-DISC]. + + An advanced disconnected mail client should use the QRESYNC and + [CONDSTORE] extensions when they are supported by the server. The + client uses the value from the HIGHESTMODSEQ OK response code + received on mailbox opening to determine if it needs to + resynchronize. Once the synchronization is complete, it MUST cache + the received value (unless the mailbox UIDVALIDITY value has changed; + see below). The client MUST update its copy of the HIGHESTMODSEQ + value whenever the server sends a subsequent HIGHESTMODSEQ OK + response code. + + After completing a full synchronization, the client MUST also take + note of any unsolicited MODSEQ FETCH data items received from the + server. Whenever the client receives a tagged response to a command, + it calculates the highest value among all MODSEQ FETCH data items + received since the last tagged response. If this value is bigger + than the client's copy of the HIGHESTMODSEQ value, then the client + MUST use this value as its new HIGHESTMODSEQ value. + + Note: It is not safe to update the client's copy of the HIGHESTMODSEQ + value with a MODSEQ FETCH data item value as soon as it is received + because servers are not required to send MODSEQ FETCH data items in + increasing modseqence order. This can lead to the client missing + some changes in case of connectivity loss. + + + +Melnikov, et al. Standards Track [Page 17] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + When opening the mailbox for synchronization, the client uses the + QRESYNC parameter to the SELECT/EXAMINE command. The QRESYNC + parameter is followed by the UIDVALIDITY and mailbox HIGHESTMODSEQ + values, as known to the client. It can be optionally followed by the + set of UIDs, for example, if the client is only interested in partial + synchronization of the mailbox. The client may also transmit a list + containing its knowledge of message numbers. + + If the SELECT/EXAMINE command is successful, the client compares + UIDVALIDITY as described in step d)1) in Section 3 of the + [IMAP-DISC]. If the cached UIDVALIDITY value matches the one + returned by the server and the server also returns the HIGHESTMODSEQ + response code, then the server reports expunged messages and returns + flag changes for all messages specified by the client in the UID set + parameter (or for all messages in the mailbox, if the client omitted + the UID set parameter). At this point, the client is synchronized, + except for maybe the new messages. + + If upon a successful SELECT/EXAMINE (QRESYNC) command the client + receives a NOMODSEQ OK untagged response (instead of the + HIGHESTMODSEQ response code), it MUST remove the last known + HIGHESTMODSEQ value from its cache and follow the more general + instructions in Section 3 of the [IMAP-DISC]. + + At this point, the client is in sync with the server regarding old + messages. This client can now fetch information about new messages + (if requested by the user). + + Step d) ("Server-to-client synchronization") in Section 4 of the + [IMAP-DISC] in the presence of the QRESYNC & CONDSTORE extensions is + amended as follows: + + d) "Server-to-client synchronization" -- for each mailbox that + requires synchronization, do the following: + + 1a) Check the mailbox UIDVALIDITY (see Section 4.1 of the [IMAP-DISC] + for more details) after issuing SELECT/EXAMINE (QRESYNC) command. + + If the UIDVALIDITY value returned by the server differs, the + client MUST + + * empty the local cache of that mailbox; + + * "forget" the cached HIGHESTMODSEQ value for the mailbox; + + + + + + + +Melnikov, et al. Standards Track [Page 18] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + * remove any pending "actions" which refer to UIDs in that + mailbox. Note, this doesn't affect actions performed on + client generated fake UIDs (see Section 5 of the + [IMAP-DISC]); + + 2) Fetch the current "descriptors"; + + I) Discover new messages. + + 3) Fetch the bodies of any "interesting" messages that the client + doesn't already have. + + Example: The UIDVALIDITY value is the same, but the HIGHESTMODSEQ + value has changed on the server while the client was + offline: + + C: A142 SELECT INBOX (QRESYNC (3857529045 20010715194032001 1:198)) + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 201] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [HIGHESTMODSEQ 20010715194045007] + S: * VANISHED (EARLIER) 1:5,7:8,10:15 + S: * 2 FETCH (UID 6 MODSEQ (20010715205008000) + FLAGS (\Deleted)) + S: * 5 FETCH (UID 9 MODSEQ (20010715195517000) + FLAGS ($NoJunk $AutoJunk $MDNSent)) + ... + S: A142 OK [READ-WRITE] SELECT completed + +6. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + Non-terminals referenced but not defined below are as defined by + [RFC3501], [CONDSTORE], or [IMAPABNF]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + + + + + +Melnikov, et al. Standards Track [Page 19] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + capability =/ "QRESYNC" + + select-param = "QRESYNC" SP "(" uidvalidity SP + mod-sequence-value [SP known-uids] + [SP seq-match-data] ")" + ;; conforms to the generic select-param + ;; syntax defined in [IMAPABNF] + + seq-match-data = "(" known-sequence-set SP known-uid-set ")" + + uidvalidity = nz-number + + known-uids = sequence-set + ;; sequence of UIDs, "*" is not allowed + + known-sequence-set = sequence-set + ;; set of message numbers corresponding to + ;; the UIDs in known-uid-set, in ascending order. + ;; * is not allowed. + + known-uid-set = sequence-set + ;; set of UIDs corresponding to the messages in + ;; known-sequence-set, in ascending order. + ;; * is not allowed. + + message-data =/ expunged-resp + + expunged-resp = "VANISHED" [SP "(EARLIER)"] SP known-uids + + rexpunges-fetch-mod = "VANISHED" + ;; VANISHED UID FETCH modifier conforms + ;; to the fetch-modifier syntax + ;; defined in [IMAPABNF]. It is only + ;; allowed in the UID FETCH command. + + resp-text-code =/ "CLOSED" + +7. Security Considerations + + As always, it is important to thoroughly test clients and servers + implementing this extension, as it changes how the server reports + expunged messages to the client. + + Security considerations relevant to [CONDSTORE] are relevant to this + extension. + + This document doesn't raise any new security concerns not already + raised by [CONDSTORE] or [RFC3501]. + + + +Melnikov, et al. Standards Track [Page 20] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +8. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + This document defines the QRESYNC IMAP capability. IANA has added + this capability to the registry. + +9. Acknowledgments + + Thanks to Steve Hole, Cyrus Daboo, and Michael Wener for encouraging + creation of this document. + + Valuable comments, both in agreement and in dissent, were received + from Timo Sirainen, Michael Wener, Randall Gellens, Arnt Gulbrandsen, + Chris Newman, Peter Coates, Mark Crispin, Elwyn Davies, Dan Karp, + Eric Rescorla, and Mike Zraly. + + This document takes substantial text from [RFC3501] by Mark Crispin. + +10. References + +10.1. Normative References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for + Conditional STORE Operation or Quick Flag Changes + Resynchronization", RFC 4551, June 2006. + + [ENABLE] Gulbrandsen, A., Ed. and A. Melnikov, Ed., "The IMAP + ENABLE Extension", RFC 5161, March 2008. + + [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to + IMAP4 ABNF", RFC 4466, April 2006. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - + UIDPLUS extension", RFC 4315, December 2005. + + + +Melnikov, et al. Standards Track [Page 21] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +10.2. Informative References + + [IMAP-DISC] Melnikov, A., Ed., "Synchronization Operations For + Disconnected Imap4 Clients", RFC 4549, June 2006. + + [UNSELECT] Melnikov, A., "Internet Message Access Protocol (IMAP) + UNSELECT command", RFC 3691, February 2004. + +Authors' Addresses + + Alexey Melnikov + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + Dave Cridland + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: dave.cridland@isode.com + + + Corby Wilson + Nokia + 5 Wayside Rd. + Burlington, MA 01803 + USA + + EMail: corby@computer.org + + + + + + + + + + + + + + +Melnikov, et al. Standards Track [Page 22] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Melnikov, et al. Standards Track [Page 23] + diff --git a/docs/rfcs/rfc5182.IMAP_extension_last_SEARCH_result.txt b/docs/rfcs/rfc5182.IMAP_extension_last_SEARCH_result.txt new file mode 100644 index 0000000..a7f9147 --- /dev/null +++ b/docs/rfcs/rfc5182.IMAP_extension_last_SEARCH_result.txt @@ -0,0 +1,731 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 5182 Isode Ltd. +Updates: 3501 March 2008 +Category: Standards Track + + + IMAP Extension for Referencing the Last SEARCH Result + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + Many IMAP clients use the result of a SEARCH command as the input to + perform another operation, for example, fetching the found messages, + deleting them, or copying them to another mailbox. + + This can be achieved using standard IMAP operations described in RFC + 3501; however, this would be suboptimal. The server will send the + list of found messages to the client; after that, the client will + have to parse the list, reformat it, and send it back to the server. + The client can't pipeline the SEARCH command with the subsequent + command, and, as a result, the server might not be able to perform + some optimizations. + + This document proposes an IMAP extension that allows a client to tell + a server to use the result of a SEARCH (or Unique Identifier (UID) + SEARCH) command as an input to any subsequent command. + +1. Introduction + + Many IMAP clients use the result of a SEARCH command as the input to + perform another operation, for example, fetching the found messages, + deleting them, or copying them to another mailbox. + + This document proposes an IMAP extension that allows a client to tell + a server to use the result of a SEARCH (or UID SEARCH) command as an + input to any subsequent command. + + The SEARCH result reference extension defines a new SEARCH result + option [IMAPABNF] "SAVE" that tells the server to remember the result + of the SEARCH or UID SEARCH command (as well as any command based on + SEARCH, e.g., SORT and THREAD [SORT]) and store it in an internal + + + +Melnikov Standards Track [Page 1] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + variable that we will reference as the "search result variable". The + client can use the "$" marker to reference the content of this + internal variable. The "$" marker can be used instead of message + sequence or UID sequence in order to indicate that the server should + substitute it with the list of messages from the search result + variable. Thus, the client can use the result of the latest + remembered SEARCH command as a parameter to another command. The + search result marker has several advantages: + + * it avoids wasted bandwidth and associated delay; + + * it allows the client to pipeline a SEARCH [IMAP4] command with a + subsequent FETCH/STORE/COPY/SEARCH [IMAP4] or UID EXPUNGE + [UIDPLUS] command; + + * the client doesn't need to spend time reformatting the result of + a SEARCH command into a message set used in the subsequent + command; + + * it allows the server to perform optimizations. For example, if + the server can execute several pipelined commands in parallel + (or out of order), presence of the search result marker can + allow the server to decide which commands may or may not be + executed out of order. + + In absence of any other SEARCH result option, the SAVE result option + also suppresses any SEARCH response that would have been otherwise + returned by the SEARCH command. + +1.1. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [KEYWORDS]. + + Explanatory comments in examples start with // and are not part of + the protocol. + + + + + + + + + + + +Melnikov Standards Track [Page 2] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +2. Overview + +2.1. Normative Description of the SEARCHRES Extension + + The SEARCH result reference extension described in this document is + present in any IMAP4 server implementation that returns "SEARCHRES" + as one of the supported capabilities in the CAPABILITY command + response. Any such server MUST also implement the [ESEARCH] + extension. + + Upon successful completion of a SELECT or an EXAMINE command (after + the tagged OK response), the current search result variable is reset + to the empty sequence. + + A successful SEARCH command with the SAVE result option sets the + value of the search result variable to the list of messages found in + the SEARCH command. For example, if no messages were found, the + search result variable will contain the empty list. + + Any of the following SEARCH commands MUST NOT change the search + result variable: + + o a SEARCH command that caused the server to return the BAD tagged + response, + + o a SEARCH command with no SAVE result option that caused the + server to return NO tagged response, + + o a successful SEARCH command with no SAVE result option. + + A SEARCH command with the SAVE result option that caused the server + to return the NO tagged response sets the value of the search result + variable to the empty sequence. + + When a message listed in the search result variable is EXPUNGEd, it + is automatically removed from the list. Implementors are reminded + that if the server stores the list as a list of message numbers, it + MUST automatically adjust them when notifying the client about + expunged messages, as described in Section 7.4.1 of [IMAP4]. + + If the server decides to send a new UIDVALIDITY value while the + mailbox is opened, this causes resetting of the search variable to + the empty list. + + + + + + + + +Melnikov Standards Track [Page 3] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + Note that even if the "$" marker contains the empty list of messages, + it must be treated by all commands accepting message sets as + parameters as a valid, but non-matching list of messages. For + example, the "FETCH $" command would return a tagged OK response and + no FETCH responses. See also the Example 5 below. + + Note that even if the "$" marker contains the empty list of messages, + it must be treated as a valid but non-matching list of messages, by + all commands that accept message sets as parameters. + + Implementation note: server implementors should note that "$" can + reference IMAP message sequences or UID sequences, depending on the + context where it is used. For example, the "$" marker can be set as + a result of a SEARCH (SAVE) command and used as a parameter to a UID + FETCH command (which accepts a UID sequence, not a message sequence), + or the "$" marker can be set as a result of a UID SEARCH (SAVE) + command and used as a parameter to a FETCH command (which accepts a + message sequence, not a UID sequence). + +2.2. Examples + + 1) The following example demonstrates how the client can use the + result of a SEARCH command to FETCH headers of interesting + messages: + + Example 1: + C: A282 SEARCH RETURN (SAVE) FLAGGED SINCE 1-Feb-1994 + NOT FROM "Smith" + S: A282 OK SEARCH completed, result saved + C: A283 FETCH $ (UID INTERNALDATE FLAGS RFC822.HEADER) + S: * 2 FETCH (UID 14 ... + S: * 84 FETCH (UID 100 ... + S: * 882 FETCH (UID 1115 ... + S: A283 OK completed + + The client can also pipeline the two commands: + + Example 2: + C: A282 SEARCH RETURN (SAVE) FLAGGED SINCE 1-Feb-1994 + NOT FROM "Smith" + C: A283 FETCH $ (UID INTERNALDATE FLAGS RFC822.HEADER) + S: A282 OK SEARCH completed + S: * 2 FETCH (UID 14 ... + S: * 84 FETCH (UID 100 ... + S: * 882 FETCH (UID 1115 ... + S: A283 OK completed + + + + + +Melnikov Standards Track [Page 4] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + 2) The following example demonstrates that the result of one SEARCH + command can be used as input to another SEARCH command: + + Example 3: + C: A300 SEARCH RETURN (SAVE) SINCE 1-Jan-2004 + NOT FROM "Smith" + S: A300 OK SEARCH completed + C: A301 UID SEARCH UID $ SMALLER 4096 + S: * SEARCH 17 900 901 + S: A301 OK completed + + Note that the second command in Example 3 can be replaced with: + C: A301 UID SEARCH $ SMALLER 4096 + and the result of the command would be the same. + + 3) The following example shows that the "$" + marker can be combined with other message numbers using the OR + SEARCH criterion. + + Example 4: + C: P282 SEARCH RETURN (SAVE) SINCE 1-Feb-1994 + NOT FROM "Smith" + S: P282 OK SEARCH completed + C: P283 SEARCH CHARSET UTF-8 (OR $ 1,3000:3021) TEXT {8} + C: YYYYYYYY + S: * SEARCH 882 1102 3003 3005 3006 + S: P283 OK completed + + Note: Since this document format is restricted to 7-bit ASCII text, + it is not possible to show actual UTF-8 data. The "YYYYYYYY" is a + placeholder for what would be 8 octets of 8-bit data in an actual + transaction. + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 5] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + 4) The following example demonstrates that a failed SEARCH sets the + search result variable to the empty list. + + Example 5: + C: B282 SEARCH RETURN (SAVE) SINCE 1-Feb-1994 + NOT FROM "Smith" + S: B282 OK SEARCH completed + C: B283 SEARCH CHARSET KOI8-R (OR $ 1,3000:3021) TEXT {4} + C: XXXX + S: B283 NO [BADCHARSET UTF-8] KOI8-R is not supported + //After this command the saved result variable contains + //no messages. A client that wants to reissue the B283 + //SEARCH command with another CHARSET would have to reissue + //the B282 command as well. One possible workaround for + //this is to include the desired CHARSET parameter + //in the earliest SEARCH RETURN (SAVE) command in a + //sequence of related SEARCH commands. + //A better approach might be to always use CHARSET UTF-8 + //instead. + + Note: Since this document format is restricted to 7-bit ASCII text, + it is not possible to show actual KOI8-R data. The "XXXX" is a + placeholder for what would be 4 octets of 8-bit data in an actual + transaction. + + 5) The following example demonstrates that it is not an error to use + the "$" marker when it contains no messages. + + Example 6: + C: E282 SEARCH RETURN (SAVE) SINCE 28-Oct-2006 + NOT FROM "Eric" + C: E283 COPY $ "Other Messages" + //The "$" contains no messages + S: E282 OK SEARCH completed + S: E283 OK COPY completed, nothing copied + +2.3. Multiple Commands in Progress + + Use of a SEARCH RETURN (SAVE) command followed by a command using the + "$" marker creates direct dependency between the two commands. As + directed by Section 5.5 of [IMAP4], a server MUST execute the two + commands in the order they were received. (A server capable of + out-of-order execution can in some cases execute the two commands in + parallel, for example, if a SEARCH RETURN (SAVE) is followed by + "SEARCH $", the search criteria from the first command can be + directly substituted into the second command.) + + + + + +Melnikov Standards Track [Page 6] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + A client supporting this extension MAY pipeline a SEARCH RETURN + (SAVE) command with one or more command using the "$" marker, as long + as this doesn't create an ambiguity, as described in Section 5.5 of + [IMAP4]. + + Example 7: + C: F282 SEARCH RETURN (SAVE) KEYWORD $Junk + C: F283 COPY $ "Junk" + C: F284 STORE $ +FLAGS.Silent (\Deleted) + S: F282 OK SEARCH completed + S: F283 OK COPY completed + S: F284 OK STORE completed + + Example 8: + C: G282 SEARCH RETURN (SAVE) KEYWORD $Junk + C: G283 SEARCH RETURN (ALL) SINCE 28-Oct-2006 + FROM "Eric" + //The server can execute the two SEARCH commands + //in any order, as they don't have any dependency. + //Note that the second command is making use of + //the [ESEARCH] extension. + S: * ESEARCH (TAG "G283") ALL 3:15,27,29:103 + S: G283 OK SEARCH completed + S: G282 OK SEARCH completed + + The following example demonstrates that the result of the second + SEARCH always overrides the result of the first. + + Example 9: + C: H282 SEARCH RETURN (SAVE) KEYWORD $Junk + C: H283 SEARCH RETURN (SAVE) SINCE 28-Oct-2006 + FROM "Eric" + S: H282 OK SEARCH completed + S: H283 OK SEARCH completed + +2.4. Interaction with ESEARCH Extension + + Servers that implement the extension defined in this document MUST + implement [ESEARCH] and conform to additional requirements listed in + this section. + + The SAVE result option doesn't change whether the server would return + items corresponding to MIN, MAX, ALL, or COUNT [ESEARCH] result + options. + + + + + + + +Melnikov Standards Track [Page 7] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + When the SAVE result option is combined with the MIN or MAX [ESEARCH] + result option, and none of the other ESEARCH result options are + present, the corresponding MIN/MAX is returned (if the search result + is not empty), but the "$" marker would contain a single message as + returned in the MIN/MAX return item. + + If the SAVE result option is combined with both MIN and MAX result + options, and none of the other ESEARCH result options are present, + the "$" marker would contain one or two messages as returned in the + MIN/MAX return items. + + If the SAVE result option is combined with the ALL and/or COUNT + result option(s), the "$" marker would always contain all messages + found by the SEARCH or UID SEARCH command. (Note that the last rule + might affect ESEARCH implementations that optimize how the COUNT + result is constructed.) + + The following table summarizes the additional requirement on ESEARCH + server implementations described in this section. + + +----------------+-------------------+ + | Combination of | "$" marker value | + | Result option | | + +----------------+-------------------+ + | SAVE MIN | MIN | + +----------------+-------------------+ + | SAVE MAX | MAX | + +----------------+-------------------+ + | SAVE MIN MAX | MIN & MAX | + +----------------+-------------------+ + | SAVE * [m] | all found messages| + +----------------+-------------------+ + + where '*' means "ALL" and/or "COUNT" + '[m]' means optional "MIN" and/or "MAX" + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 8] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + The following example demonstrates behavioral difference for + different combinations of ESEARCH result options. Explanatory + comments start with // and are not part of the protocol: + + Example 10: + C: C282 SEARCH RETURN (ALL) SINCE 12-Feb-2006 + NOT FROM "Smith" + S: * ESEARCH (TAG "C283") ALL 2,10:15,21 + //$ value hasn't changed + S: C282 OK SEARCH completed + + C: C283 SEARCH RETURN (ALL SAVE) SINCE 12-Feb-2006 + NOT FROM "Smith" + S: * ESEARCH (TAG "C283") ALL 2,10:15,21 + //$ value is 2,10:15,21 + S: C283 OK SEARCH completed + + C: C284 SEARCH RETURN (SAVE MIN) SINCE 12-Feb-2006 + NOT FROM "Smith" + S: * ESEARCH (TAG "C284") MIN 2 + //$ value is 2 + S: C284 OK SEARCH completed + + C: C285 SEARCH RETURN (MAX SAVE MIN) SINCE + 12-Feb-2006 NOT FROM "Smith" + S: * ESEARCH (TAG "C285") MIN 2 MAX 21 + //$ value is 2,21 + S: C285 OK SEARCH completed + + C: C286 SEARCH RETURN (MAX SAVE MIN COUNT) + SINCE 12-Feb-2006 NOT FROM "Smith" + S: * ESEARCH (TAG "C286") MIN 2 MAX 21 COUNT 8 + //$ value is 2,10:15,21 + S: C286 OK SEARCH completed + + C: C286 SEARCH RETURN (ALL SAVE MIN) SINCE + 12-Feb-2006 NOT FROM "Smith" + S: * ESEARCH (TAG "C286") MIN 2 ALL 2,10:15,21 + //$ value is 2,10:15,21 + S: C286 OK SEARCH completed + + + + + + + + + + + +Melnikov Standards Track [Page 9] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +2.5. Refusing to Save Search Results + + In some cases, the server MAY refuse to save a SEARCH (SAVE) result, + for example, if an internal limit on the number of saved results is + reached. + + In this case, the server MUST return a tagged NO response containing + the NOTSAVED response code and set the search result variable to the + empty sequence, as described in Section 2.1. + +3. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. Non-terminals + referenced but not defined below are as defined in [IMAP4] or + [IMAPABNF]. + + Except as noted otherwise, all alphabetic characters are + case-insensitive. The use of upper- or lower-case characters to + define token strings is for editorial clarity only. Implementations + MUST accept these strings in a case-insensitive fashion. + + capability =/ "SEARCHRES" + ;; capability is defined in [IMAP4] + + sequence-set =/ seq-last-command + ;; extends sequence-set to allow for + ;; "result of the last command" indicator. + + seq-last-command = "$" + + search-return-opt = "SAVE" + ;; conforms to generic search-return-opt + ;; syntax defined in [IMAPABNF] + + resp-text-code =/ "NOTSAVED" + ;; from [IMAP4] + + + + + + + + + + + + + + +Melnikov Standards Track [Page 10] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +4. Security Considerations + + This extension requires the server to keep additional state, that may + be used to simplify Denial of Service attacks. In order to minimize + damage from such attacks, server implementations MAY limit the number + of saved searches they allow across all connections at any given time + and return the tagged NO response containing the NOTSAVED response + code (see Section 2.5) to a SEARCH RETURN (SAVE) command when this + limit is exceeded. + + Apart from that, it is believed that this extension doesn't raise any + additional security concerns not already discussed in [IMAP4]. + +5. IANA Considerations + + This document defines the "SEARCHRES" IMAP capability. IANA has + added it to the IMAP4 Capabilities Registry, which is currently + located at: + + http://www.iana.org/assignments/imap4-capabilities + +6. Acknowledgments + + The author would like to thank Mark Crispin, Cyrus Daboo, and Curtis + King for remembering that this document had to be written, as well as + for comments and corrections received. + + The author would also like to thank Dave Cridland, Mark Crispin, + Chris Newman, Dan Karp, and Spencer Dawkins for comments and + corrections received. + + Valuable comments, both in agreement and in dissent, were received + from Arnt Gulbrandsen. + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 11] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +7. References + +7.1. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [ESEARCH] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH + Command for Controlling What Kind of Information Is + Returned", RFC 4731, November 2006. + +7.2. Informative References + + [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - + UIDPLUS extension", RFC 4315, December 2005. + + [SORT] Crispin, M. and K. Murchison, "INTERNET MESSAGE ACCESS + PROTOCOL - SORT AND THREAD EXTENSIONS", Work in Progress, + Septemeber 2007. + +Author's Address + + Alexey Melnikov + Isode Ltd. + 5 Castle Business Village, + 36 Station Road, + Hampton, Middlesex, + TW12 2BX, United Kingdom + + EMail: Alexey.Melnikov@isode.com + + + + + + + + + + + +Melnikov Standards Track [Page 12] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Melnikov Standards Track [Page 13] + diff --git a/docs/rfcs/rfc5182.Sieve_and_extensions.txt b/docs/rfcs/rfc5182.Sieve_and_extensions.txt new file mode 100644 index 0000000..e5a02c6 --- /dev/null +++ b/docs/rfcs/rfc5182.Sieve_and_extensions.txt @@ -0,0 +1,2355 @@ + + + + + + +Network Working Group P. Guenther, Ed. +Request for Comments: 5228 Sendmail, Inc. +Obsoletes: 3028 T. Showalter, Ed. +Category: Standards Track January 2008 + + + Sieve: An Email Filtering Language + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + This document describes a language for filtering email messages at + time of final delivery. It is designed to be implementable on either + a mail client or mail server. It is meant to be extensible, simple, + and independent of access protocol, mail architecture, and operating + system. It is suitable for running on a mail server where users may + not be allowed to execute arbitrary programs, such as on black box + Internet Message Access Protocol (IMAP) servers, as the base language + has no variables, loops, or ability to shell out to external + programs. + + + + + + + + + + + + + + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 1] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +Table of Contents + + 1. Introduction ....................................................4 + 1.1. Conventions Used in This Document ..........................4 + 1.2. Example Mail Messages ......................................5 + 2. Design ..........................................................6 + 2.1. Form of the Language .......................................6 + 2.2. Whitespace .................................................7 + 2.3. Comments ...................................................7 + 2.4. Literal Data ...............................................7 + 2.4.1. Numbers .............................................7 + 2.4.2. Strings .............................................8 + 2.4.2.1. String Lists ...............................9 + 2.4.2.2. Headers ....................................9 + 2.4.2.3. Addresses .................................10 + 2.4.2.4. Encoding Characters Using + "encoded-character" .......................10 + 2.5. Tests .....................................................11 + 2.5.1. Test Lists .........................................12 + 2.6. Arguments .................................................12 + 2.6.1. Positional Arguments ...............................12 + 2.6.2. Tagged Arguments ...................................12 + 2.6.3. Optional Arguments .................................13 + 2.6.4. Types of Arguments .................................13 + 2.7. String Comparison .........................................13 + 2.7.1. Match Type .........................................14 + 2.7.2. Comparisons across Character Sets ..................15 + 2.7.3. Comparators ........................................15 + 2.7.4. Comparisons against Addresses ......................16 + 2.8. Blocks ....................................................17 + 2.9. Commands ..................................................17 + 2.10. Evaluation ...............................................18 + 2.10.1. Action Interaction ................................18 + 2.10.2. Implicit Keep .....................................18 + 2.10.3. Message Uniqueness in a Mailbox ...................19 + 2.10.4. Limits on Numbers of Actions ......................19 + 2.10.5. Extensions and Optional Features ..................19 + 2.10.6. Errors ............................................20 + 2.10.7. Limits on Execution ...............................20 + 3. Control Commands ...............................................21 + 3.1. Control if ................................................21 + 3.2. Control require ...........................................22 + 3.3. Control stop ..............................................22 + 4. Action Commands ................................................23 + 4.1. Action fileinto ...........................................23 + 4.2. Action redirect ...........................................23 + 4.3. Action keep ...............................................24 + 4.4. Action discard ............................................25 + + + +Guenther & Showalter Standards Track [Page 2] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + 5. Test Commands ..................................................26 + 5.1. Test address ..............................................26 + 5.2. Test allof ................................................27 + 5.3. Test anyof ................................................27 + 5.4. Test envelope .............................................27 + 5.5. Test exists ...............................................28 + 5.6. Test false ................................................28 + 5.7. Test header ...............................................29 + 5.8. Test not ..................................................29 + 5.9. Test size .................................................29 + 5.10. Test true ................................................30 + 6. Extensibility ..................................................30 + 6.1. Capability String .........................................31 + 6.2. IANA Considerations .......................................31 + 6.2.1. Template for Capability Registrations ..............32 + 6.2.2. Handling of Existing Capability Registrations ......32 + 6.2.3. Initial Capability Registrations ...................32 + 6.3. Capability Transport ......................................33 + 7. Transmission ...................................................33 + 8. Parsing ........................................................34 + 8.1. Lexical Tokens ............................................34 + 8.2. Grammar ...................................................36 + 8.3. Statement Elements ........................................36 + 9. Extended Example ...............................................37 + 10. Security Considerations .......................................38 + 11. Acknowledgments ...............................................39 + 12. Normative References ..........................................39 + 13. Informative References ........................................40 + 14. Changes from RFC 3028 .........................................41 + + + + + + + + + + + + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 3] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +1. Introduction + + This memo documents a language that can be used to create filters for + electronic mail. It is not tied to any particular operating system + or mail architecture. It requires the use of [IMAIL]-compliant + messages, but should otherwise generalize to many systems. + + The language is powerful enough to be useful but limited in order to + allow for a safe server-side filtering system. The intention is to + make it impossible for users to do anything more complex (and + dangerous) than write simple mail filters, along with facilitating + the use of graphical user interfaces (GUIs) for filter creation and + manipulation. The base language was not designed to be Turing- + complete: it does not have a loop control structure or functions. + + Scripts written in Sieve are executed during final delivery, when the + message is moved to the user-accessible mailbox. In systems where + the Mail Transfer Agent (MTA) does final delivery, such as + traditional Unix mail, it is reasonable to filter when the MTA + deposits mail into the user's mailbox. + + There are a number of reasons to use a filtering system. Mail + traffic for most users has been increasing due to increased usage of + email, the emergence of unsolicited email as a form of advertising, + and increased usage of mailing lists. + + Experience at Carnegie Mellon has shown that if a filtering system is + made available to users, many will make use of it in order to file + messages from specific users or mailing lists. However, many others + did not make use of the Andrew system's FLAMES filtering language + [FLAMES] due to difficulty in setting it up. + + Because of the expectation that users will make use of filtering if + it is offered and easy to use, this language has been made simple + enough to allow many users to make use of it, but rich enough that it + can be used productively. However, it is expected that GUI-based + editors will be the preferred way of editing filters for a large + number of users. + +1.1. Conventions Used in This Document + + In the sections of this document that discuss the requirements of + various keywords and operators, the following conventions have been + adopted. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [KEYWORDS]. + + + +Guenther & Showalter Standards Track [Page 4] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Each section on a command (test, action, or control) has a line + labeled "Usage:". This line describes the usage of the command, + including its name and its arguments. Required arguments are listed + inside angle brackets ("<" and ">"). Optional arguments are listed + inside square brackets ("[" and "]"). Each argument is followed by + its type, so "" represents an argument called "key" that + is a string. Literal strings are represented with double-quoted + strings. Alternatives are separated with slashes, and parentheses + are used for grouping, similar to [ABNF]. + + In the "Usage:" line, there are three special pieces of syntax that + are frequently repeated, MATCH-TYPE, COMPARATOR, and ADDRESS-PART. + These are discussed in sections 2.7.1, 2.7.3, and 2.7.4, + respectively. + + The formal grammar for these commands is defined in section 8 and is + the authoritative reference on how to construct commands, but the + formal grammar does not specify the order, semantics, number or types + of arguments to commands, or the legal command names. The intent is + to allow for extension without changing the grammar. + +1.2. Example Mail Messages + + The following mail messages will be used throughout this document in + examples. + + Message A + ----------------------------------------------------------- + Date: Tue, 1 Apr 1997 09:06:31 -0800 (PST) + From: coyote@desert.example.org + To: roadrunner@acme.example.com + Subject: I have a present for you + + Look, I'm sorry about the whole anvil thing, and I really + didn't mean to try and drop it on you from the top of the + cliff. I want to try to make it up to you. I've got some + great birdseed over here at my place--top of the line + stuff--and if you come by, I'll have it all wrapped up + for you. I'm really sorry for all the problems I've caused + for you over the years, but I know we can work this out. + -- + Wile E. Coyote "Super Genius" coyote@desert.example.org + ----------------------------------------------------------- + + + + + + + + +Guenther & Showalter Standards Track [Page 5] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Message B + ----------------------------------------------------------- + From: youcouldberich!@reply-by-postal-mail.invalid + Sender: b1ff@de.res.example.com + To: rube@landru.example.com + Date: Mon, 31 Mar 1997 18:26:10 -0800 + Subject: $$$ YOU, TOO, CAN BE A MILLIONAIRE! $$$ + + YOU MAY HAVE ALREADY WON TEN MILLION DOLLARS, BUT I DOUBT + IT! SO JUST POST THIS TO SIX HUNDRED NEWSGROUPS! IT WILL + GUARANTEE THAT YOU GET AT LEAST FIVE RESPONSES WITH MONEY! + MONEY! MONEY! COLD HARD CASH! YOU WILL RECEIVE OVER + $20,000 IN LESS THAN TWO MONTHS! AND IT'S LEGAL!!!!!!!!! + !!!!!!!!!!!!!!!!!!111111111!!!!!!!11111111111!!1 JUST + SEND $5 IN SMALL, UNMARKED BILLS TO THE ADDRESSES BELOW! + ----------------------------------------------------------- + +2. Design + +2.1. Form of the Language + + The language consists of a set of commands. Each command consists of + a set of tokens delimited by whitespace. The command identifier is + the first token and it is followed by zero or more argument tokens. + Arguments may be literal data, tags, blocks of commands, or test + commands. + + With the exceptions of strings and comments, the language is limited + to US-ASCII characters. Strings and comments may contain octets + outside the US-ASCII range. Specifically, they will normally be in + UTF-8, as specified in [UTF-8]. NUL (US-ASCII 0) is never permitted + in scripts, while CR and LF can only appear as the CRLF line ending. + + Note: While this specification permits arbitrary octets to appear + in Sieve scripts inside strings and comments, this has made it + difficult to robustly handle Sieve scripts in programs that are + sensitive to the encodings used. The "encoded-character" + capability (section 2.4.2.4) provides an alternative means of + representing such octets in strings using just US-ASCII + characters. As such, the use of non-UTF-8 text in scripts should + be considered a deprecated feature that may be abandoned. + + Tokens other than strings are considered case-insensitive. + + + + + + + + +Guenther & Showalter Standards Track [Page 6] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.2. Whitespace + + Whitespace is used to separate tokens. Whitespace is made up of + tabs, newlines (CRLF, never just CR or LF), and the space character. + The amount of whitespace used is not significant. + +2.3. Comments + + Two types of comments are offered. Comments are semantically + equivalent to whitespace and can be used anyplace that whitespace is + (with one exception in multi-line strings, as described in the + grammar). + + Hash comments begin with a "#" character that is not contained within + a string and continue until the next CRLF. + + Example: if size :over 100k { # this is a comment + discard; + } + + Bracketed comments begin with the token "/*" and end with "*/" + outside of a string. Bracketed comments may span multiple lines. + Bracketed comments do not nest. + + Example: if size :over 100K { /* this is a comment + this is still a comment */ discard /* this is a comment + */ ; + } + +2.4. Literal Data + + Literal data means data that is not executed, merely evaluated "as + is", to be used as arguments to commands. Literal data is limited to + numbers, strings, and string lists. + +2.4.1. Numbers + + Numbers are given as ordinary decimal numbers. As a shorthand for + expressing larger values, such as message sizes, a suffix of "K", + "M", or "G" MAY be appended to indicate a multiple of a power of two. + To be comparable with the power-of-two-based versions of SI units + that computers frequently use, "K" specifies kibi-, or 1,024 (2^10) + times the value of the number; "M" specifies mebi-, or 1,048,576 + (2^20) times the value of the number; and "G" specifies gibi-, or + 1,073,741,824 (2^30) times the value of the number [BINARY-SI]. + + + + + + +Guenther & Showalter Standards Track [Page 7] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Implementations MUST support integer values in the inclusive range + zero to 2,147,483,647 (2^31 - 1), but MAY support larger values. + + Only non-negative integers are permitted by this specification. + +2.4.2. Strings + + Scripts involve large numbers of string values as they are used for + pattern matching, addresses, textual bodies, etc. Typically, short + quoted strings suffice for most uses, but a more convenient form is + provided for longer strings such as bodies of messages. + + A quoted string starts and ends with a single double quote (the <"> + character, US-ASCII 34). A backslash ("\", US-ASCII 92) inside of a + quoted string is followed by either another backslash or a double + quote. These two-character sequences represent a single backslash or + double quote within the value, respectively. + + Scripts SHOULD NOT escape other characters with a backslash. + + An undefined escape sequence (such as "\a" in a context where "a" has + no special meaning) is interpreted as if there were no backslash (in + this case, "\a" is just "a"), though that may be changed by + extensions. + + Non-printing characters such as tabs, CRLF, and control characters + are permitted in quoted strings. Quoted strings MAY span multiple + lines. An unencoded NUL (US-ASCII 0) is not allowed in strings; see + section 2.4.2.4 for how it can be encoded. + + As message header data is converted to [UTF-8] for comparison (see + section 2.7.2), most string values will use the UTF-8 encoding. + However, implementations MUST accept all strings that match the + grammar in section 8. The ability to use non-UTF-8 encoded strings + matches existing practice and has proven to be useful both in tests + for invalid data and in arguments containing raw MIME parts for + extension actions that generate outgoing messages. + + For entering larger amounts of text, such as an email message, a + multi-line form is allowed. It starts with the keyword "text:", + followed by a CRLF, and ends with the sequence of a CRLF, a single + period, and another CRLF. The CRLF before the final period is + considered part of the value. In order to allow the message to + contain lines with a single dot, lines are dot-stuffed. That is, + when composing a message body, an extra '.' is added before each line + that begins with a '.'. When the server interprets the script, these + extra dots are removed. Note that a line that begins with a dot + followed by a non-dot character is not interpreted as dot-stuffed; + + + +Guenther & Showalter Standards Track [Page 8] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + that is, ".foo" is interpreted as ".foo". However, because this is + potentially ambiguous, scripts SHOULD be properly dot-stuffed so such + lines do not appear. + + Note that a hashed comment or whitespace may occur in between the + "text:" and the CRLF, but not within the string itself. Bracketed + comments are not allowed here. + +2.4.2.1. String Lists + + When matching patterns, it is frequently convenient to match against + groups of strings instead of single strings. For this reason, a list + of strings is allowed in many tests, implying that if the test is + true using any one of the strings, then the test is true. + + For instance, the test 'header :contains ["To", "Cc"] + ["me@example.com", "me00@landru.example.com"]' is true if either a To + header or Cc header of the input message contains either of the email + addresses "me@example.com" or "me00@landru.example.com". + + Conversely, in any case where a list of strings is appropriate, a + single string is allowed without being a member of a list: it is + equivalent to a list with a single member. This means that the test + 'exists "To"' is equivalent to the test 'exists ["To"]'. + +2.4.2.2. Headers + + Headers are a subset of strings. In the Internet Message + Specification [IMAIL], each header line is allowed to have whitespace + nearly anywhere in the line, including after the field name and + before the subsequent colon. Extra spaces between the header name + and the ":" in a header field are ignored. + + A header name never contains a colon. The "From" header refers to a + line beginning "From:" (or "From :", etc.). No header will match + the string "From:" due to the trailing colon. + + Similarly, no header will match a syntactically invalid header name. + An implementation MUST NOT cause an error for syntactically invalid + header names in tests. + + Header lines are unfolded as described in [IMAIL] section 2.2.3. + Interpretation of header data SHOULD be done according to [MIME3] + section 6.2 (see section 2.7.2 below for details). + + + + + + + +Guenther & Showalter Standards Track [Page 9] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.4.2.3. Addresses + + A number of commands call for email addresses, which are also a + subset of strings. When these addresses are used in outbound + contexts, addresses must be compliant with [IMAIL], but are further + constrained within this document. Using the symbols defined in + [IMAIL], section 3, the syntax of an address is: + + sieve-address = addr-spec ; simple address + / phrase "<" addr-spec ">" ; name & addr-spec + + That is, routes and group syntax are not permitted. If multiple + addresses are required, use a string list. Named groups are not + permitted. + + It is an error for a script to execute an action with a value for use + as an outbound address that doesn't match the "sieve-address" syntax. + +2.4.2.4. Encoding Characters Using "encoded-character" + + When the "encoded-character" extension is in effect, certain + character sequences in strings are replaced by their decoded value. + This happens after escape sequences are interpreted and dot- + unstuffing has been done. Implementations SHOULD support "encoded- + character". + + Arbitrary octets can be embedded in strings by using the syntax + encoded-arb-octets. The sequence is replaced by the octets with the + hexadecimal values given by each hex-pair. + + blank = WSP / CRLF + encoded-arb-octets = "${hex:" hex-pair-seq "}" + hex-pair-seq = *blank hex-pair *(1*blank hex-pair) *blank + hex-pair = 1*2HEXDIG + + Where WSP and HEXDIG non-terminals are defined in Appendix B.1 of + [ABNF]. + + It may be inconvenient or undesirable to enter Unicode characters + verbatim, and for these cases the syntax encoded-unicode-char can be + used. The sequence is replaced by the UTF-8 encoding of the + specified Unicode characters, which are identified by the hexadecimal + value of unicode-hex. + + encoded-unicode-char = "${unicode:" unicode-hex-seq "}" + unicode-hex-seq = *blank unicode-hex + *(1*blank unicode-hex) *blank + unicode-hex = 1*HEXDIG + + + +Guenther & Showalter Standards Track [Page 10] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + It is an error for a script to use a hexadecimal value that isn't in + either the range 0 to D7FF or the range E000 to 10FFFF. (The range + D800 to DFFF is excluded as those character numbers are only used as + part of the UTF-16 encoding form and are not applicable to the UTF-8 + encoding that the syntax here represents.) + + Note: Implementations MUST NOT raise an error for an out-of-range + Unicode value unless the sequence containing it is well-formed + according to the grammar. + + The capability string for use with the require command is "encoded- + character". + + In the following script, message B is discarded, since the specified + test string is equivalent to "$$$". + + Example: require "encoded-character"; + if header :contains "Subject" "$${hex:24 24}" { + discard; + } + The following examples demonstrate valid and invalid encodings and + how they are handled: + + "$${hex:40}" -> "$@" + "${hex: 40 }" -> "@" + "${HEX: 40}" -> "@" + "${hex:40" -> "${hex:40" + "${hex:400}" -> "${hex:400}" + "${hex:4${hex:30}}" -> "${hex:40}" + "${unicode:40}" -> "@" + "${ unicode:40}" -> "${ unicode:40}" + "${UNICODE:40}" -> "@" + "${UnICoDE:0000040}" -> "@" + "${Unicode:40}" -> "@" + "${Unicode:Cool}" -> "${Unicode:Cool}" + "${unicode:200000}" -> error + "${Unicode:DF01} -> error + +2.5. Tests + + Tests are given as arguments to commands in order to control their + actions. In this document, tests are given to if/elsif to decide + which block of code is run. + + + + + + + + +Guenther & Showalter Standards Track [Page 11] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.5.1. Test Lists + + Some tests ("allof" and "anyof", which implement logical "and" and + logical "or", respectively) may require more than a single test as an + argument. The test-list syntax element provides a way of grouping + tests as a comma-separated list in parentheses. + + Example: if anyof (not exists ["From", "Date"], + header :contains "from" "fool@example.com") { + discard; + } + +2.6. Arguments + + In order to specify what to do, most commands take arguments. There + are three types of arguments: positional, tagged, and optional. + + It is an error for a script, on a single command, to use conflicting + arguments or to use a tagged or optional argument more than once. + +2.6.1. Positional Arguments + + Positional arguments are given to a command that discerns their + meaning based on their order. When a command takes positional + arguments, all positional arguments must be supplied and must be in + the order prescribed. + +2.6.2. Tagged Arguments + + This document provides for tagged arguments in the style of + CommonLISP. These are also similar to flags given to commands in + most command-line systems. + + A tagged argument is an argument for a command that begins with ":" + followed by a tag naming the argument, such as ":contains". This + argument means that zero or more of the next tokens have some + particular meaning depending on the argument. These next tokens may + be literal data, but they are never blocks. + + Tagged arguments are similar to positional arguments, except that + instead of the meaning being derived from the command, it is derived + from the tag. + + Tagged arguments must appear before positional arguments, but they + may appear in any order with other tagged arguments. For simplicity + of the specification, this is not expressed in the syntax definitions + + + + + +Guenther & Showalter Standards Track [Page 12] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + with commands, but they still may be reordered arbitrarily provided + they appear before positional arguments. Tagged arguments may be + mixed with optional arguments. + + Tagged arguments SHOULD NOT take tagged arguments as arguments. + +2.6.3. Optional Arguments + + Optional arguments are exactly like tagged arguments except that they + may be left out, in which case a default value is implied. Because + optional arguments tend to result in shorter scripts, they have been + used far more than tagged arguments. + + One particularly noteworthy case is the ":comparator" argument, which + allows the user to specify which comparator [COLLATION] will be used + to compare two strings, since different languages may impose + different orderings on UTF-8 [UTF-8] strings. + +2.6.4. Types of Arguments + + Abstractly, arguments may be literal data, tests, or blocks of + commands. In this way, an "if" control structure is merely a command + that happens to take a test and a block as arguments and may execute + the block of code. + + However, this abstraction is ambiguous from a parsing standpoint. + + The grammar in section 8.2 presents a parsable version of this: + Arguments are string lists (string-lists), numbers, and tags, which + may be followed by a test or a test list (test-list), which may be + followed by a block of commands. No more than one test or test list, + or more than one block of commands, may be used, and commands that + end with a block of commands do not end with semicolons. + +2.7. String Comparison + + When matching one string against another, there are a number of ways + of performing the match operation. These are accomplished with three + types of matches: an exact match, a substring match, and a wildcard + glob-style match. These are described below. + + In order to provide for matches between character sets and case + insensitivity, Sieve uses the comparators defined in the Internet + Application Protocol Collation Registry [COLLATION]. + + + + + + + +Guenther & Showalter Standards Track [Page 13] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + However, when a string represents the name of a header, the + comparator is never user-specified. Header comparisons are always + done with the "i;ascii-casemap" operator, i.e., case-insensitive + comparisons, because this is the way things are defined in the + message specification [IMAIL]. + +2.7.1. Match Type + + Commands that perform string comparisons may have an optional match + type argument. The three match types in this specification are + ":contains", ":is", and ":matches". + + The ":contains" match type describes a substring match. If the value + argument contains the key argument as a substring, the match is true. + For instance, the string "frobnitzm" contains "frob" and "nit", but + not "fbm". The empty key ("") is contained in all values. + + The ":is" match type describes an absolute match; if the contents of + the first string are absolutely the same as the contents of the + second string, they match. Only the string "frobnitzm" is the string + "frobnitzm". The empty key ("") only ":is" matches with the empty + value. + + The ":matches" match type specifies a wildcard match using the + characters "*" and "?"; the entire value must be matched. "*" + matches zero or more characters in the value and "?" matches a single + character in the value, where the comparator that is used (see + section 2.7.3) defines what a character is. For example, the + comparators "i;octet" and "i;ascii-casemap" define a character to be + a single octet, so "?" will always match exactly one octet when one + of those comparators is in use. In contrast, a Unicode-based + comparator would define a character to be any UTF-8 octet sequence + encoding one Unicode character and thus "?" may match more than one + octet. "?" and "*" may be escaped as "\\?" and "\\*" in strings to + match against themselves. The first backslash escapes the second + backslash; together, they escape the "*". This is awkward, but it is + commonplace in several programming languages that use globs and + regular expressions. + + In order to specify what type of match is supposed to happen, + commands that support matching take optional arguments ":matches", + ":is", and ":contains". Commands default to using ":is" matching if + no match type argument is supplied. Note that these modifiers + interact with comparators; in particular, only comparators that + support the "substring match" operation are suitable for matching + with ":contains" or ":matches". It is an error to use a comparator + with ":contains" or ":matches" that is not compatible with it. + + + + +Guenther & Showalter Standards Track [Page 14] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + It is an error to give more than one of these arguments to a given + command. + + For convenience, the "MATCH-TYPE" syntax element is defined here as + follows: + + Syntax: ":is" / ":contains" / ":matches" + +2.7.2. Comparisons across Character Sets + + Messages may involve a number of character sets. In order for + comparisons to work across character sets, implementations SHOULD + implement the following behavior: + + Comparisons are performed on octets. Implementations convert text + from header fields in all charsets [MIME3] to Unicode, encoded as + UTF-8, as input to the comparator (see section 2.7.3). + Implementations MUST be capable of converting US-ASCII, ISO-8859- + 1, the US-ASCII subset of ISO-8859-* character sets, and UTF-8. + Text that the implementation cannot convert to Unicode for any + reason MAY be treated as plain US-ASCII (including any [MIME3] + syntax) or processed according to local conventions. An encoded + NUL octet (character zero) SHOULD NOT cause early termination of + the header content being compared against. + + If implementations fail to support the above behavior, they MUST + conform to the following: + + No two strings can be considered equal if one contains octets + greater than 127. + +2.7.3. Comparators + + In order to allow for language-independent, case-independent matches, + the match type may be coupled with a comparator name. The Internet + Application Protocol Collation Registry [COLLATION] provides the + framework for describing and naming comparators. + + All implementations MUST support the "i;octet" comparator (simply + compares octets) and the "i;ascii-casemap" comparator (which treats + uppercase and lowercase characters in the US-ASCII subset of UTF-8 as + the same). If left unspecified, the default is "i;ascii-casemap". + + Some comparators may not be usable with substring matches; that is, + they may only work with ":is". It is an error to try to use a + comparator with ":matches" or ":contains" that is not compatible with + it. + + + + +Guenther & Showalter Standards Track [Page 15] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Sieve treats a comparator result of "undefined" the same as a result + of "no-match". That is, this base specification does not provide any + means to directly detect invalid comparator input. + + A comparator is specified by the ":comparator" option with commands + that support matching. This option is followed by a string providing + the name of the comparator to be used. For convenience, the syntax + of a comparator is abbreviated to "COMPARATOR", and (repeated in + several tests) is as follows: + + Syntax: ":comparator" + + So in this example, + + Example: if header :contains :comparator "i;octet" "Subject" + "MAKE MONEY FAST" { + discard; + } + + would discard any message with subjects like "You can MAKE MONEY + FAST", but not "You can Make Money Fast", since the comparator used + is case-sensitive. + + Comparators other than "i;octet" and "i;ascii-casemap" must be + declared with require, as they are extensions. If a comparator + declared with require is not known, it is an error, and execution + fails. If the comparator is not declared with require, it is also an + error, even if the comparator is supported. (See section 2.10.5.) + + Both ":matches" and ":contains" match types are compatible with the + "i;octet" and "i;ascii-casemap" comparators and may be used with + them. + + It is an error to give more than one of these arguments to a given + command. + +2.7.4. Comparisons against Addresses + + Addresses are one of the most frequent things represented as strings. + These are structured, and being able to compare against the local- + part or the domain of an address is useful, so some tests that act + exclusively on addresses take an additional optional argument that + specifies what the test acts on. + + These optional arguments are ":localpart", ":domain", and ":all", + which act on the local-part (left side), the domain-part (right + side), and the whole address. + + + + +Guenther & Showalter Standards Track [Page 16] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + If an address is not syntactically valid, then it will not be matched + by tests specifying ":localpart" or ":domain". + + The kind of comparison done, such as whether or not the test done is + case-insensitive, is specified as a comparator argument to the test. + + If an optional address-part is omitted, the default is ":all". + + It is an error to give more than one of these arguments to a given + command. + + For convenience, the "ADDRESS-PART" syntax element is defined here as + follows: + + Syntax: ":localpart" / ":domain" / ":all" + +2.8. Blocks + + Blocks are sets of commands enclosed within curly braces and supplied + as the final argument to a command. Such a command is a control + structure: when executed it has control over the number of times the + commands in the block are executed. + + With the commands supplied in this memo, there are no loops. The + control structures supplied--if, elsif, and else--run a block either + once or not at all. + +2.9. Commands + + Sieve scripts are sequences of commands. Commands can take any of + the tokens above as arguments, and arguments may be either tagged or + positional arguments. Not all commands take all arguments. + + There are three kinds of commands: test commands, action commands, + and control commands. + + The simplest is an action command. An action command is an + identifier followed by zero or more arguments, terminated by a + semicolon. Action commands do not take tests or blocks as arguments. + The actions referenced in this document are: + + - keep, to save the message in the default location + - fileinto, to save the message in a specific mailbox + - redirect, to forward the message to another address + - discard, to silently throw away the message + + + + + + +Guenther & Showalter Standards Track [Page 17] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + A control command is a command that affects the parsing or the flow + of execution of the Sieve script in some way. A control structure is + a control command that ends with a block instead of a semicolon. + + A test command is used as part of a control command. It is used to + specify whether or not the block of code given to the control command + is executed. + +2.10. Evaluation + +2.10.1. Action Interaction + + Some actions cannot be used with other actions because the result + would be absurd. These restrictions are noted throughout this memo. + + Extension actions MUST state how they interact with actions defined + in this specification. + +2.10.2. Implicit Keep + + Previous experience with filtering systems suggests that cases tend + to be missed in scripts. To prevent errors, Sieve has an "implicit + keep". + + An implicit keep is a keep action (see section 4.3) performed in + absence of any action that cancels the implicit keep. + + An implicit keep is performed if a message is not written to a + mailbox, redirected to a new address, or explicitly thrown out. That + is, if a fileinto, a keep, a redirect, or a discard is performed, an + implicit keep is not. + + Some actions may be defined to not cancel the implicit keep. These + actions may not directly affect the delivery of a message, and are + used for their side effects. None of the actions specified in this + document meet that criteria, but extension actions may. + + For instance, with any of the short messages offered above, the + following script produces no actions. + + Example: if size :over 500K { discard; } + + As a result, the implicit keep is taken. + + + + + + + + +Guenther & Showalter Standards Track [Page 18] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.10.3. Message Uniqueness in a Mailbox + + Implementations SHOULD NOT deliver a message to the same mailbox more + than once, even if a script explicitly asks for a message to be + written to a mailbox twice. + + The test for equality of two messages is implementation-defined. + + If a script asks for a message to be written to a mailbox twice, it + MUST NOT be treated as an error. + +2.10.4. Limits on Numbers of Actions + + Site policy MAY limit the number of actions taken and MAY impose + restrictions on which actions can be used together. In the event + that a script hits a policy limit on the number of actions taken for + a particular message, an error occurs. + + Implementations MUST allow at least one keep or one fileinto. If + fileinto is not implemented, implementations MUST allow at least one + keep. + +2.10.5. Extensions and Optional Features + + Because of the differing capabilities of many mail systems, several + features of this specification are optional. Before any of these + extensions can be executed, they must be declared with the "require" + action. + + If an extension is not enabled with "require", implementations MUST + treat it as if they did not support it at all. This protects scripts + from having their behavior altered by extensions that the script + author might not have even been aware of. + + Implementations MUST NOT execute any Sieve script test or command + subsequent to "require" if one of the required extensions is + unavailable. + + Note: The reason for this restriction is that prior experiences + with languages such as LISP and Tcl suggest that this is a + workable way of noting that a given script uses an extension. + + Extensions that define actions MUST state how they interact with + actions discussed in the base specification. + + + + + + + +Guenther & Showalter Standards Track [Page 19] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.10.6. Errors + + In any programming language, there are compile-time and run-time + errors. + + Compile-time errors are ones in syntax that are detectable if a + syntax check is done. + + Run-time errors are not detectable until the script is run. This + includes transient failures like disk full conditions, but also + includes issues like invalid combinations of actions. + + When an error occurs in a Sieve script, all processing stops. + + Implementations MAY choose to do a full parse, then evaluate the + script, then do all actions. Implementations might even go so far as + to ensure that execution is atomic (either all actions are executed + or none are executed). + + Other implementations may choose to parse and run at the same time. + Such implementations are simpler, but have issues with partial + failure (some actions happen, others don't). + + Implementations MUST perform syntactic, semantic, and run-time checks + on code that is actually executed. Implementations MAY perform those + checks or any part of them on code that is not reached during + execution. + + When an error happens, implementations MUST notify the user that an + error occurred and which actions (if any) were taken, and do an + implicit keep. + +2.10.7. Limits on Execution + + Implementations may limit certain constructs. However, this + specification places a lower bound on some of these limits. + + Implementations MUST support fifteen levels of nested blocks. + + Implementations MUST support fifteen levels of nested test lists. + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 20] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +3. Control Commands + + Control structures are needed to allow for multiple and conditional + actions. + +3.1. Control if + + There are three pieces to if: "if", "elsif", and "else". Each is + actually a separate command in terms of the grammar. However, an + elsif or else MUST only follow an if or elsif. An error occurs if + these conditions are not met. + + Usage: if + + Usage: elsif + + Usage: else + + The semantics are similar to those of any of the many other + programming languages these control structures appear in. When the + interpreter sees an "if", it evaluates the test associated with it. + If the test is true, it executes the block associated with it. + + If the test of the "if" is false, it evaluates the test of the first + "elsif" (if any). If the test of "elsif" is true, it runs the + elsif's block. An elsif may be followed by an elsif, in which case, + the interpreter repeats this process until it runs out of elsifs. + + When the interpreter runs out of elsifs, there may be an "else" case. + If there is, and none of the if or elsif tests were true, the + interpreter runs the else's block. + + This provides a way of performing exactly one of the blocks in the + chain. + + In the following example, both messages A and B are dropped. + + Example: require "fileinto"; + if header :contains "from" "coyote" { + discard; + } elsif header :contains ["subject"] ["$$$"] { + discard; + } else { + fileinto "INBOX"; + } + + + + + + +Guenther & Showalter Standards Track [Page 21] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + When the script below is run over message A, it redirects the message + to acm@example.com; message B, to postmaster@example.com; any other + message is redirected to field@example.com. + + Example: if header :contains ["From"] ["coyote"] { + redirect "acm@example.com"; + } elsif header :contains "Subject" "$$$" { + redirect "postmaster@example.com"; + } else { + redirect "field@example.com"; + } + + Note that this definition prohibits the "... else if ..." sequence + used by C. This is intentional, because this construct produces a + shift-reduce conflict. + +3.2. Control require + + Usage: require + + The require action notes that a script makes use of a certain + extension. Such a declaration is required to use the extension, as + discussed in section 2.10.5. Multiple capabilities can be declared + with a single require. + + The require command, if present, MUST be used before anything other + than a require can be used. An error occurs if a require appears + after a command other than require. + + Example: require ["fileinto", "reject"]; + + Example: require "fileinto"; + require "vacation"; + +3.3. Control stop + + Usage: stop + + The "stop" action ends all processing. If the implicit keep has not + been cancelled, then it is taken. + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 22] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +4. Action Commands + + This document supplies four actions that may be taken on a message: + keep, fileinto, redirect, and discard. + + Implementations MUST support the "keep", "discard", and "redirect" + actions. + + Implementations SHOULD support "fileinto". + + Implementations MAY limit the number of certain actions taken (see + section 2.10.4). + +4.1. Action fileinto + + Usage: fileinto + + The "fileinto" action delivers the message into the specified + mailbox. Implementations SHOULD support fileinto, but in some + environments this may be impossible. Implementations MAY place + restrictions on mailbox names; use of an invalid mailbox name MAY be + treated as an error or result in delivery to an implementation- + defined mailbox. If the specified mailbox doesn't exist, the + implementation MAY treat it as an error, create the mailbox, or + deliver the message to an implementation-defined mailbox. If the + implementation uses a different encoding scheme than UTF-8 for + mailbox names, it SHOULD reencode the mailbox name from UTF-8 to its + encoding scheme. For example, the Internet Message Access Protocol + [IMAP] uses modified UTF-7, such that a mailbox argument of "odds & + ends" would appear in IMAP as "odds &- ends". + + The capability string for use with the require command is "fileinto". + + In the following script, message A is filed into mailbox + "INBOX.harassment". + + Example: require "fileinto"; + if header :contains ["from"] "coyote" { + fileinto "INBOX.harassment"; + } + +4.2. Action redirect + + Usage: redirect + + The "redirect" action is used to send the message to another user at + a supplied address, as a mail forwarding feature does. The + "redirect" action makes no changes to the message body or existing + + + +Guenther & Showalter Standards Track [Page 23] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + headers, but it may add new headers. In particular, existing + Received headers MUST be preserved and the count of Received headers + in the outgoing message MUST be larger than the same count on the + message as received by the implementation. (An implementation that + adds a Received header before processing the message does not need to + add another when redirecting.) + + The message is sent back out with the address from the redirect + command as an envelope recipient. Implementations MAY combine + separate redirects for a given message into a single submission with + multiple envelope recipients. (This is not a Mail User Agent (MUA)- + style forward, which creates a new message with a different sender + and message ID, wrapping the old message in a new one.) + + The envelope sender address on the outgoing message is chosen by the + sieve implementation. It MAY be copied from the message being + processed. However, if the message being processed has an empty + envelope sender address the outgoing message MUST also have an empty + envelope sender address. This last requirement is imposed to prevent + loops in the case where a message is redirected to an invalid address + when then returns a delivery status notification that also ends up + being redirected to the same invalid address. + + A simple script can be used for redirecting all mail: + + Example: redirect "bart@example.com"; + + Implementations MUST take measures to implement loop control, + possibly including adding headers to the message or counting Received + headers as specified in section 6.2 of [SMTP]. If an implementation + detects a loop, it causes an error. + + Implementations MUST provide means of limiting the number of + redirects a Sieve script can perform. See section 10 for more + details. + + Implementations MAY ignore a redirect action silently due to policy + reasons. For example, an implementation MAY choose not to redirect + to an address that is known to be undeliverable. Any ignored + redirect MUST NOT cancel the implicit keep. + +4.3. Action keep + + Usage: keep + + The "keep" action is whatever action is taken in lieu of all other + actions, if no filtering happens at all; generally, this simply means + to file the message into the user's main mailbox. This command + + + +Guenther & Showalter Standards Track [Page 24] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + provides a way to execute this action without needing to know the + name of the user's main mailbox, providing a way to call it without + needing to understand the user's setup or the underlying mail system. + + For instance, in an implementation where the IMAP server is running + scripts on behalf of the user at time of delivery, a keep command is + equivalent to a fileinto "INBOX". + + Example: if size :under 1M { keep; } else { discard; } + + Note that the above script is identical to the one below. + + Example: if not size :under 1M { discard; } + +4.4. Action discard + + Usage: discard + + Discard is used to silently throw away the message. It does so by + simply canceling the implicit keep. If discard is used with other + actions, the other actions still happen. Discard is compatible with + all other actions. (For instance, fileinto+discard is equivalent to + fileinto.) + + Discard MUST be silent; that is, it MUST NOT return a non-delivery + notification of any kind ([DSN], [MDN], or otherwise). + + In the following script, any mail from "idiot@example.com" is thrown + out. + + Example: if header :contains ["from"] ["idiot@example.com"] { + discard; + } + + While an important part of this language, "discard" has the potential + to create serious problems for users: Students who leave themselves + logged in to an unattended machine in a public computer lab may find + their script changed to just "discard". In order to protect users in + this situation (along with similar situations), implementations MAY + keep messages destroyed by a script for an indefinite period, and MAY + disallow scripts that throw out all mail. + + + + + + + + + + +Guenther & Showalter Standards Track [Page 25] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +5. Test Commands + + Tests are used in conditionals to decide which part(s) of the + conditional to execute. + + Implementations MUST support these tests: "address", "allof", + "anyof", "exists", "false", "header", "not", "size", and "true". + + Implementations SHOULD support the "envelope" test. + +5.1. Test address + + Usage: address [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] + + + The "address" test matches Internet addresses in structured headers + that contain addresses. It returns true if any header contains any + key in the specified part of the address, as modified by the + comparator and the match keyword. Whether there are other addresses + present in the header doesn't affect this test; this test does not + provide any way to determine whether an address is the only address + in a header. + + Like envelope and header, this test returns true if any combination + of the header-list and key-list arguments match and returns false + otherwise. + + Internet email addresses [IMAIL] have the somewhat awkward + characteristic that the local-part to the left of the at-sign is + considered case sensitive, and the domain-part to the right of the + at-sign is case insensitive. The "address" command does not deal + with this itself, but provides the ADDRESS-PART argument for allowing + users to deal with it. + + The address primitive never acts on the phrase part of an email + address or on comments within that address. It also never acts on + group names, although it does act on the addresses within the group + construct. + + Implementations MUST restrict the address test to headers that + contain addresses, but MUST include at least From, To, Cc, Bcc, + Sender, Resent-From, and Resent-To, and it SHOULD include any other + header that utilizes an "address-list" structured header body. + + Example: if address :is :all "from" "tim@example.com" { + discard; + } + + + + +Guenther & Showalter Standards Track [Page 26] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +5.2. Test allof + + Usage: allof + + The "allof" test performs a logical AND on the tests supplied to it. + + Example: allof (false, false) => false + allof (false, true) => false + allof (true, true) => true + + The allof test takes as its argument a test-list. + +5.3. Test anyof + + Usage: anyof + + The "anyof" test performs a logical OR on the tests supplied to it. + + Example: anyof (false, false) => false + anyof (false, true) => true + anyof (true, true) => true + +5.4. Test envelope + + Usage: envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] + + + The "envelope" test is true if the specified part of the [SMTP] (or + equivalent) envelope matches the specified key. This specification + defines the interpretation of the (case insensitive) "from" and "to" + envelope-parts. Additional envelope-parts may be defined by other + extensions; implementations SHOULD consider unknown envelope parts an + error. + + If one of the envelope-part strings is (case insensitive) "from", + then matching occurs against the FROM address used in the SMTP MAIL + command. The null reverse-path is matched against as the empty + string, regardless of the ADDRESS-PART argument specified. + + If one of the envelope-part strings is (case insensitive) "to", then + matching occurs against the TO address used in the SMTP RCPT command + that resulted in this message getting delivered to this user. Note + that only the most recent TO is available, and only the one relevant + to this user. + + The envelope-part is a string list and may contain more than one + parameter, in which case all of the strings specified in the key-list + are matched against all parts given in the envelope-part list. + + + +Guenther & Showalter Standards Track [Page 27] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Like address and header, this test returns true if any combination of + the envelope-part list and key-list arguments match and returns false + otherwise. + + All tests against envelopes MUST drop source routes. + + If the SMTP transaction involved several RCPT commands, only the data + from the RCPT command that caused delivery to this user is available + in the "to" part of the envelope. + + If a protocol other than SMTP is used for message transport, + implementations are expected to adapt this command appropriately. + + The envelope command is optional. Implementations SHOULD support it, + but the necessary information may not be available in all cases. The + capability string for use with the require command is "envelope". + + Example: require "envelope"; + if envelope :all :is "from" "tim@example.com" { + discard; + } + +5.5. Test exists + + Usage: exists + + The "exists" test is true if the headers listed in the header-names + argument exist within the message. All of the headers must exist or + the test is false. + + The following example throws out mail that doesn't have a From header + and a Date header. + + Example: if not exists ["From","Date"] { + discard; + } + +5.6. Test false + + Usage: false + + The "false" test always evaluates to false. + + + + + + + + + +Guenther & Showalter Standards Track [Page 28] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +5.7. Test header + + Usage: header [COMPARATOR] [MATCH-TYPE] + + + The "header" test evaluates to true if the value of any of the named + headers, ignoring leading and trailing whitespace, matches any key. + The type of match is specified by the optional match argument, which + defaults to ":is" if not specified, as specified in section 2.6. + + Like address and envelope, this test returns true if any combination + of the header-names list and key-list arguments match and returns + false otherwise. + + If a header listed in the header-names argument exists, it contains + the empty key (""). However, if the named header is not present, it + does not match any key, including the empty key. So if a message + contained the header + + X-Caffeine: C8H10N4O2 + + these tests on that header evaluate as follows: + + header :is ["X-Caffeine"] [""] => false + header :contains ["X-Caffeine"] [""] => true + + Testing whether a given header is either absent or doesn't contain + any non-whitespace characters can be done using a negated "header" + test: + + not header :matches "Cc" "?*" + +5.8. Test not + + Usage: not + + The "not" test takes some other test as an argument, and yields the + opposite result. "not false" evaluates to "true" and "not true" + evaluates to "false". + +5.9. Test size + + Usage: size <":over" / ":under"> + + The "size" test deals with the size of a message. It takes either a + tagged argument of ":over" or ":under", followed by a number + representing the size of the message. + + + + +Guenther & Showalter Standards Track [Page 29] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + If the argument is ":over", and the size of the message is greater + than the number provided, the test is true; otherwise, it is false. + + If the argument is ":under", and the size of the message is less than + the number provided, the test is true; otherwise, it is false. + + Exactly one of ":over" or ":under" must be specified, and anything + else is an error. + + The size of a message is defined to be the number of octets in the + [IMAIL] representation of the message. + + Note that for a message that is exactly 4,000 octets, the message is + neither ":over" nor ":under" 4000 octets. + +5.10. Test true + + Usage: true + + The "true" test always evaluates to true. + +6. Extensibility + + New control commands, actions, and tests can be added to the + language. Sites must make these features known to their users; this + document does not define a way to discover the list of extensions + supported by the server. + + Any extensions to this language MUST define a capability string that + uniquely identifies that extension. Capability string are case- + sensitive; for example, "foo" and "FOO" are different capabilities. + If a new version of an extension changes the functionality of a + previously defined extension, it MUST use a different name. + Extensions may register a set of related capabilities by registering + just a unique prefix for them. The "comparator-" prefix is an + example of this. The prefix MUST end with a "-" and MUST NOT overlap + any existing registrations. + + In a situation where there is a script submission protocol and an + extension advertisement mechanism aware of the details of this + language, scripts submitted can be checked against the mail server to + prevent use of an extension that the server does not support. + + Extensions MUST state how they interact with constraints defined in + section 2.10, e.g., whether they cancel the implicit keep, and which + actions they are compatible and incompatible with. Extensions MUST + NOT change the behavior of the "require" control command or alter the + interpretation of the argument to the "require" control. + + + +Guenther & Showalter Standards Track [Page 30] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Extensions that can submit new email messages or otherwise generate + new protocol requests MUST consider loop suppression, at least to + document any security considerations. + +6.1. Capability String + + Capability strings are typically short strings describing what + capabilities are supported by the server. + + Capability strings beginning with "vnd." represent vendor-defined + extensions. Such extensions are not defined by Internet standards or + RFCs, but are still registered with IANA in order to prevent + conflicts. Extensions starting with "vnd." SHOULD be followed by the + name of the vendor and product, such as "vnd.acme.rocket-sled". + + The following capability strings are defined by this document: + + encoded-character The string "encoded-character" indicates that the + implementation supports the interpretation of + "${hex:...}" and "${unicode:...}" in strings. + + envelope The string "envelope" indicates that the implementation + supports the "envelope" command. + + fileinto The string "fileinto" indicates that the implementation + supports the "fileinto" command. + + comparator- The string "comparator-elbonia" is provided if the + implementation supports the "elbonia" comparator. + Therefore, all implementations have at least the + "comparator-i;octet" and "comparator-i;ascii-casemap" + capabilities. However, these comparators may be used + without being declared with require. + +6.2. IANA Considerations + + In order to provide a standard set of extensions, a registry is + maintained by IANA. This registry contains both vendor-controlled + capability names (beginning with "vnd.") and IETF-controlled + capability names. Vendor-controlled capability names may be + registered on a first-come, first-served basis, by applying to IANA + with the form in the following section. Registration of capability + prefixes that do not begin with "vnd." REQUIRES a standards track or + IESG-approved experimental RFC. + + Extensions designed for interoperable use SHOULD use IETF-controlled + capability names. + + + + +Guenther & Showalter Standards Track [Page 31] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +6.2.1. Template for Capability Registrations + + The following template is to be used for registering new Sieve + extensions with IANA. + + To: iana@iana.org + Subject: Registration of new Sieve extension + + Capability name: [the string for use in the 'require' statement] + Description: [a brief description of what the extension adds + or changes] + RFC number: [for extensions published as RFCs] + Contact address: [email and/or physical address to contact for + additional information] + +6.2.2. Handling of Existing Capability Registrations + + In order to bring the existing capability registrations in line with + the new template, IANA has modified each as follows: + + 1. The "capability name" and "capability arguments" fields have been + eliminated + 2. The "capability keyword" field have been renamed to "Capability + name" + 3. An empty "Description" field has been added + 4. The "Standards Track/IESG-approved experimental RFC number" field + has been renamed to "RFC number" + 5. The "Person and email address to contact for further information" + field should be renamed to "Contact address" + +6.2.3. Initial Capability Registrations + + This RFC updates the following entries in the IANA registry for Sieve + extensions. + + Capability name: encoded-character + Description: changes the interpretation of strings to allow + arbitrary octets and Unicode characters to be + represented using US-ASCII + RFC number: RFC 5228 (Sieve base spec) + Contact address: The Sieve discussion list + + Capability name: fileinto + Description: adds the 'fileinto' action for delivering to a + mailbox other than the default + RFC number: RFC 5228 (Sieve base spec) + Contact address: The Sieve discussion list + + + + +Guenther & Showalter Standards Track [Page 32] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Capability name: envelope + Description: adds the 'envelope' test for testing the message + transport sender and recipient address + RFC number: RFC 5228 (Sieve base spec) + Contact address: The Sieve discussion list + + Capability name: comparator-* (anything starting with "comparator-") + Description: adds the indicated comparator for use with the + :comparator argument + RFC number: RFC 5228 (Sieve base spec) and [COLLATION] + Contact address: The Sieve discussion list + +6.3. Capability Transport + + A method of advertising which capabilities an implementation supports + is difficult due to the wide range of possible implementations. Such + a mechanism, however, should have the property that the + implementation can advertise the complete set of extensions that it + supports. + +7. Transmission + + The [MIME] type for a Sieve script is "application/sieve". + + The registration of this type for RFC 2048 requirements is updated as + follows: + + Subject: Registration of MIME media type application/sieve + + MIME media type name: application + MIME subtype name: sieve + Required parameters: none + Optional parameters: none + Encoding considerations: Most Sieve scripts will be textual, + written in UTF-8. When non-7bit characters are used, + quoted-printable is appropriate for transport systems + that require 7bit encoding. + Security considerations: Discussed in section 10 of this RFC. + Interoperability considerations: Discussed in section 2.10.5 + of this RFC. + Published specification: this RFC. + Applications that use this media type: sieve-enabled mail + servers and clients + Additional information: + Magic number(s): + File extension(s): .siv .sieve + Macintosh File Type Code(s): + + + + +Guenther & Showalter Standards Track [Page 33] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Person & email address to contact for further information: + See the discussion list at ietf-mta-filters@imc.org. + Intended usage: + COMMON + Author/Change controller: + The SIEVE WG, delegated by the IESG. + +8. Parsing + + The Sieve grammar is separated into tokens and a separate grammar as + most programming languages are. Additional rules are supplied here + for common arguments to various language facilities. + +8.1. Lexical Tokens + + Sieve scripts are encoded in UTF-8. The following assumes a valid + UTF-8 encoding; special characters in Sieve scripts are all US-ASCII. + + The following are tokens in Sieve: + + - identifiers + - tags + - numbers + - quoted strings + - multi-line strings + - other separators + + Identifiers, tags, and numbers are case-insensitive, while quoted + strings and multi-line strings are case-sensitive. + + Blanks, horizontal tabs, CRLFs, and comments ("whitespace") are + ignored except as they separate tokens. Some whitespace is required + to separate otherwise adjacent tokens and in specific places in the + multi-line strings. CR and LF can only appear in CRLF pairs. + + The other separators are single individual characters and are + mentioned explicitly in the grammar. + + The lexical structure of sieve is defined in the following grammar + (as described in [ABNF]): + + bracket-comment = "/*" *not-star 1*STAR + *(not-star-slash *not-star 1*STAR) "/" + ; No */ allowed inside a comment. + ; (No * is allowed unless it is the last + ; character, or unless it is followed by a + ; character that isn't a slash.) + + + + +Guenther & Showalter Standards Track [Page 34] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + comment = bracket-comment / hash-comment + + hash-comment = "#" *octet-not-crlf CRLF + + identifier = (ALPHA / "_") *(ALPHA / DIGIT / "_") + + multi-line = "text:" *(SP / HTAB) (hash-comment / CRLF) + *(multiline-literal / multiline-dotstart) + "." CRLF + + multiline-literal = [ octet-not-period *octet-not-crlf ] CRLF + + multiline-dotstart = "." 1*octet-not-crlf CRLF + ; A line containing only "." ends the + ; multi-line. Remove a leading '.' if + ; followed by another '.'. + + not-star = CRLF / %x01-09 / %x0B-0C / %x0E-29 / %x2B-FF + ; either a CRLF pair, OR a single octet + ; other than NUL, CR, LF, or star + + not-star-slash = CRLF / %x01-09 / %x0B-0C / %x0E-29 / %x2B-2E / + %x30-FF + ; either a CRLF pair, OR a single octet + ; other than NUL, CR, LF, star, or slash + + number = 1*DIGIT [ QUANTIFIER ] + + octet-not-crlf = %x01-09 / %x0B-0C / %x0E-FF + ; a single octet other than NUL, CR, or LF + + octet-not-period = %x01-09 / %x0B-0C / %x0E-2D / %x2F-FF + ; a single octet other than NUL, + ; CR, LF, or period + + octet-not-qspecial = %x01-09 / %x0B-0C / %x0E-21 / %x23-5B / %x5D-FF + ; a single octet other than NUL, + ; CR, LF, double-quote, or backslash + + QUANTIFIER = "K" / "M" / "G" + + quoted-other = "\" octet-not-qspecial + ; represents just the octet-no-qspecial + ; character. SHOULD NOT be used + + quoted-safe = CRLF / octet-not-qspecial + ; either a CRLF pair, OR a single octet other + ; than NUL, CR, LF, double-quote, or backslash + + + +Guenther & Showalter Standards Track [Page 35] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + quoted-special = "\" (DQUOTE / "\") + ; represents just a double-quote or backslash + + quoted-string = DQUOTE quoted-text DQUOTE + + quoted-text = *(quoted-safe / quoted-special / quoted-other) + + STAR = "*" + + tag = ":" identifier + + white-space = 1*(SP / CRLF / HTAB) / comment + +8.2. Grammar + + The following is the grammar of Sieve after it has been lexically + interpreted. No whitespace or comments appear below. The start + symbol is "start". + + argument = string-list / number / tag + + arguments = *argument [ test / test-list ] + + block = "{" commands "}" + + command = identifier arguments (";" / block) + + commands = *command + + start = commands + + string = quoted-string / multi-line + + string-list = "[" string *("," string) "]" / string + ; if there is only a single string, the brackets + ; are optional + + test = identifier arguments + + test-list = "(" test *("," test) ")" + +8.3. Statement Elements + + These elements are collected from the "Syntax" sections elsewhere in + this document, and are provided here in [ABNF] syntax so that they + can be modified by extensions. + + ADDRESS-PART = ":localpart" / ":domain" / ":all" + + + +Guenther & Showalter Standards Track [Page 36] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + COMPARATOR = ":comparator" string + + MATCH-TYPE = ":is" / ":contains" / ":matches" + +9. Extended Example + + The following is an extended example of a Sieve script. Note that it + does not make use of the implicit keep. + + # + # Example Sieve Filter + # Declare any optional features or extension used by the script + # + require ["fileinto"]; + + # + # Handle messages from known mailing lists + # Move messages from IETF filter discussion list to filter mailbox + # + if header :is "Sender" "owner-ietf-mta-filters@imc.org" + { + fileinto "filter"; # move to "filter" mailbox + } + # + # Keep all messages to or from people in my company + # + elsif address :DOMAIN :is ["From", "To"] "example.com" + { + keep; # keep in "In" mailbox + } + + # + # Try and catch unsolicited email. If a message is not to me, + # or it contains a subject known to be spam, file it away. + # + elsif anyof (NOT address :all :contains + ["To", "Cc", "Bcc"] "me@example.com", + header :matches "subject" + ["*make*money*fast*", "*university*dipl*mas*"]) + { + fileinto "spam"; # move to "spam" mailbox + } + else + { + # Move all other (non-company) mail to "personal" + # mailbox. + fileinto "personal"; + } + + + +Guenther & Showalter Standards Track [Page 37] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +10. Security Considerations + + Users must get their mail. It is imperative that whatever + implementations use to store the user-defined filtering scripts + protect them from unauthorized modification, to preserve the + integrity of the mail system. An attacker who can modify a script + can cause mail to be discarded, rejected, or forwarded to an + unauthorized recipient. In addition, it's possible that Sieve + scripts might expose private information, such as mailbox names, or + email addresses of favored (or disfavored) correspondents. Because + of that, scripts SHOULD also be protected from unauthorized + retrieval. + + Several commands, such as "discard", "redirect", and "fileinto", + allow for actions to be taken that are potentially very dangerous. + + Use of the "redirect" command to generate notifications may easily + overwhelm the target address, especially if it was not designed to + handle large messages. + + Allowing a single script to redirect to multiple destinations can be + used as a means of amplifying the number of messages in an attack. + Moreover, if loop detection is not properly implemented, it may be + possible to set up exponentially growing message loops. Accordingly, + Sieve implementations: + + (1) MUST implement facilities to detect and break message loops. See + section 6.2 of [SMTP] for additional information on basic loop + detection strategies. + + (2) MUST provide the means for administrators to limit the ability of + users to abuse redirect. In particular, it MUST be possible to + limit the number of redirects a script can perform. + Additionally, if no use cases exist for using redirect to + multiple destinations, this limit SHOULD be set to 1. Additional + limits, such as the ability to restrict redirect to local users, + MAY also be implemented. + + (3) MUST provide facilities to log use of redirect in order to + facilitate tracking down abuse. + + (4) MAY use script analysis to determine whether or not a given + script can be executed safely. While the Sieve language is + sufficiently complex that full analysis of all possible scripts + is computationally infeasible, the majority of real-world scripts + are amenable to analysis. For example, an implementation might + + + + + +Guenther & Showalter Standards Track [Page 38] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + allow scripts that it has determined are safe to run unhindered, + block scripts that are potentially problematic, and subject + unclassifiable scripts to additional auditing and logging. + + Allowing redirects at all may not be appropriate in situations where + email accounts are freely available and/or not trackable to a human + who can be held accountable for creating message bombs or other + abuse. + + As with any filter on a message stream, if the Sieve implementation + and the mail agents 'behind' Sieve in the message stream differ in + their interpretation of the messages, it may be possible for an + attacker to subvert the filter. Of particular note are differences + in the interpretation of malformed messages (e.g., missing or extra + syntax characters) or those that exhibit corner cases (e.g., NUL + octets encoded via [MIME3]). + +11. Acknowledgments + + This document has been revised in part based on comments and + discussions that took place on and off the SIEVE mailing list. + Thanks to Sharon Chisholm, Cyrus Daboo, Ned Freed, Arnt Gulbrandsen, + Michael Haardt, Kjetil Torgrim Homme, Barry Leiba, Mark E. Mallett, + Alexey Melnikov, Eric Rescorla, Rob Siemborski, and Nigel Swinson for + reviews and suggestions. + +12. Normative References + + [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 4234, October 2005. + + [COLLATION] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet + Application Protocol Collation Registry", RFC 4790, March + 2007. + + [IMAIL] Resnick, P., Ed., "Internet Message Format", RFC 2822, + April 2001. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MIME] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [MIME3] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII + Text", RFC 2047, November 1996. + + + +Guenther & Showalter Standards Track [Page 39] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + [SMTP] Klensin, J., Ed., "Simple Mail Transfer Protocol", RFC + 2821, April 2001. + + [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + +13. Informative References + + [BINARY-SI] "Standard IEC 60027-2: Letter symbols to be used in + electrical technology - Part 2: Telecommunications and + electronics", January 1999. + + [DSN] Moore, K. and G. Vaudreuil, "An Extensible Message Format + for Delivery Status Notifications", RFC 3464, January + 2003. + + [FLAMES] Borenstein, N, and C. Thyberg, "Power, Ease of Use, and + Cooperative Work in a Practical Multimedia Message + System", Int. J. of Man-Machine Studies, April, 1991. + Reprinted in Computer-Supported Cooperative Work and + Groupware, Saul Greenberg, editor, Harcourt Brace + Jovanovich, 1991. Reprinted in Readings in Groupware and + Computer-Supported Cooperative Work, Ronald Baecker, + editor, Morgan Kaufmann, 1993. + + [IMAP] Crispin, M., "Internet Message Access Protocol - version + 4rev1", RFC 3501, March 2003. + + [MDN] Hansen, T., Ed., and G. Vaudreuil, Ed., "Message + Disposition Notification", RFC 3798, May 2004. + + [RFC3028] Showalter, T., "Sieve: A Mail Filtering Language", RFC + 3028, January 2001. + + + + + + + + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 40] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +14. Changes from RFC 3028 + + This following list is a summary of the changes that have been made + in the Sieve language base specification from [RFC3028]. + + 1. Removed ban on tests having side-effects + 2. Removed reject extension (will be specified in a separate RFC) + 3. Clarified description of comparators to match [COLLATION], the + new base specification for them + 4. Require stripping of leading and trailing whitespace in "header" + test + 5. Clarified or tightened handling of many minor items, including: + - invalid [MIME3] encoding + - invalid addresses in headers + - invalid header field names in tests + - 'undefined' comparator result + - unknown envelope parts + - null return-path in "envelope" test + 6. Capability strings are case-sensitive + 7. Clarified that fileinto should reencode non-ASCII mailbox + names to match the mailstore's conventions + 8. Errors in the ABNF were corrected + 9. The references were updated and split into normative and + informative + 10. Added encoded-character capability and deprecated (but did not + remove) use of arbitrary binary octets in Sieve scripts. + 11. Updated IANA registration template, and added IANA + considerations to permit capability prefix registrations. + 12. Added .sieve as a valid extension for Sieve scripts. + +Editors' Addresses + + Philip Guenther + Sendmail, Inc. + 6425 Christie St. Ste 400 + Emeryville, CA 94608 + EMail: guenther@sendmail.com + + Tim Showalter + EMail: tjs@psaux.com + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 41] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 42] + diff --git a/docs/rfcs/rfc5255.IMAP_i18n.txt b/docs/rfcs/rfc5255.IMAP_i18n.txt new file mode 100644 index 0000000..df76402 --- /dev/null +++ b/docs/rfcs/rfc5255.IMAP_i18n.txt @@ -0,0 +1,1123 @@ + + + + + + +Network Working Group C. Newman +Request for Comments: 5255 Sun Microsystems +Category: Standards Track A. Gulbrandsen + Oryx Mail Systems GmhH + A. Melnikov + Isode Limited + June 2008 + + + Internet Message Access Protocol Internationalization + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + Internet Message Access Protocol (IMAP) version 4rev1 has basic + support for non-ASCII characters in mailbox names and search + substrings. It also supports non-ASCII message headers and content + encoded as specified by Multipurpose Internet Mail Extensions (MIME). + This specification defines a collection of IMAP extensions that + improve international support including language negotiation for + international error text, translations for namespace prefixes, and + comparator negotiation for search, sort, and thread. + + + + + + + + + + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 1] + +RFC 5255 IMAP Internationalization June 2008 + + +Table of Contents + + 1. Introduction ....................................................3 + 2. Conventions Used in This Document ...............................3 + 3. LANGUAGE Extension ..............................................3 + 3.1. LANGUAGE Extension Requirements ............................4 + 3.2. LANGUAGE Command ...........................................4 + 3.3. LANGUAGE Response ..........................................6 + 3.4. TRANSLATION Extension to the NAMESPACE Response ............7 + 3.5. Formal Syntax ..............................................8 + 4. I18NLEVEL=1 and I18NLEVEL=2 Extensions ..........................9 + 4.1. Introduction and Overview ..................................9 + 4.2. Requirements Common to Both I18NLEVEL=1 and I18NLEVEL=2 ....9 + 4.3. I18NLEVEL=1 Extension Requirements ........................10 + 4.4. I18NLEVEL=2 Extension Requirements ........................10 + 4.5. Compatibility Notes .......................................11 + 4.6. Comparators and Character Encodings .......................11 + 4.7. COMPARATOR Command ........................................13 + 4.8. COMPARATOR Response .......................................14 + 4.9. BADCOMPARATOR Response Code ...............................14 + 4.10. Formal Syntax ............................................14 + 5. Other IMAP Internationalization Issues .........................15 + 5.1. Unicode Userids and Passwords .............................15 + 5.2. UTF-8 Mailbox Names .......................................15 + 5.3. UTF-8 Domains, Addresses, and Mail Headers ................15 + 6. IANA Considerations ............................................16 + 7. Security Considerations ........................................16 + 8. Acknowledgements ...............................................16 + 9. Relevant Sources of Documents for Internationalized IMAP + Implementations ................................................17 + 10. Normative References ..........................................17 + 11. Informative References ........................................18 + + + + + + + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 2] + +RFC 5255 IMAP Internationalization June 2008 + + +1. Introduction + + This specification defines two IMAP4rev1 [RFC3501] extensions to + enhance international support. These extensions can be advertised + and implemented separately. + + The LANGUAGE extension allows the client to request a suitable + language for protocol error messages and in combination with the + NAMESPACE extension [RFC2342] enables namespace translations. + + The I18NLEVEL=2 extension allows the client to request a suitable + collation that will modify the behavior of the base specification's + SEARCH command as well as the SORT and THREAD extensions [SORT]. + This leverages the collation registry [RFC4790]. The I18NLEVEL=1 + extension updates SEARCH/SORT/THREAD to use i;unicode-casemap + comparator, as defined in [UCM]. I18NLEVEL=1 is a simpler version of + I18NLEVEL=2 with no ability to select a different collation. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + The formal syntax uses the Augmented Backus-Naur Form (ABNF) + [RFC5234] notation including the core rules defined in Appendix A. + + The UTF-8-related productions are defined in [RFC3629]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If a single "C:" or "S:" label applies to + multiple lines, then the line breaks between those lines are for + editorial clarity only and are not part of the actual protocol + exchange. + +3. LANGUAGE Extension + + IMAP allows server responses to include human-readable text that in + many cases needs to be presented to the user. But that text is + limited to US-ASCII by the IMAP specification [RFC3501] in order to + preserve backwards compatibility with deployed IMAP implementations. + This section specifies a way for an IMAP client to negotiate which + language the server should use when sending human-readable text. + + + + + + + + +Newman, et al. Standards Track [Page 3] + +RFC 5255 IMAP Internationalization June 2008 + + + The LANGUAGE extension only provides a mechanism for altering fixed + server strings such as response text and NAMESPACE folder names. + Assigning localized language aliases to shared mailboxes would be + done with a separate mechanism such as the proposed METADATA + extension (see [METADATA]). + +3.1. LANGUAGE Extension Requirements + + IMAP servers that support this extension MUST list the keyword + LANGUAGE in their CAPABILITY response as well as in the greeting + CAPABILITY data. + + A server that advertises this extension MUST use the language + "i-default" as described in [RFC2277] as its default language until + another supported language is negotiated by the client. A server + MUST include "i-default" as one of its supported languages. IMAP + servers SHOULD NOT advertise the LANGUAGE extension if they discover + that they only support "i-default". + + Clients and servers that support this extension MUST also support the + NAMESPACE extension [RFC2342]. + + The LANGUAGE command is valid in all states. Clients SHOULD issue + LANGUAGE before authentication, since some servers send valuable user + information as part of authentication (e.g., "password is correct, + but expired"). If a security layer (such as SASL or TLS) is + subsequently negotiated by the client, it MUST re-issue the LANGUAGE + command in order to make sure that no previous active attack (if any) + on LANGUAGE negotiation has effect on subsequent error messages. + (See Section 7 for a more detailed explanation of the attack.) + +3.2. LANGUAGE Command + + Arguments: Optional language range arguments. + + Response: A possible LANGUAGE response (see Section 3.3). + A possible NAMESPACE response (see Section 3.4). + + Result: OK - Command completed + NO - Could not complete command + BAD - Arguments invalid + + The LANGUAGE command requests that human-readable text emitted by the + server be localized to a language matching one of the language range + argument as described by Section 2 of [RFC4647]. + + + + + + +Newman, et al. Standards Track [Page 4] + +RFC 5255 IMAP Internationalization June 2008 + + + If the command succeeds, the server will return human-readable + responses in the first supported language specified. These responses + will be in UTF-8 [RFC3629]. The server MUST send a LANGUAGE response + specifying the language used, and the change takes effect immediately + after the LANGUAGE response. + + If the command fails, the server continues to return human-readable + responses in the language it was previously using. + + The special "default" language range argument indicates a request to + use a language designated as preferred by the server administrator. + The preferred language MAY vary based on the currently active user. + + If a language range does not match a known language tag exactly but + does match a language by the rules of [RFC4647], the server MUST send + an untagged LANGUAGE response indicating the language selected. + + If there aren't any arguments, the server SHOULD send an untagged + LANGUAGE response listing the languages it supports. If the server + is unable to enumerate the list of languages it supports it MAY + return a tagged NO response to the enumeration request. If, after + receiving a LANGUAGE request, the server discovers that it doesn't + support any language other than i-default, it MUST return a tagged NO + response to the enumeration request. + + < The server defaults to using English i-default responses until + the user explicitly changes the language. > + + C: A001 LOGIN KAREN PASSWORD + S: A001 OK LOGIN completed + + < Client requested MUL language, which no server supports. > + + C: A002 LANGUAGE MUL + S: A002 NO Unsupported language MUL + + < A LANGUAGE command with no arguments is a request to enumerate + the list of languages the server supports. > + + C: A003 LANGUAGE + S: * LANGUAGE (EN DE IT i-default) + S: A003 OK Supported languages have been enumerated + + C: B001 LANGUAGE + S: B001 NO Server is unable to enumerate supported languages + + + + + + +Newman, et al. Standards Track [Page 5] + +RFC 5255 IMAP Internationalization June 2008 + + + < Once the client changes the language, all responses will be in + that language starting after the LANGUAGE response. Note that + this includes the NAMESPACE response. Because RFCs are in US- + ASCII, this document uses an ASCII transcription rather than + UTF-8 text, e.g., "ue" in the word "ausgefuehrt" > + + C: C001 LANGUAGE DE + S: * LANGUAGE (DE) + S: * NAMESPACE (("" "/")) (("Other Users/" "/" "TRANSLATION" + ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" + "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) + S: C001 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt + + < If a server does not support the requested primary language, + responses will continue to be returned in the current language + the server is using. > + + C: D001 LANGUAGE FR + S: D001 NO Diese Sprache ist nicht unterstuetzt + C: D002 LANGUAGE DE-IT + S: * LANGUAGE (DE-IT) + S: * NAMESPACE (("" "/"))(("Other Users/" "/" "TRANSLATION" + ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" + "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) + S: D002 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt + C: D003 LANGUAGE "default" + S: * LANGUAGE (DE) + S: D003 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt + + < Server does not speak French, but does speak English. User + speaks Canadian French and Canadian English. > + + C: E001 LANGUAGE FR-CA EN-CA + S: * LANGUAGE (EN) + S: E001 OK Now speaking English + +3.3. LANGUAGE Response + + Contents: A list of one or more language tags. + + The LANGUAGE response occurs as a result of a LANGUAGE command. A + LANGUAGE response with a list containing a single language tag + indicates that the server is now using that language. A LANGUAGE + response with a list containing multiple language tags indicates the + server is communicating a list of available languages to the client, + and no change in the active language has been made. + + + + + +Newman, et al. Standards Track [Page 6] + +RFC 5255 IMAP Internationalization June 2008 + + +3.4. TRANSLATION Extension to the NAMESPACE Response + + If localized representations of the namespace prefixes are available + in the selected language, the server SHOULD include these in the + TRANSLATION extension to the NAMESPACE response. + + The TRANSLATION extension to the NAMESPACE response returns a single + string, containing the modified UTF-7 [RFC3501] encoded translation + of the namespace prefix. It is the responsibility of the client to + convert between the namespace prefix and the translation of the + namespace prefix when presenting mailbox names to the user. + + In this example, a server supports the IMAP4 NAMESPACE command. It + uses no prefix to the user's Personal Namespace, a prefix of "Other + Users" to its Other Users' Namespace, and a prefix of "Public + Folders" to its only Shared Namespace. Since a client will often + display these prefixes to the user, the server includes a translation + of them that can be presented to the user. + + C: A001 LANGUAGE DE-IT + S: * NAMESPACE (("" "/")) (("Other Users/" "/" "TRANSLATION" + ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" + "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) + S: A001 OK LANGUAGE-Befehl ausgefuehrt + + + + + + + + + + + + + + + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 7] + +RFC 5255 IMAP Internationalization June 2008 + + +3.5. Formal Syntax + + The following syntax specification inherits ABNF [RFC5234] rules from + IMAP4rev1 [RFC3501], IMAP4 Namespace [RFC2342], Tags for the + Identifying Languages [RFC4646], UTF-8 [RFC3629], and Collected + Extensions to IMAP4 ABNF [RFC4466]. + + command-any =/ language-cmd + ; LANGUAGE command is valid in all states + + language-cmd = "LANGUAGE" *(SP lang-range-quoted) + + response-payload =/ language-data + + language-data = "LANGUAGE" SP "(" lang-tag-quoted *(SP + lang-tag-quoted) ")" + + namespace-trans = SP DQUOTE "TRANSLATION" DQUOTE SP "(" string ")" + ; the string is encoded in Modified UTF-7. + ; this is a subset of the syntax permitted by + ; the Namespace-Response-Extension rule in [RFC4466] + + lang-range-quoted = astring + ; Once any literal wrapper or quoting is removed, this + ; follows the language-range rule in [RFC4647] + + lang-tag-quoted = astring + ; Once any literal wrapper or quoting is removed, this follows + ; the Language-Tag rule in [RFC4646] + + resp-text = ["[" resp-text-code "]" SP ] UTF8-TEXT-CHAR + *(UTF8-TEXT-CHAR / "[") + ; After the server is changed to a language other than + ; i-default, this resp-text rule replaces the resp-text + ; rule from [RFC3501]. + + UTF8-TEXT-CHAR = %x20-5A / %x5C-7E / UTF8-2 / UTF8-3 / UTF8-4 + ; UTF-8 excluding 7-bit control characters and "[" + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 8] + +RFC 5255 IMAP Internationalization June 2008 + + +4. I18NLEVEL=1 and I18NLEVEL=2 Extensions + +4.1. Introduction and Overview + + IMAP4rev1 [RFC3501] includes the SEARCH command that can be used to + locate messages matching criteria including human-readable text. The + SORT extension [SORT] to IMAP allows the client to ask the server to + determine the order of messages based on criteria including human- + readable text. These mechanisms require the ability to support non- + English search and sort functions. + + Section 4 defines two IMAP extensions for internationalizing IMAP + SEARCH, SORT, and THREAD [SORT] using the comparator framework + [RFC4790]. + + The I18NLEVEL=1 extension updates SEARCH/SORT/THREAD to use + i;unicode-casemap comparator, as defined in [UCM]. See Sections 4.2 + and 4.3 for more details. + + The I18NLEVEL=2 extension is a superset of the I18NLEVEL=1 extension. + It adds to I18NLEVEL=1 extension the ability to determine the active + comparator (see definition below) and to negotiate use of comparators + using the COMPARATOR command. It also adds the COMPARATOR response + that indicates the active comparator and possibly other available + comparators. See Sections 4.2 and 4.4 for more details. + +4.2. Requirements Common to Both I18NLEVEL=1 and I18NLEVEL=2 + + The term "default comparator" refers to the comparator that is used + by SEARCH and SORT absent any negotiation using the COMPARATOR + command (see Section 4.7). The term "active comparator" refers to + the comparator which will be used within a session, e.g., by SEARCH + and SORT. The COMPARATOR command is used to change the active + comparator. + + The active comparator applies to the following SEARCH keys: "BCC", + "BODY", "CC", "FROM", "SUBJECT", "TEXT", "TO", and "HEADER". If the + server also advertises the "SORT" extension, then the active + comparator applies to the following SORT keys: "CC", "FROM", + "SUBJECT", and "TO". If the server advertises THREAD=ORDEREDSUBJECT, + then the active comparator applies to the ORDEREDSUBJECT threading + algorithm. If the server advertises THREAD=REFERENCES, then the + active comparator applies to the subject field comparisons done by + REFERENCES threading algorithm. Future extensions may choose to + apply the active comparator to their SEARCH keys. + + + + + + +Newman, et al. Standards Track [Page 9] + +RFC 5255 IMAP Internationalization June 2008 + + + For SORT and THREAD, the pre-processing necessary to extract the base + subject text from a Subject header occurs prior to the application of + a comparator. + + A server that advertises I18NLEVEL=1 or I18NLEVEL=2 extension MUST + implement the i;unicode-casemap comparator, as defined in [UCM]. + + A server that advertises I18NLEVEL=1 or I18NLEVEL=2 extension MUST + support UTF-8 as a SEARCH charset. + +4.3. I18NLEVEL=1 Extension Requirements + + An IMAP server that satisfies all requirements specified in Sections + 4.2 and 4.6 (and that doesn't support/advertise any other + I18NLEVEL= extension, where n > 1) MUST list the keyword + I18NLEVEL=1 in its CAPABILITY data once IMAP enters the authenticated + state, and MAY list that keyword in other states. + +4.4. I18NLEVEL=2 Extension Requirements + + An IMAP server that satisfies all requirements specified in Sections + 4.2, 4.4, and 4.6-4.10 (and that doesn't support/advertise any other + I18NLEVEL= extension, where n > 2) MUST list the keyword + I18NLEVEL=2 in its CAPABILITY data once IMAP enters the authenticated + state, and MAY list that keyword in other states. + + A server that advertises this extension MUST implement the + i;unicode-casemap comparator, as defined in [UCM]. It MAY implement + other comparators from the IANA registry established by [RFC4790]. + See also Section 4.5 of this document. + + A server that advertises this extension SHOULD use i;unicode-casemap + as the default comparator. (Note that i;unicode-casemap is the + default comparator for I18NLEVEL=1, but not necessarily the default + for I18NLEVEL=2.) The selection of the default comparator MAY be + adjustable by the server administrator, and MAY be sensitive to the + current user. Once the IMAP connection enters authenticated state, + the default comparator MUST remain static for the remainder of that + connection. + + Note that since SEARCH uses the substring operation, IMAP servers can + only implement collations that offer the substring operation (see + [RFC4790], Section 4.2.2). Since SORT uses the ordering operation + (which in turn uses the equality operation), IMAP servers that + advertise the SORT extension can only implement collations that offer + all three operations (see [RFC4790], Sections 4.2.2-4.2.4). + + + + + +Newman, et al. Standards Track [Page 10] + +RFC 5255 IMAP Internationalization June 2008 + + + If the active collation does not provide the operations needed by an + IMAP command, the server MUST respond with a tagged BAD. + +4.5. Compatibility Notes + + Several server implementations deployed prior to the publication of + this specification comply with I18NLEVEL=1 (see Section 4.3), but do + not advertise that. Other legacy servers use the i;ascii-casemap + comparator (see [RFC4790]). + + There is no good way for a client to know which comparator a legacy + server uses. If the client has to assume the worst, it may end up + doing expensive local operations to obtain i;unicode-casemap + comparisons even though the server implements it. + + Legacy server implementations which comply with I18NLEVEL=1 should be + updated to advertise I18NLEVEL=1. All server implementations should + eventually be updated to comply with the I18NLEVEL=2 extension. + +4.6. Comparators and Character Encodings + + RFC 3501, Section 6.4.4, says: + + In all search keys that use strings, a message matches the key + if the string is a substring of the field. The matching is + case-insensitive. + + When performing the SEARCH operation, the active comparator is + applied instead of the case-insensitive matching specified above. + + An IMAP server which performs collation operations (e.g., as part of + commands such as SEARCH, SORT, and THREAD) does so according to the + following procedure: + + (a) MIME encoding (for example, see [RFC2047] for headers and + [RFC2045] for body parts) MUST be removed in the texts being + collated. + + If MIME encoding removal fails for a message (e.g., a body part + of the message has an unsupported Content-Transfer-Encoding, uses + characters not allowed by the Content-Transfer-Encoding, etc.), + the collation of this message is undefined by this specification, + and is handled in an implementation-dependent manner. + + (b) The decoded text from (a) MUST be converted to the charset + expected by the active comparator. + + + + + +Newman, et al. Standards Track [Page 11] + +RFC 5255 IMAP Internationalization June 2008 + + + (c) For the substring operation: + + If step (b) failed (e.g., the text is in an unknown charset, + contains a sequence that is not valid according in that charset, + etc.), the original decoded text from (a) (i.e., before the + charset conversion attempt) is collated using the i;octet + comparator (see [RFC4790]). + + If step (b) was successful, the converted text from (b) is + collated according to the active comparator. + + For the ordering operation: + + All strings that were successfully converted by step (b) are + separated from all strings that failed step (b). Strings in each + group are collated independently. All strings successfully + converted by step (b) are then validated by the active + comparator. Strings that pass validation are collated using the + active comparator. All strings that either fail step (b) or fail + the active collation's validity operation are collated (after + applying step (a)) using the i;octet comparator (see [RFC4790]). + The resulting sorted list is produced by appending all collated + "failed" strings after all strings collated using the active + comparator. + + Example: The following example demonstrates ordering of 4 + different strings using the i;unicode-casemap [UCM] comparator. + Strings are represented using hexadecimal notation used by ABNF + [RFC5234]. + + (1) %xD0 %xC0 %xD0 %xBD %xD0 %xB4 %xD1 %x80 %xD0 %xB5 + %xD0 %xB9 (labeled with charset=UTF-8) + (2) %xD1 %x81 %xD0 %x95 %xD0 %xA0 %xD0 %x93 %xD0 %x95 + %xD0 %x99 (labeled with charset=UTF-8) + (3) %xD0 %x92 %xD0 %xB0 %xD1 %x81 %xD0 %xB8 %xD0 %xBB + %xD0 %xB8 %xFF %xB9 (labeled with charset=UTF-8) + (4) %xE1 %xCC %xC5 %xCB %xD3 %xC5 %xCA (labeled with + charset=KOI8-R) + + Step (b) will convert string (4) to the following sequence of + octets (in UTF-8): + + %xD0 %x90 %xD0 %xBB %xD0 %xB5 %xD0 %xBA %xD1 %x81 %xD0 + %xB5 %xD0 %xB9 + + and will reject strings (1) and (3), as they contain octets not + allowed in charset=UTF-8. + + + + +Newman, et al. Standards Track [Page 12] + +RFC 5255 IMAP Internationalization June 2008 + + + After that, using the i;unicode-casemap collation, string (4) + will collate before string (2). Using the i;octet collation on + the original strings, string (3) will collate before string (1). + So the final ordering is as follows: (4) (2) (3) (1). + + If the substring operation (e.g., IMAP SEARCH) of the active + comparator returns the "undefined" result (see Section 4.2.3 of + [RFC4790]) for either the text specified in the SEARCH command or the + message text, then the operation is repeated on the result of step + (a) using the i;octet comparator. + + The ordering operation (e.g., IMAP SORT and THREAD) SHOULD collate + the following together: strings encoded using unknown or invalid + character encodings, strings in unrecognized charsets, and invalid + input (as defined by the active collation). + +4.7. COMPARATOR Command + + Arguments: Optional comparator order arguments. + + Response: A possible COMPARATOR response (see Section 4.8). + + Result: OK - Command completed + NO - No matching comparator found + BAD - Arguments invalid + + The COMPARATOR command is valid in authenticated and selected states. + + The COMPARATOR command is used to determine or change the active + comparator. When issued with no arguments, it results in a + COMPARATOR response indicating the currently active comparator. + + When issued with one or more comparator arguments, it changes the + active comparator as directed. (If more than one installed + comparator is matched by an argument, the first argument wins.) The + COMPARATOR response lists all matching comparators if more than one + matches the specified patterns. + + The argument "default" refers to the server's default comparator. + Otherwise, each argument is a collation specification as defined in + the Internet Application Protocol Comparator Registry [RFC4790]. + + < The client requests activating a Czech comparator if possible, + or else a generic international comparator which it considers + suitable for Czech. The server picks the first supported + comparator. > + + + + + +Newman, et al. Standards Track [Page 13] + +RFC 5255 IMAP Internationalization June 2008 + + + C: A001 COMPARATOR "cz;*" i;basic + S: * COMPARATOR i;basic + S: A001 OK Will use i;basic for collation + +4.8. COMPARATOR Response + + Contents: The active comparator. An optional list of available + matching comparators + + The COMPARATOR response occurs as a result of a COMPARATOR command. + The first argument in the comparator response is the name of the + active comparator. The second argument is a list of comparators + which matched any of the arguments to the COMPARATOR command and is + present only if more than one match is found. + +4.9. BADCOMPARATOR Response Code + + This response code SHOULD be returned as a result of server failing + an IMAP command (returning NO), when the server knows that none of + the specified comparators match the requested comparator(s). + +4.10. Formal Syntax + + The following syntax specification inherits ABNF [RFC5234] rules from + IMAP4rev1 [RFC3501] and the Internet Application Protocol Comparator + Registry [RFC4790]. + + command-auth =/ comparator-cmd + + resp-text-code =/ "BADCOMPARATOR" + + comparator-cmd = "COMPARATOR" *(SP comp-order-quoted) + + response-payload =/ comparator-data + + comparator-data = "COMPARATOR" SP comp-sel-quoted [SP "(" + comp-id-quoted *(SP comp-id-quoted) ")"] + + comp-id-quoted = astring + ; Once any literal wrapper or quoting is removed, this + ; follows the collation-id rule from [RFC4790] + + comp-order-quoted = astring + ; Once any literal wrapper or quoting is removed, this + ; follows the collation-order rule from [RFC4790] + + + + + + +Newman, et al. Standards Track [Page 14] + +RFC 5255 IMAP Internationalization June 2008 + + + comp-sel-quoted = astring + ; Once any literal wrapper or quoting is removed, this + ; follows the collation-selected rule from [RFC4790] + +5. Other IMAP Internationalization Issues + + The following sections provide an overview of various other IMAP + internationalization issues. These issues are not resolved by this + specification, but could be resolved by other standards work, such as + that being done by the EAI working group (see [IMAP-EAI]). + +5.1. Unicode Userids and Passwords + + IMAP4rev1 currently restricts the userid and password fields of the + LOGIN command to US-ASCII. The "userid" and "password" fields of the + IMAP LOGIN command are restricted to US-ASCII only until a future + standards track RFC states otherwise. Servers are encouraged to + validate both fields to make sure they conform to the formal syntax + of UTF-8 and to reject the LOGIN command if that syntax is violated. + Servers MAY reject the LOGIN command if either the "userid" or + "password" field contains an octet with the highest bit set. + + When AUTHENTICATE is used, some servers may support userids and + passwords in Unicode [RFC3490] since SASL (see [RFC4422]) allows + that. However, such userids cannot be used as part of email + addresses. + +5.2. UTF-8 Mailbox Names + + The modified UTF-7 mailbox naming convention described in Section + 5.1.3 of RFC 3501 is best viewed as an transition from the status quo + in 1996 when modified UTF-7 was first specified. At that time, there + was widespread unofficial use of local character sets such as ISO- + 8859-1 and Shift-JIS for non-ASCII mailbox names, with resultant + non-interoperability. + + The requirements in Section 5.1 of RFC 3501 are very important if + we're ever going to be able to deploy UTF-8 mailbox names. Servers + are encouraged to enforce them. + +5.3. UTF-8 Domains, Addresses, and Mail Headers + + There is now an IETF standard for "Internationalizing Domain Names in + Applications (IDNA)" [RFC3490]. While IMAP clients are free to + support this standard, an argument can be made that it would be + helpful to simple clients if the IMAP server could perform this + conversion (the same argument would apply to MIME header encoding + + + + +Newman, et al. Standards Track [Page 15] + +RFC 5255 IMAP Internationalization June 2008 + + + [RFC2047]). However, it would be unwise to move forward with such + work until the work in progress to define the format of international + email addresses is complete. + +6. IANA Considerations + + IANA added LANGUAGE, I18NLEVEL=1, and I18NLEVEL=2 to the IMAP4 + Capabilities Registry. + +7. Security Considerations + + The LANGUAGE extension makes a new command available in "Not + Authenticated" state in IMAP. Some IMAP implementations run with + root privilege when the server is in "Not Authenticated" state and do + not revoke that privilege until after authentication is complete. + Such implementations are particularly vulnerable to buffer overflow + security errors at this stage and need to implement parsing of this + command with extra care. + + A LANGUAGE command issued prior to activation of a security layer is + subject to an active attack that suppresses or modifies the + negotiation, and thus makes STARTTLS or authentication error messages + more difficult to interpret. This is not a new attack as the error + messages themselves are subject to active attack. Clients MUST re- + issue the LANGUAGE command once a security layer is active, in order + to prevent this attack from impacting subsequent protocol operations. + + LANGUAGE, I18NLEVEL=1, and I18NLEVEL=2 extensions use the UTF-8 + charset; thus, the security considerations for UTF-8 [RFC3629] are + relevant. However, neither uses UTF-8 for identifiers, so the most + serious concerns do not apply. + +8. Acknowledgements + + The LANGUAGE extension is based on a previous document by Mike + Gahrns, a substantial portion of the text in that section was written + by him. Many people have participated in discussions about an IMAP + Language extension in the various fora of the IETF and Internet + working groups, so any list of contributors is bound to be + incomplete. However, the authors would like to thank Andrew McCown + for early work on the original proposal, John Myers for suggestions + regarding the namespace issue, along with Jutta Degener, Mark + Crispin, Mark Pustilnik, Larry Osterman, Cyrus Daboo, Martin Duerst, + Timo Sirainen, Ben Campbell, and Magnus Nystrom for their many + suggestions that have been incorporated into this document. + + Initial discussion of the I18NLEVEL=2 extension involved input from + Mark Crispin and other participants of the IMAP Extensions WG. + + + +Newman, et al. Standards Track [Page 16] + +RFC 5255 IMAP Internationalization June 2008 + + +9. Relevant Sources of Documents for Internationalized IMAP + Implementations + + This is a non-normative list of sources to consider when implementing + i18n-aware IMAP software. + + o The LANGUAGE and I18NLEVEL=2 extensions to IMAP (this + specification). + + o The 8-bit rules for mailbox naming in Section 5.1 of RFC 3501. + + o The Mailbox International Naming Convention in Section 5.1.3 of + RFC 3501. + + o MIME [RFC2045] for message bodies. + + o MIME header encoding [RFC2047] for message headers. + + o The IETF EAI working group. + + o MIME Parameter Value and Encoded Word Extensions [RFC2231] for + filenames. Quality IMAP server implementations will + automatically combine multipart parameters when generating the + BODYSTRUCTURE. There is also some deployed non-standard use of + MIME header encoding inside double quotes for filenames. + + o IDNA [RFC3490] and punycode [RFC3492] for domain names + (currently only relevant to IMAP clients). + + o The UTF-8 charset [RFC3629]. + + o The IETF policy on Character Sets and Languages [RFC2277]. + +10. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2277] Alvestrand, H., "IETF Policy on Character Sets and + Languages", BCP 18, RFC 2277, January 1998. + + [RFC2342] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, May + 1998. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + + + + +Newman, et al. Standards Track [Page 17] + +RFC 5255 IMAP Internationalization June 2008 + + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [RFC4422] Melnikov, A., Ed., and K. Zeilenga, Ed., "Simple + Authentication and Security Layer (SASL)", RFC 4422, June + 2006. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC4646] Phillips, A. and M. Davis, "Tags for Identifying + Languages", BCP 47, RFC 4646, September 2006. + + [RFC4647] Phillips, A. and M. Davis, "Matching of Language Tags", + BCP 47, RFC 4647, September 2006. + + [RFC4790] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet + Application Protocol Collation Registry", RFC 4790, March + 2007. + + [SORT] Crispin, M. and K. Murchison, "Internet Message Access + Protocol - SORT and THREAD Extensions", RFC 5256, June + 2008. + + [UCM] Crispin, M., "i;unicode-casemap - Simple Unicode Collation + Algorithm", RFC 5051, October 2007. + + [RFC2045] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII Text", + RFC 2047, November 1996. + +11. Informative References + + [RFC2231] Freed, N. and K. Moore, "MIME Parameter Value and Encoded + Word Extensions: Character Sets, Languages, and + Continuations", RFC 2231, November 1997. + + [RFC3490] Faltstrom, P., Hoffman, P., and A. Costello, + "Internationalizing Domain Names in Applications (IDNA)", + RFC 3490, March 2003. + + + +Newman, et al. Standards Track [Page 18] + +RFC 5255 IMAP Internationalization June 2008 + + + [RFC3492] Costello, A., "Punycode: A Bootstring encoding of Unicode + for Internationalized Domain Names in Applications + (IDNA)", RFC 3492, March 2003. + + [METADATA] Daboo, C., "IMAP METADATA Extension", Work in Progress, + April 2008. + + [IMAP-EAI] Resnick, P., and C. Newman, "IMAP Support for UTF-8", Work + in Progress, November 2007. + +Authors' Addresses + + Chris Newman + Sun Microsystems + 3401 Centrelake Dr., Suite 410 + Ontario, CA 91761 + US + + EMail: chris.newman@sun.com + + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + EMail: arnt@oryx.com + Fax: +49 89 4502 9758 + + + Alexey Melnikov + Isode Limited + 5 Castle Business Village, 36 Station Road, + Hampton, Middlesex, TW12 2BX, UK + + EMail: Alexey.Melnikov@isode.com + + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 19] + +RFC 5255 IMAP Internationalization June 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Newman, et al. Standards Track [Page 20] + diff --git a/docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt b/docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt new file mode 100644 index 0000000..1d088e5 --- /dev/null +++ b/docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt @@ -0,0 +1,1739 @@ + + + + + + +Network Working Group C. Daboo +Request for Comments: 5257 Apple Inc. +Category: Experimental R. Gellens + QUALCOMM Incorporated + June 2008 + + + Internet Message Access Protocol - ANNOTATE Extension + +Status of This Memo + + This memo defines an Experimental Protocol for the Internet + community. It does not specify an Internet standard of any kind. + Discussion and suggestions for improvement are requested. + Distribution of this memo is unlimited. + +Abstract + + The ANNOTATE extension to the Internet Message Access Protocol + permits clients and servers to maintain "meta data" for messages, or + individual message parts, stored in a mailbox on the server. For + example, this can be used to attach comments and other useful + information to a message. It is also possible to attach annotations + to specific parts of a message, so that, for example, they could be + marked as seen, or important, or a comment added. + + Note that this document was the product of a WG that had good + consensus on how to approach the problem. Nevertheless, the WG felt + it did not have enough information on implementation and deployment + hurdles to meet all of the requirements of a Proposed Standard. The + IETF solicits implementations and implementation reports in order to + make further progress. + + Implementers should be aware that this specification may change in an + incompatible manner when going to Proposed Standard status. However, + any incompatible changes will result in a new capability name being + used to prevent problems with any deployments of the experimental + extension. + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 1] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +Table of Contents + + 1. Introduction and Overview .......................................3 + 2. Conventions Used in This Document ...............................4 + 3. Data Model ......................................................4 + 3.1. Overview ...................................................4 + 3.2. Namespace of Entries and Attributes ........................4 + 3.2.1. Entry Names .........................................5 + 3.2.2. Attribute Names .....................................7 + 3.3. Private Versus Shared ......................................7 + 3.4. Access Control .............................................8 + 3.5. Access to Standard IMAP Flags and Keywords ................11 + 4. IMAP Protocol Changes ..........................................11 + 4.1. General Considerations ....................................11 + 4.2. ANNOTATE Parameter with the SELECT/EXAMINE Commands .......12 + 4.3. ANNOTATION Message Data Item in FETCH Command .............12 + 4.4. ANNOTATION Message Data Item in FETCH Response ............14 + 4.5. ANNOTATION Message Data Item in STORE .....................16 + 4.6. ANNOTATION Interaction with COPY ..........................18 + 4.7. ANNOTATION Message Data Item in APPEND ....................18 + 4.8. ANNOTATION Criterion in SEARCH ............................19 + 4.9. ANNOTATION Key in SORT ....................................20 + 4.10. New ACL Rights ...........................................21 + 5. Formal Syntax ..................................................21 + 6. IANA Considerations ............................................23 + 6.1. Entry and Attribute Registration Template .................23 + 6.2. Entry Registrations .......................................24 + 6.2.1. /comment ...........................................24 + 6.2.2. /flags .............................................24 + 6.2.3. /altsubject ........................................25 + 6.2.4. //comment ............................25 + 6.2.5. //flags/seen .........................26 + 6.2.6. //flags/answered .....................26 + 6.2.7. //flags/flagged ......................27 + 6.2.8. //flags/forwarded ....................27 + 6.3. Attribute Registrations ...................................28 + 6.3.1. value ..............................................28 + 6.3.2. size ...............................................28 + 6.4. Capability Registration ...................................28 + 7. Internationalization Considerations ............................29 + 8. Security Considerations ........................................29 + 9. References .....................................................29 + 9.1. Normative References ......................................29 + 9.2. Informative References ....................................30 + 10. Acknowledgments ...............................................30 + + + + + + +Daboo & Gellens Experimental [Page 2] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +1. Introduction and Overview + + The ANNOTATE extension is present in any IMAP [RFC3501] + implementation that returns "ANNOTATE-EXPERIMENT-1" as one of the + supported capabilities in the CAPABILITY response. + + This extension makes the following changes to the IMAP protocol: + + a. adds a new ANNOTATION message data item for use in FETCH. + + b. adds a new ANNOTATION message data item for use in STORE. + + c. adds a new ANNOTATION search criterion for use in SEARCH. + + d. adds a new ANNOTATION sort key for use in the SORT extension. + + e. adds a new ANNOTATION data item for use in APPEND. + + f. adds a new requirement on the COPY command. + + g. adds a new ANNOTATE parameter for use with the SELECT/EXAMINE + commands. + + h. adds two new response codes to indicate store failures of + annotations. + + i. adds a new untagged response code for the SELECT or EXAMINE + commands to indicate the maximum sized annotation that can be + stored. + + j. adds a new Access Control List (ACL) "bit" for use with the ACL + extensions [RFC2086] and [RFC4314]. + + The data model used for the storage of annotations is based on the + Application Configuration Access Protocol [RFC2244]. Note that there + is no inheritance in annotations. + + If a server supports annotations, then it MUST store all annotation + data permanently, i.e., there is no concept of "session only" + annotations that would correspond to the behavior of "session" flags + as defined in the IMAP base specification. + + In order to provide optimum support for a disconnected client (one + that needs to synchronize annotations for use when offline), servers + SHOULD also support the Conditional STORE [RFC4551] extension. + + The rest of this document describes the data model and protocol + changes more rigorously. + + + +Daboo & Gellens Experimental [Page 3] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +2. Conventions Used in This Document + + The examples in this document use "C:" and "S:" to indicate lines + sent by the client and server, respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + +3. Data Model + +3.1. Overview + + The data model for annotations in ANNOTATE uses a uniquely named + entry that contains a set of standard attributes. Thus, a single + coherent unit of "meta data" for a message is stored as a single + entry, made up of several attributes. + + For example, a comment annotation added to a message has an entry + name of "/comment". This entry is composed of several attributes + such as "value", "size", etc., that contain the properties and data + of the entry. + + The protocol changes to IMAP, described below, allow a client to + access or change the values of any attribute in any entry in a + message annotation, assuming it has sufficient access rights to do so + (see Section 3.4 for specifics). + +3.2. Namespace of Entries and Attributes + + A message may contain zero or more annotations, each of which is a + single uniquely named entry. Each entry has a hierarchical name, + with each component of the name separated by a slash ("/"). An entry + name MUST NOT contain two consecutive "/" characters and MUST NOT end + with a "/" character. + + Each entry is made up of a set of attributes. Each attribute has a + hierarchical name, with each component of the name separated by a + period ("."). An attribute name MUST NOT contain two consecutive "." + characters and MUST NOT end with a "." character. + + The value of an attribute is "NIL" (has no value), or is a string of + zero or more octets. + + Entry and attribute names MUST NOT contain asterisk ("*") or percent + ("%") characters, and MUST NOT contain non-ASCII characters or the + NULL octet. Invalid entry or attribute names result in a BAD + response in any IMAP commands where they are used. + + + +Daboo & Gellens Experimental [Page 4] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Attribute names MUST NOT contain any hierarchical components with the + names "priv" or "shared", as those have special meaning (see Section + 3.3). + + Entry and attribute names are case-sensitive. + + Use of control or punctuation characters in entry and attribute names + is strongly discouraged. + + This specification defines an initial set of entry and attribute + names available for use in message annotations. In addition, an + extension mechanism is described to allow additional names to be + added as needed. + +3.2.1. Entry Names + + Entry names MUST be specified in a standards track or IESG approved + experimental RFC, or fall under the vendor namespace. See Section + 6.1 for the registration template. + + / + Defines the top-level of entries associated with an entire + message. This entry itself does not contain any attributes. All + entries that start with a numeric character ("0" - "9") refer to + an annotation on a specific body part. All other entries are for + annotations on the entire message. + + /comment + Defines a comment or note associated with an entire message. + + /flags + This entry hierarchy is reserved for future use. + + /altsubject + Contains text supplied by the message recipient to be used by the + client, instead of the original message Subject. + + /vendor/ + Defines the top-level of entries associated with an entire message + as created by a particular product of some vendor. These sub- + entries can be used by vendors to provide client-specific + annotations. The vendor-token MUST be registered with IANA, using + the [RFC2244] vendor subtree registry. + + / + Defines the top-level of entries associated with a specific body + part of a message. This entry itself does not contain any + attributes. The section-part is a numeric part specifier. Its + + + +Daboo & Gellens Experimental [Page 5] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + syntax is the same as the section-part ABNF element defined in + [RFC3501]. The server MUST return a BAD response if the client + uses an incorrect part specifier (either incorrect syntax or a + specifier referring to a non-existent part). The server MUST + return a BAD response if the client uses an empty part specifier + (which is used in IMAP to represent the entire message). + + //comment + Defines a comment or note associated with a specific body part of + a message. + + //flags + Defines the top-level of entries associated with the flag state + for a specific body part of a message. All sub-entries are + maintained entirely by the client. There is no implicit change to + any flag by the server. + + //flags/seen + This is similar to the IMAP \Seen flag, except it applies + to only the body part referenced by the entry. + + //flags/answered + This is similar to the IMAP \Answered flag, except it + applies to only the body part referenced by the entry. + + //flags/flagged + This is similar to the IMAP \Flagged flag, except it + applies to only the body part referenced by the entry. + + //flags/forwarded + This is similar to the IMAP $Forwarded keyword, except it + applies to only the body part referenced by the entry. + + Defines flags for a specific body part of a message. The "value" + attribute of each of the entries described above must be either + "1", "0", or "NIL". "1" corresponds to the flag being set. + + //vendor/ + Defines the top-level of entries associated with a specific body + part of a message as created by a particular product of some + vendor. This entry can be used by vendors to provide client + specific annotations. The vendor-token MUST be registered with + IANA. + + + + + + + + +Daboo & Gellens Experimental [Page 6] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +3.2.2. Attribute Names + + Attribute names MUST be specified in a standards track or IESG + approved experimental RFC. See Section 6.1 for the registration + template. + + All attribute names implicitly have a ".priv" and a ".shared" suffix + that maps to private and shared versions of the entry. Searching or + fetching without using either suffix will include both. The client + MUST specify either a ".priv" or ".shared" suffix when storing an + annotation or sorting on annotations. + + value + A string or binary data representing the value of the annotation. + To delete an annotation, the client can store "NIL" into the + value. If the client requests the value attribute for a non- + existent entry, then the server MUST return "NIL" for the value. + The content represented by the string is determined by the + content-type used to register the entry (see Section 6.1 for entry + registration templates). Where applicable, the registered + content-type MUST include a charset parameter. Text values SHOULD + use the utf-8 [RFC3629] character set. Note that binary data + (data which may contain the NULL octet) is allowed (e.g., for + storing images), and this extension uses the "literal8" syntax + element [RFC4466] to allow such data to be written to or read from + the server. + + size + The size of the value, in octets. Set automatically by the + server, read-only to clients. If the client requests the size + attribute for a non-existent entry, then the server MUST return + "0" (zero) for the size. + +3.3. Private Versus Shared + + Some IMAP mailboxes are private, accessible only to the owning user. + Other mailboxes are not, either because the owner has set an ACL + [RFC4314] that permits access by other users, or because it is a + shared mailbox. + + This raises the issue of shared versus private annotations. + + If all annotations are private, it is then impossible to have + annotations in a shared or otherwise non-private mailbox be visible + to other users. This eliminates what could be a useful aspect of + annotations in a shared environment. An example of such use is a + shared IMAP folder containing bug reports. Engineers may want to use + + + + +Daboo & Gellens Experimental [Page 7] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + annotations to add information to existing messages, indicate + assignments, status, etc. This use requires shared annotations. + + If all annotations are shared, it is impossible to use annotations + for private notes on messages in shared mailboxes. Also, modifying + an ACL to permit access to a mailbox by other users may + unintentionally expose private information. + + There are also situations in which both shared and private + annotations are useful. For example, an administrator may want to + set shared annotations on messages in a shared folder, which + individual users may wish to supplement with additional notes. + + If shared and private annotations are to coexist, we need a clear way + to differentiate them. Also, it should be as easy as possible for a + client to access both and not overlook either. There is also a + danger in allowing a client to store an annotation without knowing if + it is shared or private. + + This document proposes two standard suffixes for all attributes: + ".shared" and ".priv". A SEARCH or FETCH command that specifies + neither, uses both. STORE, APPEND, and SORT commands MUST explicitly + use ".priv" or ".shared" suffixes. + + If the ANNOTATE extension is present, support for shared annotations + in servers is REQUIRED, while support for private annotations in + servers is OPTIONAL. This recognizes the fact that support for + private annotations may introduce a significant increase in + complexity to a server in terms of tracking ownership of the + annotations, how quota is determined for users based on their own + annotations, etc. Clients that support the ANNOTATE extension MUST + handle both shared and private annotations. + +3.4. Access Control + + A user needs to have appropriate rights in order to read or write + ".priv" or ".shared" annotation values. How those rights are + calculated depends on whether or not the ACL [RFC2086] extension or + its update [RFC4314] is present. If a client attempts to store or + fetch an annotation to which they do not have the appropriate rights, + the server MUST respond with a NO response. + + When the ACL extension is not present, access to annotation values is + governed by the nature of the selected state, in particular whether + the mailbox was SELECTED or EXAMINED in READ-ONLY or READ-WRITE mode. + + + + + + +Daboo & Gellens Experimental [Page 8] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + When the ACL extension is present, the server MUST recognize the new + ACL "n" right, in addition to the ones defined by the ACL extension + itself. + + For ".priv" annotation values, the "r" right controls both read and + write access. When it is on, access to ".priv" annotations is + allowed; when it is off, access to ".priv" annotations is disallowed. + + For ".shared" annotation values, the "r" right controls read access. + When it is on, ".shared" annotations can be read; when it is off, + ".shared" annotations cannot be read. + + For ".shared" annotation values, the "n" right controls write access. + When it is on, ".shared" annotations can be changed or created + through either a STORE or APPEND command; when it is off, ".shared" + annotations cannot be changed or created. The "n" right constitutes + a "shared flag right" as defined in Section 6.2 of [RFC4314]. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 9] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + A summary of all the access control restrictions is tabulated below + + +---------------+---------------+-----------------------------------+ + | Server Type | Action on | Type of mailbox | + | | annotation | | + +===============+===============+===================================+ + | | | | + | | read .priv | Any mailbox that can be SELECTED | + | | values | or EXAMINED. | + | | | | + | +---------------+-----------------------------------+ + | | | | + | | write .priv | Any SELECTED [READ-WRITE] mailbox.| + | | values | SELECTED [READ-ONLY] mailboxes MAY| + | Server | | also permit writes. | + | without | | | + | ACL Extension +---------------+-----------------------------------+ + | | | | + | | read .shared | Any mailbox that can be SELECTED | + | | values | or EXAMINED. | + | | | | + | +---------------+-----------------------------------+ + | | | | + | | write .shared | Any mailbox that can be SELECTED | + | | values | or EXAMINED and is [READ-WRITE]. | + | | | | + +---------------+---------------+-----------------------------------+ + | | | | + | | read .priv | Any mailbox with the "r" | + | | values | ACL right. | + | | | | + | +---------------+-----------------------------------+ + | | | | + | | write .priv | Any mailbox with the "r" | + | Server | values | ACL right. | + | with | | | + | ACL Extension +---------------+-----------------------------------+ + | | | | + | | read .shared | Any mailbox with the "r" | + | | values | ACL right. | + | | | | + | +---------------+-----------------------------------+ + | | | | + | | write .shared | Any mailbox with the "n" | + | | values | ACL right. | + | | | | + +---------------+---------------+-----------------------------------+ + + + + +Daboo & Gellens Experimental [Page 10] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +3.5. Access to Standard IMAP Flags and Keywords + + Due to the ambiguity of how private and shared values would map to + the base IMAP flag and keyword values, the ANNOTATE extension does + not expose IMAP flags or keywords as entries. However, the /flags + annotation entry is reserved for future use and MUST NOT be used by + clients or servers supporting this extension. + + Clients that need to implement shared and private "flags" can create + their own annotation entries for those, completely bypassing the base + IMAP flag/keyword behavior. + +4. IMAP Protocol Changes + +4.1. General Considerations + + Servers may be able to offer only a limited level of support for + annotations in mailboxes, and it is useful for clients to be able to + know what level of support is available. Servers MUST return an + ANNOTATIONS response code during the SELECT or EXAMINE command for a + mailbox to indicate the level of support. Possible data items used + with the ANNOTATIONS response code are: + + "NONE" - this indicates that the mailbox being selected does not + support annotations at all. Clients MUST NOT attempt to use + annotation extensions in commands for this mailbox. + + "READ-ONLY" - this indicates that the annotations supported by the + mailbox cannot be changed by the client. Clients MUST NOT attempt + to store annotations on any messages in a mailbox with this + response code. + + "NOPRIVATE" - this indicates that the server does not support + private annotations on the mailbox. Only shared annotations are + supported. Clients SHOULD only attempt to read or store + annotations attributes with the ".shared" suffix. If a client + uses an attribute with the ".priv" suffix in a FETCH command, then + servers should return the attribute value in the FETCH response as + "NIL". If a client uses an attribute with the ".priv" suffix in a + STORE command (or an APPEND command targeted at the mailbox), then + the server MUST return a NO response. + + numeric values - if servers support writable annotations, then the + server MUST indicate the maximum size in octets for an annotation + value by providing the maximum size value in the response code. + Clients MUST NOT store annotation values of a size greater than + the amount indicated by the server. Servers MUST accept a minimum + + + + +Daboo & Gellens Experimental [Page 11] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + annotation data size of at least 1024 octets if annotations can be + written. + + In addition, the server MAY limit the total number of annotations for + a single message. However, the server MUST provide a minimum + annotation count per message of at least 10. + +4.2. ANNOTATE Parameter with the SELECT/EXAMINE Commands + + The ANNOTATE extension defines a single optional SELECT parameter + [RFC4466] "ANNOTATE", which is used to turn on unsolicited responses + for annotations as described in Section 4.4. This optional parameter + results in a per-mailbox state change, i.e., it must be used in each + SELECT/EXAMINE command in order to be effective, irrespective of + whether it was used in a previous SELECT/EXAMINE during the same + session. + + Example: + + C: a SELECT INBOX (ANNOTATE) + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] + S: * 10268 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 10268] + S: * OK [UIDVALIDITY 890061587] + S: * OK [UIDNEXT 34643] + S: * OK [ANNOTATIONS 20480 NOPRIVATE] + S: a OK [READ-WRITE] Completed + + In the above example, a SELECT command with the ANNOTATE parameter + is issued. The response from the server includes the required + ANNOTATIONS response that indicates that the server supports + annotations up to a maximum size of 20480 octets, and does not + support private annotations (only shared). + +4.3. ANNOTATION Message Data Item in FETCH Command + + This extension adds an ANNOTATION message data item to the FETCH + command. This allows clients to retrieve annotations for a range of + messages in the currently selected mailbox. + + ANNOTATION + + The ANNOTATION message data item, when used by the client in the + FETCH command, takes an entry specifier and an attribute + specifier. + + + +Daboo & Gellens Experimental [Page 12] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a FETCH 1 (ANNOTATION (/comment value)) + S: * 1 FETCH (ANNOTATION (/comment + (value.priv "My comment" + value.shared "Group note"))) + S: a OK Fetch complete + + In the above example, the content of the "value" attribute for the + "/comment" entry is requested by the client and returned by the + server. Since neither ".shared" nor ".priv" was specified, both + are returned. + + "*" and "%" wild card characters can be used in entry specifiers to + match one or more characters at that position, with the exception + that "%" does not match the "/" hierarchy delimiter. Thus, an entry + specifier of "/%" matches entries such as "/comment" and + "/altsubject", but not "/1/comment". + + Example: + + C: a UID FETCH 1123 (UID ANNOTATION + (/* (value.priv size.priv))) + S: * 12 FETCH (UID 1123 ANNOTATION + (/comment (value.priv "My comment" + size.priv "10") + /altsubject (value.priv "Rhinoceroses!" + size.priv "13") + /vendor/foobar/label.priv + (value.priv "label43" + size.priv "7") + /vendor/foobar/personality + (value.priv "Tallulah Bankhead" + size.priv "17"))) + S: a OK Fetch complete + + In the above example, the contents of the private "value" and + "size" attributes for any entries in the "/" hierarchy are + requested by the client and returned by the server. + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 13] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a FETCH 1 (ANNOTATION (/% value.shared)) + S: * 1 FETCH (ANNOTATION + (/comment (value.shared "Patch Mangler") + /altsubject (value.shared "Patches? We don't + need no steenkin patches!"))) + S: a OK Fetch complete + + In the above example, the contents of the shared "value" + attributes for entries at the top level only of the "/" hierarchy + are requested by the client and returned by the server. + + Entry and attribute specifiers can be lists of atomic specifiers, so + that multiple items of each type may be returned in a single FETCH + command. + + Example: + + C: a FETCH 1 (ANNOTATION + ((/comment /altsubject) value.priv)) + S: * 1 FETCH (ANNOTATION + (/comment (value.priv "What a chowder-head") + /altsubject (value.priv "How to crush beer cans"))) + S: a OK Fetch complete + + In the above example, the contents of the private "value" + attributes for the two entries "/comment" and "/altsubject" are + requested by the client and returned by the server. + +4.4. ANNOTATION Message Data Item in FETCH Response + + The ANNOTATION message data item in the FETCH response displays + information about annotations in a message. + + ANNOTATION parenthesized list + + The response consists of a list of entries, each of which have a + list of attribute-value pairs. + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 14] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a FETCH 1 (ANNOTATION (/comment value)) + S: * 1 FETCH (ANNOTATION (/comment + (value.priv "My comment" + value.shared NIL))) + S: a OK Fetch complete + + In the above example, a single entry with a single attribute-value + pair is returned by the server. Since the client did not specify + a ".shared" or ".priv" suffix, both are returned. Only the + private attribute has a value (the shared value is "NIL"). + + Example: + + C: a FETCH 1 (ANNOTATION + ((/comment /altsubject) value)) + S: * 1 FETCH (ANNOTATION + (/comment (value.priv "My comment" + value.shared NIL) + /altsubject (value.priv "My subject" + value.shared NIL))) + S: a OK Fetch complete + + In the above example, two entries, each with a single attribute- + value pair, are returned by the server. Since the client did not + specify a ".shared" or ".priv" suffix, both are returned. Only + the private attributes have values; the shared attributes are + "NIL". + + Example: + + C: a FETCH 1 (ANNOTATION + (/comment (value size))) + S: * 1 FETCH (ANNOTATION + (/comment + (value.priv "My comment" + value.shared NIL + size.priv "10" + size.shared "0"))) + S: a OK Fetch complete + + In the above example, a single entry with two attribute-value + pairs is returned by the server. Since the client did not specify + a ".shared" or ".priv" suffix, both are returned. Only the + private attributes have values; the shared attributes are "NIL". + + + + + +Daboo & Gellens Experimental [Page 15] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Servers SHOULD send ANNOTATION message data items in unsolicited + FETCH responses if an annotation entry is changed by a third-party, + and the ANNOTATE select parameter was used. This allows servers to + keep clients updated with changes to annotations by other clients. + + Unsolicited ANNOTATION responses MUST NOT include ANNOTATION data + values -- only the entry name of the ANNOTATION that has changed. + This restriction avoids sending ANNOTATION data values (which may be + large) to a client unless the client explicitly asks for the value. + + Example: + + C: a STORE 1 +FLAGS (\Seen) + S: * 1 FETCH (FLAGS (\Seen)) + ANNOTATION (/comment)) + S: a OK STORE complete + + In the above example, an unsolicited ANNOTATION response is + returned during a STORE command. The unsolicited response + contains only the entry name of the annotation that changed, and + not its value. + +4.5. ANNOTATION Message Data Item in STORE + + ANNOTATION + + Sets the specified list of entries by adding or replacing the + specified attributes with the values provided. Clients can use + "NIL" for values of attributes it wants to remove from entries. + + The ANNOTATION message data item used with the STORE command has an + implicit ".SILENT" behavior. This means the server does not generate + an untagged FETCH in response to the STORE command and assumes that + the client updates its own cache if the command succeeds. Though + note, that if the Conditional STORE extension [RFC4551] is present, + then an untagged FETCH response with a MODSEQ data item will be + returned by the server as required by [RFC4551]. + + If the server is unable to store an annotation because the size of + its value is too large, the server MUST return a tagged NO response + with a "[ANNOTATE TOOBIG]" response code. + + If the server is unable to store a new annotation because the maximum + number of allowed annotations has already been reached, the server + MUST return a tagged NO response with a "[ANNOTATE TOOMANY]" response + code. + + + + + +Daboo & Gellens Experimental [Page 16] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a STORE 1 ANNOTATION (/comment + (value.priv "My new comment")) + S: a OK Store complete + + In the above example, the entry "/comment" is created (if not + already present). Its private attribute "value" is created if not + already present, or replaced if it exists. "value.priv" is set to + "My new comment". + + Example: + + C: a STORE 1 ANNOTATION (/comment + (value.shared NIL)) + S: a OK Store complete + + In the above example, the shared "value" attribute of the entry + "/comment" is removed by storing "NIL" into the attribute. + + Multiple entries can be set in a single STORE command by listing + entry-attribute-value pairs in the list. + + Example: + + C: a STORE 1 ANNOTATION (/comment + (value.priv "Get tix Tuesday") + /altsubject + (value.priv "Wots On")) + S: a OK Store complete + + In the above example, the entries "/comment" and "/altsubject" are + created (if not already present) and the private attribute "value" + is created or replaced for each entry. + + Multiple attributes can be set in a single STORE command by listing + multiple attribute-value pairs in the entry list. + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 17] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a STORE 1 ANNOTATION (/comment + (value.priv "My new comment" + value.shared "foo's bar")) + S: a OK Store complete + + In the above example, the entry "/comment" is created (if not + already present) and the private and shared "value" attributes are + created if not already present, or replaced if they exist. + +4.6. ANNOTATION Interaction with COPY + + The COPY command can be used to move messages from one mailbox to + another on the same server. Servers that support the ANNOTATION + extension MUST, for each message being copied, copy all ".priv" + annotation data for the current user only, and all ".shared" + annotation data along with the message to the new mailbox. The only + exceptions to this are if the destination mailbox permissions are + such that either the ".priv" or ".shared" annotations are not + allowed, or if the destination mailbox is of a type that does not + support annotations or does not support storing of annotations (a + mailbox that returns a "NONE" or "READ-ONLY" response code in its + ANNOTATIONS response), or if the destination mailbox cannot support + the size of an annotation because it exceeds the ANNOTATIONS value. + Servers MUST NOT copy ".priv" annotation data for users other than + the current user. + +4.7. ANNOTATION Message Data Item in APPEND + + ANNOTATION + + Sets the specified list of entries and attributes in the resulting + message. + + The APPEND command can include annotations for the message being + appended via the addition of a new append data item [RFC4466]. The + new data item can also be used with the multi-append [RFC3502] + extension that allows multiple messages to be appended via a single + APPEND command. + + + + + + + + + + + +Daboo & Gellens Experimental [Page 18] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a APPEND drafts ANNOTATION (/comment + (value.priv "Don't send until I say so")) {310} + S: + Ready for literal data + C: MIME-Version: 1.0 + ... + C: + S: a OK APPEND completed + + In the above example, a comment with a private value is added to a + new message appended to the mailbox. The ellipsis represents the + bulk of the message. + +4.8. ANNOTATION Criterion in SEARCH + + ANNOTATION + + The ANNOTATION criterion for the SEARCH command allows a client to + search for a specified string in the value of an annotation entry of + a message. + + Messages that have annotations with entries matching , + attributes matching , and the specified string + in their values are returned in the SEARCH results. The "*" + character can be used in the entry name field to match any content in + those items. The "%" character can be used in the entry name field + to match a single level of hierarchy only. + + Only the "value", "value.priv", and "value.shared" attributes can be + searched. Clients MUST NOT specify an attribute other than either + "value", "value.priv", or "value.shared". Servers MUST return a BAD + response if the client tries to search any other attribute. + + Example: + + C: a SEARCH ANNOTATION /comment value "IMAP4" + S: * SEARCH 2 3 5 7 11 13 17 19 23 + S: a OK Search complete + + In the above example, the message numbers of any messages + containing the string "IMAP4" in the shared or private "value" + attribute of the "/comment" entry are returned in the search + results. + + + + + + + +Daboo & Gellens Experimental [Page 19] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a SEARCH ANNOTATION * value.priv "IMAP4" + S: * SEARCH 1 2 3 5 8 13 21 34 + S: a OK Search complete + + In the above example, the message numbers of any messages + containing the string "IMAP4" in the private "value" attribute of + any entry are returned in the search results. + +4.9. ANNOTATION Key in SORT + + ANNOTATION + + The ANNOTATION criterion for the SORT command [RFC5256] instructs the + server to return the sequence numbers or Unique Identifiers (UIDs) of + messages in a mailbox, sorted using the values of the specified + annotations. The ANNOTATION criterion is available if the server + returns both ANNOTATE-EXPERIMENT-1 and SORT as supported capabilities + in the CAPABILITY command response. + + Messages are sorted using the values of the + attributes in the entries. + + Clients MUST provide either the ".priv" or ".shared" suffix to the + attribute name to ensure that the server knows which specific value + to sort on. + + Only the "value.priv" and "value.shared" attributes can be used for + sorting. Clients MUST NOT specify an attribute other than either + "value.priv" or "value.shared". Servers MUST return a BAD response + if the client tries to sort on any other attribute. + + When either "value.priv" or "value.shared" is being sorted, the + server MUST use the character set value specified in the SORT command + to determine the appropriate sort order. + + Example: + + C: a SORT (ANNOTATION /altsubject value.shared) UTF-8 ALL + S: * SORT 2 3 4 5 1 11 10 6 7 9 8 + S: a OK Sort complete + + In the above example, the message numbers of all messages are + returned, sorted according to the shared "value" attribute of the + "/altsubject" entry. + + + + + +Daboo & Gellens Experimental [Page 20] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Note that the ANNOTATION sort key must include a fully specified + entry -- wild cards are not allowed. + +4.10. New ACL Rights + + As discussed in Section 3.4, this extension adds a new "n" right to + the list of rights provided by the ACL extensions [RFC2086] and + [RFC4314]. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. + + Non-terminals referenced but not defined below are as defined by + [RFC3501] with the new definitions in [RFC4466] superseding those in + [RFC3501]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + ann-size = "NONE" / + (("READ-ONLY" / nz-number) + [SP "NOPRIVATE"]) + ; response codes indicating the level of + ; support for annotations in a mailbox + + append-ext =/ att-annotate + ; modifies [RFC3501] extension behaviour + + att-annotate = "ANNOTATION" SP + "(" entry-att *(SP entry-att) ")" + + att-search = "value" / "value.priv" / "value.shared" + ; the only attributes that can be searched + + att-sort = "value.priv" / "value.shared" + ; the only attributes that can be sorted + + att-value = attrib SP value + + attrib = astring + ; dot-separated attribute name + ; MUST NOT contain "*" or "%" + + + + + +Daboo & Gellens Experimental [Page 21] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + attribs = attrib / "(" attrib *(SP attrib) ")" + ; one or more attribute specifiers + + capability =/ "ANNOTATE-EXPERIMENT-1" + ; defines the capability for this extension + + entries = entry-match / + "(" entry-match *(SP entry-match) ")" + + entry = astring + ; slash-separated path to entry + ; MUST NOT contain "*" or "%" + + entry-att = entry SP "(" att-value *(SP att-value) ")" + + entry-match = list-mailbox + ; slash-separated path to entry + ; MAY contain "*" or "%" for use as wild cards + + fetch-att =/ "ANNOTATION" SP "(" entries SP attribs ")" + ; modifies original IMAP fetch-att + + msg-att-dynamic =/ "ANNOTATION" SP + ( "(" entry-att *(SP entry-att) ")" / + "(" entry *(SP entry) ")" ) + ; extends FETCH response with annotation data + + resp-text-code =/ "ANNOTATE" SP "TOOBIG" / + "ANNOTATE" SP "TOOMANY" / + "ANNOTATIONS" SP ann-size + ; new response codes + + search-key =/ "ANNOTATION" SP entry-match SP att-search + SP value + ; modifies original IMAP search-key + + select-param =/ "ANNOTATE" + ; defines the select parameter used with + ; ANNOTATE extension + + sort-key =/ "ANNOTATION" SP entry SP att-sort + ; modifies original sort-key + + store-att-flags =/ att-annotate + ; modifies original IMAP STORE command + + value = nstring / literal8 + + + + +Daboo & Gellens Experimental [Page 22] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6. IANA Considerations + + Entry names MUST be specified in a standards track or IESG approved + experimental RFC, or fall under the vendor namespace. Vendor names + MUST be registered. + + Attribute names MUST be specified in a standards track or IESG + approved experimental RFC. + + Each entry registration MUST include a content-type that is used to + indicate the nature of the annotation value. Where applicable, a + charset parameter MUST be included with the content-type. + +6.1. Entry and Attribute Registration Template + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [] Entry [] Attribute + + Name: ______________________________ + + Description: _______________________ + + ____________________________________ + + ____________________________________ + + Content-Type:_______________________ + + Contact person: ____________________ + + email: ____________________ + + + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 23] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.2. Entry Registrations + + The following templates specify the IANA registrations of annotation + entries specified in this document. + +6.2.1. /comment + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: /comment + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.2.2. /flags + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: /flags + + Description: Reserved entry hierarchy. + + Content-Type: - + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + + + + + + + + + +Daboo & Gellens Experimental [Page 24] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.2.3. /altsubject + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: /altsubject + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.2.4. //comment + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //comment + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 25] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.2.5. //flags/seen + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //flags/seen + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.2.6. //flags/answered + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //flags/answered + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 26] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.2.7. //flags/flagged + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //flags/flagged + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.2.8. //flags/forwarded + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //flags/forwarded + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 27] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.3. Attribute Registrations + + The following templates specify the IANA registrations of annotation + attributes specified in this document. + +6.3.1. value + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [] Entry [X] Attribute + + Name: value + + Description: Defined in IMAP ANNOTATE extension document. + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.3.2. size + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [] Entry [X] Attribute + + Name: size + + Description: Defined in IMAP ANNOTATE extension document. + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.4. Capability Registration + + This document registers "ANNOTATE-EXPERIMENT-1" as an IMAPEXT + capability. + + + + + + + + +Daboo & Gellens Experimental [Page 28] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +7. Internationalization Considerations + + Annotations may contain values that include text strings, and both + searching and sorting are possible with annotations. Servers MUST + follow standard IMAP text normalization, character set conversion, + and collation rules when such operations are carried out, as would be + done for other textual fields being searched or sorted on. + +8. Security Considerations + + Annotations whose values are intended to remain private MUST be + stored in ".priv" values instead of ".shared" values, which may be + accessible to other users. + + Excluding the above issues, the ANNOTATE extension does not raise any + security considerations that are not present in the base IMAP + protocol; these issues are discussed in [RFC3501]. + +9. References + +9.1. Normative References + + [RFC2086] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2244] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC3502] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + + +Daboo & Gellens Experimental [Page 29] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + [RFC5256] Crispin, M. and K. Murchison, "Internet Message Access + Protocol - SORT and THREAD Extensions", RFC 5256, June + 2008. + +9.2. Informative References + + [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional + STORE Operation or Quick Flag Changes Resynchronization", + RFC 4551, June 2006. + +10. Acknowledgments + + Many thanks to Chris Newman for his detailed comments on the first + draft of this document, and to the participants at the ACAP working + dinner in Pittsburgh. The participants of the IMAPext working group + made significant contributions to this work. + +Authors' Addresses + + Cyrus Daboo + Apple Inc. + 1 Infinite Loop + Cupertino, CA 95014 + USA + + EMail: cyrus@daboo.name + URI: http://www.apple.com/ + + + Randall Gellens + QUALCOMM Incorporated + 5775 Morehouse Dr. + San Diego, CA 92121-2779 + USA + + EMail: randy@qualcomm.com + + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 30] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 31] + diff --git a/docs/rfcs/rfc5258.IMAP4_LIST_command_extension.txt b/docs/rfcs/rfc5258.IMAP4_LIST_command_extension.txt new file mode 100644 index 0000000..a80ec15 --- /dev/null +++ b/docs/rfcs/rfc5258.IMAP4_LIST_command_extension.txt @@ -0,0 +1,1739 @@ + + + + + + +Network Working Group B. Leiba +Request for Comments: 5258 IBM T.J. Watson Research Center +Obsoletes: 3348 A. Melnikov +Updates: 2193 Isode Limited +Category: Standards Track June 2008 + + + Internet Message Access Protocol version 4 - LIST Command Extensions + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + IMAP4 has two commands for listing mailboxes: LIST and LSUB. As we + have added extensions, such as Mailbox Referrals, that have required + specialized lists we have had to expand the number of list commands, + since each extension must add its function to both LIST and LSUB, and + these commands are not, as they are defined, extensible. If we've + needed the extensions to work together, we've had to add a set of + commands to mix the different options, the set increasing in size + with each new extension. This document describes an extension to the + base LIST command that will allow these additions to be done with + mutually compatible options to the LIST command, avoiding the + exponential increase in specialized list commands. + + + + + + + + + + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 1] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +Table of Contents + + 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . 4 + 3. Extended LIST Command . . . . . . . . . . . . . . . . . . . . 4 + 3.1. Initial List of Selection Options . . . . . . . . . . . . 7 + 3.2. Initial List of Return Options . . . . . . . . . . . . . . 8 + 3.3. General Principles for Returning LIST Responses . . . . . 9 + 3.4. Additional Requirements on LIST-EXTENDED Clients . . . . . 9 + 3.5. CHILDINFO Extended Data Item . . . . . . . . . . . . . . . 10 + 4. The CHILDREN Return Option . . . . . . . . . . . . . . . . . . 11 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 12 + 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 19 + 7. Internationalization Considerations . . . . . . . . . . . . . 22 + 8. Security Considerations . . . . . . . . . . . . . . . . . . . 23 + 9. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 23 + 9.1. Guidelines for IANA . . . . . . . . . . . . . . . . . . . 23 + 9.2. Registration Procedure and Change Control . . . . . . . . 23 + 9.3. Registration Template for LIST-EXTENDED Options . . . . . 25 + 9.4. Initial LIST-EXTENDED Option Registrations . . . . . . . . 25 + 9.5. Registration Template for LIST-EXTENDED Extended Data + Item . . . . . . . . . . . . . . . . . . . . . . . . . . . 28 + 9.6. Initial LIST-EXTENDED Extended Data Item Registrations . . 28 + 10. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . . 29 + 11. References . . . . . . . . . . . . . . . . . . . . . . . . . . 29 + 11.1. Normative References . . . . . . . . . . . . . . . . . . . 29 + 11.2. Informative References . . . . . . . . . . . . . . . . . . 30 + + + + + + + + + + + + + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 2] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +1. Introduction and Overview + + The LIST command is extended by amending the syntax to allow options + and multiple patterns to be specified. The list of options replaces + the several commands that are currently used to mix and match the + information requested. The new syntax is backward compatible, with + no ambiguity: the new syntax is being used if one of the following + conditions is true: + + 1. if the first word after the command name begins with a + parenthesis ("LIST selection options") + + 2. if the second word after the command name begins with a + parenthesis ("multiple mailbox patterns") + + 3. if the LIST command has more than 2 parameters ("LIST return + options") + + Otherwise the original syntax is used. + + By adding options to the LIST command, we are announcing the intent + to phase out and eventually to deprecate the RLIST and RLSUB commands + described in [MBRef]. We are also defining the mechanism to request + extended mailbox information, such as is described in the Child + Mailbox Extension [CMbox]. The base LSUB command is not deprecated + by this extension; rather, this extension adds a way to obtain + subscription information with more options, with those server + implementations that support it. Clients that simply need a list of + subscribed mailboxes, as provided by the LSUB command, SHOULD + continue to use that command. + + This document defines an IMAP4 extension that is identified by the + capability string "LIST-EXTENDED". The LIST-EXTENDED extension makes + the following changes to the IMAP4 protocol, which are described in + more detail in Section 3 and Section 4: + + a. defines new syntax for LIST command options. + + b. extends LIST to allow for multiple mailbox patterns. + + c. adds LIST command selection options: SUBSCRIBED, REMOTE, and + RECURSIVEMATCH. + + d. adds LIST command return options: SUBSCRIBED and CHILDREN. + + e. adds new mailbox attributes: "\NonExistent", "\Subscribed", + "\Remote", "\HasChildren", and "\HasNoChildren". + + + + +Leiba & Melnikov Standards Track [Page 3] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + f. adds CHILDINFO extended data item. + +2. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + are used in this document as specified in RFC 2119 [Kwds]. + + The term "canonical LIST pattern" refers to the canonical pattern + constructed internally by the server from the reference and mailbox + name arguments (Section 6.3.8 of [IMAP4]). The [IMAP4] LIST command + returns only mailboxes that match the canonical LIST pattern. + + Other terms are introduced where they are referenced for the first + time. + +3. Extended LIST Command + + This extension updates the syntax of the LIST command to allow for + multiple mailbox patterns to be specified, if they are enclosed in + parentheses. A mailbox name matches a list of mailbox patterns if it + matches at least one mailbox pattern. If a mailbox name matches + multiple mailbox patterns from the list, the server SHOULD return + only a single LIST response. + + Note that the non-extended LIST command is required to treat an empty + ("" string) mailbox name argument as a special request to return the + hierarchy delimiter and the root name of the name given in the + reference parameter (as per [IMAP4]). However, ANY extended LIST + command (extended in any of 3 ways specified in Section 1, or any + combination thereof) MUST NOT treat the empty mailbox name as such a + special request, and any regular processing described in this + document applies. In particular, if an extended LIST command has + multiple mailbox names and one (or more) of them is the empty string, + the empty string MUST be ignored for the purpose of matching. + + Some servers might restrict which patterns are allowed in a LIST + command. If a server doesn't accept a particular pattern, it MUST + silently ignore it. + + The LIST command syntax is also extended in two additional ways: by + adding a parenthesized list of command options between the command + name and the reference name (LIST selection options) and an optional + + + + + + +Leiba & Melnikov Standards Track [Page 4] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + list of options at the end that control what kind of information + should be returned (LIST return options). See the formal syntax in + Section 6 for specific details. + + A LIST selection option tells the server which mailbox names should + be selected by the LIST operation. The server should return + information about all mailbox names that match any of the "canonical + LIST pattern" (as described above) and satisfy additional selection + criteria (if any) specified by the LIST selection options. Let's + call any such mailbox name a "matched mailbox name". When multiple + selection options are specified, the server MUST return information + about mailbox names that satisfy every selection option, unless a + description of a particular specified option prescribes special + rules. An example of an option prescribing special rules is the + RECURSIVEMATCH selection option described later in this section. We + will use the term "selection criteria" when referring collectively to + all selection options specified in a LIST command. + + A LIST return option controls which information is returned for each + matched mailbox name. Note that return options MUST NOT cause the + server to report information about additional mailbox names. If the + client has not specified any return option, only information about + attributes should be returned by the server. (Of course, the server + is allowed to include any other information at will.) + + Both selection and return command options will be defined in this + document and in approved extension documents; each option will be + enabled by a capability string (one capability may enable multiple + options), and a client MUST NOT send an option for which the server + has not advertised support. A server MUST respond to options it does + not recognize with a BAD response. The client SHOULD NOT specify any + option more than once; however, if the client does this, the server + MUST act as if it received the option only once. The order in which + options are specified by the client is not significant. + + In general, each selection option except RECURSIVEMATCH will have a + corresponding return option. The REMOTE selection option is an + anomaly in this regard, and does not have a corresponding return + option. That is because it expands, rather than restricts, the set + of mailboxes that are returned. Future extensions to this + specification should keep parallelism in mind and define a pair of + corresponding options. + + This extension is identified by the capability string + "LIST-EXTENDED", and support for it is a prerequisite for any future + extensions that require specialized forms of the LIST command. Such + extensions MUST refer to this document and MUST add their function + through command options as described herein. Note that extensions + + + +Leiba & Melnikov Standards Track [Page 5] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + that don't require support for an extended LIST command, but use + extended LIST responses (see below), don't need to advertise the + "LIST-EXTENDED" capability string. + + This extension also defines extensions to the LIST response, allowing + a series of extended fields at the end, a parenthesized list of + tagged data (also referred to as "extended data item"). The first + element of an extended field is a tag, which identifies the type of + data. Tags MUST be registered with IANA, as described in Section 9.5 + of this document. An example of such an extended set might be + + tablecloth (("edge" "lacy") ("color" "red"))) (X-Sample "text")) + + or + + tablecloth ("edge" "lacy")) (X-Sample "text" "more text")) + + See the formal syntax, in Section 6, for the full syntactic details. + The server MUST NOT return any extended data item unless the client + has expressed its ability to support extended LIST responses, for + example, by using an extended LIST command. The server MAY return + data in the extended fields that was not directly solicited by the + client in the corresponding LIST command. For example, the client + can enable extra extended fields by using another IMAP extension that + make use of the extended LIST responses. The client MUST ignore all + extended fields it doesn't recognize. + + The LIST-EXTENDED capability also defines several new mailbox + attributes. + + The "\NonExistent" attribute indicates that a mailbox name does not + refer to an existing mailbox. Note that this attribute is not + meaningful by itself, as mailbox names that match the canonical LIST + pattern but don't exist must not be returned unless one of the two + conditions listed below is also satisfied: + + a. The mailbox name also satisfies the selection criteria (for + example, it is subscribed and the "SUBSCRIBED" selection option + has been specified). + + b. "RECURSIVEMATCH" has been specified, and the mailbox name has at + least one descendant mailbox name that does not match the LIST + pattern and does match the selection criteria. + + In practice, this means that the "\NonExistent" attribute is usually + returned with one or more of "\Subscribed", "\Remote", + "\HasChildren", or the CHILDINFO extended data item (see their + description below). + + + +Leiba & Melnikov Standards Track [Page 6] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The "\NonExistent" attribute implies "\NoSelect". The "\NonExistent" + attribute MUST be supported and MUST be accurately computed. + +3.1. Initial List of Selection Options + + The selection options defined in this specification are as follows: + + SUBSCRIBED - causes the LIST command to list subscribed names, + rather than the existing mailboxes. This will often be a subset + of the actual mailboxes. It's also possible for this list to + contain the names of mailboxes that don't exist. In any case, the + list MUST include exactly those mailbox names that match the + canonical list pattern and are subscribed to. This option is + intended to supplement the LSUB command. Of particular note are + the mailbox attributes as returned by this option, compared with + what is returned by LSUB. With the latter, the attributes + returned may not reflect the actual attribute status on the + mailbox name, and the \NoSelect attribute has a second special + meaning (it indicates that this mailbox is not, itself, + subscribed, but that it has descendant mailboxes that are). With + the SUBSCRIBED selection option described here, the attributes are + accurate and complete, and have no special meanings. "LSUB" and + "LIST (SUBSCRIBED)" are, thus, not the same thing, and some + servers must do significant extra work to respond to "LIST + (SUBSCRIBED)". Because of this, clients SHOULD continue to use + "LSUB" unless they specifically want the additional information + offered by "LIST (SUBSCRIBED)". + + This option defines a new mailbox attribute, "\Subscribed", that + indicates that a mailbox name is subscribed to. The "\Subscribed" + attribute MUST be supported and MUST be accurately computed when + the SUBSCRIBED selection option is specified. + + Note that the SUBSCRIBED selection option implies the SUBSCRIBED + return option (see below). + + REMOTE - causes the LIST command to show remote mailboxes as well as + local ones, as described in [MBRef]. This option is intended to + replace the RLIST command and, in conjunction with the SUBSCRIBED + selection option, the RLSUB command. + + This option defines a new mailbox attribute, "\Remote", that + indicates that a mailbox is a remote mailbox. The "\Remote" + attribute MUST be accurately computed when the REMOTE option is + specified. + + The REMOTE selection option has no interaction with other options. + Its effect is to tell the server to apply the other options, if + + + +Leiba & Melnikov Standards Track [Page 7] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + any, to remote mailboxes, in addition to local ones. In + particular, it has no interaction with RECURSIVEMATCH (see below). + A request for (REMOTE RECURSIVEMATCH) is invalid, because a + request for (RECURSIVEMATCH) is. A request for (REMOTE + RECURSIVEMATCH SUBSCRIBED) is asking for all subscribed mailboxes, + both local and remote. + + RECURSIVEMATCH - this option forces the server to return information + about parent mailboxes that don't match other selection options, + but have some submailboxes that do. Information about children is + returned in the CHILDINFO extended data item, as described in + Section 3.5. + + Note 1: In order for a parent mailbox to be returned, it still has + to match the canonical LIST pattern. + + Note 2: When returning the CHILDINFO extended data item, it + doesn't matter whether or not the submailbox matches the canonical + LIST pattern. See also example 9 in Section 5. + + The RECURSIVEMATCH option MUST NOT occur as the only selection + option (or only with REMOTE), as it only makes sense when other + selection options are also used. The server MUST return BAD + tagged response in such case. + + Note that even if the RECURSIVEMATCH option is specified, the + client MUST still be able to handle a case when a CHILDINFO + extended data item is returned and there are no submailboxes that + meet the selection criteria of the subsequent LIST command, as + they can be deleted/renamed after the LIST response was sent, but + before the client had a chance to access them. + +3.2. Initial List of Return Options + + The return options defined in this specification are as follows: + + SUBSCRIBED - causes the LIST command to return subscription state + for all matching mailbox names. The "\Subscribed" attribute MUST + be supported and MUST be accurately computed when the SUBSCRIBED + return option is specified. Further, all mailbox flags MUST be + accurately computed (this differs from the behavior of the LSUB + command). + + CHILDREN - requests mailbox child information as originally proposed + in [CMbox]. See Section 4, below, for details. This option MUST + be supported by all servers. + + + + + +Leiba & Melnikov Standards Track [Page 8] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +3.3. General Principles for Returning LIST Responses + + This section outlines several principles that can be used by server + implementations of this document to decide whether a LIST response + should be returned, as well as how many responses and what kind of + information they may contain. + + 1. At most one LIST response should be returned for each mailbox + name that matches the canonical LIST pattern. Server + implementors must not assume that clients will be able to + assemble mailbox attributes and other information returned in + multiple LIST responses. + + 2. There are only two reasons for including a matching mailbox name + in the responses to the LIST command (note that the server is + allowed to return unsolicited responses at any time, and such + responses are not governed by this rule): + + A. The mailbox name also satisfies the selection criteria. + + B. The mailbox name doesn't satisfy the selection criteria, but + it has at least one descendant mailbox name that satisfies + the selection criteria and that doesn't match the canonical + LIST pattern. + + For more information on this case, see the CHILDINFO extended + data item described in Section 3.5. Note that the CHILDINFO + extended data item can only be returned when the + RECURSIVEMATCH selection option is specified. + + 3. Attributes returned in the same LIST response must be treated + additively. For example, the following response + + S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" + + means that the "Fruit/Peach" mailbox doesn't exist, but it is + subscribed. + +3.4. Additional Requirements on LIST-EXTENDED Clients + + All clients that support this extension MUST treat an attribute with + a stronger meaning as implying any attribute that can be inferred + from it. For example, the client must treat the presence of the + \NoInferiors attribute as if the \HasNoChildren attribute was also + sent by the server. + + + + + + +Leiba & Melnikov Standards Track [Page 9] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The following table summarizes inference rules described in + Section 3. + + +--------------------+-------------------+ + | returned attribute | implied attribute | + +--------------------+-------------------+ + | \NoInferiors | \HasNoChildren | + | \NonExistent | \NoSelect | + +--------------------+-------------------+ + +3.5. CHILDINFO Extended Data Item + + The CHILDINFO extended data item MUST NOT be returned unless the + client has specified the RECURSIVEMATCH selection option. + + The CHILDINFO extended data item in a LIST response describes the + selection criteria that has caused it to be returned and indicates + that the mailbox has at least one descendant mailbox that matches the + selection criteria. + + The LSUB command indicates this condition by using the "\NoSelect" + attribute, but the LIST (SUBSCRIBED) command MUST NOT do that, since + "\NoSelect" retains its original meaning here. Further, the + CHILDINFO extended data item is more general, in that it can be used + with any extended set of selection criteria. + + Note: Some servers allow for mailboxes to exist without requiring + their parent to exist. For example, a mailbox "Customers/ABC" can + exist while the mailbox "Customers" does not. As CHILDINFO extended + data item is not allowed if the RECURSIVEMATCH selection option is + not specified, such servers SHOULD use the "\NonExistent + \HasChildren" attribute pair to signal to the client that there is a + descendant mailbox that matches the selection criteria. See example + 11 in Section 5. + + The returned selection criteria allow the client to distinguish a + solicited response from an unsolicited one, as well as to distinguish + among solicited responses caused by multiple pipelined LIST commands + that specify different criteria. + + Servers SHOULD ONLY return a non-matching mailbox name along with + CHILDINFO if at least one matching child is not also being returned. + That is, servers SHOULD suppress redundant CHILDINFO responses. + + Examples 8 and 10 in Section 5 demonstrate the difference between + present CHILDINFO extended data item and the "\HasChildren" + attribute. + + + + +Leiba & Melnikov Standards Track [Page 10] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The following table summarizes interaction between the "\NonExistent" + attribute and CHILDINFO (the first column indicates whether the + parent mailbox exists): + + +--------+--------------+--------------------+----------------------+ + | exists | meets the | has a child that | returned | + | | selection | meets the | LIST-EXTENDED | + | | criteria | selection criteria | attributes and | + | | | | CHILDINFO | + +--------+--------------+--------------------+----------------------+ + | no | no | no | no LIST response | + | | | | returned | + | yes | no | no | no LIST response | + | | | | returned | + | no | yes | no | (\NonExistent | + | | | | ) | + | yes | yes | no | () | + | no | no | yes | (\NonExistent) + | + | | | | CHILDINFO | + | yes | no | yes | () + CHILDINFO | + | no | yes | yes | (\NonExistent | + | | | | ) + CHILDINFO | + | yes | yes | yes | () + CHILDINFO | + +--------+--------------+--------------------+----------------------+ + + where is one or more attributes that correspond to the + selection criteria; for example, for the SUBSCRIBED option the + is \Subscribed. + +4. The CHILDREN Return Option + + The CHILDREN return option implements the Child Mailbox Extension, + originally proposed by Mike Gahrns and Raymond Cheng, of Microsoft + Corporation. Most of the information in this section is taken + directly from their original specification [CMbox]. The CHILDREN + return option is simply an indication that the client wants this + information; a server MAY provide it even if the option is not + specified. + + Many IMAP4 [IMAP4] clients present to the user a hierarchical view of + the mailboxes that a user has access to. Rather than initially + presenting to the user the entire mailbox hierarchy, it is often + preferable to show to the user a collapsed outline list of the + mailbox hierarchy (particularly if there is a large number of + mailboxes). The user can then expand the collapsed outline hierarchy + as needed. It is common to include within the collapsed hierarchy a + visual clue (such as a ''+'') to indicate that there are child + mailboxes under a particular mailbox. When the visual clue is + + + +Leiba & Melnikov Standards Track [Page 11] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + clicked, the hierarchy list is expanded to show the child mailboxes. + The CHILDREN return option provides a mechanism for a client to + efficiently determine whether a particular mailbox has children, + without issuing a LIST "" * or a LIST "" % for each mailbox name. + The CHILDREN return option defines two new attributes that MUST be + returned within a LIST response: \HasChildren and \HasNoChildren. + Although these attributes MAY be returned in response to any LIST + command, the CHILDREN return option is provided to indicate that the + client particularly wants this information. If the CHILDREN return + option is present, the server MUST return these attributes even if + their computation is expensive. + + \HasChildren + + The presence of this attribute indicates that the mailbox has child + mailboxes. A server SHOULD NOT set this attribute if there are + child mailboxes and the user does not have permission to access + any of them. In this case, \HasNoChildren SHOULD be used. In + many cases, however, a server may not be able to efficiently + compute whether a user has access to any child mailbox. Note + that even though the \HasChildren attribute for a mailbox must + be correct at the time of processing of the mailbox, a client + must be prepared to deal with a situation when a mailbox is + marked with the \HasChildren attribute, but no child mailbox + appears in the response to the LIST command. This might happen, + for example, due to children mailboxes being deleted or made + inaccessible to the user (using access control) by another + client before the server is able to list them. + + \HasNoChildren + + The presence of this attribute indicates that the mailbox has NO + child mailboxes that are accessible to the currently + authenticated user. + + It is an error for the server to return both a \HasChildren and a + \HasNoChildren attribute in the same LIST response. + + Note: the \HasNoChildren attribute should not be confused with the + IMAP4 [IMAP4] defined attribute \NoInferiors, which indicates that no + child mailboxes exist now and none can be created in the future. + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 12] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +5. Examples + + 1: The first example shows the complete local hierarchy that will + be used for the other examples. + + C: A01 LIST "" "*" + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST () "/" "Fruit" + S: * LIST () "/" "Fruit/Apple" + S: * LIST () "/" "Fruit/Banana" + S: * LIST () "/" "Tofu" + S: * LIST () "/" "Vegetable" + S: * LIST () "/" "Vegetable/Broccoli" + S: * LIST () "/" "Vegetable/Corn" + S: A01 OK done + + 2: In the next example, we will see the subscribed mailboxes. This + is similar to, but not equivalent with, . Note + that the mailbox called "Fruit/Peach" is subscribed to, but does + not actually exist (perhaps it was deleted while still + subscribed). The "Fruit" mailbox is not subscribed to, but it + has two subscribed children. The "Vegetable" mailbox is + subscribed and has two children; one of them is subscribed as + well. + + C: A02 LIST (SUBSCRIBED) "" "*" + S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" + S: * LIST (\Subscribed) "/" "Fruit/Banana" + S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" + S: * LIST (\Subscribed) "/" "Vegetable" + S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" + S: A02 OK done + + 3: The next example shows the use of the CHILDREN option. The + client, without having to list the second level of hierarchy, + now knows which of the top-level mailboxes have submailboxes + (children) and which do not. Note that it's not necessary for + the server to return the \HasNoChildren attribute for the inbox, + because the \NoInferiors attribute already implies that, and has + a stronger meaning. + + C: A03 LIST () "" "%" RETURN (CHILDREN) + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST (\HasChildren) "/" "Fruit" + S: * LIST (\HasNoChildren) "/" "Tofu" + S: * LIST (\HasChildren) "/" "Vegetable" + S: A03 OK done + + + + +Leiba & Melnikov Standards Track [Page 13] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + 4: In this example, we see more mailboxes that reside on another + server. This is similar to the command . + + C: A04 LIST (REMOTE) "" "%" RETURN (CHILDREN) + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST (\HasChildren) "/" "Fruit" + S: * LIST (\HasNoChildren) "/" "Tofu" + S: * LIST (\HasChildren) "/" "Vegetable" + S: * LIST (\Remote) "/" "Bread" + S: * LIST (\HasChildren \Remote) "/" "Meat" + S: A04 OK done + + 5: The following example also requests the server to include + mailboxes that reside on another server. The server returns + information about all mailboxes that are subscribed. This is + similar to the command . We also see the use of + two selection options. + + C: A05 LIST (REMOTE SUBSCRIBED) "" "*" + S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" + S: * LIST (\Subscribed) "/" "Fruit/Banana" + S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" + S: * LIST (\Subscribed) "/" "Vegetable" + S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" + S: * LIST (\Remote \Subscribed) "/" "Bread" + S: A05 OK done + + 6: The following example requests the server to include mailboxes + that reside on another server. The server is asked to return + subscription information for all returned mailboxes. This is + different from the example above. + + Note that the output of this command is not a superset of the + output in the previous example, as it doesn't include LIST + response for the non-existent "Fruit/Peach". + + C: A06 LIST (REMOTE) "" "*" RETURN (SUBSCRIBED) + S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" + S: * LIST () "/" "Fruit" + S: * LIST () "/" "Fruit/Apple" + S: * LIST (\Subscribed) "/" "Fruit/Banana" + S: * LIST () "/" "Tofu" + S: * LIST (\Subscribed) "/" "Vegetable" + S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" + S: * LIST () "/" "Vegetable/Corn" + S: * LIST (\Remote \Subscribed) "/" "Bread" + S: * LIST (\Remote) "/" "Meat" + S: A06 OK done + + + +Leiba & Melnikov Standards Track [Page 14] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + 7: In the following example, the client has specified multiple + mailbox patterns. Note that this example does not use the + mailbox hierarchy used in the previous examples. + + C: BBB LIST "" ("INBOX" "Drafts" "Sent/%") + S: * LIST () "/" "INBOX" + S: * LIST (\NoInferiors) "/" "Drafts" + S: * LIST () "/" "Sent/March2004" + S: * LIST (\Marked) "/" "Sent/December2003" + S: * LIST () "/" "Sent/August2004" + S: BBB OK done + + 8: The following example demonstrates the difference between the + \HasChildren attribute and the CHILDINFO extended data item. + + Let's assume there is the following hierarchy: + + C: C01 LIST "" "*" + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST () "/" "Foo" + S: * LIST () "/" "Foo/Bar" + S: * LIST () "/" "Foo/Baz" + S: * LIST () "/" "Moo" + S: C01 OK done + + If the client asks RETURN (CHILDREN), it will get this: + + C: CA3 LIST "" "%" RETURN (CHILDREN) + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST (\HasChildren) "/" "Foo" + S: * LIST (\HasNoChildren) "/" "Moo" + S: CA3 OK done + + A) Let's also assume that the mailbox "Foo/Baz" is the only + subscribed mailbox. Then we get this result: + + C: C02 LIST (SUBSCRIBED) "" "*" + S: * LIST (\Subscribed) "/" "Foo/Baz" + S: C02 OK done + + Now, if the client issues , the server will + return no mailboxes (as the mailboxes "Moo", "Foo", and "Inbox" are + NOT subscribed). However, if the client issues this: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" + S: * LIST () "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) + S: C04 OK done + + + + +Leiba & Melnikov Standards Track [Page 15] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + (i.e., the mailbox "Foo" is not subscribed, but it has a child that + is.) + + A1) If the mailbox "Foo" had also been subscribed, the last command + would return this: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" + S: * LIST (\Subscribed) "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) + S: C04 OK done + + or even this: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" + S: * LIST (\Subscribed \HasChildren) "/" "Foo" ("CHILDINFO" + ("SUBSCRIBED")) + S: C04 OK done + + A2) If we assume instead that the mailbox "Foo" is not part of the + original hierarchy and is not subscribed, the last command will give + this result: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" + S: * LIST (\NonExistent) "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) + S: C04 OK done + + B) Now, let's assume that no mailbox is subscribed. In this case, + the command will return no + responses, as there are no subscribed children (even though "Foo" has + children). + + C) And finally, suppose that only the mailboxes "Foo" and "Moo" are + subscribed. In that case, we see this result: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" RETURN (CHILDREN) + S: * LIST (\HasChildren \Subscribed) "/" "Foo" + S: * LIST (\HasNoChildren \Subscribed) "/" "Moo" + S: C04 OK done + + (which means that the mailbox "Foo" has children, but none of them is + subscribed). + + 9: The following example demonstrates that the CHILDINFO extended + data item is returned whether or not children mailboxes match + the canonical LIST pattern. + + + + + + + +Leiba & Melnikov Standards Track [Page 16] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Let's assume there is the following hierarchy: + + C: D01 LIST "" "*" + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST () "/" "foo2" + S: * LIST () "/" "foo2/bar1" + S: * LIST () "/" "foo2/bar2" + S: * LIST () "/" "baz2" + S: * LIST () "/" "baz2/bar2" + S: * LIST () "/" "baz2/bar22" + S: * LIST () "/" "baz2/bar222" + S: * LIST () "/" "eps2" + S: * LIST () "/" "eps2/mamba" + S: * LIST () "/" "qux2/bar2" + S: D01 OK done + + And that the following mailboxes are subscribed: + + C: D02 LIST (SUBSCRIBED) "" "*" + S: * LIST (\Subscribed) "/" "foo2/bar1" + S: * LIST (\Subscribed) "/" "foo2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar22" + S: * LIST (\Subscribed) "/" "baz2/bar222" + S: * LIST (\Subscribed) "/" "eps2" + S: * LIST (\Subscribed) "/" "eps2/mamba" + S: * LIST (\Subscribed) "/" "qux2/bar2" + S: D02 OK done + + The client issues the following command first: + + C: D03 LIST (RECURSIVEMATCH SUBSCRIBED) "" "*2" + S: * LIST () "/" "foo2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "foo2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar22" + S: * LIST (\Subscribed) "/" "baz2/bar222" + S: * LIST (\Subscribed) "/" "eps2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "qux2/bar2" + S: D03 OK done + + and the server may also include (but this would violate a SHOULD NOT + in Section 3.5, because CHILDINFO is redundant) + + S: * LIST () "/" "baz2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\NonExistent) "/" "qux2" ("CHILDINFO" ("SUBSCRIBED")) + + + + + +Leiba & Melnikov Standards Track [Page 17] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The CHILDINFO extended data item is returned for mailboxes "foo2", + "baz2", and "eps2", because all of them have subscribed children, + even though for the mailbox "foo2" only one of the two subscribed + children matches the pattern, for the mailbox "baz2" all the + subscribed children match the pattern, and for the mailbox "eps2" + none of the subscribed children matches the pattern. + + Note that if the client issues + + C: D03 LIST (RECURSIVEMATCH SUBSCRIBED) "" "*" + S: * LIST () "/" "foo2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "foo2/bar1" + S: * LIST (\Subscribed) "/" "foo2/bar2" + S: * LIST () "/" "baz2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "baz2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar22" + S: * LIST (\Subscribed) "/" "baz2/bar222" + S: * LIST (\Subscribed) "/" "eps2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "eps2/mamba" + S: * LIST (\Subscribed) "/" "qux2/bar2" + S: D03 OK done + + The LIST responses for mailboxes "foo2", "baz2", and "eps2" still + have the CHILDINFO extended data item, even though this information + is redundant and the client can determine it by itself. + + 10: The following example shows usage of multiple mailbox patterns. + It also demonstrates that the presence of the CHILDINFO extended + data item doesn't necessarily imply \HasChildren. + + C: a1 LIST "" ("foo" "foo/*") + S: * LIST () "/" foo + S: a1 OK done + + C: a2 LIST (SUBSCRIBED) "" "foo/*" + S: * LIST (\Subscribed \NonExistent) "/" foo/bar + S: a2 OK done + + C: a3 LIST (SUBSCRIBED RECURSIVEMATCH) "" foo RETURN (CHILDREN) + S: * LIST (\HasNoChildren) "/" foo ("CHILDINFO" ("SUBSCRIBED")) + S: a3 OK done + + 11: The following example shows how a server that supports missing + mailbox hierarchy elements can signal to a client that didn't + specify the RECURSIVEMATCH selection option that there is a + child mailbox that matches the selection criteria. + + + + + +Leiba & Melnikov Standards Track [Page 18] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + C: a1 LIST (REMOTE) "" * + S: * LIST () "/" music/rock + S: * LIST (\Remote) "/" also/jazz + S: a1 OK done + + C: a2 LIST () "" % + S: * LIST (\NonExistent \HasChildren) "/" music + S: a2 OK done + + C: a3 LIST (REMOTE) "" % + S: * LIST (\NonExistent \HasChildren) "/" music + S: * LIST (\NonExistent \HasChildren) "/" also + S: a3 OK done + + C: a3.1 LIST "" (% music/rock) + S: * LIST () "/" music/rock + S: a3.1 OK done + + Because "music/rock" is the only mailbox under "music", there's no + need for the server to also return "music". However clients must + handle both cases. + +6. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) as described in [ABNF]. Terms not defined here are taken + from [IMAP4]. In particular, note that the version of "mailbox-list" + below, which defines the payload of the LIST response, updates the + version defined in the IMAP specification. It is pointed to by + "mailbox-data", which is defined in [IMAP4]. + + "vendor-token" is defined in [ACAP]. Note that this normative + reference to ACAP will be an issue in moving this spec forward, since + it introduces a dependency on ACAP. The definitions of + "vendor-token" and of the IANA registry must eventually go somewhere + else, in a document that can be moved forward on the standards track + independently of ACAP. + + + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 19] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + childinfo-extended-item = "CHILDINFO" SP "(" + list-select-base-opt-quoted + *(SP list-select-base-opt-quoted) ")" + ; Extended data item (mbox-list-extended-item) + ; returned when the RECURSIVEMATCH + ; selection option is specified. + ; Note 1: the CHILDINFO tag can be returned + ; with and without surrounding quotes, as per + ; mbox-list-extended-item-tag production. + ; Note 2: The selection options are always returned + ; quoted, unlike their specification in + ; the extended LIST command. + + child-mbox-flag = "\HasChildren" / "\HasNoChildren" + ; attributes for CHILDREN return option, at most one + ; possible per LIST response + + eitem-standard-tag = atom + ; a tag for extended list data defined in a Standard + ; Track or Experimental RFC. + + eitem-vendor-tag = vendor-token "-" atom + ; a vendor-specific tag for extended list data + + list = "LIST" [SP list-select-opts] SP mailbox SP mbox-or-pat + [SP list-return-opts] + + list-return-opts = "RETURN" SP + "(" [return-option *(SP return-option)] ")" + ; list return options, e.g., CHILDREN + + list-select-base-opt = "SUBSCRIBED" / option-extension + ; options that can be used by themselves + + list-select-base-opt-quoted = DQUOTE list-select-base-opt DQUOTE + + list-select-independent-opt = "REMOTE" / option-extension + ; options that do not syntactically interact with + ; other options + + list-select-mod-opt = "RECURSIVEMATCH" / option-extension + ; options that require a list-select-base-opt + ; to also be present + + list-select-opt = list-select-base-opt / list-select-independent-opt + / list-select-mod-opt + ; An option registration template is described in + ; Section 9.3 of this document. + + + +Leiba & Melnikov Standards Track [Page 20] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + list-select-opts = "(" [ + (*(list-select-opt SP) list-select-base-opt + *(SP list-select-opt)) + / (list-select-independent-opt + *(SP list-select-independent-opt)) + ] ")" + ; Any number of options may be in any order. + ; If a list-select-mod-opt appears, then a + ; list-select-base-opt must also appear. + ; This allows these: + ; () + ; (REMOTE) + ; (SUBSCRIBED) + ; (SUBSCRIBED REMOTE) + ; (SUBSCRIBED RECURSIVEMATCH) + ; (SUBSCRIBED REMOTE RECURSIVEMATCH) + ; But does NOT allow these: + ; (RECURSIVEMATCH) + ; (REMOTE RECURSIVEMATCH) + + mailbox-list = "(" [mbx-list-flags] ")" SP + (DQUOTE QUOTED-CHAR DQUOTE / nil) SP mailbox + [SP mbox-list-extended] + ; This is the list information pointed to by the ABNF + ; item "mailbox-data", which is defined in [IMAP4] + + mbox-list-extended = "(" [mbox-list-extended-item + *(SP mbox-list-extended-item)] ")" + + mbox-list-extended-item = mbox-list-extended-item-tag SP + tagged-ext-val + + mbox-list-extended-item-tag = astring + ; The content MUST conform to either "eitem-vendor-tag" + ; or "eitem-standard-tag" ABNF productions. + ; A tag registration template is described in this + ; document in Section 9.5. + + mbx-list-oflag =/ child-mbox-flag / "\Subscribed" / "\Remote" + + mbx-list-sflag =/ "\NonExistent" + + mbox-or-pat = list-mailbox / patterns + + option-extension = (option-standard-tag / option-vendor-tag) + [SP option-value] + + + + + +Leiba & Melnikov Standards Track [Page 21] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + option-standard-tag = atom + ; an option defined in a Standards Track or + ; Experimental RFC + + option-val-comp = astring / + option-val-comp *(SP option-val-comp) / + "(" option-val-comp ")" + + option-value = "(" option-val-comp ")" + + option-vendor-tag = vendor-token "-" atom + ; a vendor-specific option, non-standard + + patterns = "(" list-mailbox *(SP list-mailbox) ")" + + return-option = "SUBSCRIBED" / "CHILDREN" / option-extension + + tagged-ext-comp = astring / + tagged-ext-comp *(SP tagged-ext-comp) / + "(" tagged-ext-comp ")" + ; Extensions that follow this general + ; syntax should use nstring instead of + ; astring when appropriate in the context + ; of the extension. + ; Note that a message set or a "number" + ; can always be represented as an "atom". + ; A URL should be represented as + ; a "quoted" string. + + tagged-ext-simple = sequence-set / number + + tagged-ext-val = tagged-ext-simple / + "(" [tagged-ext-comp] ")" + +7. Internationalization Considerations + + The LIST command selection option types defined in this specification + involve simple tests of mailbox properties. However, future + extensions to LIST-EXTENDED may define selection options that do more + sophisticated tests. In the case of a test that requires matching + text, in the presence of the COMPARATOR [I18N] extension, the active + comparator must be used to do comparisons. Such LIST-EXTENDED + extensions MUST indicate in their specification the interaction with + the COMPARATOR [I18N] extension. + + + + + + + +Leiba & Melnikov Standards Track [Page 22] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +8. Security Considerations + + This document describes syntactic changes to the specification of the + IMAP4 commands LIST, LSUB, RLIST, and RLSUB, and the modified LIST + command has the same security considerations as those commands. They + are described in [IMAP4] and [MBRef]. + + The Child Mailbox Extension provides a client a more efficient means + of determining whether a particular mailbox has children. If a + mailbox has children, but the currently authenticated user does not + have access to any of them, the server SHOULD respond with a + \HasNoChildren attribute. In many cases, however, a server may not + be able to efficiently compute whether a user has access to any child + mailbox. If such a server responds with a \HasChildren attribute, + when in fact the currently authenticated user does not have access to + any child mailboxes, potentially more information is conveyed about + the mailbox than intended. In most situations, this will not be a + security concern, because if information regarding whether a mailbox + has children is considered sensitive, a user would not be granted + access to that mailbox in the first place. + + The CHILDINFO extended data item has the same security considerations + as the \HasChildren attribute described above. + +9. IANA Considerations + +9.1. Guidelines for IANA + + IANA has created two new registries for LIST-EXTENDED options and + LIST-EXTENDED response data. The templates and the initial + registrations are detailed below. + +9.2. Registration Procedure and Change Control + + Registration of a LIST-EXTENDED option is done by filling in the + template in Section 9.3 and sending it via electronic mail to + iana@iana.org. Registration of a LIST-EXTENDED extended data item is + done by filling in the template in Section 9.5 and sending it via + electronic mail to iana@iana.org. IANA has the right to reject + obviously bogus registrations, but will perform no review of claims + made in the registration form. + + A LIST-EXTENDED option/extended data item name that starts with "V-" + is reserved for vendor-specific options/extended data items. All + options, whether they are vendor specific or global, should be + registered with IANA. If a LIST-EXTENDED extended data item is + returned as a result of requesting a particular LIST-EXTENDED option, + + + + +Leiba & Melnikov Standards Track [Page 23] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + the name of the option SHOULD be used as the name of the + LIST-EXTENDED extended data item. + + Each vendor-specific option/extended data item MUST start with its + vendor-token ("vendor prefix"). The vendor-token MUST be registered + with IANA, using the [ACAP] vendor subtree registry. + + Standard LIST-EXTENDED option/extended data item names are case + insensitive. If the vendor prefix is omitted from a vendor-specific + LIST-EXTENDED option/extended data item name, the rest is case + insensitive. The vendor prefix itself is not case sensitive, as it + might contain non-ASCII characters. While the registration + procedures do not require it, authors of + LIST-EXTENDED options/extended data items are encouraged to seek + community review and comment whenever that is feasible. Authors may + seek community review by posting a specification of their proposed + mechanism as an + Internet-Draft. LIST-EXTENDED option/extended data items intended + for widespread use should be standardized through the normal IETF + process, when appropriate. + + Comments on registered LIST-EXTENDED options/extended response data + should first be sent to the "owner" of the mechanism and/or to the + IMAPEXT WG mailing list. Submitters of comments may, after a + reasonable attempt to contact the owner, request IANA to attach their + comment to the registration itself. If IANA approves of this, the + comment will be made accessible in conjunction with the registration + LIST-EXTENDED options/extended response data itself. + + Once a LIST-EXTENDED registration has been published by IANA, the + author may request a change to its definition. The change request + follows the same procedure as the registration request. + + The owner of a LIST-EXTENDED registration may pass responsibility for + the registered option/extended data item to another person or agency + by informing IANA; this can be done without discussion or review. + + The IESG may reassign responsibility for a LIST-EXTENDED + option/extended data item. The most common case of this will be to + enable changes to be made to mechanisms where the author of the + registration has died, has moved out of contact, or is otherwise + unable to make changes that are important to the community. + + LIST-EXTENDED registrations may not be deleted; mechanisms that are + no longer believed appropriate for use can be declared OBSOLETE by a + change to their "intended use" field. Such LIST-EXTENDED + options/extended data items will be clearly marked in the lists + published by IANA. + + + +Leiba & Melnikov Standards Track [Page 24] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The IESG is considered to be the owner of all LIST-EXTENDED + options/extended data items that are on the IETF standards track. + +9.3. Registration Template for LIST-EXTENDED Options + + To: iana@iana.org + Subject: Registration of LIST-EXTENDED option X + + LIST-EXTENDED option name: + + LIST-EXTENDED option type: (One of SELECTION or RETURN) + + Implied return options(s), if the option type is SELECTION: (zero or + more) + + LIST-EXTENDED option description: + + Published specification (optional, recommended): + + Security considerations: + + Intended usage: + (One of COMMON, LIMITED USE, or OBSOLETE) + + Person and email address to contact for further information: + + Owner/Change controller: + + (Any other information that the author deems interesting may be added + below this line.) + +9.4. Initial LIST-EXTENDED Option Registrations + + The LIST-EXTENDED option registry has been populated with the + following entries: + + 1. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option SUBSCRIBED + + LIST-EXTENDED option name: SUBSCRIBED + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): SUBSCRIBED + + LIST-EXTENDED option description: Causes the LIST command to list + subscribed mailboxes, rather than the actual mailboxes. + + + + +Leiba & Melnikov Standards Track [Page 25] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Published specification: RFC 5258, Section 3. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + + 2. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option REMOTE + + LIST-EXTENDED option name: REMOTE + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): (none) + + LIST-EXTENDED option description: Causes the LIST command to + return remote mailboxes as well as local ones, as described in + RFC 2193. + + Published specification: RFC 5258, Section 3. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + + 3. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option SUBSCRIBED + + LIST-EXTENDED option name: SUBSCRIBED + + LIST-EXTENDED option type: RETURN + + LIST-EXTENDED option description: Causes the LIST command to + return subscription state. + + Published specification: RFC 5258, Section 3. + + Security considerations: RFC 5258, Section 8. + + + +Leiba & Melnikov Standards Track [Page 26] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + + 4. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option RECURSIVEMATCH + + LIST-EXTENDED option name: RECURSIVEMATCH + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): (none) + + LIST-EXTENDED option description: Requests that CHILDINFO + extended data item (childinfo-extended-item) is to be returned. + + Published specification: RFC 5258, Section 3. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + + 5. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option CHILDREN + + LIST-EXTENDED option name: CHILDREN + + LIST-EXTENDED option type: RETURN + + LIST-EXTENDED option description: Requests mailbox child + information. + + Published specification: RFC 5258, Section 3 and Section 4. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + + +Leiba & Melnikov Standards Track [Page 27] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Owner/Change controller: iesg@ietf.org + +9.5. Registration Template for LIST-EXTENDED Extended Data Item + + To: iana@iana.org + Subject: Registration of X LIST-EXTENDED extended data item + + LIST-EXTENDED extended data item tag: + + LIST-EXTENDED extended data item description: + + Which LIST-EXTENDED option(s) (and their types) causes this extended + data item to be returned (if any): + + Published specification (optional, recommended): + + Security considerations: + + Intended usage: + (One of COMMON, LIMITED USE, or OBSOLETE) + + Person and email address to contact for further information: + + Owner/Change controller: + + (Any other information that the author deems interesting may be added + below this line.) + +9.6. Initial LIST-EXTENDED Extended Data Item Registrations + + The LIST-EXTENDED extended data item registry has been populated with + the following entries: + + 1. To: iana@iana.org + Subject: Registration of CHILDINFO LIST-EXTENDED extended data + item + + LIST-EXTENDED extended data item tag: CHILDINFO + + LIST-EXTENDED extended data item description: The CHILDINFO + extended data item describes the selection criteria that has + caused it to be returned and indicates that the mailbox has one + or more child mailboxes that match the selection criteria. + + Which LIST-EXTENDED option(s) (and their types) causes this + extended data item to be returned (if any): RECURSIVEMATCH + selection option + + + + +Leiba & Melnikov Standards Track [Page 28] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Published specification: RFC 5258, Section 3.5. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + +10. Acknowledgements + + Mike Gahrns and Raymond Cheng of Microsoft Corporation originally + devised the Child Mailbox Extension and proposed it in 1997; the + idea, as well as most of the text in Section 4, is theirs. + + This document is the result of discussions on the IMAP4 and IMAPEXT + mailing lists and is meant to reflect consensus of those groups. In + particular, Mark Crispin, Philip Guenther, Cyrus Daboo, Timo + Sirainen, Ken Murchison, Rob Siemborski, Steve Hole, Arnt + Gulbrandsen, Larry Greenfield, Dave Cridland, and Pete Maclean were + active participants in those discussions or made suggestions to this + document. + +11. References + +11.1. Normative References + + [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application Configuration + Access Protocol", RFC 2244, November 1997. + + [I18N] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet + Message Access Protocol Internationalization", RFC 5255, + June 2008. + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [Kwds] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, March 1997. + + [MBRef] Gahrns, M., "IMAP4 Mailbox Referrals", RFC 2193, + September 1997. + + + + +Leiba & Melnikov Standards Track [Page 29] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +11.2. Informative References + + [CMbox] Gahrns, M. and R. Cheng, "The Internet Message Action + Protocol (IMAP4) Child Mailbox Extension", RFC 3348, + July 2002. + +Authors' Addresses + + Barry Leiba + IBM T.J. Watson Research Center + 19 Skyline Drive + Hawthorne, NY 10532 + US + + Phone: +1 914 784 7941 + EMail: leiba@watson.ibm.com + + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + + + + + + + + + + + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 30] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 31] + diff --git a/docs/rfcs/rfc5423.IM_Store_Events.txt b/docs/rfcs/rfc5423.IM_Store_Events.txt new file mode 100644 index 0000000..0326d94 --- /dev/null +++ b/docs/rfcs/rfc5423.IM_Store_Events.txt @@ -0,0 +1,955 @@ + + + + + + +Network Working Group R. Gellens +Request for Comments: 5423 QUALCOMM Inc. +Category: Standards Track C. Newman + Sun Microsystems + March 2009 + + + Internet Message Store Events + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (c) 2009 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents in effect on the date of + publication of this document (http://trustee.ietf.org/license-info). + Please review these documents carefully, as they describe your rights + and restrictions with respect to this document. + +Abstract + + One of the missing features in the existing Internet mail and + messaging standards is a facility for server-to-server and server-to- + client event notifications related to message store events. As the + scope of Internet mail expands to support more diverse media (such as + voice mail) and devices (such as cell phones) and to provide rich + interactions with other services (such as web portals and legal + compliance systems), the need for an interoperable notification + system increases. This document attempts to enumerate the types of + events that interest real-world consumers of such a system. + + This document describes events and event parameters that are useful + for several cases, including notification to administrative systems + and end users. This is not intended as a replacement for a message + access facility such as IMAP. + + + + + + + +Gellens & Newman Standards Track [Page 1] + +RFC 5423 Internet Message Store Events March 2009 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 1.1. Conventions Used in This Document . . . . . . . . . . . . 3 + 2. Terminology . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 3. Event Model . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 4. Event Types . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 4.1. Message Addition and Deletion . . . . . . . . . . . . . . 5 + 4.2. Message Flags . . . . . . . . . . . . . . . . . . . . . . 7 + 4.3. Access Accounting . . . . . . . . . . . . . . . . . . . . 8 + 4.4. Mailbox Management . . . . . . . . . . . . . . . . . . . . 8 + 5. Event Parameters . . . . . . . . . . . . . . . . . . . . . . . 10 + 6. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 14 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 14 + 8. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 15 + 9. References . . . . . . . . . . . . . . . . . . . . . . . . . . 15 + 9.1. Normative References . . . . . . . . . . . . . . . . . . . 15 + 9.2. Informative References . . . . . . . . . . . . . . . . . . 15 + Appendix A. Future Extensions . . . . . . . . . . . . . . . . . . 17 + +1. Introduction + + A message store is used to organize Internet Messages [RFC5322] into + one or more mailboxes (possibly hierarchical), annotate them in + various ways, and provide access to these messages and associated + metadata. Three different standards-based protocols have been widely + deployed to remotely access a message store. The Post Office + Protocol (POP) [RFC1939] provides simple download-and-delete access + to a single mail drop (which is a subset of the functionality + typically associated with a message store). The Internet Message + Access Protocol (IMAP) [RFC3501] provides an extensible feature-rich + model for online, offline, and disconnected access to a message store + with minimal constraints on any associated "fat-client" user + interface. Finally, mail access applications built on top of the + Hypertext Transfer Protocol (HTTP) [RFC2616] that run in standards- + based web browsers provide a third standards-based access mechanism + for online-only access. + + While simple and/or ad-hoc mechanisms for notifications have sufficed + to some degree in the past (e.g., "Simple New Mail Notification" + [RFC4146], "IMAP4 IDLE Command" [RFC2177]), as the scope and + importance of message stores expand, the demand for a more complete + store notification system increases. Some of the driving forces + behind this demand include: + + o Mobile devices with intermittent network connectivity that have + "new mail" or "message count" indicators. + + + + +Gellens & Newman Standards Track [Page 2] + +RFC 5423 Internet Message Store Events March 2009 + + + o Unified messaging systems that include both Internet and voice + mail require support for a message-waiting indicator on phones. + + o Interaction with systems for event-based or utility-computing + billing. + + o Simplification of the process of passing message store events to + non-Internet notification systems. + + o A calendar system may wish to subscribe to MessageNew + notifications in order to support iMIP [RFC2447]. + + o Some jurisdictions have laws or regulations for information + protection and auditing that require interoperable protocols + between message stores built by messaging experts and compliance + auditing systems built by compliance experts. + + Vendors who have deployed proprietary notification systems for their + Internet message stores have seen significant demand to provide + notifications for more and more events. As a first step towards + building a notification system, this document attempts to enumerate + the core events that real-world customers demand. + + This document includes those events that can be generated by the use + of IMAP4rev1 [RFC3501] and some existing extensions. As new IMAP + extensions are defined, or additional event types or parameters need + to be added, the set specified here can be extended by means of an + IANA registry with update requirements, as specified in Section 6. + +1.1. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + When these words appear in lower-case or with initial capital + letters, they are not RFC 2119 key words. + +2. Terminology + + The following terminology is used in this document: + + mailbox + A container for Internet messages and/or child mailboxes. A + mailbox may or may not permit delivery of new messages via a mail + delivery agent. + + + + + + +Gellens & Newman Standards Track [Page 3] + +RFC 5423 Internet Message Store Events March 2009 + + + mailbox identifier + A mailbox identifier provides sufficient information to identify a + specific mailbox on a specific server instance. An IMAP URL can + be a mailbox identifier. + + message access protocols + Protocols that provide clients (e.g., a mail user agent or web + browser) with access to the message store, including but not + limited to IMAP, POP, and HTTP. + + message context + As defined in [RFC3458]. + + UIDVALIDITY + As defined in IMAP4rev1 [RFC3501]. UIDVALIDITY is critical to the + correct operation of a caching mail client. When it changes, the + client MUST flush its cache. It's particularly important to + include UIDVALIDITY with event notifications related to message + addition or removal in order to keep the message data correctly + synchronized. + +3. Event Model + + The events that are generated by a message store depend to some + degree on the model used to represent a message store. The model the + IETF has for a message store is implicit from IMAP4rev1 and + extensions, so that model is assumed by this document. + + A message store event typically has an associated mailbox name and + usually has an associated user name (or authorization identity if + using the terminology from "Simple Authentication and Security Layer" + (SASL) [RFC4422]). Events referring to a specific message can use an + IMAP URL [RFC5092] to do so. Events referring to a set of messages + can use an IMAP URL to the mailbox plus an IMAP UID (Unique + Identifier) set. + + Each notification has a type and parameters. The type determines the + type of event, while the parameters supply information about the + context of the event that may be used to adjust subscription + preferences or may simply supply data associated with the event. The + types and parameter names in this document are restricted to US-ASCII + printable characters, so these events can be easily mapped to an + arbitrary notification system. However, this document assumes that + arbitrary parameter values (including large and multi-line values) + can be encoded with the notification system. Systems which lack that + feature could only implement a subset of these events. + + + + + +Gellens & Newman Standards Track [Page 4] + +RFC 5423 Internet Message Store Events March 2009 + + + This document does not indicate which event parameters are mandatory + or optional. That is done in documents that specify specific message + formats or bindings to a notification system. + + For scalability reasons, some degree of filtering at event generation + is necessary. At the very least, the ability to turn on and off + groups of related events and to suppress inclusion of large + parameters (such as messageContent) is needed. A sophisticated + publish/subscribe notification system may be able to propagate + cumulative subscription information to the publisher. + + Some of these events might be logically collapsed into a single event + type with a required parameter to distinguish between the cases + (e.g., QuotaExceed and QuotaWithin). However, until such time that + an event subscription model is formulated, it's not practical to make + such decisions. We thus note only the fact that some of these events + may be viewed as a single event type. + +4. Event Types + + This section discusses the different types of events useful in a + message store event notification system. The intention is to + document the events sufficient to cover an overwhelming majority of + known use cases while leaving less common event types for the future. + This section mentions parameters that are important or specific to + the events described here. Event parameters likely to be included in + most or all notifications are discussed in the next section. + +4.1. Message Addition and Deletion + + This section includes events related to message addition and + deletion. + + MessageAppend + A message was appended or concatenated to a mailbox by a message + access client. For the most part, this is identical to the + MessageNew event type except that the SMTP envelope information is + not included as a parameter, but information about which protocol + triggered the event MAY be included. See the MessageNew event for + more information. + + MessageExpire + One or more messages were expired from a mailbox due to server + expiration policy and are no longer accessible by the end user. + + The parameters include a mailbox identifier that MUST include + UIDVALIDITY and a UID set that describes the messages. + + + + +Gellens & Newman Standards Track [Page 5] + +RFC 5423 Internet Message Store Events March 2009 + + + Information about which server expiration policy was applied may + be included in the future. + + MessageExpunge + One or more messages were expunged from a mailbox by an IMAP + CLOSE/EXPUNGE, POP3 DELE+QUIT, HTTP, or equivalent client action + and are no longer accessible by the end user. + + The parameters include a mailbox identifier that MUST include + UIDVALIDITY, a UID set, and MAY also indicate which access + protocol triggered the event. + + MessageNew + A new message was received into a mailbox via a message delivery + agent. + + The parameters include a message identifier that, for IMAP- + accessible message stores, MUST include UIDVALIDITY and a UID. + The parameters MAY also include an SMTP envelope and other + arbitrary message and mailbox metadata. In some cases, the entire + new message itself may be included. The set of parameters SHOULD + be adjustable to the client's preference, with limits set by + server policy. An interesting policy, for example, would be to + include messages up to 2K in size with the notification, but to + include a URLAUTH [RFC4467] reference for larger messages. + + QuotaExceed + An operation failed (typically MessageNew) because the user's + mailbox exceeded one of the quotas (e.g., disk quota, message + quota, quota by message context, etc.). The parameters SHOULD + include at least the relevant user and quota and, optionally, the + mailbox. Quota usage SHOULD be included if possible. Parameters + needed to extend this to support quota by context are not + presently described in this document but could be added in the + future. + + QuotaWithin + An operation occurred (typically MessageExpunge or MessageExpire) + that reduced the user's quota usage under the limit. + + QuotaChange + The user's quota was changed. + + + + + + + + + +Gellens & Newman Standards Track [Page 6] + +RFC 5423 Internet Message Store Events March 2009 + + +4.2. Message Flags + + This section includes events related to changes in message flags. + + MessageRead + One or more messages in the mailbox were marked as read or seen by + a user. Note that POP has no concept of read or seen messages, so + these events are only generated by IMAP or HTTP clients (or + equivalent). + + The parameters include a mailbox identifier and a set of message + UIDs. + + MessageTrash + One or more messages were marked for future deletion by the user + but are still accessible over the protocol (the user's client may + or may not make these messages accessible through its user + interface). + + The parameters include a mailbox identifier and a set of message + UIDs. + + FlagsSet + One or more messages in the mailbox had one or more IMAP flags or + keywords set. + + The parameters include a list of IMAP flag or keyword names that + were set, a mailbox identifier, and the set of UIDs of affected + messages. The flagNames MUST NOT include \Recent. For + compatibility with simpler clients, it SHOULD be configurable + whether setting the \Seen or \Deleted flags results in this event + or the simpler MessageRead/MessageTrash events. By default, the + simpler message forms SHOULD be used for MessageRead and + MessageTrash. + + FlagsClear + One or more messages in the mailbox had one or more IMAP flags or + keywords cleared. + + The parameters include a list of IMAP flag or keyword names that + were cleared, a mailbox identifier, and the set of UIDs of + affected messages. The flagNames parameter MUST NOT include + \Recent. + + + + + + + + +Gellens & Newman Standards Track [Page 7] + +RFC 5423 Internet Message Store Events March 2009 + + +4.3. Access Accounting + + This section lists events related to message store access accounting. + + Login + A user has logged into the system via IMAP, HTTP, POP, or some + other mechanism. + + The parameters include the domain name and port used to access the + server and the user's authorization identity. Additional possible + parameters include the client's IP address and port, the + authentication identity (if different from the authorization + identity), the service name, the authentication mechanism, + information about any negotiated security layers, a timestamp, and + other information. + + Logout + A user has logged out or otherwise been disconnected from the + message store via IMAP, HTTP, POP, or some other mechanism. + + The parameters include the server domain name and the user's + authorization identity. Additional parameters MAY include any of + the information from the "Login" event as well as information + about the type of disconnect (suggested values include graceful, + abort, timeout, and security layer error), the duration of the + connection or session, and other information. + +4.4. Mailbox Management + + This section lists events related to the management of mailboxes. + + MailboxCreate + A mailbox has been created, or an access control changed on an + existing mailbox so that it is now accessible by the user. If the + mailbox creation caused the creation of new mailboxes earlier in + the hierarchy, separate MailboxCreate events are not generated, as + their creation is implied. + + The parameters include the created mailbox identifier, its + UIDVALIDITY for IMAP-accessible message stores, and MAY also + indicate which access protocol triggered the event. Access and + permissions information (such as Access Control List (ACL) + [RFC4314] settings) require a standardized format to be included, + and so are left for future extension. + + + + + + + +Gellens & Newman Standards Track [Page 8] + +RFC 5423 Internet Message Store Events March 2009 + + + MailboxDelete + A mailbox has been deleted, or an access control changed on an + existing mailbox so that it is no longer accessible by the user. + Note that if the mailbox has child mailboxes, only the specified + mailbox has been deleted, not the children. The mailbox becomes + \NOSELECT, and the hierarchy remains unchanged, as per the + description of the DELETE command in IMAP4rev1 [RFC3501]. + + The parameters include the deleted mailbox identifier and MAY also + indicate which access protocol triggered the event. + + MailboxRename + A mailbox has been renamed. Note that, per the description of the + RENAME command in IMAP4rev1 [RFC3501], special semantics regarding + the mailbox hierarchy apply when INBOX is renamed (child mailboxes + are usually included in the rename, but are excluded when INBOX is + renamed). When a mailbox other than INBOX is renamed and its + child mailboxes are also renamed as a result, separate + MailboxRename events are not generated for the child mailboxes, as + their renaming is implied. If the rename caused the creation of + new mailboxes earlier in the hierarchy, separate MailboxCreate + events are not generated for those, as their creation is implied. + When INBOX is renamed, a new INBOX is created. A MailboxCreate + event is not generated for the new INBOX, since it is implied. + + The parameters include the old mailbox identifier, the new mailbox + identifier, and MAY also indicate which access protocol triggered + the event. + + MailboxSubscribe + A mailbox has been added to the server-stored subscription list, + such as the one managed by the IMAP SUBSCRIBE and UNSUBSCRIBE + commands. + + The parameters include the user whose subscription list has been + affected, the mailbox identifier, and MAY also indicate which + access protocol triggered the event. + + MailboxUnSubscribe + A mailbox has been removed from the subscription list. + + The parameters include the user whose subscription list has been + affected, the mailbox identifier, and MAY also indicate which + access protocol triggered the event. + + + + + + + +Gellens & Newman Standards Track [Page 9] + +RFC 5423 Internet Message Store Events March 2009 + + +5. Event Parameters + + This section lists parameters included with these events. + + admin + Included with all events generated by message access protocols. + + The authentication identity associated with this event, as + distinct from the authorization identity (see "user"). This is + not included when it is the same as the value of the user + parameter. + + bodyStructure + May be included with MessageAppend and MessageNew. + + The IMAP BODYSTRUCTURE of the message. + + clientIP + Included with all events generated by message access protocols. + + The IPv4 or IPv6 address of the message store access client that + performed the action that triggered the notification. + + clientPort + Included with all events generated by message access protocols. + + The port number of the message store access client that performed + an action that triggered the notification (the port from which the + connection occurred). + + diskQuota + Included with QuotaExceed, QuotaWithin, and QuotaChange + notifications relating to a user or mailbox disk quota. May be + included with other notifications. + + Disk quota limit in kilobytes (1024 octets). + + diskUsed + Included with QuotaExceed and QuotaWithin notifications relating + to a user or mailbox disk quota. May be included with other + notifications. + + Disk space used in kilobytes (1024 octets). Only disk space that + counts against the quota is included. + + + + + + + +Gellens & Newman Standards Track [Page 10] + +RFC 5423 Internet Message Store Events March 2009 + + + envelope + May be included with the MessageNew notification. + + The message transfer envelope associated with final delivery of + the message for the MessageNew notification. This includes the + MAIL FROM and relevant RCPT TO line(s) used for final delivery + with CRLF delimiters and any ESMTP parameters. + + flagNames + Included with FlagsSet and FlagsClear events. May be included + with MessageAppend and MessageNew to indicate flags that were set + initially by the APPEND command or delivery agent, respectively. + + A list (likely to be space-separated) of IMAP flag or keyword + names that were set or cleared. Flag names begin with a backslash + while keyword names do not. The \Recent flag is explicitly not + permitted in the list. + + mailboxID + Included in events that affect mailboxes. A URI describing the + mailbox. In the case of MailboxRename, this refers to the new + name. + + maxMessages + Included with QuotaExceed and QuotaWithin notifications relating + to a user or mailbox message count quota. May be included with + other notifications. + + Quota limit on the number of messages in the mailbox, for events + referring to a mailbox. + + messageContent + May be included with MessageAppend and MessageNew. + + The entire message itself. Size-based suppression of this SHOULD + be available. + + messageSize + May be included with MessageAppend and MessageNew. + + Size of the RFC 5322 message itself in octets. This value matches + the length of the IMAP literal returned in response to an IMAP + FETCH of BODY[] for the referenced message. + + + + + + + + +Gellens & Newman Standards Track [Page 11] + +RFC 5423 Internet Message Store Events March 2009 + + + messages + Included with QuotaExceed and QuotaWithin notifications relating + to a user or mailbox message count quota. May be included with + other notifications. + + Number of messages in the mailbox. This is typically included + with message addition and deletion events. + + modseq + May be included with any notification referring to one message. + + This is the 64-bit integer MODSEQ as defined in [RFC4551]. No + assumptions about MODSEQ can be made if this is omitted. + + oldMailboxID + A URI describing the old name of a renamed or moved mailbox. + + pid + May be included with any notification. + + The process ID of the process that generated the notification. + + process + May be included with any notification. + + The name of the process that generated the notification. + + serverDomain + Included in Login and optionally in Logout or other events. The + domain name or IP address (v4 or v6) used to access the server or + mailbox. + + serverPort + Included in Login and optionally in Logout or other events. The + port number used to access the server. This is often a well-known + port. + + serverFQDN + May be included with any notification. + + The fully qualified domain name of the server that generated the + event. Note that this may be different from the server name used + to access the mailbox included in the mailbox identifier. + + + + + + + + +Gellens & Newman Standards Track [Page 12] + +RFC 5423 Internet Message Store Events March 2009 + + + service + May be included with any notification. + + The name of the service that triggered the event. Suggested + values include "imap", "pop", "http", and "admincli" (for an + administrative client). + + tags + May be included with any notification. + + A list of UTF-8 tags (likely to be comma-separated). One or more + tags can be set at the time a notification criteria or + notification subscription is created. Subscribers can use tags + for additional client-side filtering or dispatch of events. + + timestamp + May be included with any notification. + + The time at which the event occurred that triggered the + notification (the underlying protocol carrying the notification + may contain a timestamp for when the notification was generated). + This MAY be an approximate time. + + Timestamps are expressed in local time and contain the offset from + UTC (this information is used in several places in Internet mail) + and are normally in [RFC3339] format. + + uidnext + May be included with any notification referring to a mailbox. + + The UID that is projected to be assigned next in the mailbox. + This is typically included with message addition and deletion + events. This is equivalent to the UIDNEXT status item in the IMAP + STATUS command. + + uidset + Included with MessageExpires, MessageExpunges, MessageRead, + MessageTrash, FlagsSet, and FlagsClear. + + This includes the set of IMAP UIDs referenced. + + uri + Included with all notifications. A reference to the IMAP server, + a mailbox, or a message. + + Typically an IMAP URL. This can include the name of the server + used to access the mailbox/message, the mailbox name, the + UIDVALIDITY of the mailbox, and the UID of a specific message. + + + +Gellens & Newman Standards Track [Page 13] + +RFC 5423 Internet Message Store Events March 2009 + + + user + Included with all events generated by message access protocols. + + This is the authorization identifier used when the client + connected to the access protocol that triggered the event. Some + protocols (for example, many SASL mechanisms) distinguish between + authorization and authentication identifiers. For events + associated with a mailbox, this may be different from the owner of + the mailbox specified in the IMAP URL. + +6. IANA Considerations + + The IANA has created a new registry for "Internet Message Store + Events" that contains two sub-registries: event names and event + parameters. For both event names and event parameters, entries that + do not start with "vnd." are added by the IETF and are intended for + interoperable use. Entries that start with "vnd." are intended for + private use by one or more parties and are allocated to avoid + collisions. + + The initial values are contained in this document. + + Using IANA Considerations [RFC5226] terminology, entries that do not + start with "vnd." are allocated by IETF Consensus, while those + starting with "vnd." are allocated First Come First Served. + +7. Security Considerations + + Notifications can produce a large amount of traffic and expose + sensitive information. When notification mechanisms are used to + maintain state between different entities, the ability to corrupt or + manipulate notification messages could enable an attacker to modulate + the state of these entities. For example, if an attacker were able + to modify notifications sent from a message store to an auditing + server, he could modify the "user" and "messageContent" parameters in + MessageNew notifications to create false audit log entries. + + A competent transfer protocol for notifications must consider + authentication, authorization, privacy, and message integrity, as + well as denial-of-service issues. While the IETF has adequate tools + and experience to address these issues for mechanisms that involve + only one TCP connection, notification or publish/subscribe protocols + that are more sophisticated than a single end-to-end TCP connection + will need to pay extra attention to these issues and carefully + balance requirements to successfully deploy a system with security + and privacy considerations. + + + + + +Gellens & Newman Standards Track [Page 14] + +RFC 5423 Internet Message Store Events March 2009 + + +8. Acknowledgments + + Alexey Melnikov, Arnt Gulbrandsen, and Zoltan Ordogh have reviewed + and offered improvements to this document. Richard Barnes did a nice + review during Last Call. + +9. References + +9.1. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC5092] Melnikov, A. and C. Newman, "IMAP URL Scheme", RFC 5092, + November 2007. + + [RFC5226] Narten, T. and H. Alvestrand, "Guidelines for Writing an + IANA Considerations Section in RFCs", BCP 26, RFC 5226, + May 2008. + +9.2. Informative References + + [RFC1939] Myers, J. and M. Rose, "Post Office Protocol - Version 3", + STD 53, RFC 1939, May 1996. + + [RFC2177] Leiba, B., "IMAP4 IDLE command", RFC 2177, June 1997. + + [RFC2447] Dawson, F., Mansour, S., and S. Silverberg, "iCalendar + Message-Based Interoperability Protocol (iMIP)", RFC 2447, + November 1998. + + [RFC2616] Fielding, R., Gettys, J., Mogul, J., Frystyk, H., + Masinter, L., Leach, P., and T. Berners-Lee, "Hypertext + Transfer Protocol -- HTTP/1.1", RFC 2616, June 1999. + + [RFC3339] Klyne, G., Ed. and C. Newman, "Date and Time on the + Internet: Timestamps", RFC 3339, July 2002. + + [RFC3458] Burger, E., Candell, E., Eliot, C., and G. Klyne, "Message + Context for Internet Mail", RFC 3458, January 2003. + + [RFC4146] Gellens, R., "Simple New Mail Notification", RFC 4146, + August 2005. + + + + + +Gellens & Newman Standards Track [Page 15] + +RFC 5423 Internet Message Store Events March 2009 + + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication and + Security Layer (SASL)", RFC 4422, June 2006. + + [RFC4467] Crispin, M., "Internet Message Access Protocol (IMAP) - + URLAUTH Extension", RFC 4467, May 2006. + + [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional + STORE Operation or Quick Flag Changes Resynchronization", + RFC 4551, June 2006. + + [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, + October 2008. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gellens & Newman Standards Track [Page 16] + +RFC 5423 Internet Message Store Events March 2009 + + +Appendix A. Future Extensions + + This document specifies core functionality based on events that are + believed to be well understood, have known use cases, and are + implemented by at least one deployed real-world Internet message + store. (A few events are exceptions to the last test only: FlagsSet, + FlagsClear, MailboxCreate, MailboxDelete, MailboxRename, + MailboxSubscribe, and MailboxUnSubscribe.) + + Some events have been suggested but are postponed to future + extensions because they do not meet this criteria. These events + include messages that have been moved to archive storage and may + require extra time to access, quota by message context, + authentication failure, user mail account disabled, annotations, and + mailbox ACL or metadata change. The descriptions of several events + note additional parameters that are likely candidates for future + inclusion. See Section 6 for how the list of events and parameters + can be extended. + + In order to narrow the scope of this document to something that can + be completed, only events generated from the message store (by a + message access module, administrative module, or message delivery + agent) are considered. A complete mail system is normally linked + with an identity system that would also publish events of interest to + a message store event subscriber. Events of interest include account + created/deleted/disabled and password changed/expired. + +Authors' Addresses + + Randall Gellens + QUALCOMM Incorporated + 5775 Morehouse Drive + San Diego, CA 92651 + USA + + Phone: + EMail: rg+ietf@qualcomm.com + + + Chris Newman + Sun Microsystems + 800 Royal Oaks + Monrovia, CA 91016-6347 + USA + + Phone: + EMail: chris.newman@sun.com + + + + +Gellens & Newman Standards Track [Page 17] + diff --git a/docs/rfcs/rfc5464.IMAP_METADATA_extension.txt b/docs/rfcs/rfc5464.IMAP_METADATA_extension.txt new file mode 100644 index 0000000..645bfd9 --- /dev/null +++ b/docs/rfcs/rfc5464.IMAP_METADATA_extension.txt @@ -0,0 +1,1177 @@ + + + +Network Working Group C. Daboo +Request for Comments: 5464 Apple, Inc. +Category: Standards Track February 2009 + + + The IMAP METADATA Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + The METADATA extension to the Internet Message Access Protocol + permits clients and servers to maintain "annotations" or "metadata" + on IMAP servers. It is possible to have annotations on a per-mailbox + basis or on the server as a whole. For example, this would allow + comments about the purpose of a particular mailbox to be "attached" + to that mailbox, or a "message of the day" containing server status + information to be made available to anyone logging in to the server. + + + + + + + + + + + + + + + + + + + + + + + + + + + +Daboo Standards Track [Page 1] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +Table of Contents + + 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . 3 + 3. Data Model . . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 3.1. Overview . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 3.2. Namespace of Entries . . . . . . . . . . . . . . . . . . . 4 + 3.2.1. Entry Names . . . . . . . . . . . . . . . . . . . . . 5 + 3.3. Private versus Shared and Access Control . . . . . . . . . 6 + 4. IMAP Protocol Changes . . . . . . . . . . . . . . . . . . . . 7 + 4.1. General Considerations . . . . . . . . . . . . . . . . . . 7 + 4.2. GETMETADATA Command . . . . . . . . . . . . . . . . . . . 8 + 4.2.1. MAXSIZE GETMETADATA Command Option . . . . . . . . . . 9 + 4.2.2. DEPTH GETMETADATA Command Option . . . . . . . . . . . 10 + 4.3. SETMETADATA Command . . . . . . . . . . . . . . . . . . . 10 + 4.4. METADATA Response . . . . . . . . . . . . . . . . . . . . 12 + 4.4.1. METADATA Response with Values . . . . . . . . . . . . 13 + 4.4.2. Unsolicited METADATA Response without Values . . . . . 13 + 5. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 14 + 6. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 16 + 6.1. Entry and Attribute Registration Template . . . . . . . . 16 + 6.2. Server Entry Registrations . . . . . . . . . . . . . . . . 16 + 6.2.1. /shared/comment . . . . . . . . . . . . . . . . . . . 17 + 6.2.2. /shared/admin . . . . . . . . . . . . . . . . . . . . 17 + 6.3. Mailbox Entry Registrations . . . . . . . . . . . . . . . 17 + 6.3.1. /shared/comment . . . . . . . . . . . . . . . . . . . 18 + 6.3.2. /private/comment . . . . . . . . . . . . . . . . . . . 18 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 18 + 8. Normative References . . . . . . . . . . . . . . . . . . . . . 19 + Appendix A. Acknowledgments . . . . . . . . . . . . . . . . . . . 19 + + + + + + + + + + + + + + + + + + + + + +Daboo Standards Track [Page 2] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +1. Introduction and Overview + + The goal of the METADATA extension is to provide a means for clients + to set and retrieve "annotations" or "metadata" on an IMAP server. + The annotations can be associated with specific mailboxes or the + server as a whole. The server can choose to support only server + annotations or both server and mailbox annotations. + + A server that supports both server and mailbox annotations indicates + the presence of this extension by returning "METADATA" as one of the + supported capabilities in the CAPABILITY command response. + + A server that supports only server annotations indicates the presence + of this extension by returning "METADATA-SERVER" as one of the + supported capabilities in the CAPABILITY command response. + + A server that supports unsolicited annotation change responses MUST + support the "ENABLE" [RFC5161] extension to allow clients to turn + that feature on. + + The METADATA extension adds two new commands and one new untagged + response to the IMAP base protocol. + + This extension makes the following changes to the IMAP protocol: + + o adds a new SETMETADATA command + + o adds a new GETMETADATA command + + o adds a new METADATA untagged response + + o adds a new METADATA response code + + The rest of this document describes the data model and protocol + changes more rigorously. + +2. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + Whitespace and line breaks have been added to the examples in this + document to promote readability. + + + + +Daboo Standards Track [Page 3] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +3. Data Model + +3.1. Overview + + Mailboxes or the server as a whole may have zero or more annotations + associated with them. An annotation contains a uniquely named entry, + which has a value. Annotations can be added to mailboxes when a + mailbox name is provided as the first argument to the SETMETADATA + command, or to the server as a whole when the empty string is + provided as the first argument to the command. + + For example, a general comment being added to a mailbox may have an + entry name of "/comment" and a value of "Really useful mailbox". + + The protocol changes to IMAP described below allow a client to access + or change the values of any annotation entry, assuming it has + sufficient access rights to do so. + +3.2. Namespace of Entries + + Each annotation is an entry that has a hierarchical name, with each + component of the name separated by a slash ("/"). An entry name MUST + NOT contain two consecutive "/" characters and MUST NOT end with a + "/" character. + + The value of an entry is NIL (has no value), or a string or binary + data of zero or more octets. A string MAY contain multiple lines of + text. Clients MUST use the CRLF (0x0D 0x0A) character octet sequence + to represent line ends in a multi-line string value. + + Entry names MUST NOT contain asterisk ("*") or percent ("%") + characters and MUST NOT contain non-ASCII characters or characters + with octet values in the range 0x00 to 0x19. Invalid entry names + result in a BAD response in any IMAP command in which they are used. + + Entry names are case-insensitive. + + Use of control or punctuation characters in entry names is strongly + discouraged. + + This specification defines an initial set of entry names available + for use with mailbox and server annotations. In addition, an + extension mechanism is described to allow additional names to be + added for extensibility. + + The first component in entry names defines the scope of the + annotation. Currently, only the prefixes "/private" or "/shared" are + defined. These prefixes are used to indicate whether an annotation + + + +Daboo Standards Track [Page 4] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + is stored on a per-user basis ("/private") and not visible to other + users, or whether an annotation is shared between authorized users + ("/shared") with a single value that can be read and changed by + authorized users with appropriate access. See Section 3.3 for + details. + + Entry names can have any number of components starting at 2, unless + they fall under the vendor namespaces (i.e., have a /shared/vendor/ + or /private/vendor/ prefix as described + below), in which case they have at least 4 components. + +3.2.1. Entry Names + + Entry names MUST be specified in a Standards Track or IESG-approved + Experimental RFC, or fall under the vendor namespace. See + Section 6.1 for the registration template. + +3.2.1.1. Server Entries + + These entries are set or retrieved when the mailbox name argument to + the new SETMETADATA or GETMETADATA command is the empty string. + + /shared/comment + + Defines a comment or note that is associated with the server and + that is shared with authorized users of the server. + + /shared/admin + + Indicates a method for contacting the server administrator. The + value MUST be a URI (e.g., a mailto: or tel: URL). This entry is + always read-only -- clients cannot change it. It is visible to + authorized users of the system. + + /shared/vendor/ + + Defines the top level of shared entries associated with the + server, as created by a particular product of some vendor. This + entry can be used by vendors to provide server- or client-specific + annotations. The vendor-token MUST be registered with IANA, using + the Application Configuration Access Protocol (ACAP) [RFC2244] + vendor subtree registry. + + /private/vendor/ + + Defines the top level of private entries associated with the + server, as created by a particular product of some vendor. This + entry can be used by vendors to provide server- or client-specific + + + +Daboo Standards Track [Page 5] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + annotations. The vendor-token MUST be registered with IANA, using + the ACAP [RFC2244] vendor subtree registry. + +3.2.1.2. Mailbox Entries + + These entries are set or retrieved when the mailbox name argument to + the new SETMETADATA or GETMETADATA command is not the empty string. + + /shared/comment + + Defines a shared comment or note associated with a mailbox. + + /private/comment + + Defines a private (per-user) comment or note associated with a + mailbox. + + /shared/vendor/ + + Defines the top level of shared entries associated with a specific + mailbox, as created by a particular product of some vendor. This + entry can be used by vendors to provide client-specific + annotations. The vendor-token MUST be registered with IANA, using + the ACAP [RFC2244] vendor subtree registry. + + /private/vendor/ + + Defines the top level of private entries associated with a + specific mailbox, as created by a particular product of some + vendor. This entry can be used by vendors to provide client- + specific annotations. The vendor-token MUST be registered with + IANA, using the ACAP [RFC2244] vendor subtree registry. + +3.3. Private versus Shared and Access Control + + In the absence of the ACL (Access Control List) extension [RFC4314], + users can only set and retrieve private or shared mailbox annotations + on a mailbox that exists and is returned to them via a LIST or LSUB + command, and on which they have either read or write access to the + actual message content of the mailbox (as determined by the READ-ONLY + and READ-WRITE response codes as described in Section 5.2 of + [RFC4314]). + + When the ACL extension [RFC4314] is present, users can only set and + retrieve private or shared mailbox annotations on a mailbox on which + they have the "l" right and any one of the "r", "s", "w", "i", or "p" + rights. + + + + +Daboo Standards Track [Page 6] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + If a client attempts to set or retrieve annotations on mailboxes that + do not satisfy the conditions above, the server MUST respond with a + NO response. + + Users can always retrieve private or shared server annotations if + they exist. Servers MAY restrict the creation of private or shared + server annotations as appropriate. When restricted, the server MUST + return a NO response when the SETMETADATA command is used to try to + create a server annotation. + + If the METADATA extension is present, support for shared annotations + is REQUIRED, whilst support for private annotations is OPTIONAL. + This recognizes the fact that support for private annotations may + introduce significantly more complexity to a server in terms of + tracking ownership of the annotations, how quota is determined for + users based on their own annotations, etc. + +4. IMAP Protocol Changes + +4.1. General Considerations + + The new SETMETADATA command and the METADATA response each have a + mailbox name argument. An empty string is used for the mailbox name + to signify server annotations. A non-empty string is used to signify + mailbox annotations attached to the corresponding mailbox. + + Servers SHOULD ensure that mailbox annotations are automatically + moved when the mailbox they refer to is renamed, i.e., the + annotations follow the mailbox. This applies to a rename of the + INBOX, with the additional behavior that the annotations are copied + from the original INBOX to the renamed mailbox, i.e., mailbox + annotations are preserved on the INBOX when it is renamed. + + Servers SHOULD delete annotations for a mailbox when the mailbox is + deleted, so that a mailbox created with the same name as a previously + existing mailbox does not inherit the old mailbox annotations. + + Servers SHOULD allow annotations on all 'types' of mailboxes, + including ones reporting \Noselect for their LIST response. Servers + can implicitly remove \Noselect mailboxes when all child mailboxes + are removed, and, at that time any annotations associated with the + \Noselect mailbox SHOULD be removed. + + The server is allowed to impose limitations on the size of any one + annotation or the total number of annotations for a single mailbox or + for the server as a whole. However, the server MUST accept an + annotation data size of at least 1024 bytes, and an annotation count + per server or mailbox of at least 10. + + + +Daboo Standards Track [Page 7] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + Some annotations may be "read-only" -- i.e., they are set by the + server and cannot be changed by the client. Also, such annotations + may be "computed" -- i.e., the value changes based on underlying + properties of the mailbox or server. For example, an annotation + reporting the total size of all messages in the mailbox would change + as messages are added or removed. Or, an annotation containing an + IMAP URL for the mailbox would change if the mailbox was renamed. + + Servers MAY support sending unsolicited responses for use when + annotations are changed by some "third-party" (see Section 4.4). In + order to do so, servers MUST support the ENABLE command [RFC5161] and + MUST only send unsolicited responses if the client used the ENABLE + command [RFC5161] extension with the capability string "METADATA" or + "METADATA-SERVER" earlier in the session, depending on which of those + capabilities is supported by the server. + +4.2. GETMETADATA Command + + This extension adds the GETMETADATA command. This allows clients to + retrieve server or mailbox annotations. + + This command is only available in authenticated or selected state + [RFC3501]. + + Arguments: mailbox-name + options + entry-specifier + + Responses: required METADATA response + + Result: OK - command completed + NO - command failure: can't access annotations on + the server + BAD - command unknown or arguments invalid + + When the mailbox name is the empty string, this command retrieves + server annotations. When the mailbox name is not empty, this command + retrieves annotations on the specified mailbox. + + Options MAY be included with this command and are defined below. + + Example: + + C: a GETMETADATA "" /shared/comment + S: * METADATA "" (/shared/comment "Shared comment") + S: a OK GETMETADATA complete + + + + + +Daboo Standards Track [Page 8] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + In the above example, the contents of the value of the "/shared/ + comment" server entry is requested by the client and returned by + the server. + + Example: + + C: a GETMETADATA "INBOX" /private/comment + S: * METADATA "INBOX" (/private/comment "My own comment") + S: a OK GETMETADATA complete + + In the above example, the contents of the value of the "/private/ + comment" mailbox entry for the mailbox "INBOX" is requested by the + client and returned by the server. + + Entry specifiers can be lists of atomic specifiers, so that multiple + annotations may be returned in a single GETMETADATA command. + + Example: + + C: a GETMETADATA "INBOX" (/shared/comment /private/comment) + S: * METADATA "INBOX" (/shared/comment "Shared comment" + /private/comment "My own comment") + S: a OK GETMETADATA complete + + In the above example, the values of the two server entries + "/shared/comment" and "/private/comment" on the mailbox "INBOX" + are requested by the client and returned by the server. + +4.2.1. MAXSIZE GETMETADATA Command Option + + When the MAXSIZE option is specified with the GETMETADATA command, it + restricts which entry values are returned by the server. Only entry + values that are less than or equal in octet size to the specified + MAXSIZE limit are returned. If there are any entries with values + larger than the MAXSIZE limit, the server MUST include the METADATA + LONGENTRIES response code in the tagged OK response for the + GETMETADATA command. The METADATA LONGENTRIES response code returns + the size of the biggest entry value requested by the client that + exceeded the MAXSIZE limit. + + Example: + + C: a GETMETADATA "INBOX" (MAXSIZE 1024) + (/shared/comment /private/comment) + S: * METADATA "INBOX" (/private/comment "My own comment") + S: a OK [METADATA LONGENTRIES 2199] GETMETADATA complete + + + + + +Daboo Standards Track [Page 9] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + In the above example, the values of the two server entries + "/shared/comment" and "/private/comment" on the mailbox "INBOX" + are requested by the client, which wants to restrict the size of + returned values to 1024 octets. In this case, the "/shared/ + comment" entry value is 2199 octets and is not returned. + +4.2.2. DEPTH GETMETADATA Command Option + + When the DEPTH option is specified with the GETMETADATA command, it + extends the list of entry values returned by the server. For each + entry name specified in the GETMETADATA command, the server returns + the value of the specified entry name (if it exists), plus all + entries below the entry name up to the specified DEPTH. Three values + are allowed for DEPTH: + + "0" - no entries below the specified entry are returned + "1" - only entries immediately below the specified entry are returned + "infinity" - all entries below the specified entry are returned + + Thus, "depth 1" for an entry "/a" will match "/a" as well as its + children entries (e.g., "/a/b"), but will not match grandchildren + entries (e.g., "/a/b/c"). + + If the DEPTH option is not specified, this is the same as specifying + "DEPTH 0". + + Example: + + C: a GETMETADATA "INBOX" (DEPTH 1) + (/private/filters/values) + S: * METADATA "INBOX" (/private/filters/values/small + "SMALLER 5000" /private/filters/values/boss + "FROM \"boss@example.com\"") + S: a OK GETMETADATA complete + + In the above example, 2 entries below the /private/filters/values + entry exist on the mailbox "INBOX": "/private/filters/values/ + small" and "/private/filters/values/boss". + +4.3. SETMETADATA Command + + This extension adds the SETMETADATA command. This allows clients to + set annotations. + + This command is only available in authenticated or selected state + [RFC3501]. + + + + + +Daboo Standards Track [Page 10] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + Arguments: mailbox-name + entry + value + list of entry, values + + Responses: no specific responses for this command + + Result: OK - command completed + NO - command failure: can't set annotations, + or annotation too big or too many + BAD - command unknown or arguments invalid + + This command sets the specified list of entries by adding or + replacing the specified values provided, on the specified existing + mailboxes or on the server (if the mailbox argument is the empty + string). Clients can use NIL for the value of entries it wants to + remove. The server SHOULD NOT return a METADATA response containing + the updated annotation data. Clients MUST NOT assume that a METADATA + response will be sent, and MUST assume that if the command succeeds, + then the annotation has been changed. + + If the server is unable to set an annotation because the size of its + value is too large, the server MUST return a tagged NO response with + a "[METADATA MAXSIZE NNN]" response code when NNN is the maximum + octet count that it is willing to accept. + + If the server is unable to set a new annotation because the maximum + number of allowed annotations has already been reached, the server + MUST return a tagged NO response with a "[METADATA TOOMANY]" response + code. + + If the server is unable to set a new annotation because it does not + support private annotations on one of the specified mailboxes, the + server MUST return a tagged NO response with a "[METADATA NOPRIVATE]" + response code. + + When any one annotation fails to be set, resulting in a tagged NO + response from the server, then the server MUST NOT change the values + for other annotations specified in the SETMETADATA command. + + Example: + + C: a SETMETADATA INBOX (/private/comment {33} + S: + ready for data + My new comment across + two lines. + ) + S: a OK SETMETADATA complete + + + +Daboo Standards Track [Page 11] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + In the above example, the entry "/private/comment" for the mailbox + "INBOX" is created (if not already present) and the value set to a + multi-line string. + + Example: + + C: a SETMETADATA INBOX (/private/comment NIL) + S: a OK SETMETADATA complete + + In the above example, the entry "/private/comment" is removed from + the mailbox "INBOX". + + Multiple entries can be set in a single SETMETADATA command by + listing entry-value pairs in the list. + + Example: + + C: a SETMETADATA INBOX (/private/comment "My new comment" + /shared/comment "This one is for you!") + S: a OK SETMETADATA complete + + In the above example, the entries "/private/comment" and "/shared/ + comment" for the mailbox "INBOX" are created (if not already + present) and the values set as specified. + + Example: + + C: a SETMETADATA INBOX (/private/comment "My new comment") + S: a NO [METADATA TOOMANY] SETMETADATA failed + + In the above example, the server is unable to set the requested + (new) annotation as it has reached the limit on the number of + annotations it can support on the specified mailbox. + +4.4. METADATA Response + + The METADATA response displays results of a GETMETADATA command, or + can be returned as an unsolicited response at any time by the server + in response to a change in a server or mailbox annotation. + + When unsolicited responses are activated by the ENABLE [RFC5161] + command for this extension, servers MUST send unsolicited METADATA + responses if server or mailbox annotations are changed by a third- + party, allowing servers to keep clients updated with changes. + + Unsolicited METADATA responses MUST only contain entry names, not the + values. If the client wants to update any cached values, it must + explicitly retrieve those using a GETMETADATA command. + + + +Daboo Standards Track [Page 12] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + The METADATA response can contain multiple entries in a single + response, but the server is free to return multiple responses for + each entry or group of entries, if it desires. + + This response is only available in authenticated or selected state + [RFC3501]. + +4.4.1. METADATA Response with Values + + The response consists of a list of entry-value pairs. + + Example: + + C: a GETMETADATA "" /shared/comment + S: * METADATA "" (/shared/comment "My comment") + S: a OK GETMETADATA complete + + In the above example, a single entry with its value is returned by + the server. + + Example: + + C: a GETMETADATA "INBOX" /private/comment /shared/comment + S: * METADATA "INBOX" (/private/comment "My comment" + /shared/comment "Its sunny outside!") + S: a OK GETMETADATA complete + + In the above example, two entries and their values are returned by + the server. + + Example: + + C: a GETMETADATA "INBOX" /private/comment /shared/comment + S: * METADATA "INBOX" (/private/comment "My comment") + S: * METADATA "INBOX" (/shared/comment "Its sunny outside!") + S: a OK GETMETADATA complete + + In the above example, the server returns two separate responses + for each of the two entries requested. + +4.4.2. Unsolicited METADATA Response without Values + + The response consists of a list of entries, each of which have + changed on the server or mailbox. + + Example: + + + + + +Daboo Standards Track [Page 13] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + C: a NOOP + S: * METADATA "" /shared/comment + S: a OK NOOP complete + + In the above example, the server indicates that the "/shared/ + comment" server entry has been changed. + + Example: + + C: a NOOP + S: * METADATA "INBOX" /shared/comment /private/comment + S: a OK NOOP complete + + In the above example, the server indicates a change to two mailbox + entries. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. + + Non-terminals referenced but not defined below are as defined by + [RFC3501], with the new definitions in [RFC4466] superseding those in + [RFC3501]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + capability =/ "METADATA" / "METADATA-SERVER" + ; defines the capabilities for this extension. + + command-auth =/ setmetadata / getmetadata + ; adds to original IMAP command + + entries = entry / + "(" entry *(SP entry) ")" + ; entry specifiers + + entry = astring + ; slash-separated path to entry + ; MUST NOT contain "*" or "%" + + entry-value = entry SP value + + entry-values = "(" entry-value *(SP entry-value) ")" + + + + +Daboo Standards Track [Page 14] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + entry-list = entry *(SP entry) + ; list of entries used in unsolicited + ; METADATA response + + getmetadata = "GETMETADATA" [SP getmetadata-options] + SP mailbox SP entries + ; empty string for mailbox implies + ; server annotation. + + getmetadata-options = "(" getmetadata-option + *(SP getmetadata-option) ")" + + getmetadata-option = tagged-ext-label [SP tagged-ext-val] + ; tagged-ext-label and tagged-ext-val + ; are defined in [RFC4466]. + + maxsize-opt = "MAXSIZE" SP number + ; Used as a getmetadata-option + + metadata-resp = "METADATA" SP mailbox SP + (entry-values / entry-list) + ; empty string for mailbox implies + ; server annotation. + + response-payload =/ metadata-resp + ; adds to original IMAP data responses + + resp-text-code =/ "METADATA" SP "LONGENTRIES" SP number + ; new response codes for GETMETADATA + + resp-text-code =/ "METADATA" SP ("MAXSIZE" SP number / + "TOOMANY" / "NOPRIVATE") + ; new response codes for SETMETADATA + ; failures + + scope-opt = "DEPTH" SP ("0" / "1" / "infinity") + ; Used as a getmetadata-option + + setmetadata = "SETMETADATA" SP mailbox + SP entry-values + ; empty string for mailbox implies + ; server annotation. + + value = nstring / literal8 + + + + + + + +Daboo Standards Track [Page 15] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +6. IANA Considerations + + All entries MUST have either "/shared" or "/private" as a prefix. + Entry names MUST be specified in a Standards Track or IESG-approved + Experimental RFC, or fall under the vendor namespace (i.e., use + /shared/vendor/ or /private/vendor/ as + the prefix). + + Each entry registration MUST include a content-type that is used to + indicate the nature of the annotation value. Where applicable, a + charset parameter MUST be included with the content-type. + +6.1. Entry and Attribute Registration Template + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: [Either "Mailbox" or "Server"] + + Name: [the name of the entry] + + Description: [a description of what the entry is for] + + Content-type: [MIME Content-Type and charset for the entry value] + + RFC Number: [for entries published as RFCs] + + Contact: [email and/or physical address to contact for + additional information] + +6.2. Server Entry Registrations + + The following templates specify the IANA registrations of annotation + entries specified in this document. + + + + + + + + + + + + + + + + + +Daboo Standards Track [Page 16] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +6.2.1. /shared/comment + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: Server + + Name: /shared/comment + + Description: Defines a comment or note that is associated + with the server and that is shared with + authorized users of the server. + + Content-type: text/plain; charset=utf-8 + + RFC Number: RFC 5464 + + Contact: IMAP Extensions mailto:ietf-imapext@imc.org + +6.2.2. /shared/admin + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: Server + + Name: /shared/admin + + Description: Indicates a method for contacting the server + administrator. The value MUST be a URI (e.g., a + mailto: or tel: URL). This entry is always + read-only -- clients cannot change it. It is visible + to authorized users of the system. + + Content-type: text/plain; charset=utf-8 + + RFC Number: RFC 5464 + + Contact: IMAP Extensions mailto:ietf-imapext@imc.org + +6.3. Mailbox Entry Registrations + + The following templates specify the IANA registrations of annotation + entries specified in this document. + + + + + + + +Daboo Standards Track [Page 17] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +6.3.1. /shared/comment + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: Mailbox + + Name: /shared/comment + + Description: Defines a shared comment or note associated with a + mailbox. + + Content-type: text/plain; charset=utf-8 + + RFC Number: RFC 5464 + + Contact: IMAP Extensions mailto:ietf-imapext@imc.org + +6.3.2. /private/comment + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: Mailbox + + Name: /private/comment + + Description: Defines a private comment or note associated with a + mailbox. + + Content-type: text/plain; charset=utf-8 + + RFC Number: RFC 5464 + + Contact: IMAP Extensions mailto:ietf-imapext@imc.org + +7. Security Considerations + + The security considerations in Section 11 of [RFC3501] apply here + with respect to protecting annotations from snooping. Servers MAY + choose to only support the METADATA and/or METADATA-SERVER extensions + after a privacy layer has been negotiated by the client. + + Annotations can contain arbitrary data of varying size. As such, + servers MUST ensure that size limits are enforced to prevent a user + from using up all available space on a server and preventing use by + others. Clients MUST treat annotation data values as an "untrusted" + source of data as it is possible for it to contain malicious content. + + + +Daboo Standards Track [Page 18] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + Annotations whose values are intended to remain private MUST be + stored only in entries that have the "/private" prefix on the entry + name. + + Excluding the above issues, the METADATA extension does not raise any + security considerations that are not present in the base IMAP + protocol, and these issues are discussed in [RFC3501]. + +8. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2244] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC5161] Gulbrandsen, A. and A. Melnikov, "The IMAP ENABLE + Extension", RFC 5161, March 2008. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + +Appendix A. Acknowledgments + + The ideas expressed in this document are based on the message + annotation document that was co-authored by Randall Gellens. The + author would like to thank the following individuals for contributing + their ideas and support for writing this specification: Dave + Cridland, Arnt Gulbrandsen, Dan Karp, Alexey Melnikov, Ken Murchison, + Chris Newman, and Michael Wener. + + + + + + + + + + + + +Daboo Standards Track [Page 19] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +Author's Address + + Cyrus Daboo + Apple Inc. + 1 Infinite Loop + Cupertino, CA 95014 + USA + + EMail: cyrus@daboo.name + URI: http://www.apple.com/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Daboo Standards Track [Page 20] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2009). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Daboo Standards Track [Page 21] + + diff --git a/docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt b/docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt new file mode 100644 index 0000000..3fe5bcf --- /dev/null +++ b/docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt @@ -0,0 +1,1235 @@ + + + + + + +Network Working Group A. Gulbrandsen +Request for Comments: 5465 Oryx Mail Systems GmbH +Updates: 5267 C. King +Category: Standards Track A. Melnikov + Isode Ltd. + February 2009 + + + The IMAP NOTIFY Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (c) 2009 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents (http://trustee.ietf.org/ + license-info) in effect on the date of publication of this document. + Please review these documents carefully, as they describe your rights + and restrictions with respect to this document. + +Abstract + + This document defines an IMAP extension that allows a client to + request specific kinds of unsolicited notifications for specified + mailboxes, such as messages being added to or deleted from such + mailboxes. + + + + + + + + + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 1] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +Table of Contents + + 1. Overview and Rationale ..........................................3 + 2. Conventions Used in This Document ...............................4 + 3. The NOTIFY Extension ............................................4 + 3.1. The NOTIFY Command .........................................4 + 4. Interaction with the IDLE Command ...............................8 + 5. Event Types .....................................................8 + 5.1. FlagChange and AnnotationChange ............................9 + 5.2. MessageNew .................................................9 + 5.3. MessageExpunge ............................................10 + 5.4. MailboxName ...............................................11 + 5.5. SubscriptionChange ........................................12 + 5.6. MailboxMetadataChange .....................................12 + 5.7. ServerMetadataChange ......................................13 + 5.8. Notification Overflow .....................................13 + 5.9. ACL (Access Control List) Changes .........................13 + 6. Mailbox Specification ..........................................14 + 6.1. Mailbox Specifiers Affecting the Currently + Selected Mailbox ..........................................14 + 6.2. Personal ..................................................15 + 6.3. Inboxes ...................................................15 + 6.4. Subscribed ................................................15 + 6.5. Subtree ...................................................15 + 6.6. Mailboxes .................................................16 + 7. Extension to SEARCH and SORT Commands ..........................16 + 8. Formal Syntax ..................................................16 + 9. Security Considerations ........................................19 + 10. IANA Considerations ...........................................19 + 10.1. Initial LIST-EXTENDED Extended Data Item Registrations ...19 + 11. Acknowledgements ..............................................20 + 12. Normative References ..........................................20 + 13. Informative References ........................................21 + + + + + + + + + + + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 2] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +1. Overview and Rationale + + The IDLE command (defined in [RFC2177]) provides a way for the client + to go into a mode where the IMAP server pushes it notifications about + IMAP mailstore events for the selected mailbox. However, the IDLE + extension doesn't restrict or control which server events can be + sent, or what information the server sends in response to each event. + Also, IDLE only applies to the selected mailbox, thus requiring an + additional TCP connection per mailbox. + + This document defines an IMAP extension that allows clients to + express their preferences about unsolicited events generated by the + server. The extension allows clients to only receive events that + they are interested in, while servers know that they don't need to go + to the effort of generating certain types of untagged responses. + + Without the NOTIFY command defined in this document, an IMAP server + will only send information about mailstore changes to the client in + the following cases: + + - as the result of a client command (e.g., FETCH responses to a + FETCH or STORE command), + - as unsolicited responses sent just before the end of a command + (e.g., EXISTS or EXPUNGE) as the result of changes in other + sessions, and + - during an IDLE command. + + The NOTIFY command extends what information may be returned in those + last two cases, and also permits and requires the server to send + information about updates between commands. The NOTIFY command also + allows for the client to extend what information is sent unsolicited + about the selected mailbox and to request some update information to + be sent regarding other mailboxes. + + The interaction between IDLE and NOTIFY commands is described in + Section 4. + + For the new messages delivered to or appended to the selected + mailbox, the NOTIFY command can be used to request that a set of + attributes be sent to the client in an unsolicited FETCH response. + This allows a client to be a passive recipient of events and new mail + and to be able to maintain full synchronisation without having to + issue any subsequent commands except to modify the state of the + mailbox on the server. + + + + + + + +Gulbrandsen, et al. Standards Track [Page 3] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + Some mobile clients, however, may want mail "pushed" only for mail + that matches a SEARCH pattern. To meet that need, [RFC5267] is + augmented by this document to extend the UPDATE return option to + specify a list of fetch-atts to be returned when a new message is + delivered or appended in another session. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + The acronym MSN stands for Message Sequence Numbers (see Section + 2.3.1.2 of [RFC3501]). + + Example lines prefaced by "C:" are sent by the client and ones + prefaced by "S:", by the server. "[...]" means elision. + +3. The NOTIFY Extension + + IMAP servers that support this extension advertise the NOTIFY + capability. This extension adds the NOTIFY command as defined in + Section 5.1. + + A server implementing this extension is not required to implement + LIST-EXTENDED [RFC5258], even though a NOTIFY-compliant server must + be able to return extended LIST responses, defined in [RFC5258]. + +3.1. The NOTIFY Command + + Arguments: "SET" + Optional STATUS indicator + Mailboxes to be watched + Events about which to notify the client + + Or + Arguments: "NONE" + + Responses: Possibly untagged STATUS responses (for SET) + + Result: OK - The server will notify the client as requested. + NO - Unsupported NOTIFY event, NOTIFY too complex or + expensive, etc. + BAD - Command unknown, invalid, unsupported, or has + unknown arguments. + + + + + + +Gulbrandsen, et al. Standards Track [Page 4] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + The NOTIFY command informs the server that the client listens for + event notifications all the time (even when no command is in + progress), and requests the server to notify it about the specified + set of events. The NOTIFY command has two forms. NOTIFY NONE + specifies that the client is not interested in any kind of event + happening on the server. NOTIFY SET replaces the current list of + interesting events with a new list of events. + + Until the NOTIFY command is used for the first time, the server only + sends notifications while a command is being processed, and notifies + the client about these events on the selected mailbox (see Section 5 + for definitions): MessageNew, MessageExpunge, or FlagChange. It does + not notify the client about any events on other mailboxes. + + The effect of a successful NOTIFY command lasts until the next NOTIFY + command or until the IMAP connection is closed. + + A successful NOTIFY SET command MUST cause the server to immediately + return any accumulated changes to the currently selected mailbox (if + any), such as flag changes and new or expunged messages. Thus, a + successful NOTIFY SET command implies an implicit NOOP command. + + The NOTIFY SET command can request notifications of message-related + changes to the selected mailbox, whatever that may be at the time the + message notifications are being generated. This is done by + specifying either the SELECTED or the SELECTED-DELAYED mailbox + selector (see Section 6.1) in the NOTIFY SET command. If the + SELECTED/SELECTED-DELAYED mailbox selector is not specified in the + NOTIFY SET command, this means that the client doesn't want to + receive any s for the currently selected mailbox. + This is the same as specifying SELECTED NONE. + + The client can also request notifications on other mailboxes by name + or by a limited mailbox pattern match. Message-related notifications + returned for the currently selected mailbox will be those specified + by the SELECTED/SELECTED-DELAYED mailbox specifier, even if the + selected mailbox also appears by name (or matches a pattern) in the + command. Non-message-related notifications are controlled by mailbox + specifiers other than SELECTED/SELECTED-DELAYED. + + If the NOTIFY command enables MessageNew, MessageExpunge, + AnnotationChange, or FlagChange notifications for a mailbox other + than the currently selected mailbox, and the client has specified the + STATUS indicator parameter, then the server MUST send a STATUS + response for that mailbox before NOTIFY's tagged OK. If MessageNew + is enabled, the STATUS response MUST contain MESSAGES, UIDNEXT, and + UIDVALIDITY. If MessageExpunge is enabled, the STATUS response MUST + contain MESSAGES. If either AnnotationChange or FlagChange are + + + +Gulbrandsen, et al. Standards Track [Page 5] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + included and the server also supports the CONDSTORE [RFC4551] and/or + QRESYNC [RFC5162] extensions, the STATUS response MUST contain + UIDVALIDITY and HIGHESTMODSEQ. Absence of the STATUS indicator + parameter allows the client to avoid the additional STATUS responses. + This might be useful if the client already retrieved this information + before issuing the NOTIFY command. + + Clients are advised to limit the number of mailboxes used with + NOTIFY. Particularly, if a client asks for events for all accessible + mailboxes, the server may swamp the client with updates about shared + mailboxes. This may reduce the client's battery life. Also, this + wastes both server and network resources. + + For each mailbox specified, the server verifies that the client has + access using the following test: + + - If the name does not refer to an existing mailbox, the server MUST + ignore it. + + - If the name refers to a mailbox that the client can't LIST, the + server MUST ignore it. For a server that implements [RFC4314], + this means that if the client doesn't have the 'l' (lookup) right + for the name, then the server MUST ignore the mailbox. This + behavior prevents disclosure of potentially confidential + information to clients who don't have rights to know it. + + - If the name refers to a mailbox that the client can LIST (e.g., it + has the 'l' right from [RFC4314]), but the client doesn't have + another right required for processing of the specified event(s), + then the server MUST respond with an untagged extended LIST + response containing the \NoAccess name attribute. + + The server SHOULD return the tagged OK response if the client has + access to at least one of the mailboxes specified in the current list + of interesting events. The server MAY return the tagged NO response + if the client has no access to any of the specified mailboxes and no + access can ever be granted in the future (e.g., the client specified + an event for 'Subtree Bar/Foo', 'Bar/Foo' doesn't exist, and LIST + returns \Noinferiors for the parent 'Bar'). + + If the notification would be prohibitively expensive for the server + (e.g., "notify me of all flag changes in all mailboxes"), the server + MAY refuse the command with a tagged NO [NOTIFICATIONOVERFLOW] + response. + + + + + + + +Gulbrandsen, et al. Standards Track [Page 6] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + If the client requests information for events of an unsupported type, + the server MUST refuse the command with a tagged NO response (not a + BAD). This response SHOULD contain the BADEVENT response code, which + MUST list names of all events supported by the server. + + Here's an example: + + S: * OK [CAPABILITY IMAP4REV1 NOTIFY] + C: a login bob alice + S: a OK Password matched + C: b notify set status (selected MessageNew (uid + body.peek[header.fields (from to subject)]) MessageExpunge) + (subtree Lists MessageNew) + S: * STATUS Lists/Lemonade (UIDVALIDITY 4 UIDNEXT 9999 MESSAGES + 500) + S: [...] + S: * STATUS Lists/Im2000 (UIDVALIDITY 901 UIDNEXT 1 MESSAGES 0) + S: b OK done + C: c select inbox + S: [...] (the usual 7-8 responses to SELECT) + S: c OK INBOX selected + (Time passes. A new message is delivered to mailbox + Lists/Lemonade.) + S: * STATUS Lists/Lemonade (UIDVALIDITY 4 UIDNEXT 10000 + MESSAGES 501) + (Time passes. A new message is delivered to inbox.) + S: * 127 FETCH (UID 127001 BODY[HEADER.FIELDS (From To + Subject)] {75} + S: Subject: Re: good morning + S: From: alice@example.org + S: To: bob@example.org + S: + S: ) + (Time passes. The client decides it wants to know about + one more mailbox. As the client already knows necessary + STATUS information for all mailboxes below the Lists + mailbox, and because "notify set status" would cause + STATUS responses for *all* mailboxes specified in the + NOTIFY command, including the ones for which the client + already knows STATUS information, the client issues an + explicit STATUS request for the mailbox to be added to + the watch list, followed by the NOTIFY SET without the + STATUS parameter.) + C: d STATUS misc (UIDVALIDITY UIDNEXT MESSAGES) + S: * STATUS misc (UIDVALIDITY 1 UIDNEXT 999) + S: d STATUS completed + + + + + +Gulbrandsen, et al. Standards Track [Page 7] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + C: e notify set (selected MessageNew (uid + body.peek[header.fields (from to subject)]) MessageExpunge) + (subtree Lists MessageNew) (mailboxes misc MessageNew) + S: e OK done + +4. Interaction with the IDLE Command + + If IDLE [RFC2177] (as well as this extension) is supported, then + while processing any IDLE command, the server MUST send exactly the + same events as instructed by the client using the NOTIFY command. + + NOTIFY makes IDLE unnecessary for some clients. If a client does not + use MSNs and '*' in commands, it can request MessageExpunge and + MessageNew for the selected mailbox by using the NOTIFY command + instead of entering the IDLE mode. + + A client that uses MSNs and '*' in commands can still use the NOTIFY + command if it specifies the SELECTED-DELAYED mailbox specifier in the + NOTIFY command. + +5. Event Types + + Only some of the events in [RFC5423] can be expressed in IMAP, and + for some of them there are several possible ways to express the + event. + + This section specifies the events of which an IMAP server can notify + an IMAP client, and how. + + The server SHOULD omit notifying the client if the event is caused by + this client. For example, if the client issues CREATE and has + requested a MailboxName event that would cover the newly created + mailbox, the server SHOULD NOT notify the client of the MailboxName + change. + + All event types described in this document require the 'l' and 'r' + rights (see [RFC4314]) on all observed mailboxes. Servers that don't + implement [RFC4314] should map the above rights to their access- + control model. + + If the FlagChange and/or AnnotationChange events are specified, + MessageNew and MessageExpunge MUST also be specified by the client. + Otherwise, the server MUST respond with the tagged BAD response. + + If one of MessageNew or MessageExpunge is specified, then both events + MUST be specified. Otherwise, the server MUST respond with the + tagged BAD response. + + + + +Gulbrandsen, et al. Standards Track [Page 8] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + The client can instruct the server not to send an event by omitting + the necessary event from the list of events specified in NOTIFY SET, + by using the NONE event specifier in the NOTIFY SET, or by using + NOTIFY NONE. In particular, NOTIFY SET ... NONE can be used as a + snapshot facility by clients. + +5.1. FlagChange and AnnotationChange + + If the flag and/or message annotation change happens in the selected + mailbox, the server MUST notify the client by sending an unsolicited + FETCH response, which MUST include UID and FLAGS/ANNOTATION FETCH + data items. It MAY also send new FLAGS and/or OK [PERMANENTFLAGS + ...] responses. + + If a search context is in effect as specified in [RFC5267], an + ESEARCH ADDTO or ESEARCH REMOVEFROM will also be generated, if + appropriate. In this case, the FETCH response MUST precede the + ESEARCH response. + + If the change happens in another mailbox, then the server responds + with a STATUS response. The exact content of the STATUS response + depends on various factors. If CONDSTORE [RFC4551] and/or QRESYNC + [RFC5162] are enabled by the client, then the server sends a STATUS + response that includes at least HIGHESTMODSEQ and UIDVALIDITY status + data items. If the number of messages with the \Seen flag changes, + the server MAY also include the UNSEEN data item in the STATUS + response. If CONDSTORE/QRESYNC is not enabled by the client and the + server chooses not to include the UNSEEN data item, the server does + not notify the client. When this event is requested, the server MUST + notify the client about mailbox UIDVALIDITY changes. This is done by + sending a STATUS response that includes UIDVALIDITY. + + FlagChange covers the MessageRead, MessageTrash, FlagsSet, and + FlagsClear events in [RFC5423]. + + Example in the selected mailbox: + S: * 99 FETCH (UID 9999 FLAGS ($Junk)) + + And in another mailbox, with CONDSTORE in use: + S: * STATUS Lists/Lemonade (HIGHESTMODSEQ 65666665 UIDVALIDITY + 101) + + + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 9] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +5.2. MessageNew + + This covers both MessageNew and MessageAppend in [RFC5423]. + + If the new/appended message is in the selected mailbox, the server + notifies the client by sending an unsolicited EXISTS response, + followed by an unsolicited FETCH response containing the information + requested by the client. A FETCH response SHOULD NOT be generated + for a new message created by the client on this particular + connection, for instance, as the result of an APPEND or COPY command + to the selected mailbox performed by the client itself. The server + MAY also send a RECENT response, if the server marks the message as + \Recent. + + Note that a single EXISTS response can be returned for multiple + MessageAppend/MessageNew events. + + If a search context is in effect as specified in [RFC5267], an + ESEARCH ADDTO will also be generated, if appropriate. In this case, + the EXISTS response MUST precede the ESEARCH response. Both the + NOTIFY command and the SEARCH and SORT commands (see Section 7) can + specify attributes to be returned for new messages. These attributes + SHOULD be combined into a single FETCH response. The server SHOULD + avoid sending duplicate data. The FETCH response(s) MUST follow any + ESEARCH ADDTO responses. + + If the new/appended message is in another mailbox, the server sends + an unsolicited STATUS (UIDNEXT MESSAGES) response for the relevant + mailbox. If the CONDSTORE extension [RFC4551] and/or the QRESYNC + extension [RFC5162] is enabled, the HIGHESTMODSEQ status data item + MUST be included in the STATUS response. + + The client SHOULD NOT use FETCH attributes that implicitly set the + \seen flag, or that presuppose the existence of a given bodypart. + UID, MODSEQ, FLAGS, ENVELOPE, BODY.PEEK[HEADER.FIELDS... and + BODY/BODYSTRUCTURE may be the most useful attributes. + + Note that if a client asks to be notified of MessageNew events with + the SELECTED mailbox specifier, the number of messages can increase + at any time, and therefore the client cannot refer to a specific + message using the MSN/UID '*'. + + Example in the selected mailbox: + S: * 444 EXISTS + S: * 444 FETCH (UID 9999) + + And in another mailbox, without CONDSTORE enabled: + S: * STATUS Lists/Lemonade (UIDNEXT 10002 MESSAGES 503) + + + +Gulbrandsen, et al. Standards Track [Page 10] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +5.3. MessageExpunge + + If the expunged message or messages are in the selected mailbox, the + server notifies the client using EXPUNGE (or VANISHED, if [RFC5162] + is supported by the server and enabled by the client). + + If a search context is in effect, as specified in [RFC5267], an + ESEARCH REMOVEFROM will also be generated, if appropriate. + + If the expunged message or messages are in another mailbox, the + server sends an unsolicited STATUS (UIDNEXT MESSAGES) response for + the relevant mailbox. If the QRESYNC [RFC5162] extension is enabled, + the HIGHESTMODSEQ data item MUST be included in the STATUS response + as well. + + Note that if a client requests MessageExpunge with the SELECTED + mailbox specifier, the meaning of an MSN can change at any time, so + the client cannot use MSNs in commands anymore. For example, such a + client cannot use FETCH, but has to use UID FETCH. The meaning of + '*' can also change when messages are added or expunged. A client + wishing to keep using MSNs can either use the SELECTED-DELAYED + mailbox specifier or can avoid using the MessageExpunge event + entirely. + + The MessageExpunge notification covers both MessageExpunge and + MessageExpire events from [RFC5423]. + + Example in the selected mailbox, without QRESYNC: + S: * 444 EXPUNGE + + The same example in the selected mailbox, with QRESYNC: + S: * VANISHED 5444 + + And in another mailbox, when QRESYNC is not enabled: + S: * STATUS misc (UIDNEXT 999 MESSAGES 554) + +5.4. MailboxName + + These notifications are sent if an affected mailbox name was created + (with CREATE), deleted (with DELETE), or renamed (with RENAME). For + a server that implements [RFC4314], granting or revocation of the 'l' + right to the current user on the affected mailbox MUST be considered + mailbox creation or deletion, respectively. If a mailbox is created + or deleted, the mailbox itself and its direct parent (whether it is + an existing mailbox or not) are considered to be affected. + + + + + + +Gulbrandsen, et al. Standards Track [Page 11] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + The server notifies the client by sending an unsolicited LIST + response for each affected mailbox name. If, after the event, the + mailbox name does not refer to a mailbox accessible to the client, + the \Nonexistent flag MUST be included. + + For each LISTable mailbox renamed, the server sends an extended LIST + response [RFC5258] for the new mailbox name, containing the OLDNAME + extended data item with the old mailbox name. When a mailbox is + renamed, its children are renamed too. No additional MailboxName + events are sent for children in this case. When INBOX is renamed, a + new INBOX is assumed to be created. No MailboxName event is sent for + INBOX in this case. + + If the server automatically subscribes a mailbox when it is created + or renamed, then the unsolicited LIST response for each affected + subscribed mailbox name MUST include the \Subscribed attribute (see + [RFC5258]). The server SHOULD also include \HasChildren or + \HasNoChildren attributes [RFC5258] as appropriate. + + Example of a newly created mailbox (or granting of the 'l' right on + the mailbox): + S: * LIST () "/" "NewMailbox" + + And a deleted mailbox (or revocation of the 'l' right on the + mailbox): + S: * LIST (\NonExistent) "." "INBOX.DeletedMailbox" + + Example of a renamed mailbox: + S: * LIST () "/" "NewMailbox" ("OLDNAME" ("OldMailbox")) + +5.5. SubscriptionChange + + The server notifies the client by sending an unsolicited LIST + response for each affected mailbox name. If and only if the mailbox + is subscribed after the event, the \Subscribed attribute (see + [RFC5258]) is included. Note that in the LIST response, all mailbox + attributes MUST be accurately computed (this differs from the + behavior of the LSUB command). + + Example: + S: * LIST (\Subscribed) "/" "SubscribedMailbox" + +5.6. MailboxMetadataChange + + Support for this event type is OPTIONAL unless the METADATA extension + [RFC5464] is also supported by the server, in which case support for + this event type is REQUIRED. + + + + +Gulbrandsen, et al. Standards Track [Page 12] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + A client willing to receive unsolicited METADATA responses as a + result of using the MailboxMetadataChange event in the NOTIFY command + doesn't have to issue ENABLE METADATA. + + The server sends an unsolicited METADATA response (as per Section + 4.4.2 of [RFC5464]). If possible, only the changed metadata SHOULD + be included, but if the server can't detect a change to a single + metadata item, it MAY include all metadata items set on the mailbox. + If a metadata item is deleted (set to NIL), it MUST always be + included in the METADATA response. + + Example: + S: * METADATA "INBOX" /shared/comment + +5.7. ServerMetadataChange + + Support for this event type is OPTIONAL unless the METADATA or the + METADATA-SERVER extension [RFC5464] is also supported by the server, + in which case support for this event type is REQUIRED. + + A client willing to receive unsolicited METADATA responses as a + result of using the ServerMetadataChange event in the NOTIFY command + doesn't have to issue ENABLE METADATA or ENABLE METADATA-SERVER. + + The server sends an unsolicited METADATA response (as per Section + 4.4.2 of [RFC5464]). Only the names of changed metadata entries + SHOULD be returned in such METADATA responses. If a metadata item is + deleted (set to NIL), it MUST always be included in the METADATA + response. + + Example: + S: * METADATA "" /shared/comment + +5.8. Notification Overflow + + If the server is unable or unwilling to deliver as many notifications + as it is being asked to, it may disable notifications for some or all + clients. It MUST notify these clients by sending an untagged "OK + [NOTIFICATIONOVERFLOW]" response and behave as if a NOTIFY NONE + command had just been received. + + Example: + S: * OK [NOTIFICATIONOVERFLOW] ...A comment can go here... + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 13] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +5.9. ACL (Access Control List) Changes + + Even if NOTIFY succeeds, it is still possible to lose access to the + mailboxes being monitored at a later time. If this happens, the + server MUST stop monitoring these mailboxes. If access is later + granted, the server MUST restart event monitoring. + + The server SHOULD return the LIST response with the \NoAccess name + attribute if and when the mailbox loses the 'l' right. Similarly, + the server SHOULD return the LIST response with no \NoAccess name + attribute if the mailbox was previously reported as having \NoAccess + and the 'l' right is later granted. + +6. Mailbox Specification + + Mailboxes to be monitored can be specified in several different ways. + + Only 'SELECTED' and 'SELECTED-DELAYED' (Section 6.1) match the + currently selected mailbox. All other mailbox specifications affect + other (non-selected) mailboxes. + + Note that multiple s can apply to the same mailbox. The + following example demonstrates this. In this example, MessageNew and + MessageExpunge events are reported for INBOX, due to the first + . A SubscriptionChange event will also be reported for + INBOX, due to the second . + + C: a notify set (mailboxes INBOX (Messagenew messageExpunge)) + (personal (SubscriptionChange)) + + A typical client that supports the NOTIFY extension would ask for + events on the selected mailbox and some named mailboxes. + + In the next example, the client asks for FlagChange events for all + personal mailboxes except the currently selected mailbox. This is + different from the previous example because SELECTED overrides all + other message event definitions for the currently selected mailbox + (see Section 3.1). + + C: a notify set (selected (Messagenew (uid flags) messageExpunge)) + (personal (MessageNew FlagChange MessageExpunge)) + +6.1. Mailbox Specifiers Affecting the Currently Selected Mailbox + + Only one of the mailbox specifiers affecting the currently selected + mailbox can be specified in any NOTIFY command. The two such mailbox + specifiers (SELECTED and SELECTED-DELAYED) are described below. + + + + +Gulbrandsen, et al. Standards Track [Page 14] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + Both refer to the mailbox that was selected using either SELECT or + EXAMINE (see [RFC3501], Sections 6.3.1 and 6.3.2). When the IMAP + connection is not in the selected state, such mailbox specifiers + don't refer to any mailbox. + + The mailbox specifiers only apply to s. It is an + error to specify other types of events with either the SELECTED or + the SELECTED-DELAYED selector. + +6.1.1. Selected + + The SELECTED mailbox specifier requires the server to send immediate + notifications for the currently selected mailbox about all specified + s. + +6.1.2. Selected-Delayed + + The SELECTED-DELAYED mailbox specifier requires the server to delay a + MessageExpunge event until the client issues a command that allows + returning information about expunged messages (see Section 7.4.1 of + [RFC3501] for more details), for example, till a NOOP or an IDLE + command has been issued. When SELECTED-DELAYED is specified, the + server MAY also delay returning other s until the + client issues one of the commands specified above, or it MAY return + them immediately. + +6.2. Personal + + Personal refers to all selectable mailboxes in the user's personal + namespace(s), as defined in [RFC2342]. + +6.3. Inboxes + + Inboxes refers to all selectable mailboxes in the user's personal + namespace(s) to which messages may be delivered by a Message Delivery + Agent (MDA) (see [EMAIL-ARCH], particularly Section 4.3.3). + + If the IMAP server cannot easily compute this set, it MUST treat + "inboxes" as equivalent to "personal". + +6.4. Subscribed + + Subscribed refers to all mailboxes subscribed to by the user. + + If the subscription list changes, the server MUST reevaluate the + list. + + + + + +Gulbrandsen, et al. Standards Track [Page 15] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +6.5. Subtree + + Subtree is followed by a mailbox name or list of mailbox names. A + subtree refers to all selectable mailboxes that are subordinate to + the specified mailbox plus the specified mailbox itself. + +6.6. Mailboxes + + Mailboxes is followed by a mailbox name or a list of mailbox names. + The server MUST NOT do a wildcard expansion. This means there is no + special treatment for the LIST wildcard characters ('*' and '%') if + they are present in mailbox names. + +7. Extension to SEARCH and SORT Commands + + If the server that supports the NOTIFY extension also supports + CONTEXT=SEARCH and/or CONTEXT=SORT as defined in [RFC5267], the + UPDATE return option is extended so that a client can request that + FETCH attributes be returned when a new message is added to the + context result set. + + For example: + + C: a00 SEARCH RETURN (COUNT UPDATE (UID BODY[HEADER.FIELDS (TO + FROM SUBJECT)])) FROM "boss" + S: * ESEARCH (TAG "a00") (COUNT 17) + S: a00 OK + [...a new message is delivered...] + S: * EXISTS 93 + S: * 93 FETCH (UID 127001 BODY[HEADER.FIELDS (FROM TO SUBJECT)] + {76} + S: Subject: Re: good morning + S: From: myboss@example.org + S: To: bob@example.org + S: + S: ) + S: * ESEARCH (TAG "a00") ADDTO (0 93) + + Note that the EXISTS response MUST precede any FETCH responses, and + together they MUST precede the ESEARCH response. + + No untagged FETCH response SHOULD be returned if a message becomes a + member of UPDATE SEARCH due to flag or annotation changes. + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 16] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +8. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. [RFC3501] defines + the non-terminals "capability", "command-auth", "mailbox", "mailbox- + data", "resp-text-code", and "search-key". The "modifier-update" + non-terminal is defined in [RFC5267]. "mbx-list-oflag" is defined in + [RFC3501] and updated by [RFC5258]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. For example, the + non-terminal value "SELECTED" must be + treated in the same way as "Selected" or "selected". + + capability =/ "NOTIFY" + + command-auth =/ notify + + notify = "NOTIFY" SP + (notify-set / notify-none) + + notify-set = "SET" [status-indicator] SP event-groups + ; Replace registered notification events + ; with the specified list of events + + notify-none = "NONE" + ; Cancel all registered notification + ; events. The client is not interested + ; in receiving any events. + + status-indicator = SP "STATUS" + + one-or-more-mailbox = mailbox / many-mailboxes + + many-mailboxes = "(" mailbox *(SP mailbox) ")" + + event-groups = event-group *(SP event-group) + + event-group = "(" filter-mailboxes SP events ")" + ;; Only s are allowed in + ;; when is used. + + filter-mailboxes = filter-mailboxes-selected / + filter-mailboxes-other + + + + + +Gulbrandsen, et al. Standards Track [Page 17] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + filter-mailboxes-other = "inboxes" / "personal" / "subscribed" / + ( "subtree" SP one-or-more-mailbox ) / + ( "mailboxes" SP one-or-more-mailbox ) + + filter-mailboxes-selected = "selected" / "selected-delayed" + ;; Apply to the currently selected mailbox only. + ;; Only one of them can be specified in a NOTIFY + ;; command. + + events = ( "(" event *(SP event) ")" ) / "NONE" + ;; As in [MSGEVENT]. + ;; "NONE" means that the client does not wish + ;; to receive any events for the specified + ;; mailboxes. + + event = message-event / + mailbox-event / user-event / event-ext + + message-event = ( "MessageNew" [SP + "(" fetch-att *(SP fetch-att) ")" ] ) + / "MessageExpunge" + / "FlagChange" + / "AnnotationChange" + ;; "MessageNew" includes "MessageAppend" from + ;; [MSGEVENT]. "FlagChange" is any of + ;; "MessageRead", "MessageTrash", "FlagsSet", + ;; "FlagsClear" [MSGEVENT]. "MessageExpunge" + ;; includes "MessageExpire" [MSGEVENT]. + ;; MessageNew and MessageExpunge MUST always + ;; be specified together. If FlagChange is + ;; specified, then MessageNew and MessageExpunge + ;; MUST be specified as well. + ;; The fett-att list may only be present for the + ;; SELECTED/SELECTED-DELAYED mailbox filter + ;; (). + + mailbox-event = "MailboxName" / + "SubscriptionChange" / "MailboxMetadataChange" + ; "SubscriptionChange" includes + ; MailboxSubscribe and MailboxUnSubscribe. + ; "MailboxName" includes MailboxCreate, + ; "MailboxDelete" and "MailboxRename". + + user-event = "ServerMetadataChange" + + event-ext = atom + ;; For future extensions + + + + +Gulbrandsen, et al. Standards Track [Page 18] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + oldname-extended-item = "OLDNAME" SP "(" mailbox ")" + ;; Extended data item (mbox-list-extended-item) + ;; returned in a LIST response when a mailbox is + ;; renamed. + ;; Note 1: the OLDNAME tag can be returned + ;; with or without surrounding quotes, as per + ;; mbox-list-extended-item-tag production. + + resp-text-code =/ "NOTIFICATIONOVERFLOW" / + unsupported-events-code + + message-event-name = "MessageNew" / + "MessageExpunge" / "FlagChange" / + "AnnotationChange" + + event-name = message-event-name / mailbox-event / + user-event + + unsupported-events-code = "BADEVENT" + SP "(" event-name *(SP event-name) ")" + + modifier-update = "UPDATE" + [ "(" fetch-att *(SP fetch-att) ")" ] + + mbx-list-oflag =/ "\NoAccess" + +9. Security Considerations + + It is very easy for a client to deny itself service using NOTIFY. + Asking for all events on all mailboxes may work on a small server, + but with a big server, can swamp the client's network connection or + processing capability. In the worst case, the server's processing + could also degrade the service it offers to other clients. + + Server authors should be aware that if a client issues requests and + does not listen to the resulting responses, the TCP window can easily + fill up, and a careless server might block. This problem also exists + in plain IMAP; however, this extension magnifies the problem. + + This extension makes it possible to retrieve messages immediately + when they are added to the mailbox. This makes it wholly impractical + to delete sensitive messages using programs like imapfilter. Using + SIEVE [RFC5228] or similar is much better. + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 19] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +10. IANA Considerations + + The IANA has added NOTIFY to the list of IMAP extensions. + +10.1. Initial LIST-EXTENDED Extended Data Item Registrations + + The following entry has been added to the LIST-EXTENDED response + registry [RFC5258]: + + To: iana@iana.org + Subject: Registration of OLDNAME LIST-EXTENDED extended data item + + LIST-EXTENDED extended data item tag: OLDNAME + + LIST-EXTENDED extended data item description: The OLDNAME extended + data item describes the old mailbox name for the mailbox + identified by the LIST response. + + Which LIST-EXTENDED option(s) (and their types) causes this extended + data item to be returned (if any): none + + Published specification : RFC 5465, Section 5.4. + + Security considerations: none + + Intended usage: COMMON + + Person and email address to contact for further information: Alexey + Melnikov + + Owner/Change controller: iesg@ietf.org + +11. Acknowledgments + + The authors gratefully acknowledge the help of Peter Coates, Dave + Cridland, Mark Crispin, Cyrus Daboo, Abhijit Menon-Sen, Timo + Sirainen, and Eric Burger. In particular, Peter Coates contributed + lots of text and useful suggestions to this document. + + Various examples are copied from other RFCs. + + This document builds on one published and two unpublished drafts by + the same authors. + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 20] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +12. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2177] Leiba, B., "IMAP4 IDLE command", RFC 2177, June 1997. + + [RFC2342] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, + May 1998. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) + Extension", RFC 4314, December 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to + IMAP4 ABNF", RFC 4466, April 2006. + + [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for + Conditional STORE Operation or Quick Flag Changes + Resynchronization", RFC 4551, June 2006. + + [RFC5162] Melnikov, A., Cridland, D., and C. Wilson, "IMAP4 + Extensions for Quick Mailbox Resynchronization", RFC + 5162, March 2008. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access + Protocol version 4 - LIST Command Extensions", RFC 5258, + June 2008. + + [RFC5267] Cridland, D. and C. King, "Contexts for IMAP4", RFC + 5267, July 2008. + + [RFC5423] Newman, C. and R. Gellens, "Internet Message Store + Events", RFC 5423, Month 2009. + + [RFC5464] Daboo, C., "The IMAP METADATA Extension", RFC 5464, + February 2009. + +13. Informative References + + [RFC5228] Guenther, P., Ed., and T. Showalter, Ed., "Sieve: An + Email Filtering Language", RFC 5228, January 2008. + + + +Gulbrandsen, et al. Standards Track [Page 21] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + [EMAIL-ARCH] Crocker, D., "Internet Mail Architecture", Work in + Progress, October 2008. + +Authors' Addresses + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + EMail: arnt@oryx.com + + + Curtis King + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Curtis.King@isode.com + + + Alexey Melnikov + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + + + + + + + + + + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 22] + diff --git a/docs/rfcs/rfc5530.IMAP_Response_codes.txt b/docs/rfcs/rfc5530.IMAP_Response_codes.txt new file mode 100644 index 0000000..946fbb5 --- /dev/null +++ b/docs/rfcs/rfc5530.IMAP_Response_codes.txt @@ -0,0 +1,507 @@ + + + + + + +Network Working Group A. Gulbrandsen +Request for Comments: 5530 Oryx Mail Systems GmbH +Category: Standards Track May 2009 + + + IMAP Response Codes + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (c) 2009 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents in effect on the date of + publication of this document (http://trustee.ietf.org/license-info). + Please review these documents carefully, as they describe your rights + and restrictions with respect to this document. + +Abstract + + IMAP responses consist of a response type (OK, NO, BAD), an optional + machine-readable response code, and a human-readable text. + + This document collects and documents a variety of machine-readable + response codes, for better interoperation and error reporting. + + + + + + + + + + + + + + + + + + +Gulbrandsen Standards Track [Page 1] + +RFC 5530 IMAP Response Codes May 2009 + + +1. Introduction + + Section 7.1 of [RFC3501] defines a number of response codes that can + help tell an IMAP client why a command failed. However, experience + has shown that more codes are useful. For example, it is useful for + a client to know that an authentication attempt failed because of a + server problem as opposed to a password problem. + + Currently, many IMAP servers use English-language, human-readable + text to describe these errors, and a few IMAP clients attempt to + translate this text into the user's language. + + This document names a variety of errors as response codes. It is + based on errors that have been checked and reported on in some IMAP + server implementations, and on the needs of some IMAP clients. + + This document doesn't require any servers to test for these errors or + any clients to test for these names. It only names errors for better + reporting and handling. + +2. Conventions Used in This Document + + Formal syntax is defined by [RFC5234] as modified by [RFC3501]. + + Example lines prefaced by "C:" are sent by the client and ones + prefaced by "S:" by the server. "[...]" means elision. + +3. Response Codes + + This section defines all the new response codes. Each definition is + followed by one or more examples. + + UNAVAILABLE + Temporary failure because a subsystem is down. For example, an + IMAP server that uses a Lightweight Directory Access Protocol + (LDAP) or Radius server for authentication might use this + response code when the LDAP/Radius server is down. + + C: a LOGIN "fred" "foo" + S: a NO [UNAVAILABLE] User's backend down for maintenance + + AUTHENTICATIONFAILED + Authentication failed for some reason on which the server is + unwilling to elaborate. Typically, this includes "unknown + user" and "bad password". + + + + + + +Gulbrandsen Standards Track [Page 2] + +RFC 5530 IMAP Response Codes May 2009 + + + This is the same as not sending any response code, except that + when a client sees AUTHENTICATIONFAILED, it knows that the + problem wasn't, e.g., UNAVAILABLE, so there's no point in + trying the same login/password again later. + + C: b LOGIN "fred" "foo" + S: b NO [AUTHENTICATIONFAILED] Authentication failed + + AUTHORIZATIONFAILED + Authentication succeeded in using the authentication identity, + but the server cannot or will not allow the authentication + identity to act as the requested authorization identity. This + is only applicable when the authentication and authorization + identities are different. + + C: c1 AUTHENTICATE PLAIN + [...] + S: c1 NO [AUTHORIZATIONFAILED] No such authorization-ID + + C: c2 AUTHENTICATE PLAIN + [...] + S: c2 NO [AUTHORIZATIONFAILED] Authenticator is not an admin + + + EXPIRED + Either authentication succeeded or the server no longer had the + necessary data; either way, access is no longer permitted using + that passphrase. The client or user should get a new + passphrase. + + C: d login "fred" "foo" + S: d NO [EXPIRED] That password isn't valid any more + + PRIVACYREQUIRED + The operation is not permitted due to a lack of privacy. If + Transport Layer Security (TLS) is not in use, the client could + try STARTTLS (see Section 6.2.1 of [RFC3501]) and then repeat + the operation. + + C: d login "fred" "foo" + S: d NO [PRIVACYREQUIRED] Connection offers no privacy + + C: d select inbox + S: d NO [PRIVACYREQUIRED] Connection offers no privacy + + + + + + + +Gulbrandsen Standards Track [Page 3] + +RFC 5530 IMAP Response Codes May 2009 + + + CONTACTADMIN + The user should contact the system administrator or support + desk. + + C: e login "fred" "foo" + S: e OK [CONTACTADMIN] + + NOPERM + The access control system (e.g., Access Control List (ACL), see + [RFC4314]) does not permit this user to carry out an operation, + such as selecting or creating a mailbox. + + C: f select "/archive/projects/experiment-iv" + S: f NO [NOPERM] Access denied + + INUSE + An operation has not been carried out because it involves + sawing off a branch someone else is sitting on. Someone else + may be holding an exclusive lock needed for this operation, or + the operation may involve deleting a resource someone else is + using, typically a mailbox. + + The operation may succeed if the client tries again later. + + C: g delete "/archive/projects/experiment-iv" + S: g NO [INUSE] Mailbox in use + + EXPUNGEISSUED + Someone else has issued an EXPUNGE for the same mailbox. The + client may want to issue NOOP soon. [RFC2180] discusses this + subject in depth. + + C: h search from fred@example.com + S: * SEARCH 1 2 3 5 8 13 21 42 + S: h OK [EXPUNGEISSUED] Search completed + + CORRUPTION + The server discovered that some relevant data (e.g., the + mailbox) are corrupt. This response code does not include any + information about what's corrupt, but the server can write that + to its logfiles. + + C: i select "/archive/projects/experiment-iv" + S: i NO [CORRUPTION] Cannot open mailbox + + + + + + + +Gulbrandsen Standards Track [Page 4] + +RFC 5530 IMAP Response Codes May 2009 + + + SERVERBUG + The server encountered a bug in itself or violated one of its + own invariants. + + C: j select "/archive/projects/experiment-iv" + S: j NO [SERVERBUG] This should not happen + + CLIENTBUG + The server has detected a client bug. This can accompany all + of OK, NO, and BAD, depending on what the client bug is. + + C: k1 select "/archive/projects/experiment-iv" + [...] + S: k1 OK [READ-ONLY] Done + C: k2 status "/archive/projects/experiment-iv" (messages) + [...] + S: k2 OK [CLIENTBUG] Done + + CANNOT + The operation violates some invariant of the server and can + never succeed. + + C: l create "///////" + S: l NO [CANNOT] Adjacent slashes are not supported + + LIMIT + The operation ran up against an implementation limit of some + kind, such as the number of flags on a single message or the + number of flags used in a mailbox. + + C: m STORE 42 FLAGS f1 f2 f3 f4 f5 ... f250 + S: m NO [LIMIT] At most 32 flags in one mailbox supported + + OVERQUOTA + The user would be over quota after the operation. (The user + may or may not be over quota already.) + + Note that if the server sends OVERQUOTA but doesn't support the + IMAP QUOTA extension defined by [RFC2087], then there is a + quota, but the client cannot find out what the quota is. + + C: n1 uid copy 1:* oldmail + S: n1 NO [OVERQUOTA] Sorry + + C: n2 uid copy 1:* oldmail + S: n2 OK [OVERQUOTA] You are now over your soft quota + + + + + +Gulbrandsen Standards Track [Page 5] + +RFC 5530 IMAP Response Codes May 2009 + + + ALREADYEXISTS + The operation attempts to create something that already exists, + such as when the CREATE or RENAME directories attempt to create + a mailbox and there is already one of that name. + + C: o RENAME this that + S: o NO [ALREADYEXISTS] Mailbox "that" already exists + + NONEXISTENT + The operation attempts to delete something that does not exist. + Similar to ALREADYEXISTS. + + C: p RENAME this that + S: p NO [NONEXISTENT] No such mailbox + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. [RFC3501] defines + the non-terminal "resp-text-code". + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lowercase characters to define + token strings is for editorial clarity only. + + resp-text-code =/ "UNAVAILABLE" / "AUTHENTICATIONFAILED" / + "AUTHORIZATIONFAILED" / "EXPIRED" / + "PRIVACYREQUIRED" / "CONTACTADMIN" / "NOPERM" / + "INUSE" / "EXPUNGEISSUED" / "CORRUPTION" / + "SERVERBUG" / "CLIENTBUG" / "CANNOT" / + "LIMIT" / "OVERQUOTA" / "ALREADYEXISTS" / + "NONEXISTENT" + +5. Security Considerations + + Revealing information about a passphrase to unauthenticated IMAP + clients causes bad karma. + + Response codes are easier to parse than human-readable text. This + can amplify the consequences of an information leak. For example, + selecting a mailbox can fail because the mailbox doesn't exist, + because the user doesn't have the "l" right (right to know the + mailbox exists) or "r" right (right to read the mailbox). If the + server sent different responses in the first two cases in the past, + only malevolent clients would discover it. With response codes it's + possible, perhaps probable, that benevolent clients will forward the + + + + + +Gulbrandsen Standards Track [Page 6] + +RFC 5530 IMAP Response Codes May 2009 + + + leaked information to the user. Server authors are encouraged to be + particularly careful with the NOPERM and authentication-related + responses. + +6. IANA Considerations + + The IANA has created the IMAP Response Codes registry. The registry + has been populated with the following codes: + + NEWNAME RFC 2060 (obsolete) + REFERRAL RFC 2221 + ALERT RFC 3501 + BADCHARSET RFC 3501 + PARSE RFC 3501 + PERMANENTFLAGS RFC 3501 + READ-ONLY RFC 3501 + READ-WRITE RFC 3501 + TRYCREATE RFC 3501 + UIDNEXT RFC 3501 + UIDVALIDITY RFC 3501 + UNSEEN RFC 3501 + UNKNOWN-CTE RFC 3516 + UIDNOTSTICKY RFC 4315 + APPENDUID RFC 4315 + COPYUID RFC 4315 + URLMECH RFC 4467 + TOOBIG RFC 4469 + BADURL RFC 4469 + HIGHESTMODSEQ RFC 4551 + NOMODSEQ RFC 4551 + MODIFIED RFC 4551 + COMPRESSIONACTIVE RFC 4978 + CLOSED RFC 5162 + NOTSAVED RFC 5182 + BADCOMPARATOR RFC 5255 + ANNOTATE RFC 5257 + ANNOTATIONS RFC 5257 + TEMPFAIL RFC 5259 + MAXCONVERTMESSAGES RFC 5259 + MAXCONVERTPARTS RFC 5259 + NOUPDATE RFC 5267 + METADATA RFC 5464 + NOTIFICATIONOVERFLOW RFC 5465 + BADEVENT RFC 5465 + UNDEFINED-FILTER RFC 5466 + UNAVAILABLE RFC 5530 + AUTHENTICATIONFAILED RFC 5530 + AUTHORIZATIONFAILED RFC 5530 + + + +Gulbrandsen Standards Track [Page 7] + +RFC 5530 IMAP Response Codes May 2009 + + + EXPIRED RFC 5530 + PRIVACYREQUIRED RFC 5530 + CONTACTADMIN RFC 5530 + NOPERM RFC 5530 + INUSE RFC 5530 + EXPUNGEISSUED RFC 5530 + CORRUPTION RFC 5530 + SERVERBUG RFC 5530 + CLIENTBUG RFC 5530 + CANNOT RFC 5530 + LIMIT RFC 5530 + OVERQUOTA RFC 5530 + ALREADYEXISTS RFC 5530 + NONEXISTENT RFC 5530 + + The new registry can be extended by sending a registration request to + IANA. IANA will forward this request to a Designated Expert, + appointed by the responsible IESG Area Director, CCing it to the IMAP + Extensions mailing list at (or a successor + designated by the Area Director). After either allowing 30 days for + community input on the IMAP Extensions mailing list or a successful + IETF Last Call, the expert will determine the appropriateness of the + registration request and either approve or disapprove the request by + sending a notice of the decision to the requestor, CCing the IMAP + Extensions mailing list and IANA. A denial notice must be justified + by an explanation, and, in cases where it is possible, concrete + suggestions on how the request can be modified so as to become + acceptable should be provided. + + For each response code, the registry contains a list of relevant RFCs + that describe (or extend) the response code and an optional response + code status description, such as "obsolete" or "reserved to prevent + collision with deployed software". (Note that in the latter case, + the RFC number can be missing.) Presence of the response code status + description means that the corresponding response code is NOT + RECOMMENDED for widespread use. + + The intention is that any future allocation will be accompanied by a + published RFC (including direct submissions to the RFC Editor). But + in order to allow for the allocation of values prior to the RFC being + approved for publication, the Designated Expert can approve + allocations once it seems clear that an RFC will be published, for + example, before requesting IETF LC for the document. + + The Designated Expert can also approve registrations for response + codes used in deployed software when no RFC exists. Such + registrations must be marked as "reserved to prevent collision with + deployed software". + + + +Gulbrandsen Standards Track [Page 8] + +RFC 5530 IMAP Response Codes May 2009 + + + Response code registrations may not be deleted; response codes that + are no longer believed appropriate for use (for example, if there is + a problem with the syntax of said response code or if the + specification describing it was moved to Historic) should be marked + "obsolete" in the registry, clearly marking the lists published by + IANA. + +7. Acknowledgements + + Peter Coates, Mark Crispin, Philip Guenther, Alexey Melnikov, Ken + Murchison, Chris Newman, Timo Sirainen, Philip Van Hoof, Dale + Wiggins, and Sarah Wilkin helped with this document. + +8. References + +8.1. Normative References + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + +9. Informative References + + [RFC2087] Myers, J., "IMAP4 QUOTA extension", RFC 2087, January + 1997. + + [RFC2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC + 2180, July 1997. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + +Author's Address + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + Fax: +49 89 4502 9758 + EMail: arnt@oryx.com + + + + + + +Gulbrandsen Standards Track [Page 9] + diff --git a/docs/rfcs/rfc5738.IMAP_UTF8.txt b/docs/rfcs/rfc5738.IMAP_UTF8.txt new file mode 100644 index 0000000..2b5daaa --- /dev/null +++ b/docs/rfcs/rfc5738.IMAP_UTF8.txt @@ -0,0 +1,843 @@ + + + + + + +Network Working Group P. Resnick +Request for Comments: 5738 Qualcomm Incorporated +Updates: 3501 C. Newman +Category: Experimental Sun Microsystems + March 2010 + + + IMAP Support for UTF-8 + +Abstract + + This specification extends the Internet Message Access Protocol + version 4rev1 (IMAP4rev1) to support UTF-8 encoded international + characters in user names, mail addresses, and message headers. + +Status of This Memo + + This document is not an Internet Standards Track specification; it is + published for examination, experimental implementation, and + evaluation. + + This document defines an Experimental Protocol for the Internet + community. This document is a product of the Internet Engineering + Task Force (IETF). It represents the consensus of the IETF + community. It has received public review and has been approved for + publication by the Internet Engineering Steering Group (IESG). Not + all documents approved by the IESG are a candidate for any level of + Internet Standard; see Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc5738. + +Copyright Notice + + Copyright (c) 2010 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + +Resnick & Newman Experimental [Page 1] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + This document may contain material from IETF Documents or IETF + Contributions published or made publicly available before November + 10, 2008. The person(s) controlling the copyright in some of this + material may not have granted the IETF Trust the right to allow + modifications of such material outside the IETF Standards Process. + Without obtaining an adequate license from the person(s) controlling + the copyright in such materials, this document may not be modified + outside the IETF Standards Process, and derivative works of it may + not be created outside the IETF Standards Process, except to format + it for publication as an RFC or to translate it into languages other + than English. + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . 3 + 3. UTF8=ACCEPT IMAP Capability . . . . . . . . . . . . . . . . . 3 + 3.1. IMAP UTF-8 Quoted Strings . . . . . . . . . . . . . . . . 3 + 3.2. UTF8 Parameter to SELECT and EXAMINE . . . . . . . . . . . 5 + 3.3. UTF-8 LIST and LSUB Responses . . . . . . . . . . . . . . 5 + 3.4. UTF-8 Interaction with IMAP4 LIST Command Extensions . . . 6 + 3.4.1. UTF8 and UTF8ONLY LIST Selection Options . . . . . . . 6 + 3.4.2. UTF8 LIST Return Option . . . . . . . . . . . . . . . 6 + 4. UTF8=APPEND Capability . . . . . . . . . . . . . . . . . . . . 7 + 5. UTF8=USER Capability . . . . . . . . . . . . . . . . . . . . . 7 + 6. UTF8=ALL Capability . . . . . . . . . . . . . . . . . . . . . 7 + 7. UTF8=ONLY Capability . . . . . . . . . . . . . . . . . . . . . 8 + 8. Up-Conversion Server Requirements . . . . . . . . . . . . . . 8 + 9. Issues with UTF-8 Header Mailstore . . . . . . . . . . . . . . 9 + 10. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 9 + 11. Security Considerations . . . . . . . . . . . . . . . . . . . 11 + 12. References . . . . . . . . . . . . . . . . . . . . . . . . . . 11 + 12.1. Normative References . . . . . . . . . . . . . . . . . . . 11 + 12.2. Informative References . . . . . . . . . . . . . . . . . . 13 + Appendix A. Design Rationale . . . . . . . . . . . . . . . . . . 14 + Appendix B. Examples Demonstrating Relationships between + UTF8= Capabilities . . . . . . . . . . . . . . . . . 15 + Appendix C. Acknowledgments . . . . . . . . . . . . . . . . . . . 15 + + + + + + + + + + + + + +Resnick & Newman Experimental [Page 2] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +1. Introduction + + This specification extends IMAP4rev1 [RFC3501] to permit UTF-8 + [RFC3629] in headers as described in "Internationalized Email + Headers" [RFC5335]. It also adds a mechanism to support mailbox + names, login names, and passwords using the UTF-8 charset. This + specification creates five new IMAP capabilities to allow servers to + advertise these new extensions, along with two new IMAP LIST + selection options and a new IMAP LIST return option. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in "Key words for + use in RFCs to Indicate Requirement Levels" [RFC2119]. + + The formal syntax uses the Augmented Backus-Naur Form (ABNF) + [RFC5234] notation including the core rules defined in Appendix B of + [RFC5234]. In addition, rules from IMAP4rev1 [RFC3501], UTF-8 + [RFC3629], "Collected Extensions to IMAP4 ABNF" [RFC4466], and IMAP4 + LIST Command Extensions [RFC5258] are also referenced. + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. If a single "C:" or "S:" label applies to + multiple lines, then the line breaks between those lines are for + editorial clarity only and are not part of the actual protocol + exchange. + +3. UTF8=ACCEPT IMAP Capability + + The "UTF8=ACCEPT" capability indicates that the server supports UTF-8 + quoted strings, the "UTF8" parameter to SELECT and EXAMINE, and UTF-8 + responses from the LIST and LSUB commands. + + A client MUST use the "ENABLE UTF8=ACCEPT" command (defined in + [RFC5161]) to indicate to the server that the client accepts UTF-8 + quoted-strings. The "ENABLE UTF8=ACCEPT" command MUST only be used + in the authenticated state. (Note that the "UTF8=ONLY" capability + described in Section 7 and the "UTF8=ALL" capability described in + Section 6 imply the "UTF8=ACCEPT" capability. See additional + information in these sections.) + +3.1. IMAP UTF-8 Quoted Strings + + The IMAP4rev1 [RFC3501] base specification forbids the use of 8-bit + characters in atoms or quoted strings. Thus, a UTF-8 string can only + be sent as a literal. This can be inconvenient from a coding + standpoint, and unless the server offers IMAP4 non-synchronizing + + + +Resnick & Newman Experimental [Page 3] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + literals [RFC2088], this requires an extra round trip for each UTF-8 + string sent by the client. When the IMAP server advertises the + "UTF8=ACCEPT" capability, it informs the client that it supports + native UTF-8 quoted-strings with the following syntax: + + string =/ utf8-quoted + + utf8-quoted = "*" DQUOTE *UQUOTED-CHAR DQUOTE + + UQUOTED-CHAR = QUOTED-CHAR / UTF8-2 / UTF8-3 / UTF8-4 + ; UTF8-2, UTF8-3, and UTF8-4 are as defined in RFC 3629 + + When this quoting mechanism is used by the client (specifically an + octet sequence beginning with *" and ending with "), then the server + MUST reject octet sequences with the high bit set that fail to comply + with the formal syntax in [RFC3629] with a BAD response. + + The IMAP server MUST NOT send utf8-quoted syntax to the client unless + the client has indicated support for that syntax by using the "ENABLE + UTF8=ACCEPT" command. + + If the server advertises the "UTF8=ACCEPT" capability, the client MAY + use utf8-quoted syntax with any IMAP argument that permits a string + (including astring and nstring). However, if characters outside the + US-ASCII repertoire are used in an inappropriate place, the results + would be the same as if other syntactically valid but semantically + invalid characters were used. For example, if the client includes + UTF-8 characters in the user or password arguments (and the server + has not advertised "UTF8=USER"), the LOGIN command will fail as it + would with any other invalid user name or password. Specific cases + where UTF-8 characters are permitted or not permitted are described + in the following paragraphs. + + All IMAP servers that advertise the "UTF8=ACCEPT" capability SHOULD + accept UTF-8 in mailbox names, and those that also support the + "Mailbox International Naming Convention" described in RFC 3501, + Section 5.1.3 MUST accept utf8-quoted mailbox names and convert them + to the appropriate internal format. Mailbox names MUST comply with + the Net-Unicode Definition (Section 2 of [RFC5198]) with the specific + exception that they MUST NOT contain control characters (0000-001F, + 0080-009F), delete (007F), line separator (2028), or paragraph + separator (2029). + + An IMAP client MUST NOT issue a SEARCH command that uses a mixture of + utf8-quoted syntax and a SEARCH CHARSET other than UTF-8. If an IMAP + server receives such a SEARCH command, it SHOULD reject the command + with a BAD response (due to the conflicting charset labels). + + + + +Resnick & Newman Experimental [Page 4] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +3.2. UTF8 Parameter to SELECT and EXAMINE + + The "UTF8=ACCEPT" capability also indicates that the server supports + the "UTF8" parameter to SELECT and EXAMINE. When a mailbox is + selected with the "UTF8" parameter, it alters the behavior of all + IMAP commands related to message sizes, message headers, and MIME + body headers so they refer to the message with UTF-8 headers. If the + mailstore is not UTF-8 header native and the SELECT or EXAMINE + command with UTF-8 header modifier succeeds, then the server MUST + return results as if the mailstore were UTF-8 header native with + upconversion requirements as described in Section 8. The server MAY + reject the SELECT or EXAMINE command with the [NOT-UTF-8] response + code, unless the "UTF8=ALL" or "UTF8=ONLY" capability is advertised. + + Servers MAY include mailboxes that can only be selected or examined + if the "UTF8" parameter is provided. However, such mailboxes MUST + NOT be included in the output of an unextended LIST, LSUB, or + equivalent command. If a client attempts to SELECT or EXAMINE such + mailboxes without the "UTF8" parameter, the server MUST reject the + command with a [UTF-8-ONLY] response code. As a result, such + mailboxes will not be accessible by IMAP clients written prior to + this specification and are discouraged unless the server advertises + "UTF8=ONLY" or the server implements IMAP4 LIST Command Extensions + [RFC5258]. + + utf8-select-param = "UTF8" + ;; Conforms to from RFC 4466 + + C: a SELECT newmailbox (UTF8) + S: ... + S: a OK SELECT completed + C: b FETCH 1 (SIZE ENVELOPE BODY) + S: ... < UTF-8 header native results > + S: b OK FETCH completed + + C: c EXAMINE legacymailbox (UTF8) + S: c NO [NOT-UTF-8] Mailbox does not support UTF-8 access + + C: d SELECT funky-new-mailbox + S: d NO [UTF-8-ONLY] Mailbox requires UTF-8 client + +3.3. UTF-8 LIST and LSUB Responses + + After an IMAP client successfully issues an "ENABLE UTF8=ACCEPT" + command, the server MUST NOT return in LIST results any mailbox names + to the client following the IMAP4 Mailbox International Naming + Convention. Instead, the server MUST return any mailbox names with + characters outside the US-ASCII repertoire using utf8-quoted syntax. + + + +Resnick & Newman Experimental [Page 5] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + (The IMAP4 Mailbox International Naming Convention has proved + problematic in the past, so the desire is to make this syntax + obsolete as quickly as possible.) + +3.4. UTF-8 Interaction with IMAP4 LIST Command Extensions + + When an IMAP server advertises both the "UTF8=ACCEPT" capability and + the "LIST-EXTENDED" [RFC5258] capability, the server MUST support the + LIST extensions described in this section. + +3.4.1. UTF8 and UTF8ONLY LIST Selection Options + + The "UTF8" LIST selection option tells the server to include + mailboxes that only support UTF-8 headers in the output of the list + command. The "UTF8ONLY" LIST selection option tells the server to + include all mailboxes that support UTF-8 headers and to exclude + mailboxes that don't support UTF-8 headers. Note that "UTF8ONLY" + implies "UTF8", so it is not necessary for the client to request + both. Use of either selection option will also result in UTF-8 + mailbox names in the result as described in Section 3.3 and implies + the "UTF8" List return option described in Section 3.4.2. + +3.4.2. UTF8 LIST Return Option + + If the client supplies the "UTF8" LIST return option, then the server + MUST include either the "\NoUTF8" or the "\UTF8Only" mailbox + attribute as appropriate. The "\NoUTF8" mailbox attribute indicates + that an attempt to SELECT or EXAMINE that mailbox with the "UTF8" + parameter will fail with a [NOT-UTF-8] response code. The + "\UTF8Only" mailbox attribute indicates that an attempt to SELECT or + EXAMINE that mailbox without the "UTF8" parameter will fail with a + [UTF-8-ONLY] response code. Note that computing this information may + be expensive on some server implementations, so this return option + should not be used unless necessary. + + The ABNF [RFC5234] for these LIST extensions follows: + + list-select-independent-opt =/ "UTF8" + + list-select-base-opt =/ "UTF8ONLY" + + mbx-list-oflag =/ "\NoUTF8" / "\UTF8Only" + + return-option =/ "UTF8" + + resp-text-code =/ "NOT-UTF-8" / "UTF-8-ONLY" + + + + + +Resnick & Newman Experimental [Page 6] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +4. UTF8=APPEND Capability + + If the "UTF8=APPEND" capability is advertised, then the server + accepts UTF-8 headers in the APPEND command message argument. A + client that sends a message with UTF-8 headers to the server MUST + send them using the "UTF8" APPEND data extension. If the server also + advertises the CATENATE capability (as specified in [RFC4469]), the + client can use the same data extension to include such a message in a + CATENATE message part. The ABNF for the APPEND data extension and + CATENATE extension follows: + + utf8-literal = "UTF8" SP "(" literal8 ")" + + append-data =/ utf8-literal + + cat-part =/ utf8-literal + + A server that advertises "UTF8=APPEND" has to comply with the + requirements of the IMAP base specification and [RFC5322] for message + fetching. Mechanisms for 7-bit downgrading to help comply with the + standards are discussed in Downgrading mechanism for + Internationalized eMail Address (IMA) [RFC5504]. + + IMAP servers that do not advertise the "UTF8=APPEND" or "UTF8=ONLY" + capability SHOULD reject an APPEND command that includes any 8-bit in + the message headers with a "NO" response. + + Note that the "UTF8=ONLY" capability described in Section 7 implies + the "UTF8=APPEND" capability. See additional information in that + section. + +5. UTF8=USER Capability + + If the "UTF8=USER" capability is advertised, that indicates the + server accepts UTF-8 user names and passwords and applies SASLprep + [RFC4013] to both arguments of the LOGIN command. The server MUST + reject UTF-8 that fails to comply with the formal syntax in RFC 3629 + [RFC3629] or if it encounters Unicode characters listed in Section + 2.3 of SASLprep RFC 4013 [RFC4013]. + +6. UTF8=ALL Capability + + The "UTF8=ALL" capability indicates all server mailboxes support + UTF-8 headers. Specifically, SELECT and EXAMINE with the "UTF8" + parameter will never fail with a [NOT-UTF-8] response code. + + + + + + +Resnick & Newman Experimental [Page 7] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + Note that the "UTF8=ONLY" capability described in Section 7 implies + the "UTF8=ALL" capability. See additional information in that + section. + + Note that the "UTF8=ALL" capability implies the "UTF8=ACCEPT" + capability. + +7. UTF8=ONLY Capability + + The "UTF8=ONLY" capability permits an IMAP server to advertise that + it does not support the international mailbox name convention + (modified UTF-7), and does not permit selection or examination of any + mailbox unless the "UTF8" parameter is provided. As this is an + incompatible change to IMAP, a clear warning is necessary. IMAP + clients that find implementation of the "UTF8=ONLY" capability + problematic are encouraged to at least detect the "UTF8=ONLY" + capability and provide an informative error message to the end-user. + + When an IMAP mailbox internally uses UTF-8 header native storage, the + down-conversion step is necessary to permit selection or examination + of the mailbox in a backwards compatible fashion will become more + difficult to support. Although it is hoped that deployed IMAP + servers will not advertise "UTF8=ONLY" for some years, this + capability is intended to minimize the disruption when legacy support + finally goes away. + + The "UTF8=ONLY" capability implies the "UTF8=ACCEPT" capability, the + "UTF8=ALL" capability, and the "UTF8=APPEND" capability. A server + that advertises "UTF8=ONLY" need not advertise the three implicit + capabilities. + +8. Up-Conversion Server Requirements + + When an IMAP4 server uses a traditional mailbox format that includes + 7-bit headers and it chooses to permit access to that mailbox with + the "UTF8" parameter, it MUST support minimal up-conversion as + described in this section. + + The server MUST support up-conversion of the following address + header-fields in the message header: From, Sender, To, CC, Bcc, + Resent-From, Resent-Sender, Resent-To, Resent-CC, Resent-Bcc, and + Reply-To. This up-conversion MUST include address local-parts in + fields downgraded according to [RFC5504], address domains encoded + according to Internationalizing Domain Names in Applications (IDNA) + [RFC3490], and MIME header encoding [RFC2047] of display-names and + any [RFC5322] comments. + + + + + +Resnick & Newman Experimental [Page 8] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + The following charsets MUST be supported for up-conversion of MIME + header encoding [RFC2047]: UTF-8, US-ASCII, ISO-8859-1, ISO-8859-2, + ISO-8859-3, ISO-8859-4, ISO-8859-5, ISO-8859-6, ISO-8859-7, + ISO-8859-8, ISO-8859-9, ISO-8859-10, ISO-8859-14, and ISO-8859-15. + If the server supports other charsets in IMAP SEARCH or IMAP CONVERT + [RFC5259], it SHOULD also support those charsets in this conversion. + + Up-conversion of MIME header encoding of the following headers MUST + also be implemented: Subject, Date ([RFC5322] comments only), + Comments, Keywords, and Content-Description. + + Server implementations also SHOULD up-convert all MIME body headers + [RFC2045], SHOULD up-convert or remove the deprecated (and misused) + "name" parameter [RFC1341] on Content-Type, and MUST up-convert the + Content-Disposition [RFC2183] "filename" parameter, except when any + of these are contained within a multipart/signed MIME body part (see + below). These parameters can be encoded using the standard MIME + parameter encoding [RFC2231] mechanism, or via non-standard use of + MIME header encoding [RFC2047] in quoted strings. + + The IMAP server MUST NOT perform up-conversion of headers and content + of multipart/signed, as well as Original-Recipient and Return-Path. + +9. Issues with UTF-8 Header Mailstore + + When an IMAP server uses a mailbox format that supports UTF-8 headers + and it permits selection or examination of that mailbox without the + "UTF8" parameter, it is the responsibility of the server to comply + with the IMAP4rev1 base specification [RFC3501] and [RFC5322] with + respect to all header information transmitted over the wire. + Mechanisms for 7-bit downgrading to help comply with the standards + are discussed in "Downgrading Mechanism for Email Address + Internationalization" [RFC5504]. + + An IMAP server with a mailbox that supports UTF-8 headers MUST comply + with the protocol requirements implicit from Section 8. However, the + code necessary for such compliance need not be part of the IMAP + server itself in this case. For example, the minimal required up- + conversion could be performed when a message is inserted into the + IMAP-accessible mailbox. + +10. IANA Considerations + + This adds five new capabilities ("UTF8=ACCEPT", "UTF8=USER", + "UTF8=APPEND", "UTF8=ALL", and "UTF8=ONLY") to the IMAP4rev1 + Capabilities registry [RFC3501]. + + + + + +Resnick & Newman Experimental [Page 9] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + This adds two new IMAP4 list selection options and one new IMAP4 list + return option. + + 1. LIST-EXTENDED option name: UTF8 + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): UTF8 + + LIST-EXTENDED option description: Causes the LIST response to + include mailboxes that mandate the UTF8 SELECT/EXAMINE parameter. + + Published specification: RFC 5738, Section 3.4.1 + + Security considerations: RFC 5738, Section 11 + + Intended usage: COMMON + + Person and email address to contact for further information: see + the Authors' Addresses at the end of this specification + + Owner/Change controller: iesg@ietf.org + + 2. LIST-EXTENDED option name: UTF8ONLY + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): UTF8 + + LIST-EXTENDED option description: Causes the LIST response to + include mailboxes that mandate the UTF8 SELECT/EXAMINE parameter + and exclude mailboxes that do not support the UTF8 SELECT/EXAMINE + parameter. + + Published specification: RFC 5738, Section 3.4.1 + + Security considerations: RFC 5738, Section 11 + + Intended usage: COMMON + + Person and email address to contact for further information: see + the Authors' Addresses at the end of this specification + + Owner/Change controller: iesg@ietf.org + + + + + + + +Resnick & Newman Experimental [Page 10] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + 3. LIST-EXTENDED option name: UTF8 + + LIST-EXTENDED option type: RETURN + + Implied return options(s): none + + LIST-EXTENDED option description: Causes the LIST response to + include \NoUTF8 and \UTF8Only mailbox attributes. + + Published specification: RFC 5738, Section 3.4.1 + + Security considerations: RFC 5738, Section 11 + + Intended usage: COMMON + + Person and email address to contact for further information: see + the Authors' Addresses at the end of this specification + + Owner/Change controller: iesg@ietf.org + +11. Security Considerations + + The security considerations of UTF-8 [RFC3629] and SASLprep [RFC4013] + apply to this specification, particularly with respect to use of + UTF-8 in user names and passwords. Otherwise, this is not believed + to alter the security considerations of IMAP4rev1. + +12. References + +12.1. Normative References + + [RFC1341] Borenstein, N. and N. Freed, "MIME (Multipurpose Internet + Mail Extensions): Mechanisms for Specifying and Describing + the Format of Internet Message Bodies", RFC 1341, + June 1992. + + [RFC2045] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII Text", + RFC 2047, November 1996. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + + + + +Resnick & Newman Experimental [Page 11] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + [RFC2183] Troost, R., Dorner, S., and K. Moore, "Communicating + Presentation Information in Internet Messages: The + Content-Disposition Header Field", RFC 2183, August 1997. + + [RFC2231] Freed, N. and K. Moore, "MIME Parameter Value and Encoded + Word Extensions: + Character Sets, Languages, and Continuations", RFC 2231, + November 1997. + + [RFC3490] Faltstrom, P., Hoffman, P., and A. Costello, + "Internationalizing Domain Names in Applications (IDNA)", + RFC 3490, March 2003. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC4013] Zeilenga, K., "SASLprep: Stringprep Profile for User Names + and Passwords", RFC 4013, February 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC4469] Resnick, P., "Internet Message Access Protocol (IMAP) + CATENATE Extension", RFC 4469, April 2006. + + [RFC5161] Gulbrandsen, A. and A. Melnikov, "The IMAP ENABLE + Extension", RFC 5161, March 2008. + + [RFC5198] Klensin, J. and M. Padlipsky, "Unicode Format for Network + Interchange", RFC 5198, March 2008. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access + Protocol version 4 - LIST Command Extensions", RFC 5258, + June 2008. + + [RFC5259] Melnikov, A. and P. Coates, "Internet Message Access + Protocol - CONVERT Extension", RFC 5259, July 2008. + + [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, + October 2008. + + + + + +Resnick & Newman Experimental [Page 12] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + [RFC5335] Abel, Y., "Internationalized Email Headers", RFC 5335, + September 2008. + + [RFC5504] Fujiwara, K. and Y. Yoneya, "Downgrading Mechanism for + Email Address Internationalization", RFC 5504, March 2009. + +12.2. Informative References + + [RFC2049] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part Five: Conformance Criteria and + Examples", RFC 2049, November 1996. + + [RFC2088] Myers, J., "IMAP4 non-synchronizing literals", RFC 2088, + January 1997. + + [RFC2277] Alvestrand, H., "IETF Policy on Character Sets and + Languages", BCP 18, RFC 2277, January 1998. + + [RFC5721] Gellens, R. and C. Newman, "POP3 Support for UTF-8", + RFC 5721, February 2010. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick & Newman Experimental [Page 13] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +Appendix A. Design Rationale + + This non-normative section discusses the reasons behind some of the + design choices in the above specification. + + The basic approach of advertising the ability to access a mailbox in + UTF-8 mode is intended to permit graceful upgrade, including servers + that support multiple mailbox formats. In particular, it would be + undesirable to force conversion of an entire server mailstore to + UTF-8 headers, so being able to phase-in support for new mailboxes + and gradually migrate old mailboxes is permitted by this design. + + "UTF8=USER" is optional because many identity systems are US-ASCII + only, so it's helpful to inform the client up front that UTF-8 won't + work. + + "UTF8=APPEND" is optional because it effectively requires IMAP server + support for down-conversion, which is a much more complex operation + than up-conversion. + + The "UTF8=ONLY" mechanism simplifies diagnosis of interoperability + problems when legacy support goes away. In the situation where + backwards compatibility is broken anyway, just-send-UTF-8 IMAP has + the advantage that it might work with some legacy clients. However, + the difficulty of diagnosing interoperability problems caused by a + just-send-UTF-8 IMAP mechanism is the reason the "UTF8=ONLY" + capability mechanism was chosen. + + The up-conversion requirements are designed to balance the desire to + deprecate and eventually eliminate complicated encodings (like MIME + header encodings) without creating a significant deployment burden + for servers. As IMAP4 servers already require a MIME parser, this + includes additional server up-conversion requirements not present in + POP3 Support for UTF-8 [RFC5721]. + + The set of mandatory charsets comes from two sources: MIME + requirements [RFC2049] and IETF Policy on Character Sets [RFC2277]. + Including a requirement to up-convert widely deployed encoded + ideographic charsets to UTF-8 would be reasonable for most scenarios, + but may require unacceptable table sizes for some embedded devices. + The open-ended recommendation to support widely deployed charsets + avoids the political ramifications of attempting to list such + charsets. The authors believe market forces, existing open-source + software, and public conversion tables are sufficient to deploy the + appropriate charsets. + + + + + + +Resnick & Newman Experimental [Page 14] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +Appendix B. Examples Demonstrating Relationships between UTF8= + Capabilities + + + UTF8=ACCEPT UTF8=USER UTF8=APPEND + UTF8=ACCEPT UTF8=ALL + UTF8=ALL ; Note, same as above + UTF8=ACCEPT UTF8=USER UTF8=APPEND UTF8=ALL UTF8=ONLY + UTF8=USER UTF8=ONLY ; Note, same as above + +Appendix C. Acknowledgments + + The authors wish to thank the participants of the EAI working group + for their contributions to this document with particular thanks to + Harald Alvestrand, David Black, Randall Gellens, Arnt Gulbrandsen, + Kari Hurtta, John Klensin, Xiaodong Lee, Charles Lindsey, Alexey + Melnikov, Subramanian Moonesamy, Shawn Steele, Daniel Taharlev, and + Joseph Yee for their specific contributions to the discussion. + +Authors' Addresses + + Pete Resnick + Qualcomm Incorporated + 5775 Morehouse Drive + San Diego, CA 92121-1714 + US + + Phone: +1 858 651 4478 + EMail: presnick@qualcomm.com + URI: http://www.qualcomm.com/~presnick/ + + Chris Newman + Sun Microsystems + 800 Royal Oaks + Monrovia, CA 91016 + US + + EMail: chris.newman@sun.com + + + + + + + + + + + + + +Resnick & Newman Experimental [Page 15] + diff --git a/docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt b/docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt new file mode 100644 index 0000000..fe0de70 --- /dev/null +++ b/docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt @@ -0,0 +1,619 @@ + + + + + + +Internet Engineering Task Force (IETF) A. Melnikov +Request for Comments: 5788 D. Cridland +Category: Standards Track Isode Limited +ISSN: 2070-1721 March 2010 + + + IMAP4 Keyword Registry + +Abstract + + The aim of this document is to establish a new IANA registry for IMAP + keywords and to define a procedure for keyword registration, in order + to improve interoperability between different IMAP clients. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc5788. + +Copyright Notice + + Copyright (c) 2010 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + + + + + +Melnikov & Cridland Standards Track [Page 1] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + +Table of Contents + + 1. Introduction ....................................................2 + 2. Conventions Used in This Document ...............................2 + 3. IANA Considerations .............................................3 + 3.1. Review Guidelines for the Designated Expert Reviewer .......4 + 3.2. Comments on IMAP Keywords' Registrations ...................5 + 3.3. Change Control .............................................5 + 3.4. Initial Registrations ......................................6 + 3.4.1. $MDNSent IMAP Keyword Registration ..................6 + 3.4.2. $Forwarded IMAP Keyword Registration ................7 + 3.4.3. $SubmitPending IMAP Keyword Registration ............8 + 3.4.4. $Submitted IMAP Keyword Registration ................9 + 4. Security Considerations ........................................10 + 5. Acknowledgements ...............................................10 + 6. References .....................................................10 + 6.1. Normative References ......................................10 + 6.2. Informative References ....................................11 + +1. Introduction + + IMAP keywords [RFC3501] are boolean named flags that can be used by + clients to annotate messages in an IMAP mailbox. Although IMAP + keywords are an optional feature of IMAP, the majority of IMAP + servers can store arbitrary keywords. Many mainstream IMAP clients + use a limited set of specific keywords, and some can manage (create, + edit, display) arbitrary IMAP keywords. + + Over the years, some IMAP keywords have become de-facto standards, + with some specific semantics associated with them. In some cases, + different client implementors decided to define and use keywords with + different names, but the same semantics. Some server implementors + decided to map such keywords automatically in order to improve cross- + client interoperability. + + In other cases, the same keywords have been used with different + semantics, thus causing interoperability problems. + + This document attempts to prevent further incompatible uses of IMAP + keywords by establishing an "IMAP Keywords" registry and allocating a + special prefix for standardized keywords. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [Kwds]. + + + + +Melnikov & Cridland Standards Track [Page 2] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + +3. IANA Considerations + + IANA has established a new registry for IMAP keywords. + + Registration of an IMAP keyword is requested by filling in the + following template and following the instructions on the IANA pages + for the registry to submit it to IANA: + + Subject: Registration of IMAP keyword X + + IMAP keyword name: + + Purpose (description): + + Private or Shared on a server: (One of PRIVATE, SHARED, or BOTH. + PRIVATE means that each different + user has a distinct copy of the + keyword. SHARED means that all + different users see the same value of + the keyword. BOTH means that an IMAP + server can have the keyword as either + private or shared.) + + Is it an advisory keyword or may it cause an automatic action: + + When/by whom the keyword is set/cleared: + + Related keywords: (for example, "mutually exclusive with keywords Y + and Z") + + Related IMAP capabilities: + + Security considerations: + + Published specification (recommended): + + Person & email address to contact for further information: + + Intended usage: (One of COMMON, LIMITED USE, or DEPRECATED (i.e., + not recommended for use)) + + Owner/Change controller: (MUST be "IESG" for any "common use" + keyword registration specified in an IETF + Review document. See definition of "common + use" below in this section. When the + Owner/Change controller is not a + Standardization Organization, the + registration request MUST make it clear if + + + +Melnikov & Cridland Standards Track [Page 3] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + the registration is controlled by a + company, or the individual performing the + registration.) + + Note: (Any other information that the author deems interesting + may be added here, for example, if the keyword(s) is + supported by existing clients.) + + Registration of an IMAP keyword requires Expert Review [RFC5226]. + Registration of any IMAP keyword is initiated by posting a + registration request to the Message Organization WG mailing list + (or its replacement as chosen by the responsible + Application Area Director) and CCing IANA (). After + allowing for at least two weeks for community input on the designated + mailing list, the expert will determine the appropriateness of the + registration request and either approve or disapprove the request + with notice to the requestor, the mailing list, and IANA. Any + refusal must come with a clear explanation. + + The IESG appoints one or more Expert Reviewers for the IMAP keyword + registry established by this document. + + The Expert Reviewer should strive for timely reviews. The Expert + Reviewer should take no longer than six weeks to make and announce + the decision, or notify the mailing list that more time is required. + + Decisions (or lack of) made by an Expert Reviewer can be first + appealed to Application Area Directors and, if the appellant is not + satisfied with the response, to the full IESG. + + There are two types of IMAP keywords in the "IMAP Keywords" registry: + intended for "common use" and vendor-/organization-specific use (also + known as "limited use"). An IMAP keyword is said to be for "common + use" if it is reasonably expected to be implemented in at least two + independent client implementations. The two types of IMAP keywords + have different levels of requirements for registration (see below). + +3.1. Review Guidelines for the Designated Expert Reviewer + + Expert Reviewers should focus on the following requirements. + + Registration of a vendor-/organization-specific ("limited use") IMAP + keyword is easier. The Expert Reviewer only needs to check that the + requested name doesn't conflict with an already registered name, and + that the name is not too generic, misleading, etc. The Expert + Reviewer MAY request the IMAP keyword name change before approving + + + + + +Melnikov & Cridland Standards Track [Page 4] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + the registration. The Expert Reviewer SHOULD refuse a registration + if there is an already registered IMAP keyword that serves the same + purpose, but has a different name. + + When registering an IMAP keyword for "common use", the Expert + Reviewer performs the checks described for vendor-/ + organization-specific IMAP keywords, plus additional checks as + detailed below. + + Keywords intended for "common use" SHOULD start with the "$" prefix. + (Note that this is a SHOULD because some of the commonly used IMAP + keywords in widespread use don't follow this convention.) + + IMAP keywords intended for "common use" SHOULD be standardized in + IETF Review [RFC5226] documents. (Note that IETF Review documents + still require Expert Review.) + + Values in the "IMAP Keywords" IANA registry intended for "common use" + must be clearly documented and likely to ensure interoperability. + They must be useful, not harmful to the Internet. In cases when an + IMAP keyword being registered is already deployed, Expert Reviewers + should favor registering it over requiring perfect documentation + and/or requesting a change to the name of the IMAP keyword. + + The Expert Reviewer MAY automatically "upgrade" registration requests + for a "limited use" IMAP keyword to "common use" level. The Expert + Reviewer MAY also request that a registration targeted for "common + use" be registered as "limited use" instead. + +3.2. Comments on IMAP Keywords' Registrations + + Comments on registered IMAP keywords should be sent to both the + "owner" of the mechanism and to the mailing list designated to IMAP + keyword review (see Section 3). This improves the chances of getting + a timely response. + + Submitters of comments may, after a reasonable attempt to contact the + owner and after soliciting comments on the IMAP mailing list, request + the designated Expert Reviewer to attach their comment to the IMAP + keyword registration itself. The procedure is similar to requesting + an Expert Review for the affected keyword. + +3.3. Change Control + + Once an IMAP keyword registration has been published by IANA, the + owner may request a change to its definition. The change request + (including a change to the "intended usage" field) follows the same + procedure as the initial registration request, with the exception of + + + +Melnikov & Cridland Standards Track [Page 5] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + changes to the "Person & email address to contact for further + information" and "Owner/Change controller" fields. The latter can be + changed by the owner by informing IANA; this can be done without + discussion or review. + + The IESG may reassign responsibility for an IMAP keyword. The most + common case of this will be to enable clarifications to be made to + keywords where the owner of the registration has died, moved out of + contact, or is otherwise unable to make changes that are important to + the community. + + IMAP keyword registrations MUST NOT be deleted; keywords that are no + longer believed appropriate for use can be declared DEPRECATED by a + change to their "intended usage" field. + + The IESG is considered the owner of all "common use" IMAP keywords + that are published in an IETF Review document. + +3.4. Initial Registrations + + IANA has registered the IMAP keywords specified in following + subsections in the registry established by this document. + +3.4.1. $MDNSent IMAP Keyword Registration + + Subject: Registration of IMAP keyword $MDNSent + + + IMAP keyword name: $MDNSent + + Purpose (description): Specifies that a Message Disposition + Notification (MDN) must not be sent for any + message annotated with the $MDNSent IMAP + keyword. + + Private or Shared on a server: SHARED + + Is it an advisory keyword or may it cause an automatic action: + This keyword can cause automatic action by + the client. See [RFC3503] for more details. + + When/by whom the keyword is set/cleared: + This keyword is set by an IMAP client when it + decides to act on an MDN request, or when + uploading a sent or draft message. It can + also be set by a delivery agent. Once set, + the flag SHOULD NOT be cleared. + + + + +Melnikov & Cridland Standards Track [Page 6] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + Related keywords: None + + Related IMAP capabilities: None + + Security considerations: See Section 6 of [RFC3503] + + Published specification (recommended): [RFC3503] + + Person & email address to contact for further information: + Alexey Melnikov + + Intended usage: COMMON + + Owner/Change controller: IESG + + Note: + +3.4.2. $Forwarded IMAP Keyword Registration + + Subject: Registration of the IMAP keyword $Forwarded + + IMAP keyword name: $Forwarded + + Purpose (description): $Forwarded is used by several IMAP clients to + specify that the message was resent to + another email address, embedded within or + attached to a new message. This keyword is + set by the mail client when it successfully + forwards the message to another email + address. Typical usage of this keyword is to + show a different (or additional) icon for a + message that has been forwarded. + + Private or Shared on a server: BOTH + + Is it an advisory keyword or may it cause an automatic action: + advisory + + When/by whom the keyword is set/cleared: + This keyword can be set by either a delivery + agent or a mail client. Once set, the flag + SHOULD NOT be cleared. Notes: There is no + way to tell if a message with $Forwarded + keyword set was forwarded more than once. + + Related keywords: None + + Related IMAP capabilities: None + + + +Melnikov & Cridland Standards Track [Page 7] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + Security considerations: A server implementing this keyword as a + shared keyword may disclose that a + confidential message was forwarded. + + Published specification (recommended): [RFC5550] + + Person & email address to contact for further information: + Alexey Melnikov + + Intended usage: COMMON + + Owner/Change controller: IESG + + Note: + +3.4.3. $SubmitPending IMAP Keyword Registration + + Subject: Registration of IMAP keyword $SubmitPending + + IMAP keyword name: $SubmitPending + + Purpose (description): The $SubmitPending IMAP keyword designates + the message as awaiting to be submitted. + This keyword allows storing messages waiting + to be submitted in the same mailbox where + messages that were already submitted and/or + are being edited are stored. + + A message that has both $Submitted and + $SubmitPending IMAP keywords set is a message + being actively submitted. + + Private or Shared on a server: SHARED + + Is it an advisory keyword or may it cause an automatic action: + This keyword can cause automatic action by + the client. See Section 5.10 of [RFC5550] + for more details. + + When/by whom the keyword is set/cleared: + This keyword is set by a mail client when it + decides that the message needs to be sent + out. + + Related keywords: $Submitted + + Related IMAP capabilities: None + + + + +Melnikov & Cridland Standards Track [Page 8] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + Security considerations: A server implementing this keyword as a + shared keyword may disclose that a + confidential message is scheduled to be + sent out or is being actively sent out. + + Published specification (recommended): [RFC5550] + + Person & email address to contact for further information: + Alexey Melnikov + + Intended usage: COMMON + + Owner/Change controller: IESG + + Note: + +3.4.4. $Submitted IMAP Keyword Registration + + Subject: Registration of IMAP keyword $Submitted + + IMAP keyword name: $Submitted + + Purpose (description): The $Submitted IMAP keyword designates the + message as being sent out. + + A message that has both $Submitted and + $SubmitPending IMAP keywords set is a message + being actively submitted. + + Private or Shared on a server: SHARED + + Is it an advisory keyword or may it cause an automatic action: + This keyword can cause automatic action by + the client. See Section 5.10 of [RFC5550] + for more details. + + When/by whom the keyword is set/cleared: + This keyword is set by a mail client when it + decides to start sending it. + + Related keywords: $SubmitPending + + Related IMAP capabilities: None + + Security considerations: A server implementing this keyword as a + shared keyword may disclose that a + confidential message was sent or is being + actively sent out. + + + +Melnikov & Cridland Standards Track [Page 9] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + Published specification (recommended): [RFC5550] + + Person & email address to contact for further information: + Alexey Melnikov + + Intended usage: COMMON + + Owner/Change controller: IESG + + Note: + +4. Security Considerations + + IMAP keywords are one of the base IMAP features [RFC3501]. This + document doesn't change their behavior, so it does not add new + security issues. + + A particular IMAP keyword might have specific security + considerations, which are documented in the IMAP keyword + registration template standardized by this document. + +5. Acknowledgements + + The creation of this document was prompted by one of many discussions + on the IMAP mailing list. + + John Neystadt co-authored the first version of this document. + + Special thanks to Chris Newman, David Harris, Lyndon Nerenberg, Mark + Crispin, Samuel Weiler, Alfred Hoenes, Lars Eggert, and Cullen + Jennings for reviewing different versions of this document. However, + all errors or omissions must be attributed to the authors of this + document. + + The authors would also like to thank the developers of Mozilla mail + clients for providing food for thought. + +6. References + +6.1. Normative References + + [Kwds] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + + + + +Melnikov & Cridland Standards Track [Page 10] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + [RFC5226] Narten, T. and H. Alvestrand, "Guidelines for Writing an + IANA Considerations Section in RFCs", BCP 26, RFC 5226, + May 2008. + +6.2. Informative References + + [RFC3503] Melnikov, A., "Message Disposition Notification (MDN) + profile for Internet Message Access Protocol (IMAP)", + RFC 3503, March 2003. + + [RFC5550] Cridland, D., Melnikov, A., and S. Maes, "The Internet + Email to Support Diverse Service Environments (Lemonade) + Profile", RFC 5550, August 2009. + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + Dave Cridland + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: dave.cridland@isode.com + + + + + + + + + + + + + + + + +Melnikov & Cridland Standards Track [Page 11] + diff --git a/docs/rfcs/rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt b/docs/rfcs/rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt new file mode 100644 index 0000000..e45b855 --- /dev/null +++ b/docs/rfcs/rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt @@ -0,0 +1,339 @@ + + + + + + +Internet Engineering Task Force (IETF) A. Melnikov +Request for Comments: 5819 Isode Limited +Category: Standards Track T. Sirainen +ISSN: 2070-1721 Unaffiliated + March 2010 + + + IMAP4 Extension for Returning STATUS Information in Extended LIST + +Abstract + + Many IMAP clients display information about total number of + messages / total number of unseen messages in IMAP mailboxes. In + order to do that, they are forced to issue a LIST or LSUB command and + to list all available mailboxes, followed by a STATUS command for + each mailbox found. This document provides an extension to LIST + command that allows the client to request STATUS information for + mailboxes together with other information typically returned by the + LIST command. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc5819. + +Copyright Notice + + Copyright (c) 2010 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + +Melnikov & Sirainen Standards Track [Page 1] + +RFC 5819 TITLE* March 2010 + + +Table of Contents + + 1. Introduction ....................................................2 + 1.1. Conventions Used in This Document ..........................2 + 2. STATUS Return Option to LIST Command ............................2 + 3. Examples ........................................................3 + 4. Formal Syntax ...................................................4 + 5. Security Considerations .........................................4 + 6. IANA Considerations .............................................4 + 7. Acknowledgements ................................................5 + 8. Normative References ............................................5 + +1. Introduction + + Many IMAP clients display information about the total number of + messages / total number of unseen messages in IMAP mailboxes. In + order to do that, they are forced to issue a LIST or LSUB command and + to list all available mailboxes, followed by a STATUS command for + each mailbox found. This document provides an extension to LIST + command that allows the client to request STATUS information for + mailboxes together with other information typically returned by the + LIST command. + +1.1. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [Kwds]. + +2. STATUS Return Option to LIST Command + + [RFC3501] explicitly disallows mailbox patterns in the STATUS + command. The main reason was to discourage frequent use of the + STATUS command by clients, as it might be quite expensive for an IMAP + server to perform. However, this prohibition had resulted in an + opposite effect: a new generation of IMAP clients appeared, that + issues a STATUS command for each mailbox returned by the LIST + command. This behavior is suboptimal to say at least. It wastes + extra bandwidth and, in the case of a client that doesn't support + IMAP pipelining, also degrades performance by using too many round + trips. This document tries to remedy the situation by specifying a + single command that can be used by the client to request all the + necessary information. In order to achieve this goal, this document + is extending the LIST command with a new return option, STATUS. This + option takes STATUS data items as parameters. For each selectable + + + +Melnikov & Sirainen Standards Track [Page 2] + +RFC 5819 TITLE* March 2010 + + + mailbox matching the list pattern and selection options, the server + MUST return an untagged LIST response followed by an untagged STATUS + response containing the information requested in the STATUS return + option. + + If an attempted STATUS for a listed mailbox fails because the mailbox + can't be selected (e.g., if the "l" ACL right [ACL] is granted to the + mailbox and the "r" right is not granted, or due to a race condition + between LIST and STATUS changing the mailbox to \NoSelect), the + STATUS response MUST NOT be returned and the LIST response MUST + include the \NoSelect attribute. This means the server may have to + buffer the LIST reply until it has successfully looked up the + necessary STATUS information. + + If the server runs into unexpected problems while trying to look up + the STATUS information, it MAY drop the corresponding STATUS reply. + In such a situation, the LIST command would still return a tagged OK + reply. + +3. Examples + + C: A01 LIST "" % RETURN (STATUS (MESSAGES UNSEEN)) + S: * LIST () "." "INBOX" + S: * STATUS "INBOX" (MESSAGES 17 UNSEEN 16) + S: * LIST () "." "foo" + S: * STATUS "foo" (MESSAGES 30 UNSEEN 29) + S: * LIST (\NoSelect) "." "bar" + S: A01 OK List completed. + + The "bar" mailbox isn't selectable, so it has no STATUS reply. + + C: A02 LIST (SUBSCRIBED RECURSIVEMATCH)"" % RETURN (STATUS + (MESSAGES)) + S: * LIST (\Subscribed) "." "INBOX" + S: * STATUS "INBOX" (MESSAGES 17) + S: * LIST () "." "foo" (CHILDINFO ("SUBSCRIBED")) + S: A02 OK List completed. + + The LIST reply for "foo" is returned because it has matching + children, but no STATUS reply is returned because "foo" itself + doesn't match the selection criteria. + + + + + + + + + + +Melnikov & Sirainen Standards Track [Page 3] + +RFC 5819 TITLE* March 2010 + + +4. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. Terms not defined here are taken + from [RFC3501] and [LISTEXT]. + + return-option =/ status-option + + status-option = "STATUS" SP "(" status-att *(SP status-att) ")" + ;; This ABNF production complies with + ;; syntax. + +5. Security Considerations + + This extension makes it a bit easier for clients to overload the + server by requesting STATUS information for a large number of + mailboxes. However, as already noted in the introduction, existing + clients already try to do that by generating a large number of STATUS + commands for each mailbox in which they are interested. While + performing STATUS information retrieval for big lists of mailboxes, a + server implementation needs to make sure that it can still serve + other IMAP connections and yield execution to other connections, when + necessary. + +6. IANA Considerations + + IMAP4 capabilities are registered by publishing a Standards Track or + IESG-approved Experimental RFC. The "IMAP 4 Capabilities" registry + is available from the IANA webiste: + + http://www.iana.org + + This document defines the LIST-STATUS IMAP capability. IANA has + added it to the registry. + + IANA has also added the following new LIST-EXTENDED option to the + IANA registry established by [LISTEXT]: + + To: iana@iana.org + Subject: Registration of LIST-EXTENDED option STATUS + + LIST-EXTENDED option name: STATUS + + LIST-EXTENDED option type: RETURN + + LIST-EXTENDED option description: Causes the LIST command to return + STATUS responses in addition to LIST responses. + + + + +Melnikov & Sirainen Standards Track [Page 4] + +RFC 5819 TITLE* March 2010 + + + Published specification: RFC 5819 + + Security considerations: RFC 5819 + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + +7. Acknowledgements + + Thanks to Philip Van Hoof who pointed out that STATUS and LIST + commands should be combined in order to optimize traffic and number + of round trips. + +8. Normative References + + [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [Kwds] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [LISTEXT] Leiba, B. and A. Melnikov, "Internet Message Access + Protocol version 4 - LIST Command Extensions", RFC 5258, + June 2008. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + + + + + + + + + + + + + + + +Melnikov & Sirainen Standards Track [Page 5] + +RFC 5819 TITLE* March 2010 + + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + Timo Sirainen + + EMail: tss@iki.fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Sirainen Standards Track [Page 6] + diff --git a/docs/rfcs/rfc5957.IMAP4_SORT_extension.txt b/docs/rfcs/rfc5957.IMAP4_SORT_extension.txt new file mode 100644 index 0000000..32adb49 --- /dev/null +++ b/docs/rfcs/rfc5957.IMAP4_SORT_extension.txt @@ -0,0 +1,283 @@ + + + + + + +Internet Engineering Task Force (IETF) D. Karp +Request for Comments: 5957 Zimbra +Updates: 5256 July 2010 +Category: Standards Track +ISSN: 2070-1721 + + + Display-Based Address Sorting for the IMAP4 SORT Extension + +Abstract + + This document describes an IMAP protocol extension enabling server- + side message sorting on the commonly displayed portion of the From + and To header fields. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc5957. + +Copyright Notice + + Copyright (c) 2010 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + + + + +Karp IMAP4 Display-Based Address Sorting [Page 1] + +RFC 5957 July 2010 + + +Table of Contents + + 1. Introduction ....................................................2 + 2. Conventions Used in This Document ...............................2 + 3. DISPLAY Sort Value for an Address ...............................2 + 4. The DISPLAYFROM and DISPLAYTO Sort Criteria .....................3 + 5. Formal Syntax ...................................................3 + 6. Security Considerations .........................................3 + 7. Internationalization Considerations .............................4 + 8. IANA Considerations .............................................4 + 9. Normative References ............................................4 + +1. Introduction + + The [SORT] extension to the [IMAP] protocol provides a means for the + server-based sorting of messages. It defines a set of sort criteria + and the mechanism for determining the sort value of a message for + each such ordering. + + The [SORT] FROM and TO orderings sort messages lexically on the + [IMAP] addr-mailbox of the first address in the message's From and To + headers, respectively. This document provides two alternative + orderings, DISPLAYFROM and DISPLAYTO, which sort messages based on + the first From or To address's [IMAP] addr-name (generally the same + as its [RFC5322] display-name), when present. + + A server that supports the full [SORT] extension as well as both the + DISPLAYFROM and DISPLAYTO sort criteria indicates this by returning + "SORT=DISPLAY" in its CAPABILITY response. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + +3. DISPLAY Sort Value for an Address + + For the purposes of the sort criteria defined in this document, the + sort value for an [IMAP] address structure is defined as follows: + + o If the address structure's [IMAP] addr-name is non-NIL, apply the + procedure from [RFC5255], Section 4.6. (That is, decode any + [RFC2047] encoded-words and convert the resulting character string + into a charset valid for the currently active [RFC4790] collation, + with a default of UTF-8.) If the resulting octet string is not + the empty string, use it as the sort value for the address. + + + + +Karp IMAP4 Display-Based Address Sorting [Page 2] + +RFC 5957 July 2010 + + + o Otherwise, if the address structure's [IMAP] addr-mailbox and + [IMAP] addr-host are both non-NIL, the sort value for the address + is addr-mailbox@addr-host. + + o Otherwise, if the address structure's [IMAP] addr-mailbox is non- + NIL, the sort value for the address is its addr-mailbox. + + o If none of the above conditions are met, the sort value for the + address is the empty string. + +4. The DISPLAYFROM and DISPLAYTO Sort Criteria + + This document introduces two new [SORT] sort criteria, DISPLAYFROM + and DISPLAYTO. A message's sort value under these orderings MUST be + derived as follows: + + A "derived-addr" value is created from the [IMAP] envelope structure + resulting from a FETCH ENVELOPE on the message. For DISPLAYFROM, the + derived-addr value is the [IMAP] env-from value. For DISPLAYTO, the + derived-addr value is the [IMAP] env-to value. + + o If the derived-addr value is NIL, the message's sort value is the + empty string. + + o Otherwise, the message's sort value is the DISPLAY sort value of + the first [IMAP] address in the derived-addr value. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. [IMAP] defines the + non-terminal "capability", and [SORT] defines "sort-key". + + capability =/ "SORT=DISPLAY" + + sort-key =/ "DISPLAYFROM" / "DISPLAYTO" + +6. Security Considerations + + This document defines an additional IMAP4 capability. As such, it + does not change the underlying security considerations of [IMAP]. + The author believes that no new security issues are introduced with + this additional IMAP4 capability. + + + + + + + + +Karp IMAP4 Display-Based Address Sorting [Page 3] + +RFC 5957 July 2010 + + +7. Internationalization Considerations + + DISPLAYFROM and DISPLAYTO are string-based sort criteria. As stated + in [SORT], the active [RFC4790] collation as per [RFC5255] MUST be + used when sorting such strings. + + The DISPLAYFROM and DISPLAYTO orderings sort on the full decoded + [IMAP] addr-name, when present. They do not attempt to parse this + string in a locale- or language-dependent manner in order to + determine and sort on some semantically meaningful substring such as + the surname. + +8. IANA Considerations + + [IMAP] capabilities are registered by publishing a Standards Track or + IESG-approved Experimental RFC. This document constitutes + registration of the SORT=DISPLAY capability in the [IMAP] + capabilities registry. + +9. Normative References + + [IMAP] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII Text", + RFC 2047, November 1996. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC4790] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet + Application Protocol Collation Registry", RFC 4790, March + 2007. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [RFC5255] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet + Message Access Protocol Internationalization", RFC 5255, + June 2008. + + [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, + October 2008. + + + + + + +Karp IMAP4 Display-Based Address Sorting [Page 4] + +RFC 5957 July 2010 + + + [SORT] Crispin, M. and K. Murchison, "Internet Message Access + Protocol - SORT and THREAD Extensions", RFC 5256, June + 2008. + +Author's Address + + Dan Karp + Zimbra + 3401 Hillview Avenue + Palo Alto, CA 94304 + USA + + EMail: dkarp@zimbra.com + URI: http://www.zimbra.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Karp IMAP4 Display-Based Address Sorting [Page 5] + diff --git a/docs/rfcs/rfc6154.IMAP_LIST_Special-use_Mailboxes.txt b/docs/rfcs/rfc6154.IMAP_LIST_Special-use_Mailboxes.txt new file mode 100644 index 0000000..14d3e76 --- /dev/null +++ b/docs/rfcs/rfc6154.IMAP_LIST_Special-use_Mailboxes.txt @@ -0,0 +1,675 @@ + + + + + + +Internet Engineering Task Force (IETF) B. Leiba +Request for Comments: 6154 Huawei Technologies +Category: Standards Track J. Nicolson +ISSN: 2070-1721 Google + March 2011 + + + IMAP LIST Extension for Special-Use Mailboxes + +Abstract + + Some IMAP message stores include special-use mailboxes, such as those + used to hold draft messages or sent messages. Many mail clients + allow users to specify where draft or sent messages should be put, + but configuring them requires that the user know which mailboxes the + server has set aside for these purposes. This extension adds new + optional mailbox attributes that a server may include in IMAP LIST + command responses to identify special-use mailboxes to the client, + easing configuration. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc6154. + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + +Leiba & Nicolson Standards Track [Page 1] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 1.1. Conventions Used in This Document . . . . . . . . . . . . 3 + 2. New Mailbox Attributes Identifying Special-Use Mailboxes . . . 3 + 3. Extension to IMAP CREATE Command to Set Special-Use + Attributes . . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 4. IMAP METADATA Entry for Special-Use Attributes . . . . . . . . 6 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 7 + 5.1. Example of an IMAP LIST Command . . . . . . . . . . . . . 7 + 5.2. Example of an Extended IMAP LIST Command . . . . . . . . . 7 + 5.3. Example of an IMAP CREATE Command . . . . . . . . . . . . 8 + 5.4. Example of Using IMAP METADATA to Manipulate + Special-Use Attributes . . . . . . . . . . . . . . . . . . 8 + 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 9 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 9 + 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 10 + 8.1. Registration of USEATTR IMAP Response Code . . . . . . . . 10 + 8.2. Registration of CREATE-SPECIAL-USE IMAP Capability . . . . 10 + 8.3. Registration of SPECIAL-USE IMAP Capability . . . . . . . 10 + 8.4. Registration of SPECIAL-USE Selection Option . . . . . . . 10 + 8.5. Registration of SPECIAL-USE Return Option . . . . . . . . 11 + 8.6. Registration of SPECIAL-USE Metadata . . . . . . . . . . . 11 + 9. References . . . . . . . . . . . . . . . . . . . . . . . . . . 12 + 9.1. Normative References . . . . . . . . . . . . . . . . . . . 12 + 9.2. Informative References . . . . . . . . . . . . . . . . . . 12 + + + + + + + + + + + + + + + + + + + + + + + + + +Leiba & Nicolson Standards Track [Page 2] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +1. Introduction + + Some IMAP message stores include special-use mailboxes, such as those + used to hold draft messages or sent messages. Many mail clients + allow users to specify where draft or sent messages should be put, + but configuring them requires that the user know which mailboxes the + server has set aside for these purposes. This extension adds new + optional mailbox attributes that a server may include in IMAP LIST + command responses to identify special-use mailboxes to the client, + easing configuration. + + In addition, this extension adds an optional parameter on the IMAP + CREATE command, allowing a client to assign a special use to a + mailbox when it is created. Servers may choose to support this part + of the extension, but are not required to. + +1.1. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + +2. New Mailbox Attributes Identifying Special-Use Mailboxes + + An IMAP server that supports this extension MAY include any or all of + the following attributes in responses to the non-extended IMAP LIST + command. The new attributes are included along with existing + attributes, such as "\Marked" and "\Noselect". A given mailbox may + have none, one, or more than one of these attributes. In some cases, + a special use is advice to a client about what to put in that + mailbox. In other cases, it's advice to a client about what to + expect to find there. There is no capability string related to the + support of special-use attributes on the non-extended LIST command. + + For the extended list command [RFC5258], this extension adds a new + capability string, a new selection option, and a new return option, + all called "SPECIAL-USE". Supporting implementations MUST include + the "SPECIAL-USE" capability string in response to an IMAP CAPABILITY + command. If the client specifies the "SPECIAL-USE" selection option, + the LIST command MUST return only those mailboxes that have a + special-use attribute set. If the client specifies the "SPECIAL-USE" + return option, the LIST command MUST return the new special-use + attributes on those mailboxes that have them set. The "SPECIAL-USE" + + + + + +Leiba & Nicolson Standards Track [Page 3] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + return option is implied by the "SPECIAL-USE" selection option. The + extended LIST command MAY return SPECIAL-USE attributes even if the + client does not specify the return option. + + The new attributes defined here are as follows: + + \All + This mailbox presents all messages in the user's message store. + Implementations MAY omit some messages, such as, perhaps, those + in \Trash and \Junk. When this special use is supported, it is + almost certain to represent a virtual mailbox. + + \Archive + This mailbox is used to archive messages. The meaning of an + "archival" mailbox is server-dependent; typically, it will be + used to get messages out of the inbox, or otherwise keep them + out of the user's way, while still making them accessible. + + \Drafts + This mailbox is used to hold draft messages -- typically, + messages that are being composed but have not yet been sent. In + some server implementations, this might be a virtual mailbox, + containing messages from other mailboxes that are marked with + the "\Draft" message flag. Alternatively, this might just be + advice that a client put drafts here. + + \Flagged + This mailbox presents all messages marked in some way as + "important". When this special use is supported, it is likely + to represent a virtual mailbox collecting messages (from other + mailboxes) that are marked with the "\Flagged" message flag. + + \Junk + This mailbox is where messages deemed to be junk mail are held. + Some server implementations might put messages here + automatically. Alternatively, this might just be advice to a + client-side spam filter. + + \Sent + This mailbox is used to hold copies of messages that have been + sent. Some server implementations might put messages here + automatically. Alternatively, this might just be advice that a + client save sent messages here. + + \Trash + This mailbox is used to hold messages that have been deleted or + marked for deletion. In some server implementations, this might + be a virtual mailbox, containing messages from other mailboxes + + + +Leiba & Nicolson Standards Track [Page 4] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + that are marked with the "\Deleted" message flag. + Alternatively, this might just be advice that a client that + chooses not to use the IMAP "\Deleted" model should use this as + its trash location. In server implementations that strictly + expect the IMAP "\Deleted" model, this special use is likely not + to be supported. + + All of the above attributes are OPTIONAL, and any given server or + message store may support any combination of the attributes, or none + at all. In most cases, there will likely be at most one mailbox with + a given attribute for a given user, but in some server or message + store implementations it might be possible for multiple mailboxes to + have the same special-use attribute. + + Special-use attributes are likely to be user-specific. User Adam + might share his \Sent mailbox with user Barb, but that mailbox is + unlikely to also serve as Barb's \Sent mailbox. It's certainly + possible for Adam and Barb to each set the \Sent use on the same + mailbox, but that would be done by specific action (see the sections + below). + +3. Extension to IMAP CREATE Command to Set Special-Use Attributes + + As an OPTIONAL feature, a server MAY allow clients to designate a + mailbox, at creation, as having one or more special uses. This + extension defines the "USE" parameter to the IMAP CREATE command for + that purpose (using the syntax defined in RFC 4466 section 2.2 + [RFC4466]). The new OPTIONAL "USE" parameter is followed by a + parenthesized list of zero or more special-use attributes, as defined + above. + + In some server implementations, some special uses may imply automatic + action by the server. For example, creation of a "\Junk" mailbox + might cause the server to start placing messages that have been + evaluated as spam into the mailbox. + + In some server implementations, some special uses may result in a + mailbox with unusual characteristics or side effects. For example, + creation of an "\All" mailbox might cause the server to create a + virtual mailbox, rather than a standard one, and that mailbox might + behave in unexpected ways (COPY into it might fail, for example). + + Servers MAY allow the creation of a special-use mailbox even if one + so designated already exists. This might have the effect of moving + the special use from the old mailbox to the new one, or might create + multiple mailboxes with the same special use. Alternatively, servers + MAY refuse the creation, considering the designation to be a + conflict. + + + +Leiba & Nicolson Standards Track [Page 5] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + If the server cannot create a mailbox with the designated special use + defined, for whatever reason, it MUST NOT create the mailbox, and + MUST respond to the CREATE command with a tagged NO response. If the + reason for the failure is related to the special-use attribute (the + specified special use is not supported or cannot be assigned to the + specified mailbox), the server SHOULD include the new "USEATTR" + response code in the tagged response (see Section 5.3 for an + example). + + An IMAP server that supports this OPTIONAL feature will advertise the + "CREATE-SPECIAL-USE" capability string. Clients MUST NOT use the + "USE" parameter unless the server advertises the capability. Note + that this capability string is different from the "SPECIAL-USE" + string defined above, and a server that supports both functions MUST + advertise both capability strings. + +4. IMAP METADATA Entry for Special-Use Attributes + + If a server supports this extension and the METADATA extension + [RFC5464], it SHOULD tie the special-use attributes for a mailbox to + its metadata entry "/private/specialuse". The value of /private/ + specialuse is either NIL (if there are no special-use attributes for + that mailbox) or a space-separated list of special-use attributes, + presented the same way they would be presented in the LIST command + response. + + Such a server MAY allow the setting of special-use attributes through + the METADATA mechanisms, thereby allowing clients to change the + special uses of existing mailboxes. These changes might have side + effects, as the server automatically adjusts the special uses + accordingly, just as it might do with CREATE USE, above. See + Section 5.4 for an example. + + A server that supports this MUST check the validity of changes to the + special-use attributes that are done through the metadata in the same + way that it checks validity for the CREATE command and for any + internal mechanisms for setting special uses on mailboxes. It MUST + NOT just blindly accept setting of these metadata by clients, which + might result in the setting of special uses that the implementation + does not support, multiple mailboxes with the same special use, or + other situations that the implementation considers invalid. + + + + + + + + + + +Leiba & Nicolson Standards Track [Page 6] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +5. Examples + +5.1. Example of an IMAP LIST Command + + This example shows an IMAP LIST response from a server that supports + this extension. Note that not all of the attributes are used. This + server also supports the Child Mailbox extension [RFC3348]. + + C: t1 LIST "" "%" + S: * LIST (\Marked \HasNoChildren) "/" Inbox + S: * LIST (\HasNoChildren) "/" ToDo + S: * LIST (\HasChildren) "/" Projects + S: * LIST (\Sent \HasNoChildren) "/" SentMail + S: * LIST (\Marked \Drafts \HasNoChildren) "/" MyDrafts + S: * LIST (\Trash \HasNoChildren) "/" Trash + S: t1 OK done + +5.2. Example of an Extended IMAP LIST Command + + This example shows an IMAP LIST response from a server that supports + this extension. The client uses the extended IMAP LIST command. + + C: t1 CAPABILITY + S: * CAPABILITY IMAP4rev1 SPECIAL-USE + S: t1 OK done + + C: t2 LIST "" "%" RETURN (SPECIAL-USE) + S: * LIST (\Marked) "/" Inbox + S: * LIST () "/" ToDo + S: * LIST () "/" Projects + S: * LIST (\Sent) "/" SentMail + S: * LIST (\Marked \Drafts) "/" MyDrafts + S: * LIST (\Trash) "/" Trash + S: t2 OK done + + Here, the client also includes the "SPECIAL-USE" selection option for + the same list. The "SPECIAL-USE" return option could also have been + specified, but it is unnecessary, as it is implied by the selection + option. Note that in this case, mailboxes that do not have a + special-use attribute are not listed. Also note that we've used the + wildcard "*", rather than "%", to make sure we see all special-use + mailboxes, even ones that might not be at the namespace's root. + + C: t3 LIST (SPECIAL-USE) "" "*" + S: * LIST (\Sent) "/" SentMail + S: * LIST (\Marked \Drafts) "/" MyDrafts + S: * LIST (\Trash) "/" Trash + S: t3 OK done + + + +Leiba & Nicolson Standards Track [Page 7] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +5.3. Example of an IMAP CREATE Command + + This example shows an IMAP CREATE command that might be used to + create a mailbox designated to hold draft and sent messages. It also + attempts to create a mailbox that will contain all the user's + messages, but the server does not support that special use for this + user's message store. + + C: t1 CAPABILITY + S: * CAPABILITY IMAP4rev1 CREATE-SPECIAL-USE + S: t1 OK done + + C: t2 CREATE MySpecial (USE (\Drafts \Sent)) + S: t2 OK MySpecial created + + C: t3 CREATE Everything (USE (\All)) + S: t3 NO [USEATTR] \All not supported + +5.4. Example of Using IMAP METADATA to Manipulate Special-Use + Attributes + + This example shows how IMAP METADATA can be used to manipulate + special-use attributes, if the operation is supported on the server. + + ==> Starting point: + C: t1 LIST "" "%" RETURN (SPECIAL-USE) + S: * LIST (\Sent) "/" SentMail + S: * LIST (\Drafts) "/" MyDrafts + S: * LIST () "/" SavedDrafts + S: * LIST (\Trash) "/" Trash + S: t1 OK done + + ==> Demonstrate the connection: + C: t2 GETMETADATA "MyDrafts" /private/specialuse + S: * METADATA "MyDrafts" (/private/specialuse "\\Drafts") + S: t2 OK done + + ==> Set new use for SavedDrafts; MyDrafts changes automatically: + C: t3 SETMETADATA "SavedDrafts" (/private/specialuse "\\Drafts") + S: * METADATA "MyDrafts" (/private/specialuse NIL) + S: t3 OK SETMETADATA complete + + ==> Remove special use for SentMail: + C: t4 SETMETADATA "SentMail" (/private/specialuse NIL) + S: t4 OK SETMETADATA complete + + + + + + +Leiba & Nicolson Standards Track [Page 8] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + ==> Check the results: + C: t5 LIST "" "%" RETURN (SPECIAL-USE) + S: * LIST () "/" SentMail + S: * LIST () "/" MyDrafts + S: * LIST (\Drafts) "/" SavedDrafts + S: * LIST (\Trash) "/" Trash + S: t5 OK done + +6. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [RFC5234]. + + create-param =/ "USE" SP "(" [use-attr *(SP use-attr)] ")" + ; Extends "create-param" from RFC 4466 [RFC4466] + + mbx-list-oflag =/ use-attr + ; Extends "mbx-list-oflag" from IMAP base [RFC3501] + + list-select-independent-opt =/ "SPECIAL-USE" + ; Extends "list-select-independent-opt" from + ; LIST-extended [RFC5258] + + return-option =/ "SPECIAL-USE" + ; Extends "return-option" from + ; LIST-extended [RFC5258] + + resp-text-code =/ "USEATTR" + ; Extends "resp-text-code" from + ; IMAP [RFC3501] + + use-attr = "\All" / "\Archive" / "\Drafts" / "\Flagged" / + "\Junk" / "\Sent" / "\Trash" / use-attr-ext + + use-attr-ext = "\" atom + ; Reserved for future extensions. Clients + ; MUST ignore list attributes they do not understand + ; Server implementations MUST NOT generate + ; extension attributes except as defined by + ; future Standards-Track revisions of or + ; extensions to this specification. + +7. Security Considerations + + LIST response: + Conveying special-use information to a client exposes a small bit of + extra information that could be of value to an attacker. Knowing, + for example, that a particular mailbox (\All) contains pointers to + + + +Leiba & Nicolson Standards Track [Page 9] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + every message the user has might be of particular value. If the IMAP + channel is not protected from passive eavesdropping, this could be an + issue. + + CREATE command "USE" parameter and metadata extension: In some server + implementations, some special uses may imply automatic action by the + server. For example, creation of a "\Junk" mailbox might cause the + server to start placing messages that have been evaluated as spam + into the mailbox. Implementors SHOULD consider the consequences of + allowing a user (or client program) to designate the target of such + automatic action. + + Example: If a user is allowed to give the "\Junk" attribute to a + shared mailbox, legitimate mail that's misclassified as junk (false + positives) will be put into that shared mailbox, exposing the user's + private mail to others. The server might warn a user of that + possibility, or might refuse to allow the specification to be made on + a shared mailbox. (Note that this problem exists independent of this + specification, if the server allows a user to share a mailbox that's + already in use for such a function.) + +8. IANA Considerations + +8.1. Registration of USEATTR IMAP Response Code + + This document defines a new IMAP response code, "USEATTR", which IANA + added to the IMAP Response Codes registry. + +8.2. Registration of CREATE-SPECIAL-USE IMAP Capability + + This document defines a new IMAP capability, "CREATE-SPECIAL-USE", + which IANA added to the IMAP 4 Capabilities registry. + +8.3. Registration of SPECIAL-USE IMAP Capability + + This document defines a new IMAP capability, "SPECIAL-USE", which + IANA added to the IMAP 4 Capabilities registry. + +8.4. Registration of SPECIAL-USE Selection Option + + This document defines a new IMAP4 List Extended selection option, + "SPECIAL-USE", which IANA added to the IMAP4 List Extended registry, + as follows: + + To: iana@iana.org + Subject: Registration of LIST-EXTENDED selection option SPECIAL-USE + LIST-EXTENDED option name: SPECIAL-USE + LIST-EXTENDED option type: SELECTION + + + +Leiba & Nicolson Standards Track [Page 10] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + Implied return option(s): SPECIAL-USE + LIST-EXTENDED option description: Limit the list to special-use + mailboxes only + Published specification: RFC 6154 + Security considerations: none + Intended usage: COMMON + Person and email address to contact for further information: Authors' + Addresses at the end of RFC 6154 + Owner/Change controller: iesg@ietf.org + +8.5. Registration of SPECIAL-USE Return Option + + This document defines a new IMAP4 List Extended return option, + "SPECIAL-USE", which IANA added to the IMAP4 List Extended registry, + as follows: + + To: iana@iana.org + Subject: Registration of LIST-EXTENDED return option SPECIAL-USE + LIST-EXTENDED option name: SPECIAL-USE + LIST-EXTENDED option type: RETURN + LIST-EXTENDED option description: Request special-use mailbox + information + Published specification: RFC 6154 + Security considerations: none + Intended usage: COMMON + Person and email address to contact for further information: Authors' + Addresses at the end of RFC 6154 + Owner/Change controller: iesg@ietf.org + +8.6. Registration of SPECIAL-USE Metadata + + This document defines a new IMAP METADATA entry. IANA added the + following to the IMAP METADATA Mailbox Entry registry: + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + Type: Mailbox + Name: /private/specialuse + Description: Defines any special-use features of a mailbox. See the + reference specification for details of its use. + Content-type: text/plain; charset=us-ascii + RFC Number: RFC 6154 + Contact: MORG mailing list mailto:morg@ietf.org + + + + + + + + +Leiba & Nicolson Standards Track [Page 11] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +9. References + +9.1. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access + Protocol version 4 - LIST Command Extensions", RFC 5258, + June 2008. + + [RFC5464] Daboo, C., "The IMAP METADATA Extension", RFC 5464, + February 2009. + +9.2. Informative References + + [RFC3348] Gahrns, M. and R. Cheng, "The Internet Message Action + Protocol (IMAP4) Child Mailbox Extension", RFC 3348, + July 2002. + +Authors' Addresses + + Barry Leiba + Huawei Technologies + + Phone: +1 646 827 0648 + EMail: barryleiba@computer.org + URI: http://internetmessagingtechnology.org/ + + + Jamie Nicolson + Google + + EMail: nicolson@google.com + + + + + + + + +Leiba & Nicolson Standards Track [Page 12] + diff --git a/docs/rfcs/rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt b/docs/rfcs/rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt new file mode 100644 index 0000000..e4ed001 --- /dev/null +++ b/docs/rfcs/rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt @@ -0,0 +1,395 @@ + + + + + + +Internet Engineering Task Force (IETF) T. Sirainen +Request for Comments: 6203 March 2011 +Category: Standards Track +ISSN: 2070-1721 + + + IMAP4 Extension for Fuzzy Search + +Abstract + + This document describes an IMAP protocol extension enabling a server + to perform searches with inexact matching and assigning relevancy + scores for matched messages. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc6203. + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + + + + + +Sirainen Standards Track [Page 1] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + +1. Introduction + + When humans perform searches in IMAP clients, they typically want to + see the most relevant search results first. IMAP servers are able to + do this in the most efficient way when they're free to internally + decide how searches should match messages. This document describes a + new SEARCH=FUZZY extension that provides such functionality. + +2. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + +3. The FUZZY Search Key + + The FUZZY search key takes another search key as its argument. The + server is allowed to perform all matching in an implementation- + defined manner for this search key, including ignoring the active + comparator as defined by [RFC5255]. Typically, this would be used to + search for strings. For example: + + C: A1 SEARCH FUZZY (SUBJECT "IMAP break") + S: * SEARCH 1 5 10 + S: A1 OK Search completed. + + Besides matching messages with a subject of "IMAP break", the above + search may also match messages with subjects "broken IMAP", "IMAP is + broken", or anything else the server decides that might be a good + match. + + This example does a fuzzy SUBJECT search, but a non-fuzzy FROM + search: + + C: A2 SEARCH FUZZY SUBJECT work FROM user@example.com + S: * SEARCH 1 4 + S: A2 OK Search completed. + + How the server handles multiple separate FUZZY search keys is + implementation-defined. + + Fuzzy search algorithms might change, or the results of the + algorithms might be different from search to search, so that fuzzy + searches with the same parameters might give different results for + 1) the same user at different times, 2) different users (searches + + + +Sirainen Standards Track [Page 2] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + executed simultaneously), or 3) different users (searches executed at + different times). For example, a fuzzy search might adapt to a + user's search habits in an attempt to give more relevant results (in + a "learning" manner). Such differences can also occur because of + operational decisions, such as load balancing. Clients asking for + "fuzzy" really are requesting search results in a not-necessarily- + deterministic way and need to give the user appropriate warning about + that. + +4. Relevancy Scores for Search Results + + Servers SHOULD assign a search relevancy score for each matched + message when the FUZZY search key is given. Relevancy scores are + given in the range 1-100, where 100 is the highest relevancy. The + relevancy scores SHOULD use the full 1-100 range, so that clients can + show them to users in a meaningful way, e.g., as a percentage value. + + As the name already indicates, relevancy scores specify how relevant + to the search the matched message is. It's not necessarily the same + as how precisely the message matched. For example, a message whose + subject fuzzily matches the search string might get a higher + relevancy score than a message whose body had the exact string in the + middle of a sentence. When multiple search keys are matched fuzzily, + how the relevancy score is calculated is server-dependent. + + If the server also advertises the ESEARCH capability as defined by + [ESEARCH], the relevancy scores can be retrieved using the new + RELEVANCY return option for SEARCH: + + C: B1 SEARCH RETURN (RELEVANCY ALL) FUZZY TEXT "Helo" + S: * ESEARCH (TAG "B1") ALL 1,5,10 RELEVANCY (4 99 42) + S: B1 OK Search completed. + + In the example above, the server would treat "hello", "help", and + other similar strings as fuzzily matching the misspelled "Helo". + + The RELEVANCY return option MUST NOT be used unless a FUZZY search + key is also given. Note that SEARCH results aren't sorted by + relevancy; SORT is needed for that. + +5. Fuzzy Matching with Non-String Search Keys + + Fuzzy matching is not limited to just string matching. All search + keys SHOULD be matched fuzzily, although exactly what that means for + different search keys is left for server implementations to decide -- + including deciding that fuzzy matching is meaningless for a + particular key, and falling back to exact matching. Some suggestions + are given below. + + + +Sirainen Standards Track [Page 3] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + Dates: + A typical example could be when a user wants to find a message + "from Dave about a week ago". A client could perform this search + using SEARCH FUZZY (FROM "Dave" SINCE 21-Jan-2009 BEFORE + 24-Jan-2009). The server could return messages outside the + specified date range, but the further away the message is, the + lower the relevancy score. + + Sizes: + These should be handled similarly to dates. If a user wants to + search for "about 1 MB attachments", the client could do this by + sending SEARCH FUZZY (LARGER 900000 SMALLER 1100000). Again, the + further away the message size is from the specified range, the + lower the relevancy score. + + Flags: + If other search criteria match, the server could return messages + that don't have the specified flags set, but with lower relevancy + scores. SEARCH SUBJECT "xyz" FUZZY ANSWERED, for example, might + be useful if the user thinks the message he is looking for has the + ANSWERED flag set, but he isn't sure. + + Unique Identifiers (UIDs), sequences, modification sequences: These + are examples of keys for which exact matching probably makes sense. + Alternatively, a server might choose, for instance, to expand a UID + range by 5% on each side. + +6. Extensions to SORT and SEARCH + + If the server also advertises the SORT capability as defined by + [SORT], the results can be sorted by the new RELEVANCY sort criteria: + + C: C1 SORT (RELEVANCY) UTF-8 FUZZY SUBJECT "Helo" + S: * SORT 5 10 1 + S: C1 OK Sort completed. + + The message with the highest score is returned first. As with the + RELEVANCY return option, RELEVANCY sort criteria MUST NOT be used + unless a FUZZY search key is also given. + + If the server also advertises the ESORT capability as defined by + [CONTEXT], the relevancy scores can be retrieved using the new + RELEVANCY return option for SORT: + + C: C2 SORT RETURN (RELEVANCY ALL) (RELEVANCY) UTF-8 FUZZY TEXT + "Helo" + S: * ESEARCH (TAG "C2") ALL 5,10,1 RELEVANCY (99 42 4) + S: C2 OK Sort completed. + + + +Sirainen Standards Track [Page 4] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + Furthermore, if the server advertises the CONTEXT=SORT (or + CONTEXT=SEARCH) capability, then the client can limit the number of + returned messages to a SORT (or a SEARCH) by using the PARTIAL return + option. For example, this returns the 10 most relevant messages: + + C: C3 SORT RETURN (PARTIAL 1:10) (RELEVANCY) UTF-8 FUZZY TEXT + "World" + S: * ESEARCH (TAG "C3") PARTIAL (1:10 42,9,34,13,15,4,2,7,23,82) + S: C3 OK Sort completed. + +7. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. It includes definitions from + [RFC3501], [IMAP-ABNF], and [SORT]. + + capability =/ "SEARCH=FUZZY" + + score = 1*3DIGIT + ;; (1 <= n <= 100) + + score-list = "(" [score *(SP score)] ")" + + search-key =/ "FUZZY" SP search-key + + search-return-data =/ "RELEVANCY" SP score-list + ;; Conforms to , from [IMAP-ABNF] + + search-return-opt =/ "RELEVANCY" + ;; Conforms to , from [IMAP-ABNF] + + sort-key =/ "RELEVANCY" + +8. Security Considerations + + Implementation of this extension might enable denial-of-service + attacks against server resources. Servers MAY limit the resources + that a single search (or a single user) may use. Additionally, + implementors should be aware of the following: Fuzzy search engines + are often complex with non-obvious disk space, memory, and/or CPU + usage patterns. Server implementors should at least test the fuzzy- + search behavior with large messages that contain very long words + and/or unique random strings. Also, very long search keys might + cause excessive memory or CPU usage. + + Invalid input may also be problematic. For example, if the search + engine takes a UTF-8 stream as input, it might fail more or less + badly when illegal UTF-8 sequences are fed to it from a message whose + + + +Sirainen Standards Track [Page 5] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + character set was claimed to be UTF-8. This could be avoided by + validating all the input and, for example, replacing illegal UTF-8 + sequences with the Unicode replacement character (U+FFFD). + + Search relevancy rankings might be susceptible to "poisoning" by + smart attackers using certain keywords or hidden markup (e.g., HTML) + in their messages to boost the rankings. This can't be fully + prevented by servers, so clients should prepare for it by at least + allowing users to see all the search results, rather than hiding + results below a certain score. + +9. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG-approved experimental RFC. The "Internet Message Access + Protocol (IMAP) 4 Capabilities Registry" is available from + http://www.iana.org/. + + This document defines the SEARCH=FUZZY IMAP capability. IANA has + added it to the registry. + +10. Acknowledgements + + Alexey Melnikov, Zoltan Ordogh, Barry Leiba, Cyrus Daboo, and Dave + Cridland have helped with this document. + +11. Normative References + + [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, + January 2008. + + [CONTEXT] Cridland, D. and C. King, "Contexts for IMAP4", + RFC 5267, July 2008. + + [ESEARCH] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH + Command for Controlling What Kind of Information Is + Returned", RFC 4731, November 2006. + + [IMAP-ABNF] Melnikov, A. and C. Daboo, "Collected Extensions to + IMAP4 ABNF", RFC 4466, April 2006. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + + + +Sirainen Standards Track [Page 6] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + [RFC5255] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet + Message Access Protocol Internationalization", RFC 5255, + June 2008. + + [SORT] Crispin, M. and K. Murchison, "Internet Message Access + Protocol - SORT and THREAD Extensions", RFC 5256, + June 2008. + +Author's Address + + Timo Sirainen + + EMail: tss@iki.fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Sirainen Standards Track [Page 7] + diff --git a/docs/rfcs/rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt b/docs/rfcs/rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt new file mode 100644 index 0000000..b5a3a61 --- /dev/null +++ b/docs/rfcs/rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt @@ -0,0 +1,563 @@ + + + + + + +Internet Engineering Task Force (IETF) B. Leiba +Request for Comments: 6237 Huawei Technologies +Updates: 4466 A. Melnikov +Category: Experimental Isode Limited +ISSN: 2070-1721 May 2011 + + + IMAP4 Multimailbox SEARCH Extension + +Abstract + + The IMAP4 specification allows the searching of only the selected + mailbox. A user often wants to search multiple mailboxes, and a + client that wishes to support this must issue a series of SELECT and + SEARCH commands, waiting for each to complete before moving on to the + next. This extension allows a client to search multiple mailboxes + with one command, limiting the round trips and waiting for various + searches to complete, and not requiring disruption of the currently + selected mailbox. This extension also uses MAILBOX and TAG fields in + ESEARCH responses, allowing a client to pipeline the searches if it + chooses. This document updates RFC 4466. + +Status of This Memo + + This document is not an Internet Standards Track specification; it is + published for examination, experimental implementation, and + evaluation. + + This document defines an Experimental Protocol for the Internet + community. This document is a product of the Internet Engineering + Task Force (IETF). It represents the consensus of the IETF + community. It has received public review and has been approved for + publication by the Internet Engineering Steering Group (IESG). Not + all documents approved by the IESG are a candidate for any level of + Internet Standard; see Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc6237. + + + + + + + + + + + + +Leiba & Melnikov Experimental [Page 1] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + +Table of Contents + + 1. Introduction ....................................................2 + 1.1. Conventions Used in This Document ..........................3 + 2. New ESEARCH Command .............................................3 + 2.1. The ESEARCH Response .......................................4 + 2.2. Source Options: Specifying Mailboxes to Search .............5 + 3. Examples ........................................................6 + 4. Formal Syntax ...................................................7 + 5. Security Considerations .........................................8 + 6. IANA Considerations .............................................9 + 7. Acknowledgements ................................................9 + 8. Normative References ............................................9 + +1. Introduction + + The IMAP4 specification allows the searching of only the selected + mailbox. A user often wants to search multiple mailboxes, and a + client that wishes to support this must issue a series of SELECT and + SEARCH commands, waiting for each to complete before moving on to the + next. The commands can't be pipelined, because the server might run + them in parallel, and the untagged SEARCH responses could not then be + distinguished from each other. + + This extension allows a client to search multiple mailboxes with one + command, and includes MAILBOX and TAG fields in the ESEARCH response, + yielding the following advantages: + + o A single command limits the number of round trips needed to search + a set of mailboxes. + + o A single command eliminates the need to wait for one search to + complete before starting the next. + + + +Leiba & Melnikov Experimental [Page 2] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + o A single command allows the server to optimize the search, if it + can. + + o A command that is not dependent upon the selected mailbox + eliminates the need to disrupt the selection state or to open + another IMAP connection. + + o The MAILBOX, UIDVALIDITY, and TAG fields in the responses allow a + client to distinguish which responses go with which search (and + which mailbox). A client can safely pipeline these search + commands without danger of confusion. The addition of the MAILBOX + and UIDVALIDITY fields updates the search-correlator item defined + in [RFC4466]. + +1.1. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + +2. New ESEARCH Command + + Arguments: OPTIONAL source options + OPTIONAL result options + OPTIONAL charset specification (see [RFC2978]) + searching criteria (one or more) + + Responses: REQUIRED untagged response: ESEARCH + + Result: OK -- search completed + NO -- error: cannot search that charset or criteria + BAD -- command unknown or arguments invalid + + This section defines a new ESEARCH command, which works similarly to + the UID SEARCH command described in Section 2.6.1 of [RFC4466] + (initially described in Section 6.4.4 of [RFC3501] and extended by + [RFC4731]). + + The ESEARCH command further extends searching by allowing for + optional source and result options. This document does not define + any new result options (see Section 3.1 of [RFC4731]). A server that + supports this extension includes "MULTISEARCH" in its IMAP capability + string. + + + + + +Leiba & Melnikov Experimental [Page 3] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + Because there has been confusion about this, it is worth pointing out + that with ESEARCH, as with *any* SEARCH or UID SEARCH command, it + MUST NOT be considered an error if the search terms include a range + of message numbers that extends (or, in fact, starts) beyond the end + of the mailbox. For example, a client might want to establish a + rolling window through the search results this way: + + C: tag1 UID ESEARCH FROM "frobozz" 1:100 + + ...followed later by this: + + C: tag1 UID ESEARCH FROM "frobozz" 101:200 + + ...and so on. This tells the server to match only the first hundred + messages in the mailbox the first time, the second hundred the second + time, etc. In fact, it might likely allow the server to optimize the + search significantly. In the above example, whether the mailbox + contains 50 or 150 or 250 messages, neither of the search commands + shown will result in an error. It is up to the client to know when + to stop moving its search window. + +2.1. The ESEARCH Response + + In response to an ESEARCH command, the server MUST return ESEARCH + responses [RFC4731] (that is, not SEARCH responses). Because message + numbers are not useful for mailboxes that are not selected, the + responses MUST contain information about UIDs, not message numbers. + This is true even if the source options specify that only the + selected mailbox be searched. + + Presence of a source option in the absence of a result option implies + the "ALL" result option (see Section 3.1 of [RFC4731]). Note that + this is not the same as the result from the SEARCH command described + in the IMAP base protocol [RFC3501]. + + Source options describe which mailboxes must be searched for + messages. An ESEARCH command with source options does not affect + which mailbox, if any, is currently selected, regardless of which + mailboxes are searched. + + For each mailbox satisfying the source options, a single ESEARCH + response MUST be returned if any messages in that mailbox match the + search criteria. An ESEARCH response MUST NOT be returned for + mailboxes that contain no matching messages. This is true even when + result options such as MIN, MAX, and COUNT are specified (see + Section 3.1 of [RFC4731]), and the values returned (lowest UID + matched, highest UID matched, and number of messages matched, + respectively) apply to the mailbox reported in that ESEARCH response. + + + +Leiba & Melnikov Experimental [Page 4] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + Note that it is possible for an ESEARCH command to return *no* + untagged responses (no ESEARCH responses at all), in the case that + there are no matches to the search in any of the mailboxes that + satisfy the source options. Clients can detect this situation by + finding the tagged OK response without having received any matching + untagged ESEARCH responses. + + Each ESEARCH response MUST contain the MAILBOX, TAG, and UIDVALIDITY + correlators. Correlators allow clients to issue several ESEARCH + commands at once (pipelined). If the SEARCHRES [RFC5182] extension + is used in an ESEARCH command, that ESEARCH command MUST be executed + by the server after all previous SEARCH/ESEARCH commands have + completed and before any subsequent SEARCH/ESEARCH commands are + executed. The server MAY perform consecutive ESEARCH commands in + parallel as long as none of them use the SEARCHRES extension. + +2.2. Source Options: Specifying Mailboxes to Search + + The source options, if present, MUST contain a mailbox specifier as + defined in the IMAP NOTIFY extension [RFC5465], Section 6 (using the + "filter-mailboxes" ABNF item), with the following differences: + + 1. The "selected-delayed" specifier is not valid here. + + 2. A "subtree-one" specifier is added. The "subtree" specifier + results in a search of the specified mailbox and all selectable + mailboxes that are subordinate to it, through an indefinitely + deep hierarchy. The "subtree-one" specifier results in a search + of the specified mailbox and all selectable child mailboxes, one + hierarchy level down. + + If "subtree" is specified, the server MUST defend against loops in + the hierarchy (for example, those caused by recursive file-system + links within the message store). The server SHOULD do this by + keeping track of the mailboxes that have been searched, and + terminating the hierarchy traversal when a repeat is found. If it + cannot do that, it MAY do it by limiting the hierarchy depth. + + If the source options are not present, the value "selected" is + assumed -- that is, only the currently selected mailbox is searched. + + The "personal" source option is a particularly convenient way to + search all of the current user's mailboxes. Note that there is no + way to use wildcard characters to search all mailboxes; the + "mailboxes" source option does not do wildcard expansion. + + + + + + +Leiba & Melnikov Experimental [Page 5] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + If the source options include (or default to) "selected", the IMAP + session MUST be in "selected" state. If the source options specify + other mailboxes and NOT "selected", then the IMAP session MUST be in + either "selected" or "authenticated" state. If the session is not in + a correct state, the ESEARCH command MUST return a "BAD" result. + + If the server supports the SEARCHRES [RFC5182] extension, then the + "SAVE" result option is valid *only* if "selected" is specified or + defaulted as the sole mailbox to be searched. If any source option + other than "selected" is specified, the ESEARCH command MUST return a + "BAD" result. + + If the server supports the CONTEXT=SEARCH and/or CONTEXT=SORT + extension [RFC5267], then the following additional rules apply: + + o The CONTEXT return option (Section 4.2 of [RFC5267]) can be used + with an ESEARCH command. + + o If the UPDATE return option is used (Section 4.3 of [RFC5267]), it + MUST apply ONLY to the currently selected mailbox. If UPDATE is + used and there is no mailbox currently selected, the ESEARCH + command MUST return a "BAD" result. + + o The PARTIAL search return option (Section 4.4 of [RFC5267]) can be + used and applies to each mailbox searched by the ESEARCH command. + + If the server supports the Access Control List (ACL) [RFC4314] + extension, then the logged-in user is required to have the "r" right + for each mailbox she wants to search. In addition, any mailboxes + that are not explicitly named (accessed through "personal" or + "subtree", for example) are required to have the "l" right. + Mailboxes matching the source options for which the logged-in user + lacks sufficient rights MUST be ignored by the ESEARCH command + processing. In particular, ESEARCH responses MUST NOT be returned + for those mailboxes. + +3. Examples + + In the following example, note that two ESEARCH commands are + pipelined, and that the server is running them in parallel, + interleaving a response to the second search amid the responses to + the first (watch the tags). + + C: tag1 ESEARCH IN (mailboxes "folder1" subtree "folder2") unseen + C: tag2 ESEARCH IN (mailboxes "folder1" subtree-one "folder2") + subject "chad" + S: * ESEARCH (TAG "tag1" MAILBOX "folder1" UIDVALIDITY 1) UID ALL + 4001,4003,4005,4007,4009 + + + +Leiba & Melnikov Experimental [Page 6] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + S: * ESEARCH (TAG "tag2" MAILBOX "folder1" UIDVALIDITY 1) UID ALL + 3001:3004,3788 + S: * ESEARCH (TAG "tag1" MAILBOX "folder2/banana" UIDVALIDITY 503) + UID ALL 3002,4004 + S: * ESEARCH (TAG "tag1" MAILBOX "folder2/peach" UIDVALIDITY 3) UID + ALL 921691 + S: tag1 OK done + S: * ESEARCH (TAG "tag2" MAILBOX "folder2/salmon" UIDVALIDITY + 1111111) UID ALL 50003,50006,50009,50012 + S: tag2 OK done + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) as described in [RFC5234]. Terms not defined here are + taken from [RFC3501], [RFC5465], or [RFC4466]. + + command-auth =/ esearch + ; Update definition from IMAP base [RFC3501]. + ; Add new "esearch" command. + + command-select =/ esearch + ; Update definition from IMAP base [RFC3501]. + ; Add new "esearch" command. + + filter-mailboxes-other =/ ("subtree-one" SP one-or-more-mailbox) + ; Update definition from IMAP Notify [RFC5465]. + ; Add new "subtree-one" selector. + + filter-mailboxes-selected = "selected" + ; Update definition from IMAP Notify [RFC5465]. + ; We forbid the use of "selected-delayed". + + one-correlator = ("TAG" SP tag-string) / ("MAILBOX" SP astring) / + ("UIDVALIDITY" SP nz-number) + ; Each correlator MUST appear exactly once. + + scope-option = scope-option-name [SP scope-option-value] + ; No options defined here. Syntax for future extensions. + + scope-option-name = tagged-ext-label + ; No options defined here. Syntax for future extensions. + + scope-option-value = tagged-ext-val + ; No options defined here. Syntax for future extensions. + + + + + + +Leiba & Melnikov Experimental [Page 7] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + scope-options = scope-option *(SP scope-option) + ; A given option may only appear once. + ; No options defined here. Syntax for future extensions. + + esearch = "ESEARCH" [SP esearch-source-opts] + [SP search-return-opts] SP search-program + + search-correlator = SP "(" one-correlator *(SP one-correlator) ")" + ; Updates definition in IMAP4 ABNF [RFC4466]. + + esearch-source-opts = "IN" SP "(" source-mbox [SP + "(" scope-options ")"] ")" + + source-mbox = filter-mailboxes *(SP filter-mailboxes) + ; "filter-mailboxes" is defined in IMAP Notify [RFC5465]. + ; See updated definition of filter-mailboxes-other, above. + ; See updated definition of filter-mailboxes-selected, above. + +5. Security Considerations + + This new IMAP ESEARCH command allows a single command to search many + mailboxes at once. On the one hand, a client could do that by + sending many IMAP SEARCH commands. On the other hand, this makes it + easier for a client to overwork a server, by sending a single command + that results in an expensive search of tens of thousands of + mailboxes. Server implementations need to be aware of that, and + provide mechanisms that prevent a client from adversely affecting + other users. Limitations on the number of mailboxes that may be + searched in one command, and/or on the server resources that will be + devoted to responding to a single client, are reasonable limitations + for an implementation to impose. + + Implementations MUST, of course, apply access controls appropriately, + limiting a user's access to ESEARCH in the same way its access is + limited for any other IMAP commands. This extension has no data- + access risks beyond what may be there in the unextended IMAP + implementation. + + Mailboxes matching the source options for which the logged-in user + lacks sufficient rights MUST be ignored by the ESEARCH command + processing (see the paragraph about this in Section 2.2). In + particular, any attempt to distinguish insufficient access from + non-existent mailboxes may expose information about the mailbox + hierarchy that isn't otherwise available to the client. + + If "subtree" is specified, the server MUST defend against loops in + the hierarchy (see the paragraph about this in Section 2.2). + + + + +Leiba & Melnikov Experimental [Page 8] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + +6. IANA Considerations + + IMAP4 capabilities are registered by publishing a Standards Track or + IESG-approved Experimental RFC. The "IMAP 4 Capabilities" registry + is currently located here: + + http://www.iana.org/ + + This document defines the IMAP capability "MULTISEARCH", and IANA has + added it to the registry. + +7. Acknowledgements + + The authors gratefully acknowledge feedback provided by Timo + Sirainen, Peter Coates, and Arnt Gulbrandsen. + +8. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2978] Freed, N. and J. Postel, "IANA Charset Registration + Procedures", BCP 19, RFC 2978, October 2000. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC4731] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH + Command for Controlling What Kind of Information Is + Returned", RFC 4731, November 2006. + + [RFC5182] Melnikov, A., "IMAP Extension for Referencing the Last + SEARCH Result", RFC 5182, March 2008. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, + January 2008. + + [RFC5267] Cridland, D. and C. King, "Contexts for IMAP4", RFC 5267, + July 2008. + + + + + +Leiba & Melnikov Experimental [Page 9] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + [RFC5465] Gulbrandsen, A., King, C., and A. Melnikov, "The IMAP + NOTIFY Extension", RFC 5465, February 2009. + +Authors' Addresses + + Barry Leiba + Huawei Technologies + + Phone: +1 646 827 0648 + EMail: barryleiba@computer.org + URI: http://internetmessagingtechnology.org/ + + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Leiba & Melnikov Experimental [Page 10] + diff --git a/docs/rfcs/rfc6331.Moving_Digest-MD5_to_Historic b/docs/rfcs/rfc6331.Moving_Digest-MD5_to_Historic new file mode 100644 index 0000000..3d4172f --- /dev/null +++ b/docs/rfcs/rfc6331.Moving_Digest-MD5_to_Historic @@ -0,0 +1,339 @@ + + + + + + +Internet Engineering Task Force (IETF) A. Melnikov +Request for Comments: 6331 Isode Limited +Obsoletes: 2831 July 2011 +Category: Informational +ISSN: 2070-1721 + + + Moving DIGEST-MD5 to Historic + +Abstract + + This memo describes problems with the DIGEST-MD5 Simple + Authentication and Security Layer (SASL) mechanism as specified in + RFC 2831. It marks DIGEST-MD5 as OBSOLETE in the IANA Registry of + SASL mechanisms and moves RFC 2831 to Historic status. + +Status of This Memo + + This document is not an Internet Standards Track specification; it is + published for informational purposes. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Not all documents + approved by the IESG are a candidate for any level of Internet + Standard; see Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc6331. + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + +Melnikov Informational [Page 1] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + + This document may contain material from IETF Documents or IETF + Contributions published or made publicly available before November + 10, 2008. The person(s) controlling the copyright in some of this + material may not have granted the IETF Trust the right to allow + modifications of such material outside the IETF Standards Process. + Without obtaining an adequate license from the person(s) controlling + the copyright in such materials, this document may not be modified + outside the IETF Standards Process, and derivative works of it may + not be created outside the IETF Standards Process, except to format + it for publication as an RFC or to translate it into languages other + than English. + +Table of Contents + + 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 2 + 2. Security Considerations . . . . . . . . . . . . . . . . . . . 5 + 3. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 5 + 4. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . 5 + 5. References . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 5.1. Normative References . . . . . . . . . . . . . . . . . . . . 5 + 5.2. Informative References . . . . . . . . . . . . . . . . . . . 5 + +1. Introduction and Overview + + [RFC2831] defines how HTTP Digest Authentication [RFC2617] can be + used as a Simple Authentication and Security Layer (SASL) [RFC4422] + mechanism for any protocol that has a SASL profile. It was intended + both as an improvement over CRAM-MD5 [RFC2195] and as a convenient + way to support a single authentication mechanism for web, email, the + Lightweight Directory Access Protocol (LDAP), and other protocols. + While it can be argued that it is an improvement over CRAM-MD5, many + implementors commented that the additional complexity of DIGEST-MD5 + makes it difficult to implement fully and securely. + + Below is an incomplete list of problems with the DIGEST-MD5 mechanism + as specified in [RFC2831]: + + 1. The mechanism has too many options and modes. Some of them are + not well described and are not widely implemented. For example, + DIGEST-MD5 allows the "qop" directive to contain multiple values, + but it also allows for multiple qop directives to be specified. + The handling of multiple options is not specified, which results + in minor interoperability problems. Some implementations + amalgamate multiple qop values into one, while others treat + multiple qops as an error. Another example is the use of an + empty authorization identity. In SASL, an empty authorization + identity means that the client is willing to authorize as the + authentication identity. The document is not clear on whether + + + +Melnikov Informational [Page 2] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + + the authzid must be omitted or if it can be specified with an + empty value to convey this. The requirement for backward + compatibility with HTTP Digest means that the situation is even + worse. For example, DIGEST-MD5 requires all usernames/passwords + that can be entirely represented in the ISO-8859-1 charset to be + down converted from UTF-8 [RFC3629] to ISO-8859-1 [ISO-8859-1]. + Another example is the use of quoted strings. Handling of + characters that need escaping is not properly described, and the + DIGEST-MD5 document has no examples to demonstrate correct + behavior. + + 2. The DIGEST-MD5 document uses ABNF from RFC 822 [RFC0822], which + allows an extra construct and allows for "implied folding + whitespace" to be inserted in many places. The difference from a + more common ABNF defined in [RFC5234] is confusing for some + implementors. As a result, many implementations do not accept + folding whitespace in many places where it is allowed. + + 3. The DIGEST-MD5 document uses the concept of a "realm" to define a + collection of accounts. A DIGEST-MD5 server can support one or + more realms. The DIGEST-MD5 document does not provide any + guidance on how realms should be named and, more importantly, how + they can be entered in User Interfaces (UIs). As a result, many + DIGEST-MD5 clients have confusing UIs, do not allow users to + enter a realm, and/or do not allow users to pick one of the + server-supported realms. + + 4. Use of username in the inner hash is problematic. The inner hash + of DIGEST-MD5 is an MD5 hash of colon-separated username, realm, + and password. Implementations may choose to store inner hashes + instead of clear text passwords. This has some useful + properties, such as protection from compromise of authentication + databases containing the same username and password on other + servers if a server with the username and password is + compromised; however, this is rarely done in practice. First, + the inner hash is not compatible with widely deployed Unix + password databases, and second, changing the username would + invalidate the inner hash. + + 5. Description of DES/3DES [DES] and RC4 security layers are + inadequate to produce independently developed interoperable + implementations. In the DES/3DES case, this is partly a problem + with existing DES APIs. + + 6. DIGEST-MD5 outer hash (the value of the "response" directive) + does not protect the whole authentication exchange, which makes + the mechanism vulnerable to "man-in-the-middle" (MITM) attacks, + such as modification of the list of supported qops or ciphers. + + + +Melnikov Informational [Page 3] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + + 7. The following features are missing from DIGEST-MD5, making it + insecure or unsuitable for use in protocols: + + A. Channel bindings [RFC5056]. + + B. Hash agility (i.e., no easy way to replace the MD5 hash + function with another one). + + C. Support for SASLPrep [RFC4013] or any other type of Unicode + character normalization of usernames and passwords. The + original DIGEST-MD5 document predates SASLPrep and does not + recommend any Unicode character normalization. + + 8. The cryptographic primitives in DIGEST-MD5 are not up to today's + standards, in particular: + + A. The MD5 hash is sufficiently weak to make a brute force + attack on DIGEST-MD5 easy with common hardware [RFC6151]. + + B. The RC4 algorithm is prone to attack when used as the + security layer without discarding the initial key stream + output [RFC6229]. + + C. The DES cipher for the security layer is considered insecure + due to its small key space [RFC3766]. + + Note that most of the problems listed above are already present in + the HTTP Digest authentication mechanism. + + Because DIGEST-MD5 is defined as an extensible mechanism, it is + possible to fix most of the problems listed above. However, this + would increase implementation complexity of an already complex + mechanism even further, so the effort is not worth the cost. In + addition, an implementation of a "fixed" DIGEST-MD5 specification + would likely either not interoperate with any existing implementation + of [RFC2831] or would be vulnerable to various downgrade attacks. + + Note that despite DIGEST-MD5 seeing some deployment on the Internet, + this specification recommends obsoleting DIGEST-MD5 because DIGEST- + MD5, as implemented, is not a reasonable candidate for further + standardization and should be deprecated in favor of one or more new + password-based mechanisms currently being designed. + + The Salted Challenge Response Authentication Mechanism (SCRAM) family + of SASL mechanisms [RFC5802] has been developed to provide similar + features as DIGEST-MD5 but with a better design. + + + + + +Melnikov Informational [Page 4] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + +2. Security Considerations + + Security issues are discussed throughout this document. + +3. IANA Considerations + + IANA has changed the "Intended usage" of the DIGEST-MD5 mechanism + registration in the SASL mechanism registry to OBSOLETE. The SASL + mechanism registry is specified in [RFC4422] and is currently + available at: + + http://www.iana.org/assignments/sasl-mechanisms + +4. Acknowledgements + + The author gratefully acknowledges the feedback provided by Chris + Newman, Simon Josefsson, Kurt Zeilenga, Sean Turner, and Abhijit + Menon-Sen. Various text was copied from other RFCs, in particular, + from [RFC2831]. + +5. References + +5.1. Normative References + + [RFC2617] Franks, J., Hallam-Baker, P., Hostetler, J., Lawrence, + S., Leach, P., Luotonen, A., and L. Stewart, "HTTP + Authentication: Basic and Digest Access + Authentication", RFC 2617, June 1999. + + [RFC2831] Leach, P. and C. Newman, "Using Digest Authentication + as a SASL Mechanism", RFC 2831, May 2000. + +5.2. Informative References + + [DES] National Institute of Standards and Technology, "Data + Encryption Standard (DES)", FIPS PUB 46-3, + October 1999. + + [ISO-8859-1] International Organization for Standardization, + "Information technology - 8-bit single-byte coded + graphic character sets - Part 1: Latin alphabet No. 1", + ISO/IEC 8859-1, 1998. + + [RFC0822] Crocker, D., "Standard for the format of ARPA Internet + text messages", STD 11, RFC 822, August 1982. + + + + + + +Melnikov Informational [Page 5] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + + [RFC2195] Klensin, J., Catoe, R., and P. Krumviede, "IMAP/POP + AUTHorize Extension for Simple Challenge/Response", + RFC 2195, September 1997. + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC3766] Orman, H. and P. Hoffman, "Determining Strengths For + Public Keys Used For Exchanging Symmetric Keys", + BCP 86, RFC 3766, April 2004. + + [RFC4013] Zeilenga, K., "SASLprep: Stringprep Profile for User + Names and Passwords", RFC 4013, February 2005. + + [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication + and Security Layer (SASL)", RFC 4422, June 2006. + + [RFC5056] Williams, N., "On the Use of Channel Bindings to Secure + Channels", RFC 5056, November 2007. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [RFC5802] Newman, C., Menon-Sen, A., Melnikov, A., and N. + Williams, "Salted Challenge Response Authentication + Mechanism (SCRAM) SASL and GSS-API Mechanisms", + RFC 5802, July 2010. + + [RFC6151] Turner, S. and L. Chen, "Updated Security + Considerations for the MD5 Message-Digest and the HMAC- + MD5 Algorithms", RFC 6151, March 2011. + + [RFC6229] Strombergson, J. and S. Josefsson, "Test Vectors for + the Stream Cipher RC4", RFC 6229, May 2011. + +Author's Address + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + + + +Melnikov Informational [Page 6] + From e303c6c5d0217d35b6ca2c37806f6f21431a0a8c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 7 Mar 2015 19:08:22 +0100 Subject: [PATCH 728/817] rework documentation - Add CONTRIBUTING.rst. - Improve MAINTAINERS.rst. - Split long HACKING.rst into: * dco.rst * GitAdvanced.rst Update content. Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 103 +++++ MAINTAINERS | 31 -- MAINTAINERS.rst | 93 ++++ docs/doc-src/API.rst | 14 +- docs/doc-src/FAQ.rst | 89 ++-- docs/doc-src/GitAdvanced.rst | 679 ++++++++++++++++++++++++++++++ docs/doc-src/HACKING.rst | 792 ----------------------------------- docs/doc-src/dco.rst | 68 +++ docs/doc-src/index.rst | 22 +- 9 files changed, 1004 insertions(+), 887 deletions(-) create mode 100644 CONTRIBUTING.rst delete mode 100644 MAINTAINERS create mode 100644 MAINTAINERS.rst create mode 100644 docs/doc-src/GitAdvanced.rst delete mode 100644 docs/doc-src/HACKING.rst create mode 100644 docs/doc-src/dco.rst diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..66ac2f4 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,103 @@ +.. -*- coding: utf-8 -*- +.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap +.. _Github: https://github.com/OfflineIMAP/offlineimap +.. _repository: git://github.com/OfflineIMAP/offlineimap.git +.. _maintainers: https://github.com/OfflineIMAP/offlineimap/blob/next/MAINTAINERS.rst +.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project +.. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst + + +================= +HOW TO CONTRIBUTE +================= + +You'll find here the basics to contribute to OfflineIMAP_, addressed to users +and both experienced/learning developers to quickly provide contributions. + +.. contents:: :depth: 3 + +Submit issues +============= + +Issues are welcome to both Github_ and the `mailing list`_, at your own +convenience. + + +Community +========= + +All contributors to OfflineIMAP_ are benevolent volunteers. This makes hacking +to OfflineIMAP_ **fun and open**. + +Thanks to Python, almost every developer can quickly become productive. Students +and novices are welcome. Third-parties patches are essential and proved to be a +wonderful source of changes for both fixes and new features. + +OfflineIMAP_ is entirely written in Python, works on IMAP and source code is +tracked with Git. + +*It is expected that most contributors don't have skills to all of these areas.* +That's why the best thing you could do for you, is to ask us about any +difficulty or question raising in your mind. We actually do our best to help new +comers. **We've all started like this.** + +- The official repository_ is maintained by the core team maintainers_. + +- The `mailing list`_ is where all the exciting things happen. + + +Getting started +=============== + +Occasional contributors +----------------------- + +* Clone the official repository_. + +Regular contributors +-------------------- + +* Create an account and login to Github. +* Fork the official repository_. +* Clone your own fork to your local workspace. +* Add a reference to your fork (once):: + + $ git remote add myfork https://github.com//offlineimap.git + +* Regularly fetch the changes applied by the maintainers:: + + $ git fetch origin + $ git checkout master + $ git merge offlineimap/master + $ git checkout next + $ git merge offlineimap/next + + +Making changes (all contributors) +--------------------------------- + +1. Create your own topic branch off of ``next`` (recently updated) via:: + + $ git checkout -b my_topic next + +2. Check for unnecessary whitespaces with ``git diff --check`` before committing. +3. Commit your changes into logical/atomic commits. **Sign-off your work** to + confirm you agree with the `Developer's Certificate of Origin`_. +4. Write a good *commit message* about **WHY** this patch (take samples from + the ``git log``). + + +Learn more +========== + +There is already a lot of documentation. Here's where you might want to look +first: + +- The directory ``offlineimap/docs`` has all kind of additional documentation + (advanced Git, RFCs, APIs, coding guidelines, etc). + +- The file ``offlineimap.conf`` allows to know all the supported features. + +- The file ``TODO.rst`` express code changes we'd like and current *Work In + Progress* (WIP). + diff --git a/MAINTAINERS b/MAINTAINERS deleted file mode 100644 index 53b54d8..0000000 --- a/MAINTAINERS +++ /dev/null @@ -1,31 +0,0 @@ - -Official maintainers -==================== - -Dmitrijs Ledkovs - email: xnox at debian.org - github: xnox - -Eygene Ryabinkin - email: rea at freebsd.org - github: konvpalto - -Sebastian Spaeth - email: sebastian at sspaeth.de - github: spaetz - -Nicolas Sebrecht - email: nicolas.s-dev at laposte.net - github: nicolas33 - -MAILING LIST MAINTAINERS -======================== - -Eygene Ryabinkin - email: rea at freebsd.org - -Sebastian Spaeth - email: sebastian at sspaeth.de - -Nicolas Sebrecht - email: nicolas.s-dev at laposte.net diff --git a/MAINTAINERS.rst b/MAINTAINERS.rst new file mode 100644 index 0000000..4ff0cda --- /dev/null +++ b/MAINTAINERS.rst @@ -0,0 +1,93 @@ +.. -*- coding: utf-8 -*- + +Official maintainers +==================== + +Dmitrijs Ledkovs + email: xnox at debian.org + github: xnox + +Eygene Ryabinkin + email: rea at freebsd.org + github: konvpalto + +Sebastian Spaeth + email: sebastian at sspaeth.de + github: spaetz + +Nicolas Sebrecht + email: nicolas.s-dev at laposte.net + github: nicolas33 + +Mailing List maintainers +======================== + +Eygene Ryabinkin + email: rea at freebsd.org + +Sebastian Spaeth + email: sebastian at sspaeth.de + +Nicolas Sebrecht + email: nicolas.s-dev at laposte.net + + +How to maintain the source code +=============================== + +Rolling out a new release +------------------------- + +1. Update the Changelogs. +2. Update the version in ``offlineimap/__init__.py``. +3. Commit the changes with the version in the commit message. +4. Tag the release. +5. Wait the next day before pushing out the release. +6. Make an announce to the mailing list. + + +Tagging stable releases or release candidates +''''''''''''''''''''''''''''''''''''''''''''' + +It is done via Git's ``tag`` command, but you **must** do ``git tag -a`` +to create annotated tag. + +Release tags are named ``vX.Y.Z`` and release candidate tags are named +``vX.Y.Z-rcN``. + + +How to become a core team maintainer +==================================== + +Express your desire to one of the current official maintainers. This is all you +have to do. + +We don't require years of contributions. Of course, we will pay attention a bit +to your past interest to the project and your skills. But nothing is mandatory +and it's fine to still be learning while being an official maintainer. Nobody is +all-knowing, even with years of experience. So, if one current maintainer +agrees, he will grant you the write access to the official repository. + +Contrary to what most people might think, becoming an official maintainer is not +something hard. You'll first be asked to merge contributions and make one or two +official releases. We will review your first steps and nothing more. + +Be aware that there is no more leader maintainer, neither is a leader in the +team. Once you become a team member and did some maintenance for the project, +you're free to take all the strong decisions. + +Since we are a team, we always try to discuss with others before taking any +strong decision, including the users. This prevents from breaking things inside +the whole community and the proximity we have with our users. Even most discreet +of them will raise one day with good hints, review, feedback or opinion. This is +how we are running in OfflineIMAP. And yes, we are really proud about the solid +relationship we have with the users. + +Also, we are all benevolent volunteers. We are in a very good position to know +that we are not always free to contribute as much as we'd like to. Being away +for some (long) time is not a problem. At the time of this writing, each +maintainer took some vacations from the project at some point in time. You'll +contribute as much as your free time permits. It's fine! We are a team, so we +help each other. + +Welcome! ,-) diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index 774f47b..de8b500 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -7,7 +7,13 @@ :mod:`offlineimap's` API documentation ====================================== -Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. Email repositories are represented by a :class:`offlineimap.repository.Base.BaseRepository` or derivatives (see :mod:`offlineimap.repository` for details). A folder within a repository is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative from :mod:`offlineimap.folder`. +Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the +high-level functionality. The rest of the classes should usually not needed to +be touched by the user. Email repositories are represented by a +:class:`offlineimap.repository.Base.BaseRepository` or derivatives (see +:mod:`offlineimap.repository` for details). A folder within a repository is +represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative +from :mod:`offlineimap.folder`. This page contains the main API overview of OfflineImap |release|. @@ -15,9 +21,6 @@ OfflineImap can be imported as:: from offlineimap import OfflineImap -The file ``HACKING.rst`` in the source distribution documents a -number of resources and conventions you may find useful. - :mod:`offlineimap` -- The OfflineImap module ============================================= @@ -34,7 +37,8 @@ number of resources and conventions you may find useful. :class:`offlineimap.account` ============================ -An :class:`accounts.Account` connects two email repositories that are to be synced. It comes in two flavors, normal and syncable. +An :class:`accounts.Account` connects two email repositories that are to be +synced. It comes in two flavors, normal and syncable. .. autoclass:: offlineimap.accounts.Account diff --git a/docs/doc-src/FAQ.rst b/docs/doc-src/FAQ.rst index 29fa928..49fa140 100644 --- a/docs/doc-src/FAQ.rst +++ b/docs/doc-src/FAQ.rst @@ -3,6 +3,15 @@ .. NOTE TO MAINTAINERS: Please add new questions to the end of their sections, so section/question numbers remain stable. +.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project +.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap +.. _ssl.wrap_socket: http://docs.python.org/library/ssl.html#ssl.wrap_socket +.. _Advanced Git: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/GitAdvanced.rst +.. _FAQ: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/FAQ.rst +.. _Contributing: https://github.com/OfflineIMAP/offlineimap/blob/next/CONTRIBUTING.rst + + + ============================================= OfflineIMAP FAQ (Frequently Asked Questions) @@ -15,14 +24,10 @@ .. sectnum:: -This is a work in progress. - Please feel free to ask questions and/or provide answers; send email to the `mailing list`_. -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project -.. _OfflineIMAP: https://github.com/nicolas33/offlineimap -.. _ssl.wrap_socket: http://docs.python.org/library/ssl.html#ssl.wrap_socket +Most recent `FAQ`_. OfflineIMAP @@ -206,8 +211,8 @@ How is OfflineIMAP conformance? * Internet Message Access Protocol version 4rev1 (IMAP 4rev1) as specified in `2060`:RFC: and `3501`:RFC: * CRAM-MD5 as specified in `2195`:RFC: -* Maildir as specified in the Maildir manpage and the qmail website -* Standard Python 2.6 as implemented on POSIX-compliant systems +* Maildir as specified in the Maildir manpage and the qmail website. +* Standard Python 2.7 as implemented on POSIX-compliant systems. Can I force OfflineIMAP to sync a folder right now? --------------------------------------------------- @@ -226,12 +231,13 @@ accounts as follows:: resync that account immediately. This will be ignored if a resync is already in progress for that account. -2) while in sleep mode, you can also send a SIGUSR1. See the :ref:`UNIX +2) While in sleep mode, you can also send a SIGUSR1. See the :ref:`UNIX signals` section in the MANUAL for details. I get a "Mailbox already exists" error -------------------------------------- + **Q:** When synchronizing, I receive errors such as:: Folder 'sent'[main-remote] could not be created. Server responded: @@ -241,7 +247,7 @@ I get a "Mailbox already exists" error to treat "special" folders as case insensitive (e.g. the initial INBOX. part, or folders such as "Sent" or "Trash"). If you happen to have a folder "sent" on one side of things and a folder called "Sent" - on the other side, offlineimap will try to create those folders on + on the other side, OfflineIMAP will try to create those folders on both sides. If you server happens to treat those folders as case-insensitive you can then see this warning. @@ -426,20 +432,24 @@ To be brief: (security and bugfixes) for users who don't want or can't upgrade to the latest release. -For more information about the branching model and workflow, see the HACKING page. +For more information about the branching model and workflow, see `Advanced +Git`_. Why are your Maildir message filenames so long? ----------------------------------------------- -OfflineIMAP has two relevant principles: 1) never modifying your messages in -any way and 2) ensuring 100% reliable synchronizations. In order to do a -reliable sync, OfflineIMAP must have a way to uniquely identify each e-mail. -Three pieces of information are required to do this: your account name, the -folder name, and the message UID. The account name can be calculated from the -path in which your messages are. The folder name can usually be as well, BUT -some mail clients move messages between folders by simply moving the file, -leaving the name intact. +OfflineIMAP has two relevant principles: + +1. Never modifying your messages in any way. +2. Ensure 100% reliable synchronizations. + +In order to do a reliable sync, OfflineIMAP must have a way to uniquely identify +each e-mail. Three pieces of information are required to do this: your account +name, the folder name, and the message UID. The account name can be calculated +from the path in which your messages are. The folder name can usually be as +well, BUT some mail clients move messages between folders by simply moving the +file, leaving the name intact. So, OfflineIMAP must store both a message UID and a folder ID. The folder ID is necessary so OfflineIMAP can detect a message being moved @@ -481,55 +491,26 @@ How to test OfflineIMAP? We don't have a testing tool, for now. As a IMAP client, we need an available IMAP server for that purpose. But it doesn't mean you can do anything. -Recent patches are merged in the next branch before beeing in the mainline. Once +Recent patches are merged in the next branch before being in the mainline. Once you have your own copy of the official repository, track this next branch:: - git checkout -t origin/next + $ git checkout -t origin/next Update this branch in a regular basis with:: - git checkout next - git pull + $ git checkout next + $ git pull Notice you're not supposed to install OfflineIMAP each time. You may simply run it like this:: - ./offlineimap.py + $ ./offlineimap.py The choice is up to you. :-) + How to submit a patch? ---------------------- -If you want to send regular patches, you should first subscribe to the `mailing -list`_. This is not a pre-requisite, though. - -Next, you'll find documentation in the docs/ directory, especially the HACKING -page. - -You'll need to get a clone from the official `OfflineIMAP`_ repository and -configure Git. Then, read the SubmittingPatches.rst page in your local -repository or at -https://github.com/nicolas33/offlineimap/blob/master/SubmittingPatches.rst#readme -. - -To send a patch, we recommend using 'git send-email'. - - -Where from should my patches be based on? ------------------------------------------ - -Depends. If you're not sure, it should start off of the master -branch. master is the branch where new patches should be based on by -default. - -Obvious materials for next release (e.g. new features) start off of -current next. Also, next is the natural branch to write patches on top -of commits not already in master. - -A fix for a very old bug or security issue may start off of maint. This isn't -needed since such fix are backported by the maintainer, though. - -Finally, a work on very active or current development can start from a topic -next. This clearly means you **need** this topic as a base for what is intended. +Read `Contributing`_. diff --git a/docs/doc-src/GitAdvanced.rst b/docs/doc-src/GitAdvanced.rst new file mode 100644 index 0000000..17686c4 --- /dev/null +++ b/docs/doc-src/GitAdvanced.rst @@ -0,0 +1,679 @@ +.. -*- coding: utf-8 -*- +.. _OfflineIMAP: http://offlineimap.org +.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project +.. _Developers's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst + +============ +Git Advanced +============ + +.. contents:: :depth: 2 + +Git: OfflineImap's branching Model And Workflow +=============================================== + + +Git Branching model +------------------- + +OfflineIMAP_ uses the following branches: + +master + This is **the mainline**. Simple users should use this branch. + +next + **The development branch** for developers and testers. The content of ``next`` is + merged into the mainline ``master`` at release time for both stable and releases + candidates. When patches are sent to the mailing list, contributors discuss + about them. Once done and when patches looks ready for the mainline, patches + are first merged into ``next``. Advanced developers and testers use this branch to + test the last merged patches before they hit the mainline. This helps not + introducing strong breakages directly in the mainline. + +maint + This is **the maintenance branch**. It gets its own releases starting off of an old + stable release. + Notice that this branch tend to be more or less abandoned when context does not + force the maintainers to take care of it. + +pu + Don't care much about this branch unless you're asked to use it. It's almost + abandoned nowadays. ``pu`` stands for *"proposed updates"* and helps + **tracking of topics**. If a topic is not ready for the ``next`` release, it + might be merged into ``pu``. This branch only help developers to work on + someone else topic or an earlier pending topic. Developers can extract a topic + from this branch to work on it. This branch is **not intended to be + checkouted**; never. Even developers don't do that. Due to the way ``pu`` is + built you can't expect content there to work in any way... unless you clearly + want to run into troubles. + + +Release cycles +-------------- + +A typical release cycle works like this: + +1. A stable release is out. + +2. Feature topics are sent, discussed and merged. + +3. When enough work was merged, we start the freeze cycle: the first release + candidate is out. + +4. During the freeze cycle, no more features are merged. It's time to test + OfflineIMAP_. The more we are late in *-rc* releases, the less patches are + merged but bug fixes. + +5. When we think a release is stable enough, we restart from step 1. + + +Because third-parties tend to not always follow the cycles, it's fine to send +your patches as soon as they are ready. Any maintainer might prefer to pend your +contributions before merging it at a better time. You'll always be notified if +such decision is made for your work. + +Know about where we are in the release cycle:: + + $ git tag + + +Create commits +-------------- + +* Make commits of logical units. +* If you change, add, or remove a command line option or + make some other user interface change, the associated + documentation should be updated as well. +* Check for unnecessary whitespace with ``git diff --check`` + before committing. +* Do not check in commented out code or unneeded files. +* the first line of the commit message should be a short + description (50 characters is the soft limit, see DISCUSSION + in git-commit(1)), and should skip the full stop +* The body should provide a meaningful commit message, which: + * uses the imperative, present tense: **change**, + not **changed** or **changes**; + * includes motivation for the change, and contrasts + its implementation with previous behaviour. +* Add a ``Signed-off-by: Your Name `` line to + to confirm that you agree to the `Developer's Certificate of Origin`_. +* Make sure that you have tests for the bug you are fixing. +* Make sure that the test suite passes after your commit. + + +Make a pull request +------------------- + +TODO. + + +Export commits as patches +------------------------- + +* Use ``git format-patch -M`` to create the patch. +* Do not attach your patch, but read in the mail + body, unless you cannot teach your mailer to + leave the formatting of the patch alone. +* Be careful doing cut & paste into your mailer, not to + corrupt whitespaces. + + +Export commits as patches (experts) +----------------------------------- + +* Do not PGP sign your patch. +* Provide additional information (which is unsuitable for + the commit message) between the ``---`` and the diffstat. +* If your name is not writable in ASCII, make sure that + you send off a message in the correct encoding. +* Send the patch to the `mailing list`_ if (and only if) + the patch is ready for inclusion. +* If you use `git-send-email(1)` which is a good idea, + please test it first by sending email to yourself. +* See below for instructions specific to your mailer. + + +Extract a topic from pu +----------------------- + +To find the tip of a topic branch, run ``git log --first-parent next..pu`` and +look for the merge commit. The second parent of this commit is the tip of the +topic branch. + + +``pu`` is built this way:: + + $ git checkout pu + $ git reset --keep next + $ git merge --no-ff -X theirs topic1 + $ git merge --no-ff -X theirs topic2 + $ git merge --no-ff -X theirs blue + $ git merge --no-ff -X theirs orange + ... + +As a consequence: + +1. Each topic merged uses a merge commit. A merge commit is a commit having 2 + ancestors. Actually, Git allows more than 2 parents but we don't use this + feature. It's intended. + +2. Paths in ``pu`` may mix up multiple versions if all the topics don't use the same + base commit. This is very often the case as topics aren't rebased: it guarantees + each topic is strictly identical to the last version sent to the mailing list. + No surprise. + + +What you need to extract a particular topic is the *sha1* of the tip of that +branch (the last commit of the topic). Assume you want the branch of the topic +called 'blue'. First, look at the log given by this command:: + + $ git log --reverse --merges --parents origin/next..origin/pu + +With this command you ask for the log: + +* from next to pu +* in reverse order (older first) +* merge commits only +* with the sha1 of the ancestors + +From this list, find the topic you're looking for, basing you search on the lines +like:: + + Merge branch 'topic/name' into pu + +By convention, it has the form /. When you're at +it, pick the topic ancestor sha1. It's always the last sha1 in the line starting +by 'commit'. For you to know: + +* The first sha1 is the commit you see: the merge commit. +* The following sha1 is the ancestor of the branch checkouted at merge time + (always the previous merged topic or the ancien next in our case). +* Last is the branch merged. + +Giving:: + + commit sha1_of_merge_commit sha1_of_ancient_pu sha1_of_topic_blue + +Then, you only have to checkout the topic from there:: + + $ git checkout -b blue sha1_of_topic_blue + +You're done! You've just created a new branch called "blue" with the blue +content. Be aware this topic is not updated against the **current** next branch. +,-) + + + +Very detailed version +===================== + +I started reading over the SubmittingPatches document for Git, primarily because +I wanted to have a document similar to it for OfflineIMAP to make sure people +understand what they are doing when they write `Signed-off-by` line. + +But the patch submission requirements are a lot more relaxed here on the +technical/contents front, because the OfflineIMAP is a lot smaller ;-). So here +are only the relevant bits. + + +Decide what branch to base your work on +--------------------------------------- + +In general, base your work on the ``next`` branch. Otherwise, start off of the +latest commit your change is relevant to. + + +Make separate commits for logically separate changes +---------------------------------------------------- + +Unless your patch is really trivial, you should not be sending your +changes in a single patch. Instead, always make a commit with +complete commit message and generate a series of small patches from +your repository. + +Describe the technical detail of the change(s). + +If your description starts to get too long, that's a sign that you probably need +to split up your commit to finer grained pieces. That being said, patches which +plainly describe the things that help reviewers check the patch, and future +maintainers understand the code, are the most beautiful patches. + +Descriptions that summarise the point in the subject well, and describe the +motivation for the change, the approach taken by the change, and if relevant how +this differs substantially from the prior version, can be found on Usenet +archives back into the late 80's. Consider it like good Netiquette, but for +code. + + +Generate your patch using git tools out of your commits +------------------------------------------------------- + +* ``git`` based diff tools (git, Cogito, and StGIT included) generate *unidiff* +which is the preferred format. + +* You do not have to be afraid to use ``-M`` option to ``git diff`` or ``git +format-patch``, if your patch involves file renames. The receiving end can +handle them just fine. + +* Please make sure your patch does not include any extra files which do not +belong in a patch submission. + +* Make sure to review your patch after generating it, to ensure accuracy. + +* Before sending out, please make sure it cleanly applies to the ``next`` branch +head. If you are preparing a work based on somewhere else, that is fine, but +please mark it as such. + + +Sending your patches +-------------------- + +The mailing list is the preferred way for sending patches. This allows easier +review and comments on the code. + +People on the mailing list need to be able to read and +comment on the changes you are submitting. It is important for +a developer to be able to "quote" your changes, using standard +e-mail tools, so that they may comment on specific portions of +your code. For this reason, all patches should be submitted +"inline". WARNING: Be wary of your MUAs word-wrap +corrupting your patch. Do not cut-n-paste your patch; you can +lose tabs that way if you are not careful. + +It is a common convention to prefix your subject line with +[PATCH]. This lets people easily distinguish patches from other +e-mail discussions. Use of additional markers after PATCH and +the closing bracket to mark the nature of the patch is also +encouraged. E.g. [PATCH/RFC] is often used when the patch is +not ready to be applied but it is for discussion, [PATCH v2], +[PATCH v3] etc. are often seen when you are sending an update to +what you have previously sent. + +* ``git format-patch`` command follows the best current practice to + format the body of an e-mail message. At the beginning of the + patch should come your commit message, ending with the + ``Signed-off-by:`` lines, a line that consists of three dashes, + followed by the diffstat information and the patch itself. + +* If you are forwarding a patch from somebody else, optionally, at + the beginning of the e-mail message just before the commit + message starts, you can put a ``From:`` line to name that person. + +* You often want to add additional explanation about the patch, + other than the commit message itself. Place such "cover letter" + material between the three dash lines and the diffstat. + +* Do not attach the patch as a MIME attachment, compressed or not. + Many popular e-mail applications will not always + transmit a MIME attachment as plain text, making it impossible to comment on + your code. A MIME attachment also takes a bit more time to process. This does + not decrease the likelihood of your MIME-attached change being accepted, but it + makes it more likely that it will be postponed. + + Exception: If your mailer is mangling patches then someone may ask + you to re-send them using MIME, that is OK. + +* Do not let your e-mail client send quoted-printable. + +* Do not let your e-mail client send format=flowed which would destroy +whitespaces in your patches. + +* Do not PGP sign your patch, at least for now. Most likely, your + maintainer or other people on the list would not have your PGP + key and would not bother obtaining it anyway. Your patch is not + judged by who you are; a good patch from an unknown origin has a + far better chance of being accepted than a patch from a known, + respected origin that is done poorly or does incorrect things. + + If you really really really really want to do a PGP signed + patch, format it as "multipart/signed", not a text/plain message + that starts with '-----BEGIN PGP SIGNED MESSAGE-----'. That is + not a text/plain, it's something else. + +* Unless your patch is a very trivial and an obviously correct one, + first send it with "To:" set to the mailing list, with "cc:" listing + people who are involved in the area you are touching (the output from + "git blame $path" and "git shortlog --no-merges $path" would help to + identify them), to solicit comments and reviews. After the list + reached a consensus that it is a good idea to apply the patch, re-send + it with "To:" set to the maintainer and optionally "cc:" the list for + inclusion. Do not forget to add trailers such as "Acked-by:", + "Reviewed-by:" and "Tested-by:" after your "Signed-off-by:" line as + necessary. + + +Sign your work +-------------- + +To improve tracking of who did what, we've borrowed the +"sign-off" procedure from the Linux kernel project on patches +that are being emailed around. Although OfflineIMAP is a lot +smaller project it is a good discipline to follow it. + +The sign-off is a simple line at the end of the explanation for +the patch, which **certifies that you wrote it or otherwise have +the right to pass it on as a open-source patch**. The rules are +pretty simple: if you can certify the below: + + +An ideal patch flow +=================== + +Here is an ideal patch flow for this project the current maintainers +suggests to the contributors: + +0. You come up with an itch. You code it up. + +1. Send it to the list and cc people who may need to know about + the change. + + The people who may need to know are the ones whose code you + are butchering. These people happen to be the ones who are + most likely to be knowledgeable enough to help you, but + they have no obligation to help you (i.e. you ask for help, + don't demand). ``git log -p -- $area_you_are_modifying`` would + help you find out who they are. + +2. You get comments and suggestions for improvements. You may + even get them in a "on top of your change" patch form. + +3. Polish, refine, and re-send to the list and the people who + spend their time to improve your patch. Go back to step (2). + +4. The list forms consensus that the last round of your patch is + good. Send it to the list and cc the maintainers. + +5. A topic branch is created with the patch and is merged to ``next``, + and cooked further and eventually graduates to ``master``. + + +In any time between the (2)-(3) cycle, the maintainer may pick it up +from the list and queue it to ``pu``, in order to make it easier for +people play with it without having to pick up and apply the patch to +their trees themselves. + + +Know the status of your patch after submission +---------------------------------------------- + +You can use Git itself to find out when your patch is merged in +master. ``git pull --rebase`` will automatically skip already-applied +patches, and will let you know. This works only if you rebase on top +of the branch in which your patch has been merged (i.e. it will not +tell you if your patch is merged in ``pu`` if you rebase on top of +``next``). + +.. Read the git mailing list, the maintainer regularly posts messages + entitled "What's cooking in git.git" and "What's in git.git" giving + the status of various proposed changes. + + +MUA specific hints +================== + +Some of patches I receive or pick up from the list share common +patterns of breakage. Please make sure your MUA is set up +properly not to corrupt whitespaces. Here are two common ones +I have seen: + +* Empty context lines that do not have _any_ whitespace. + +* Non empty context lines that have one extra whitespace at the + beginning. + +One test you could do yourself if your MUA is set up correctly is: + +* Send the patch to yourself, exactly the way you would, except + To: and Cc: lines, which would not contain the list and + maintainer address. + +* Save that patch to a file in UNIX mailbox format. Call it say + a.patch. + +* Try to apply to the tip of the "master" branch from the + git.git public repository:: + + $ git fetch http://kernel.org/pub/scm/git/git.git master:test-apply + $ git checkout test-apply + $ git reset --hard + $ git am a.patch + +If it does not apply correctly, there can be various reasons. + +* Your patch itself does not apply cleanly. That is _bad_ but + does not have much to do with your MUA. Please rebase the + patch appropriately. + +* Your MUA corrupted your patch; "am" would complain that + the patch does not apply. Look at .git/rebase-apply/ subdirectory and + see what 'patch' file contains and check for the common + corruption patterns mentioned above. + +* While you are at it, check what are in 'info' and + 'final-commit' files as well. If what is in 'final-commit' is + not exactly what you would want to see in the commit log + message, it is very likely that your maintainer would end up + hand editing the log message when he applies your patch. + Things like "Hi, this is my first patch.\n", if you really + want to put in the patch e-mail, should come after the + three-dash line that signals the end of the commit message. + + +Pine +---- + +(Johannes Schindelin) + I don't know how many people still use pine, but for those poor souls it may + be good to mention that the quell-flowed-text is needed for recent versions. + + ... the "no-strip-whitespace-before-send" option, too. AFAIK it was introduced + in 4.60. + +(Linus Torvalds) + And 4.58 needs at least this + +:: + + --- + diff-tree 8326dd8350be64ac7fc805f6563a1d61ad10d32c (from e886a61f76edf5410573e92e38ce22974f9c40f1) + Author: Linus Torvalds + Date: Mon Aug 15 17:23:51 2005 -0700 + + Fix pine whitespace-corruption bug + + There's no excuse for unconditionally removing whitespace from + the pico buffers on close. + + diff --git a/pico/pico.c b/pico/pico.c + --- a/pico/pico.c + +++ b/pico/pico.c + @@ -219,7 +219,9 @@ PICO *pm; + switch(pico_all_done){ /* prepare for/handle final events */ + case COMP_EXIT : /* already confirmed */ + packheader(); + +#if 0 + stripwhitespace(); + +#endif + c |= COMP_EXIT; + break; + +(Daniel Barkalow) + > A patch to SubmittingPatches, MUA specific help section for + > users of Pine 4.63 would be very much appreciated. + + Ah, it looks like a recent version changed the default behavior to do the + right thing, and inverted the sense of the configuration option. (Either + that or Gentoo did it.) So you need to set the + "no-strip-whitespace-before-send" option, unless the option you have is + "strip-whitespace-before-send", in which case you should avoid checking + it. + + +Thunderbird +----------- + +(A Large Angry SCM) + By default, Thunderbird will both wrap emails as well as flag them as + being 'format=flowed', both of which will make the resulting email unusable + by git. + + Here are some hints on how to successfully submit patches inline using + Thunderbird. + + There are two different approaches. One approach is to configure + Thunderbird to not mangle patches. The second approach is to use + an external editor to keep Thunderbird from mangling the patches. + +**Approach #1 (configuration):** + + This recipe is current as of Thunderbird 2.0.0.19. Three steps: + + 1. Configure your mail server composition as plain text + Edit...Account Settings...Composition & Addressing, + uncheck 'Compose Messages in HTML'. + 2. Configure your general composition window to not wrap + Edit..Preferences..Composition, wrap plain text messages at 0 + 3. Disable the use of format=flowed + Edit..Preferences..Advanced..Config Editor. Search for: + mailnews.send_plaintext_flowed + toggle it to make sure it is set to 'false'. + + After that is done, you should be able to compose email as you + otherwise would (cut + paste, git-format-patch | git-imap-send, etc), + and the patches should not be mangled. + +**Approach #2 (external editor):** + +This recipe appears to work with the current [*1*] Thunderbird from Suse. + +The following Thunderbird extensions are needed: + AboutConfig 0.5 + http://aboutconfig.mozdev.org/ + External Editor 0.7.2 + http://globs.org/articles.php?lng=en&pg=8 + + +1) Prepare the patch as a text file using your method of choice. + +2) Before opening a compose window, use Edit->Account Settings to + uncheck the "Compose messages in HTML format" setting in the + "Composition & Addressing" panel of the account to be used to send the + patch. [*2*] + +3) In the main Thunderbird window, _before_ you open the compose window + for the patch, use Tools->about:config to set the following to the + indicated values:: + + mailnews.send_plaintext_flowed => false + mailnews.wraplength => 0 + +4) Open a compose window and click the external editor icon. + +5) In the external editor window, read in the patch file and exit the + editor normally. + +6) Back in the compose window: Add whatever other text you wish to the + message, complete the addressing and subject fields, and press send. + +7) Optionally, undo the about:config/account settings changes made in + steps 2 & 3. + + +[Footnotes] + +*1* Version 1.0 (20041207) from the MozillaThunderbird-1.0-5 rpm of Suse +9.3 professional updates. + +*2* It may be possible to do this with about:config and the following +settings but I haven't tried, yet:: + + mail.html_compose => false + mail.identity.default.compose_html => false + mail.identity.id?.compose_html => false + +(Lukas Sandström) + There is a script in contrib/thunderbird-patch-inline which can help you + include patches with Thunderbird in an easy way. To use it, do the steps above + and then use the script as the external editor. + +Gnus +---- + +'|' in the *Summary* buffer can be used to pipe the current +message to an external program, and this is a handy way to drive +"git am". However, if the message is MIME encoded, what is +piped into the program is the representation you see in your +*Article* buffer after unwrapping MIME. This is often not what +you would want for two reasons. It tends to screw up non ASCII +characters (most notably in people's names), and also +whitespaces (fatal in patches). Running 'C-u g' to display the +message in raw form before using '|' to run the pipe can work +this problem around. + + +KMail +----- + +This should help you to submit patches inline using KMail. + +1) Prepare the patch as a text file. + +2) Click on New Mail. + +3) Go under "Options" in the Composer window and be sure that + "Word wrap" is not set. + +4) Use Message -> Insert file... and insert the patch. + +5) Back in the compose window: add whatever other text you wish to the + message, complete the addressing and subject fields, and press send. + + +Gmail +----- + +GMail does not appear to have any way to turn off line wrapping in the web +interface, so this will mangle any emails that you send. You can however +use "git send-email" and send your patches through the GMail SMTP server, or +use any IMAP email client to connect to the google IMAP server and forward +the emails through that. + +To use ``git send-email`` and send your patches through the GMail SMTP server, +edit `~/.gitconfig` to specify your account settings:: + + [sendemail] + smtpencryption = tls + smtpserver = smtp.gmail.com + smtpuser = user@gmail.com + smtppass = p4ssw0rd + smtpserverport = 587 + +Once your commits are ready to be sent to the mailing list, run the +following commands:: + + $ git format-patch --cover-letter -M origin/master -o outgoing/ + $ edit outgoing/0000-* + $ git send-email outgoing/* + +To submit using the IMAP interface, first, edit your `~/.gitconfig` to specify your +account settings:: + + [imap] + folder = "[Gmail]/Drafts" + host = imaps://imap.gmail.com + user = user@gmail.com + pass = p4ssw0rd + port = 993 + sslverify = false + +You might need to instead use: folder = "[Google Mail]/Drafts" if you get an error +that the "Folder doesn't exist". + +Once your commits are ready to be sent to the mailing list, run the +following commands:: + + $ git format-patch --cover-letter -M --stdout origin/master | git imap-send + +Just make sure to disable line wrapping in the email client (GMail web +interface will line wrap no matter what, so you need to use a real +IMAP client). diff --git a/docs/doc-src/HACKING.rst b/docs/doc-src/HACKING.rst deleted file mode 100644 index d4b3001..0000000 --- a/docs/doc-src/HACKING.rst +++ /dev/null @@ -1,792 +0,0 @@ -.. -*- coding: utf-8 -*- -.. _OfflineIMAP: http://offlineimap.org -.. _commits mailing list: http://lists.offlineimap.org/listinfo.cgi/commits-offlineimap.org -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project - -Hacking OfflineIMAP -=================== - -In this section you'll find all the information you need to start -hacking `OfflineIMAP`_. Be aware there are a lot of very usefull tips -in the mailing list. You may want to subscribe if you didn't, -yet. This is where you will get help. - -.. contents:: :depth: 2 - -API ---- - -:ref:`OfflineImap's API ` documentation is included in the user -documentation (next section) and online browsable at -``_. It is mostly auto-generated from the -source code and is a work in progress. Contributions in this area -would be very appreciated. - -Following new commits ---------------------- - -You can follow upstream commits on - - `CIA.vc `, - - `Ohloh `, - - `GitHub `, - - or on the `commits mailing list`_. - - - -Git: OfflineImap's branching Model And Workflow -=============================================== - -Introduction ------------- - -This optional section provides you with information on how we use git -branches and do releases. You will need to know very little about git -to get started. - -For the impatient, see the :ref:`contribution checklist` below. - -Git Branching model --------------------- - -OfflineIMAP uses the following branches: - - * master - * next - * maint - * (pu) - * & several topic oriented feature branches. A topic may consist of - one or more patches. - -master -++++++ - -If you're not sure what branch you should use, this one is for you. -This is the mainline. Simple users should use this branch to follow -OfflineIMAP's evolution. - -Usually, patches submitted to the mailing list should start off of -this branch. - -next -++++ - -Patches recently merged are good candidates for this branch. The content of next -is merged into the mainline (master) at release time for both stable and -rc -releases. - -When patches are sent to the mailing list, contributors discuss about them. Once -done and when patches looks ready for mainline, patches are first merged into -next. Advanced users and testers use this branch to test last merged patches -before they hit the mainline. This helps not introducing strong breackages -directly in master. - -pu -+++ - -pu stands for "proposed updates". If a topic is not ready for master nor next, -it may be merged into pu. This branch only help developers to work on someone -else topic or an earlier pending topic. - -This branch is **not intended to be checkouted**; never. Even developers don't -do that. Due to the way pu is built you can't expect content there to work in -any way... unless you clearly want to run into troubles. - -Developers can extract a topic from this branch to work on it. See the following -section "Extract a topic from pu" in this documentation. - -maint -+++++ - -This is the maintenance branch. It gets its own releases starting from an old -stable release. It helps both users having troubles with last stable releases -and users not wanting latest features or so to still benefit from strong bug -fixes and security fixes. - -Release cycles --------------- - -A typical release cycle works like this: - -1. A stable release is out. - -2. Feature topics are sent, discussed and merged. - -3. When enough work was merged, we start the freeze cycle: the first release - candidate is out. - -4. During the freeze cycle, no more features are merged. It's time to test - OfflineIMAP. New candidates version are released. The more we are late in -rc - releases the less patches are merged but bug fixes. - -5. When we think a release is stable enough, we restart from step 1. - -Tagging release or RC ---------------------- - -It is done via Git's ``tag`` command, but you must do ``git tag -a`` -to create annotated tag. - -Release tags are named ``vX.Y.Z`` and release candidate tags are named -``vX.Y.Z-rcN``. - - -.. _contribution checklist: - - -Contribution Checklist (and a short version for the impatient) -=============================================================== - -Create commits --------------- - -* make commits of logical units -* check for unnecessary whitespace with ``git diff --check`` - before committing -* do not check in commented out code or unneeded files -* the first line of the commit message should be a short - description (50 characters is the soft limit, see DISCUSSION - in git-commit(1)), and should skip the full stop -* the body should provide a meaningful commit message, which: - * uses the imperative, present tense: **change**, - not **changed** or **changes**. - * includes motivation for the change, and contrasts - its implementation with previous behaviour -* add a ``Signed-off-by: Your Name `` line to the - commit message (or just use the option `-s` when committing) - to confirm that you agree to the **Developer's Certificate of Origin** -* make sure that you have tests for the bug you are fixing -* make sure that the test suite passes after your commit - - -Export commits as patches -------------------------- - -* use ``git format-patch -M`` to create the patch -* do not PGP sign your patch -* do not attach your patch, but read in the mail - body, unless you cannot teach your mailer to - leave the formatting of the patch alone. -* be careful doing cut & paste into your mailer, not to - corrupt whitespaces. -* provide additional information (which is unsuitable for - the commit message) between the ``---`` and the diffstat -* if you change, add, or remove a command line option or - make some other user interface change, the associated - documentation should be updated as well. -* if your name is not writable in ASCII, make sure that - you send off a message in the correct encoding. -* send the patch to the `mailing list`_ and the - maintainer (nicolas.s-dev@laposte.net) if (and only if) - the patch is ready for inclusion. If you use `git-send-email(1)`, - please test it first by sending email to yourself. -* see below for instructions specific to your mailer - - - -Long version ------------- - -I started reading over the SubmittingPatches document for Git, primarily because -I wanted to have a document similar to it for OfflineIMAP to make sure people -understand what they are doing when they write `Signed-off-by` line. - -But the patch submission requirements are a lot more relaxed here on the -technical/contents front, because the OfflineIMAP is a lot smaller ;-). So here -is only the relevant bits. - -Decide what branch to base your work on -+++++++++++++++++++++++++++++++++++++++ - -In general, always base your work on the oldest branch that your -change is relevant to. - -* A bugfix should be based on 'maint' in general. If the bug is not - present in 'maint', base it on 'master'. For a bug that's not yet - in 'master', find the topic that introduces the regression, and - base your work on the tip of the topic. -* A new feature should be based on 'master' in general. If the new - feature depends on a topic that is in 'pu', but not in 'master', - base your work on the tip of that topic. -* Corrections and enhancements to a topic not yet in 'master' should - be based on the tip of that topic. If the topic has not been merged - to 'next', it's alright to add a note to squash minor corrections - into the series. -* In the exceptional case that a new feature depends on several topics - not in 'master', start working on 'next' or 'pu' privately and send - out patches for discussion. Before the final merge, you may have to - wait until some of the dependent topics graduate to 'master', and - rebase your work. - -To find the tip of a topic branch, run ``git log --first-parent -master..pu`` and look for the merge commit. The second parent of this -commit is the tip of the topic branch. - -Make separate commits for logically separate changes -++++++++++++++++++++++++++++++++++++++++++++++++++++ - -Unless your patch is really trivial, you should not be sending your -changes in a single patch. Instead, always make a commit with -complete commit message and generate a series of small patches from -your repository. - -Describe the technical detail of the change(s). - -If your description starts to get too long, that's a sign that you -probably need to split up your commit to finer grained pieces. -That being said, patches which plainly describe the things that -help reviewers check the patch, and future maintainers understand -the code, are the most beautiful patches. Descriptions that summarise -the point in the subject well, and describe the motivation for the -change, the approach taken by the change, and if relevant how this -differs substantially from the prior version, can be found on Usenet -archives back into the late 80's. Consider it like good Netiquette, -but for code. - - -Generate your patch using git tools out of your commits -+++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -git based diff tools (git, Cogito, and StGIT included) generate -unidiff which is the preferred format. - -You do not have to be afraid to use -M option to ``git diff`` or -``git format-patch``, if your patch involves file renames. The -receiving end can handle them just fine. - -Please make sure your patch does not include any extra files -which do not belong in a patch submission. Make sure to review -your patch after generating it, to ensure accuracy. Before -sending out, please make sure it cleanly applies to the "master" -branch head. If you are preparing a work based on "next" branch, -that is fine, but please mark it as such. - - -Sending your patches -++++++++++++++++++++ - -People on the mailing list need to be able to read and -comment on the changes you are submitting. It is important for -a developer to be able to "quote" your changes, using standard -e-mail tools, so that they may comment on specific portions of -your code. For this reason, all patches should be submitted -"inline". WARNING: Be wary of your MUAs word-wrap -corrupting your patch. Do not cut-n-paste your patch; you can -lose tabs that way if you are not careful. - -It is a common convention to prefix your subject line with -[PATCH]. This lets people easily distinguish patches from other -e-mail discussions. Use of additional markers after PATCH and -the closing bracket to mark the nature of the patch is also -encouraged. E.g. [PATCH/RFC] is often used when the patch is -not ready to be applied but it is for discussion, [PATCH v2], -[PATCH v3] etc. are often seen when you are sending an update to -what you have previously sent. - -``git format-patch`` command follows the best current practice to -format the body of an e-mail message. At the beginning of the -patch should come your commit message, ending with the -Signed-off-by: lines, and a line that consists of three dashes, -followed by the diffstat information and the patch itself. If -you are forwarding a patch from somebody else, optionally, at -the beginning of the e-mail message just before the commit -message starts, you can put a "From: " line to name that person. - -You often want to add additional explanation about the patch, -other than the commit message itself. Place such "cover letter" -material between the three dash lines and the diffstat. - -Do not attach the patch as a MIME attachment, compressed or not. -Do not let your e-mail client send quoted-printable. Do not let -your e-mail client send format=flowed which would destroy -whitespaces in your patches. Many -popular e-mail applications will not always transmit a MIME -attachment as plain text, making it impossible to comment on -your code. A MIME attachment also takes a bit more time to -process. This does not decrease the likelihood of your -MIME-attached change being accepted, but it makes it more likely -that it will be postponed. - -Exception: If your mailer is mangling patches then someone may ask -you to re-send them using MIME, that is OK. - -Do not PGP sign your patch, at least for now. Most likely, your -maintainer or other people on the list would not have your PGP -key and would not bother obtaining it anyway. Your patch is not -judged by who you are; a good patch from an unknown origin has a -far better chance of being accepted than a patch from a known, -respected origin that is done poorly or does incorrect things. - -If you really really really really want to do a PGP signed -patch, format it as "multipart/signed", not a text/plain message -that starts with '-----BEGIN PGP SIGNED MESSAGE-----'. That is -not a text/plain, it's something else. - -Unless your patch is a very trivial and an obviously correct one, -first send it with "To:" set to the mailing list, with "cc:" listing -people who are involved in the area you are touching (the output from -"git blame $path" and "git shortlog --no-merges $path" would help to -identify them), to solicit comments and reviews. After the list -reached a consensus that it is a good idea to apply the patch, re-send -it with "To:" set to the maintainer and optionally "cc:" the list for -inclusion. Do not forget to add trailers such as "Acked-by:", -"Reviewed-by:" and "Tested-by:" after your "Signed-off-by:" line as -necessary. - - -Sign your work -++++++++++++++ - -To improve tracking of who did what, we've borrowed the -"sign-off" procedure from the Linux kernel project on patches -that are being emailed around. Although OfflineIMAP is a lot -smaller project it is a good discipline to follow it. - -The sign-off is a simple line at the end of the explanation for -the patch, which **certifies that you wrote it or otherwise have -the right to pass it on as a open-source patch**. The rules are -pretty simple: if you can certify the below: - -**Developer's Certificate of Origin 1.1** -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - By making a contribution to this project, I certify that: - - (a) The contribution was created in whole or in part by me and I - have the right to submit it under the open source license - indicated in the file; or - - (b) The contribution is based upon previous work that, to the best - of my knowledge, is covered under an appropriate open source - license and I have the right under that license to submit that - work with modifications, whether created in whole or in part - by me, under the same open source license (unless I am - permitted to submit under a different license), as indicated - in the file; or - - (c) The contribution was provided directly to me by some other - person who certified (a), (b) or (c) and I have not modified - it. - - (d) I understand and agree that this project and the contribution - are public and that a record of the contribution (including all - personal information I submit with it, including my sign-off) is - maintained indefinitely and may be redistributed consistent with - this project or the open source license(s) involved. - -then you just add a line saying - - Signed-off-by: Random J Developer - -This line can be automatically added by git if you run the git-commit -command with the -s option. - -Notice that you can place your own Signed-off-by: line when -forwarding somebody else's patch with the above rules for -D-C-O. Indeed you are encouraged to do so. Do not forget to -place an in-body "From: " line at the beginning to properly attribute -the change to its true author (see above). - -Also notice that a real name is used in the Signed-off-by: line. Please -don't hide your real name. - -If you like, you can put extra tags at the end: - -* "Reported-by:" is used to to credit someone who found the bug that - the patch attempts to fix. -* "Acked-by:" says that the person who is more familiar with the area - the patch attempts to modify liked the patch. -* "Reviewed-by:", unlike the other tags, can only be offered by the - reviewer and means that she is completely satisfied that the patch - is ready for application. It is usually offered only after a - detailed review. -* "Tested-by:" is used to indicate that the person applied the patch - and found it to have the desired effect. - -You can also create your own tag or use one that's in common usage -such as "Thanks-to:", "Based-on-patch-by:", or "Mentored-by:". - -An ideal patch flow -=================== - -Here is an ideal patch flow for this project the current maintainer -suggests to the contributors: - - (0) You come up with an itch. You code it up. - - (1) Send it to the list and cc people who may need to know about - the change. - - The people who may need to know are the ones whose code you - are butchering. These people happen to be the ones who are - most likely to be knowledgeable enough to help you, but - they have no obligation to help you (i.e. you ask for help, - don't demand). ``git log -p -- $area_you_are_modifying`` would - help you find out who they are. - - (2) You get comments and suggestions for improvements. You may - even get them in a "on top of your change" patch form. - - (3) Polish, refine, and re-send to the list and the people who - spend their time to improve your patch. Go back to step (2). - - (4) The list forms consensus that the last round of your patch is - good. Send it to the list and cc the maintainer. - - (5) A topic branch is created with the patch and is merged to 'next', - and cooked further and eventually graduates to 'master'. - -In any time between the (2)-(3) cycle, the maintainer may pick it up -from the list and queue it to 'pu', in order to make it easier for -people play with it without having to pick up and apply the patch to -their trees themselves. - -Know the status of your patch after submission ----------------------------------------------- - -* You can use Git itself to find out when your patch is merged in - master. ``git pull --rebase`` will automatically skip already-applied - patches, and will let you know. This works only if you rebase on top - of the branch in which your patch has been merged (i.e. it will not - tell you if your patch is merged in pu if you rebase on top of - master). - -.. * Read the git mailing list, the maintainer regularly posts messages - entitled "What's cooking in git.git" and "What's in git.git" giving - the status of various proposed changes. - -MUA specific hints -================== - -Some of patches I receive or pick up from the list share common -patterns of breakage. Please make sure your MUA is set up -properly not to corrupt whitespaces. Here are two common ones -I have seen: - -* Empty context lines that do not have _any_ whitespace. - -* Non empty context lines that have one extra whitespace at the - beginning. - -One test you could do yourself if your MUA is set up correctly is: - -* Send the patch to yourself, exactly the way you would, except - To: and Cc: lines, which would not contain the list and - maintainer address. - -* Save that patch to a file in UNIX mailbox format. Call it say - a.patch. - -* Try to apply to the tip of the "master" branch from the - git.git public repository:: - - $ git fetch http://kernel.org/pub/scm/git/git.git master:test-apply - $ git checkout test-apply - $ git reset --hard - $ git am a.patch - -If it does not apply correctly, there can be various reasons. - -* Your patch itself does not apply cleanly. That is _bad_ but - does not have much to do with your MUA. Please rebase the - patch appropriately. - -* Your MUA corrupted your patch; "am" would complain that - the patch does not apply. Look at .git/rebase-apply/ subdirectory and - see what 'patch' file contains and check for the common - corruption patterns mentioned above. - -* While you are at it, check what are in 'info' and - 'final-commit' files as well. If what is in 'final-commit' is - not exactly what you would want to see in the commit log - message, it is very likely that your maintainer would end up - hand editing the log message when he applies your patch. - Things like "Hi, this is my first patch.\n", if you really - want to put in the patch e-mail, should come after the - three-dash line that signals the end of the commit message. - - -Pine ----- - -(Johannes Schindelin) - I don't know how many people still use pine, but for those poor souls it may - be good to mention that the quell-flowed-text is needed for recent versions. - - ... the "no-strip-whitespace-before-send" option, too. AFAIK it was introduced - in 4.60. - -(Linus Torvalds) - And 4.58 needs at least this - -:: - - --- - diff-tree 8326dd8350be64ac7fc805f6563a1d61ad10d32c (from e886a61f76edf5410573e92e38ce22974f9c40f1) - Author: Linus Torvalds - Date: Mon Aug 15 17:23:51 2005 -0700 - - Fix pine whitespace-corruption bug - - There's no excuse for unconditionally removing whitespace from - the pico buffers on close. - - diff --git a/pico/pico.c b/pico/pico.c - --- a/pico/pico.c - +++ b/pico/pico.c - @@ -219,7 +219,9 @@ PICO *pm; - switch(pico_all_done){ /* prepare for/handle final events */ - case COMP_EXIT : /* already confirmed */ - packheader(); - +#if 0 - stripwhitespace(); - +#endif - c |= COMP_EXIT; - break; - -(Daniel Barkalow) - > A patch to SubmittingPatches, MUA specific help section for - > users of Pine 4.63 would be very much appreciated. - - Ah, it looks like a recent version changed the default behavior to do the - right thing, and inverted the sense of the configuration option. (Either - that or Gentoo did it.) So you need to set the - "no-strip-whitespace-before-send" option, unless the option you have is - "strip-whitespace-before-send", in which case you should avoid checking - it. - - -Thunderbird ------------ - -(A Large Angry SCM) - By default, Thunderbird will both wrap emails as well as flag them as - being 'format=flowed', both of which will make the resulting email unusable - by git. - - Here are some hints on how to successfully submit patches inline using - Thunderbird. - - There are two different approaches. One approach is to configure - Thunderbird to not mangle patches. The second approach is to use - an external editor to keep Thunderbird from mangling the patches. - -**Approach #1 (configuration):** - - This recipe is current as of Thunderbird 2.0.0.19. Three steps: - - 1. Configure your mail server composition as plain text - Edit...Account Settings...Composition & Addressing, - uncheck 'Compose Messages in HTML'. - 2. Configure your general composition window to not wrap - Edit..Preferences..Composition, wrap plain text messages at 0 - 3. Disable the use of format=flowed - Edit..Preferences..Advanced..Config Editor. Search for: - mailnews.send_plaintext_flowed - toggle it to make sure it is set to 'false'. - - After that is done, you should be able to compose email as you - otherwise would (cut + paste, git-format-patch | git-imap-send, etc), - and the patches should not be mangled. - -**Approach #2 (external editor):** - -This recipe appears to work with the current [*1*] Thunderbird from Suse. - -The following Thunderbird extensions are needed: - AboutConfig 0.5 - http://aboutconfig.mozdev.org/ - External Editor 0.7.2 - http://globs.org/articles.php?lng=en&pg=8 - - -1) Prepare the patch as a text file using your method of choice. - -2) Before opening a compose window, use Edit->Account Settings to - uncheck the "Compose messages in HTML format" setting in the - "Composition & Addressing" panel of the account to be used to send the - patch. [*2*] - -3) In the main Thunderbird window, _before_ you open the compose window - for the patch, use Tools->about:config to set the following to the - indicated values:: - - mailnews.send_plaintext_flowed => false - mailnews.wraplength => 0 - -4) Open a compose window and click the external editor icon. - -5) In the external editor window, read in the patch file and exit the - editor normally. - -6) Back in the compose window: Add whatever other text you wish to the - message, complete the addressing and subject fields, and press send. - -7) Optionally, undo the about:config/account settings changes made in - steps 2 & 3. - - -[Footnotes] - -*1* Version 1.0 (20041207) from the MozillaThunderbird-1.0-5 rpm of Suse -9.3 professional updates. - -*2* It may be possible to do this with about:config and the following -settings but I haven't tried, yet:: - - mail.html_compose => false - mail.identity.default.compose_html => false - mail.identity.id?.compose_html => false - -(Lukas Sandström) - There is a script in contrib/thunderbird-patch-inline which can help you - include patches with Thunderbird in an easy way. To use it, do the steps above - and then use the script as the external editor. - -Gnus ----- - -'|' in the *Summary* buffer can be used to pipe the current -message to an external program, and this is a handy way to drive -"git am". However, if the message is MIME encoded, what is -piped into the program is the representation you see in your -*Article* buffer after unwrapping MIME. This is often not what -you would want for two reasons. It tends to screw up non ASCII -characters (most notably in people's names), and also -whitespaces (fatal in patches). Running 'C-u g' to display the -message in raw form before using '|' to run the pipe can work -this problem around. - - -KMail ------ - -This should help you to submit patches inline using KMail. - -1) Prepare the patch as a text file. - -2) Click on New Mail. - -3) Go under "Options" in the Composer window and be sure that - "Word wrap" is not set. - -4) Use Message -> Insert file... and insert the patch. - -5) Back in the compose window: add whatever other text you wish to the - message, complete the addressing and subject fields, and press send. - - -Gmail ------ - -GMail does not appear to have any way to turn off line wrapping in the web -interface, so this will mangle any emails that you send. You can however -use "git send-email" and send your patches through the GMail SMTP server, or -use any IMAP email client to connect to the google IMAP server and forward -the emails through that. - -To use ``git send-email`` and send your patches through the GMail SMTP server, -edit `~/.gitconfig` to specify your account settings:: - - [sendemail] - smtpencryption = tls - smtpserver = smtp.gmail.com - smtpuser = user@gmail.com - smtppass = p4ssw0rd - smtpserverport = 587 - -Once your commits are ready to be sent to the mailing list, run the -following commands:: - - $ git format-patch --cover-letter -M origin/master -o outgoing/ - $ edit outgoing/0000-* - $ git send-email outgoing/* - -To submit using the IMAP interface, first, edit your `~/.gitconfig` to specify your -account settings:: - - [imap] - folder = "[Gmail]/Drafts" - host = imaps://imap.gmail.com - user = user@gmail.com - pass = p4ssw0rd - port = 993 - sslverify = false - -You might need to instead use: folder = "[Google Mail]/Drafts" if you get an error -that the "Folder doesn't exist". - -Once your commits are ready to be sent to the mailing list, run the -following commands:: - - $ git format-patch --cover-letter -M --stdout origin/master | git imap-send - -Just make sure to disable line wrapping in the email client (GMail web -interface will line wrap no matter what, so you need to use a real -IMAP client). - -Working with Git -================ - -Extract a topic from pu ------------------------ - -pu is built this way:: - - git checkout pu - git reset --keep next - git merge --no-ff -X theirs topic1 - git merge --no-ff -X theirs topic2 - git merge --no-ff -X theirs blue - git merge --no-ff -X theirs orange - ... - -As a consequence: - -1. Each topic merged uses a merge commit. A merge commit is a commit having 2 - ancestors. Actually, Git allows more than 2 parents but we don't use this - feature. It's intended. - -2. Paths in pu may mix up multiple versions if all the topics don't use the same - base commit. This is very often the case as topics aren't rebased: it guarantees - each topic is strictly identical to the last version sent to the mailing list. - No surprise. - - -What you need to extract a particular topic is the sha1 of the tip of that -branch (the last commit of the topic). Assume you want the branch of the topic -called 'blue'. First, look at the log given by this command:: - - git log --reverse --merges --parents origin/next..origin/pu - -With this command you ask for the log: - -* from next to pu -* in reverse order (older first) -* merge commits only -* with the sha1 of the ancestors - -In this list, find the topic you're looking for, basing you search on the lines -like:: - - Merge branch 'topic/name' into pu - -By convention, it has the form /. When you're at -it, pick the topic ancestor sha1. It's always the last sha1 in the line starting -by 'commit'. For you to know: - -* the first is the sha1 of the commit you see: the merge commit -* the following sha1 is the ancestor of the branch checkouted at merge time - (always the previous merged topic or the ancien next in our case) -* last is the branch merged - -Giving:: - - commit sha1_of_merge_commit sha1_of_ancient_pu sha1_of_topic_blue - -Then, you only have to checkout the topic from there:: - - git checkout -b blue sha1_of_topic_blue - -and you're done! You've just created a new branch called "blue" with the blue -content. Be aware this topic is almostly not updated against current next -branch. ,-) diff --git a/docs/doc-src/dco.rst b/docs/doc-src/dco.rst new file mode 100644 index 0000000..83ae888 --- /dev/null +++ b/docs/doc-src/dco.rst @@ -0,0 +1,68 @@ + +Developer's Certificate of Origin +================================= + +v1.1:: + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. + + +Then, you just add a line saying:: + + Signed-off-by: Random J Developer + +This line can be automatically added by git if you run the git-commit command +with the ``-s`` option. Signing can made be afterword with ``--amend -s``. + +Notice that you can place your own ``Signed-off-by:`` line when forwarding +somebody else's patch with the above rules for D-C-O. Indeed you are encouraged +to do so. Do not forget to place an in-body ``From:`` line at the beginning to +properly attribute the change to its true author (see above). + +Also notice that a real name is used in the ``Signed-off-by:`` line. Please +don't hide your real name. + +If you like, you can put extra tags at the end: + +Reported-by + is used to to credit someone who found the bug that the patch attempts to fix. + +Acked-by + says that the person who is more familiar with the area the patch attempts to + modify liked the patch. + +Reviewed-by + unlike the other tags, can only be offered by the reviewer and means that she + is completely satisfied that the patch is ready for application. It is + usually offered only after a detailed review. + +Tested-by + is used to indicate that the person applied the patch and found it to have the + desired effect. + +You can also create your own tag or use one that's in common usage such as +``Thanks-to:``, ``Based-on-patch-by:``, or ``Mentored-by:``. + + diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index e7d639e..0f6d8c4 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -5,12 +5,22 @@ Welcome to :mod:`offlineimaps`'s documentation ============================================== -`OfflineImap`_ synchronizes email between an IMAP server and a MailDir or between two IMAP servers. It offers very powerful and flexible configuration options, that allow things such as the filtering of folders, transposing of names via static configuration or python scripting. It plays well with mutt and other MailDir consuming email clients. +`OfflineImap`_ synchronizes email between an IMAP server and a MailDir or +between two IMAP servers. It offers very powerful and flexible configuration +options, that allow things such as the filtering of folders, transposing of +names via static configuration or python scripting. It plays well with mutt and +other MailDir consuming email clients. -The documentation contains the end user documentation in a first part. It also contains use cases and example configurations. It is followed by the internal :doc:`API documentation ` for those interested in modifying the source code or otherwise peek into the OfflineImap internals in a second part. +The documentation contains the end user documentation in a first part. It also +contains use cases and example configurations. It is followed by the internal +:doc:`API documentation ` for those interested in modifying the source code +or otherwise peek into the OfflineImap internals in a second part. -If you just want to get started with minimal fuzz, have a look at our `online quick start guide `_. Do note though, that our configuration options are many and powerful. Perusing our precious documentation does often pay off! +If you just want to get started with minimal fuzz, have a look at our `online +quick start guide `_. Do note though, +that our configuration options are many and powerful. Perusing our precious +documentation does often pay off! More information on specific topics can be found on the following pages: @@ -26,7 +36,8 @@ More information on specific topics can be found on the following pages: * :doc:`Frequently Asked Questions ` **Developer documentation** - * :doc:`HACKING HowTo & git workflows ` + * :doc:`Contributing ` + * :doc:`Advanced Git ` * :doc:`API documentation ` for internal details on the :mod:`offlineimap` module @@ -41,7 +52,8 @@ More information on specific topics can be found on the following pages: offlineimap FAQ - HACKING + CONTRIBUTING + GitAdvanced API repository ui From 61d292bb85a0f3748277656dad50af7d8bf9ed67 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 7 Mar 2015 19:56:05 +0100 Subject: [PATCH 729/817] add TODO.rst Signed-off-by: Nicolas Sebrecht --- TODO.rst | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 TODO.rst diff --git a/TODO.rst b/TODO.rst new file mode 100644 index 0000000..73f80da --- /dev/null +++ b/TODO.rst @@ -0,0 +1,127 @@ +.. vim: spelllang=en ts=2 expandtab : + +.. _coding style: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/CodingGuidelines.rst + +============================ +TODO list by relevance order +============================ + +Should be the starting point to improve the `coding style`_. + +Write your WIP directly in this file. + +TODO list +--------- + +* Better names for variables, objects, etc. + + +* Improve comments. + + Most of the current comments assume a very good + knowledge of the internals. That sucks because I guess nobody is + anymore aware of ALL of them. Time when this was a one guy made + project has long passed. + + +* Better policy on objects. + + - Turn ALL attributes private and use accessors. This is not + "pythonic" but such pythonic thing turn the code into intricated + code. + + - Turn ALL methods not intended to be used outside, private. + + +* Revamp the factorization. + + It's not unusual to find "factorized" code + for bad reasons: because it made the code /look/ nicer, but the + factorized function/methods is actually called from ONE place. While it + might locally help, such practice globally defeat the purpose because + we lose the view of what is true factorized code and what is not. + + +* Namespace the factorized code. + + If a method require a local function, DON'T USE yet another method. Use a + local namespaced function.:: + + class BLah(object): + def _internal_method(self, arg): + def local_factorized(local_arg): + # local_factorized's code + # _internal_method's code. + + Python allows local namespaced functions for good reasons. + + +* Better inheritance policy. + + Take the sample of the folder/LocalStatus(SQlite) and folder/Base stuffs. It's + *nearly IMPOSSIBLE* to know and understand what parent method is used by what + child, for what purpose, etc. So, instead of (re)defining methods in the wild, + keep the well common NON-redefined stuff into the parent and define the + required methods in the childs. We really don't want anything like:: + + def method(self): + raise NotImplemented + + While this is common practice in Python, think about that again: how a + parent object should know all the expected methods/accessors of all the + possible kind of childs? + + Inheritance is about factorizing, certainly **NOT** about **defining the + interface** of the childs. + + +* Introduce as many as intermediate inherited objects as required. + + Keeping linear inheritance is good because Python sucks at playing + with multiple parents and it keeps things simple. But a parent should + have ALL its methods used in ALL the childs. If not, it's a good + sign that a new intermediate object should be introduced in the + inheritance line. + +* Don't blindly inherit from library objects. + + We do want **well defined interfaces**. For example, we do too much things + like imapobj.methodcall() while the imapobj is far inherited from imaplib2. + + We have NO clue about what we currently use from the library. + Having a dump wrappper for each call should be made mandatory for + objects inherited from a library. Using composed objects should be + seriously considered in this case, instead of using inheritance. + +* Use factories. + + Current objects do too much initialization stuff varying with the context it + is used. Move things like that into factories and keep the objects definitions + clean. + + +* Make it clear when we expect a composite object and what we expect + exactly. + + Even the more obvious composed objects are badly defined. For example, + the ``conf`` instances are spread across a lot of objects. Did you know + that such composed objects are sometimes restricted to the section the + object works on, and most of the time it's not restricted at all? + How many time it requires to find and understand on what we are + currently working? + + +* Seriously improve our debugging/hacking sessions (AGAIN). + + Until now, we have limited the improvements to allow better/full stack traces. + While this was actually required, we now hit some limitations of the whole + exception-based paradigm. For example, it's very HARD to follow an instance + during its life time. I have a good overview of what we could do in this area, + so don't matter much about that if you don't get the point or what could be + done. + + +* Support Python 3. + + +* Support Unicode. From 846ebeb2aa90f288364f9ab7bc23a21e2f07f89b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 7 Mar 2015 20:05:17 +0100 Subject: [PATCH 730/817] docs: GitAdvanced: write the "sending pull request" section Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 2 ++ docs/doc-src/GitAdvanced.rst | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 66ac2f4..af4ab6b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,4 +1,6 @@ .. -*- coding: utf-8 -*- +.. vim: spelllang=en ts=2 expandtab: + .. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap .. _Github: https://github.com/OfflineIMAP/offlineimap .. _repository: git://github.com/OfflineIMAP/offlineimap.git diff --git a/docs/doc-src/GitAdvanced.rst b/docs/doc-src/GitAdvanced.rst index 17686c4..402f68d 100644 --- a/docs/doc-src/GitAdvanced.rst +++ b/docs/doc-src/GitAdvanced.rst @@ -1,4 +1,6 @@ .. -*- coding: utf-8 -*- +.. vim: spelllang=en ts=2 expandtab: + .. _OfflineIMAP: http://offlineimap.org .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _Developers's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst @@ -104,7 +106,10 @@ Create commits Make a pull request ------------------- -TODO. +* Push your changes to a topic branch in your public fork of OfflineIMAP. +* Submit a pull request to the OfflineIMAP_ maintainers. +* If a ticket is open in the issues, add a comment with the link to your pull + request. Export commits as patches From 25513e90387f60831b94c6041de08db5ba12e95f Mon Sep 17 00:00:00 2001 From: Janna Martl Date: Sat, 7 Mar 2015 04:50:33 -0500 Subject: [PATCH 731/817] fix: don't loose local mails because of maxage Suppose messages A and B were delivered to the remote folder at "maxage + 1" days ago. A was downloaded to the local folder "maxage + 1" days ago, but B was only downloaded "maxage - 1" days ago (contrived scenario to illustrate the two things that could happen). The behavior was that B gets deleted from the local folder, but A did not. The expected behavior is that neither is deleted. Starting where Base.py: __syncmessagesto_delete(self, dstfolder, statusfolder) is called where: - self is the remote folder and - dstfolder is the local folder. It defines deletelist to be the list of messages in the status folder messagelist that aren't in the remote folder messagelist with not self.uidexists(uid) A and B are both in the status folder. They're also both *NOT* in the remote folder messagelist: this list is formed in IMAP.py: cachemessagelist(), which calls _msgs_to_fetch(), which only asks the IMAP server for messages that are "< maxage" days old. Back to Base.py __syncmessagesto_delete(), look at the call folder.deletemessages(deletelist), where folder is the local folder. This ends up calling Maildir.py deletemessage() for each message on the deletelist. But we see that this methods returns (instead of deleting anything) if the message is in the local folder's messagelist. This messagelist was created by Maildir.py's cachemessagelist(), which calls _scanfolder(), which tries to exclude messages based on maxage. So at this point, we *WANT* A and B to be excluded -- then they will be spared from deletion. This maxage check calls _iswithinmaxage(), and actually does the date comparison based on the time found at the beginning of the message's filename. These filenames were originally created in Maildir.py's new_message_filename(), which calls _gettimeseq() to get the current time (i.e. the time of retrieval). Upshot: A's filename has an older timestamp than B's filename. A is excluded from the local folder messagelist in _scanfolder(), hence spared from deletion in deletemessage(); B is not excluded, and is deleted. This patch does not address the timezone issue. As for the IMAP/timezone issue, a similar issue is discussed in the thunderbird bug tracker here: https://bugzilla.mozilla.org/show_bug.cgi?id=886534 In the end, they're solving a different problem, but they agree that there is really no reliable way of guessing the IMAP server's internal timezone. Signed-off-by: Janna Martl Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 69 +++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 4bbfe7e..77f7ebb 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -31,7 +31,7 @@ try: # python 2.6 has set() built in except NameError: from sets import Set as set -from offlineimap import OfflineImapError +from offlineimap import OfflineImapError, emailutil # Find the UID in a message filename re_uidmatch = re.compile(',U=(\d+)') @@ -73,9 +73,9 @@ class MaildirFolder(BaseFolder): #self.ui is set in BaseFolder.init() # Everything up to the first comma or colon (or ! if Windows): self.re_prefixmatch = re.compile('([^'+ self.infosep + ',]*)') - #folder's md, so we can match with recorded file md5 for validity + # folder's md, so we can match with recorded file md5 for validity. self._foldermd5 = md5(self.getvisiblename()).hexdigest() - # Cache the full folder path, as we use getfullname() very often + # Cache the full folder path, as we use getfullname() very often. self._fullname = os.path.join(self.getroot(), self.getname()) # Interface from BaseFolder @@ -91,12 +91,12 @@ class MaildirFolder(BaseFolder): token.""" return 42 - #Checks to see if the given message is within the maximum age according - #to the maildir name which should begin with a timestamp + # Checks to see if the given message is within the maximum age according + # to the maildir name which should begin with a timestamp def _iswithinmaxage(self, messagename, maxage): - #In order to have the same behaviour as SINCE in an IMAP search - #we must convert this to the oldest time and then strip off hrs/mins - #from that day + # In order to have the same behaviour as SINCE in an IMAP search + # we must convert this to the oldest time and then strip off hrs/mins + # from that day. oldest_time_utc = time.time() - (60*60*24*maxage) oldest_time_struct = time.gmtime(oldest_time_utc) oldest_time_today_seconds = ((oldest_time_struct[3] * 3600) \ @@ -173,7 +173,7 @@ class MaildirFolder(BaseFolder): for dirannex, filename in files: # We store just dirannex and filename, ie 'cur/123...' filepath = os.path.join(dirannex, filename) - # check maxage/maxsize if this message should be considered + # Check maxage/maxsize if this message should be considered. if maxage and not self._iswithinmaxage(filename, maxage): continue if maxsize and (os.path.getsize(os.path.join( @@ -181,7 +181,7 @@ class MaildirFolder(BaseFolder): continue (prefix, uid, fmd5, flags) = self._parse_filename(filename) - if uid is None: # assign negative uid to upload it. + if uid is None: # Assign negative uid to upload it. uid = nouidcounter nouidcounter -= 1 else: # It comes from our folder. @@ -202,22 +202,21 @@ class MaildirFolder(BaseFolder): def quickchanged(self, statusfolder): """Returns True if the Maildir has changed""" self.cachemessagelist() - # Folder has different uids than statusfolder => TRUE + # Folder has different uids than statusfolder => TRUE. if sorted(self.getmessageuidlist()) != \ sorted(statusfolder.getmessageuidlist()): return True - # Also check for flag changes, it's quick on a Maildir + # Also check for flag changes, it's quick on a Maildir. for (uid, message) in self.getmessagelist().iteritems(): if message['flags'] != statusfolder.getmessageflags(uid): return True - return False #Nope, nothing changed + return False # Nope, nothing changed. # Interface from BaseFolder def msglist_item_initializer(self, uid): return {'flags': set(), 'filename': '/no-dir/no-such-file/'} - # Interface from BaseFolder def cachemessagelist(self): if self.ismessagelistempty(): @@ -246,16 +245,35 @@ class MaildirFolder(BaseFolder): filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) - def new_message_filename(self, uid, flags=set()): + def new_message_filename(self, uid, flags=set(), rtime=None): """Creates a new unique Maildir filename :param uid: The UID`None`, or a set of maildir flags :param flags: A set of maildir flags :returns: String containing unique message filename""" - timeval, timeseq = _gettimeseq() - return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \ - (timeval, timeseq, os.getpid(), socket.gethostname(), + # Prefer internal time (rtime) over retrieval time (_gettimeseq()) for + # consistency in the maxage check, which compares times of local mail + # (gotten from the filename) to the internal time of the remote mail. + if rtime is None: + timeval, timeseq = _gettimeseq() + else: + timeval, timeseq = rtime, 0 + + def build_filename(timeval, timeseq, pid, hostname, uid, folder_id, + infosep, flags): + """Compute a new filename with 'time sequence' uniqueness in mind.""" + filename = '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% ( + timeval, timeseq, pid, hostname, uid, folder_id, infosep, flags) + for imap_dir in ['cur', 'new', 'tmp']: + if os.path.exists(os.path.join(self.getfullname(), imap_dir, + filename)): + # Increase timeseq. + return build_filename(timeval, timeseq + 1, pid, hostname, + uid, folder_id, infosep, flags) + return filename + + return build_filename(timeval, timeseq, os.getpid(), socket.gethostname(), uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) @@ -285,14 +303,14 @@ class MaildirFolder(BaseFolder): time.sleep(0.23) continue severity = OfflineImapError.ERROR.MESSAGE - raise OfflineImapError("Unique filename %s already exists." % \ - filename, severity), None, exc_info()[2] + raise OfflineImapError("Unique filename %s already exists."% + filename, severity), None, exc_info()[2] else: raise fd = os.fdopen(fd, 'wt') fd.write(content) - # Make sure the data hits the disk + # Make sure the data hits the disk. fd.flush() if self.dofsync: os.fsync(fd) @@ -323,7 +341,10 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - messagename = self.new_message_filename(uid, flags) + # Make sure rtime is the internal date, so it can be put in the filename + if rtime is None: + rtime = emailutil.get_message_date(content) + messagename = self.new_message_filename(uid, flags, rtime=rtime) tmpname = self.save_to_tmp_file(messagename, content) if rtime != None: os.utime(os.path.join(self.getfullname(), tmpname), (rtime, rtime)) @@ -399,8 +420,10 @@ class MaildirFolder(BaseFolder): oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) flags = self.getmessageflags(uid) + content = self.getmessage(uid) + rtime = emailutil.get_message_date(content) newfilename = os.path.join(dir_prefix, - self.new_message_filename(new_uid, flags)) + self.new_message_filename(new_uid, flags, rtime=rtime)) os.rename(os.path.join(self.getfullname(), oldfilename), os.path.join(self.getfullname(), newfilename)) self.messagelist[new_uid] = self.messagelist[uid] From 171a7a07972fec0ca0485ff9e598f094fee14a49 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 04:00:03 +0100 Subject: [PATCH 732/817] allow fetching the website in our git work tree Signed-off-by: Nicolas Sebrecht --- .gitignore | 14 ++++++++------ get-website.sh | 9 +++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) create mode 100755 get-website.sh diff --git a/.gitignore b/.gitignore index c5af353..235f5fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ -# Generated files -/docs/dev-doc/ -/build/ -*.pyc -offlineimap.1 -# backups +# Backups. .*.swp .*.swo *.html *~ +# website. +/website/ +# Generated files. +/docs/dev-doc/ +/build/ +*.pyc +offlineimap.1 diff --git a/get-website.sh b/get-website.sh new file mode 100755 index 0000000..fe6e9f7 --- /dev/null +++ b/get-website.sh @@ -0,0 +1,9 @@ +#!/bin/zsh + +if test -d website +then + echo "Directory 'website' already exists..." + exit 1 +else + git clone TODO +fi From bbc84ead0f9d6f86bbe29307d95d15bd66e0d6bf Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 03:54:12 +0100 Subject: [PATCH 733/817] move some documentation to the website Signed-off-by: Nicolas Sebrecht --- docs/INSTALL.rst | 161 ----------- docs/doc-src/FAQ.rst | 516 ----------------------------------- docs/doc-src/INSTALL.rst | 1 - docs/doc-src/index.rst | 46 +--- docs/doc-src/nametrans.rst | 233 ---------------- docs/doc-src/offlineimap.rst | 92 ------- 6 files changed, 5 insertions(+), 1044 deletions(-) delete mode 100644 docs/INSTALL.rst delete mode 100644 docs/doc-src/FAQ.rst delete mode 120000 docs/doc-src/INSTALL.rst delete mode 100644 docs/doc-src/nametrans.rst delete mode 100644 docs/doc-src/offlineimap.rst diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst deleted file mode 100644 index d4670c5..0000000 --- a/docs/INSTALL.rst +++ /dev/null @@ -1,161 +0,0 @@ -.. -*- coding: utf-8 -*- -.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap -.. _OLI_git_repo: git://github.com/OfflineIMAP/offlineimap.git - -============ -Installation -============ - -.. contents:: -.. .. sectnum:: - ------------------ -Development state ------------------ - -Since several months, the official maintainers Nicolas Sebrecht and Sebatian -Spaeth are too much busy to actively contribute to this project. - -In order to preserve contributions, a team of official maintainers have been -promoted to have write access to the official repository at `OfflineIMAP`_. -OfflineIMAP is now maintained by occasional contributors and the official -maintainers. All the documentation links might not be up-to-date to reflect -this change. - -The best place to get the latest news about the development state is at the -mailing list. - -------------- -Prerequisites -------------- - -In order to use `OfflineIMAP`_, you need to have these conditions satisfied: - -1. Your mail server must support IMAP. Mail access via POP is not - supported. A special Gmail mailbox type is available to interface - with Gmail's IMAP front-end, although Gmail has a very peculiar and - non-standard implementation of its IMAP interface. - -2. You must have Python version 2.6 or above installed. If you are - running on Debian GNU/Linux, this requirement will automatically be - taken care of for you. If you intend to use the SSL interface, - your Python must have been built with SSL support. - -3. If you use OfflineImap as an IMAP<->Maildir synchronizer, you will - obviously need to have a mail reader that supports the Maildir - mailbox format. Most modern mail readers have this support built-in, - so you can choose from a wide variety of mail servers. This format - is also known as the "qmail" format, so any mail reader compatible - with it will work with `OfflineIMAP`_. - - ------------- -Installation ------------- - -Installing OfflineImap should usually be quite easy, as you can simply unpack -and run OfflineImap in place if you wish to do so. There are a number of options -though: - -#. system-wide :ref:`installation via your distribution package manager ` -#. system-wide or single user :ref:`installation from the source package ` -#. system-wide or single user :ref:`installation from a git checkout ` - -Having installed OfflineImap, you will need to configure it, to be actually -useful. Please check the :ref:`Configuration` section in the :doc:`MANUAL` for -more information on the configuration step. - -.. _inst_pkg_man: - -System-Wide Installation via distribution -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The easiest way to install OfflineIMAP is via your distribution's package -manager. OfflineImap is available under the name `offlineimap` in most Linux and -BSD distributions. - - -.. _inst_src_tar: - -Installation from source package -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Download the latest source archive from our `download page -`_. Simply click the "Download -as .zip" or "Download as .tar.gz" buttons to get the latest "stable" code from -the master branch. If you prefer command line, you will want to use: wget -https://github.com/spaetz/offlineimap/tarball/master - -Unpack and continue with the :ref:`system-wide installation ` -or the :ref:`single-user installation ` section. - - -.. _inst_git: - -Installation from git checkout -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Get your own copy of the `official git repository `_ at `OfflineIMAP`_:: - - git clone https://github.com/OfflineIMAP/offlineimap.git - -This will download the source with history. By default, git sets up the -`master` branch up, which is most likely what you want. If not, you can -checkout a particular release like this:: - - cd offlineimap - git checkout v6.5.2.1 - -You have now a source tree available and proceed with either the -:ref:`system-wide installation ` or the :ref:`single-user -installation `. - - -.. _system_wide_inst: - -System-wide installation -++++++++++++++++++++++++ - -Then run these commands, to build the python package:: - - make clean - make - -Finally, install the program (as root):: - - python setup.py install - -Next, proceed to below. Type `offlineimap` to invoke the program. - - -.. _single_user_inst: - -Single-user installation -++++++++++++++++++++++++ - -Download the git repository as described above. Instead of installing the -program as root, you type `./offlineimap.py`; there is no installation step -necessary. - ---------- -Uninstall ---------- - -If you installed a system-wide installation via "python setup.py -install", there are a few files to purge to cleanly uninstall -`OfflineImap`_ again. Assuming that `/usr/local` is the standard prefix of -your system and that you use python 2.7, you need to: - -#) Delete the OfflineImap installation itself:: - - /usr/local/lib/python2.7/dist-packages/offlineimap-6.4.4.egg-info - /usr/local/lib/python2.7/dist-packages/offlineimap - - In case, you did the single-user installation, simply delete your - offlineimap directory. - -#) Delete all files that OfflineImap creates during its operation. - - The cache at (default location) ~/.offlineimap - - Your manually created (default loc) ~/.offlineimaprc - (It is possible that you created those in different spots) - -That's it. Have fun without OfflineImap. diff --git a/docs/doc-src/FAQ.rst b/docs/doc-src/FAQ.rst deleted file mode 100644 index 49fa140..0000000 --- a/docs/doc-src/FAQ.rst +++ /dev/null @@ -1,516 +0,0 @@ -.. -*- coding: utf-8 -*- - -.. NOTE TO MAINTAINERS: Please add new questions to the end of their - sections, so section/question numbers remain stable. - -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project -.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap -.. _ssl.wrap_socket: http://docs.python.org/library/ssl.html#ssl.wrap_socket -.. _Advanced Git: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/GitAdvanced.rst -.. _FAQ: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/FAQ.rst -.. _Contributing: https://github.com/OfflineIMAP/offlineimap/blob/next/CONTRIBUTING.rst - - - - -============================================= - OfflineIMAP FAQ (Frequently Asked Questions) -============================================= - -:Web site: https://github.com/nicolas33/offlineimap -:Copyright: This document is licensed under GPLv2. - -.. contents:: -.. sectnum:: - - -Please feel free to ask questions and/or provide answers; send email to the -`mailing list`_. - -Most recent `FAQ`_. - - -OfflineIMAP -=========== - -Where do I get OfflineIMAP? ---------------------------- - -See the information on the Home page `OfflineIMAP`_. - -How fast is it? ---------------- - -OfflineIMAP has a multithreaded sync, so it should have very nice performance. - -OfflineIMAP versions 2.0 and above contain a multithreaded system. A good way -to experiment is by setting maxsyncaccounts to 3 and maxconnections to 3 in -each account clause. - -This lets OfflineIMAP open up multiple connections simultaneously. That will -let it process multiple folders and messages at once. In most cases, this will -increase performance of the sync. - -Don’t set the number too high. If you do that, things might actually slow down -as your link gets saturated. Also, too many connections can cause mail servers -to have excessive load. Administrators might take unkindly to this, and the -server might bog down. There are many variables in the optimal setting; experimentation may help. - -See the Performance section in the MANUAL for some tips. - -What platforms does OfflineIMAP support? ----------------------------------------- - -It should run on most platforms supported by Python, with one exception: we do not support Windows, but some have made it work there. - -The following has been reported by OfflineIMAP users. We do not test -OfflineIMAP on Windows, so we can’t directly address their accuracy. - -The basic answer is that it’s possible and doesn’t require hacking OfflineIMAP -source code. However, it’s not necessarily trivial. The information below is -based in instructions submitted by Chris Walker:: - - First, you must run OfflineIMAP in the Cygwin environment. The Windows - filesystem is not powerful enough to accomodate Maildir by itself. - - Next, you’ll need to mount your Maildir directory in a special - way. There is information for doing that at - http://barnson.org/node/295. That site gives this example:: - - mount -f -s -b -o managed "d:/tmp/mail" "/home/of/mail" - - That URL also has more details on making OfflineIMAP work with Windows. - - -Does OfflineIMAP supports XDG Base Directory specification? ------------------------------------------------------------ - -Yes. We are trying to use `$XDG_CONFIG_HOME/offlineimap/config` -as the primary configuration file, falling back to `~/.offlineimaprc` -if configuration file location was not explicitely specified at the -command line. - - -Does OfflineIMAP support mbox, mh, or anything else other than Maildir? ------------------------------------------------------------------------ - -Not directly. Maildir was the easiest to implement. We are not planning -to write an mbox-backend, though if someone sent me well-written mbox -support and pledged to support it, it would be committed it to the tree. - -However, OfflineIMAP can directly sync accounts on two different IMAP servers -together. So you could install an IMAP server on your local machine that -supports mbox, sync to it, and then instruct your mail readers to use the -mboxes. - -Or you could install whatever IMAP server you like on the local machine, and -point your mail readers to that IMAP server on localhost. - -What is the UID validity problem for folder? --------------------------------------------- - -IMAP servers use a folders UIDVALIDITY value in combination with a -unique ID (UID) to refer to a specific message. This is guaranteed to -be unique to a particular message forever. No other message in the same -folder will ever get the same UID as long as UIDVALIDITY remains -unchanged. UIDs are an integral part of `OfflineIMAP`_'s -synchronization scheme; they are used to match up messages on your -computer to messages on the server. - -Sometimes, the UIDs on the server might get reset. Usually this will -happen if you delete and then recreate a folder. When you create a -folder, the server will often start the UID back from 1. But -`OfflineIMAP`_ might still have the UIDs from the previous folder by the -same name stored. `OfflineIMAP`_ will detect this condition because of -the changed UIDVALIDITY value and skip the folder. This is GOOD, -because it prevents data loss. - -In the IMAP<->Maildir case, you can fix it by removing your local folder -and cache data. For instance, if your folders are under `~/Folders` and -the folder with the problem is INBOX, you'd type this:: - - rm -r ~/Folders/INBOX - rm -r ~/.offlineimap/Account-AccountName/LocalStatus/INBOX - rm -r ~/.offlineimap/Repository-RemoteRepositoryName/FolderValidity/INBOX - -(Of course, replace AccountName and RemoteRepositoryName with the names as -specified in `~/.offlineimaprc`). - -Next time you run `OfflineIMAP`_, it will re-download the folder with the new -UIDs. Note that the procedure specified above will lose any local changes made -to the folder. - -Some IMAP servers are broken and do not support UIDs properly. If you continue -to get this error for all your folders even after performing the above -procedure, it is likely that your IMAP server falls into this category. -`OfflineIMAP`_ is incompatible with such servers. Using `OfflineIMAP`_ with -them will not destroy any mail, but at the same time, it will not actually -synchronize it either. (`OfflineIMAP`_ will detect this condition and abort -prior to synchronization.) - - -This question comes up frequently on the `mailing list`_. You can find a detailed -discussion of the problem there -http://lists.complete.org/offlineimap@complete.org/2003/04/msg00012.html.gz. - -How do I automatically delete a folder? ---------------------------------------- - -OfflineIMAP does not currently provide this feature. You will have to delete folders manually. See next entry too. - -May I delete local folders? ---------------------------- - -`OfflineIMAP`_ does a two-way synchronization. That is, if you make a change -to the mail on the server, it will be propagated to your local copy, and -vise-versa. Some people might think that it would be wise to just delete all -their local mail folders periodically. If you do this with `OfflineIMAP`_, -remember to also remove your local status cache (`~/.offlineimap` by default). -Otherwise, `OfflineIMAP`_ will take this as an intentional deletion of many -messages and will interpret your action as requesting them to be deleted from -the server as well. (If you don't understand this, don't worry; you probably -won't encounter this situation.) - -Can I run multiple instances? ------------------------------ - -`OfflineIMAP`_ is not designed to have several instances (for instance, a cron -job and an interactive invocation) run over the same mailbox simultaneously. -It will perform a check on startup and abort if another `OfflineIMAP`_ is -already running. If you need to schedule synchronizations, you'll probably -find autorefresh settings more convenient than cron. Alternatively, you can -set a separate metadata directory for each instance. -In the future, we will lock each account individually rather than having one global lock. - -Can I copy messages between folders? ---------------------------------------- - -Normally, when you copy a message between folders or add a new message to a -folder locally, `OfflineIMAP`_ will just do the right thing. However, -sometimes this can be tricky ― if your IMAP server does not provide the SEARCH -command, or does not return something useful, `OfflineIMAP`_ cannot determine -the new UID of the message. So, in these rare instances, OfflineIMAP will -upload the message to the IMAP server and delete it from your local folder. -Then, on your next sync, the message will be re-downloaded with the proper UID. -`OfflineIMAP`_ makes sure that the message was properly uploaded before -deleting it, so there should be no risk of data loss. - -But if you try to sync between two IMAP servers, where both are unable to -provide you with UID of the new message, then this will lead to infinite loop. -`OfflineIMAP`_ will upload the message to one server and delete on second. On -next run it will upload the message to second server and delete on first, etc. - -Does OfflineIMAP support POP? ------------------------------ - -No. - -How is OfflineIMAP conformance? -------------------------------- - -* Internet Message Access Protocol version 4rev1 (IMAP 4rev1) as specified in - `2060`:RFC: and `3501`:RFC: -* CRAM-MD5 as specified in `2195`:RFC: -* Maildir as specified in the Maildir manpage and the qmail website. -* Standard Python 2.7 as implemented on POSIX-compliant systems. - -Can I force OfflineIMAP to sync a folder right now? ---------------------------------------------------- - -Yes: - -1) if you use the `Blinkenlights` UI. That UI shows the active -accounts as follows:: - - 4: [active] *Control: . - 3: [ 4:36] personal: - 2: [ 3:37] work: . - 1: [ 6:28] uni: - - Simply press the appropriate digit (`3` for `personal`, etc.) to - resync that account immediately. This will be ignored if a resync is - already in progress for that account. - -2) While in sleep mode, you can also send a SIGUSR1. See the :ref:`UNIX - signals` section in the MANUAL for details. - - -I get a "Mailbox already exists" error --------------------------------------- - -**Q:** When synchronizing, I receive errors such as:: - - Folder 'sent'[main-remote] could not be created. Server responded: - ('NO', ['Mailbox already exists.']) - -**A:** IMAP folders are usually case sensitive. But some IMAP servers seem - to treat "special" folders as case insensitive (e.g. the initial - INBOX. part, or folders such as "Sent" or "Trash"). If you happen to - have a folder "sent" on one side of things and a folder called "Sent" - on the other side, OfflineIMAP will try to create those folders on - both sides. If you server happens to treat those folders as - case-insensitive you can then see this warning. - - You can solve this by excluding the "sent" folder by filtering it from - the repository settings:: - - folderfilter= lambda f: f not in ['sent'] - - -Configuration Questions -======================= - -Can I synchronize multiple accounts with OfflineIMAP? ------------------------------------------------------ - -Of course! - -Just name them all in the accounts line in the general section of the -configuration file, and add a per-account section for each one. - -You can also optionally use the -a option when you run OfflineIMAP to request -that it only operate upon a subset of the accounts for a particular run. - -How do I specify the names of folders? --------------------------------------- - -You do not need to. OfflineIMAP is smart enough to automatically figure out -what folders are present on the IMAP server and synchronize them. You can use -the folderfilter and nametrans configuration file options to request only -certain folders and rename them as they come in if you like. - -Also you can configure OfflineImap to only synchronize "subscribed" folders. - -How do I prevent certain folders from being synced? ---------------------------------------------------- - -Use the folderfilter option. See the MANUAL for details and examples. - -What is the mailbox name recorder (mbnames) for? ------------------------------------------------- - -Some mail readers, such as mutt, are not capable of automatically determining the names of your mailboxes. OfflineIMAP can help these programs by writing the names of the folders in a format you specify. See the example offlineimap.conf for details. - -Does OfflineIMAP verify SSL certificates? ------------------------------------------ - -You can verify an imapserver's certificate by specifying the CA -certificate on a per-repository basis by setting the `sslcacertfile` -option in the config file. (See the example offlineimap.conf for -details.) If you do not specify any CA certificate, you will be presented with the server's certificate fingerprint and add that to the configuration file, to make sure it remains unchanged. -No verification happens if connecting via STARTTLS. - -How do I generate an `sslcacertfile` file? ------------------------------------------- - -The `sslcacertfile` file must contain an SSL certificate (or a concatenated -certificates chain) in PEM format. (See the documentation of -`ssl.wrap_socket`_'s `certfile` parameter for the gory details.) You can use either openssl or gnutls to create a certificate file in the required format. - -#. via openssl:: - - openssl s_client -CApath /etc/ssl/certs -connect ${hostname}:imaps -showcerts \ - | perl -ne 'print if /BEGIN/../END/; print STDERR if /return/' > $sslcacertfile - ^D - - -#. via gnutls:: - gnutls-cli --print-cert -p imaps ${host} $sslcacertfile - - -The path `/etc/ssl/certs` is not standardized; your system may store -SSL certificates elsewhere. (On some systems it may be in -`/usr/local/share/certs/`.) - -Before using the resulting file, ensure that openssl verified the certificate -successfully. In case of problems, you can test the certificate using a command such as (credits to Daniel Shahaf for this) to verify the certificate:: - - % openssl s_client -CAfile $sslcacertfile -connect ${hostname}:imaps 2>&1 ` for those interested in modifying the source code -or otherwise peek into the OfflineImap internals in a second part. - - -If you just want to get started with minimal fuzz, have a look at our `online -quick start guide `_. Do note though, -that our configuration options are many and powerful. Perusing our precious -documentation does often pay off! - -More information on specific topics can be found on the following pages: - -**User documentation** - * :doc:`Overview and features ` - * :doc:`installation/uninstall ` +Welcome to OfflineIMAP's documentation +====================================== **Configuration** * :doc:`user manual/Configuration ` * :doc:`Folder filtering & name transformation guide ` - * :doc:`maxage ` * :doc:`command line options ` - * :doc:`Frequently Asked Questions ` **Developer documentation** - * :doc:`Contributing ` * :doc:`Advanced Git ` * :doc:`API documentation ` for internal details on the :mod:`offlineimap` module +**OfflineIMAP APIs** + .. toctree:: - :hidden: - - features - INSTALL - MANUAL - nametrans - advanced_config - offlineimap - FAQ - - CONTRIBUTING - GitAdvanced API repository ui diff --git a/docs/doc-src/nametrans.rst b/docs/doc-src/nametrans.rst deleted file mode 100644 index f802c05..0000000 --- a/docs/doc-src/nametrans.rst +++ /dev/null @@ -1,233 +0,0 @@ -.. _folder_filtering_and_name_translation: - -Folder filtering and Name translation -===================================== - -OfflineImap provides advanced and potentially complex possibilities for -filtering and translating folder names. If you don't need any of this, you can -safely skip this section. - -.. warning:: - Starting with v6.4.0, OfflineImap supports the creation of folders on the remote repostory. This change means that people that only had a nametrans option on the remote repository (everyone) will need to have a nametrans setting on the local repository too that will reverse the name transformation. See section `Reverse nametrans`_ for details. - -folderfilter ------------- - -If you do not want to synchronize all your folders, you can specify a -`folderfilter`_ function that determines which folders to include in a sync and -which to exclude. Typically, you would set a folderfilter option on the remote -repository only, and it would be a lambda or any other python function. - -The only parameter to that function is the folder name. If the filter -function returns True, the folder will be synced, if it returns False, -it. will be skipped. The folderfilter operates on the *UNTRANSLATED* -name (before any `nametrans`_ fudging takes place). Consider the -examples below to get an idea of what they do. - -Example 1: synchronizing only INBOX and Sent:: - - folderfilter = lambda folder: folder in ['INBOX', 'Sent'] - -Example 2: synchronizing everything except Trash:: - - folderfilter = lambda folder: folder not in ['Trash'] - -Example 3: Using a regular expression to exclude Trash and all folders -containing the characters "Del":: - - folderfilter = lambda folder: not re.search('(^Trash$|Del)', folder) - -.. note:: - If folderfilter is not specified, ALL remote folders will be - synchronized. - -You can span multiple lines by indenting the others. (Use backslashes -at the end when required by Python syntax) For instance:: - - folderfilter = lambda foldername: foldername in - ['INBOX', 'Sent Mail', 'Deleted Items', - 'Received'] - -Usually it suffices to put a `folderfilter`_ setting in the remote repository -section. You might want to put a folderfilter option on the local repository if -you want to prevent some folders on the local repository to be created on the -remote one. (Even in this case, folder filters on the remote repository will -prevent that) - -folderincludes --------------- - -You can specify `folderincludes`_ to manually include additional folders to be -synced, even if they had been filtered out by a folderfilter setting. -`folderincludes`_ should return a Python list. - -This can be used to 1) add a folder that was excluded by your -folderfilter rule, 2) to include a folder that your server does not specify -with its LIST option, or 3) to include a folder that is outside your basic -`reference`. The `reference` value will not be prefixed to this folder -name, even if you have specified one. For example:: - - folderincludes = ['debian.user', 'debian.personal'] - -This will add the "debian.user" and "debian.personal" folders even if you -have filtered out everything starting with "debian" in your folderfilter -settings. - - -createfolders -------------- - -By default OfflineImap propagates new folders in both -directions. Sometimes this is not what you want. E.g. you might want -new folders on your IMAP server to propagate to your local MailDir, -but not the other way around. The 'readonly' setting on a repository -will not help here, as it prevents any change from occuring on that -repository. This is what the `createfolders` setting is for. By -default it is `True`, meaning that new folders can be created on this -repository. To prevent folders from ever being created on a -repository, set this to `False`. If you set this to False on the -REMOTE repository, you will not have to create the `Reverse -nametrans`_ rules on the LOCAL repository. - - -nametrans ----------- - -Sometimes, folders need to have different names on the remote and the local -repositories. To achieve this you can specify a folder name translator. This -must be a eval-able Python expression that takes a foldername arg and returns -the new value. We suggest a lambda function, but it could be any python -function really. If you use nametrans rules, you will need to set them both on -the remote and the local repository, see `Reverse nametrans`_ just below for -details. The following examples are thought to be put in the remote repository -section. - -The below will remove "INBOX." from the leading edge of folders (great -for Courier IMAP users):: - - nametrans = lambda folder: re.sub('^INBOX\.', '', folder) - -Using Courier remotely and want to duplicate its mailbox naming -locally? Try this:: - - nametrans = lambda folder: re.sub('^INBOX\.*', '.', folder) - -.. warning:: - You MUST construct nametrans rules such that it NEVER returns the - same value for two folders, UNLESS the second values are filtered - out by folderfilter below. That is, two filters on one side may - never point to the same folder on the other side. Failure to follow - this rule will result in undefined behavior. See also *Sharing a - maildir with multiple IMAP servers* in the :ref:`pitfalls` section. - - -Reverse nametrans -+++++++++++++++++ - -Since 6.4.0, OfflineImap supports the creation of folders on the remote -repository and that complicates things. Previously, only one nametrans setting -on the remote repository was needed and that transformed a remote to a local -name. However, nametrans transformations are one-way, and OfflineImap has no way -using those rules on the remote repository to back local names to remote names. - -Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts off -any existing INBOX prefix. Now, if we parse a list of local folders, finding -e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We -have no way of knowing. This is why **every nametrans setting on a remote -repository requires an equivalent nametrans rule on the local repository that -reverses the transformation**. - -Take the above examples. If your remote nametrans setting was:: - - nametrans = lambda folder: re.sub('^INBOX\.', '', folder) - -then you will want to have this in your local repository, prepending "INBOX." to -any local folder name:: - - nametrans = lambda folder: 'INBOX.' + folder - -Failure to set the local nametrans rule will lead to weird-looking error -messages of -for instance- this type:: - - ERROR: Creating folder moo.foo on repository remote - Folder 'moo.foo'[remote] could not be created. Server responded: ('NO', ['Unknown namespace.']) - -(This indicates that you attempted to create a folder "Sent" when all remote -folders needed to be under the prefix of "INBOX."). - -OfflineImap will make some sanity checks if it needs to create a new -folder on the remote side and a back-and-forth nametrans-lation does not -yield the original foldername (as that could potentially lead to -infinite folder creation cycles). - -You can probably already see now that creating nametrans rules can be a pretty -daunting and complex endeavour. Check out the Use cases in the manual. If you -have some interesting use cases that we can present as examples here, please let -us know. - -Debugging folderfilter and nametrans ------------------------------------- - -Given the complexity of the functions and regexes involved, it is easy to -misconfigure things. One way to test your configuration without danger to -corrupt anything or to create unwanted folders is to invoke offlineimap with the -`--info` option. - -It will output a list of folders and their transformations on the screen (save -them to a file with -l info.log), and will help you to tweak your rules as well -as to understand your configuration. It also provides good output for bug -reporting. - -FAQ on nametrans ----------------- - -Where to put nametrans rules, on the remote and/or local repository? -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -If you never intend to create new folders on the LOCAL repository that -need to be synced to the REMOTE repository, it is sufficient to create a -nametrans rule on the remote Repository section. This will be used to -determine the names of new folder names on the LOCAL repository, and to -match existing folders that correspond. - -*IF* you create folders on the local repository, that are supposed to be - automatically created on the remote repository, you will need to create - a nametrans rule that provides the reverse name translation. - -(A nametrans rule provides only a one-way translation of names and in -order to know which names folders on the LOCAL side would have on the -REMOTE side, you need to specify the reverse nametrans rule on the local -repository) - -OfflineImap will complain if it needs to create a new folder on the -remote side and a back-and-forth nametrans-lation does not yield the -original foldername (as that could potentially lead to infinite folder -creation cycles). - -What folder separators do I need to use in nametrans rules? -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -**Q:** If I sync from an IMAP server with folder separator '/' to a - Maildir using the default folder separator '.' which do I need to use - in nametrans rules?:: - - nametrans = lambda f: "INBOX/" + f - - or:: - nametrans = lambda f: "INBOX." + f - -**A:** Generally use the folder separator as defined in the repository - you write the nametrans rule for. That is, use '/' in the above - case. We will pass in the untranslated name of the IMAP folder as - parameter (here `f`). The translated name will ultimately have all - folder separators be replaced with the destination repositories' - folder separator. - -So if 'f' was "Sent", the first nametrans yields the translated name -"INBOX/Sent" to be used on the other side. As that repository uses the -folder separator '.' rather than '/', the ultimate name to be used will -be "INBOX.Sent". - -(As a final note, the smart will see that both variants of the above -nametrans rule would have worked identically in this case) - diff --git a/docs/doc-src/offlineimap.rst b/docs/doc-src/offlineimap.rst deleted file mode 100644 index 37d78dc..0000000 --- a/docs/doc-src/offlineimap.rst +++ /dev/null @@ -1,92 +0,0 @@ -The offlineimap 'binary' command line options -============================================= - -Offlineimap is invoked with the following pattern: `offlineimap [args...]`. - -Where [args...] are as follows: - -Options: - --dry-run This mode protects us from performing any actual action. - It will not precisely give the exact information what - will happen. If e.g. it would need to create a folder, - it merely outputs "Would create folder X", but not how - many and which mails it would transfer. - --info Output information on the configured email - repositories. Useful for debugging and bug reporting. - Use in conjunction with the -a option to limit the - output to a single account. - --version show program's version number and exit - -h, --help show this help message and exit - -1 Disable all multithreading operations and use solely a - single-thread sync. This effectively sets the - maxsyncaccounts and all maxconnections configuration - file variables to 1. - -P DIR Sets OfflineIMAP into profile mode. The program will - create DIR (it must not already exist). As it runs, - Python profiling information about each thread is - logged into profiledir. Please note: This option is - present for debugging and optimization only, and - should NOT be used unless you have a specific reason - to do so. It will significantly slow program - performance, may reduce reliability, and can generate - huge amounts of data. This option implies the - singlethreading option (-1). - -a ACCOUNTS Overrides the accounts section in the config file. - Lets you specify a particular account or set of - accounts to sync without having to edit the config - file. You might use this to exclude certain accounts, - or to sync some accounts that you normally prefer not - to. - -c FILE Specifies a configuration file to use in lieu of - ~/.offlineimaprc. - - -d type1,[type2...] Enables debugging for OfflineIMAP. This is useful if - you are trying to track down a malfunction or figure - out what is going on under the hood. I suggest that - you use this with -1 in order to make the results more - sensible. This option requires one or more debugtypes, - separated by commas. These define what exactly will - be debugged, and so far include the options: imap, - thread,maildir or ALL. The imap option will enable - IMAP protocol stream and parsing debugging. Note that - the output may contain passwords, so take care to - remove that from the debugging output before sending - it to anyone else. The maildir option will enable - debugging for certain Maildir operations. - - -l FILE Log to FILE - - -f folder1,[folder2...] - Only sync the specified folders. The 'folder's are the - *untranslated* foldernames. This command-line option - overrides any 'folderfilter' and 'folderincludes' - options in the configuration file. - - -k `[section:]option=value` - Override configuration file option. If"section" is - omitted, it defaults to "general". Any underscores - "_" in the section name are replaced with spaces: - for instance, to override option "autorefresh" in - the "[Account Personal]" section in the config file - one would use "-k Account_Personal:autorefresh=30". - - -o Run only once, ignoring any autorefresh setting in the - configuration file. - -q Run only quick synchronizations. Ignore any flag - updates on IMAP servers. - -u INTERFACE Specifies an alternative user interface to use. This - overrides the default specified in the configuration - file. The UI specified with -u will be forced to be - used, even if checks determine that it is not usable. - Possible interface choices are: Curses.Blinkenlights, - TTY.TTYUI, Noninteractive.Basic, Noninteractive.Quiet, - Machine.MachineUI - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`search` - From d5cea5037016b85d04f81fe2e6a33d5f84247834 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 03:56:58 +0100 Subject: [PATCH 734/817] doc: move all sources to the same directory Signed-off-by: Nicolas Sebrecht --- docs/CodingGuidelines.rst | 51 ------------------------------ docs/doc-src/CodingGuidelines.rst | 52 ++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 52 deletions(-) delete mode 100644 docs/CodingGuidelines.rst mode change 120000 => 100644 docs/doc-src/CodingGuidelines.rst diff --git a/docs/CodingGuidelines.rst b/docs/CodingGuidelines.rst deleted file mode 100644 index 92cc3be..0000000 --- a/docs/CodingGuidelines.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. -*- coding: utf-8 -*- -.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap -.. _OLI_git_repo: git://github.com/OfflineIMAP/offlineimap.git - -================================= -Coding guidelines for OfflineIMAP -================================= - -.. contents:: -.. .. sectnum:: - -This document contains assorted guidelines for programmers that want -to hack OfflineIMAP. - - ------------------- -Exception handling ------------------- - -OfflineIMAP on many occasions re-raises various exceptions and often -changes exception type to `OfflineImapError`. This is not a problem -per se, but you must always remember that we need to preserve original -tracebacks. This is not hard if you follow these simple rules. - -For re-raising original exceptions, just use:: - - raise - -from inside your exception handling code. - -If you need to change exception type, or its argument, or whatever, -use this three-argument form:: - - raise YourExceptionClass(argum, ents), None, sys.exc_info()[2] - -In this form, you're creating an instance of new exception, so ``raise`` -will deduce its ``type`` and ``value`` parameters from the first argument, -thus the second expression passed to ``raise`` is always ``None``. -And the third one is the traceback object obtained from the thread-safe -``exc_info()`` function. - -In fact, if you hadn't already imported the whole ``sys`` module, it will -be better to import just ``exc_info()``:: - - from sys import exc_info - -and raise like this:: - - raise YourExceptionClass(argum, ents), None, exc_info()[2] - -since this is the historically-preferred style in the OfflineIMAP code. diff --git a/docs/doc-src/CodingGuidelines.rst b/docs/doc-src/CodingGuidelines.rst deleted file mode 120000 index 7117956..0000000 --- a/docs/doc-src/CodingGuidelines.rst +++ /dev/null @@ -1 +0,0 @@ -../CodingGuidelines.rst \ No newline at end of file diff --git a/docs/doc-src/CodingGuidelines.rst b/docs/doc-src/CodingGuidelines.rst new file mode 100644 index 0000000..92cc3be --- /dev/null +++ b/docs/doc-src/CodingGuidelines.rst @@ -0,0 +1,51 @@ +.. -*- coding: utf-8 -*- +.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap +.. _OLI_git_repo: git://github.com/OfflineIMAP/offlineimap.git + +================================= +Coding guidelines for OfflineIMAP +================================= + +.. contents:: +.. .. sectnum:: + +This document contains assorted guidelines for programmers that want +to hack OfflineIMAP. + + +------------------ +Exception handling +------------------ + +OfflineIMAP on many occasions re-raises various exceptions and often +changes exception type to `OfflineImapError`. This is not a problem +per se, but you must always remember that we need to preserve original +tracebacks. This is not hard if you follow these simple rules. + +For re-raising original exceptions, just use:: + + raise + +from inside your exception handling code. + +If you need to change exception type, or its argument, or whatever, +use this three-argument form:: + + raise YourExceptionClass(argum, ents), None, sys.exc_info()[2] + +In this form, you're creating an instance of new exception, so ``raise`` +will deduce its ``type`` and ``value`` parameters from the first argument, +thus the second expression passed to ``raise`` is always ``None``. +And the third one is the traceback object obtained from the thread-safe +``exc_info()`` function. + +In fact, if you hadn't already imported the whole ``sys`` module, it will +be better to import just ``exc_info()``:: + + from sys import exc_info + +and raise like this:: + + raise YourExceptionClass(argum, ents), None, exc_info()[2] + +since this is the historically-preferred style in the OfflineIMAP code. From f46f08ecf8d9bb729d690e27dae682c60cb2ed12 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 04:01:22 +0100 Subject: [PATCH 735/817] README: deep cleanup Documentation refactorized. Signed-off-by: Nicolas Sebrecht --- README.md | 263 +++++++++++------------------------------------------- 1 file changed, 54 insertions(+), 209 deletions(-) diff --git a/README.md b/README.md index 7748f62..ef815c3 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,62 @@ -OfflineImap -=========== +[offlineimap]: https://github.com/OfflineIMAP/offlineimap +[website]: http://offlineimap.github.io +[wiki]: http://github.com/OfflineIMAP/offlineimap/wiki -Description ------------ +# OfflineImap -OfflineIMAP is a tool to simplify your e-mail reading. With OfflineIMAP, you can -read the same mailbox from multiple computers. You get a current copy of your -messages on each computer, and changes you make one place will be visible on all -other systems. For instance, you can delete a message on your home computer, and -it will appear deleted on your work computer as well. OfflineIMAP is also useful -if you want to use a mail reader that does not have IMAP support, has poor IMAP -support, or does not provide disconnected operation. It's homepage at -http://offlineimap.org contains more information, source code, and online -documentation. +## Description -OfflineIMAP does not require additional python dependencies beyond python >=2.6 -(although python-sqlite is strongly recommended). +OfflineIMAP is a software to dispose your e-mail mailbox(es) as a local Maildir. -OfflineIMAP is a Free Software project licensed under the GNU General -Public License version 2 (or later) with a special exception that allows -the OpenSSL library to be used. You can download it for free, and you -can modify it. In fact, you are encouraged to contribute to OfflineIMAP. +For example, this allows reading the mails while offline without the need for your mail reader (MUA) to support disconnected operations. -Documentation -------------- +OfflineIMAP will synchronize both sides via *IMAP*. + + +## License + +GNU General Public License v2. + + +## Why should I use OfflineIMAP? + +* It is **fast**. +* It is **reliable**. +* It is **flexible**. +* It is **safe**. + + +## Downloads + +You should first check if your distribution already package OfflineIMAP for you. +Downloads releases as [tarball or zipball](https://github.com/OfflineIMAP/offlineimap/tags). + + +## Feedbacks and contributions + +**The user discussions, development, announces and all the exciting stuff take +place in the mailing list.** While not mandatory to send emails, you can +[subscribe +here](http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project). + +Bugs, issues and contributions can be requested to both the mailing list or the +[official Github project][offlineimap]. + + +## The community + +* OfflineIMAP's main site is the [project page at Github][offlineimap]. +* There is the [OfflineIMAP community's website][website]. +* And finally, [the wiki][wiki]. + + +## Requirements + +* Python v2.7 +* Python SQlite (optional while recommended) + + +## Documentation The documentation is included (in .rst format) in the `docs` directory. Read it directly or generate nice html docs (python-sphinx needed) and/or @@ -36,191 +69,3 @@ the man page (python-docutils needed) while being in the `docs` dir via: The resulting user documentation will be in `docs/html`. The full user docs are also at: http://docs.offlineimap.org. Please see there for detailed information on how to install and configure OfflineImap. - -Quick Start -=========== - -First, install OfflineIMAP. See `docs/INSTALL.rst` or read - -(hint: `sudo python setup.py install`). - -Second, set up your configuration file and run it! The distribution -includes offlineimap.conf.minimal (Debian users may find this at -`/usr/share/doc/offlineimap/examples/offlineimap.conf.minimal`) that -provides you with the bare minimum of setting up OfflineIMAP. You can -simply copy this file into your home directory and name it -`.offlineimaprc`. A command such as `cp offlineimap.conf.minimal -~/.offlineimaprc` will do it. Or, if you prefer, you can just copy -this text to `~/.offlineimaprc`: - - [general] - accounts = Test - - [Account Test] - localrepository = Local - remoterepository = Remote - - [Repository Local] - type = Maildir - localfolders = ~/Test - - [Repository Remote] - type = IMAP - remotehost = examplehost - remoteuser = jgoerzen - - -Now, edit the `~/.offlineimaprc` file with your favorite editor. All you have -to do is specify a directory for your folders to be in (on the `localfolders` -line), the host name of your IMAP server (on the `remotehost` line), and your -login name on the remote (on the `remoteuser` line). That's it! - -If you prefer to be compatible with the [XDG Base Directory -spec](http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html), -then substitute the above `~/.offlineimaprc` with -`$XDG_CONFIG_HOME/offlineimap/config` and don't forget to set -`XDG_CONFIG_HOME` properly if you want it to be different from -the default `$HOME/.config` for any reason. - -To run OfflineIMAP, you just have to say `offlineimap` ― it will fire -up, ask you for a login password if necessary, synchronize your folders, -and exit. See? - -You can just throw away the rest of the finely-crafted, perfectly-honed user -manual! Of course, if you want to see how you can make OfflineIMAP -FIVE TIMES FASTER FOR JUST $19.95 (err, well, $0), you have to read on our -full user documentation and peruse the sample offlineimap.conf (which -includes all available options) for further tweaks! - - -Mailing list & bug reporting ----------------------------- - -The user discussion, development and all exciting stuff take place in the -OfflineImap mailing list at -. You do not -need to subscribe to send emails. - -Bugs, issues and contributions should be reported to the mailing list. Bugs can -also be reported in the issue tracker at -. - -Configuration Examples -====================== - -Here are some example configurations for various situations. Please e-mail any -other examples you have that may be useful to me. - - -Multiple Accounts with Mutt ---------------------------- - -This example shows you how to set up OfflineIMAP to synchronize multiple -accounts with the mutt mail reader. - -Start by creating a directory to hold your folders by running `mkdir ~/Mail`. -Then, in your `~/.offlineimaprc`, specify: - - accounts = Personal, Work - - -Make sure that you have both an `[Account Personal]` and an `[Account Work]` -section. The local repository for each account must have different `localfolder` -path names. Also, make sure to enable `[mbnames]`. - -In each local repository section, write something like this: - - localfolders = ~/Mail/Personal - - -Finally, add these lines to your `~/.muttrc`: - - source ~/path-to-mbnames-muttrc-mailboxes - folder-hook Personal set from="youremail@personal.com" - folder-hook Work set from="youremail@work.com" - set mbox_type=Maildir - set folder=$HOME/Mail - spoolfile=+Personal/INBOX - - -That's it! - - -UW-IMAPD and References ------------------------ - -Some users with a UW-IMAPD server need to use OfflineIMAP's "reference" feature -to get at their mailboxes, specifying a reference of `~/Mail` or `#mh/` -depending on the configuration. The below configuration from (originally from -docwhat@gerf.org) shows using a reference of Mail, a `nametrans` that strips the -leading `Mail/` off incoming folder names, and a `folderfilter` that limits the -folders synced to just three: - - [Account Gerf] - localrepository = GerfLocal - remoterepository = GerfRemote - - [Repository GerfLocal] - type = Maildir - localfolders = ~/Mail - - [Repository GerfRemote] - type = IMAP - remotehost = gerf.org - ssl = yes - remoteuser = docwhat - reference = Mail - # Trims off the preceeding Mail on all the folder names. - nametrans = lambda foldername: \ - re.sub('^Mail/', '', foldername) - # Yeah, you have to mention the Mail dir, even though it - # would seem intuitive that reference would trim it. - folderfilter = lambda foldername: foldername in [ - 'Mail/INBOX', - 'Mail/list/zaurus-general', - 'Mail/list/zaurus-dev', - ] - maxconnections = 1 - holdconnectionopen = no - - -pythonfile Configuration File Option -------------------------------------- - -You can have OfflineIMAP load up a Python file before evaluating the -configuration file options that are Python expressions. This example is based -on one supplied by Tommi Virtanen for this feature. - - -In `~/.offlineimaprc`, he adds these options: - - [general] - pythonfile=~/.offlineimap.py - [Repository foo] - foldersort=mycmp - -Then, the `~/.offlineimap.py` file will contain: - - prioritized = ['INBOX', 'personal', 'announce', 'list'] - - def mycmp(x, y): - for prefix in prioritized: - xsw = x.startswith(prefix) - ysw = y.startswith(prefix) - if xsw and ysw: - return cmp(x, y) - elif xsw: - return -1 - elif ysw: - return +1 - return cmp(x, y) - - def test_mycmp(): - import os, os.path - folders=os.listdir(os.path.expanduser('~/data/mail/tv@hq.yok.utu.fi')) - folders.sort(mycmp) - print folders - - -This code snippet illustrates how the `foldersort` option can be customized with a -Python function from the `pythonfile` to always synchronize certain folders first. From 7c7e7f92b1997abd8bce552a00ce0a7d9b3c2dc4 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 05:19:59 +0100 Subject: [PATCH 736/817] website: learn to build the sphinx documentation Signed-off-by: Nicolas Sebrecht --- .gitignore | 1 + Makefile | 3 +- docs/Makefile | 3 + docs/conf.py | 201 +++++++++++++++++++++++++++++++++++++++++++ docs/doc-src/conf.py | 14 +-- docs/website-doc.sh | 37 ++++++++ get-website.sh | 9 +- 7 files changed, 259 insertions(+), 9 deletions(-) create mode 100644 docs/conf.py create mode 100755 docs/website-doc.sh diff --git a/.gitignore b/.gitignore index 235f5fa..cecdf30 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /build/ *.pyc offlineimap.1 +offlineimapui.7 diff --git a/Makefile b/Makefile index c355583..bfe4523 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,8 @@ clean: -find . -name '*.pygc' -exec rm -f {} \; -find . -name '*.class' -exec rm -f {} \; -find . -name '.cache*' -exec rm -f {} \; - -find . -name '*.html' -exec rm -f {} \; + # Care with that. We have html in subdirs we want to keep. + -find ./docs -name '*.html' -exec rm -f {} \; -rm -f manpage.links manpage.refs -find . -name auth -exec rm -vf {}/password {}/username \; @$(MAKE) -C docs clean diff --git a/docs/Makefile b/docs/Makefile index cd688f7..6063dc7 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -25,6 +25,9 @@ offlineimap.1: MANUAL.rst doc: $(SPHINXBUILD) -b html -d html/doctrees doc-src html +websitedoc: + ./website-doc.sh + clean: $(RM) -f $(HTML_TARGETS) $(RM) -f offlineimap.1 ../offlineimap.1 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..3f4bb6c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- +# +# pyDNS documentation build configuration file, created by +# sphinx-quickstart on Tue Feb 2 10:00:47 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0,os.path.abspath('../..')) + +from offlineimap import __version__,__author__ +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] +autoclass_content = "both" + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'OfflineImap' +copyright = u'2002-2010, ' + __author__ + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = __version__ +# The full version, including alpha/beta/rc tags. +release = __version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = False + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' +#html_style = '' +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['html'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +html_use_modindex = False + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'dev-doc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'offlineimap.tex', u'OfflineImap Documentation', + u'OfflineImap contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/doc-src/conf.py b/docs/doc-src/conf.py index 3f4bb6c..0886d4a 100644 --- a/docs/doc-src/conf.py +++ b/docs/doc-src/conf.py @@ -16,9 +16,9 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0,os.path.abspath('../..')) +sys.path.insert(0, os.path.abspath('../..')) -from offlineimap import __version__,__author__ +from offlineimap import __version__, __bigversion__, __author__, __copyright__ # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions @@ -40,8 +40,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'OfflineImap' -copyright = u'2002-2010, ' + __author__ +project = u'OfflineIMAP' +copyright = __copyright__ # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -50,7 +50,7 @@ copyright = u'2002-2010, ' + __author__ # The short X.Y version. version = __version__ # The full version, including alpha/beta/rc tags. -release = __version__ +release = __bigversion__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -175,8 +175,8 @@ htmlhelp_basename = 'dev-doc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'offlineimap.tex', u'OfflineImap Documentation', - u'OfflineImap contributors', 'manual'), + ('index', 'offlineimap.tex', u'OfflineIMAP Documentation', + u'OfflineIMAP contributors', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/docs/website-doc.sh b/docs/website-doc.sh new file mode 100755 index 0000000..f705b10 --- /dev/null +++ b/docs/website-doc.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# +# vim: expandtab ts=2 : + +SPHINXBUILD=sphinx-build +TMPDIR='/tmp/offlineimap-sphinx-doctrees' +WEBSITE='../website' +DESTBASE="${WEBSITE}/_doc/versions" +VERSIONS_YML="${WEBSITE}/_data/versions.yml" + +version="v$(../offlineimap.py --version)" + +test -d "$DESTBASE" || exit 1 +dest="${DESTBASE}/${version}" + +# +# Build sphinx documentation. +# +echo "Cleaning target directory: $dest" +rm -rf "$dest" +$SPHINXBUILD -b html -d "$TMPDIR" doc-src "$dest" + +# +# Dynamically build JSON definitions for Jekyll. +# +echo "Building Jekyll data: $VERSIONS_YML" +for version in $(ls "$DESTBASE") +do + echo "- $version" +done > "$VERSIONS_YML" + +# +# Copy usefull sources of documentation. +# + +# User doc +for foo in ../Changelog.rst ../Changelog.maint.rst diff --git a/get-website.sh b/get-website.sh index fe6e9f7..74ec3d4 100755 --- a/get-website.sh +++ b/get-website.sh @@ -5,5 +5,12 @@ then echo "Directory 'website' already exists..." exit 1 else - git clone TODO + git clone https://github.com/OfflineIMAP/offlineimap.github.io.git +cat </offlineimap.github.io.git +EOF fi From 0b43418911e8ba84ac40bb18a3fdd9a869ff618e Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 14:55:54 +0100 Subject: [PATCH 737/817] docs: full refactoring of the MANUAL Signed-off-by: Nicolas Sebrecht --- docs/MANUAL.rst | 623 ---------------------------------------- docs/Makefile | 10 +- docs/conf.py | 201 ------------- docs/doc-src/MANUAL.rst | 1 - docs/doc-src/index.rst | 13 +- docs/offlineimap.txt | 408 ++++++++++++++++++++++++++ docs/offlineimapui.txt | 140 +++++++++ offlineimap.conf | 28 +- offlineimap/init.py | 80 +----- 9 files changed, 592 insertions(+), 912 deletions(-) delete mode 100644 docs/MANUAL.rst delete mode 100644 docs/conf.py delete mode 120000 docs/doc-src/MANUAL.rst create mode 100644 docs/offlineimap.txt create mode 100644 docs/offlineimapui.txt diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst deleted file mode 100644 index 5bdc067..0000000 --- a/docs/MANUAL.rst +++ /dev/null @@ -1,623 +0,0 @@ -==================== - OfflineIMAP Manual -==================== - -.. _OfflineIMAP: http://offlineimap.org - --------------------------------------------------------- -Powerful IMAP/Maildir synchronization and reader support --------------------------------------------------------- - -:Author: John Goerzen & contributors -:Date: 2012-02-23 - -DESCRIPTION -=========== - -OfflineImap operates on a REMOTE and a LOCAL repository and synchronizes -emails between them, so that you can read the same mailbox from multiple -computers. The REMOTE repository is some IMAP server, while LOCAL can be -either a local Maildir or another IMAP server. - -Missing folders will be automatically created on both sides if -needed. No folders will be deleted at the moment. - -Configuring OfflineImap in basic mode is quite easy, however it provides -an amazing amount of flexibility for those with special needs. You can -specify the number of connections to your IMAP server, use arbitrary -python functions (including regular expressions) to limit the number of -folders being synchronized. You can transpose folder names between -repositories using any python function, to mangle and modify folder -names on the LOCAL repository. There are six different ways to hand the -IMAP password to OfflineImap from console input, specifying in the -configuration file, .netrc support, specifying in a separate file, to -using arbitrary python functions that somehow return the -password. Finally, you can use IMAPs IDLE infrastructure to always keep -a connection to your IMAP server open and immediately be notified (and -synchronized) when a new mail arrives (aka Push mail). - -Most configuration is done via the configuration file. However, any setting can -also be overriden by command line options handed to OfflineIMAP. - -OfflineImap is well suited to be frequently invoked by cron jobs, or can run in -daemon mode to periodically check your email (however, it will exit in some -error situations). - -The documentation is included in the git repository and can be created by -issueing `make doc` in the `doc` folder (python-sphinx required), or it can -be viewed online at http://docs.offlineimap.org. - -.. _configuration: - -Configuration -============= - -`OfflineIMAP`_ is regulated by a configuration file that is normally stored in -`~/.offlineimaprc`. `OfflineIMAP`_ ships with a file named `offlineimap.conf` -that you should copy to that location and then edit. This file is vital to -proper operation of the system; it sets everything you need to run -`OfflineIMAP`_. Full documentation for the configuration file is included -within the sample file. - - -`OfflineIMAP`_ also ships a file named `offlineimap.conf.minimal` that you can -also try. It's useful if you want to get started with the most basic feature -set, and you can read about other features later with `offlineimap.conf`. - -Check out the `Use Cases`_ section for some example configurations. - -If you want to be XDG-compatible, you can put your configuration file into -`$XDG_CONFIG_HOME/offlineimap/config`. - - -OPTIONS -======= - -The command line options are described by issueing `offlineimap --help`. -Details on their use can be found either in the sample offlineimap.conf file or -in the user docs at http://docs.offlineimap.org. - -User Interfaces -=============== - -OfflineIMAP has various user interfaces that let you choose how the -program communicates information to you. The 'ui' option in the -configuration file specifies the user interface. The -u command-line -option overrides the configuration file setting. The available values -for the configuration file or command-line are described in this -section. - - -Blinkenlights ---------------- - -Blinkenlights is an interface designed to be sleek, fun to watch, and -informative of the overall picture of what OfflineIMAP is doing. - -Blinkenlights contains a row of "LEDs" with command buttons and a log. -The log shows more detail about what is happening and is color-coded to match -the color of the lights. - -Each light in the Blinkenlights interface represents a thread of execution -- -that is, a particular task that OfflineIMAP is performing right now. The colors -indicate what task the particular thread is performing, and are as follows: - -* Black: - indicates that this light's thread has terminated; it will light up again - later when new threads start up. So, black indicates no activity. - -* Red (Meaning 1): - is the color of the main program's thread, which basically does nothing but - monitor the others. It might remind you of HAL 9000 in 2001. - -* Gray: - indicates that the thread is establishing a new connection to the IMAP - server. - -* Purple: - is the color of an account synchronization thread that is monitoring the - progress of the folders in that account (not generating any I/O). - -* Cyan: - indicates that the thread is syncing a folder. - -* Green: - means that a folder's message list is being loaded. - -* Blue: - is the color of a message synchronization controller thread. - -* Orange: - indicates that an actual message is being copied. (We use fuchsia for fake - messages.) - -* Red (meaning 2): - indicates that a message is being deleted. - -* Yellow / bright orange: - indicates that message flags are being added. - -* Pink / bright red: - indicates that message flags are being removed. - -* Red / Black Flashing: - corresponds to the countdown timer that runs between synchronizations. - - -The name of this interfaces derives from a bit of computer history. Eric -Raymond's Jargon File defines blinkenlights, in part, as: - - Front-panel diagnostic lights on a computer, esp. a dinosaur. Now that - dinosaurs are rare, this term usually refers to status lights on a modem, - network hub, or the like. - -This term derives from the last word of the famous blackletter-Gothic sign in -mangled pseudo-German that once graced about half the computer rooms in the -English-speaking world. One version ran in its entirety as follows: - -| ACHTUNG! ALLES LOOKENSPEEPERS! -| -| Das computermachine ist nicht fuer gefingerpoken und mittengrabben. -| Ist easy schnappen der springenwerk, blowenfusen und poppencorken -| mit spitzensparken. Ist nicht fuer gewerken bei das dumpkopfen. -| Das rubbernecken sichtseeren keepen das cotten-pickenen hans in das -| pockets muss; relaxen und watchen das blinkenlichten. - - -TTYUI ------- - -TTYUI interface is for people running in terminals. It prints out basic -status messages and is generally friendly to use on a console or xterm. - - -Basic ------- - -Basic is designed for situations in which OfflineIMAP will be run non-attended -and the status of its execution will be logged. This user interface is not -capable of reading a password from the keyboard; account passwords must be -specified using one of the configuration file options. For example, it will not -print periodic sleep announcements and tends to be a tad less verbose, in -general. - - -Quiet ------ - -It will output nothing except errors and serious warnings. Like Basic, -this user interface is not capable of reading a password from the -keyboard; account passwords must be specified using one of the -configuration file options. - -MachineUI ---------- - -MachineUI generates output in a machine-parsable format. It is designed -for other programs that will interface to OfflineIMAP. - - -Synchronization Performance -=========================== - -By default, we use fairly conservative settings that are safe for -syncing but that might not be the best performing one. Once you got -everything set up and running, you might want to look into speeding up -your synchronization. Here are a couple of hints and tips on how to -achieve this. - - 1) Use maxconnections > 1. By default we only use one connection to an - IMAP server. Using 2 or even 3 speeds things up considerably in most - cases. This setting goes into the [Repository XXX] section. - - 2) Use folderfilters. The quickest sync is a sync that can ignore some - folders. I sort my inbox into monthly folders, and ignore every - folder that is more than 2-3 months old, this lets me only inspect a - fraction of my Mails on every sync. If you haven't done this yet, do - it :). See the folderfilter section the example offlineimap.conf. - - 3) The default status cache is a plain text file that will write out - the complete file for each single new message (or even changed flag) - to a temporary file. If you have plenty of files in a folder, this - is a few hundred kilo to megabytes for each mail and is bound to - make things slower. I recommend to use the sqlite backend for - that. See the status_backend = sqlite setting in the example - offlineimap.conf. You will need to have python-sqlite installed in - order to use this. This will save you plenty of disk activity. Do - note that the sqlite backend is still considered experimental as it - has only been included recently (although a loss of your status - cache should not be a tragedy as that file can be rebuilt - automatically) - - 4) Use quick sync. A regular sync will request all flags and all UIDs - of all mails in each folder which takes quite some time. A 'quick' - sync only compares the number of messages in a folder on the IMAP - side (it will detect flag changes on the Maildir side of things - though). A quick sync on my smallish account will take 7 seconds - rather than 40 seconds. Eg, I run a cron script that does a regular - sync once a day, and does quick syncs (-q) only synchronizing the - "-f INBOX" in between. - - 5) Turn off fsync. In the [general] section you can set fsync to True - or False. If you want to play 110% safe and wait for all operations - to hit the disk before continueing, you can set this to True. If you - set it to False, you lose some of that safety, trading it for speed. - - -Upgrading from plain text cache to SQLITE based cache -===================================================== - -OfflineImap uses a cache to store the last know status of mails (flags etc). -Historically that has meant plain text files, but recently we introduced -sqlite-based cache, which helps with performance and CPU usage on large folders. -Here is how to upgrade existing plain text cache installations to sqlite based -one: - -1) Sync to make sure things are reasonably similar - -2) Change the account section to status_backend = sqlite - -3) A new sync will convert your plain text cache to an sqlite cache - (but leave the old plain text cache around for easy reverting) This - should be quick and not involve any mail up/downloading. - -4) See if it works :-) - -5) If it does not work, go back to the old version or set - status_backend=plain - -6) Or, once you are sure it works, you can delete the - .offlineimap/Account-foo/LocalStatus folder (the new cache will be - in the LocalStatus-sqlite folder) - -Security and SSL -================ - -Some words on OfflineImap and its use of SSL/TLS. By default, we will -connect using any method that openssl supports, that is SSLv2, SSLv3, or -TLSv1. Do note that SSLv2 is notoriously insecure and deprecated. -Unfortunately, python2 does not offer easy ways to disable SSLv2. It is -recommended you test your setup and make sure that the mail server does -not use an SSLv2 connection. Use e.g. "openssl s_client -host -mail.server -port 443" to find out the connection that is used by -default. - -Certificate checking --------------------- - -Unfortunately, by default we will not verify the certificate of an IMAP -TLS/SSL server we connect to, so connecting by SSL is no guarantee -against man-in-the-middle attacks. While verifying a server certificate -fingerprint is being planned, it is not implemented yet. There is -currently only one safe way to ensure that you connect to the correct -server in an encrypted manner: You can specify a 'sslcacertfile' setting -in your repository section of offlineimap.conf pointing to a file that -contains (among others) a CA Certificate in PEM format which validating -your server certificate. In this case, we will check that: 1) The server -SSL certificate is validated by the CA Certificate 2) The server host -name matches the SSL certificate 3) The server certificate is not past -its expiration date. The FAQ contains an entry on how to create your own -certificate and CA certificate. - -StartTLS --------- - -If you have not configured your account to connect via SSL anyway, -OfflineImap will still attempt to set up an SSL connection via the -STARTTLS function, in case the imap server supports it. Do note, that -there is no certificate or fingerprint checking involved at all, when -using STARTTLS (the underlying imaplib library does not support this -yet). This means that you will be protected against passively listening -eavesdroppers and they will not be able to see your password or email -contents. However, this will not protect you from active attacks, such -as Man-In-The-Middle attacks which cause you to connect to the wrong -server and pretend to be your mail server. DO NOT RELY ON STARTTLS AS A -SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF YOUR IMAP SERVER! - -.. _UNIX signals: - -UNIX Signals -============ - -OfflineImap listens to the unix signals SIGUSR1, SIGUSR2, SIGTERM, -SIGINT, SIGHUP, SIGQUIT: - -If sent a SIGUSR1 it will abort any current (or next future) sleep of all -accounts that are configured to "autorefresh". In effect, this will trigger a -full sync of all accounts to be performed as soon as possible. - -If sent a SIGUSR2, it will stop "autorefresh mode" for all accounts. That is, -accounts will abort any current sleep and will exit after a currently running -synchronization has finished. This signal can be used to gracefully exit out of -a running offlineimap "daemon". - -SIGTERM, SIGINT, SIGHUP are all treated to gracefully terminate as -soon as possible. This means it will finish syncing the current folder -in each account, close keep alive connections, remove locks on the -accounts and exit. It may take up to 10 seconds, if autorefresh option -is used. - -SIGQUIT dumps stack traces for all threads and tries to dump process -core. - -Folder filtering and nametrans -============================== - -OfflineImap offers flexible (and complex) ways of filtering and transforming -folder names. Please see the docs/doc-src/nametrans.rst document about details -how to use folder filters and name transformations. The documentation will be -autogenerated by a "make doc" in the docs directory. It is also viewable at -:ref:`folder_filtering_and_name_translation`. - -KNOWN ISSUES -============ - -* SSL3 write pending: - users enabling SSL may hit a bug about "SSL3 write pending". If so, the - account(s) will stay unsynchronised from the time the bug appeared. Running - OfflineIMAP again can help. We are still working on this bug. Patches or - detailed bug reports would be appreciated. Please check you're running the - last stable version and send us a report to the mailing list including the - full log. - -* IDLE support is incomplete and experimental. Bugs may be encountered. - - * No hook exists for "run after an IDLE response". Email will - show up, but may not be processed until the next refresh cycle. - - * nametrans may not be supported correctly. - - * IMAP IDLE <-> IMAP IDLE doesn't work yet. - - * IDLE may only work "once" per refresh. If you encounter this bug, - please send a report to the list! - -* Maildir support in Windows drive - Maildir uses colon caracter (:) in message file names. Colon is however - forbidden character in windows drives. There are several workarounds for - that situation: - - * Use "maildir-windows-compatible = yes" account OfflineIMAP configuration. - - That makes OfflineIMAP to use exclamation mark (!) instead of colon for - storing messages. Such files can be written to windows partitions. But - you will probably loose compatibility with other programs trying to - read the same Maildir. - - Exclamation mark was chosen because of the note in - http://docs.python.org/library/mailbox.html - - If you have some messages already stored without this option, you will - have to re-sync them again - - * Enable file name character translation in windows registry (not tested) - - http://support.microsoft.com/kb/289627 - - * Use cygwin managed mount (not tested) - - not available anymore since cygwin 1.7 - -* OfflineIMAP confused after system suspend. - When resuming a suspended session, OfflineIMAP does not cleanly handles the - broken socket(s) if socktimeout option is not set. - You should enable this option with a value like 10. - -.. _pitfalls: - -PITFALLS & ISSUES -================= - -Sharing a maildir with multiple IMAP servers --------------------------------------------- - - Generally a word of caution mixing IMAP repositories on the same - Maildir root. You have to be careful that you *never* use the same - maildir folder for 2 IMAP servers. In the best case, the folder MD5 - will be different, and you will get a loop where it will upload your - mails to both servers in turn (infinitely!) as it thinks you have - placed new mails in the local Maildir. In the worst case, the MD5 is - the same (likely) and mail UIDs overlap (likely too!) and it will fail to - sync some mails as it thinks they are already existent. - - I would create a new local Maildir Repository for the Personal Gmail and - use a different root to be on the safe side here. You could e.g. use - `~/mail/Pro` as Maildir root for the ProGmail and - `~/mail/Personal` as root for the personal one. - - If you then point your local mutt, or whatever MUA you use to `~/mail/` - as root, it should still recognize all folders. (see the 2 IMAP setup - in the `Use Cases`_ section. - -USE CASES -========= - -Sync from GMail to another IMAP server --------------------------------------- - -This is an example of a setup where "TheOtherImap" requires all folders to be under INBOX:: - - [Repository Gmailserver-foo] - #This is the remote repository - type = Gmail - remotepass = XXX - remoteuser = XXX - # The below will put all GMAIL folders as sub-folders of the 'local' INBOX, - # assuming that your path separator on 'local' is a dot. - nametrans = lambda x: 'INBOX.' + x - - [Repository TheOtherImap] - #This is the 'local' repository - type = IMAP - remotehost = XXX - remotepass = XXX - remoteuser = XXX - #Do not use nametrans here. - - -Sync from Gmail to a local Maildir with labels ----------------------------------------------- - -This is an example of a setup where GMail gets synced with a local Maildir. -It also keeps track of GMail labels, that get embedded into the messages -under the header configured in the labelsheader setting, and syncs them back -and forth the same way as flags. This is particularly useful with an email -client that indexes your email and recognizes the labels header, so that you -can sync a single "All Mail" folder, and navigate your email via searches. - -The header used to store the labels depends on the email client you plan to use. -Some choices that may be recognized by email clients are X-Keywords -(the default) or X-Labels. Note that if you need to change the label header -after the labels have already been synced, you will have to change the header -manually on all messages, otherwise offlineimap will not pick up the labels under -the old header. - -The first time OfflineIMAP runs with synclabels enabled on a large repository it -may take some time as the labels are read / embedded on every message. -Afterwards local label changes are detected using modification times, which is -much faster:: - - [Account Gmail-mine] - localrepository = Gmaillocal-mine - remoterepository = Gmailserver-mine - synclabels = yes - # This header is where labels go. Usually you will be fine - # with default value (X-Keywords), but in case you want it - # different, here we go: - labelsheader = X-Keywords - - [Repository Gmailserver-mine] - #This is the remote repository - type = Gmail - remotepass = XXX - remoteuser = XXX - - [Repository Gmaillocal-mine] - #This is the 'local' repository - type = GmailMaildir - -There are some labels, that gmail treats in a special way. All internal gmail -labels start with "\\". Those labels include: \\Drafts, \\Important, \\Inbox, -\\Sent, \\Junk, \\Flagged, \\Trash. You can add and remove those labels -locally, and when synced, will have special actions on the gmail side. For instance, -adding the label \Trash to an email will move it to the trash, and be permanantly -deleted after some time. This is relevant, since gmail's IMAP prevents from removing -messages from the "All Mail" folder the usual way. - - -Selecting only a few folders to sync ------------------------------------- -Add this to the remote gmail repository section to only sync mails which are in a certain folder:: - - folderfilter = lambda folder: folder.startswith('MyLabel') - -To only get the All Mail folder from a Gmail account, you would e.g. do:: - - folderfilter = lambda folder: folder.startswith('[Gmail]/All Mail') - - -Another nametrans transpose example ------------------------------------ - -Put everything in a GMX. subfolder except for the boxes INBOX, Draft, -and Sent which should keep the same name:: - - nametrans: lambda folder: folder if folder in ['INBOX', 'Drafts', 'Sent'] \ - else re.sub(r'^', r'GMX.', folder) - -2 IMAP using name translations ------------------------------- - -Synchronizing 2 IMAP accounts to local Maildirs that are "next to each -other", so that mutt can work on both. Full email setup described by -Thomas Kahle at ``_ - -offlineimap.conf:: - - [general] - accounts = acc1, acc2 - maxsyncaccounts = 2 - ui = ttyui - pythonfile=~/bin/offlineimap-helpers.py - socktimeout = 90 - - [Account acc1] - localrepository = acc1local - remoterepository = acc1remote - autorefresh = 2 - - [Account acc2] - localrepository = acc2local - remoterepository = acc2remote - autorefresh = 4 - - [Repository acc1local] - type = Maildir - localfolders = ~/Mail/acc1 - - [Repository acc2local] - type = Maildir - localfolders = ~/Mail/acc2 - - [Repository acc1remote] - type = IMAP - remotehost = imap.acc1.com - remoteusereval = get_username("imap.acc1.net") - remotepasseval = get_password("imap.acc1.net") - nametrans = oimaptransfolder_acc1 - ssl = yes - maxconnections = 2 - # Folders to get: - folderfilter = lambda foldername: foldername in [ - 'INBOX', 'Drafts', 'Sent', 'archiv'] - - [Repository acc2remote] - type = IMAP - remotehost = imap.acc2.net - remoteusereval = get_username("imap.acc2.net") - remotepasseval = get_password("imap.acc2.net") - nametrans = oimaptransfolder_acc2 - ssl = yes - maxconnections = 2 - -One of the coolest things about offlineimap is that you can call -arbitrary python code from your configuration. To do this, specify a -pythonfile with:: - - pythonfile=~/bin/offlineimap-helpers.py - -Here is a basic content sample:: - - import commands - - def get_password(account_name): - cmd = "security find-internet-password -w -a '%s'"% account_name - (status, output) = commands.getstatusoutput(cmd) - return output - -From this sample, replace the cmd line with whatever can retrieve your password. -Your pythonfile needs to contain implementations for the functions -that you want to use in offflineimaprc. The example uses it for two -purposes: Fetching passwords from the gnome-keyring and translating -folder names on the server to local foldernames. An example -implementation of get_username and get_password showing how to query -gnome-keyring is contained in -``_ The folderfilter is -a lambda term that, well, filters which folders to get. The function -`oimaptransfolder_acc2` translates remote folders into local folders -with a very simple logic. The `INBOX` folder will have the same name -as the account while any other folder will have the account name and a -dot as a prefix. This is useful for hierarchichal display in mutt. -Offlineimap handles the renaming correctly in both directions:: - - import re - def oimaptransfolder_acc1(foldername): - if(foldername == "INBOX"): - retval = "acc1" - else: - retval = "acc1." + foldername - retval = re.sub("/", ".", retval) - return retval - - def oimaptransfolder_acc2(foldername): - if(foldername == "INBOX"): - retval = "acc2" - else: - retval = "acc2." + foldername - retval = re.sub("/", ".", retval) - return retval diff --git a/docs/Makefile b/docs/Makefile index 6063dc7..ec6aaaa 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,11 +16,13 @@ html: $(HTML_TARGETS) $(HTML_TARGETS): %.html : %.rst $(RST2HTML) $? $@ -man: offlineimap.1 +man: offlineimap.1 offlineimapui.7 -offlineimap.1: MANUAL.rst - $(RST2MAN) MANUAL.rst offlineimap.1 - cp -f offlineimap.1 .. +offlineimap.1: offlineimap.txt + a2x -v -f manpage $? + +offlineimapui.7: offlineimapui.txt + a2x -v -f manpage $? doc: $(SPHINXBUILD) -b html -d html/doctrees doc-src html diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 3f4bb6c..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,201 +0,0 @@ -# -*- coding: utf-8 -*- -# -# pyDNS documentation build configuration file, created by -# sphinx-quickstart on Tue Feb 2 10:00:47 2010. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0,os.path.abspath('../..')) - -from offlineimap import __version__,__author__ -# -- General configuration ----------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] -autoclass_content = "both" - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'OfflineImap' -copyright = u'2002-2010, ' + __author__ - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = __version__ -# The full version, including alpha/beta/rc tags. -release = __version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -#unused_docs = [] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = [] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -add_module_names = False - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. Major themes that come with -# Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'default' -#html_style = '' -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['html'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -html_use_modindex = False - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'dev-doc' - - -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'offlineimap.tex', u'OfflineImap Documentation', - u'OfflineImap contributors', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_use_modindex = True - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/doc-src/MANUAL.rst b/docs/doc-src/MANUAL.rst deleted file mode 120000 index a45b3ee..0000000 --- a/docs/doc-src/MANUAL.rst +++ /dev/null @@ -1 +0,0 @@ -../MANUAL.rst \ No newline at end of file diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index 3e656a5..7335827 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -2,20 +2,13 @@ .. _OfflineIMAP: http://offlineimap.github.io -Welcome to OfflineIMAP's documentation -====================================== - -**Configuration** - * :doc:`user manual/Configuration ` - * :doc:`Folder filtering & name transformation guide ` - * :doc:`command line options ` +Welcome to OfflineIMAP's developer documentation +================================================ **Developer documentation** * :doc:`Advanced Git ` - * :doc:`API documentation ` for internal details on the - :mod:`offlineimap` module -**OfflineIMAP APIs** +**Documented APIs** .. toctree:: API diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt new file mode 100644 index 0000000..7708792 --- /dev/null +++ b/docs/offlineimap.txt @@ -0,0 +1,408 @@ + +offlineimap(1) +============== + +NAME +---- +offlineimap - Synchronize mailboxes and Maildirs + +SYNOPSIS +-------- +[verse] +'offlineimap' (options) + +DESCRIPTION +----------- + +Synchronize the accounts configured in the configuration file via IMAP. Each +account has two sides. + +One of the side must be an IMAP server. +The other side can either be a Maildir or another IMAP server. + + +OPTIONS +------- + +-h:: +--help:: + + Display summary of options. + +--version:: + + Output version. + +--dry-run:: + + Run in dry run mode. ++ +Do not actually modify any store but check and print what synchronization +actions would be taken if a sync would be performed. It will not precisely +give the exact information what will happen. If e.g. we need to create a +folder, it merely outputs 'Would create folder X', but not how many and which +mails it would transfer. + + +--info:: + + Output information on the configured email repositories. ++ +Useful for debugging and bug reporting. Use in conjunction with the -a option +to limit the output to a single account. This mode will prevent any actual +sync to occur and exits after it outp ut the debug information. + + +-1:: + + Limit multithreading operations and run solely a single-thread sync. ++ +This effectively sets the maxsyncaccounts and all maxconnections configuration +file variables to 1. This is 1, the number. + + +-P :: + + Set OfflineIMAP into profile mode. ++ +The program will create DIR (it must not already exist). As it runs, Python +profiling information about each thread is logged into profiledir. Please +note: This option is present for debugging and optimization only, and should +NOT be used unless you have a specific reason to do so. It will significantly +decrease program performance, may reduce reliability, and can generate huge +amounts of data. This option implies the -1 option. + + +-a :: + + Overrides the accounts section in the config file. ++ +Allows to specify a particular account or set of accounts to sync without +having to edit the config file. + + +-c :: + + Specifies a configuration file to use. + + +-d :: + + Enables debugging for OfflineIMAP. ++ +This is useful if you are to track down a malfunction or figure out what is +going on under the hood. This option requires one or more debugtypes, +separated by commas. These define what exactly will be debugged, and so far +include two options: imap, thread, maildir or ALL. The imap option will enable +IMAP protocol stream and parsing debugging. Note that the output may contain +passwords, so take care to remove that from the debugging output before +sending it to anyone else. The maildir option will enable debugging for +certain Maildir operations. The use of any debug option (unless 'thread' is +included), implies the single-thread option -1. + + +-l :: + + Send logs to . + + +-f :: + + Only sync the specified folders. ++ +The folder names are the untranslated foldernames of the remote repository. +This command-line option overrides any 'folderfilter' and 'folderincludes' +options in the configuration file. + +-k <[section:]option=value:: + + Override any configuration file option. ++ +If "section" is omitted, it defaults to "general". Any underscores in the +section name are replaced with spaces: for instance, to override option +"autorefresh" in the "[Account Personal]" section in the config file one would +use "-k Account_Personal:autorefresh=30". Repeat this option as much as +necessary to redefine multiple options. + +-o:: + + Run only once. ++ +Ignore any autorefresh setting in the configuration file. + +-q:: + + Run only quick synchronizations. ++ +Ignore any flag updates on IMAP servers. If a flag on the remote IMAP changes, +and we have the message locally, it will be left untouched in a quick run. + + +-u :: + + Specifies an alternative user interface to use. ++ +This overrides the default specified in the configuration file. The UI +specified with -u will be forced to be used, even if checks determine that it +is not usable. Possible interface choices are: quiet, basic, ttyui, +blinkenlights, machineui. + + + +--column[=]:: +--no-column:: + Display branch listing in columns. See configuration variable + column.branch for option syntax.`--column` and `--no-column` + without options are equivalent to 'always' and 'never' respectively. ++ +This option is only applicable in non-verbose mode. + + +Synchronization Performance +--------------------------- + +By default, we use fairly conservative settings that are safe for syncing but +that might not be the best performing one. Once you got everything set up and +running, you might want to look into speeding up your synchronization. Here +are a couple of hints and tips on how to achieve this. + +1. Use maxconnections > 1. ++ +By default we only use one connection to an IMAP server. Using 2 or even 3 +speeds things up considerably in most cases. This setting goes into the +[Repository XXX] section. + +2. Use folderfilters. ++ +The quickest sync is a sync that can ignore some folders. I sort my inbox into +monthly folders, and ignore every folder that is more than 2-3 months old, +this lets me only inspect a fraction of my Mails on every sync. If you haven't +done this yet, do it :). See the folderfilter section the example +offlineimap.conf. + +3. The cache. ++ +The default status cache is a plain text file that will write out the complete +file for each single new message (or even changed flag) to a temporary file. +If you have plenty of files in a folder, this is a few hundred kilo to +megabytes for each mail and is bound to make things slower. I recommend to use +the sqlite backend for that. See the status_backend = sqlite setting in the +example offlineimap.conf. You will need to have python-sqlite installed in +order to use this. This will save you plenty of disk activity. Do note that +the sqlite backend is still considered experimental as it has only been +included recently (although a loss of your status cache should not be a +tragedy as that file can be rebuilt automatically) + +4. Use quick sync. ++ +A regular sync will request all flags and all UIDs of all mails in each folder +which takes quite some time. A 'quick' sync only compares the number of +messages in a folder on the IMAP side (it will detect flag changes on the +Maildir side of things though). A quick sync on my smallish account will take +7 seconds rather than 40 seconds. Eg, I run a cron script that does a regular +sync once a day, and does quick syncs (-q) only synchronizing the "-f INBOX" +in between. + +5. Turn off fsync. ++ +In the [general] section you can set fsync to True or False. If you want to +play 110% safe and wait for all operations to hit the disk before continueing, +you can set this to True. If you set it to False, you lose some of that +safety, trading it for speed. + + +Upgrading from plain text to SQLite cache format +------------------------------------------------ + +OfflineImap uses a cache to store the last know status of mails (flags etc). + +Historically that has meant plain text files, but recently we introduced +sqlite-based cache, which helps with performance and CPU usage on large +folders. Here is how to upgrade existing plain text cache installations to +sqlite based one: + +1. Sync to make sure things are reasonably similar. + +2. Change the account section to "status_backend = sqlite". + +3. Run a new sync. ++ +This will convert your plain text cache to an sqlite cache (but leave +the old plain text cache around for easy reverting). This must be quick and +not involve any mail up/downloading. + +4. See if it works! :-) + +5. If it does not work, go back to the old version or set "status_backend = plain" + +6. Delete the old text cache files. + +Once you are sure it works, you can delete the +~/.offlineimap/Account-foo/LocalStatus folder (the new cache will be in the +LocalStatus-sqlite folder) + + +Security and SSL +---------------- + +By default, OfflineIMAP will connect using any method that 'openssl' supports, +that is SSLv2, SSLv3, or TLSv1. + +Do note that SSLv2 is notoriously insecure and deprecated. Unfortunately, +python2 does not offer easy ways to disable SSLv2. It is recommended you test +your setup and make sure that the mail server does not use an SSLv2 +connection. Use e.g. "openssl s_client -host mail.server -port 443" to find +out the connection that is used by default. + +* Certificate checking ++ +Unfortunately, by default we will not verify the certificate of an IMAP +TLS/SSL server we connect to, so connecting by SSL is no guarantee against +man-in-the-middle attacks. While verifying a server certificate fingerprint is +being planned, it is not implemented yet. There is currently only one safe way +to ensure that you connect to the correct server in an encrypted manner: you +can specify a 'sslcacertfile' setting in your repository section of +offlineimap.conf pointing to a file that contains (among others) a CA +Certificate in PEM format which validating your server certificate. In this +case, we will check that: + +1. The server SSL certificate is validated by the CA Certificate. + +2. The server host name matches the SSL certificate. + +3. The server certificate is not past its expiration date. + +The FAQ has an entry on how to create your own certificate and CA certificate. + +* StartTLS ++ +If you have not configured your account to connect via SSL anyway, OfflineImap +will still attempt to set up an SSL connection via the STARTTLS function, in +case the imap server supports it. ++ +There is no certificate or fingerprint checking involved at all, when using +STARTTLS (the underlying imaplib library does not support this yet). This +means that you will be protected against passively listening eavesdroppers and +they will not be able to see your password or email contents. However, this +will not protect you from active attacks, such as Man-In-The-Middle attacks +which cause you to connect to the wrong server and pretend to be your mail +server. ++ +DO NOT RELY ON STARTTLS AS A SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF +YOUR IMAP SERVER! + + +Unix Signals +------------ + +OfflineImap listens to the unix signals SIGUSR1, SIGUSR2, SIGTERM, SIGINT, +SIGHUP, SIGQUIT. + +* If sent a SIGUSR1 it will abort any current (or next future) sleep of all +accounts that are configured to "autorefresh". In effect, this will trigger a +full sync of all accounts to be performed as soon as possible. + +* If sent a SIGUSR2, it will stop "autorefresh mode" for all accounts. That +is, accounts will abort any current sleep and will exit after a currently +running synchronization has finished. This signal can be used to gracefully +exit out of a running offlineimap "daemon". + +* SIGTERM, SIGINT, SIGHUP are all treated to gracefully terminate as soon as +possible. This means it will finish syncing the current folder in each +account, close keep alive connections, remove locks on the accounts and exit. ++ +It may take up to 10 seconds, if autorefresh option is used. + +* If sent SIGQUIT, dumps stack traces for all threads and tries to dump +process core. + + +Known Issues +------------ + +* SSL3 write pending. ++ +Users enabling SSL may hit a bug about "SSL3 write pending". If so, the +account(s) will stay unsynchronised from the time the bug appeared. Running +OfflineIMAP again can help. We are still working on this bug. Patches or +detailed bug reports would be appreciated. Please check you're running the +last stable version and send us a report to the mailing list including the +full log. + +* IDLE support is incomplete and experimental. Bugs may be encountered. + + * No hook exists for "run after an IDLE response". ++ +Email will show up, but may not be processed until the next refresh cycle. + + * nametrans may not be supported correctly. + + * IMAP IDLE <-> IMAP IDLE doesn't work yet. + + * IDLE may only work "once" per refresh. ++ +If you encounter this bug, please send a report to the list! + +* Maildir support in Windows drive. ++ +Maildir uses colon caracter (:) in message file names. Colon is however +forbidden character in windows drives. There are several workarounds for that +situation: + + * Use "maildir-windows-compatible = yes" account OfflineIMAP configuration. + + - That makes OfflineIMAP to use exclamation mark (!) instead of colon for + storing messages. Such files can be written to windows partitions. But + you will probably loose compatibility with other programs trying to + read the same Maildir. + + - Exclamation mark was chosen because of the note in + http://docs.python.org/library/mailbox.html + + - If you have some messages already stored without this option, you will + have to re-sync them again + + * Enable file name character translation in windows registry (not tested). + - http://support.microsoft.com/kb/289627 + + * Use cygwin managed mount (not tested). + - not available anymore since cygwin 1.7 + +* OfflineIMAP confused after system suspend. ++ +When resuming a suspended session, OfflineIMAP does not cleanly handles the +broken socket(s) if socktimeout option is not set. +You should enable this option with a value like 10. + + +* Sharing a maildir with multiple IMAP servers. ++ +Generally a word of caution mixing IMAP repositories on the same Maildir root. +You have to be careful that you *never* use the same maildir folder for 2 IMAP +servers. In the best case, the folder MD5 will be different, and you will get +a loop where it will upload your mails to both servers in turn (infinitely!) +as it thinks you have placed new mails in the local Maildir. In the worst +case, the MD5 is the same (likely) and mail UIDs overlap (likely too!) and it +will fail to sync some mails as it thinks they are already existent. ++ +I would create a new local Maildir Repository for the Personal Gmail and +use a different root to be on the safe side here. You could e.g. use + + `~/mail/Pro` as Maildir root for the ProGmail and + `~/mail/Personal` as root for the personal one. ++ +If you then point your local mutt, or whatever MUA you use to `~/mail/` +as root, it should still recognize all folders. + + +Authors +------- + + John Goerzen, Sebastian Spaetz, Eygene Ryabinkin, Nicolas Sebrecht. + + +See Also +-------- + + offlineimapui(7), openssl(1), signal(7), sqlite3(1). + http://offlineimap.github.io diff --git a/docs/offlineimapui.txt b/docs/offlineimapui.txt new file mode 100644 index 0000000..c5a755b --- /dev/null +++ b/docs/offlineimapui.txt @@ -0,0 +1,140 @@ + +offlineimapui(7) +================ + +NAME +---- +offlineimapui - The User Interfaces + +DESCRIPTION +----------- + +OfflineIMAP comes with differents UI, each aiming its own purpose. + + +TTYUI +------ + +TTYUI interface is for people running in terminals. It prints out basic +status messages and is generally friendly to use on a console or xterm. + + +Basic +------ + +Basic is designed for situations in which OfflineIMAP will be run non-attended +and the status of its execution will be logged. ++ +This user interface is not capable of reading a password from the keyboard; +account passwords must be specified using one of the configuration file +options. For example, it will not print periodic sleep announcements and tends +to be a tad less verbose, in general. + + +Blinkenlights +------------- + +Blinkenlights is an interface designed to be sleek, fun to watch, and +informative of the overall picture of what OfflineIMAP is doing. + +Blinkenlights contains a row of "LEDs" with command buttons and a log. The +log shows more detail about what is happening and is color-coded to match the +color of the lights. + +Each light in the Blinkenlights interface represents a thread of execution -- +that is, a particular task that OfflineIMAP is performing right now. The +colors indicate what task the particular thread is performing, and are as +follows: + +* Black + + indicates that this light's thread has terminated; it will light up again + later when new threads start up. So, black indicates no activity. + +* Red (Meaning 1) + + is the color of the main program's thread, which basically does nothing but + monitor the others. It might remind you of HAL 9000 in 2001. + +* Gray + + indicates that the thread is establishing a new connection to the IMAP + server. + +* Purple + + is the color of an account synchronization thread that is monitoring the + progress of the folders in that account (not generating any I/O). + +* Cyan + + indicates that the thread is syncing a folder. + +* Green + + means that a folder's message list is being loaded. + +* Blue + + is the color of a message synchronization controller thread. + +* Orange + + indicates that an actual message is being copied. (We use fuchsia for fake + messages.) + +* Red (meaning 2) + + indicates that a message is being deleted. + +* Yellow / bright orange + + indicates that message flags are being added. + +* Pink / bright red + + indicates that message flags are being removed. + +* Red / Black Flashing + + corresponds to the countdown timer that runs between synchronizations. + + +The name of this interfaces derives from a bit of computer history. Eric +Raymond's Jargon File defines blinkenlights, in part, as: + + Front-panel diagnostic lights on a computer, esp. a dinosaur. Now that + dinosaurs are rare, this term usually refers to status lights on a modem, + network hub, or the like. + +This term derives from the last word of the famous blackletter-Gothic sign in +mangled pseudo-German that once graced about half the computer rooms in the +English-speaking world. One version ran in its entirety as follows: + + ACHTUNG! ALLES LOOKENSPEEPERS! + + Das computermachine ist nicht fuer gefingerpoken und mittengrabben. + Ist easy schnappen der springenwerk, blowenfusen und poppencorken + mit spitzensparken. Ist nicht fuer gewerken bei das dumpkopfen. + Das rubbernecken sichtseeren keepen das cotten-pickenen hans in das + pockets muss; relaxen und watchen das blinkenlichten. + + +Quiet +----- + +It will output nothing except errors and serious warnings. Like Basic, this +user interface is not capable of reading a password from the keyboard; account +passwords must be specified using one of the configuration file options. + +MachineUI +--------- + +MachineUI generates output in a machine-parsable format. It is designed +for other programs that will interface to OfflineIMAP. + + +See Also +-------- + + offlineima(1) diff --git a/offlineimap.conf b/offlineimap.conf index 1b3214a..4c8e5d2 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -2,9 +2,27 @@ # This file documents *all* possible options and can be quite scary. # Looking for a quick start? Take a look at offlineimap.conf.minimal. -# More details can be found in the included user documention, which is -# also available at: http://docs.offlineimap.org/en/latest/ +# More details can be found at http://offlineimap.github.io. +################################################## +# Overview +################################################## + +# The default configuration file is "~/.offlineimaprc". +# +# OfflineIMAP ships with a file named "offlineimap.conf" that you should copy to +# that location and then edit. +# +# OfflineIMAP also ships a file named "offlineimap.conf.minimal" that you can +# also try. It's useful if you want to get started with the most basic feature +# set, and you can read about other features later with "offlineimap.conf". +# +# If you want to be XDG-compatible, you can put your configuration file into +# "$XDG_CONFIG_HOME/offlineimap/config". + +################################################## +# General definitions +################################################## # NOTE 1: Settings generally support python interpolation. This means # values can contain python format strings which refer to other values @@ -31,10 +49,6 @@ # as it coincides with typical shell expansion strategy. -################################################## -# General definitions -################################################## - [general] # This specifies where OfflineIMAP is to store its metadata. @@ -84,6 +98,8 @@ accounts = Test # MachineUI -- Interactive interface suitable for machine # parsing. # +# See also offlineimapui(7) +# # You can override this with a command-line option -u. # #ui = basic diff --git a/offlineimap/init.py b/offlineimap/init.py index 7f6a679..579878d 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -58,108 +58,54 @@ class OfflineImap: parser.add_option("--dry-run", action="store_true", dest="dryrun", default=False, - help="Do not actually modify any store but check and print " - "what synchronization actions would be taken if a sync would be" - " performed. It will not precisely give the exact information w" - "hat will happen. If e.g. we need to create a folder, it merely" - " outputs 'Would create folder X', but not how many and which m" - "ails it would transfer.") + help="dry run mode") parser.add_option("--info", action="store_true", dest="diagnostics", default=False, - help="Output information on the configured email repositories" - ". Useful for debugging and bug reporting. Use in conjunction wit" - "h the -a option to limit the output to a single account. This mo" - "de will prevent any actual sync to occur and exits after it outp" - "ut the debug information.") + help="output information on the configured email repositories") parser.add_option("-1", action="store_true", dest="singlethreading", default=False, - help="Disable all multithreading operations and use " - "solely a single-thread sync. This effectively sets the " - "maxsyncaccounts and all maxconnections configuration file " - "variables to 1.") + help="disable all multithreading operations") parser.add_option("-P", dest="profiledir", metavar="DIR", - help="Sets OfflineIMAP into profile mode. The program " - "will create DIR (it must not already exist). " - "As it runs, Python profiling information about each " - "thread is logged into profiledir. Please note: " - "This option is present for debugging and optimization " - "only, and should NOT be used unless you have a " - "specific reason to do so. It will significantly " - "decrease program performance, may reduce reliability, " - "and can generate huge amounts of data. This option " - "implies the -1 option.") + help="sets OfflineIMAP into profile mode.") parser.add_option("-a", dest="accounts", metavar="ACCOUNTS", - help="Overrides the accounts section in the config file. " - "Lets you specify a particular account or set of " - "accounts to sync without having to edit the config " - "file. You might use this to exclude certain accounts, " - "or to sync some accounts that you normally prefer not to.") + help="overrides the accounts section in the config file") parser.add_option("-c", dest="configfile", metavar="FILE", default=None, - help="Specifies a configuration file to use") + help="specifies a configuration file to use") parser.add_option("-d", dest="debugtype", metavar="type1,[type2...]", - help="Enables debugging for OfflineIMAP. This is useful " - "if you are to track down a malfunction or figure out what is " - "going on under the hood. This option requires one or more " - "debugtypes, separated by commas. These define what exactly " - "will be debugged, and so far include two options: imap, thread, " - "maildir or ALL. The imap option will enable IMAP protocol " - "stream and parsing debugging. Note that the output may contain " - "passwords, so take care to remove that from the debugging " - "output before sending it to anyone else. The maildir option " - "will enable debugging for certain Maildir operations. " - "The use of any debug option (unless 'thread' is included), " - "implies the single-thread option -1.") + help="enables debugging for OfflineIMAP.") parser.add_option("-l", dest="logfile", metavar="FILE", - help="Log to FILE") + help="log to FILE") parser.add_option("-f", dest="folders", metavar="folder1,[folder2...]", - help="Only sync the specified folders. The folder names " - "are the *untranslated* foldernames of the remote repository. " - "This command-line option overrides any 'folderfilter' " - "and 'folderincludes' options in the configuration file.") + help="only sync the specified folders") parser.add_option("-k", dest="configoverride", action="append", metavar="[section:]option=value", - help= - """Override configuration file option. If"section" is - omitted, it defaults to "general". Any underscores - in the section name are replaced with spaces: - for instance, to override option "autorefresh" in - the "[Account Personal]" section in the config file - one would use "-k Account_Personal:autorefresh=30".""") + help="override configuration file option") parser.add_option("-o", action="store_true", dest="runonce", default=False, - help="Run only once, ignoring any autorefresh setting " - "in the configuration file.") + help="run only once") parser.add_option("-q", action="store_true", dest="quick", default=False, - help="Run only quick synchronizations. Ignore any " - "flag updates on IMAP servers (if a flag on the remote IMAP " - "changes, and we have the message locally, it will be left " - "untouched in a quick run.") + help="run only quick synchronizations") parser.add_option("-u", dest="interface", - help="Specifies an alternative user interface to " - "use. This overrides the default specified in the " - "configuration file. The UI specified with -u will " - "be forced to be used, even if checks determine that it is " - "not usable. Possible interface choices are: %s " % - ", ".join(UI_LIST.keys())) + help="specifies an alternative user interface") (options, args) = parser.parse_args() globals.set_options (options) From f4579d966047c6e72400518c7c7006190b6919cc Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 15:43:01 +0100 Subject: [PATCH 738/817] learn how to fork the wiki Signed-off-by: Nicolas Sebrecht --- .gitignore | 3 ++- get-wiki.sh | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100755 get-wiki.sh diff --git a/.gitignore b/.gitignore index cecdf30..f118931 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,9 @@ .*.swo *.html *~ -# website. +# websites. /website/ +/wiki/ # Generated files. /docs/dev-doc/ /build/ diff --git a/get-wiki.sh b/get-wiki.sh new file mode 100755 index 0000000..92d48de --- /dev/null +++ b/get-wiki.sh @@ -0,0 +1,16 @@ +#!/bin/zsh + +if test -d wiki +then + echo "Directory 'wiki' already exists..." + exit 1 +else + git clone https://github.com/OfflineIMAP/offlineimap.wiki.git wiki + cat </offlineimap-wiki.git +EOF +fi From c9ce43313f124bc5e49e0fed59fa857cc8017016 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 16:07:00 +0100 Subject: [PATCH 739/817] fix the Makefile(s) Signed-off-by: Nicolas Sebrecht --- Makefile | 7 ++----- docs/Makefile | 4 +++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index bfe4523..a1ccee7 100644 --- a/Makefile +++ b/Makefile @@ -35,18 +35,15 @@ clean: -find . -name '*.pygc' -exec rm -f {} \; -find . -name '*.class' -exec rm -f {} \; -find . -name '.cache*' -exec rm -f {} \; - # Care with that. We have html in subdirs we want to keep. - -find ./docs -name '*.html' -exec rm -f {} \; -rm -f manpage.links manpage.refs -find . -name auth -exec rm -vf {}/password {}/username \; - @$(MAKE) -C docs clean + @$(MAKE) -C clean man: - @$(MAKE) -C docs man + @$(MAKE) -C man doc: @$(MAKE) -C docs - $(RST2HTML) Changelog.rst Changelog.html targz: ../$(TARGZ) ../$(TARGZ): diff --git a/docs/Makefile b/docs/Makefile index ec6aaaa..1311bbf 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -32,7 +32,9 @@ websitedoc: clean: $(RM) -f $(HTML_TARGETS) - $(RM) -f offlineimap.1 ../offlineimap.1 + $(RM) -f offlineimap.1 + $(RM) -f offlineimap.7 $(RM) -rf html/* + -find ./docs -name '*.html' -exec rm -f {} \; .PHONY: clean doc From 8229800e0ce52508d6b86be7bde52438f831ffd3 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 16:08:19 +0100 Subject: [PATCH 740/817] Changelogs: move format back to markdown/kramdown to be more compatible with Jekyll Learn how to export them to the website. Signed-off-by: Nicolas Sebrecht --- Changelog.maint.md | 21 +++ Changelog.maint.rst | 24 --- Changelog.rst => Changelog.md | 324 ++++++++++++---------------------- docs/website-doc.sh | 10 +- 4 files changed, 137 insertions(+), 242 deletions(-) create mode 100644 Changelog.maint.md delete mode 100644 Changelog.maint.rst rename Changelog.rst => Changelog.md (88%) diff --git a/Changelog.maint.md b/Changelog.maint.md new file mode 100644 index 0000000..57c3e64 --- /dev/null +++ b/Changelog.maint.md @@ -0,0 +1,21 @@ +--- +layout: page +title: Changelog of the stable branch +--- + +This is the Changelog of the maintenance branch. + +**NOTE FROM THE MAINTAINER:** + + This branch comes almost as-is. With no URGENT requirements to update this + branch (e.g. big security fix), it is left behind. + If anyone volunteers to maintain it and backport patches, let us know! + + +### OfflineIMAP v6.3.2.1 (2011-03-23) + +#### Bug Fixes + +* Sanity checks for SSL cacertfile configuration. +* Fix regression (UIBase is no more). +* Make profiling mode really enforce single-threading. diff --git a/Changelog.maint.rst b/Changelog.maint.rst deleted file mode 100644 index 605ee8c..0000000 --- a/Changelog.maint.rst +++ /dev/null @@ -1,24 +0,0 @@ -========= -ChangeLog -========= - -:website: http://offlineimap.org - -This is the Changelog of the maintenance branch. - -**NOTE FROM THE MAINTAINER:** - Contributors should use the `WIP` section in Changelog.draft.rst in order to - add changes they are working on. I will use it to make the new changelog entry - on releases. And because I'm lazy, it will also be used as a draft for the - releases announces. - - -OfflineIMAP v6.3.2.1 (2011-03-23) -================================= - -Bug Fixes ---------- - -* Sanity checks for SSL cacertfile configuration. -* Fix regression (UIBase is no more). -* Make profiling mode really enforce single-threading. diff --git a/Changelog.rst b/Changelog.md similarity index 88% rename from Changelog.rst rename to Changelog.md index 5e499c1..2a55d48 100644 --- a/Changelog.rst +++ b/Changelog.md @@ -1,20 +1,16 @@ -========= -ChangeLog -========= - -:website: http://offlineimap.org +--- +layout: page +title: Changelog of mainline +--- -OfflineIMAP v6.5.7-rc3 (2015- - ) -=================================== +### OfflineIMAP v6.5.7-rc3 (2015- - ) -OfflineIMAP v6.5.7-rc2 (2015-01-18) -=================================== +### OfflineIMAP v6.5.7-rc2 (2015-01-18) -Notes ------ +#### Notes This release candidate should be minor for most users. @@ -25,15 +21,13 @@ Documentation had our attention, too. There's some code cleanups and code refactoring, as usual. -Features --------- +#### Features * Do not keep reloading pyhtonfile, make it stateful. * HACKING: how to create tags. * MANUAL: add minor sample on how to retrieve a password with a helper python file. -Fixes ------ +#### Fixes * Make OS-default CA certificate file to be requested explicitely. * SSL: do not fallback on other authentication mode if it fails. @@ -42,8 +36,7 @@ Fixes * ui: Machine: remove offending param for a _printData() call. * Drop caches after having processed folders. -Changes -------- +#### Changes * Fix unexpected garbage code. * Properly re-raise exception to save original tracebacks. @@ -59,11 +52,9 @@ Changes * COPYING: fix unexpected characters. -OfflineIMAP v6.5.7-rc1 (2015-01-07) -=================================== +### OfflineIMAP v6.5.7-rc1 (2015-01-07) -Notes ------ +#### Notes I think it's time for a new release candidate. Our release cycles are long enough and users are asked to use the current TIP of the next branch to test @@ -77,8 +68,7 @@ Debugging messages are added and polished. There's some code cleanups and refactoring, also. -Features --------- +#### Features * Expand environment variables in the following configuration items: @@ -96,8 +86,7 @@ Features * Added default CA bundle location for OpenBSD (GitHub pull #120) and DragonFlyBSD. -Fixes ------ +#### Fixes * Fix unbounded recursion during flag update (Josh Berry). * Do not ignore gmail labels if header appears multiple times @@ -122,8 +111,7 @@ Fixes * Add missing version bump for 6.5.6 (it was released with 6.5.5 in setup.py and other places). -Changes -------- +#### Changes * Warn about a tricky piece of code in addmessageheader * Rename addmessageheader()'s crlf parameter to linebreak @@ -160,16 +148,14 @@ Changes -OfflineIMAP v6.5.6 (2014-05-14) -=============================== +### OfflineIMAP v6.5.6 (2014-05-14) * Fix IDLE mode regression (it didn't worked) introduced after v6.5.5 (pointy hat goes to Eygene Ryabinkin, kudos -- to Tomasz Żok) -OfflineIMAP v6.5.6-rc1 (2014-05-14) -=================================== +### OfflineIMAP v6.5.6-rc1 (2014-05-14) * Add knob to invoke folderfilter dynamically on each sync (GitHub#73) * Add knob to apply compression to IMAP connections (Abdó Roig-Maranges) @@ -187,8 +173,7 @@ OfflineIMAP v6.5.6-rc1 (2014-05-14) 'cert_fingerprint' -OfflineIMAP v6.5.5 (2013-10-07) -=============================== +### OfflineIMAP v6.5.5 (2013-10-07) * Avoid lockups for IMAP synchronizations running with the "-1" command-line switch (X-Ryl669 ) @@ -239,8 +224,7 @@ OfflineIMAP v6.5.5 (2013-10-07) 'group' (D. Franke) * Improved error throwing on repository misconfiguration -OfflineIMAP v6.5.4 (2012-06-02) -=============================== +### OfflineIMAP v6.5.4 (2012-06-02) * bump bundled imaplib2 library 2.29 --> 2.33 * Actually perform the SSL fingerprint check (reported by J. Cook) @@ -249,15 +233,13 @@ OfflineIMAP v6.5.4 (2012-06-02) * Fix crash when IMAP.quickchanged() led to an Error (reported by sharat87) * Implement the createfolders setting to disable folder propagation (see docs) -OfflineIMAP v6.5.3.1 (2012-04-03) -================================= +### OfflineIMAP v6.5.3.1 (2012-04-03) * Don't fail if no dry-run setting exists in offlineimap.conf (introduced in 6.5.3) -OfflineIMAP v6.5.3 (2012-04-02) -=============================== +### OfflineIMAP v6.5.3 (2012-04-02) * --dry-run mode protects us from performing any actual action. It will not precisely give the exact information what will happen. If e.g. it @@ -290,8 +272,7 @@ OfflineIMAP v6.5.3 (2012-04-02) * Improve compatability of the curses UI with python 2.6 -OfflineIMAP v6.5.2.1 (2012-04-04) -================================= +### OfflineIMAP v6.5.2.1 (2012-04-04) * Fix python2.6 compatibility with the TTYUI backend (crash) @@ -312,8 +293,7 @@ OfflineIMAP v6.5.2.1 (2012-04-04) * Remove the Gmail "realdelete" option, as it could lead to potential data loss. -OfflineIMAP v6.5.2 (2012-01-17) -=============================== +### OfflineIMAP v6.5.2 (2012-01-17) * Gmail "realdelete" option is considered harmful and has the potential for data loss. Analysis at @@ -334,8 +314,7 @@ OfflineIMAP v6.5.2 (2012-01-17) * Don't emit noisy regular sleeping announcements in Basic UI. -OfflineIMAP v6.5.1.2 (2012-01-07) - "Baby steps" -================================================ +### OfflineIMAP v6.5.1.2 (2012-01-07) - "Baby steps" Smallish bug fixes that deserve to be put out. @@ -346,8 +325,7 @@ Smallish bug fixes that deserve to be put out. support them (yet) (this prevents many scary bogus sync messages) * Add filter information to the filter list in --info output -OfflineIMAP v6.5.1.1 (2012-01-07) - "Das machine control is nicht fur gerfinger-poken und mittengrabben" -======================================================================================================== +### OfflineIMAP v6.5.1.1 (2012-01-07) - "Das machine control is nicht fur gerfinger-poken und mittengrabben" Blinkenlights UI 6.5.0 regression fixes only. @@ -355,8 +333,7 @@ Blinkenlights UI 6.5.0 regression fixes only. * Make exit via 'q' key work again cleanly -OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" -======================================================= +### OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" * Fixed Maildir regression "flagmatchre" not found. (regressed in 6.5.0) @@ -373,8 +350,7 @@ OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" all releases >= 6.4.0, so don't run older releases simultanous to this one. -OfflineIMAP v6.5.0 (2012-01-06) -=============================== +### OfflineIMAP v6.5.0 (2012-01-06) This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. Please upgrade to avoid potential data loss! The version has been bumped to @@ -391,8 +367,7 @@ Please upgrade to avoid potential data loss! The version has been bumped to * Fix regression that broke MachineUI -OfflineIMAP v6.4.4 (2012-01-06) -=============================== +### OfflineIMAP v6.4.4 (2012-01-06) This is a bugfix release, fixing regressions occurring in or since 6.4.0. @@ -408,17 +383,14 @@ This is a bugfix release, fixing regressions occurring in or since 6.4.0. (such as 'a'), note that they will still be deleted as they are not supported in the sync to an IMAP server. -OfflineIMAP v6.4.3 (2012-01-04) -=============================== +### OfflineIMAP v6.4.3 (2012-01-04) -New Features ------------- +#### New Features * add a --info command line switch that outputs useful information about the server and the configuration for all enabled accounts. -Changes -------- +#### Changes * Reworked logging which was reported to e.g. not flush output to files often enough. User-visible changes: @@ -435,8 +407,7 @@ Changes SIGTERM and CTRL-C can take a little longer, but will be clean. -OfflineIMAP v6.4.2 (2011-12-01) -=============================== +### OfflineIMAP v6.4.2 (2011-12-01) * IMAP<->IMAP sync with a readonly local IMAP repository failed with a rather mysterious "TypeError: expected a character buffer object" @@ -447,18 +418,15 @@ OfflineIMAP v6.4.2 (2011-12-01) folder separator, all folder names would get a trailing slash appended, which is plain wrong. -OfflineIMAP v6.4.1 (2011-11-17) -=============================== +### OfflineIMAP v6.4.1 (2011-11-17) -Changes -------- +#### Changes * Indicate progress when copying many messages (slightly change log format) * Output how long an account sync took (min:sec). -Bug Fixes ---------- +#### Bug Fixes * Syncing multiple accounts in single-threaded mode would fail as we try to "register" a thread as belonging to two accounts which was @@ -471,35 +439,29 @@ Bug Fixes which could result in some folders being opened read-only when we really needed read-write. -OfflineIMAP v6.4.0 (2011-09-29) -=============================== +### OfflineIMAP v6.4.0 (2011-09-29) This is the first stable release to support the forward-compatible per-account locks and remote folder creation that has been introduced in the 6.3.5 series. * Various regression and bug fixes from the last couple of RCs -OfflineIMAP v6.3.5-rc3 (2011-09-21) -=================================== +### OfflineIMAP v6.3.5-rc3 (2011-09-21) -Changes -------- +#### Changes * Refresh server capabilities after login, so we know that Gmail supports UIDPLUS (it only announces that after login, not before). This prevents us from adding custom headers to Gmail uploads. -Bug Fixes ---------- +#### Bug Fixes * Fix the creation of folders on remote repositories, which was still botched on rc2. -OfflineIMAP v6.3.5-rc2 (2011-09-19) -=================================== +### OfflineIMAP v6.3.5-rc2 (2011-09-19) -New Features ------------- +#### New Features * Implement per-account locking, so that it will possible to sync different accounts at the same time. The old global lock is still in @@ -519,29 +481,24 @@ New Features repository, you might need a nametrans setting on the local repository that leads to the original name (reverse nametrans). -Changes -------- +#### Changes * Documentation improvements concerning 'restoreatime' and some code cleanup * Maildir repositories now also respond to folderfilter= configurations. -Bug Fixes ---------- +#### Bug Fixes * New emails are not created with "-rwxr-xr-x" but as "-rw-r--r--" anymore, fixing a regression in 6.3.4. -OfflineIMAP v6.3.5-rc1 (2011-09-12) -=================================== +### OfflineIMAP v6.3.5-rc1 (2011-09-12) -Notes ------ +#### Notes Idle feature and SQLite backend leave the experimental stage! ,-) -New Features ------------- +#### New Features * When a message upload/download fails, we do not abort the whole folder synchronization, but only skip that message, informing the user at the @@ -551,8 +508,7 @@ New Features that the server certificate is actually known and identical by comparing the stored sha1 fingerprint with the current one. -Changes -------- +#### Changes * Refactor our IMAPServer class. Background work without user-visible changes. @@ -560,8 +516,7 @@ Changes cluttered the main configuration file for little gain. * Updated bundled imaplib2 to version 2.28. -Bug Fixes ---------- +#### Bug Fixes * We protect more robustly against asking for inexistent messages from the IMAP server, when someone else deletes or moves messages while we sync. @@ -571,46 +526,37 @@ Bug Fixes -OfflineIMAP v6.3.4 (2011-08-10) -=============================== +### OfflineIMAP v6.3.4 (2011-08-10) -Notes ------ +#### Notes Here we are. A nice release since v6.3.3, I think. -Changes -------- +#### Changes * Handle when UID can't be found on saved messages. -OfflineIMAP v6.3.4-rc4 (2011-07-27) -=================================== +### OfflineIMAP v6.3.4-rc4 (2011-07-27) -Notes ------ +#### Notes There is nothing exciting in this release. This is somewhat expected due to the late merge on -rc3. -New Features ------------- +#### New Features * Support maildir for Windows. -Changes -------- +#### Changes * Manual improved. -OfflineIMAP v6.3.4-rc3 (2011-07-07) -=================================== +### OfflineIMAP v6.3.4-rc3 (2011-07-07) -Notes ------ +#### Notes Here is a surprising release. :-) @@ -625,24 +571,20 @@ We usually don't do much changes so late in a cycle. Now, things are highly calming down and I hope a lot of people will test this release. Next one could be the stable! -New Features ------------- +#### New Features * Added StartTLS support, it will automatically be used if the server supports it. -Bug Fixes ---------- +#### Bug Fixes * We protect more robustly against asking for inexistent messages from the IMAP server, when someone else deletes or moves messages while we sync. -OfflineIMAP v6.3.4-rc2 (2011-06-15) -=================================== +### OfflineIMAP v6.3.4-rc2 (2011-06-15) -Notes ------ +#### Notes This was a very active rc1 and we could expect a lot of new fixes for the next release. @@ -654,18 +596,15 @@ information about his bug here: The IDLE support is merged as experimental feature. -New Features ------------- +#### New Features * Implement experimental IDLE feature. -Changes -------- +#### Changes * Maildirs use less memory while syncing. -Bug Fixes ---------- +#### Bug Fixes * Saving to Maildirs now checks for file existence without race conditions. * A bug in the underlying imap library has been fixed that could @@ -675,11 +614,9 @@ Bug Fixes release is strongly recommended. -OfflineIMAP v6.3.4-rc1 (2011-05-16) -=================================== +### OfflineIMAP v6.3.4-rc1 (2011-05-16) -Notes ------ +#### Notes Welcome to the v6.3.4 pre-release cycle. Your favorite IMAP tool wins 2 new features which were asked for a long time: @@ -696,8 +633,7 @@ releases. As usual, we ask our users to test this release as much as possible, especially the SQL backend. Have fun! -New Features ------------- +#### New Features * Begin sphinx-based documentation for the code. * Enable 1-way synchronization by settting a [Repository ...] to @@ -707,8 +643,7 @@ New Features * Optional: experimental SQLite-based backend for the LocalStatus cache. Plain text remains the default. -Changes -------- +#### Changes * Start a enhanced error handling background system. This is designed to not stop a whole sync process on all errors (not much used, yet). @@ -722,8 +657,7 @@ Changes the -f option. * Give more detailed error when encountering a corrupt UID mapping file. -Bug Fixes ---------- +#### Bug Fixes * Drop connection if synchronization failed. This is needed if resuming the system from suspend mode gives a wrong connection. @@ -731,11 +665,9 @@ Bug Fixes * Make 'thread' command line option work. -OfflineIMAP v6.3.3 (2011-04-24) -=============================== +### OfflineIMAP v6.3.3 (2011-04-24) -Notes ------ +#### Notes Make this last candidate cycle short. It looks like we don't need more tests as most issues were raised and solved in the second round. Also, we have huge work @@ -744,24 +676,20 @@ to merge big and expected features into OfflineIMAP. Thanks to all contributors, again. With such a contribution rate, we can release stable faster. I hope it will be confirmed in the longer run! -Changes -------- +#### Changes * Improved documentation for querying password. -OfflineIMAP v6.3.3-rc3 (2011-04-19) -=================================== +### OfflineIMAP v6.3.3-rc3 (2011-04-19) -Notes ------ +#### Notes It's more than a week since the previous release. Most of the issues raised were discussed and fixed since last release. I think we can be glad and confident for the future while the project live his merry life. -Changes -------- +#### Changes * The -f option did not work with Folder names with spaces. It works now, use with quoting e.g. -f "INBOX, Deleted Mails". @@ -769,17 +697,14 @@ Changes * Bump from imaplib2 v2.20 to v2.22. * Code refactoring. -Bug Fixes ---------- +#### Bug Fixes * Fix IMAP4 tunnel with imaplib2. -OfflineIMAP v6.3.3-rc2 (2011-04-07) -=================================== +### OfflineIMAP v6.3.3-rc2 (2011-04-07) -Notes ------ +#### Notes We are now at the third week of the -rc1 cycle. I think it's welcome to begin the -rc2 cycle. Things are highly calming down in the code even if we had @@ -802,23 +727,19 @@ I'd like to thank reporters who involved in this cycle: The imaplib2 migration looks to go the right way to be definetly released but still needs more tests. So, here we go... -Changes -------- +#### Changes * Increase compatability with Gmail servers which claim to not support the UIDPLUS extension but in reality do. -Bug Fixes ---------- +#### Bug Fixes * Fix hang when using Ctrl+C in some cases. -OfflineIMAP v6.3.3-rc1 (2011-03-16) -=================================== +### OfflineIMAP v6.3.3-rc1 (2011-03-16) -Notes ------ +#### Notes Here is time to begin the tests cycle. If feature topics are sent, I may merge or delay them until the next stable release. @@ -833,16 +754,14 @@ hang and consuming a lot of CPU are asked to update. That beeing said, this is still an early release candidate you should use for non-critical data only! -New Features ------------- +#### New Features * Implement UIDPLUS extension support. OfflineIMAP will now not insert an X-OfflineIMAP header if the mail server supports the UIDPLUS extension. * SSL: support subjectAltName. -Changes -------- +#### Changes * Use imaplib2 instead of imaplib. * Makefile use magic to find the version number. @@ -857,8 +776,7 @@ Changes * TTYUI ouput improved. * Code cleanups. -Bug Fixes ---------- +#### Bug Fixes * Fix ignoring output while determining the rst2xxx command name to build documentation. @@ -870,11 +788,9 @@ Bug Fixes either ignore or refuse them. -OfflineIMAP v6.3.2 (2010-02-21) -=============================== +### OfflineIMAP v6.3.2 (2010-02-21) -Notes ------ +#### Notes First of all I'm really happy to announce our new official `website `_. Most of the work started from the impulse @@ -892,19 +808,16 @@ This release will also be the root of our long maintenance support. Other bugs were fixed. -Bug Fixes ---------- +#### Bug Fixes * Fix craches for getglobalui(). * Fix documentation build. * Restore compatibiliy with python 2.5. -OfflineIMAP v6.3.2-rc3 (2010-02-06) -=================================== +### OfflineIMAP v6.3.2-rc3 (2010-02-06) -Notes ------ +#### Notes We are still touched by the "SSL3 write pending" bug it would be really nice to fix before releasing the coming stable. In the worse case, we'll have to add the @@ -919,40 +832,34 @@ In this release I won't merge any patch not fixing a bug or a security issue. More feedbacks on the main issue would be appreciated. -Changes -------- +#### Changes * Sample offlineimap.conf states it expects a PEM formatted certificat. * Give better trace information if an error occurs. * Have --version ONLY print the version number. * Code cleanups. -Bug Fixes ---------- +#### Bug Fixes * Fix Curses UI (simplified by moving from MultiLock to Rlock implementation). * Makefile: docutils build work whether python extension command is stripped or not. * Makefile: clean now removes HTML documentation files. -OfflineIMAP v6.3.2-rc2 (2010-12-21) -=================================== +### OfflineIMAP v6.3.2-rc2 (2010-12-21) -Notes ------ +#### Notes We are beginning a new tests cycle. At this stage, I expect most people will try to intensively stuck OfflineIMAP. :-) -New Features ------------- +#### New Features * Makefile learn to build the package and make it the default. * Introduce a Changelog to involve community in the releasing process. * Migrate documentation to restructuredtext. -Changes -------- +#### Changes * Improve CustomConfig documentation. * Imply single threading mode in debug mode exept for "-d thread". @@ -962,8 +869,7 @@ Changes * Improve version managment and make it easier. * Introduce a true single threading mode. -Bug Fixes ---------- +#### Bug Fixes * Understand multiple EXISTS replies from servers like Zimbra. * Only verify hostname if we actually use CA cert. @@ -975,37 +881,30 @@ Bug Fixes * Remove uneeded files. -OfflineIMAP v6.3.2-rc1 (2010-12-19) -=================================== +### OfflineIMAP v6.3.2-rc1 (2010-12-19) -Notes ------ +#### Notes We are beginning a tests cycle. If feature topics are sent, I may merge or delay them until the next stable release. -New Features ------------- +#### New Features * Primitive implementation of SSL certificates check. -Changes -------- +#### Changes * Use OptionParser instead of getopts. * Code cleanups. -Bug Fixes ---------- +#### Bug Fixes * Fix reading password from UI. -OfflineIMAP v6.3.1 (2010-12-11) -=============================== +### OfflineIMAP v6.3.1 (2010-12-11) -Notes ------ +#### Notes Yes, I know I've just annouced the v6.3.0 in the same week. As said, it was not really a true release for the software. This last release @@ -1014,26 +913,22 @@ includes fixes and improvements it might be nice to update to. Thanks to every body who helped to make this release with patches and tips through the mailing list. This is clearly a release they own. -Changes -------- +#### Changes * cProfile becomes the default profiler. Sebastian Spaeth did refactoring to prepare to the coming unit test suites. * UI output formating enhanced. * Some code cleanups. -Bug Fixes ---------- +#### Bug Fixes * Fix possible overflow while working with Exchange. * Fix time sleep while exiting threads. -OfflineIMAP v6.3.0 (2010-12-09) -=============================== +### OfflineIMAP v6.3.0 (2010-12-09) -Notes ------ +#### Notes This release is more "administrative" than anything else and mainly marks the change of the maintainer. New workflow and policy for developers come in. BTW, @@ -1041,8 +936,7 @@ I don't think I'll maintain debian/changelog. At least, not in the debian way. Most users and maintainers may rather want to skip this release. -Bug Fixes ---------- +#### Bug Fixes * Fix terminal display on exit. * netrc password authentication. diff --git a/docs/website-doc.sh b/docs/website-doc.sh index f705b10..f5f1c85 100755 --- a/docs/website-doc.sh +++ b/docs/website-doc.sh @@ -5,7 +5,8 @@ SPHINXBUILD=sphinx-build TMPDIR='/tmp/offlineimap-sphinx-doctrees' WEBSITE='../website' -DESTBASE="${WEBSITE}/_doc/versions" +DOCBASE="${WEBSITE}/_doc" +DESTBASE="${DOCBASE}/versions" VERSIONS_YML="${WEBSITE}/_data/versions.yml" version="v$(../offlineimap.py --version)" @@ -33,5 +34,8 @@ done > "$VERSIONS_YML" # Copy usefull sources of documentation. # -# User doc -for foo in ../Changelog.rst ../Changelog.maint.rst +# Changelogs. +for foo in ../Changelog.md ../Changelog.maint.md +do + cp -afv "$foo" "$DOCBASE" +done From 3a00ddaa846e66de6cc7480fd0145f7cefd1a1a9 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 16:49:53 +0100 Subject: [PATCH 741/817] docs: minor improvements Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 9 +++++++-- README.md | 3 +-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index af4ab6b..af24292 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -7,17 +7,22 @@ .. _maintainers: https://github.com/OfflineIMAP/offlineimap/blob/next/MAINTAINERS.rst .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst +.. _Community's website: https://offlineimap.github.io ================= HOW TO CONTRIBUTE ================= -You'll find here the basics to contribute to OfflineIMAP_, addressed to users -and both experienced/learning developers to quickly provide contributions. +You'll find here the **basics** to contribute to OfflineIMAP_, addressed to +users as well as learning or experienced developers to quickly provide +contributions. + +**For more detailed documentation, see the `Community's website`_.** .. contents:: :depth: 3 + Submit issues ============= diff --git a/README.md b/README.md index ef815c3..062cfdd 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,7 @@ Downloads releases as [tarball or zipball](https://github.com/OfflineIMAP/offlin **The user discussions, development, announces and all the exciting stuff take place in the mailing list.** While not mandatory to send emails, you can -[subscribe -here](http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project). +[subscribe here](http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project). Bugs, issues and contributions can be requested to both the mailing list or the [official Github project][offlineimap]. From 4d0a7eae42157c3f2f60060258a43d1ec0b1296f Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 10 Mar 2015 18:31:18 +0100 Subject: [PATCH 742/817] website-doc.sh: write releases in reversed chronological order Signed-off-by: Nicolas Sebrecht --- docs/website-doc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/website-doc.sh b/docs/website-doc.sh index f5f1c85..f232e1d 100755 --- a/docs/website-doc.sh +++ b/docs/website-doc.sh @@ -25,7 +25,7 @@ $SPHINXBUILD -b html -d "$TMPDIR" doc-src "$dest" # Dynamically build JSON definitions for Jekyll. # echo "Building Jekyll data: $VERSIONS_YML" -for version in $(ls "$DESTBASE") +for version in $(ls "$DESTBASE" -1 | sort -nr) do echo "- $version" done > "$VERSIONS_YML" From a629b4e0e72156413b88807022497e27ea6d1c3f Mon Sep 17 00:00:00 2001 From: Mark Oteiza Date: Sat, 31 May 2014 10:42:25 -0400 Subject: [PATCH 743/817] do not error if `remoteuser` is not configured Signed-off-by: Mark Oteiza Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/IMAP.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index ff1d5e2..430c6dc 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -156,7 +156,8 @@ class IMAPRepository(BaseRepository): if user != None: return localeval.eval(user) - user = self.getconf('remoteuser') + if self.config.has_option(self.getsection(), 'remoteuser'): + user = self.getconf('remoteuser') if user != None: return user @@ -284,7 +285,7 @@ class IMAPRepository(BaseRepository): raise else: if netrcentry: - user = self.getconf('remoteuser') + user = self.getuser() if user == None or user == netrcentry[0]: return netrcentry[2] # 5. read password from /etc/netrc @@ -295,7 +296,7 @@ class IMAPRepository(BaseRepository): raise else: if netrcentry: - user = self.getconf('remoteuser') + user = self.getuser() if user == None or user == netrcentry[0]: return netrcentry[2] # no strategy yielded a password! From fcd22a201ff27f5d2e2f39b79f6d72ad88475962 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 11 Mar 2015 11:38:56 +0100 Subject: [PATCH 744/817] get-website/wiki.sh: improve scripts and messages Signed-off-by: Nicolas Sebrecht --- get-website.sh | 17 ++++++++++++----- get-wiki.sh | 13 ++++++++++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/get-website.sh b/get-website.sh index 74ec3d4..336e226 100755 --- a/get-website.sh +++ b/get-website.sh @@ -5,12 +5,19 @@ then echo "Directory 'website' already exists..." exit 1 else - git clone https://github.com/OfflineIMAP/offlineimap.github.io.git -cat </offlineimap.github.io.git + +You can now push your WIPs with: + + $ git push myfork : EOF fi diff --git a/get-wiki.sh b/get-wiki.sh index 92d48de..609d764 100755 --- a/get-wiki.sh +++ b/get-wiki.sh @@ -7,10 +7,17 @@ then else git clone https://github.com/OfflineIMAP/offlineimap.wiki.git wiki cat </offlineimap-wiki.git + +You can now push your WIPs with: + + $ git push myfork : EOF fi From 7292342cd056ea863d4db6802c668b3cc674dd45 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 12 Mar 2015 15:15:16 +0100 Subject: [PATCH 745/817] scripts: merge get-website.sh and get-wiki.sh into scripts/get-repository.sh Signed-off-by: Nicolas Sebrecht --- get-website.sh | 23 ------- get-wiki.sh | 23 ------- scripts/get-repository.sh | 138 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 46 deletions(-) delete mode 100755 get-website.sh delete mode 100755 get-wiki.sh create mode 100755 scripts/get-repository.sh diff --git a/get-website.sh b/get-website.sh deleted file mode 100755 index 336e226..0000000 --- a/get-website.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/zsh - -if test -d website -then - echo "Directory 'website' already exists..." - exit 1 -else - git clone https://github.com/OfflineIMAP/offlineimap.github.io.git website - cat </offlineimap.github.io.git - -You can now push your WIPs with: - - $ git push myfork : -EOF -fi diff --git a/get-wiki.sh b/get-wiki.sh deleted file mode 100755 index 609d764..0000000 --- a/get-wiki.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/zsh - -if test -d wiki -then - echo "Directory 'wiki' already exists..." - exit 1 -else - git clone https://github.com/OfflineIMAP/offlineimap.wiki.git wiki - cat </offlineimap-wiki.git - -You can now push your WIPs with: - - $ git push myfork : -EOF -fi diff --git a/scripts/get-repository.sh b/scripts/get-repository.sh new file mode 100755 index 0000000..65704ad --- /dev/null +++ b/scripts/get-repository.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# +# Licence: this file is in the public deomain. +# +# Download and configure the repositories of the website or wiki. + +repository=$1 +github_remote=$2 + +# +# TODO +# +function final_note () { + cat < + $ cd ./$1 + $ git remote add myfork https://github.com//.git +EOF +} + +function setup () { + target_dir=$1 + remote_url=$2 + + # Adjust $PWD if necessary. + test -d scripts || cd .. + if test ! -d scripts + then + echo "cannot figure the correct workdir..." + exit 2 + fi + + if test -d $target_dir + then + echo "Directory '$target_dir' already exists..." + exit 3 + fi + + git clone "${remote_url}.git" "$1" + echo '' + if test $? -gt 0 + then + echo "Cannot fork $remote_url to $1" + exit 4 + fi +} + +function write_renderer () { + cat < "$renderer" + chmod u+x "$renderer" + cd .. + else + echo "ERROR: could not write renderer." + fi +} + +function configure_wiki () { + : # noop +} + +test n$github_remote = 'n' && github_remote='origin' + +# Get Github username. +#offlineimap_url="$(git config --local --get remote.origin.url)" +offlineimap_url="$(git config --local --get remote.nicolas33.url)" +username=$(echo $offlineimap_url | sed -r -e 's,.*github.com.([^/]+)/.*,\1,') + + +case n$repository in + nwebsite) + upstream=https://github.com/OfflineIMAP/offlineimap.github.io + setup website "$upstream" + configure_website "$username" + final_note website "$upstream" + ;; + nwiki) + upstream=https://github.com/OfflineIMAP/offlineimap.wiki + setup wiki "$upstream" + configure_wiki + final_note wiki "$upstream" + ;; + *) + cat <] + +: The name of the Git repository of YOUR fork at Github. + Default: origin +EOF + exit 1 + ;; +esac + From 65274312a6c60476a25b2e664ae6cfe2e8d55b86 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 12 Mar 2015 17:53:28 +0100 Subject: [PATCH 746/817] README: improve description Signed-off-by: Nicolas Sebrecht --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 062cfdd..1bbbd7f 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,17 @@ ## Description -OfflineIMAP is a software to dispose your e-mail mailbox(es) as a local Maildir. +OfflineIMAP is a software to dispose your e-mail mailbox(es) as a **local +Maildir**. OfflineIMAP will synchronize both sides via *IMAP*. -For example, this allows reading the mails while offline without the need for your mail reader (MUA) to support disconnected operations. +The main downside about IMAP is that you have to **trust** your MAIL provider to +not loose your mails. This is not something impossible while not very common. +With OfflineIMAP, you can download your Mailboxes and make you own backups of +the Maildir. -OfflineIMAP will synchronize both sides via *IMAP*. +This allows reading your mails while offline without the need for the mail +reader (MUA) to support IMAP disconnected operations. Need an attachement from a +message without internet? It's fine, the message is still there. ## License From e8280e642cc24035cb8505301ea027f2c8182974 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 12 Mar 2015 17:54:15 +0100 Subject: [PATCH 747/817] fix scripts/get-repository.sh to match website/render.sh Signed-off-by: Nicolas Sebrecht --- scripts/get-repository.sh | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/scripts/get-repository.sh b/scripts/get-repository.sh index 65704ad..080815c 100755 --- a/scripts/get-repository.sh +++ b/scripts/get-repository.sh @@ -49,40 +49,6 @@ function setup () { fi } -function write_renderer () { - cat < "$renderer" - chmod u+x "$renderer" + sed -r -i -e "s,{{USERNAME}},$1," "$renderer" cd .. else - echo "ERROR: could not write renderer." + echo "ERROR: could not enter ./website. (?)" fi } From 54784816175ebdadf5c4032363fc74d64402c2d6 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Mar 2015 18:42:53 +0100 Subject: [PATCH 748/817] Makefile: avoid packaging the website and the wiki Signed-off-by: Nicolas Sebrecht --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a1ccee7..4095171 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ targz: ../$(TARGZ) echo "Containing directory must be called offlineimap-$(VERSION)"; \ exit 1; \ fi; \ - pwd && cd .. && pwd && tar -zhcv --exclude '.git' -f $(TARGZ) offlineimap-$(VERSION) + pwd && cd .. && pwd && tar -zhcv --exclude '.git' --exclude 'website' --exclude 'wiki' -f $(TARGZ) offlineimap-$(VERSION) rpm: targz cd .. && sudo rpmbuild -ta $(TARGZ) From 2c259369c076140753f29ad1751fb3ada5da3b21 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Mar 2015 18:45:51 +0100 Subject: [PATCH 749/817] man: offlineimapui: minor typo fix Signed-off-by: Nicolas Sebrecht --- docs/offlineimapui.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/offlineimapui.txt b/docs/offlineimapui.txt index c5a755b..c087ece 100644 --- a/docs/offlineimapui.txt +++ b/docs/offlineimapui.txt @@ -137,4 +137,4 @@ for other programs that will interface to OfflineIMAP. See Also -------- - offlineima(1) + offlineimap(1) From 09fca0ccb1395851f9ff66fbe57ab6b401cbc5ca Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Mar 2015 18:51:14 +0100 Subject: [PATCH 750/817] remove obsolete documentation Signed-off-by: Nicolas Sebrecht --- docs/doc-src/advanced_config.rst | 18 --------- docs/doc-src/features.rst | 66 -------------------------------- 2 files changed, 84 deletions(-) delete mode 100644 docs/doc-src/advanced_config.rst delete mode 100644 docs/doc-src/features.rst diff --git a/docs/doc-src/advanced_config.rst b/docs/doc-src/advanced_config.rst deleted file mode 100644 index 49f2ea4..0000000 --- a/docs/doc-src/advanced_config.rst +++ /dev/null @@ -1,18 +0,0 @@ -Message filtering -================= - -There are two ways to selectively filter messages out of a folder, using -`maxsize` and `maxage`. Setting each option will basically ignore all messages -that are on the server by pretending they don't exist. - -:todo: explain them and give tips on how to use and not use them. Use cases! - -maxage ------- - -:todo: ! - -maxsize -------- - -:todo: ! diff --git a/docs/doc-src/features.rst b/docs/doc-src/features.rst deleted file mode 100644 index e9272d1..0000000 --- a/docs/doc-src/features.rst +++ /dev/null @@ -1,66 +0,0 @@ -Description -=========== - -OfflineIMAP is a tool to simplify your e-mail reading. With OfflineIMAP, you can -read the same mailbox from multiple computers. You get a current copy of your -messages on each computer, and changes you make one place will be visible on all -other systems. For instance, you can delete a message on your home computer, and -it will appear deleted on your work computer as well. OfflineIMAP is also useful -if you want to use a mail reader that does not have IMAP support, has poor IMAP -support, or does not provide disconnected operation. - -OfflineIMAP works on pretty much any POSIX operating system, such as Linux, BSD -operating systems, MacOS X, Solaris, etc. - -OfflineIMAP is a Free Software project licensed under the GNU General Public -License. You can download it for free, and you can modify it. In fact, you are -encouraged to contribute to OfflineIMAP, and doing so is fast and easy. - -OfflineIMAP is FAST; it synchronizes my two accounts with over 50 folders in 3 -seconds. Other similar tools might take over a minute, and achieve a -less-reliable result. Some mail readers can take over 10 minutes to do the same -thing, and some don't even support it at all. Unlike other mail tools, -OfflineIMAP features a multi-threaded synchronization algorithm that can -dramatically speed up performance in many situations by synchronizing several -different things simultaneously. - -OfflineIMAP is FLEXIBLE; you can customize which folders are synced via regular -expressions, lists, or Python expressions; a versatile and comprehensive -configuration file is used to control behavior; two user interfaces are -built-in; fine-tuning of synchronization performance is possible; internal or -external automation is supported; SSL and PREAUTH tunnels are both supported; -offline (or "unplugged") reading is supported; and esoteric IMAP features are -supported to ensure compatibility with the widest variety of IMAP servers. - -OfflineIMAP is SAFE; it uses an algorithm designed to prevent mail loss at all -costs. Because of the design of this algorithm, even programming errors should -not result in loss of mail. I am so confident in the algorithm that I use my -own personal and work accounts for testing of OfflineIMAP pre-release, -development, and beta releases. Of course, legally speaking, OfflineIMAP comes -with no warranty, so I am not responsible if this turns out to be wrong. - -.. note: OfflineImap was written by John Goerzen, who retired from - maintaining. It is now maintained by Nicolas Sebrecht & Sebastian - Spaeth at https://github.com/spaetz/offlineimap. Thanks to John - for his great job and to have share this project with us. - -Method of Operation -=================== - -OfflineIMAP traditionally operates by maintaining a hierarchy of mail folders in -Maildir format locally. Your own mail reader will read mail from this tree, and -need never know that the mail comes from IMAP. OfflineIMAP will detect changes -to the mail folders on your IMAP server and your own computer and -bi-directionally synchronize them, copying, marking, and deleting messages as -necessary. - -With OfflineIMAP 4.0, a powerful new ability has been introduced ― the program -can now synchronize two IMAP servers with each other, with no need to have a -Maildir layer in-between. Many people use this if they use a mail reader on -their local machine that does not support Maildirs. People may install an IMAP -server on their local machine, and point both OfflineIMAP and their mail reader -of choice at it. This is often preferable to the mail reader's own IMAP support -since OfflineIMAP supports many features (offline reading, for one) that most -IMAP-aware readers don't. However, this feature is not as time-tested as -traditional syncing, so my advice is to stick with normal methods of operation -for the time being. From 5eeea9ed83fe4b778c95d5bcd96a8fe7a6b81c1c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 13 Mar 2015 19:10:48 +0100 Subject: [PATCH 751/817] improve documentation - README - restrict sphinx documentation to the API - update the Makefiles Signed-off-by: Nicolas Sebrecht --- Makefile | 5 +---- README.md | 20 +++++++++++--------- docs/Makefile | 4 ++-- docs/doc-src/index.rst | 3 --- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 4095171..3aca20a 100644 --- a/Makefile +++ b/Makefile @@ -39,10 +39,7 @@ clean: -find . -name auth -exec rm -vf {}/password {}/username \; @$(MAKE) -C clean -man: - @$(MAKE) -C man - -doc: +docs: @$(MAKE) -C docs targz: ../$(TARGZ) diff --git a/README.md b/README.md index 1bbbd7f..cd97db2 100644 --- a/README.md +++ b/README.md @@ -63,14 +63,16 @@ Bugs, issues and contributions can be requested to both the mailing list or the ## Documentation -The documentation is included (in .rst format) in the `docs` directory. -Read it directly or generate nice html docs (python-sphinx needed) and/or -the man page (python-docutils needed) while being in the `docs` dir via: +All the current and updated documentation is at the [community's website][website]. - 'make doc' (user docs), 'make man' (man page only) or 'make' (both) +### Dispose locally - (`make html` will simply create html versions of all *.rst files in /docs) - -The resulting user documentation will be in `docs/html`. The full user -docs are also at: http://docs.offlineimap.org. Please see there for -detailed information on how to install and configure OfflineImap. +You might want to dispose the documentation locally. Get the sources of the website. +For the other documentations, run the approppriate make target: +``` +$ ./scripts/get-repository.sh website +$ cd docs +$ make html # Require rst2html +$ make man # Require a2x +$ make api # Require sphinx +``` diff --git a/docs/Makefile b/docs/Makefile index 1311bbf..6a7f8ea 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -9,7 +9,7 @@ RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` RST2MAN=`type rst2man >/dev/null 2>&1 && echo rst2man || echo rst2man.py` SPHINXBUILD = sphinx-build -all: man doc +docs: man api html: $(HTML_TARGETS) @@ -24,7 +24,7 @@ offlineimap.1: offlineimap.txt offlineimapui.7: offlineimapui.txt a2x -v -f manpage $? -doc: +api: $(SPHINXBUILD) -b html -d html/doctrees doc-src html websitedoc: diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index 7335827..89f9ff0 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -5,9 +5,6 @@ Welcome to OfflineIMAP's developer documentation ================================================ -**Developer documentation** - * :doc:`Advanced Git ` - **Documented APIs** .. toctree:: From 6be2656350f551fc6e60ffb975fa090780c04702 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 14 Mar 2015 00:13:14 +0100 Subject: [PATCH 752/817] doc: move non-API documentation to the website Signed-off-by: Nicolas Sebrecht --- docs/doc-src/CodingGuidelines.rst | 51 --- docs/doc-src/GitAdvanced.rst | 684 ------------------------------ docs/doc-src/dco.rst | 1 + docs/doc-src/index.rst | 3 + 4 files changed, 4 insertions(+), 735 deletions(-) delete mode 100644 docs/doc-src/CodingGuidelines.rst delete mode 100644 docs/doc-src/GitAdvanced.rst diff --git a/docs/doc-src/CodingGuidelines.rst b/docs/doc-src/CodingGuidelines.rst deleted file mode 100644 index 92cc3be..0000000 --- a/docs/doc-src/CodingGuidelines.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. -*- coding: utf-8 -*- -.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap -.. _OLI_git_repo: git://github.com/OfflineIMAP/offlineimap.git - -================================= -Coding guidelines for OfflineIMAP -================================= - -.. contents:: -.. .. sectnum:: - -This document contains assorted guidelines for programmers that want -to hack OfflineIMAP. - - ------------------- -Exception handling ------------------- - -OfflineIMAP on many occasions re-raises various exceptions and often -changes exception type to `OfflineImapError`. This is not a problem -per se, but you must always remember that we need to preserve original -tracebacks. This is not hard if you follow these simple rules. - -For re-raising original exceptions, just use:: - - raise - -from inside your exception handling code. - -If you need to change exception type, or its argument, or whatever, -use this three-argument form:: - - raise YourExceptionClass(argum, ents), None, sys.exc_info()[2] - -In this form, you're creating an instance of new exception, so ``raise`` -will deduce its ``type`` and ``value`` parameters from the first argument, -thus the second expression passed to ``raise`` is always ``None``. -And the third one is the traceback object obtained from the thread-safe -``exc_info()`` function. - -In fact, if you hadn't already imported the whole ``sys`` module, it will -be better to import just ``exc_info()``:: - - from sys import exc_info - -and raise like this:: - - raise YourExceptionClass(argum, ents), None, exc_info()[2] - -since this is the historically-preferred style in the OfflineIMAP code. diff --git a/docs/doc-src/GitAdvanced.rst b/docs/doc-src/GitAdvanced.rst deleted file mode 100644 index 402f68d..0000000 --- a/docs/doc-src/GitAdvanced.rst +++ /dev/null @@ -1,684 +0,0 @@ -.. -*- coding: utf-8 -*- -.. vim: spelllang=en ts=2 expandtab: - -.. _OfflineIMAP: http://offlineimap.org -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project -.. _Developers's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst - -============ -Git Advanced -============ - -.. contents:: :depth: 2 - -Git: OfflineImap's branching Model And Workflow -=============================================== - - -Git Branching model -------------------- - -OfflineIMAP_ uses the following branches: - -master - This is **the mainline**. Simple users should use this branch. - -next - **The development branch** for developers and testers. The content of ``next`` is - merged into the mainline ``master`` at release time for both stable and releases - candidates. When patches are sent to the mailing list, contributors discuss - about them. Once done and when patches looks ready for the mainline, patches - are first merged into ``next``. Advanced developers and testers use this branch to - test the last merged patches before they hit the mainline. This helps not - introducing strong breakages directly in the mainline. - -maint - This is **the maintenance branch**. It gets its own releases starting off of an old - stable release. - Notice that this branch tend to be more or less abandoned when context does not - force the maintainers to take care of it. - -pu - Don't care much about this branch unless you're asked to use it. It's almost - abandoned nowadays. ``pu`` stands for *"proposed updates"* and helps - **tracking of topics**. If a topic is not ready for the ``next`` release, it - might be merged into ``pu``. This branch only help developers to work on - someone else topic or an earlier pending topic. Developers can extract a topic - from this branch to work on it. This branch is **not intended to be - checkouted**; never. Even developers don't do that. Due to the way ``pu`` is - built you can't expect content there to work in any way... unless you clearly - want to run into troubles. - - -Release cycles --------------- - -A typical release cycle works like this: - -1. A stable release is out. - -2. Feature topics are sent, discussed and merged. - -3. When enough work was merged, we start the freeze cycle: the first release - candidate is out. - -4. During the freeze cycle, no more features are merged. It's time to test - OfflineIMAP_. The more we are late in *-rc* releases, the less patches are - merged but bug fixes. - -5. When we think a release is stable enough, we restart from step 1. - - -Because third-parties tend to not always follow the cycles, it's fine to send -your patches as soon as they are ready. Any maintainer might prefer to pend your -contributions before merging it at a better time. You'll always be notified if -such decision is made for your work. - -Know about where we are in the release cycle:: - - $ git tag - - -Create commits --------------- - -* Make commits of logical units. -* If you change, add, or remove a command line option or - make some other user interface change, the associated - documentation should be updated as well. -* Check for unnecessary whitespace with ``git diff --check`` - before committing. -* Do not check in commented out code or unneeded files. -* the first line of the commit message should be a short - description (50 characters is the soft limit, see DISCUSSION - in git-commit(1)), and should skip the full stop -* The body should provide a meaningful commit message, which: - * uses the imperative, present tense: **change**, - not **changed** or **changes**; - * includes motivation for the change, and contrasts - its implementation with previous behaviour. -* Add a ``Signed-off-by: Your Name `` line to - to confirm that you agree to the `Developer's Certificate of Origin`_. -* Make sure that you have tests for the bug you are fixing. -* Make sure that the test suite passes after your commit. - - -Make a pull request -------------------- - -* Push your changes to a topic branch in your public fork of OfflineIMAP. -* Submit a pull request to the OfflineIMAP_ maintainers. -* If a ticket is open in the issues, add a comment with the link to your pull - request. - - -Export commits as patches -------------------------- - -* Use ``git format-patch -M`` to create the patch. -* Do not attach your patch, but read in the mail - body, unless you cannot teach your mailer to - leave the formatting of the patch alone. -* Be careful doing cut & paste into your mailer, not to - corrupt whitespaces. - - -Export commits as patches (experts) ------------------------------------ - -* Do not PGP sign your patch. -* Provide additional information (which is unsuitable for - the commit message) between the ``---`` and the diffstat. -* If your name is not writable in ASCII, make sure that - you send off a message in the correct encoding. -* Send the patch to the `mailing list`_ if (and only if) - the patch is ready for inclusion. -* If you use `git-send-email(1)` which is a good idea, - please test it first by sending email to yourself. -* See below for instructions specific to your mailer. - - -Extract a topic from pu ------------------------ - -To find the tip of a topic branch, run ``git log --first-parent next..pu`` and -look for the merge commit. The second parent of this commit is the tip of the -topic branch. - - -``pu`` is built this way:: - - $ git checkout pu - $ git reset --keep next - $ git merge --no-ff -X theirs topic1 - $ git merge --no-ff -X theirs topic2 - $ git merge --no-ff -X theirs blue - $ git merge --no-ff -X theirs orange - ... - -As a consequence: - -1. Each topic merged uses a merge commit. A merge commit is a commit having 2 - ancestors. Actually, Git allows more than 2 parents but we don't use this - feature. It's intended. - -2. Paths in ``pu`` may mix up multiple versions if all the topics don't use the same - base commit. This is very often the case as topics aren't rebased: it guarantees - each topic is strictly identical to the last version sent to the mailing list. - No surprise. - - -What you need to extract a particular topic is the *sha1* of the tip of that -branch (the last commit of the topic). Assume you want the branch of the topic -called 'blue'. First, look at the log given by this command:: - - $ git log --reverse --merges --parents origin/next..origin/pu - -With this command you ask for the log: - -* from next to pu -* in reverse order (older first) -* merge commits only -* with the sha1 of the ancestors - -From this list, find the topic you're looking for, basing you search on the lines -like:: - - Merge branch 'topic/name' into pu - -By convention, it has the form /. When you're at -it, pick the topic ancestor sha1. It's always the last sha1 in the line starting -by 'commit'. For you to know: - -* The first sha1 is the commit you see: the merge commit. -* The following sha1 is the ancestor of the branch checkouted at merge time - (always the previous merged topic or the ancien next in our case). -* Last is the branch merged. - -Giving:: - - commit sha1_of_merge_commit sha1_of_ancient_pu sha1_of_topic_blue - -Then, you only have to checkout the topic from there:: - - $ git checkout -b blue sha1_of_topic_blue - -You're done! You've just created a new branch called "blue" with the blue -content. Be aware this topic is not updated against the **current** next branch. -,-) - - - -Very detailed version -===================== - -I started reading over the SubmittingPatches document for Git, primarily because -I wanted to have a document similar to it for OfflineIMAP to make sure people -understand what they are doing when they write `Signed-off-by` line. - -But the patch submission requirements are a lot more relaxed here on the -technical/contents front, because the OfflineIMAP is a lot smaller ;-). So here -are only the relevant bits. - - -Decide what branch to base your work on ---------------------------------------- - -In general, base your work on the ``next`` branch. Otherwise, start off of the -latest commit your change is relevant to. - - -Make separate commits for logically separate changes ----------------------------------------------------- - -Unless your patch is really trivial, you should not be sending your -changes in a single patch. Instead, always make a commit with -complete commit message and generate a series of small patches from -your repository. - -Describe the technical detail of the change(s). - -If your description starts to get too long, that's a sign that you probably need -to split up your commit to finer grained pieces. That being said, patches which -plainly describe the things that help reviewers check the patch, and future -maintainers understand the code, are the most beautiful patches. - -Descriptions that summarise the point in the subject well, and describe the -motivation for the change, the approach taken by the change, and if relevant how -this differs substantially from the prior version, can be found on Usenet -archives back into the late 80's. Consider it like good Netiquette, but for -code. - - -Generate your patch using git tools out of your commits -------------------------------------------------------- - -* ``git`` based diff tools (git, Cogito, and StGIT included) generate *unidiff* -which is the preferred format. - -* You do not have to be afraid to use ``-M`` option to ``git diff`` or ``git -format-patch``, if your patch involves file renames. The receiving end can -handle them just fine. - -* Please make sure your patch does not include any extra files which do not -belong in a patch submission. - -* Make sure to review your patch after generating it, to ensure accuracy. - -* Before sending out, please make sure it cleanly applies to the ``next`` branch -head. If you are preparing a work based on somewhere else, that is fine, but -please mark it as such. - - -Sending your patches --------------------- - -The mailing list is the preferred way for sending patches. This allows easier -review and comments on the code. - -People on the mailing list need to be able to read and -comment on the changes you are submitting. It is important for -a developer to be able to "quote" your changes, using standard -e-mail tools, so that they may comment on specific portions of -your code. For this reason, all patches should be submitted -"inline". WARNING: Be wary of your MUAs word-wrap -corrupting your patch. Do not cut-n-paste your patch; you can -lose tabs that way if you are not careful. - -It is a common convention to prefix your subject line with -[PATCH]. This lets people easily distinguish patches from other -e-mail discussions. Use of additional markers after PATCH and -the closing bracket to mark the nature of the patch is also -encouraged. E.g. [PATCH/RFC] is often used when the patch is -not ready to be applied but it is for discussion, [PATCH v2], -[PATCH v3] etc. are often seen when you are sending an update to -what you have previously sent. - -* ``git format-patch`` command follows the best current practice to - format the body of an e-mail message. At the beginning of the - patch should come your commit message, ending with the - ``Signed-off-by:`` lines, a line that consists of three dashes, - followed by the diffstat information and the patch itself. - -* If you are forwarding a patch from somebody else, optionally, at - the beginning of the e-mail message just before the commit - message starts, you can put a ``From:`` line to name that person. - -* You often want to add additional explanation about the patch, - other than the commit message itself. Place such "cover letter" - material between the three dash lines and the diffstat. - -* Do not attach the patch as a MIME attachment, compressed or not. - Many popular e-mail applications will not always - transmit a MIME attachment as plain text, making it impossible to comment on - your code. A MIME attachment also takes a bit more time to process. This does - not decrease the likelihood of your MIME-attached change being accepted, but it - makes it more likely that it will be postponed. - - Exception: If your mailer is mangling patches then someone may ask - you to re-send them using MIME, that is OK. - -* Do not let your e-mail client send quoted-printable. - -* Do not let your e-mail client send format=flowed which would destroy -whitespaces in your patches. - -* Do not PGP sign your patch, at least for now. Most likely, your - maintainer or other people on the list would not have your PGP - key and would not bother obtaining it anyway. Your patch is not - judged by who you are; a good patch from an unknown origin has a - far better chance of being accepted than a patch from a known, - respected origin that is done poorly or does incorrect things. - - If you really really really really want to do a PGP signed - patch, format it as "multipart/signed", not a text/plain message - that starts with '-----BEGIN PGP SIGNED MESSAGE-----'. That is - not a text/plain, it's something else. - -* Unless your patch is a very trivial and an obviously correct one, - first send it with "To:" set to the mailing list, with "cc:" listing - people who are involved in the area you are touching (the output from - "git blame $path" and "git shortlog --no-merges $path" would help to - identify them), to solicit comments and reviews. After the list - reached a consensus that it is a good idea to apply the patch, re-send - it with "To:" set to the maintainer and optionally "cc:" the list for - inclusion. Do not forget to add trailers such as "Acked-by:", - "Reviewed-by:" and "Tested-by:" after your "Signed-off-by:" line as - necessary. - - -Sign your work --------------- - -To improve tracking of who did what, we've borrowed the -"sign-off" procedure from the Linux kernel project on patches -that are being emailed around. Although OfflineIMAP is a lot -smaller project it is a good discipline to follow it. - -The sign-off is a simple line at the end of the explanation for -the patch, which **certifies that you wrote it or otherwise have -the right to pass it on as a open-source patch**. The rules are -pretty simple: if you can certify the below: - - -An ideal patch flow -=================== - -Here is an ideal patch flow for this project the current maintainers -suggests to the contributors: - -0. You come up with an itch. You code it up. - -1. Send it to the list and cc people who may need to know about - the change. - - The people who may need to know are the ones whose code you - are butchering. These people happen to be the ones who are - most likely to be knowledgeable enough to help you, but - they have no obligation to help you (i.e. you ask for help, - don't demand). ``git log -p -- $area_you_are_modifying`` would - help you find out who they are. - -2. You get comments and suggestions for improvements. You may - even get them in a "on top of your change" patch form. - -3. Polish, refine, and re-send to the list and the people who - spend their time to improve your patch. Go back to step (2). - -4. The list forms consensus that the last round of your patch is - good. Send it to the list and cc the maintainers. - -5. A topic branch is created with the patch and is merged to ``next``, - and cooked further and eventually graduates to ``master``. - - -In any time between the (2)-(3) cycle, the maintainer may pick it up -from the list and queue it to ``pu``, in order to make it easier for -people play with it without having to pick up and apply the patch to -their trees themselves. - - -Know the status of your patch after submission ----------------------------------------------- - -You can use Git itself to find out when your patch is merged in -master. ``git pull --rebase`` will automatically skip already-applied -patches, and will let you know. This works only if you rebase on top -of the branch in which your patch has been merged (i.e. it will not -tell you if your patch is merged in ``pu`` if you rebase on top of -``next``). - -.. Read the git mailing list, the maintainer regularly posts messages - entitled "What's cooking in git.git" and "What's in git.git" giving - the status of various proposed changes. - - -MUA specific hints -================== - -Some of patches I receive or pick up from the list share common -patterns of breakage. Please make sure your MUA is set up -properly not to corrupt whitespaces. Here are two common ones -I have seen: - -* Empty context lines that do not have _any_ whitespace. - -* Non empty context lines that have one extra whitespace at the - beginning. - -One test you could do yourself if your MUA is set up correctly is: - -* Send the patch to yourself, exactly the way you would, except - To: and Cc: lines, which would not contain the list and - maintainer address. - -* Save that patch to a file in UNIX mailbox format. Call it say - a.patch. - -* Try to apply to the tip of the "master" branch from the - git.git public repository:: - - $ git fetch http://kernel.org/pub/scm/git/git.git master:test-apply - $ git checkout test-apply - $ git reset --hard - $ git am a.patch - -If it does not apply correctly, there can be various reasons. - -* Your patch itself does not apply cleanly. That is _bad_ but - does not have much to do with your MUA. Please rebase the - patch appropriately. - -* Your MUA corrupted your patch; "am" would complain that - the patch does not apply. Look at .git/rebase-apply/ subdirectory and - see what 'patch' file contains and check for the common - corruption patterns mentioned above. - -* While you are at it, check what are in 'info' and - 'final-commit' files as well. If what is in 'final-commit' is - not exactly what you would want to see in the commit log - message, it is very likely that your maintainer would end up - hand editing the log message when he applies your patch. - Things like "Hi, this is my first patch.\n", if you really - want to put in the patch e-mail, should come after the - three-dash line that signals the end of the commit message. - - -Pine ----- - -(Johannes Schindelin) - I don't know how many people still use pine, but for those poor souls it may - be good to mention that the quell-flowed-text is needed for recent versions. - - ... the "no-strip-whitespace-before-send" option, too. AFAIK it was introduced - in 4.60. - -(Linus Torvalds) - And 4.58 needs at least this - -:: - - --- - diff-tree 8326dd8350be64ac7fc805f6563a1d61ad10d32c (from e886a61f76edf5410573e92e38ce22974f9c40f1) - Author: Linus Torvalds - Date: Mon Aug 15 17:23:51 2005 -0700 - - Fix pine whitespace-corruption bug - - There's no excuse for unconditionally removing whitespace from - the pico buffers on close. - - diff --git a/pico/pico.c b/pico/pico.c - --- a/pico/pico.c - +++ b/pico/pico.c - @@ -219,7 +219,9 @@ PICO *pm; - switch(pico_all_done){ /* prepare for/handle final events */ - case COMP_EXIT : /* already confirmed */ - packheader(); - +#if 0 - stripwhitespace(); - +#endif - c |= COMP_EXIT; - break; - -(Daniel Barkalow) - > A patch to SubmittingPatches, MUA specific help section for - > users of Pine 4.63 would be very much appreciated. - - Ah, it looks like a recent version changed the default behavior to do the - right thing, and inverted the sense of the configuration option. (Either - that or Gentoo did it.) So you need to set the - "no-strip-whitespace-before-send" option, unless the option you have is - "strip-whitespace-before-send", in which case you should avoid checking - it. - - -Thunderbird ------------ - -(A Large Angry SCM) - By default, Thunderbird will both wrap emails as well as flag them as - being 'format=flowed', both of which will make the resulting email unusable - by git. - - Here are some hints on how to successfully submit patches inline using - Thunderbird. - - There are two different approaches. One approach is to configure - Thunderbird to not mangle patches. The second approach is to use - an external editor to keep Thunderbird from mangling the patches. - -**Approach #1 (configuration):** - - This recipe is current as of Thunderbird 2.0.0.19. Three steps: - - 1. Configure your mail server composition as plain text - Edit...Account Settings...Composition & Addressing, - uncheck 'Compose Messages in HTML'. - 2. Configure your general composition window to not wrap - Edit..Preferences..Composition, wrap plain text messages at 0 - 3. Disable the use of format=flowed - Edit..Preferences..Advanced..Config Editor. Search for: - mailnews.send_plaintext_flowed - toggle it to make sure it is set to 'false'. - - After that is done, you should be able to compose email as you - otherwise would (cut + paste, git-format-patch | git-imap-send, etc), - and the patches should not be mangled. - -**Approach #2 (external editor):** - -This recipe appears to work with the current [*1*] Thunderbird from Suse. - -The following Thunderbird extensions are needed: - AboutConfig 0.5 - http://aboutconfig.mozdev.org/ - External Editor 0.7.2 - http://globs.org/articles.php?lng=en&pg=8 - - -1) Prepare the patch as a text file using your method of choice. - -2) Before opening a compose window, use Edit->Account Settings to - uncheck the "Compose messages in HTML format" setting in the - "Composition & Addressing" panel of the account to be used to send the - patch. [*2*] - -3) In the main Thunderbird window, _before_ you open the compose window - for the patch, use Tools->about:config to set the following to the - indicated values:: - - mailnews.send_plaintext_flowed => false - mailnews.wraplength => 0 - -4) Open a compose window and click the external editor icon. - -5) In the external editor window, read in the patch file and exit the - editor normally. - -6) Back in the compose window: Add whatever other text you wish to the - message, complete the addressing and subject fields, and press send. - -7) Optionally, undo the about:config/account settings changes made in - steps 2 & 3. - - -[Footnotes] - -*1* Version 1.0 (20041207) from the MozillaThunderbird-1.0-5 rpm of Suse -9.3 professional updates. - -*2* It may be possible to do this with about:config and the following -settings but I haven't tried, yet:: - - mail.html_compose => false - mail.identity.default.compose_html => false - mail.identity.id?.compose_html => false - -(Lukas Sandström) - There is a script in contrib/thunderbird-patch-inline which can help you - include patches with Thunderbird in an easy way. To use it, do the steps above - and then use the script as the external editor. - -Gnus ----- - -'|' in the *Summary* buffer can be used to pipe the current -message to an external program, and this is a handy way to drive -"git am". However, if the message is MIME encoded, what is -piped into the program is the representation you see in your -*Article* buffer after unwrapping MIME. This is often not what -you would want for two reasons. It tends to screw up non ASCII -characters (most notably in people's names), and also -whitespaces (fatal in patches). Running 'C-u g' to display the -message in raw form before using '|' to run the pipe can work -this problem around. - - -KMail ------ - -This should help you to submit patches inline using KMail. - -1) Prepare the patch as a text file. - -2) Click on New Mail. - -3) Go under "Options" in the Composer window and be sure that - "Word wrap" is not set. - -4) Use Message -> Insert file... and insert the patch. - -5) Back in the compose window: add whatever other text you wish to the - message, complete the addressing and subject fields, and press send. - - -Gmail ------ - -GMail does not appear to have any way to turn off line wrapping in the web -interface, so this will mangle any emails that you send. You can however -use "git send-email" and send your patches through the GMail SMTP server, or -use any IMAP email client to connect to the google IMAP server and forward -the emails through that. - -To use ``git send-email`` and send your patches through the GMail SMTP server, -edit `~/.gitconfig` to specify your account settings:: - - [sendemail] - smtpencryption = tls - smtpserver = smtp.gmail.com - smtpuser = user@gmail.com - smtppass = p4ssw0rd - smtpserverport = 587 - -Once your commits are ready to be sent to the mailing list, run the -following commands:: - - $ git format-patch --cover-letter -M origin/master -o outgoing/ - $ edit outgoing/0000-* - $ git send-email outgoing/* - -To submit using the IMAP interface, first, edit your `~/.gitconfig` to specify your -account settings:: - - [imap] - folder = "[Gmail]/Drafts" - host = imaps://imap.gmail.com - user = user@gmail.com - pass = p4ssw0rd - port = 993 - sslverify = false - -You might need to instead use: folder = "[Google Mail]/Drafts" if you get an error -that the "Folder doesn't exist". - -Once your commits are ready to be sent to the mailing list, run the -following commands:: - - $ git format-patch --cover-letter -M --stdout origin/master | git imap-send - -Just make sure to disable line wrapping in the email client (GMail web -interface will line wrap no matter what, so you need to use a real -IMAP client). diff --git a/docs/doc-src/dco.rst b/docs/doc-src/dco.rst index 83ae888..ad8a3b0 100644 --- a/docs/doc-src/dco.rst +++ b/docs/doc-src/dco.rst @@ -1,3 +1,4 @@ +.. _dco Developer's Certificate of Origin ================================= diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index 89f9ff0..fb8bdc9 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -5,6 +5,9 @@ Welcome to OfflineIMAP's developer documentation ================================================ +**License** + :doc:`dco` (dco) + **Documented APIs** .. toctree:: From cb3f13a331c8752258e528f6fd41c975e0ca280e Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 14 Mar 2015 13:55:34 +0100 Subject: [PATCH 753/817] Changelogs: improve website formatting Signed-off-by: Nicolas Sebrecht --- Changelog.maint.md | 3 +++ Changelog.md | 2 ++ 2 files changed, 5 insertions(+) diff --git a/Changelog.maint.md b/Changelog.maint.md index 57c3e64..344987c 100644 --- a/Changelog.maint.md +++ b/Changelog.maint.md @@ -3,6 +3,9 @@ layout: page title: Changelog of the stable branch --- +* The following excerpt is only usefull when rendered in the website. +{:toc} + This is the Changelog of the maintenance branch. **NOTE FROM THE MAINTAINER:** diff --git a/Changelog.md b/Changelog.md index 2a55d48..b2746fe 100644 --- a/Changelog.md +++ b/Changelog.md @@ -3,6 +3,8 @@ layout: page title: Changelog of mainline --- +* The following excerpt is only usefull when rendered in the website. +{:toc} ### OfflineIMAP v6.5.7-rc3 (2015- - ) From b9d647a4a24a8c31ec044826dccb4b4bd5ce5407 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 15 Mar 2015 19:39:18 +0100 Subject: [PATCH 754/817] CONTRIBUTING: add section "for the impatients" Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index af24292..f2469ce 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,6 +8,8 @@ .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst .. _Community's website: https://offlineimap.github.io +.. _APIs in OfflineIMAP: http://offlineimap.github.io/documentation.html#available-apis +.. _documentation: https://offlineimap.github.io/documentation.html ================= @@ -18,11 +20,19 @@ You'll find here the **basics** to contribute to OfflineIMAP_, addressed to users as well as learning or experienced developers to quickly provide contributions. -**For more detailed documentation, see the `Community's website`_.** +**For more detailed documentation, see the** `Community's website`_. .. contents:: :depth: 3 +For the imaptients +================== + +- `Coding Guidelines`_ +- `APIs in OfflineIMAP_` +- All the `documentation`_ + + Submit issues ============= From 10aa7c19e6ae7c4e42c884e0368a4bb363b21d26 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 15 Mar 2015 19:41:17 +0100 Subject: [PATCH 755/817] CONTRIBUTING: fix links Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f2469ce..91fd1f5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -10,6 +10,7 @@ .. _Community's website: https://offlineimap.github.io .. _APIs in OfflineIMAP: http://offlineimap.github.io/documentation.html#available-apis .. _documentation: https://offlineimap.github.io/documentation.html +.. _Coding Guidelines: http://offlineimap.github.io/doc/CodingGuidelines.html ================= @@ -29,7 +30,7 @@ For the imaptients ================== - `Coding Guidelines`_ -- `APIs in OfflineIMAP_` +- `APIs in OfflineIMAP`_ - All the `documentation`_ From c89d17dacb9e11b2e2b9d40b48a4abaaff952d3d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 15 Mar 2015 22:49:19 +0100 Subject: [PATCH 756/817] CONTRIBUTING: know the status of the patches after submission Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 91fd1f5..9f8f669 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -11,6 +11,7 @@ .. _APIs in OfflineIMAP: http://offlineimap.github.io/documentation.html#available-apis .. _documentation: https://offlineimap.github.io/documentation.html .. _Coding Guidelines: http://offlineimap.github.io/doc/CodingGuidelines.html +.. _Know the status of your patches: http://localhost:4000/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission ================= @@ -31,6 +32,7 @@ For the imaptients - `Coding Guidelines`_ - `APIs in OfflineIMAP`_ +- `Know the status of your patches`_ after submission - All the `documentation`_ From dd2a70fc68bc52f783a0df5c1f8c5399d80ae707 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 17 Mar 2015 02:37:44 +0100 Subject: [PATCH 757/817] website-doc.sh: learn to export Changelog's definitions Signed-off-by: Nicolas Sebrecht --- Changelog.md | 10 ++++- docs/Makefile | 3 +- docs/website-doc.sh | 89 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/Changelog.md b/Changelog.md index b2746fe..5fa6c38 100644 --- a/Changelog.md +++ b/Changelog.md @@ -3,13 +3,21 @@ layout: page title: Changelog of mainline --- + + + * The following excerpt is only usefull when rendered in the website. {:toc} ### OfflineIMAP v6.5.7-rc3 (2015- - ) - ### OfflineIMAP v6.5.7-rc2 (2015-01-18) #### Notes diff --git a/docs/Makefile b/docs/Makefile index 6a7f8ea..0efd63b 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -28,7 +28,8 @@ api: $(SPHINXBUILD) -b html -d html/doctrees doc-src html websitedoc: - ./website-doc.sh + ./website-doc.sh releases + ./website-doc.sh api clean: $(RM) -f $(HTML_TARGETS) diff --git a/docs/website-doc.sh b/docs/website-doc.sh index f232e1d..dc1b883 100755 --- a/docs/website-doc.sh +++ b/docs/website-doc.sh @@ -2,40 +2,91 @@ # # vim: expandtab ts=2 : +ARGS=$* + SPHINXBUILD=sphinx-build TMPDIR='/tmp/offlineimap-sphinx-doctrees' WEBSITE='../website' DOCBASE="${WEBSITE}/_doc" DESTBASE="${DOCBASE}/versions" VERSIONS_YML="${WEBSITE}/_data/versions.yml" +ANNOUNCES_YML="${WEBSITE}/_data/announces.yml" version="v$(../offlineimap.py --version)" test -d "$DESTBASE" || exit 1 -dest="${DESTBASE}/${version}" # -# Build sphinx documentation. +# Build the sphinx documentation. # -echo "Cleaning target directory: $dest" -rm -rf "$dest" -$SPHINXBUILD -b html -d "$TMPDIR" doc-src "$dest" +function sphinx_doc () { + # Build the doc with sphinx. + dest="${DESTBASE}/${version}" + echo "Cleaning target directory: $dest" + rm -rf "$dest" + $SPHINXBUILD -b html -d "$TMPDIR" doc-src "$dest" + + # Build the JSON definitions for Jekyll. + # This let know the website about the available APIs documentations. + echo "Building Jekyll data: $VERSIONS_YML" + echo "# DO NOT EDIT MANUALLY: it is generated by a script (website-doc.sh)" > "$VERSIONS_YML" + for version in $(ls "$DESTBASE" -1 | sort -nr) + do + echo "- $version" + done >> "$VERSIONS_YML" +} + + # -# Dynamically build JSON definitions for Jekyll. +# Make Changelog public and save links to them as JSON. # -echo "Building Jekyll data: $VERSIONS_YML" -for version in $(ls "$DESTBASE" -1 | sort -nr) +function releases () { + # Copy the Changelogs. + for foo in ../Changelog.md ../Changelog.maint.md + do + cp -afv "$foo" "$DOCBASE" + done + + # Build the announces JSON list. Format is JSON: + # - {version: '', link: ''} + # - ... + echo "# DO NOT EDIT MANUALLY: it is generated by a script (website-doc.sh)" > "$ANNOUNCES_YML" + grep -E '^### OfflineIMAP' ../Changelog.md | while read title + do + link="$(echo $title | sed -r -e 's,^### (OfflineIMAP.*)\),\1,' \ + | tr '[:upper:]' '[:lower:]' \ + | sed -r -e 's,[\.("],,g' \ + | sed -r -e 's, ,-,g' + )" + v="$(echo $title \ + | sed -r -e 's,^### [a-Z]+ (v[^ ]+).*,\1,' + )" + echo "- {version: '${v}', link: '$link'}" + done | tee -a "$ANNOUNCES_YML" +} + + + +exit_code=0 +test "n$ARGS" = 'n' && ARG='usage' # no option passed +for arg in $ARGS do - echo "- $version" -done > "$VERSIONS_YML" - -# -# Copy usefull sources of documentation. -# - -# Changelogs. -for foo in ../Changelog.md ../Changelog.maint.md -do - cp -afv "$foo" "$DOCBASE" + case "n$arg" in + "nreleases") + releases + ;; + "napi") + api + ;; + "nusage") + echo "Usage: website-doc.sh " + ;; + *) + echo "unkown option $arg" + exit_code=$(( $exit_code + 1 )) + ;; + esac done + +exit $exit_code From 11fd0eb5a4aa3873a252e6cce5f73998c025b766 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 17 Mar 2015 12:27:34 +0100 Subject: [PATCH 758/817] --help: put a bit more informations Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 579878d..a5277cb 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -68,25 +68,29 @@ class OfflineImap: parser.add_option("-1", action="store_true", dest="singlethreading", default=False, - help="disable all multithreading operations") + help="(the number one) disable all multithreading operations") parser.add_option("-P", dest="profiledir", metavar="DIR", help="sets OfflineIMAP into profile mode.") - parser.add_option("-a", dest="accounts", metavar="ACCOUNTS", - help="overrides the accounts section in the config file") + parser.add_option("-a", dest="accounts", + metavar="account1[,account2[,...]]", + help="list of accounts to sync") parser.add_option("-c", dest="configfile", metavar="FILE", default=None, help="specifies a configuration file to use") - parser.add_option("-d", dest="debugtype", metavar="type1,[type2...]", - help="enables debugging for OfflineIMAP.") + parser.add_option("-d", dest="debugtype", + metavar="type1[,type2[,...]]", + help="enables debugging for OfflineIMAP " + " (types: imap, maildir, thread)") parser.add_option("-l", dest="logfile", metavar="FILE", help="log to FILE") - parser.add_option("-f", dest="folders", metavar="folder1,[folder2...]", + parser.add_option("-f", dest="folders", + metavar="folder1[,folder2[,...]]", help="only sync the specified folders") parser.add_option("-k", dest="configoverride", @@ -97,15 +101,16 @@ class OfflineImap: parser.add_option("-o", action="store_true", dest="runonce", default=False, - help="run only once") + help="run only once (ignore autorefresh)") parser.add_option("-q", action="store_true", dest="quick", default=False, - help="run only quick synchronizations") + help="run only quick synchronizations (don't update flags)") parser.add_option("-u", dest="interface", - help="specifies an alternative user interface") + help="specifies an alternative user interface" + " (quiet, basic, ttyui, blinkenlights, machineui)") (options, args) = parser.parse_args() globals.set_options (options) From cf2a2c769cee47b42681fd9f6516cf79d843eade Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 17 Mar 2015 12:40:34 +0100 Subject: [PATCH 759/817] offlineimap(1): fix formatting Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 7708792..84a5341 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -331,15 +331,15 @@ full log. * IDLE support is incomplete and experimental. Bugs may be encountered. - * No hook exists for "run after an IDLE response". + - No hook exists for "run after an IDLE response". + Email will show up, but may not be processed until the next refresh cycle. - * nametrans may not be supported correctly. + - nametrans may not be supported correctly. - * IMAP IDLE <-> IMAP IDLE doesn't work yet. + - IMAP IDLE <-> IMAP IDLE doesn't work yet. - * IDLE may only work "once" per refresh. + - IDLE may only work "once" per refresh. + If you encounter this bug, please send a report to the list! @@ -349,25 +349,24 @@ Maildir uses colon caracter (:) in message file names. Colon is however forbidden character in windows drives. There are several workarounds for that situation: - * Use "maildir-windows-compatible = yes" account OfflineIMAP configuration. + . Enable file name character translation in windows registry (not tested). + - + . Use cygwin managed mount (not tested). + - not available anymore since cygwin 1.7 + + . Use "maildir-windows-compatible = yes" account OfflineIMAP configuration. - That makes OfflineIMAP to use exclamation mark (!) instead of colon for storing messages. Such files can be written to windows partitions. But you will probably loose compatibility with other programs trying to read the same Maildir. - ++ - Exclamation mark was chosen because of the note in http://docs.python.org/library/mailbox.html - ++ - If you have some messages already stored without this option, you will have to re-sync them again - * Enable file name character translation in windows registry (not tested). - - http://support.microsoft.com/kb/289627 - - * Use cygwin managed mount (not tested). - - not available anymore since cygwin 1.7 - * OfflineIMAP confused after system suspend. + When resuming a suspended session, OfflineIMAP does not cleanly handles the From 43dbe1578c10bc8bf90f614ff765e878b6241a00 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 18 Mar 2015 21:54:36 +0100 Subject: [PATCH 760/817] imaplib2: bump to v2.41 Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 247 +++++++++++++++++++++++----------------- 1 file changed, 140 insertions(+), 107 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 6d2805a..eddc3c4 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.37" +__version__ = "2.41" __release__ = "2" -__revision__ = "37" +__revision__ = "41" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -43,12 +43,22 @@ Single quoting introduced with the help of Vladimir Marek July 2013. Fix for gmail "read 0" error provided by Jim Greenleaf August 2013. Fix for offlineimap "indexerror: string index out of range" bug provided by Eygene Ryabinkin August 2013. -Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014""" +Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014. +Conversion to Python3 provided by F. Malina February 2015. +Fix for READ-ONLY error from multiple EXAMINE/SELECT calls for same mailbox by March 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" -import binascii, errno, os, Queue, random, re, select, socket, sys, time, threading, traceback, zlib +import binascii, errno, os, random, re, select, socket, sys, time, threading, zlib + +try: + import queue # py3 + string_types = str +except ImportError: + import Queue as queue # py2 + string_types = basestring + select_module = select @@ -160,13 +170,9 @@ class Request(object): self.data = None - def abort_tb(self, typ, val, tb): - self.aborted = (typ, val, tb) - self.deliver(None) - - def abort(self, typ, val): - self.abort_tb(typ, val, traceback.extract_stack()) + self.aborted = (typ, val) + self.deliver(None) def get_response(self, exc_fmt=None): @@ -175,10 +181,10 @@ class Request(object): self.ready.wait() if self.aborted is not None: - typ, val, tb = self.aborted + typ, val = self.aborted if exc_fmt is None: exc_fmt = '%s - %%s' % typ - raise typ, typ(exc_fmt % str(val)), tb + raise typ(exc_fmt % str(val)) return self.response @@ -256,7 +262,7 @@ class IMAP4(object): containing the wildcard character '*', then enclose the argument in single quotes: the quotes will be removed and the resulting string passed unquoted. Note also that you can pass in an argument - with a type that doesn't evaluate to 'basestring' (eg: 'bytearray') + with a type that doesn't evaluate to 'string_types' (eg: 'bytearray') and it will be converted to a string without quoting. There is one instance variable, 'state', that is useful for tracking @@ -301,7 +307,6 @@ class IMAP4(object): self.tagged_commands = {} # Tagged commands awaiting response self.untagged_responses = [] # [[typ: [data, ...]], ...] self.mailbox = None # Current mailbox selected - self.mailboxes = {} # Untagged responses state per mailbox self.is_readonly = False # READ-ONLY desired state self.idle_rqb = None # Server IDLE Request - see _IdleCont self.idle_timeout = None # Must prod server occasionally @@ -357,8 +362,8 @@ class IMAP4(object): self.commands_lock = threading.Lock() self.idle_lock = threading.Lock() - self.ouq = Queue.Queue(10) - self.inq = Queue.Queue() + self.ouq = queue.Queue(10) + self.inq = queue.Queue() self.wrth = threading.Thread(target=self._writer) self.wrth.setDaemon(True) @@ -435,19 +440,19 @@ class IMAP4(object): af, socktype, proto, canonname, sa = res try: s = socket.socket(af, socktype, proto) - except socket.error, msg: + except socket.error as msg: continue try: for i in (0, 1): try: s.connect(sa) break - except socket.error, msg: + except socket.error as msg: if len(msg.args) < 2 or msg.args[0] != errno.EINTR: raise else: raise socket.error(msg) - except socket.error, msg: + except socket.error as msg: s.close() continue break @@ -527,7 +532,10 @@ class IMAP4(object): data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) - self.sock.sendall(data) + if bytes != str: + self.sock.sendall(bytes(data, 'utf8')) + else: + self.sock.sendall(data) def shutdown(self): @@ -813,7 +821,8 @@ class IMAP4(object): data = kv_pairs[0] # Assume invoker passing correctly formatted string (back-compat) else: data = '(%s)' % ' '.join([(arg and self._quote(arg) or 'NIL') for arg in kv_pairs]) - return self._simple_command(name, (data,), **kw) + + return self._simple_command(name, data, **kw) def idle(self, timeout=None, **kw): @@ -981,20 +990,14 @@ class IMAP4(object): def select(self, mailbox='INBOX', readonly=False, **kw): """(typ, [data]) = select(mailbox='INBOX', readonly=False) - Select a mailbox. (Restores any previous untagged responses.) + Select a mailbox. (Flushes all untagged responses.) 'data' is count of messages in mailbox ('EXISTS' response). Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so other responses should be obtained via "response('FLAGS')" etc.""" - self.commands_lock.acquire() - # Save state of old mailbox, restore state for new... - self.mailboxes[self.mailbox] = self.untagged_responses - self.untagged_responses = self.mailboxes.setdefault(mailbox, []) - self.commands_lock.release() - self.mailbox = mailbox - self.is_readonly = readonly and True or False + self.is_readonly = bool(readonly) if readonly: name = 'EXAMINE' else: @@ -1240,7 +1243,7 @@ class IMAP4(object): # Must quote command args if "atom-specials" present, # and not already quoted. NB: single quotes are removed. - if not isinstance(arg, basestring): + if not isinstance(arg, string_types): return arg if len(arg) >= 2 and (arg[0],arg[-1]) in (('(',')'),('"','"')): return arg @@ -1252,8 +1255,8 @@ class IMAP4(object): def _choose_nonull_or_dflt(self, dflt, *args): - if isinstance(dflt, basestring): - dflttyp = basestring # Allow any string type + if isinstance(dflt, string_types): + dflttyp = string_types # Allow any string type else: dflttyp = type(dflt) for arg in args: @@ -1303,12 +1306,16 @@ class IMAP4(object): self._check_bye() - for typ in ('OK', 'NO', 'BAD'): - self._get_untagged_response(typ) + if name in ('EXAMINE', 'SELECT'): + self.untagged_responses = [] # Flush all untagged responses + else: + for typ in ('OK', 'NO', 'BAD'): + while self._get_untagged_response(typ): + continue - if self._get_untagged_response('READ-ONLY', leave=True) and not self.is_readonly: - self.literal = None - raise self.readonly('mailbox status changed to READ-ONLY') + if self._get_untagged_response('READ-ONLY', leave=True) and not self.is_readonly: + self.literal = None + raise self.readonly('mailbox status changed to READ-ONLY') if self.Terminate: raise self.abort('connection closed') @@ -1323,7 +1330,7 @@ class IMAP4(object): literal = self.literal if literal is not None: self.literal = None - if isinstance(literal, basestring): + if isinstance(literal, string_types): literator = None data = '%s {%s}' % (data, len(literal)) else: @@ -1389,9 +1396,10 @@ class IMAP4(object): return typ, dat - def _command_completer(self, (response, cb_arg, error)): + def _command_completer(self, cb_arg_list): # Called for callback commands + (response, cb_arg, error) = cb_arg_list rqb, kw = cb_arg rqb.callback = kw['callback'] rqb.callback_arg = kw.get('cb_arg') @@ -1665,7 +1673,7 @@ class IMAP4(object): if __debug__: self._log(1, 'starting') - typ, val, tb = self.abort, 'connection terminated', None + typ, val = self.abort, 'connection terminated' while not self.Terminate: @@ -1683,12 +1691,11 @@ class IMAP4(object): try: line = self.inq.get(True, timeout) - except Queue.Empty: + except queue.Empty: if self.idle_rqb is None: if resp_timeout is not None and self.tagged_commands: if __debug__: self._log(1, 'response timeout') typ, val = self.abort, 'no response after %s secs' % resp_timeout - tb = traceback.extract_stack() break continue if self.idle_timeout > time.time(): @@ -1700,41 +1707,33 @@ class IMAP4(object): if __debug__: self._log(1, 'inq None - terminating') break - # self.inq can contain tuple, it means we got an exception. - # For example of code that produces tuple see IMAP4._reader() - # and IMAP4._writer(). - # - # XXX: may be it will be more explicit to create own exception - # XXX: class and pass it instead of tuple. - if not isinstance(line, basestring): - typ, val, tb = line + if not isinstance(line, string_types): + typ, val = line break try: self._put_response(line) except: typ, val = self.error, 'program error: %s - %s' % sys.exc_info()[:2] - tb = sys.exc_info()[2] break self.Terminate = True - if not tb: - tb = traceback.extract_stack() - if __debug__: self._log(1, 'terminating: %s' % repr(val)) while not self.ouq.empty(): try: - self.ouq.get_nowait().abort_tb(typ, val, tb) - except Queue.Empty: + qel = self.ouq.get_nowait() + if qel is not None: + qel.abort(typ, val) + except queue.Empty: break self.ouq.put(None) self.commands_lock.acquire() - for name in self.tagged_commands.keys(): + for name in list(self.tagged_commands.keys()): rqb = self.tagged_commands.pop(name) - rqb.abort_tb(typ, val, tb) + rqb.abort(typ, val) self.state_change_free.set() self.commands_lock.release() if __debug__: self._log(3, 'state_change_free.set') @@ -1795,13 +1794,22 @@ class IMAP4(object): rxzero = 0 while True: - stop = data.find('\n', start) - if stop < 0: - line_part += data[start:] - break - stop += 1 - line_part, start, line = \ - '', stop, line_part + data[start:stop] + if bytes != str: + stop = data.find(b'\n', start) + if stop < 0: + line_part += data[start:].decode() + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop].decode() + else: + stop = data.find('\n', start) + if stop < 0: + line_part += data[start:] + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop] if __debug__: self._log(4, '< %s' % line) self.inq.put(line) if self.TerminateReader: @@ -1816,7 +1824,7 @@ class IMAP4(object): self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) - self.inq.put((self.abort, reason, sys.exc_info()[2])) + self.inq.put((self.abort, reason)) break poll.unregister(self.read_fd) @@ -1862,13 +1870,22 @@ class IMAP4(object): rxzero = 0 while True: - stop = data.find('\n', start) - if stop < 0: - line_part += data[start:] - break - stop += 1 - line_part, start, line = \ - '', stop, line_part + data[start:stop] + if bytes != str: + stop = data.find(b'\n', start) + if stop < 0: + line_part += data[start:].decode() + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop].decode() + else: + stop = data.find('\n', start) + if stop < 0: + line_part += data[start:] + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop] if __debug__: self._log(4, '< %s' % line) self.inq.put(line) if self.TerminateReader: @@ -1880,7 +1897,7 @@ class IMAP4(object): self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) - self.inq.put((self.abort, reason, sys.exc_info()[2])) + self.inq.put((self.abort, reason)) break if __debug__: self._log(1, 'finished') @@ -1893,7 +1910,6 @@ class IMAP4(object): if __debug__: self._log(1, 'starting') reason = 'Terminated' - tb = None while not self.Terminate: rqb = self.ouq.get() @@ -1905,19 +1921,15 @@ class IMAP4(object): if __debug__: self._log(4, '> %s' % rqb.data) except: reason = 'socket error: %s - %s' % sys.exc_info()[:2] - tb = sys.exc_info()[2] if __debug__: if not self.Terminate: self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) - rqb.abort_tb(self.abort, reason, tb) + rqb.abort(self.abort, reason) break - if not tb: - tb = traceback.extract_stack() - - self.inq.put((self.abort, reason, tb)) + self.inq.put((self.abort, reason)) if __debug__: self._log(1, 'finished') @@ -1952,7 +1964,7 @@ class IMAP4(object): return t = '\n\t\t' - l = map(lambda x:'%s: "%s"' % (x[0], x[1][0] and '" "'.join(x[1]) or ''), l) + l = ['%s: "%s"' % (x[0], x[1][0] and '" "'.join(x[1]) or '') for x in l] self.debug_lock.acquire() self._mesg('untagged responses dump:%s%s' % (t, t.join(l))) self.debug_lock.release() @@ -2082,16 +2094,28 @@ class IMAP4_SSL(IMAP4): data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) - if hasattr(self.sock, "sendall"): - self.sock.sendall(data) + if bytes != str: + if hasattr(self.sock, "sendall"): + self.sock.sendall(bytes(data, 'utf8')) + else: + dlen = len(data) + while dlen > 0: + sent = self.sock.write(bytes(data, 'utf8')) + if sent == dlen: + break # avoid copy + data = data[sent:] + dlen = dlen - sent else: - bytes = len(data) - while bytes > 0: - sent = self.sock.write(data) - if sent == bytes: - break # avoid copy - data = data[sent:] - bytes = bytes - sent + if hasattr(self.sock, "sendall"): + self.sock.sendall(data) + else: + dlen = len(data) + while dlen > 0: + sent = self.sock.write(data) + if sent == dlen: + break # avoid copy + data = data[sent:] + dlen = dlen - sent def ssl(self): @@ -2165,7 +2189,10 @@ class IMAP4_stream(IMAP4): data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) - self.writefile.write(data) + if bytes != str: + self.writefile.write(bytes(data, 'utf8')) + else: + self.writefile.write(data) self.writefile.flush() @@ -2243,7 +2270,7 @@ class _IdleCont(object): MonthNames = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] -Mon2num = dict(zip((x.encode() for x in MonthNames[1:]), range(1, 13))) +Mon2num = dict(list(zip((x.encode() for x in MonthNames[1:]), list(range(1, 13))))) InternalDate = re.compile(r'.*INTERNALDATE "' r'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' @@ -2347,7 +2374,7 @@ if __name__ == '__main__': try: optlist, args = getopt.getopt(sys.argv[1:], 'd:il:s:p:') - except getopt.error, val: + except getopt.error as val: optlist, args = (), () debug, debug_buf_lvl, port, stream_command, keyfile, certfile, idle_intr = (None,)*7 @@ -2380,13 +2407,14 @@ if __name__ == '__main__': % {'user':USER, 'lf':'\n', 'data':data} test_seq1 = [ + ('list', ('""', '""')), ('list', ('""', '%')), - ('create', ('/tmp/imaplib2_test.0',)), - ('rename', ('/tmp/imaplib2_test.0', '/tmp/imaplib2_test.1')), - ('CREATE', ('/tmp/imaplib2_test.2',)), - ('append', ('/tmp/imaplib2_test.2', None, None, test_mesg)), - ('list', ('/tmp', 'imaplib2_test*')), - ('select', ('/tmp/imaplib2_test.2',)), + ('create', ('imaplib2_test0',)), + ('rename', ('imaplib2_test0', 'imaplib2_test1')), + ('CREATE', ('imaplib2_test2',)), + ('append', ('imaplib2_test2', None, None, test_mesg)), + ('list', ('', 'imaplib2_test%')), + ('select', ('imaplib2_test2',)), ('search', (None, 'SUBJECT', 'IMAP4 test')), ('fetch', ("'1:*'", '(FLAGS INTERNALDATE RFC822)')), ('store', ('1', 'FLAGS', '(\Deleted)')), @@ -2405,12 +2433,17 @@ if __name__ == '__main__': ('uid', ('SEARCH', 'ALL')), ('uid', ('THREAD', 'references', 'UTF-8', '(SEEN)')), ('recent', ()), + ('examine', ()), + ('select', ()), + ('examine', ()), + ('select', ()), ) AsyncError = None - def responder((response, cb_arg, error)): + def responder(cb_arg_list): + (response, cb_arg, error) = cb_arg_list global AsyncError cmd, args = cb_arg if error is not None: @@ -2467,7 +2500,7 @@ if __name__ == '__main__': for cmd,args in test_seq1: run(cmd, args) - for ml in run('list', ('/tmp/', 'imaplib2_test%'), cb=False): + for ml in run('list', ('', 'imaplib2_test%'), cb=False): mo = re.match(r'.*"([^"]+)"$', ml) if mo: path = mo.group(1) else: path = ml.split()[-1] @@ -2475,7 +2508,7 @@ if __name__ == '__main__': if 'ID' in M.capabilities: run('id', ()) - run('id', ('("name", "imaplib2")',)) + run('id', ("(name imaplib2)",)) run('id', ("version", __version__, "os", os.uname()[0])) for cmd,args in test_seq2: @@ -2527,16 +2560,16 @@ if __name__ == '__main__': M._mesg('unused untagged responses in order, most recent last:') for typ,dat in M.pop_untagged_responses(): M._mesg('\t%s %s' % (typ, dat)) - print 'All tests OK.' + print('All tests OK.') except: if not idle_intr or not 'IDLE' in M.capabilities: - print 'Tests failed.' + print('Tests failed.') if not debug: - print ''' + print(''' If you would like to see debugging output, try: %s -d5 -''' % sys.argv[0] +''' % sys.argv[0]) raise From 0dfe57d0ca425356e01c0fbec8c18da6ebf925be Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 18 Mar 2015 22:36:01 +0100 Subject: [PATCH 761/817] offlineimap.conf: explain the remotehost line must match the domain defined in the certificate Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 4c8e5d2..27ad3c6 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -492,6 +492,11 @@ remotehost = examplehost # # Whether or not to use SSL. # +# Note: be care to configure the 'remotehost' line with the domain name defined +# in the certificate. E.g., if you trust your provider and want to use the +# certificate it provides on a shared server. Otherwise, OfflineIMAP will stop +# and say that the domain is not named in the certificate. +# #ssl = yes From 300f884c097593d181ca46fcfaa7779237ee8dbb Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 18 Mar 2015 23:09:34 +0100 Subject: [PATCH 762/817] idle: continue trying selecting the folder on OfflineImapError.Error Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 351ee9f..f0b2248 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -710,9 +710,10 @@ class IdleThread(object): ui.unregisterthread(currentThread()) #syncfolder registered the thread def __idle(self): - """Invoke IDLE mode until timeout or self.stop() is invoked""" + """Invoke IDLE mode until timeout or self.stop() is invoked.""" + def callback(args): - """IDLE callback function invoked by imaplib2 + """IDLE callback function invoked by imaplib2. This is invoked when a) The IMAP server tells us something while in IDLE mode, b) we get an Exception (e.g. on dropped @@ -722,22 +723,26 @@ class IdleThread(object): if exc_data is None and not self.stop_sig.isSet(): # No Exception, and we are not supposed to stop: self.needsync = True - self.stop_sig.set() # continue to sync + self.stop_sig.set() # Continue to sync. while not self.stop_sig.isSet(): self.needsync = False - success = False # successfully selected FOLDER? + success = False # Successfully selected FOLDER? while not success: imapobj = self.parent.acquireconnection() try: imapobj.select(self.folder) except OfflineImapError as e: if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: - # Connection closed, release connection and retry + # Connection closed, release connection and retry. self.ui.error(e, exc_info()[2]) self.parent.releaseconnection(imapobj, True) + elif e.severity == OfflineImapError.ERROR.FOLDER: + # Just continue the process on such error for now. + self.ui.error(e, exc_info()[2]) else: + # Stops future attempts to sync this account. raise else: success = True @@ -747,7 +752,7 @@ class IdleThread(object): self.ui.warn("IMAP IDLE not supported on server '%s'." "Sleep until next refresh cycle."% imapobj.identifier) imapobj.noop() - self.stop_sig.wait() # self.stop() or IDLE callback are invoked + self.stop_sig.wait() # self.stop() or IDLE callback are invoked. try: # End IDLE mode with noop, imapobj can point to a dropped conn. imapobj.noop() @@ -759,7 +764,7 @@ class IdleThread(object): self.parent.releaseconnection(imapobj) if self.needsync: - # here not via self.stop, but because IDLE responded. Do + # Here not via self.stop, but because IDLE responded. Do # another round and invoke actual syncing. self.stop_sig.clear() self.__dosync() From 8bcefc7558ed7db7c73e89c06f1c82ff99b0a402 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 19 Mar 2015 16:59:46 +0100 Subject: [PATCH 763/817] offlineimap.conf: add note about known issues Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 27ad3c6..bd84d6e 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -48,6 +48,11 @@ # expanded in the resulting string. This behaviour is intentional # as it coincides with typical shell expansion strategy. +# NOTE 4: multiple same-named sections. +# The library used to parse the configuration file has known issue when multiple +# sections have the same name. In such case, only the last section is considered. +# It is strongly discouraged to have multiple sections with the same name. +# See https://github.com/OfflineIMAP/offlineimap/issues/143 for more details. [general] From b7f245b02eb119cef4f1cf0a26bffce973a6e45a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 19 Mar 2015 17:06:11 +0100 Subject: [PATCH 764/817] offlineimap(1): add known issue when mails change since invokation Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 84a5341..f4b844c 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -373,6 +373,12 @@ When resuming a suspended session, OfflineIMAP does not cleanly handles the broken socket(s) if socktimeout option is not set. You should enable this option with a value like 10. +* OfflineIMAP confused when mails change while in a sync. ++ +When OfflineIMAP is syncing, some events happening since the invokation on +remote or local side are badly handled. OfflineIMAP won't track for changes +during the sync. + * Sharing a maildir with multiple IMAP servers. + From ebf1a9300a9449055de8ef1c1e451af3f0816f1c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 20 Mar 2015 11:02:17 +0100 Subject: [PATCH 765/817] imaplib2: bump to version 2.42 Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index eddc3c4..fa4d071 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.41" +__version__ = "2.42" __release__ = "2" -__revision__ = "41" +__revision__ = "42" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -45,7 +45,7 @@ Fix for gmail "read 0" error provided by Jim Greenleaf August 2013. Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014. Conversion to Python3 provided by F. Malina February 2015. -Fix for READ-ONLY error from multiple EXAMINE/SELECT calls for same mailbox by March 2015.""" +Fix for READ-ONLY error from multiple EXAMINE/SELECT calls by March 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -1307,7 +1307,9 @@ class IMAP4(object): self._check_bye() if name in ('EXAMINE', 'SELECT'): + self.commands_lock.acquire() self.untagged_responses = [] # Flush all untagged responses + self.commands_lock.release() else: for typ in ('OK', 'NO', 'BAD'): while self._get_untagged_response(typ): @@ -1496,6 +1498,7 @@ class IMAP4(object): # Protocol mandates all lines terminated by CRLF resp = resp[:-2] + if __debug__: self._log(5, '_put_response(%s)' % resp) if 'continuation' in self.tagged_commands: continuation_expected = True @@ -1520,6 +1523,8 @@ class IMAP4(object): tag = self.mo.group('tag') typ = self.mo.group('type') dat = self.mo.group('data') + if typ in ('OK', 'NO', 'BAD') and self._match(self.response_code_cre, dat): + self._append_untagged(self.mo.group('type'), self.mo.group('data')) if not tag in self.tagged_commands: if __debug__: self._log(1, 'unexpected tagged response: %s' % resp) else: @@ -1561,15 +1566,12 @@ class IMAP4(object): return self._append_untagged(typ, dat) + if typ in ('OK', 'NO', 'BAD') and self._match(self.response_code_cre, dat): + self._append_untagged(self.mo.group('type'), self.mo.group('data')) if typ != 'OK': # NO, BYE, IDLE self._end_idle() - # Bracketed response information? - - if typ in ('OK', 'NO', 'BAD') and self._match(self.response_code_cre, dat): - self._append_untagged(self.mo.group('type'), self.mo.group('data')) - # Command waiting for aborted continuation response? if continuation_expected: @@ -2435,6 +2437,7 @@ if __name__ == '__main__': ('recent', ()), ('examine', ()), ('select', ()), + ('fetch', ("'1:*'", '(FLAGS UID)')), ('examine', ()), ('select', ()), ) From 9b55cc547545490407fad83c70778ff284aa1621 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 19 Mar 2015 23:35:03 +0100 Subject: [PATCH 766/817] website-doc.sh: fix function call Signed-off-by: Nicolas Sebrecht --- docs/website-doc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/website-doc.sh b/docs/website-doc.sh index dc1b883..ca387c3 100755 --- a/docs/website-doc.sh +++ b/docs/website-doc.sh @@ -19,7 +19,7 @@ test -d "$DESTBASE" || exit 1 # # Build the sphinx documentation. # -function sphinx_doc () { +function api () { # Build the doc with sphinx. dest="${DESTBASE}/${version}" echo "Cleaning target directory: $dest" From f11eaaea6f9f3af237d3cd5cd5ce0e317f15ed8b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 14 Mar 2015 15:45:56 +0100 Subject: [PATCH 767/817] contrib: introduce script to make new releases Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 432 ++++++++++++++++++++++++++++++++++++++++ docs/website-doc.sh | 1 + offlineimap/__init__.py | 2 +- 3 files changed, 434 insertions(+), 1 deletion(-) create mode 100755 contrib/release.sh diff --git a/contrib/release.sh b/contrib/release.sh new file mode 100755 index 0000000..0bf0444 --- /dev/null +++ b/contrib/release.sh @@ -0,0 +1,432 @@ +#!/bin/sh +# +# Put into Public Domain, by Nicolas Sebrecht +# +# Create new releases in OfflineIMAP. + +# TODO: https://developer.github.com/v3/repos/releases/#create-a-release +# https://developer.github.com/libraries/ +# https://github.com/turnkeylinux/octohub +# https://github.com/michaelliao/githubpy (onefile) +# https://github.com/sigmavirus24/github3.py +# https://github.com/copitux/python-github3 +# https://github.com/PyGithub/PyGithub +# https://github.com/micha/resty (curl) + +# TODO: move configuration out and source it. +# TODO: implement rollback. + +__VERSION__='v0.1' + +SEND_BY_MAIL='git send-email' +SPHINXBUILD=sphinx-build + +MAILING_LIST='offlineimap-project@lists.alioth.debian.org' + +DOCSDIR='docs' +ANNOUNCE_MAGIC='#### Notes ' +CHANGELOG_MAGIC='{:toc}' +CHANGELOG='Changelog.md' +CACHEDIR='.git/offlineimap-release' +WEBSITE='website' +WEBSITE_LATEST="${WEBSITE}/_data/latest.yml" + +TMP_CHANGELOG_EXCERPT="${CACHEDIR}/changelog.excerpt.md" +TMP_CHANGELOG_EXCERPT_OLD="${TMP_CHANGELOG_EXCERPT}.old" +TMP_CHANGELOG="${CACHEDIR}/changelog.md" +TMP_ANNOUNCE="${CACHEDIR}/announce.txt" + +True=0 +False=1 +Yes=$True +No=$False + +DEBUG=$True + +# +# $1: EXIT_CODE +# $2..: message +function die () { + n=$1 + shift + echo $* + exit $n +} + + +function debug () { + if test $DEBUG -eq $True + then + echo "DEBUG: $*" >&2 + fi +} + + + +# +# $1: question +# $2: message on abort +# +function ask () { + echo + echo -n "--- $1 " + read -r ans + test "n$ans" = 'n' -o "n$ans" = 'ny' && return $Yes + test "n$ans" = "ns" -o "n$ans" = 'nn' && return $No + die 1 "! $2" +} + + + +# +# $1: message +# $1: path to file +# +function edit_file () { + ask "Press Enter to $1" + test $? -eq $Yes && { + $EDITOR "$2" + reset + } +} + + + +function fix_pwd () { + debug 'in fix_pwd' + if test ! -d .git -a ! -f offlineimap.py + then + test "$PWD" = '/' && die 2 "You're not in the offlineimap repository..." + cd .. + fix_pwd + fi +} + + +function prepare_env () { + debug 'in prepare_env' + mkdir "$CACHEDIR" 2>/dev/null + test ! -d "$CACHEDIR" && die 5 "Could not make cache directory $CACHEDIR" +} + + +function check_dirty () { + debug 'in check_dirty' + git diff --quiet 2>/dev/null && git diff --quiet --cached 2>/dev/null || { + die 4 "Commit all your changes first!" + } +} + + +function welcome () { + debug 'in welcome' +cat <' : yes, continue +- 'n' : no +- 's' : skip (ONLY where applicable, otherwise continue) + +Any other key will abort the program. +EOF + ask 'Ready?' +} + + +function checkout_next () { + debug 'in checkout_next' + git checkout --quiet next || { + die 6 "Could not checkout 'next' branch" + } +} + + +function get_version () { + debug 'in get_version' + echo "v$(./offlineimap.py --version)" +} + + +function update_offlineimap_version () { + debug 'in update_offlineimap_version' + edit_file 'update the version in __init__.py' offlineimap/__init__.py +} + + +# +# $1: previous version +# +function get_shortlog () { + debug 'in get_shortlog' + git shortlog "${1}.." | grep -v '^[^ ]' | sed -r -e 's,^[ ]*,\- ,' +} + + +# +# $1: new version +# $2: shortlog +function changelog_template () { + debug 'in changelog_template' + cat < "$TMP_CHANGELOG_EXCERPT" + get_shortlog "$2" >> "$TMP_CHANGELOG_EXCERPT" + edit_file "the Changelog excerpt" $TMP_CHANGELOG_EXCERPT + + # Remove comments. + grep -v '//' "$TMP_CHANGELOG_EXCERPT" > "${TMP_CHANGELOG_EXCERPT}.nocomment" + mv -f "${TMP_CHANGELOG_EXCERPT}.nocomment" "$TMP_CHANGELOG_EXCERPT" + fi + + # Write new Changelog. + cat "$CHANGELOG" > "$TMP_CHANGELOG" + debug "include excerpt $TMP_CHANGELOG_EXCERPT to $TMP_CHANGELOG" + sed -i -e "/${CHANGELOG_MAGIC}/ r ${TMP_CHANGELOG_EXCERPT}" "$TMP_CHANGELOG" + debug 'remove trailing whitespaces' + sed -i -r -e 's, +$,,' "$TMP_CHANGELOG" # Remove trailing whitespaces. + debug "copy to $TMP_ANNOUNCE -> $CHANGELOG" + cp -f "$TMP_CHANGELOG" "$CHANGELOG" + + # Check and edit Changelog. + ask "Next step: you'll be asked to review the diff of $CHANGELOG" + action=$No + while test ! $action -eq $Yes + do + git diff -- "$CHANGELOG" | less + ask 'edit Changelog?' $CHANGELOG + action=$? + done +} + + +# +# $1: new version +# +function git_release () { + debug 'in git_release' + git commit -as -m"$1" + git tag -a "$1" -m"$1" + git checkout master + git merge next + git checkout next +} + + +function get_last_rc () { + git tag | grep -E '^v([0-9][\.-]){3}rc' | sort -n | tail -n1 +} + +# +# $1: new version +# +function update_website_releases_info() { + cat > "$WEBSITE_LATEST" < /dev/null 2>&1 && { + cd website || echo "ERROR: cannot go to website" + git diff --quiet 2>/dev/null && git diff --quiet --cached 2>/dev/null || { + echo "There is WIP in the website repository, stashing" + echo "git stash create 'WIP during offlineimap API import'" + git stash create 'WIP during offlineimap API import' + } + cd "../$DOCSDIR" + make websitedoc && { + cd ../website && { + branch_name="import-$1" + git checkout -b "$branch_name" + git add '_doc/versions' + git commit -s -m"doc: import of developer documentation for offlineimap $1" + git checkout master + git merge "$branch_name" + echo "website: master branch ready for a push!" + } + } + } || { + echo "Oops! you don't have $SPHINXBUILD installed?" + echo "Cannot update the webite documentation..." + echo "You should install it and run:" + echo " $ cd docs" + echo " $ make websitedoc" + echo "Then, commit and push changes of the website." + } + ask 'continue' + fi +} + + +function git_username () { + git config --get user.name +} +function git_usermail () { + git config --get user.email +} + +# +# $1: new version +# +function announce_header () { + cat < +Date: $(git log HEAD~1.. --oneline --pretty='%cD') +From: $(git_username) <$(git_usermail)> +To: $MAILING_LIST +Subject: [ANNOUNCE] OfflineIMAP $1 released + +OfflineIMAP $1 is out. + +Downloads: + http://github.com/OfflineIMAP/offlineimap/archive/${1}.tar.gz + http://github.com/OfflineIMAP/offlineimap/archive/${1}.zip + +EOF +} + + +# +# $1: previous version +# +function announce_footer () { + cat < "$TMP_ANNOUNCE" + cat "$TMP_CHANGELOG_EXCERPT" >> "$TMP_ANNOUNCE" + sed -i -r -e "s,^$ANNOUNCE_MAGIC,," "$TMP_ANNOUNCE" + announce_footer "$2" >> "$TMP_ANNOUNCE" +} + + +function edit_announce () { + edit_file 'edit announce' "$TMP_ANNOUNCE" +} + + +function send_announce () { + ask 'Press Enter to to send announce' + test $? -eq $Yes && $SEND_BY_MAIL "$TMP_ANNOUNCE" +} + + +function clear_env () { + rm -f "$TMP_CHANGELOG_EXCERPT" +} + + + +function run () { + debug 'in run' + fix_pwd + check_dirty + prepare_env + checkout_next + clear + welcome + + if test -f "$TMP_CHANGELOG_EXCERPT" + then + head "$TMP_CHANGELOG_EXCERPT" + ask "A previous Changelog excerpt (head above) was found, use it?" + if test ! $? -eq $Yes + then + mv -f "$TMP_CHANGELOG_EXCERPT" "$TMP_CHANGELOG_EXCERPT_OLD" + fi + fi + + previous_version="$(get_version)" + message="Safety check: release after version:" + ask "$message $previous_version ?" + update_offlineimap_version + new_version="$(get_version)" + ask "Safety check: make a new release with version: '$new_version'" "Clear changes and restart" + + update_changelog "$new_version" "$previous_version" + build_announce "$new_version" "$previous_version" + edit_announce + + git_release $new_version + + update_website $new_version + send_announce + clear_env +} + +run +cat < "$VERSIONS_YML" for version in $(ls "$DESTBASE" -1 | sort -nr) do diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 441af44..d0bd0b5 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -3,7 +3,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.5.7" __revision__ = "-rc2" -__bigversion__ = __version__ + __revision__ +__bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 97689ee484fd4fd39824c58957ef430b420f94c3 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 20 Mar 2015 11:33:29 +0100 Subject: [PATCH 768/817] v6.5.7-rc3 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 64 ++++++++++++++++++++++++++++++++++++++++- offlineimap/__init__.py | 2 +- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index 5fa6c38..8d3d0b1 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,7 +15,69 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} -### OfflineIMAP v6.5.7-rc3 (2015- - ) + +### OfflineIMAP v6.5.7-rc3 (2015-03-19) + +#### Notes + +Here comes a much bigger release than expected! With this release, the new +website is made official. + +Distribution maintainers, be aware that we now have a new man page +offlineimapui(7)! + +Also, the man page offlineimap(1) is sightly revised to explain the command line +options. Since `offlineimap --help` won't detail the options anymore, it becomes +critical. + +The maxage feature was broken by design and could delete mails on one side. It +is still under heavy work to fix issues when timezones are not synced. Gmail is +known to use different timezones accross mailboxes. + +The IMAP library imaplib2 was updated for the upcoming course to Python 3. + +The most other important changes are: + +- Possibility to use a proxy. +- All the documentation are SIGHTLY revisited and updated from all the available + places (sources files in the repository, wiki, website). A lot was moved from + the wiki and the sources to the website. +- the RFCs are available in the repository. + +#### Features + +- Add proxy support powered by PySocks. +- New man page offlineimapui to explain the available UIs. +- Add a CONTRIBUTING.rst file. +- Add a `TODO.rst` list for the contributors. +- Add a script for maintainers to roll out new releases. +- Add the `scripts/get-repository.sh` script to work on the website and the wiki. +- Doc: add IMAP RFCs. + +#### Fixes + +- Don't loose local mails because of maxage. +- Properly handle the cached messagelist. +- Do not error if `remoteuser` is not configured. +- imaplibutil: add missing errno import. +- LocalStatusSQLite: labels: don't fail if database returns unexpected None value. +- IDLE: continue trying selecting the folder on `OfflineImapError.Error`. + +#### Changes + +- imaplib2: bump to v2.42 +- `--help` becomes concise. +- Changelogs: move format back to markdown/kramdown to be more compatible with Jekyll. +- README: deep cleanups. +- code cleanups. +- code: more style consistency. +- sqlite: provide offending filename when open fails. +- MANUAL: full refactoring, change format to asciidoc. +- MANUAL: rename "KNOWN BUGS" TO "KNOWN ISSUES". +- MANUAL: add known issues entry about socktimeout for suspended sessions. +- offlineimap.conf: say what is the default value for the sep option. +- sqlite: provide information on what is failing for `OperationalError`. +- remove obsolete documentation. ### OfflineIMAP v6.5.7-rc2 (2015-01-18) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index d0bd0b5..b0ef6b2 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,7 +2,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.5.7" -__revision__ = "-rc2" +__revision__ = "-rc3" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" From 4015c1d84cffe26b005f5480a998b6d6f48b6832 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 20 Mar 2015 12:08:39 +0100 Subject: [PATCH 769/817] release.sh: dont publish anything by script Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index 0bf0444..283c965 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -18,7 +18,6 @@ __VERSION__='v0.1' -SEND_BY_MAIL='git send-email' SPHINXBUILD=sphinx-build MAILING_LIST='offlineimap-project@lists.alioth.debian.org' @@ -293,13 +292,12 @@ function update_website () { cd "../$DOCSDIR" make websitedoc && { cd ../website && { - branch_name="import-$1" - git checkout -b "$branch_name" - git add '_doc/versions' - git commit -s -m"doc: import of developer documentation for offlineimap $1" - git checkout master - git merge "$branch_name" - echo "website: master branch ready for a push!" + branch_name="import-$1" + git checkout -b "$branch_name" + git add '_doc/versions' + git commit -a -s -m"update for offlineimap $1" + git checkout master + echo "website: branch '$branch_name' ready for a merge in master!" } } } || { @@ -372,12 +370,6 @@ function edit_announce () { } -function send_announce () { - ask 'Press Enter to to send announce' - test $? -eq $Yes && $SEND_BY_MAIL "$TMP_ANNOUNCE" -} - - function clear_env () { rm -f "$TMP_CHANGELOG_EXCERPT" } @@ -417,7 +409,6 @@ function run () { git_release $new_version update_website $new_version - send_announce clear_env } @@ -426,6 +417,7 @@ cat < Date: Fri, 20 Mar 2015 12:46:08 +0100 Subject: [PATCH 770/817] release.sh: website: fix feeding the latest stable and rc Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index 283c965..be53792 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -257,17 +257,19 @@ function git_release () { } + function get_last_rc () { git tag | grep -E '^v([0-9][\.-]){3}rc' | sort -n | tail -n1 } -# -# $1: new version -# +function get_last_stable () { + git tag | grep -E '^v([0-9][\.])+' | grep -v '\-rc' | sort -n | tail -n1 +} + function update_website_releases_info() { cat > "$WEBSITE_LATEST" < Date: Fri, 20 Mar 2015 13:05:00 +0100 Subject: [PATCH 771/817] release.sh: don't remove temporary files at the end Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index be53792..af098cd 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -373,11 +373,6 @@ function edit_announce () { } -function clear_env () { - rm -f "$TMP_CHANGELOG_EXCERPT" -} - - function run () { debug 'in run' @@ -412,7 +407,6 @@ function run () { git_release $new_version update_website $new_version - clear_env } run From 60cc58c38e7bbce8c03a785ebafbf5e8d1571a82 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 17 Mar 2015 14:38:01 +0100 Subject: [PATCH 772/817] DNS of the website changed, reflect it Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 10 +++++----- README.md | 2 +- docs/doc-src/index.rst | 2 +- docs/offlineimap.txt | 2 +- offlineimap.conf | 2 +- scripts/get-repository.sh | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9f8f669..489f06b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -7,11 +7,11 @@ .. _maintainers: https://github.com/OfflineIMAP/offlineimap/blob/next/MAINTAINERS.rst .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst -.. _Community's website: https://offlineimap.github.io -.. _APIs in OfflineIMAP: http://offlineimap.github.io/documentation.html#available-apis -.. _documentation: https://offlineimap.github.io/documentation.html -.. _Coding Guidelines: http://offlineimap.github.io/doc/CodingGuidelines.html -.. _Know the status of your patches: http://localhost:4000/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission +.. _Community's website: https://offlineimap.org +.. _APIs in OfflineIMAP: http://offlineimap.org/documentation.html#available-apis +.. _documentation: https://offlineimap.org/documentation.html +.. _Coding Guidelines: http://offlineimap.org/doc/CodingGuidelines.html +.. _Know the status of your patches: http://offlineimap.org/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission ================= diff --git a/README.md b/README.md index cd97db2..ae9c6b6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [offlineimap]: https://github.com/OfflineIMAP/offlineimap -[website]: http://offlineimap.github.io +[website]: http://offlineimap.org [wiki]: http://github.com/OfflineIMAP/offlineimap/wiki # OfflineImap diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index fb8bdc9..e923a64 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -1,5 +1,5 @@ .. OfflineImap documentation master file -.. _OfflineIMAP: http://offlineimap.github.io +.. _OfflineIMAP: http://offlineimap.org Welcome to OfflineIMAP's developer documentation diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index f4b844c..858fc0b 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -410,4 +410,4 @@ See Also -------- offlineimapui(7), openssl(1), signal(7), sqlite3(1). - http://offlineimap.github.io + http://offlineimap.org diff --git a/offlineimap.conf b/offlineimap.conf index bd84d6e..7fbbdf7 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -2,7 +2,7 @@ # This file documents *all* possible options and can be quite scary. # Looking for a quick start? Take a look at offlineimap.conf.minimal. -# More details can be found at http://offlineimap.github.io. +# More details can be found at http://offlineimap.org . ################################################## # Overview diff --git a/scripts/get-repository.sh b/scripts/get-repository.sh index 080815c..a6dad95 100755 --- a/scripts/get-repository.sh +++ b/scripts/get-repository.sh @@ -79,7 +79,7 @@ username=$(echo $offlineimap_url | sed -r -e 's,.*github.com.([^/]+)/.*,\1,') case n$repository in nwebsite) - upstream=https://github.com/OfflineIMAP/offlineimap.github.io + upstream=https://github.com/OfflineIMAP/offlineimap.github.io setup website "$upstream" configure_website "$username" final_note website "$upstream" From 428349e3f4fcfc702acd307af4c4d5692bc7e469 Mon Sep 17 00:00:00 2001 From: Janna Martl Date: Sat, 21 Mar 2015 02:15:25 -0400 Subject: [PATCH 773/817] fix inaccurate UI messages when some messages are excluded from the cached lists Some messages were excluded from the copy/delete list after the UI message said they were copied/deleted. Also fix Internaldate2epoch(), which was wrong. Signed-off-by: Janna Martl Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 39 ++++++++++++++++++++++------------- offlineimap/folder/IMAP.py | 2 -- offlineimap/folder/Maildir.py | 3 --- offlineimap/imaplibutil.py | 4 +++- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index df2e654..e033ab2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -365,7 +365,8 @@ class BaseFolder(object): dryrun mode.""" for uid in uidlist: - self.addmessageflags(uid, flags) + if self.uidexists(uid): + self.addmessageflags(uid, flags) def deletemessageflags(self, uid, flags): """Removes each flag given from the message's flag set. If a given @@ -695,11 +696,6 @@ class BaseFolder(object): content = self.getmessage(uid) rtime = emailutil.get_message_date(content, 'Date') - if uid > 0 and dstfolder.uidexists(uid): - # dst has message with that UID already, only update status - statusfolder.savemessage(uid, None, flags, rtime) - return - # If any of the destinations actually stores the message body, # load it up. if dstfolder.storesmessages(): @@ -766,6 +762,16 @@ class BaseFolder(object): # bail out on CTRL-C or SIGTERM if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break + if uid > 0 and dstfolder.uidexists(uid): + # dst has message with that UID already, only update status + flags = self.getmessageflags(uid) + rtime = self.getmessagetime(uid) + if dstfolder.utime_from_message: + content = self.getmessage(uid) + rtime = emailutil.get_message_date(content, 'Date') + statusfolder.savemessage(uid, None, flags, rtime) + copylist.remove(uid) + self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # exceptions are caught in copymessageto() if self.suggeststhreads() and not globals.options.singlethreading: @@ -790,19 +796,24 @@ class BaseFolder(object): that were deleted in 'self'. Delete those from dstfolder and statusfolder. - This function checks and protects us from action in ryrun mode. + This function checks and protects us from action in dryrun mode. """ deletelist = filter(lambda uid: uid >= 0 and not self.uidexists(uid), statusfolder.getmessageuidlist()) if len(deletelist): - self.ui.deletingmessages(deletelist, [dstfolder]) - if self.repository.account.dryrun: - return #don't delete messages in dry-run mode - # delete in statusfolder first to play safe. In case of abort, we - # won't lose message, we will just retransmit some unneccessary. - for folder in [statusfolder, dstfolder]: - folder.deletemessages(deletelist) + # Delete in statusfolder first to play safe. In case of abort, we + # won't lose message, we will just unneccessarily retransmit some. + # Delete messages from statusfolder that were either deleted by the + # user, or not being tracked (e.g. because of maxage). + statusfolder.deletemessages(deletelist) + # Filter out untracked messages + deletelist = filter(lambda uid: dstfolder.uidexists(uid), deletelist) + if len(deletelist): + self.ui.deletingmessages(deletelist, [dstfolder]) + if self.repository.account.dryrun: + return #don't delete messages in dry-run mode + dstfolder.deletemessages(deletelist) def __syncmessagesto_flags(self, dstfolder, statusfolder): """Pass 3: Flag synchronization. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index ccc83b0..4b470a2 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -836,8 +836,6 @@ class IMAPFolder(BaseFolder): self.__deletemessages_noconvert(uidlist) def __deletemessages_noconvert(self, uidlist): - # Weed out ones not in self.messagelist - uidlist = [uid for uid in uidlist if self.uidexists(uid)] if not len(uidlist): return diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 77f7ebb..6df8180 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -439,9 +439,6 @@ class MaildirFolder(BaseFolder): :return: Nothing, or an Exception if UID but no corresponding file found. """ - if not self.uidexists(uid): - return - filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) try: diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 30aed9a..4f33be1 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -222,6 +222,8 @@ def Internaldate2epoch(resp): Returns seconds since the epoch.""" + from calendar import timegm + mo = InternalDate.match(resp) if not mo: return None @@ -245,4 +247,4 @@ def Internaldate2epoch(resp): tt = (year, mon, day, hour, min, sec, -1, -1, -1) - return time.mktime(tt) + return timegm(tt) - zone From fe92025c8ff14d6bf8afbbaed05fa1895fb5875e Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 21 Mar 2015 14:15:42 +0100 Subject: [PATCH 774/817] release.sh: git_shortlog() -> get_git_history() Use 'log --oneline' rather than 'shortlog'. Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index af098cd..e2e482b 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -156,12 +156,13 @@ function update_offlineimap_version () { # # $1: previous version # -function get_shortlog () { - debug 'in get_shortlog' - git shortlog "${1}.." | grep -v '^[^ ]' | sed -r -e 's,^[ ]*,\- ,' +function get_git_history () { + debug 'in get_git_history' + git log --oneline "${1}.." | sed -r -e 's,^(.),\- \1,' } + # # $1: new version # $2: shortlog @@ -215,7 +216,7 @@ function update_changelog () { if test ! -f "$TMP_CHANGELOG_EXCERPT" then changelog_template "$1" > "$TMP_CHANGELOG_EXCERPT" - get_shortlog "$2" >> "$TMP_CHANGELOG_EXCERPT" + get_git_history "$2" >> "$TMP_CHANGELOG_EXCERPT" edit_file "the Changelog excerpt" $TMP_CHANGELOG_EXCERPT # Remove comments. From 731129396c3a610a147a7b7e798e5a41de9153f0 Mon Sep 17 00:00:00 2001 From: Janna Martl Date: Sat, 21 Mar 2015 21:23:29 -0400 Subject: [PATCH 775/817] fix: don't copy messages if not necessary Fix regresssion introduced in 428349e3. Prevent messages with UID's already in the destination folder from getting excluded from the copy list. Signed-off-by: Janna Martl Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index e033ab2..e3c80b7 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -770,7 +770,7 @@ class BaseFolder(object): content = self.getmessage(uid) rtime = emailutil.get_message_date(content, 'Date') statusfolder.savemessage(uid, None, flags, rtime) - copylist.remove(uid) + continue self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # exceptions are caught in copymessageto() From 2d14f005d7f74c70269a8626b749782bcaaab037 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 22 Mar 2015 12:58:56 +0100 Subject: [PATCH 776/817] contrib: add systemd configuration files Signed-off-by: Nicolas Sebrecht --- contrib/systemd/offlineimap.service | 11 +++++++++++ contrib/systemd/offlineimap.timer | 9 +++++++++ contrib/systemd/offlineimap@.service | 11 +++++++++++ contrib/systemd/offlineimap@.timer | 9 +++++++++ 4 files changed, 40 insertions(+) create mode 100644 contrib/systemd/offlineimap.service create mode 100644 contrib/systemd/offlineimap.timer create mode 100644 contrib/systemd/offlineimap@.service create mode 100644 contrib/systemd/offlineimap@.timer diff --git a/contrib/systemd/offlineimap.service b/contrib/systemd/offlineimap.service new file mode 100644 index 0000000..c76a5f9 --- /dev/null +++ b/contrib/systemd/offlineimap.service @@ -0,0 +1,11 @@ +[Unit] +Description=Offlineimap Service +After=xdg-env.service +Requires=xdg-env.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/offlineimap -o -c ${XDG_CONFIG_HOME}/offlineimap/offlineimaprc + +[Install] +WantedBy=mail.target diff --git a/contrib/systemd/offlineimap.timer b/contrib/systemd/offlineimap.timer new file mode 100644 index 0000000..ef84a3d --- /dev/null +++ b/contrib/systemd/offlineimap.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Offlineimap Query Timer + +[Timer] +OnUnitInactiveSec=15m +Unit=offlineimap.service + +[Install] +WantedBy=mail.target diff --git a/contrib/systemd/offlineimap@.service b/contrib/systemd/offlineimap@.service new file mode 100644 index 0000000..fa1eea0 --- /dev/null +++ b/contrib/systemd/offlineimap@.service @@ -0,0 +1,11 @@ +[Unit] +Description=Offlineimap Service for account %i +After=xdg-env.service +Requires=xdg-env.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/offlineimap -o -a %i -c ${XDG_CONFIG_HOME}/offlineimap/offlineimaprc + +[Install] +WantedBy=mail.target diff --git a/contrib/systemd/offlineimap@.timer b/contrib/systemd/offlineimap@.timer new file mode 100644 index 0000000..6cdbac4 --- /dev/null +++ b/contrib/systemd/offlineimap@.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Offlineimap Query Timer for account %i + +[Timer] +OnUnitInactiveSec=15m +Unit=offlineimap@%i.service + +[Install] +WantedBy=mail.target From 4217fccb821f35238bc63c3e672093911a6ddc1d Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Sun, 22 Mar 2015 14:34:24 -0400 Subject: [PATCH 777/817] mbnames: add option to write the file once per account The basic problem is in the context of syncing multiple accounts where one is fast and the others are slower (due to the number of folders). When the fast account completes, the other accounts are partially written through the list and if the file is read during this time, the list can be useless. However, in the general case, the file is probably left around from a previous run of offlineimap and is more correct, so add an option to leave it alone until all syncing is done. Incremental is still the default since this running offlineimap using its own timer setup is likely the most common setup. Turning it off works best with one-shot mode triggered by cron or systemd timers. Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 8 ++++++++ offlineimap/accounts.py | 2 +- offlineimap/init.py | 4 ++++ offlineimap/mbnames.py | 12 +++++++++++- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 7fbbdf7..c447d7b 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -182,6 +182,13 @@ accounts = Test # The header, peritem, sep, and footer are all Python expressions passed # through eval, so you can (and must) use Python quoting. # +# The incremental setting controls whether the file is written after each +# account completes or once all synced accounts are complete. This is usefull if +# an account is sightly slower than the other. It allows keeping the previous +# file rather than having it partially written. +# This works best with "no" if in one-shot mode started by cron or systemd +# timers. Default: no. +# # The following hash key are available to the expansion for 'peritem': # - accountname: the name of the corresponding account; # - foldername: the name of the folder; @@ -197,6 +204,7 @@ accounts = Test #peritem = "+%(accountname)s/%(foldername)s" #sep = " " #footer = "\n" +#incremental = no # This option stands in the [mbnames] section. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 62ed5c3..cac4d88 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -359,7 +359,7 @@ class SyncableAccount(Account): thr.join() # Write out mailbox names if required and not in dry-run mode if not self.dryrun: - mbnames.write() + mbnames.write(False) localrepos.forgetfolders() remoterepos.forgetfolders() except: diff --git a/offlineimap/init.py b/offlineimap/init.py index a5277cb..a48c152 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -334,6 +334,10 @@ class OfflineImap: 'config': self.config}) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) + + if not options.dryrun: + offlineimap.mbnames.write(True) + self.ui.terminate() except (SystemExit): raise diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 936a110..8829ee5 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -38,7 +38,17 @@ def add(accountname, foldername, localfolders): if not foldername in boxes[accountname]: boxes[accountname].append(foldername) -def write(): +def write(allcomplete): + incremental = config.getdefaultboolean("mbnames", "incremental", False) + + # Skip writing if we don't want incremental writing and we're not done. + if not incremental and not allcomplete: + return + + # Skip writing if we want incremental writing and we're done. + if incremental and allcomplete: + return + # See if we're ready to write it out. for account in accounts: if account not in boxes: From 4f19927cee34ecc5b1556a0d8423856633cff266 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Sun, 22 Mar 2015 14:52:05 -0400 Subject: [PATCH 778/817] systemd: remove explicit `-c` argument Removes the need for the non-existent xdg-env.service file and offlineimap reads these files by default now anyways. Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- contrib/systemd/offlineimap.service | 4 +--- contrib/systemd/offlineimap@.service | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/contrib/systemd/offlineimap.service b/contrib/systemd/offlineimap.service index c76a5f9..f29f93c 100644 --- a/contrib/systemd/offlineimap.service +++ b/contrib/systemd/offlineimap.service @@ -1,11 +1,9 @@ [Unit] Description=Offlineimap Service -After=xdg-env.service -Requires=xdg-env.service [Service] Type=oneshot -ExecStart=/usr/bin/offlineimap -o -c ${XDG_CONFIG_HOME}/offlineimap/offlineimaprc +ExecStart=/usr/bin/offlineimap -o [Install] WantedBy=mail.target diff --git a/contrib/systemd/offlineimap@.service b/contrib/systemd/offlineimap@.service index fa1eea0..7be965a 100644 --- a/contrib/systemd/offlineimap@.service +++ b/contrib/systemd/offlineimap@.service @@ -1,11 +1,9 @@ [Unit] Description=Offlineimap Service for account %i -After=xdg-env.service -Requires=xdg-env.service [Service] Type=oneshot -ExecStart=/usr/bin/offlineimap -o -a %i -c ${XDG_CONFIG_HOME}/offlineimap/offlineimaprc +ExecStart=/usr/bin/offlineimap -o -a %i [Install] WantedBy=mail.target From 9c2981915fceb7641a855b7fc30d92d1ac146318 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Sun, 22 Mar 2015 14:52:44 -0400 Subject: [PATCH 779/817] systemd: add documentation and a sample mail.target Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- contrib/systemd/README.md | 16 ++++++++++++++++ contrib/systemd/mail.target | 5 +++++ 2 files changed, 21 insertions(+) create mode 100644 contrib/systemd/README.md create mode 100644 contrib/systemd/mail.target diff --git a/contrib/systemd/README.md b/contrib/systemd/README.md new file mode 100644 index 0000000..5ae31e4 --- /dev/null +++ b/contrib/systemd/README.md @@ -0,0 +1,16 @@ +Systemd units +============= + +These unit files are meant to be used in the user session. You may drop them +into `${XDG_DATA_HOME}/systemd/user` followed by `systemctl --user +daemon-reload` to have systemd aware of the unit files. + +These files are meant to be triggered either manually using `systemctl --user +start offlineimap.service` or by enabling the timer unit using `systemctl +--user enable offlineimap.timer`. Additionally, specific accounts may be +triggered by using `offlineimap@myaccount.timer` or +`offlineimap@myaccount.service`. + +These unit files are installed as being enabled via a `mail.target` unit which +is intended to be a catch-all for mail-related unit files. A simple +`mail.target` file is also provided. diff --git a/contrib/systemd/mail.target b/contrib/systemd/mail.target new file mode 100644 index 0000000..5a408b2 --- /dev/null +++ b/contrib/systemd/mail.target @@ -0,0 +1,5 @@ +[Unit] +Description=Mail Target + +[Install] +WantedBy=default.target From d4ea6da8573788e8f054c2d3ef6a4c6e66abecd1 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 23 Mar 2015 10:50:21 +0100 Subject: [PATCH 780/817] CONTRIBUTING: some documentations have move Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 489f06b..9bc6f7f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -114,7 +114,7 @@ There is already a lot of documentation. Here's where you might want to look first: - The directory ``offlineimap/docs`` has all kind of additional documentation - (advanced Git, RFCs, APIs, coding guidelines, etc). + (man pages, RFCs). - The file ``offlineimap.conf`` allows to know all the supported features. From 1708fd09d1ea73d35686f3d5c6b81c0715e16889 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 25 Mar 2015 13:39:13 +0100 Subject: [PATCH 781/817] imaplib2: bump to v2.43 Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index fa4d071..eeb4792 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.42" +__version__ = "2.43" __release__ = "2" -__revision__ = "42" +__revision__ = "43" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -45,7 +45,8 @@ Fix for gmail "read 0" error provided by Jim Greenleaf August 2013. Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014. Conversion to Python3 provided by F. Malina February 2015. -Fix for READ-ONLY error from multiple EXAMINE/SELECT calls by March 2015.""" +Fix for READ-ONLY error from multiple EXAMINE/SELECT calls by Pierre-Louis Bonicoli March 2015. +Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli March 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -1483,6 +1484,7 @@ class IMAP4(object): if self._expecting_data: rlen = len(resp) dlen = min(self._expecting_data_len, rlen) + if __debug__: self._log(5, '_put_response expecting data len %s, got %s' % (self._expecting_data_len, rlen)) self._expecting_data_len -= dlen self._expecting_data = (self._expecting_data_len != 0) if rlen <= dlen: @@ -1515,7 +1517,8 @@ class IMAP4(object): return typ = self._literal_expected[0] self._literal_expected = None - self._append_untagged(typ, dat) # Tail + if dat: + self._append_untagged(typ, dat) # Tail if __debug__: self._log(4, 'literal completed') else: # Command completion response? @@ -2431,15 +2434,15 @@ if __name__ == '__main__': ('response', ('UIDVALIDITY',)), ('response', ('EXISTS',)), ('append', (None, None, None, test_mesg)), - ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')), - ('uid', ('SEARCH', 'ALL')), - ('uid', ('THREAD', 'references', 'UTF-8', '(SEEN)')), - ('recent', ()), ('examine', ()), ('select', ()), ('fetch', ("'1:*'", '(FLAGS UID)')), ('examine', ()), ('select', ()), + ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')), + ('uid', ('SEARCH', 'ALL')), + ('uid', ('THREAD', 'references', 'UTF-8', '(SEEN)')), + ('recent', ()), ) @@ -2460,7 +2463,7 @@ if __name__ == '__main__': def run(cmd, args, cb=True): if AsyncError: - M._log(1, 'AsyncError') + M._log(1, 'AsyncError %s' % repr(AsyncError)) M.logout() typ, val = AsyncError raise typ(val) From 478091f9ac68c2f44d7d1a6392141eba2ba9fdbc Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Wed, 25 Mar 2015 12:15:44 +0100 Subject: [PATCH 782/817] fix paths in systemd README The right places to manually put systemd user units is: * /etc/systemd/user if you want them to be available to all users, * ${XDG_CONFIG_HOME}/systemd/user for a single user. The upstream rationale is: user configuration goes to /etc/systemd or $XDG_CONFIG_HOME/systemd, while package provided config goes to /usr/lib/systemd or $XDG_DATA_HOME/systemd. If offlineimap ever installs systemd units from the install scripts, it should install them to /usr/lib/systemd/user. Signed-off-by: Abdo Roig-Maranges --- contrib/systemd/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/systemd/README.md b/contrib/systemd/README.md index 5ae31e4..589d0ee 100644 --- a/contrib/systemd/README.md +++ b/contrib/systemd/README.md @@ -2,8 +2,8 @@ Systemd units ============= These unit files are meant to be used in the user session. You may drop them -into `${XDG_DATA_HOME}/systemd/user` followed by `systemctl --user -daemon-reload` to have systemd aware of the unit files. +into `/etc/systemd/user` or `${XDG_CONFIG_HOME}/systemd/user` followed by +`systemctl --user daemon-reload` to have systemd aware of the unit files. These files are meant to be triggered either manually using `systemctl --user start offlineimap.service` or by enabling the timer unit using `systemctl From aec5430457819ee538f22577852c3cf93288b1a3 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 25 Mar 2015 20:09:59 +0100 Subject: [PATCH 783/817] website-doc.sh: include user contributions Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 8 ++----- contrib/systemd/README.md | 27 ++++++++++++----------- docs/Makefile | 1 + docs/website-doc.sh | 46 ++++++++++++++++++++++++++++++--------- 4 files changed, 53 insertions(+), 29 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index e2e482b..e740cdc 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -93,12 +93,8 @@ function edit_file () { function fix_pwd () { debug 'in fix_pwd' - if test ! -d .git -a ! -f offlineimap.py - then - test "$PWD" = '/' && die 2 "You're not in the offlineimap repository..." - cd .. - fix_pwd - fi + cd "$(git rev-parse --show-toplevel)" || \ + die 2 "cannot determine the root of the repository" } diff --git a/contrib/systemd/README.md b/contrib/systemd/README.md index 5ae31e4..04f39f6 100644 --- a/contrib/systemd/README.md +++ b/contrib/systemd/README.md @@ -1,16 +1,17 @@ -Systemd units -============= +--- +layout: page +title: Integrating OfflineIMAP into systemd +author: Ben Boeckel +date: 2015-03-22 +--- -These unit files are meant to be used in the user session. You may drop them -into `${XDG_DATA_HOME}/systemd/user` followed by `systemctl --user -daemon-reload` to have systemd aware of the unit files. + -These files are meant to be triggered either manually using `systemctl --user -start offlineimap.service` or by enabling the timer unit using `systemctl ---user enable offlineimap.timer`. Additionally, specific accounts may be -triggered by using `offlineimap@myaccount.timer` or -`offlineimap@myaccount.service`. -These unit files are installed as being enabled via a `mail.target` unit which -is intended to be a catch-all for mail-related unit files. A simple -`mail.target` file is also provided. +## Systemd units + +These unit files are meant to be used in the user session. You may drop them into `${XDG_DATA_HOME}/systemd/user` followed by `systemctl --user daemon-reload` to have systemd aware of the unit files. + +These files are meant to be triggered either manually using `systemctl --user start offlineimap.service` or by enabling the timer unit using `systemctl --user enable offlineimap.timer`. Additionally, specific accounts may be triggered by using `offlineimap@myaccount.timer` or `offlineimap@myaccount.service`. + +These unit files are installed as being enabled via a `mail.target` unit which is intended to be a catch-all for mail-related unit files. A simple `mail.target` file is also provided. diff --git a/docs/Makefile b/docs/Makefile index 0efd63b..aadfd66 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -30,6 +30,7 @@ api: websitedoc: ./website-doc.sh releases ./website-doc.sh api + ./website-doc.sh contrib clean: $(RM) -f $(HTML_TARGETS) diff --git a/docs/website-doc.sh b/docs/website-doc.sh index abfa096..0565c71 100755 --- a/docs/website-doc.sh +++ b/docs/website-doc.sh @@ -6,15 +6,38 @@ ARGS=$* SPHINXBUILD=sphinx-build TMPDIR='/tmp/offlineimap-sphinx-doctrees' -WEBSITE='../website' +WEBSITE='./website' DOCBASE="${WEBSITE}/_doc" DESTBASE="${DOCBASE}/versions" VERSIONS_YML="${WEBSITE}/_data/versions.yml" ANNOUNCES_YML="${WEBSITE}/_data/announces.yml" +CONTRIB_YML="${WEBSITE}/_data/contribs.yml" +CONTRIB="${DOCBASE}/contrib" +HEADER="# DO NOT EDIT MANUALLY: it is generated by a script (website-doc.sh)." + + +function fix_pwd () { + cd "$(git rev-parse --show-toplevel)" || \ + exit 2 "cannot determine the root of the repository" + test -d "$DESTBASE" || exit 1 +} + +fix_pwd +version="v$(./offlineimap.py --version)" + + + +# +# Add the doc for the contrib files. +# +function contrib () { + echo $HEADER > "$CONTRIB_YML" + # systemd + cp -afv "./contrib/systemd/README.md" "${CONTRIB}/systemd.md" + echo "- {filename: 'systemd', linkname: 'Integrate with systemd'}" >> "$CONTRIB_YML" +} -version="v$(../offlineimap.py --version)" -test -d "$DESTBASE" || exit 1 # # Build the sphinx documentation. @@ -24,13 +47,13 @@ function api () { dest="${DESTBASE}/${version}" echo "Cleaning target directory: $dest" rm -rf "$dest" - $SPHINXBUILD -b html -d "$TMPDIR" doc-src "$dest" + $SPHINXBUILD -b html -d "$TMPDIR" ./docs/doc-src "$dest" # Build the JSON definitions for Jekyll. # This let know the website about the available APIs documentations. echo "Building Jekyll data: $VERSIONS_YML" # Erase previous content. - echo "# DO NOT EDIT MANUALLY: it is generated by a script (website-doc.sh)" > "$VERSIONS_YML" + echo "$HEADER" > "$VERSIONS_YML" for version in $(ls "$DESTBASE" -1 | sort -nr) do echo "- $version" @@ -44,7 +67,7 @@ function api () { # function releases () { # Copy the Changelogs. - for foo in ../Changelog.md ../Changelog.maint.md + for foo in ./Changelog.md ./Changelog.maint.md do cp -afv "$foo" "$DOCBASE" done @@ -52,8 +75,8 @@ function releases () { # Build the announces JSON list. Format is JSON: # - {version: '', link: ''} # - ... - echo "# DO NOT EDIT MANUALLY: it is generated by a script (website-doc.sh)" > "$ANNOUNCES_YML" - grep -E '^### OfflineIMAP' ../Changelog.md | while read title + echo "$HEADER" > "$ANNOUNCES_YML" + grep -E '^### OfflineIMAP' ./Changelog.md | while read title do link="$(echo $title | sed -r -e 's,^### (OfflineIMAP.*)\),\1,' \ | tr '[:upper:]' '[:lower:]' \ @@ -68,11 +91,11 @@ function releases () { } - exit_code=0 test "n$ARGS" = 'n' && ARG='usage' # no option passed for arg in $ARGS do + # PWD was fixed at the very beginning. case "n$arg" in "nreleases") releases @@ -80,8 +103,11 @@ do "napi") api ;; + "ncontrib") + contrib + ;; "nusage") - echo "Usage: website-doc.sh " + echo "Usage: website-doc.sh " ;; *) echo "unkown option $arg" From f1525094469db1a26ece022964465f90a1db1080 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 25 Mar 2015 20:10:29 +0100 Subject: [PATCH 784/817] Makefile: learn to build websitedoc Signed-off-by: Nicolas Sebrecht --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 3aca20a..da5c660 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,9 @@ clean: docs: @$(MAKE) -C docs +websitedoc: + @$(MAKE) -C websitedoc + targz: ../$(TARGZ) ../$(TARGZ): if ! pwd | grep -q "/offlineimap-$(VERSION)$$"; then \ From a70971585d9d8a0d9693719d5939f763ad1a9621 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 25 Mar 2015 22:19:42 +0100 Subject: [PATCH 785/817] contrib: systemd: give credits Signed-off-by: Nicolas Sebrecht --- contrib/systemd/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/systemd/README.md b/contrib/systemd/README.md index 05ccb30..5394945 100644 --- a/contrib/systemd/README.md +++ b/contrib/systemd/README.md @@ -3,6 +3,8 @@ layout: page title: Integrating OfflineIMAP into systemd author: Ben Boeckel date: 2015-03-22 +contributors: Abdo Roig-Maranges +updated: 2015-03-25 --- From 05fa3118787e2e14d7f10560cdd16f48ca8ad06a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 26 Mar 2015 12:02:56 +0100 Subject: [PATCH 786/817] Revert "fix: don't loose local mails because of maxage" This partially reverts commit 25513e90387f60831b94c6041de08db5ba12e95f. Only changes about dates and times are reverted. The changes about the style are kept. Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 38 +++++++---------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 6df8180..79c34a7 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -31,7 +31,7 @@ try: # python 2.6 has set() built in except NameError: from sets import Set as set -from offlineimap import OfflineImapError, emailutil +from offlineimap import OfflineImapError # Find the UID in a message filename re_uidmatch = re.compile(',U=(\d+)') @@ -245,35 +245,16 @@ class MaildirFolder(BaseFolder): filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) - def new_message_filename(self, uid, flags=set(), rtime=None): + def new_message_filename(self, uid, flags=set()): """Creates a new unique Maildir filename :param uid: The UID`None`, or a set of maildir flags :param flags: A set of maildir flags :returns: String containing unique message filename""" - # Prefer internal time (rtime) over retrieval time (_gettimeseq()) for - # consistency in the maxage check, which compares times of local mail - # (gotten from the filename) to the internal time of the remote mail. - if rtime is None: - timeval, timeseq = _gettimeseq() - else: - timeval, timeseq = rtime, 0 - - def build_filename(timeval, timeseq, pid, hostname, uid, folder_id, - infosep, flags): - """Compute a new filename with 'time sequence' uniqueness in mind.""" - filename = '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% ( - timeval, timeseq, pid, hostname, uid, folder_id, infosep, flags) - for imap_dir in ['cur', 'new', 'tmp']: - if os.path.exists(os.path.join(self.getfullname(), imap_dir, - filename)): - # Increase timeseq. - return build_filename(timeval, timeseq + 1, pid, hostname, - uid, folder_id, infosep, flags) - return filename - - return build_filename(timeval, timeseq, os.getpid(), socket.gethostname(), + timeval, timeseq = _gettimeseq() + return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \ + (timeval, timeseq, os.getpid(), socket.gethostname(), uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) @@ -341,10 +322,7 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - # Make sure rtime is the internal date, so it can be put in the filename - if rtime is None: - rtime = emailutil.get_message_date(content) - messagename = self.new_message_filename(uid, flags, rtime=rtime) + messagename = self.new_message_filename(uid, flags) tmpname = self.save_to_tmp_file(messagename, content) if rtime != None: os.utime(os.path.join(self.getfullname(), tmpname), (rtime, rtime)) @@ -420,10 +398,8 @@ class MaildirFolder(BaseFolder): oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) flags = self.getmessageflags(uid) - content = self.getmessage(uid) - rtime = emailutil.get_message_date(content) newfilename = os.path.join(dir_prefix, - self.new_message_filename(new_uid, flags, rtime=rtime)) + self.new_message_filename(new_uid, flags)) os.rename(os.path.join(self.getfullname(), oldfilename), os.path.join(self.getfullname(), newfilename)) self.messagelist[new_uid] = self.messagelist[uid] From 5a4e4fd0c2b2c369040bb5b3f51aaf9c5a87d9c1 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 27 Mar 2015 16:50:17 +0100 Subject: [PATCH 787/817] rename utime_from_message to utime_from_header Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index e3c80b7..16b5819 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -48,11 +48,11 @@ class BaseFolder(object): self.visiblename = '' self.config = repository.getconfig() - utime_from_message_global = self.config.getdefaultboolean( - "general", "utime_from_message", False) + utime_from_header_global = self.config.getdefaultboolean( + "general", "utime_from_header", False) repo = "Repository " + repository.name - self._utime_from_message = self.config.getdefaultboolean(repo, - "utime_from_message", utime_from_message_global) + self._utime_from_header = self.config.getdefaultboolean(repo, + "utime_from_header", utime_from_header_global) # Determine if we're running static or dynamic folder filtering # and check filtering status @@ -95,8 +95,8 @@ class BaseFolder(object): return self.repository.should_sync_folder(self.ffilter_name) @property - def utime_from_message(self): - return self._utime_from_message + def utime_from_header(self): + return self._utime_from_header def suggeststhreads(self): """Returns true if this folder suggests using threads for actions; @@ -692,7 +692,7 @@ class BaseFolder(object): message = None flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - if dstfolder.utime_from_message: + if dstfolder.utime_from_header: content = self.getmessage(uid) rtime = emailutil.get_message_date(content, 'Date') @@ -766,7 +766,7 @@ class BaseFolder(object): # dst has message with that UID already, only update status flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - if dstfolder.utime_from_message: + if dstfolder.utime_from_header: content = self.getmessage(uid) rtime = emailutil.get_message_date(content, 'Date') statusfolder.savemessage(uid, None, flags, rtime) From 8fc806113301c93f4dad7ddc1a21fca246e9933c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 27 Mar 2015 17:00:14 +0100 Subject: [PATCH 788/817] offlineimap.conf: document the "utime_from_header" option This was introduced in 3bc66c0858ecc2e69ff5e06b27cdf03a69128441 and never documented. Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index c447d7b..5bc48a8 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -458,6 +458,22 @@ localfolders = ~/Test #restoreatime = no +# This option stands in the [Repository LocalExample] section. +# +# Set modification time of messages basing on the message's "Date" header. This +# option makes sense for the Maildir type, only. +# +# This is useful if you are doing some processing/finding on your Maildir (for +# example, finding messages older than 3 months), without parsing each +# file/message content. +# +# If enabled, this forbid the -q (quick mode) CLI option to work correctly. +# +# Default: no. +# +#utime_from_header = no + + [Repository GmailLocalExample] # This type of repository enables syncing of Gmail. All Maildir From 1cdf34e100782f627fcb2b1d3cc8ff554b905d0b Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 12:00:22 +0100 Subject: [PATCH 789/817] remove unnecessary imaplib2 workaround Upstream bug has been fixed, since imaplib2 v2.42 the untagged responses are flushed (as stated by the documentation). See https://sourceforge.net/p/imaplib2/bugs/7/ Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 30aed9a..3719673 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -46,8 +46,6 @@ class UsefulIMAPMixIn(object): not force: # No change; return. return - # Wipe out all old responses, to maintain semantics with old imaplib2 - del self.untagged_responses[:] try: result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) except self.readonly as e: From afcd64003567857523a4d48bdc2a36c6d952e902 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 12:07:39 +0100 Subject: [PATCH 790/817] test: folder separator could be dot or slash Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/OLItest/TestRunner.py | 2 +- test/OLItest/globals.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 828df59..f768018 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -159,7 +159,7 @@ class OLITestLib(): #folder = folder.replace(br'\"', b'"') # remove quoting dirs.append(folder) # 2) filter out those not starting with INBOX.OLItest and del... - dirs = [d for d in dirs if d.startswith(b'INBOX.OLItest')] + dirs = [d for d in dirs if d.startswith(b'INBOX.OLItest') or d.startswith(b'INBOX/OLItest')] for folder in dirs: res_t, data = imapobj.delete(b'\"'+folder+b'\"') assert res_t == 'OK', "Folder deletion of {0} failed with error"\ diff --git a/test/OLItest/globals.py b/test/OLItest/globals.py index 8e69ee8..ba072e2 100644 --- a/test/OLItest/globals.py +++ b/test/OLItest/globals.py @@ -38,5 +38,5 @@ localfolders = type=IMAP # Don't hammer the server with too many connection attempts: maxconnections=1 -folderfilter= lambda f: f.startswith('INBOX.OLItest') +folderfilter= lambda f: f.startswith('INBOX.OLItest') or f.startswith('INBOX/OLItest') """) From be89a7605f7a011345e51a780ea270fce0f5cef2 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 12:13:23 +0100 Subject: [PATCH 791/817] test: folders: keep quotes Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/OLItest/TestRunner.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index f768018..33ba6fa 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -148,20 +148,18 @@ class OLITestLib(): assert res_t == 'OK' dirs = [] for d in data: - m = re.search(br''' # Find last quote - "((?: # Non-tripple quoted can contain... - [^"] | # a non-quote - \\" # a backslashded quote - )*)" # closing quote - [^"]*$ # followed by no more quotes + m = re.search(br''' + (" # starting quote + ([^"]|\\")* # a non-quote or a backslashded quote + ")$ # ending quote ''', d, flags=re.VERBOSE) folder = bytearray(m.group(1)) #folder = folder.replace(br'\"', b'"') # remove quoting dirs.append(folder) # 2) filter out those not starting with INBOX.OLItest and del... - dirs = [d for d in dirs if d.startswith(b'INBOX.OLItest') or d.startswith(b'INBOX/OLItest')] + dirs = [d for d in dirs if d.startswith(b'"INBOX.OLItest') or d.startswith(b'"INBOX/OLItest')] for folder in dirs: - res_t, data = imapobj.delete(b'\"'+folder+b'\"') + res_t, data = imapobj.delete(folder) assert res_t == 'OK', "Folder deletion of {0} failed with error"\ ":\n{1} {2}".format(folder.decode('utf-8'), res_t, data) imapobj.logout() From 8fbf741f773bab5bc1f1c7a7b028329003aedec5 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 12:16:00 +0100 Subject: [PATCH 792/817] test: remove commented statements Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/tests/test_01_basic.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 3cbd5c9..0a11be7 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -31,7 +31,6 @@ def setUpModule(): def tearDownModule(): logging.info("Tear Down test module") - # comment out next line to keep testdir after test runs. TODO: make nicer OLITestLib.delete_test_dir() #Stuff that can be used @@ -42,19 +41,7 @@ def tearDownModule(): #self.assertFalse(element in self.seq) class TestBasicFunctions(unittest.TestCase): - #@classmethod - #def setUpClass(cls): - #This is run before all tests in this class - # cls._connection = createExpensiveConnectionObject() - #@classmethod - #This is run after all tests in this class - #def tearDownClass(cls): - # cls._connection.destroy() - - # This will be run before each test - #def setUp(self): - # self.seq = range(10) def test_01_olistartup(self): """Tests if OLI can be invoked without exceptions @@ -78,7 +65,6 @@ class TestBasicFunctions(unittest.TestCase): OLITestLib.create_maildir('INBOX.OLItest 1') OLITestLib.create_maildir('INBOX.OLItest "1"') code, res = OLITestLib.run_OLI() - #logging.warn("%s %s "% (code, res)) self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(2,0), msg="Expected 2 folders and 0 " From eb16e5a86eeda8c16be6d1d9a3842c0626e59da7 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 12:16:53 +0100 Subject: [PATCH 793/817] test: always remove remote test folders Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/tests/test_01_basic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 0a11be7..ff4f44a 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -41,7 +41,11 @@ def tearDownModule(): #self.assertFalse(element in self.seq) class TestBasicFunctions(unittest.TestCase): + def setUp(self): + OLITestLib.delete_remote_testfolders() + def tearDown(self): + OLITestLib.delete_remote_testfolders() def test_01_olistartup(self): """Tests if OLI can be invoked without exceptions @@ -49,7 +53,6 @@ class TestBasicFunctions(unittest.TestCase): Cleans existing remote tet folders. Then syncs all "OLItest* (specified in the default config) to our local Maildir. The result should be 0 folders and 0 mails.""" - OLITestLib.delete_remote_testfolders() code, res = OLITestLib.run_OLI() self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') From 8be040d3a01a44de3754dc0106c9337ae2725674 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 12:17:45 +0100 Subject: [PATCH 794/817] fix typo Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/tests/test_01_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index ff4f44a..63a1bc9 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -50,7 +50,7 @@ class TestBasicFunctions(unittest.TestCase): def test_01_olistartup(self): """Tests if OLI can be invoked without exceptions - Cleans existing remote tet folders. Then syncs all "OLItest* + Cleans existing remote test folders. Then syncs all "OLItest* (specified in the default config) to our local Maildir. The result should be 0 folders and 0 mails.""" code, res = OLITestLib.run_OLI() From 4a5f32febd6c005812c2560a601b1a3d5471ee6e Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 14:26:46 +0100 Subject: [PATCH 795/817] maildir deletion must not fail Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/tests/test_01_basic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index 63a1bc9..a1147c0 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -140,6 +140,7 @@ class TestBasicFunctions(unittest.TestCase): self.assertEqual(res, "") OLITestLib.delete_maildir('INBOX.OLItest') code, res = OLITestLib.run_OLI() + self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0 " "mails, but sync led to {} folders and {} mails".format( From ecc2af9f99523cdba137ac059056de8f9ed16c5e Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 17:43:55 +0100 Subject: [PATCH 796/817] test: handle literal string Tuple are used for literal strings (see RFC3501 Section 4.3). Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/OLItest/TestRunner.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 33ba6fa..e7fbaa7 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -148,12 +148,16 @@ class OLITestLib(): assert res_t == 'OK' dirs = [] for d in data: - m = re.search(br''' - (" # starting quote - ([^"]|\\")* # a non-quote or a backslashded quote - ")$ # ending quote - ''', d, flags=re.VERBOSE) - folder = bytearray(m.group(1)) + if isinstance(d, tuple): + # literal (unquoted) + folder = b'"%s"' % d[1].replace('"', '\\"') + else: + m = re.search(br''' + (" # starting quote + ([^"]|\\")* # a non-quote or a backslashded quote + ")$ # ending quote + ''', d, flags=re.VERBOSE) + folder = bytearray(m.group(1)) #folder = folder.replace(br'\"', b'"') # remove quoting dirs.append(folder) # 2) filter out those not starting with INBOX.OLItest and del... From ff069e1d8691737fa7c1e1a52e326ceecdeb9c9e Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 17:48:43 +0100 Subject: [PATCH 797/817] test: workaround for imaplib behaviour an empty string appears after each literal string. Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/OLItest/TestRunner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index e7fbaa7..320ddcf 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -148,6 +148,8 @@ class OLITestLib(): assert res_t == 'OK' dirs = [] for d in data: + if d == '': + continue if isinstance(d, tuple): # literal (unquoted) folder = b'"%s"' % d[1].replace('"', '\\"') From b7789268b7c904a50c804b53b2c57996c797c7ba Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Mon, 23 Mar 2015 18:20:46 +0100 Subject: [PATCH 798/817] test: handle imap.gmx.net server: workarounds Signed-off-by: Pierre-Louis Bonicoli Signed-off-by: Nicolas Sebrecht --- test/OLItest/TestRunner.py | 10 +++++++--- test/tests/test_01_basic.py | 31 ++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py index 320ddcf..e5fc030 100644 --- a/test/OLItest/TestRunner.py +++ b/test/OLItest/TestRunner.py @@ -155,11 +155,15 @@ class OLITestLib(): folder = b'"%s"' % d[1].replace('"', '\\"') else: m = re.search(br''' - (" # starting quote + [ ] # space + (?P + (?P"?) # starting quote ([^"]|\\")* # a non-quote or a backslashded quote - ")$ # ending quote + (?P=quote))$ # ending quote ''', d, flags=re.VERBOSE) - folder = bytearray(m.group(1)) + folder = bytearray(m.group('dir')) + if not m.group('quote'): + folder = '"%s"' % folder #folder = folder.replace(br'\"', b'"') # remove quoting dirs.append(folder) # 2) filter out those not starting with INBOX.OLItest and del... diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py index a1147c0..629131c 100644 --- a/test/tests/test_01_basic.py +++ b/test/tests/test_01_basic.py @@ -61,20 +61,33 @@ class TestBasicFunctions(unittest.TestCase): boxes, mails)) def test_02_createdir(self): - """Create local OLItest 1 & OLItest "1" maildir, sync - - Folder names with quotes used to fail and have been fixed, so - one is included here as a small challenge.""" + """Create local 'OLItest 1', sync""" + OLITestLib.delete_maildir('') #Delete all local maildir folders OLITestLib.create_maildir('INBOX.OLItest 1') - OLITestLib.create_maildir('INBOX.OLItest "1"') code, res = OLITestLib.run_OLI() self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') - self.assertTrue((boxes, mails)==(2,0), msg="Expected 2 folders and 0 " + self.assertTrue((boxes, mails)==(1,0), msg="Expected 1 folders and 0 " "mails, but sync led to {0} folders and {1} mails".format( boxes, mails)) - def test_03_nametransmismatch(self): + def test_03_createdir_quote(self): + """Create local 'OLItest "1"' maildir, sync + + Folder names with quotes used to fail and have been fixed, so + one is included here as a small challenge.""" + OLITestLib.delete_maildir('') #Delete all local maildir folders + OLITestLib.create_maildir('INBOX.OLItest "1"') + code, res = OLITestLib.run_OLI() + if 'unallowed folder' in res: + raise unittest.SkipTest("remote server doesn't handle quote") + self.assertEqual(res, "") + boxes, mails = OLITestLib.count_maildir_mails('') + self.assertTrue((boxes, mails)==(1,0), msg="Expected 1 folders and 0 " + "mails, but sync led to {0} folders and {1} mails".format( + boxes, mails)) + + def test_04_nametransmismatch(self): """Create mismatching remote and local nametrans rules This should raise an error.""" @@ -95,7 +108,7 @@ class TestBasicFunctions(unittest.TestCase): OLITestLib.write_config_file() - def test_04_createmail(self): + def test_05_createmail(self): """Create mail in OLItest 1, sync, wipe folder sync Currently, this will mean the folder will be recreated @@ -118,7 +131,7 @@ class TestBasicFunctions(unittest.TestCase): "assigned the IMAP's UID number, but {0} messages had no valid ID "\ .format(len([None for x in uids if x==None]))) - def test_05_createfolders(self): + def test_06_createfolders(self): """Test if createfolders works as expected Create a local Maildir, then sync with remote "createfolders" From 0c17350e4f72282a7a8ac3180de713b3b322dcea Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 2 Apr 2015 14:41:47 +0200 Subject: [PATCH 799/817] decouple utime_from_header from rtime We were using rtime for two different purposes: - to store remote internal date - to use in the utime_from_header option Let's decouple the utime_from_header logic from rtime, now rtime means remote internal date. Signed-off-by: Abdo Roig-Maranges Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 6 ------ offlineimap/folder/Maildir.py | 7 +++++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 16b5819..5d81af4 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -692,9 +692,6 @@ class BaseFolder(object): message = None flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - if dstfolder.utime_from_header: - content = self.getmessage(uid) - rtime = emailutil.get_message_date(content, 'Date') # If any of the destinations actually stores the message body, # load it up. @@ -766,9 +763,6 @@ class BaseFolder(object): # dst has message with that UID already, only update status flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - if dstfolder.utime_from_header: - content = self.getmessage(uid) - rtime = emailutil.get_message_date(content, 'Date') statusfolder.savemessage(uid, None, flags, rtime) continue diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 79c34a7..f1b8e8e 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -324,8 +324,11 @@ class MaildirFolder(BaseFolder): tmpdir = os.path.join(self.getfullname(), 'tmp') messagename = self.new_message_filename(uid, flags) tmpname = self.save_to_tmp_file(messagename, content) - if rtime != None: - os.utime(os.path.join(self.getfullname(), tmpname), (rtime, rtime)) + + if self.utime_from_header: + date = emailutil.get_message_date(content, 'Date') + if date != None: + os.utime(os.path.join(self.getfullname(), tmpname), (date, date)) self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags From 03d963913c3281301bd96304cfc6950c5f9c23f9 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 3 Apr 2015 19:04:55 +0200 Subject: [PATCH 800/817] Dmitrijs has left the organization Signed-off-by: Nicolas Sebrecht --- MAINTAINERS.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/MAINTAINERS.rst b/MAINTAINERS.rst index 4ff0cda..be86144 100644 --- a/MAINTAINERS.rst +++ b/MAINTAINERS.rst @@ -3,10 +3,6 @@ Official maintainers ==================== -Dmitrijs Ledkovs - email: xnox at debian.org - github: xnox - Eygene Ryabinkin email: rea at freebsd.org github: konvpalto From eb90932d4a38b1724aeb1dcbdc0de2ef775f694a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 3 Apr 2015 23:59:35 +0200 Subject: [PATCH 801/817] MAINTAINERS: move documentation to the website Signed-off-by: Nicolas Sebrecht --- MAINTAINERS.rst | 60 ------------------------------------------------- 1 file changed, 60 deletions(-) diff --git a/MAINTAINERS.rst b/MAINTAINERS.rst index be86144..2cff161 100644 --- a/MAINTAINERS.rst +++ b/MAINTAINERS.rst @@ -27,63 +27,3 @@ Sebastian Spaeth Nicolas Sebrecht email: nicolas.s-dev at laposte.net - -How to maintain the source code -=============================== - -Rolling out a new release -------------------------- - -1. Update the Changelogs. -2. Update the version in ``offlineimap/__init__.py``. -3. Commit the changes with the version in the commit message. -4. Tag the release. -5. Wait the next day before pushing out the release. -6. Make an announce to the mailing list. - - -Tagging stable releases or release candidates -''''''''''''''''''''''''''''''''''''''''''''' - -It is done via Git's ``tag`` command, but you **must** do ``git tag -a`` -to create annotated tag. - -Release tags are named ``vX.Y.Z`` and release candidate tags are named -``vX.Y.Z-rcN``. - - -How to become a core team maintainer -==================================== - -Express your desire to one of the current official maintainers. This is all you -have to do. - -We don't require years of contributions. Of course, we will pay attention a bit -to your past interest to the project and your skills. But nothing is mandatory -and it's fine to still be learning while being an official maintainer. Nobody is -all-knowing, even with years of experience. So, if one current maintainer -agrees, he will grant you the write access to the official repository. - -Contrary to what most people might think, becoming an official maintainer is not -something hard. You'll first be asked to merge contributions and make one or two -official releases. We will review your first steps and nothing more. - -Be aware that there is no more leader maintainer, neither is a leader in the -team. Once you become a team member and did some maintenance for the project, -you're free to take all the strong decisions. - -Since we are a team, we always try to discuss with others before taking any -strong decision, including the users. This prevents from breaking things inside -the whole community and the proximity we have with our users. Even most discreet -of them will raise one day with good hints, review, feedback or opinion. This is -how we are running in OfflineIMAP. And yes, we are really proud about the solid -relationship we have with the users. - -Also, we are all benevolent volunteers. We are in a very good position to know -that we are not always free to contribute as much as we'd like to. Being away -for some (long) time is not a problem. At the time of this writing, each -maintainer took some vacations from the project at some point in time. You'll -contribute as much as your free time permits. It's fine! We are a team, so we -help each other. - -Welcome! ,-) From c094304f937e13e80ce464ff7819affd39d298cc Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Fri, 3 Apr 2015 03:29:36 +0200 Subject: [PATCH 802/817] make savemessagelabels honor utime_from_header Previously, syncing labels on a message always resulted in updating the file modification time, even with utime_from_headers=true This patch restores the file mtime to the previous value when utime_from_headers=true, preventing a label synchronization from breaking the promise that the file mtimes coincide with the header date. Signed-off-by: Abdo Roig-Maranges Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/GmailMaildir.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 894792d..a8b607a 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -159,7 +159,7 @@ class GmailMaildirFolder(MaildirFolder): content = self.deletemessageheaders(content, self.labelsheader) content = self.addmessageheader(content, '\n', self.labelsheader, labels_str) - rtime = self.messagelist[uid].get('rtime', None) + mtime = long(os.stat(filepath).st_mtime) # write file with new labels to a unique file in tmp messagename = self.new_message_filename(uid, set()) @@ -174,8 +174,9 @@ class GmailMaildirFolder(MaildirFolder): (tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER), \ None, exc_info()[2] - if rtime != None: - os.utime(filepath, (rtime, rtime)) + # if utime_from_header=true, we don't want to change the mtime. + if self.utime_from_header and mtime: + os.utime(filepath, (mtime, mtime)) # save the new mtime and labels self.messagelist[uid]['mtime'] = long(os.stat(filepath).st_mtime) From 05d6357adce228817a41c733ce4a7f00264ce17f Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 6 Apr 2015 14:17:59 +0200 Subject: [PATCH 803/817] mark utim_from_header as TESTING This is really in for a long time but due to recent changes and because it was undocumented, mark it non-stable. Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap.conf b/offlineimap.conf index 5bc48a8..3f8e39e 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -468,6 +468,7 @@ localfolders = ~/Test # file/message content. # # If enabled, this forbid the -q (quick mode) CLI option to work correctly. +# This option is still "TESTING" feature. # # Default: no. # From 71693b7d8c6cd51219e811eee7bb03af341ae544 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 6 Apr 2015 23:48:48 +0200 Subject: [PATCH 804/817] Maildir: fix imports of emailutil Commit 0c17350e4f72282a7a revisited utime_for_header without taking care of the imports. Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 2 +- offlineimap/folder/Maildir.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 5d81af4..1113ce7 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -19,7 +19,7 @@ import os.path import re from sys import exc_info -from offlineimap import threadutil, emailutil +from offlineimap import threadutil from offlineimap import globals from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index f1b8e8e..d574ce5 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -19,6 +19,7 @@ import socket import time import re import os +import emailutil from sys import exc_info from .Base import BaseFolder from threading import Lock From 8096f6cd5bf59013741a0758fb04fc4508712a5c Mon Sep 17 00:00:00 2001 From: Janna Martl Date: Tue, 7 Apr 2015 01:14:11 -0700 Subject: [PATCH 805/817] maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option 1. When using maxage, local and remote messagelists are supposed to only contain messages from at most maxage days ago. But local and remote used different timezones to calculate what "maxage days ago" means, resulting in removals on one side. Now, we ask the local folder for maxage days' worth of mail, find the lowest UID, and then ask the remote folder for all UID's starting with that lowest one. 2. maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that remote messages have UIDs in the same order as their local counterparts, which could be false, e.g. when messages are copied in quick succession. So, remove support for maxage in the IMAP-IMAP case. 3. Add startdate option for IMAP-IMAP syncs: use messages from the given repository starting at startdate, and all messages from the other repository. In the first sync, the other repository must be empty. 4. Allow maxage to be specified either as number of days to sync (as previously) or as a fixed date. Signed-off-by: Janna Martl Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 18 +++- offlineimap.conf | 46 ++++++--- offlineimap/accounts.py | 160 +++++++++++++++++++++++------ offlineimap/folder/Base.py | 71 +++++++++++++ offlineimap/folder/Gmail.py | 8 +- offlineimap/folder/GmailMaildir.py | 4 +- offlineimap/folder/IMAP.py | 95 +++++++++-------- offlineimap/folder/Maildir.py | 82 +++++++++------ offlineimap/folder/UIDMaps.py | 5 +- 9 files changed, 361 insertions(+), 128 deletions(-) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 858fc0b..618f2ab 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -135,7 +135,8 @@ Ignore any autorefresh setting in the configuration file. Run only quick synchronizations. + Ignore any flag updates on IMAP servers. If a flag on the remote IMAP changes, -and we have the message locally, it will be left untouched in a quick run. +and we have the message locally, it will be left untouched in a quick run. This +option is ignored if maxage is set. -u :: @@ -400,8 +401,19 @@ If you then point your local mutt, or whatever MUA you use to `~/mail/` as root, it should still recognize all folders. -Authors -------- +* Edge cases with maxage causing too many messages to be synced. ++ +All messages from at most maxage days ago (+/- a few hours, depending on +timezones) are synced, but there are cases in which older messages can also be +synced. This happens when a message's UID is significantly higher than those of +other messages with similar dates, e.g. when messages are added to the local +folder behind offlineimap's back, causing them to get assigned a new UID, or +when offlineimap first syncs a pre-existing Maildir. In the latter case, it +could appear as if a noticeable and random subset of old messages are synced. + + +Main authors +------------ John Goerzen, Sebastian Spaetz, Eygene Ryabinkin, Nicolas Sebrecht. diff --git a/offlineimap.conf b/offlineimap.conf index 3f8e39e..bc43eaa 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -260,6 +260,8 @@ remoterepository = RemoteExample # This option stands in the [Account Test] section. # # OfflineImap can replace a number of full updates by quick synchronizations. +# This option is ignored if maxage or startdate are used. +# # It only synchronizes a folder if # # 1) a Maildir folder has changed @@ -327,21 +329,26 @@ remoterepository = RemoteExample # This option stands in the [Account Test] section. # -# When you are starting to sync an already existing account you can tell -# OfflineIMAP to sync messages from only the last x days. When you do this, -# messages older than x days will be completely ignored. This can be useful for -# importing existing accounts when you do not want to download large amounts of -# archive email. +# maxage enables you to sync only recent messages. There are two ways to specify +# what "recent" means: if maxage is given as an integer, then only messages from +# the last maxage days will be synced. If maxage is given as a date, then only +# messages later than that date will be synced. # -# Messages older than maxage days will not be synced, their flags will not be -# changed, they will not be deleted, etc. For OfflineIMAP it will be like these -# messages do not exist. This will perform an IMAP search in the case of IMAP -# or Gmail and therefore requires that the server support server side searching. -# This will calculate the earliest day that would be included in the search and -# include all messages from that day until today. The maxage option expects an -# integer (for the number of days). +# Messages older than the cutoff will not be synced, their flags will not be +# changed, they will not be deleted, etc. For OfflineIMAP it will be like these +# messages do not exist. This will perform an IMAP search in the case of IMAP or +# Gmail and therefore requires that the server support server side searching. +# +# Known edge cases are described in offlineimap(1). +# +# maxage is allowed only when the local folder is of type Maildir. It can't be +# used with startdate. +# +# The maxage option expects an integer (for the number of days) or a date of the +# form yyyy-mm-dd. # #maxage = 3 +#maxage = 2015-04-01 # This option stands in the [Account Test] section. @@ -446,6 +453,21 @@ localfolders = ~/Test #sep = "." +# This option stands in the [Repository LocalExample] section. +# +# startdate syncs mails starting from a given date. It applies the date +# restriction to LocalExample only. The remote repository MUST be empty +# at the first sync where this option is used. +# +# Unlike maxage, this is supported for IMAP-IMAP sync. +# +# startdate can't be used with maxage. +# +# The startdate option expects a date in the format yyyy-mm-dd. +# +#startdate = 2015-04-01 + + # This option stands in the [Repository LocalExample] section. # # Some users may not want the atime (last access time) of folders to be diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index cac4d88..872b835 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -17,10 +17,11 @@ from subprocess import Popen, PIPE from threading import Event import os +import time from sys import exc_info import traceback -from offlineimap import mbnames, CustomConfig, OfflineImapError +from offlineimap import mbnames, CustomConfig, OfflineImapError, imaplibutil from offlineimap import globals from offlineimap.repository import Repository from offlineimap.ui import getglobalui @@ -402,6 +403,96 @@ def syncfolder(account, remotefolder, quick): Filtered folders on the remote side will not invoke this function.""" + def check_uid_validity(localfolder, remotefolder, statusfolder): + # If either the local or the status folder has messages and + # there is a UID validity problem, warn and abort. If there are + # no messages, UW IMAPd loses UIDVALIDITY. But we don't really + # need it if both local folders are empty. So, in that case, + # just save it off. + if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount() > 0: + if not localfolder.check_uidvalidity(): + ui.validityproblem(localfolder) + localfolder.repository.restore_atime() + return + if not remotefolder.check_uidvalidity(): + ui.validityproblem(remotefolder) + localrepos.restore_atime() + return + else: + # Both folders empty, just save new UIDVALIDITY + localfolder.save_uidvalidity() + remotefolder.save_uidvalidity() + + def save_min_uid(folder, min_uid): + uidfile = folder.get_min_uid_file() + fd = open(uidfile, 'wt') + fd.write(str(min_uid) + "\n") + fd.close() + + def cachemessagelists_upto_date(localfolder, remotefolder, date): + """ Returns messages with uid > min(uids of messages newer than date).""" + + localfolder.cachemessagelist(min_date=date) + check_uid_validity(localfolder, remotefolder, statusfolder) + # local messagelist had date restriction applied already. Restrict + # sync to messages with UIDs >= min_uid from this list. + # + # local messagelist might contain new messages (with uid's < 0). + positive_uids = filter( + lambda uid: uid > 0, localfolder.getmessageuidlist()) + if len(positive_uids) > 0: + remotefolder.cachemessagelist(min_uid=min(positive_uids)) + else: + # No messages with UID > 0 in range in localfolder. + # date restriction was applied with respect to local dates but + # remote folder timezone might be different from local, so be + # safe and make sure the range isn't bigger than in local. + remotefolder.cachemessagelist( + min_date=time.gmtime(time.mktime(date) + 24*60*60)) + + def cachemessagelists_startdate(new, partial, date): + """ Retrieve messagelists when startdate has been set for + the folder 'partial'. + + Idea: suppose you want to clone the messages after date in one + account (partial) to a new one (new). If new is empty, then copy + messages in partial newer than date to new, and keep track of the + min uid. On subsequent syncs, sync all the messages in new against + those after that min uid in partial. This is a partial replacement + for maxage in the IMAP-IMAP sync case, where maxage doesn't work: + the UIDs of the messages in localfolder might not be in the same + order as those of corresponding messages in remotefolder, so if L in + local corresponds to R in remote, the ranges [L, ...] and [R, ...] + might not correspond. But, if we're cloning a folder into a new one, + [min_uid, ...] does correspond to [1, ...]. + + This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead. + """ + + new.cachemessagelist() + min_uid = partial.retrieve_min_uid() + if min_uid == None: # min_uid file didn't exist + if len(new.getmessageuidlist()) > 0: + raise OfflineImapError("To use startdate on Repository %s, " + "Repository %s must be empty"% + (partial.repository.name, new.repository.name), + OfflineImapError.ERROR.MESSAGE) + else: + partial.cachemessagelist(min_date=date) + # messagelist.keys() instead of getuidmessagelist() because in + # the UID mapped case we want the actual local UIDs, not their + # remote counterparts + positive_uids = filter( + lambda uid: uid > 0, partial.messagelist.keys()) + if len(positive_uids) > 0: + min_uid = min(positive_uids) + else: + min_uid = 1 + save_min_uid(partial, min_uid) + else: + partial.cachemessagelist(min_uid=min_uid) + + remoterepos = account.remoterepos localrepos = account.localrepos statusrepos = account.statusrepos @@ -429,43 +520,46 @@ def syncfolder(account, remotefolder, quick): statusfolder.cachemessagelist() - if quick: - if (not localfolder.quickchanged(statusfolder) and - not remotefolder.quickchanged(statusfolder)): - ui.skippingfolder(remotefolder) - localrepos.restore_atime() - return # Load local folder. ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) - ui.loadmessagelist(localrepos, localfolder) - localfolder.cachemessagelist() - ui.messagelistloaded(localrepos, localfolder, localfolder.getmessagecount()) - # If either the local or the status folder has messages and - # there is a UID validity problem, warn and abort. If there are - # no messages, UW IMAPd loses UIDVALIDITY. But we don't really - # need it if both local folders are empty. So, in that case, - # just save it off. - if localfolder.getmessagecount() or statusfolder.getmessagecount(): - if not localfolder.check_uidvalidity(): - ui.validityproblem(localfolder) - localrepos.restore_atime() - return - if not remotefolder.check_uidvalidity(): - ui.validityproblem(remotefolder) - localrepos.restore_atime() - return + # Retrieve messagelists, taking into account age-restriction + # options + maxage = localfolder.getmaxage() + localstart = localfolder.getstartdate() + remotestart = remotefolder.getstartdate() + if (maxage != None) + (localstart != None) + (remotestart != None) > 1: + raise OfflineImapError("You can set at most one of the " + "following: maxage, startdate (for the local folder), " + "startdate (for the remote folder)", + OfflineImapError.ERROR.REPO), None, exc_info()[2] + if (maxage != None or localstart or remotestart) and quick: + # IMAP quickchanged isn't compatible with options that + # involve restricting the messagelist, since the "quick" + # check can only retrieve a full list of UIDs in the folder. + ui.warn("Quick syncs (-q) not supported in conjunction " + "with maxage or startdate; ignoring -q.") + if maxage != None: + cachemessagelists_upto_date(localfolder, remotefolder, maxage) + elif localstart != None: + cachemessagelists_startdate(remotefolder, localfolder, + localstart) + check_uid_validity(localfolder, remotefolder, statusfolder) + elif remotestart != None: + cachemessagelists_startdate(localfolder, remotefolder, + remotestart) + check_uid_validity(localfolder, remotefolder, statusfolder) else: - # Both folders empty, just save new UIDVALIDITY - localfolder.save_uidvalidity() - remotefolder.save_uidvalidity() - - # Load remote folder. - ui.loadmessagelist(remoterepos, remotefolder) - remotefolder.cachemessagelist() - ui.messagelistloaded(remoterepos, remotefolder, - remotefolder.getmessagecount()) + localfolder.cachemessagelist() + if quick: + if (not localfolder.quickchanged(statusfolder) and + not remotefolder.quickchanged(statusfolder)): + ui.skippingfolder(remotefolder) + localrepos.restore_atime() + return + check_uid_validity(localfolder, remotefolder, statusfolder) + remotefolder.cachemessagelist() # Synchronize remote changes. if not localrepos.getconfboolean('readonly', False): diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 1113ce7..7957b83 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -17,6 +17,7 @@ import os.path import re +import time from sys import exc_info from offlineimap import threadutil @@ -298,6 +299,76 @@ class BaseFolder(object): raise NotImplementedError + def getmaxage(self): + """ maxage is allowed to be either an integer or a date of the + form YYYY-mm-dd. This returns a time_struct. """ + + maxagestr = self.config.getdefault("Account %s"% + self.accountname, "maxage", None) + if maxagestr == None: + return None + # is it a number? + try: + maxage = int(maxagestr) + if maxage < 1: + raise OfflineImapError("invalid maxage value %d"% maxage, + OfflineImapError.ERROR.MESSAGE) + return time.gmtime(time.time() - 60*60*24*maxage) + except ValueError: + pass # maybe it was a date + # is it a date string? + try: + date = time.strptime(maxagestr, "%Y-%m-%d") + if date[0] < 1900: + raise OfflineImapError("maxage led to year %d. " + "Abort syncing."% date[0], + OfflineImapError.ERROR.MESSAGE) + return date + except ValueError: + raise OfflineImapError("invalid maxage value %s"% maxagestr, + OfflineImapError.ERROR.MESSAGE) + + def getmaxsize(self): + return self.config.getdefaultint("Account %s"% + self.accountname, "maxsize", None) + + def getstartdate(self): + """ Retrieve the value of the configuration option startdate """ + datestr = self.config.getdefault("Repository " + self.repository.name, + 'startdate', None) + try: + if not datestr: + return None + date = time.strptime(datestr, "%Y-%m-%d") + if date[0] < 1900: + raise OfflineImapError("startdate led to year %d. " + "Abort syncing."% date[0], + OfflineImapError.ERROR.MESSAGE) + return date + except ValueError: + raise OfflineImapError("invalid startdate value %s", + OfflineImapError.ERROR.MESSAGE) + + def get_min_uid_file(self): + startuiddir = os.path.join(self.config.getmetadatadir(), + 'Repository-' + self.repository.name, 'StartUID') + if not os.path.exists(startuiddir): + os.mkdir(startuiddir, 0o700) + return os.path.join(startuiddir, self.getfolderbasename()) + + def retrieve_min_uid(self): + uidfile = self.get_min_uid_file() + if not os.path.exists(uidfile): + return None + try: + fd = open(uidfile, 'rt') + min_uid = long(fd.readline().strip()) + fd.close() + return min_uid + except: + raise IOError("Can't read %s"% uidfile) + + def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 1afbe47..3d83b91 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -121,16 +121,18 @@ class GmailFolder(IMAPFolder): # TODO: merge this code with the parent's cachemessagelist: # TODO: they have too much common logics. - def cachemessagelist(self): + def cachemessagelist(self, min_date=None, min_uid=None): if not self.synclabels: - return super(GmailFolder, self).cachemessagelist() + return super(GmailFolder, self).cachemessagelist( + min_date=min_date, min_uid=min_uid) self.messagelist = {} self.ui.collectingdata(None, self) imapobj = self.imapserver.acquireconnection() try: - msgsToFetch = self._msgs_to_fetch(imapobj) + msgsToFetch = self._msgs_to_fetch( + imapobj, min_date=min_date, min_uid=min_uid) if not msgsToFetch: return # No messages to sync diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index a8b607a..05f7d79 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -64,9 +64,9 @@ class GmailMaildirFolder(MaildirFolder): 'filename': '/no-dir/no-such-file/', 'mtime': 0} - def cachemessagelist(self): + def cachemessagelist(self, min_date=None, min_uid=None): if self.ismessagelistempty(): - self.messagelist = self._scanfolder() + self.messagelist = self._scanfolder(min_date=min_date, min_uid=min_uid) # Get mtimes if self.synclabels: diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 4b470a2..253ac97 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -18,6 +18,7 @@ import random import binascii import re +import os import time from sys import exc_info @@ -79,6 +80,12 @@ class IMAPFolder(BaseFolder): def waitforthread(self): self.imapserver.connectionwait() + def getmaxage(self): + if self.config.getdefault("Account %s"% + self.accountname, "maxage", None): + raise OfflineImapError("maxage is not supported on IMAP-IMAP sync", + OfflineImapError.ERROR.REPO), None, exc_info()[2] + # Interface from BaseFolder def getcopyinstancelimit(self): return 'MSGCOPY_' + self.repository.getname() @@ -143,8 +150,7 @@ class IMAPFolder(BaseFolder): return True return False - - def _msgs_to_fetch(self, imapobj): + def _msgs_to_fetch(self, imapobj, min_date=None, min_uid=None): """Determines sequence numbers of messages to be fetched. Message sequence numbers (MSNs) are more easily compacted @@ -152,57 +158,55 @@ class IMAPFolder(BaseFolder): Arguments: - imapobj: instance of IMAPlib + - min_date (optional): a time_struct; only fetch messages newer than this + - min_uid (optional): only fetch messages with UID >= min_uid + + This function should be called with at MOST one of min_date OR + min_uid set but not BOTH. Returns: range(s) for messages or None if no messages are to be fetched.""" - res_type, imapdata = imapobj.select(self.getfullname(), True, True) - if imapdata == [None] or imapdata[0] == '0': - # Empty folder, no need to populate message list - return None + def search(search_conditions): + """Actually request the server with the specified conditions. - # By default examine all messages in this folder - msgsToFetch = '1:*' - - maxage = self.config.getdefaultint( - "Account %s"% self.accountname, "maxage", -1) - maxsize = self.config.getdefaultint( - "Account %s"% self.accountname, "maxsize", -1) - - # Build search condition - if (maxage != -1) | (maxsize != -1): - search_cond = "("; - - if(maxage != -1): - #find out what the oldest message is that we should look at - oldest_struct = time.gmtime(time.time() - (60*60*24*maxage)) - if oldest_struct[0] < 1900: - raise OfflineImapError("maxage setting led to year %d. " - "Abort syncing."% oldest_struct[0], - OfflineImapError.ERROR.REPO) - search_cond += "SINCE %02d-%s-%d"% ( - oldest_struct[2], - MonthNames[oldest_struct[1]], - oldest_struct[0]) - - if(maxsize != -1): - if(maxage != -1): # There are two conditions, add space - search_cond += " " - search_cond += "SMALLER %d"% maxsize - - search_cond += ")" - - res_type, res_data = imapobj.search(None, search_cond) + Returns: range(s) for messages or None if no messages + are to be fetched.""" + res_type, res_data = imapobj.search(None, search_conditions) if res_type != 'OK': raise OfflineImapError("SEARCH in folder [%s]%s failed. " "Search string was '%s'. Server responded '[%s] %s'"% ( self.getrepository(), self, search_cond, res_type, res_data), OfflineImapError.ERROR.FOLDER) + return res_data[0].split() - # Resulting MSN are separated by space, coalesce into ranges - msgsToFetch = imaputil.uid_sequence(res_data[0].split()) + res_type, imapdata = imapobj.select(self.getfullname(), True, True) + if imapdata == [None] or imapdata[0] == '0': + # Empty folder, no need to populate message list. + return None - return msgsToFetch + conditions = [] + # 1. min_uid condition. + if min_uid != None: + conditions.append("UID %d:*"% min_uid) + # 2. date condition. + elif min_date != None: + # Find out what the oldest message is that we should look at. + conditions.append("SINCE %02d-%s-%d"% ( + min_date[2], MonthNames[min_date[1]], min_date[0])) + # 3. maxsize condition. + maxsize = self.getmaxsize() + if maxsize != None: + conditions.append("SMALLER %d"% maxsize) + + if len(conditions) >= 1: + # Build SEARCH command. + search_cond = "(%s)"% ' '.join(conditions) + search_result = search(search_cond) + return imaputil.uid_sequence(search_result) + + # By default consider all messages in this folder. + return '1:*' # Interface from BaseFolder def msglist_item_initializer(self, uid): @@ -210,19 +214,21 @@ class IMAPFolder(BaseFolder): # Interface from BaseFolder - def cachemessagelist(self): + def cachemessagelist(self, min_date=None, min_uid=None): + self.ui.loadmessagelist(self.repository, self) self.messagelist = {} imapobj = self.imapserver.acquireconnection() try: - msgsToFetch = self._msgs_to_fetch(imapobj) + msgsToFetch = self._msgs_to_fetch( + imapobj, min_date=min_date, min_uid=min_uid) if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. res_type, response = imapobj.fetch("'%s'"% - msgsToFetch, '(FLAGS UID)') + msgsToFetch, '(FLAGS UID INTERNALDATE)') if res_type != 'OK': raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " "Server responded '[%s] %s'"% (self.getrepository(), self, @@ -247,6 +253,7 @@ class IMAPFolder(BaseFolder): flags = imaputil.flagsimap2maildir(options['FLAGS']) rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) def dropmessagelistcache(self): self.messagelist = {} diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index d574ce5..fb39335 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -92,25 +92,17 @@ class MaildirFolder(BaseFolder): token.""" return 42 - # Checks to see if the given message is within the maximum age according - # to the maildir name which should begin with a timestamp - def _iswithinmaxage(self, messagename, maxage): - # In order to have the same behaviour as SINCE in an IMAP search - # we must convert this to the oldest time and then strip off hrs/mins - # from that day. - oldest_time_utc = time.time() - (60*60*24*maxage) - oldest_time_struct = time.gmtime(oldest_time_utc) - oldest_time_today_seconds = ((oldest_time_struct[3] * 3600) \ - + (oldest_time_struct[4] * 60) \ - + oldest_time_struct[5]) - oldest_time_utc -= oldest_time_today_seconds + def _iswithintime(self, messagename, date): + """Check to see if the given message is newer than date (a + time_struct) according to the maildir name which should begin + with a timestamp.""" timestampmatch = re_timestampmatch.search(messagename) if not timestampmatch: return True timestampstr = timestampmatch.group() timestamplong = long(timestampstr) - if(timestamplong < oldest_time_utc): + if(timestamplong < time.mktime(date)): return False else: return True @@ -151,18 +143,21 @@ class MaildirFolder(BaseFolder): flags = set((c for c in flagmatch.group(1) if not c.islower())) return prefix, uid, fmd5, flags - def _scanfolder(self): + def _scanfolder(self, min_date=None, min_uid=None): """Cache the message list from a Maildir. + If min_date is set, this finds the min UID of all messages newer than + min_date and uses it as the real cutoff for considering messages. + This handles the edge cases where the date is much earlier than messages + with similar UID's (e.g. the UID was reassigned much later). + Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F (flagged). :returns: dict that can be used as self.messagelist. """ - maxage = self.config.getdefaultint("Account " + self.accountname, - "maxage", None) - maxsize = self.config.getdefaultint("Account " + self.accountname, - "maxsize", None) + maxsize = self.getmaxsize() + retval = {} files = [] nouidcounter = -1 # Messages without UIDs get negative UIDs. @@ -171,12 +166,11 @@ class MaildirFolder(BaseFolder): files.extend((dirannex, filename) for filename in os.listdir(fulldirname)) + date_excludees = {} for dirannex, filename in files: # We store just dirannex and filename, ie 'cur/123...' filepath = os.path.join(dirannex, filename) - # Check maxage/maxsize if this message should be considered. - if maxage and not self._iswithinmaxage(filename, maxage): - continue + # Check maxsize if this message should be considered. if maxsize and (os.path.getsize(os.path.join( self.getfullname(), filepath)) > maxsize): continue @@ -193,16 +187,43 @@ class MaildirFolder(BaseFolder): nouidcounter -= 1 else: uid = long(uidmatch.group(1)) - # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S - retval[uid] = self.msglist_item_initializer(uid) - retval[uid]['flags'] = flags - retval[uid]['filename'] = filepath + if min_uid != None and uid > 0 and uid < min_uid: + continue + if min_date != None and not self._iswithintime(filename, min_date): + # Keep track of messages outside of the time limit, because they + # still might have UID > min(UIDs of within-min_date). We hit + # this case for maxage if any message had a known/valid datetime + # and was re-uploaded because the UID in the filename got lost + # (e.g. local copy/move). On next sync, it was assigned a new + # UID from the server and will be included in the SEARCH + # condition. So, we must re-include them later in this method + # in order to avoid inconsistent lists of messages. + date_excludees[uid] = self.msglist_item_initializer(uid) + date_excludees[uid]['flags'] = flags + date_excludees[uid]['filename'] = filepath + else: + # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S + retval[uid] = self.msglist_item_initializer(uid) + retval[uid]['flags'] = flags + retval[uid]['filename'] = filepath + if min_date != None: + # Re-include messages with high enough uid's. + positive_uids = filter(lambda uid: uid > 0, retval) + if positive_uids: + min_uid = min(positive_uids) + for uid in date_excludees.keys(): + if uid > min_uid: + # This message was originally excluded because of + # its date. It is re-included now because we want all + # messages with UID > min_uid. + retval[uid] = date_excludees[uid] return retval # Interface from BaseFolder def quickchanged(self, statusfolder): - """Returns True if the Maildir has changed""" - self.cachemessagelist() + """Returns True if the Maildir has changed + + Assumes cachemessagelist() has already been called """ # Folder has different uids than statusfolder => TRUE. if sorted(self.getmessageuidlist()) != \ sorted(statusfolder.getmessageuidlist()): @@ -219,9 +240,12 @@ class MaildirFolder(BaseFolder): return {'flags': set(), 'filename': '/no-dir/no-such-file/'} # Interface from BaseFolder - def cachemessagelist(self): + def cachemessagelist(self, min_date=None, min_uid=None): if self.ismessagelistempty(): - self.messagelist = self._scanfolder() + self.ui.loadmessagelist(self.repository, self) + self.messagelist = self._scanfolder(min_date=min_date, + min_uid=min_uid) + self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) # Interface from BaseFolder def getmessagelist(self): diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 04a986b..1e54f2b 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -94,9 +94,10 @@ class MappedIMAPFolder(IMAPFolder): OfflineImapError.ERROR.MESSAGE), None, exc_info()[2] # Interface from BaseFolder - def cachemessagelist(self): - self._mb.cachemessagelist() + def cachemessagelist(self, min_date=None, min_uid=None): + self._mb.cachemessagelist(min_date=min_date, min_uid=min_uid) reallist = self._mb.getmessagelist() + self.messagelist = self._mb.messagelist self.maplock.acquire() try: From a9ed2ff98a1661a4b8549df6d8e723ce7a99415c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 7 Apr 2015 11:56:10 +0200 Subject: [PATCH 806/817] folder/Maildir: fix emailutil import Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index fb39335..3f5c071 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -19,7 +19,6 @@ import socket import time import re import os -import emailutil from sys import exc_info from .Base import BaseFolder from threading import Lock @@ -32,7 +31,7 @@ try: # python 2.6 has set() built in except NameError: from sets import Set as set -from offlineimap import OfflineImapError +from offlineimap import OfflineImapError, emailutil # Find the UID in a message filename re_uidmatch = re.compile(',U=(\d+)') From 548fa6e57f39aedeeb5b1c404f4f4e24e1ed3f6b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 7 Apr 2015 12:31:51 +0200 Subject: [PATCH 807/817] v6.5.7-rc4 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 38 ++++++++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 8d3d0b1..317f421 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,44 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} +### OfflineIMAP v6.5.7-rc4 (2015-04-07) + +#### Notes + +Contrary to what the detailed following changes look like, here is a much bigger +release than expected. + +Most important change is about maxage being sightly revisited. The whole +internal logic was found broken. Janna Martl did the hard work of raising the +issues and get them fixed. + +New configuration options are added. + +Maintainer Dmitrijs Ledkovs has left the organization. We wish you well! ,-) +Sebastian Spaeth let us know he will be almost inactive. We wish you well, too! + +#### Features + +- Add configuration option "utime_from_header" (TESTING). +- Add systemd integration files. +- mbnames: add new option "incremental" to write the file once per account. + +#### Fixes + +- maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option. +- Test suites fixed and improved. +- Fix inaccurate UI messages when some messages are internally excluded from the + cached lists. + +#### Changes + +- imaplib2: bump to v2.43. +- More documentations moves to the website. +- Maintainer Dmitrijs has left the organization. +- Remove unnecessary imaplib2 workaround. +- release.sh: script for maintainers improved. + + ### OfflineIMAP v6.5.7-rc3 (2015-03-19) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index b0ef6b2..ee4b8e7 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,7 +2,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.5.7" -__revision__ = "-rc3" +__revision__ = "-rc4" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" From ad12c125a4e50185c599c6eb8277fd3da3753584 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 7 Apr 2015 14:23:20 +0200 Subject: [PATCH 808/817] contrib/release.sh: fix latest.yml update Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 60 +++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index e740cdc..cf029f0 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -271,6 +271,7 @@ rc: $(get_last_rc) EOF } + # # $1: new version # @@ -280,35 +281,50 @@ function update_website () { ask "update API of the website? (require $SPHINXBUILD)" if test $? -eq $Yes then - $SPHINXBUILD --version > /dev/null 2>&1 && { - cd website || echo "ERROR: cannot go to website" - git diff --quiet 2>/dev/null && git diff --quiet --cached 2>/dev/null || { - echo "There is WIP in the website repository, stashing" - echo "git stash create 'WIP during offlineimap API import'" - git stash create 'WIP during offlineimap API import' - } - - update_website_releases_info - cd "../$DOCSDIR" - make websitedoc && { - cd ../website && { - branch_name="import-$1" - git checkout -b "$branch_name" - git add '_doc/versions' - git commit -a -s -m"update for offlineimap $1" - git checkout master - echo "website: branch '$branch_name' ready for a merge in master!" - } - } - } || { + # Check sphinx is available. + $SPHINXBUILD --version > /dev/null 2>&1 + if test ! $? -eq 0 + then echo "Oops! you don't have $SPHINXBUILD installed?" echo "Cannot update the webite documentation..." echo "You should install it and run:" echo " $ cd docs" echo " $ make websitedoc" echo "Then, commit and push changes of the website." + ask 'continue' + return + fi + + # Check website sources are available. + cd website + if test ! $? -eq 0 + then + echo "ERROR: cannot go to the website sources" + ask 'continue' + return + fi + # Stash any WIP in the website sources. + git diff --quiet 2>/dev/null && git diff --quiet --cached 2>/dev/null || { + echo "There is WIP in the website repository, stashing" + echo "git stash create 'WIP during offlineimap API import'" + git stash create 'WIP during offlineimap API import' + ask 'continue' } - ask 'continue' + + cd .. # Back to offlineimap.git. + update_website_releases_info + cd "./$DOCSDIR" # Enter the docs directory in offlineimap.git. + # Build the docs! + make websitedoc && { + # Commit changes in a branch. + cd ../website # Enter the website sources. + branch_name="import-$1" + git checkout -b "$branch_name" + git add '_doc/versions' + git commit -a -s -m"update for offlineimap $1" + echo "website: branch '$branch_name' ready for a merge in master!" + } + ask 'website updated locally; continue' fi } From 2c3ef50a3dc6da2c66968c28d64f46e25b3506bd Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 7 Apr 2015 14:25:22 +0200 Subject: [PATCH 809/817] contrib/release.sh: fix debug statement Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/release.sh b/contrib/release.sh index cf029f0..5ff5548 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -226,7 +226,7 @@ function update_changelog () { sed -i -e "/${CHANGELOG_MAGIC}/ r ${TMP_CHANGELOG_EXCERPT}" "$TMP_CHANGELOG" debug 'remove trailing whitespaces' sed -i -r -e 's, +$,,' "$TMP_CHANGELOG" # Remove trailing whitespaces. - debug "copy to $TMP_ANNOUNCE -> $CHANGELOG" + debug "copy to $TMP_CHANGELOG -> $CHANGELOG" cp -f "$TMP_CHANGELOG" "$CHANGELOG" # Check and edit Changelog. From c8fea7ae188bde75fed4e369fe42147eabbb2c53 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 7 Apr 2015 14:35:39 +0200 Subject: [PATCH 810/817] contrib/release.sh (v0.2): better format for announces Signed-off-by: Nicolas Sebrecht --- Changelog.md | 1 - contrib/release.sh | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Changelog.md b/Changelog.md index 317f421..91006b9 100644 --- a/Changelog.md +++ b/Changelog.md @@ -53,7 +53,6 @@ Sebastian Spaeth let us know he will be almost inactive. We wish you well, too! - release.sh: script for maintainers improved. - ### OfflineIMAP v6.5.7-rc3 (2015-03-19) #### Notes diff --git a/contrib/release.sh b/contrib/release.sh index 5ff5548..72e9936 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -16,7 +16,7 @@ # TODO: move configuration out and source it. # TODO: implement rollback. -__VERSION__='v0.1' +__VERSION__='v0.2' SPHINXBUILD=sphinx-build @@ -352,14 +352,10 @@ OfflineIMAP $1 is out. Downloads: http://github.com/OfflineIMAP/offlineimap/archive/${1}.tar.gz http://github.com/OfflineIMAP/offlineimap/archive/${1}.zip - EOF } -# -# $1: previous version -# function announce_footer () { cat < "$TMP_ANNOUNCE" - cat "$TMP_CHANGELOG_EXCERPT" >> "$TMP_ANNOUNCE" + grep -v '^### OfflineIMAP' "$TMP_CHANGELOG_EXCERPT" | \ + grep -v '^#### Notes' >> "$TMP_ANNOUNCE" sed -i -r -e "s,^$ANNOUNCE_MAGIC,," "$TMP_ANNOUNCE" - announce_footer "$2" >> "$TMP_ANNOUNCE" + sed -i -r -e "s,^#### ,# ," "$TMP_ANNOUNCE" + announce_footer >> "$TMP_ANNOUNCE" } @@ -387,6 +385,9 @@ function edit_announce () { +# +# run +# function run () { debug 'in run' fix_pwd From c3f3012cfcfa353b66b81d83e1c1351a1a5a758c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 7 Apr 2015 16:34:35 +0200 Subject: [PATCH 811/817] accounts: remove unused imaplibutil import Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 872b835..515c11d 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -21,7 +21,7 @@ import time from sys import exc_info import traceback -from offlineimap import mbnames, CustomConfig, OfflineImapError, imaplibutil +from offlineimap import mbnames, CustomConfig, OfflineImapError from offlineimap import globals from offlineimap.repository import Repository from offlineimap.ui import getglobalui From 8dd146d8fe9a5e8409a5bec5a983b7e920df8b2a Mon Sep 17 00:00:00 2001 From: Michael Hoy Date: Wed, 8 Apr 2015 14:51:46 -0400 Subject: [PATCH 812/817] folder/IMAP: fix datetuple dst check Signed-off-by: Michael Hoy Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 253ac97..9ed7fec 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -489,7 +489,7 @@ class IMAPFolder(BaseFolder): # tm_isdst coming from email.parsedate is not usable, we still use it # here, mhh. - if datetuple.tm_isdst == '1': + if datetuple.tm_isdst == 1: zone = -time.altzone else: zone = -time.timezone From 5f0915d06ee3597b97c3b972e4eff4534aca222a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 13 Apr 2015 00:05:51 +0200 Subject: [PATCH 813/817] offlineimap.conf: proxy leaks DNS support Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap.conf b/offlineimap.conf index bc43eaa..8f74c68 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -423,7 +423,9 @@ remoterepository = RemoteExample # Available proxy types are SOCKS5, SOCKS4, HTTP. # You also need to install PySocks through pip. # -#proxy = SOCKS5:localhost:9999 +# Currently, this feature leaks DNS support. +# +#proxy = SOCKS5:IP:9999 [Repository LocalExample] From a817400d768a13e168a028e28475dd0c013cb5a8 Mon Sep 17 00:00:00 2001 From: Alex Kapranoff Date: Tue, 5 May 2015 16:34:58 +0300 Subject: [PATCH 814/817] report exceptions via exit code Now it is possible to handle failed syncs in scripts. Signed-off-by: Alex Kapranoff Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 9285b51..14e9304 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -500,6 +500,8 @@ class UIBase(object): #print any exceptions that have occurred over the run if not self.exc_queue.empty(): self.warn("ERROR: Exceptions occurred during the run!") + if exitstatus == 0: + exitstatus = 1 while not self.exc_queue.empty(): msg, exc, exc_traceback = self.exc_queue.get() if msg: From 07af72081a3d86feafe72cd2aef197993e8d0a1a Mon Sep 17 00:00:00 2001 From: Alex Kapranoff Date: Tue, 5 May 2015 20:35:37 +0300 Subject: [PATCH 815/817] report UID validity problem in exit code Since skipping a folder means no new data is downloaded, the UID validity problem is a backup failure. Make it possible to alert or work around it in scripts by signaling with the exit code. Signed-off-by: Alex Kapranoff Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 14e9304..3e698ab 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -65,6 +65,8 @@ class UIBase(object): self.logfile = None self.exc_queue = Queue() # saves all occuring exceptions, so we can output them at the end + self.uidval_problem = False + # at least one folder skipped due to UID validity problem # create logger with 'OfflineImap' app self.logger = logging.getLogger('OfflineImap') self.logger.setLevel(loglevel) @@ -345,6 +347,7 @@ class UIBase(object): self.logger.info("Skipping %s (not changed)" % folder) def validityproblem(self, folder): + self.uidval_problem = True self.logger.warning("UID validity problem for folder %s (repo %s) " "(saved %d; got %d); skipping it. Please see FAQ " "and manual on how to handle this."% \ @@ -516,6 +519,10 @@ class UIBase(object): self.warn('ERROR: %s\n\n%s\n'% (errortitle, errormsg)) elif errormsg: self.warn('%s\n'% errormsg) + if self.uidval_problem: + self.warn('At least one folder skipped due to UID validity problem') + if exitstatus == 0: + exitstatus = 2 sys.exit(exitstatus) def threadExited(self, thread): From 115d141218c3deb7d2fc92ea87732b4420c4a5d2 Mon Sep 17 00:00:00 2001 From: mobamoba Date: Thu, 14 May 2015 08:28:02 -0400 Subject: [PATCH 816/817] Update offlineimap.conf Updated to add reference to how Windows refers to a SHA fingerprint. Submitted-by: mobamoba Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 8f74c68..69ede92 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -612,6 +612,8 @@ remotehost = examplehost # # Multiple fingerprints can be specified, separated by commas. # +# In Windows, Microsoft uses the term "thumbprint" instead of "fingerprint". +# # Fingerprints must be in hexadecimal form without leading '0x': # 40 hex digits like bbfe29cf97acb204591edbafe0aa8c8f914287c9. # From ca1ce256ecb1d56f00c7342b121b6079b536ef0f Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 15 May 2015 15:20:18 +0200 Subject: [PATCH 817/817] v6.5.7 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 43 +++++++++++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 4 ++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index 91006b9..bda0b3d 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,49 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} + +### OfflineIMAP v6.5.7 (2015-05-15) + +#### Notes + +Almost no change since last release candidate. This is a sign that this release +is stable. ,-) + +There was big changes since previous stable and users - especially distribution +maintainers - should really read the intermediate changelogs. + +At the beginning of this year, I've tried to implement Unicode support. As you +know, I was not satisfied with the result. Then, I've published my code analysis +where I talk about doing a lot of refactoring for more proper OOP practices. +What's new is that I've actually done it and stopped this work as soon as I +realized that it means entirely rewriting the software. + +On top of this, I'm not fully satisfied with other current limitations: +- old legacy support; +- migration to Python 3; +- complex multithreading design; +- some restrictions of the GPLv2 license; +- etc. + +That's why I've started a new product. I'll publish it in the coming weeks under +the MIT license. + +#### Features + +- Better documentation for Windows users. +- contrib/release.sh (v0.2): fixes and improvements. + +#### Fixes + +- Report exceptions via exit code. +- Proxy feature leaks DNS support: offlineimap.conf talks about this. +- Email parsing for date coudn't work: fix datetuple dst check. + +#### Changes + +- Little code refactoring. + + ### OfflineIMAP v6.5.7-rc4 (2015-04-07) #### Notes diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index ee4b8e7..8b93c96 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,11 +2,11 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.5.7" -__revision__ = "-rc4" +__revision__ = "" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" -__author_email__= "john@complete.org" +__author_email__= "offlineimap-project@lists.alioth.debian.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" __license__ = "Licensed under the GNU GPL v2 or any later version (with an OpenSSL exception)" __bigcopyright__ = """%(__productname__)s %(__bigversion__)s