diff --git a/.gitignore b/.gitignore index 553b655..f118931 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,14 @@ -# Generated files +# Backups. +.*.swp +.*.swo +*.html +*~ +# websites. +/website/ +/wiki/ +# Generated files. +/docs/dev-doc/ /build/ *.pyc offlineimap.1 -# backups -.*.swp -.*.swo -*~ +offlineimapui.7 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..9bc6f7f --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,123 @@ +.. -*- coding: utf-8 -*- +.. vim: spelllang=en ts=2 expandtab: + +.. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap +.. _Github: https://github.com/OfflineIMAP/offlineimap +.. _repository: git://github.com/OfflineIMAP/offlineimap.git +.. _maintainers: https://github.com/OfflineIMAP/offlineimap/blob/next/MAINTAINERS.rst +.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project +.. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst +.. _Community's website: https://offlineimap.org +.. _APIs in OfflineIMAP: http://offlineimap.org/documentation.html#available-apis +.. _documentation: https://offlineimap.org/documentation.html +.. _Coding Guidelines: http://offlineimap.org/doc/CodingGuidelines.html +.. _Know the status of your patches: http://offlineimap.org/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission + + +================= +HOW TO CONTRIBUTE +================= + +You'll find here the **basics** to contribute to OfflineIMAP_, addressed to +users as well as learning or experienced developers to quickly provide +contributions. + +**For more detailed documentation, see the** `Community's website`_. + +.. contents:: :depth: 3 + + +For the imaptients +================== + +- `Coding Guidelines`_ +- `APIs in OfflineIMAP`_ +- `Know the status of your patches`_ after submission +- All the `documentation`_ + + +Submit issues +============= + +Issues are welcome to both Github_ and the `mailing list`_, at your own +convenience. + + +Community +========= + +All contributors to OfflineIMAP_ are benevolent volunteers. This makes hacking +to OfflineIMAP_ **fun and open**. + +Thanks to Python, almost every developer can quickly become productive. Students +and novices are welcome. Third-parties patches are essential and proved to be a +wonderful source of changes for both fixes and new features. + +OfflineIMAP_ is entirely written in Python, works on IMAP and source code is +tracked with Git. + +*It is expected that most contributors don't have skills to all of these areas.* +That's why the best thing you could do for you, is to ask us about any +difficulty or question raising in your mind. We actually do our best to help new +comers. **We've all started like this.** + +- The official repository_ is maintained by the core team maintainers_. + +- The `mailing list`_ is where all the exciting things happen. + + +Getting started +=============== + +Occasional contributors +----------------------- + +* Clone the official repository_. + +Regular contributors +-------------------- + +* Create an account and login to Github. +* Fork the official repository_. +* Clone your own fork to your local workspace. +* Add a reference to your fork (once):: + + $ git remote add myfork https://github.com//offlineimap.git + +* Regularly fetch the changes applied by the maintainers:: + + $ git fetch origin + $ git checkout master + $ git merge offlineimap/master + $ git checkout next + $ git merge offlineimap/next + + +Making changes (all contributors) +--------------------------------- + +1. Create your own topic branch off of ``next`` (recently updated) via:: + + $ git checkout -b my_topic next + +2. Check for unnecessary whitespaces with ``git diff --check`` before committing. +3. Commit your changes into logical/atomic commits. **Sign-off your work** to + confirm you agree with the `Developer's Certificate of Origin`_. +4. Write a good *commit message* about **WHY** this patch (take samples from + the ``git log``). + + +Learn more +========== + +There is already a lot of documentation. Here's where you might want to look +first: + +- The directory ``offlineimap/docs`` has all kind of additional documentation + (man pages, RFCs). + +- The file ``offlineimap.conf`` allows to know all the supported features. + +- The file ``TODO.rst`` express code changes we'd like and current *Work In + Progress* (WIP). + diff --git a/COPYING b/COPYING index e70efd4..bb94f18 100644 --- a/COPYING +++ b/COPYING @@ -1,3 +1,13 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + GNU GENERAL PUBLIC LICENSE Version 2, June 1991 @@ -55,7 +65,7 @@ patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. - + GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION @@ -110,7 +120,7 @@ above, provided that you also meet all of these conditions: License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) - + These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in @@ -168,7 +178,7 @@ access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. - + 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is @@ -225,7 +235,7 @@ impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - + 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License @@ -278,7 +288,7 @@ PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS - + How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest @@ -338,3 +348,17 @@ proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. + +---------------------------------------------------------------- +In addition, as a special exception, the copyright holders give +permission to link the code of portions of this program with the OpenSSL +library under certain conditions as described in each individual source +file, and distribute linked combinations including the two. + +You must obey the GNU General Public License in all respects for all of +the code used other than OpenSSL. If you modify file(s) with this +exception, you may extend this exception to your version of the file(s), +but you are not obligated to do so. If you do not wish to do so, delete +this exception statement from your version. If you delete this exception +statement from all source files in the program, then also delete it +here. diff --git a/COPYRIGHT b/COPYRIGHT deleted file mode 100644 index 4a11bce..0000000 --- a/COPYRIGHT +++ /dev/null @@ -1,17 +0,0 @@ -offlineimap Mail syncing software -Copyright (C) 2002 - 2009 John Goerzen - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - diff --git a/Changelog.draft.rst b/Changelog.draft.rst deleted file mode 100644 index 2aee8d7..0000000 --- a/Changelog.draft.rst +++ /dev/null @@ -1,36 +0,0 @@ -========= -ChangeLog -========= - -Users should ignore this content: **it is draft**. - -Contributors should add entries here in the following section, on top of the -others. - -`WIP (coming releases)` -======================= - -New Features ------------- - -Changes -------- - -* Give more detailed error when encountering a corrupt UID mapping file. - -Bug Fixes ---------- - - -Pending for the next major release -================================== - -* UIs get shorter and nicer names. (API changing) - - -Stalled -======= - -* Learn Sqlite support. - Stalled: it would need to learn the ability to choose between the current - format and SQL to help testing the long term. diff --git a/Changelog.maint.md b/Changelog.maint.md new file mode 100644 index 0000000..344987c --- /dev/null +++ b/Changelog.maint.md @@ -0,0 +1,24 @@ +--- +layout: page +title: Changelog of the stable branch +--- + +* The following excerpt is only usefull when rendered in the website. +{:toc} + +This is the Changelog of the maintenance branch. + +**NOTE FROM THE MAINTAINER:** + + This branch comes almost as-is. With no URGENT requirements to update this + branch (e.g. big security fix), it is left behind. + If anyone volunteers to maintain it and backport patches, let us know! + + +### OfflineIMAP v6.3.2.1 (2011-03-23) + +#### Bug Fixes + +* Sanity checks for SSL cacertfile configuration. +* Fix regression (UIBase is no more). +* Make profiling mode really enforce single-threading. diff --git a/Changelog.maint.rst b/Changelog.maint.rst deleted file mode 100644 index 89db798..0000000 --- a/Changelog.maint.rst +++ /dev/null @@ -1,48 +0,0 @@ -========= -ChangeLog -========= - -:website: http://offlineimap.org - -This is the Changelog of the maintenance branch. - -**NOTE FROM THE MAINTAINER:** - Contributors should use the `WIP` section in Changelog.draft.rst in order to - add changes they are working on. I will use it to make the new changelog entry - on releases. And because I'm lazy, it will also be used as a draft for the - releases announces. - - -OfflineIMAP v6.3.2.3 (2011-08-10) -================================= - -Changes -------- - -* Output more detailed error on corrupt LocalStatus. -* More detailed error output on corrupt UID mapping files. - -Bug Fixes ---------- - -* Fix typo to force singlethreading in debug mode. - - -OfflineIMAP v6.3.2.2 (2011-04-24) -================================= - -Changes -------- - -* Improve traceback on some crashes. - - -OfflineIMAP v6.3.2.1 (2011-03-23) -================================= - -Bug Fixes ---------- - -* Sanity checks for SSL cacertfile configuration. -* Fix regression (UIBase is no more). -* Make profiling mode really enforce single-threading. diff --git a/Changelog.md b/Changelog.md new file mode 100644 index 0000000..bda0b3d --- /dev/null +++ b/Changelog.md @@ -0,0 +1,1095 @@ +--- +layout: page +title: Changelog of mainline +--- + + + + +* The following excerpt is only usefull when rendered in the website. +{:toc} + + +### OfflineIMAP v6.5.7 (2015-05-15) + +#### Notes + +Almost no change since last release candidate. This is a sign that this release +is stable. ,-) + +There was big changes since previous stable and users - especially distribution +maintainers - should really read the intermediate changelogs. + +At the beginning of this year, I've tried to implement Unicode support. As you +know, I was not satisfied with the result. Then, I've published my code analysis +where I talk about doing a lot of refactoring for more proper OOP practices. +What's new is that I've actually done it and stopped this work as soon as I +realized that it means entirely rewriting the software. + +On top of this, I'm not fully satisfied with other current limitations: +- old legacy support; +- migration to Python 3; +- complex multithreading design; +- some restrictions of the GPLv2 license; +- etc. + +That's why I've started a new product. I'll publish it in the coming weeks under +the MIT license. + +#### Features + +- Better documentation for Windows users. +- contrib/release.sh (v0.2): fixes and improvements. + +#### Fixes + +- Report exceptions via exit code. +- Proxy feature leaks DNS support: offlineimap.conf talks about this. +- Email parsing for date coudn't work: fix datetuple dst check. + +#### Changes + +- Little code refactoring. + + +### OfflineIMAP v6.5.7-rc4 (2015-04-07) + +#### Notes + +Contrary to what the detailed following changes look like, here is a much bigger +release than expected. + +Most important change is about maxage being sightly revisited. The whole +internal logic was found broken. Janna Martl did the hard work of raising the +issues and get them fixed. + +New configuration options are added. + +Maintainer Dmitrijs Ledkovs has left the organization. We wish you well! ,-) +Sebastian Spaeth let us know he will be almost inactive. We wish you well, too! + +#### Features + +- Add configuration option "utime_from_header" (TESTING). +- Add systemd integration files. +- mbnames: add new option "incremental" to write the file once per account. + +#### Fixes + +- maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option. +- Test suites fixed and improved. +- Fix inaccurate UI messages when some messages are internally excluded from the + cached lists. + +#### Changes + +- imaplib2: bump to v2.43. +- More documentations moves to the website. +- Maintainer Dmitrijs has left the organization. +- Remove unnecessary imaplib2 workaround. +- release.sh: script for maintainers improved. + + +### OfflineIMAP v6.5.7-rc3 (2015-03-19) + +#### Notes + +Here comes a much bigger release than expected! With this release, the new +website is made official. + +Distribution maintainers, be aware that we now have a new man page +offlineimapui(7)! + +Also, the man page offlineimap(1) is sightly revised to explain the command line +options. Since `offlineimap --help` won't detail the options anymore, it becomes +critical. + +The maxage feature was broken by design and could delete mails on one side. It +is still under heavy work to fix issues when timezones are not synced. Gmail is +known to use different timezones accross mailboxes. + +The IMAP library imaplib2 was updated for the upcoming course to Python 3. + +The most other important changes are: + +- Possibility to use a proxy. +- All the documentation are SIGHTLY revisited and updated from all the available + places (sources files in the repository, wiki, website). A lot was moved from + the wiki and the sources to the website. +- the RFCs are available in the repository. + +#### Features + +- Add proxy support powered by PySocks. +- New man page offlineimapui to explain the available UIs. +- Add a CONTRIBUTING.rst file. +- Add a `TODO.rst` list for the contributors. +- Add a script for maintainers to roll out new releases. +- Add the `scripts/get-repository.sh` script to work on the website and the wiki. +- Doc: add IMAP RFCs. + +#### Fixes + +- Don't loose local mails because of maxage. +- Properly handle the cached messagelist. +- Do not error if `remoteuser` is not configured. +- imaplibutil: add missing errno import. +- LocalStatusSQLite: labels: don't fail if database returns unexpected None value. +- IDLE: continue trying selecting the folder on `OfflineImapError.Error`. + +#### Changes + +- imaplib2: bump to v2.42 +- `--help` becomes concise. +- Changelogs: move format back to markdown/kramdown to be more compatible with Jekyll. +- README: deep cleanups. +- code cleanups. +- code: more style consistency. +- sqlite: provide offending filename when open fails. +- MANUAL: full refactoring, change format to asciidoc. +- MANUAL: rename "KNOWN BUGS" TO "KNOWN ISSUES". +- MANUAL: add known issues entry about socktimeout for suspended sessions. +- offlineimap.conf: say what is the default value for the sep option. +- sqlite: provide information on what is failing for `OperationalError`. +- remove obsolete documentation. + + +### OfflineIMAP v6.5.7-rc2 (2015-01-18) + +#### Notes + +This release candidate should be minor for most users. + +The best points are about SSL not falling back on other authentication methods +when failing, better RAM footprint and reduced I/O access. + +Documentation had our attention, too. + +There's some code cleanups and code refactoring, as usual. + +#### Features + +* Do not keep reloading pyhtonfile, make it stateful. +* HACKING: how to create tags. +* MANUAL: add minor sample on how to retrieve a password with a helper python file. + +#### Fixes + +* Make OS-default CA certificate file to be requested explicitely. +* SSL: do not fallback on other authentication mode if it fails. +* Fix regression introduced while style patching. +* API documentation: properly auto-document main class, fixes. +* ui: Machine: remove offending param for a _printData() call. +* Drop caches after having processed folders. + +#### Changes + +* Fix unexpected garbage code. +* Properly re-raise exception to save original tracebacks. +* Refactoring: avoid redefining various Python keywords. +* Code: improvements of comments and more style consistency. +* Configuration file: better design and other small improvements. +* nametrans documentation: fix minor error. +* Unused import removal. +* Add a note about the incorrect rendering of the docstring with Sphinx. +* Errors handling: log the messages with level ERROR. +* MAINTAINERS: add mailing list maintainers. +* Fixed copyright statement. +* COPYING: fix unexpected characters. + + +### OfflineIMAP v6.5.7-rc1 (2015-01-07) + +#### Notes + +I think it's time for a new release candidate. Our release cycles are long +enough and users are asked to use the current TIP of the next branch to test +our recent patches. + +The current version makes better support for environment variable expansion and +improves OS portability. Gmail should be better supported: we are still +expecting feedbacks. Embedded library imaplib2 is updated to v2.37. +Debugging messages are added and polished. + +There's some code cleanups and refactoring, also. + + +#### Features + +* Expand environment variables in the following + configuration items: + - general.pythonfile; + - general.metadata; + - mbnames.filename; + - Repository.localfolders. + - Repository.sslcacertfile. + Make tilde and environment variable expansion in the following + configuration items: + - Repository.sslclientcert; + - Repository.sslclientkey. +* Support default CA bundle locations for a couple of + known Unix systems (Michael Vogt, GutHub pull #19) +* Added default CA bundle location for OpenBSD + (GitHub pull #120) and DragonFlyBSD. + +#### Fixes + +* Fix unbounded recursion during flag update (Josh Berry). +* Do not ignore gmail labels if header appears multiple times +* Delete gmail labels header before adding a new one +* Fix improper header separator for X-OfflineIMAP header +* Match header names case-insensitively +* Create SQLite database directory if it doesn't exist + yet; warn if path is not a directory (Nick Farrell, + GutHub pull #102) +* Properly manipulate contents of messagelist for folder +* Fix label processing in GmailMaildir +* Properly capitalize OpenSSL +* Fix warning-level message processing by MachineUI + (GitHub pull #64, GitHub pull #118). +* Properly generate tarball from "sdist" command (GitHub #137) +* Fix Markdown formatting +* Fix typo in apply_xforms invocation +* Merge pull request #136 from aroig/gh/label-fix +* Fix mangled message headers for servers without UIDPLUS: + X-OfflineIMAP was added with preceeding '\n' instead of + '\r\n' just before message was uploaded to the IMAP server. +* Add missing version bump for 6.5.6 (it was released with + 6.5.5 in setup.py and other places). + +#### Changes + +* Warn about a tricky piece of code in addmessageheader +* Rename addmessageheader()'s crlf parameter to linebreak +* addmessageheader: fix case #2 and flesh out docstring +* addmessageheader(): add debug for header insertion +* Add version qualifier to differentiate releases and development ones +* More clearly show results of folder name translation +* IMAP: provide message-id in error messages +* Trade recursion by plain old cycle +* Avoid copying array every time, just slice it +* Added OpenSSL exception clause to our main GPL to allow + people to link with OpenSSL in run-time. It is needed + at least for Debian, see + https://lists.debian.org/debian-legal/2002/10/msg00113.html + for details. +* Brought CustomConfig.py into more proper shape +* Updated bundled imaplib2 to 2.37: + - add missing idle_lock in _handler() +* Imaplib2: trade backticks to repr() +* Introduce CustomConfig method that applies set of transforms +* imaplibutil.py: remove unused imports +* CustomConfig.py: remove unused imports +* init.py: remove unused import +* repository/Base.py: remove unused import +* repository/GmailMaildir.py: remove unused import +* repository/LocalStatus.py: remove unused import +* ui/Curses.py: remove unused import +* ui/UIBase.py: remove unused import +* localeval: comment on security issues +* docs: remove obsolete comment about SubmittingPatches.rst +* utils/const.py: fix ident +* ui/UIBase: folderlist(): avoid built-in list() redefinition +* more consistent style + + + +### OfflineIMAP v6.5.6 (2014-05-14) + +* Fix IDLE mode regression (it didn't worked) introduced + after v6.5.5 (pointy hat goes to Eygene Ryabinkin, kudos -- + to Tomasz Żok) + + +### OfflineIMAP v6.5.6-rc1 (2014-05-14) + +* Add knob to invoke folderfilter dynamically on each sync (GitHub#73) +* Add knob to apply compression to IMAP connections (Abdó Roig-Maranges) +* Add knob to filter some headers before uploading message + to IMAP server (Abdó Roig-Maranges) +* Allow to sync GMail labels and implement GmailMaildir repository that + adds mechanics to change message labels (Abdó Roig-Maranges) +* Allow to migrate status data across differend backends + (Abdó Roig-Maranges) +* Support XDG Base Directory Specification + (if $XDG_CONFIG_HOME/offlineimap/config exists, use it as the + default configuration path; ~/.offlineimaprc is still tried after + XDG location) (GitHub#32) +* Allow multiple certificate fingerprints to be specified inside + 'cert_fingerprint' + + +### OfflineIMAP v6.5.5 (2013-10-07) + +* Avoid lockups for IMAP synchronizations running with the + "-1" command-line switch (X-Ryl669 ) +* Dump stacktrace for all threads on SIGQUIT: ease debugging + of threading and other issues +* SIGHUP is now handled as the termination notification rather than + the signal to reread the configuration (Dmitrijs Ledkovs) +* Honor the timezone of emails (Tobias Thierer) +* Allow mbnames output to be sorted by a custom sort key by specifying + a 'sort_keyfunc' function in the [mbnames] section of the config. +* Support SASL PLAIN authentication method. (Andreas Mack) +* Support transport-only tunnels that requre full IMAP authentication. + (Steve Purcell) +* Make the list of authentication mechanisms to be configurable. + (Andreas Mack) +* Allow to set message access and modification timestamps based + on the "Date" header of the message itself. (Cyril Russo) +* "peritem" format string for [mbnames] got new expansion key + "localfolders" that corresponds to the same parameter of the + local repository for the account being processed. +* [regression] pass folder names to the foldersort function, + revert the documented behaviour +* Fix handling of zero-sized IMAP data items (GitHub#15). +* Updated bundled imaplib2 to 2.35: + - fix for Gmail sending a BYE response after reading >100 messages + in a session; + - includes fix for GitHub#15: patch was accepted upstream. +* Updated bundled imaplib2 to 2.36: it includes support for SSL + version override that was integrated into our code before, + no other changes. +* Fixed parsing of quoted strings in IMAP responses: strings like "\\" + were treated as having \" as the escaped quote, rather than treating + it as the quoted escaped backslash (GitHub#53). +* Execute pre/post-sync hooks during synchronizations + toggled by IMAP IDLE message processing. (maxgerer@gmail.com) +* Catch unsuccessful local mail uploads when IMAP server + responds with "NO" status; that resulted in a loss of such + local messages. (Adam Spiers) +* Don't create folders if readonly is enabled. +* Learn to deal with readonly folders to properly detect this + condition and act accordingly. One example is Gmail's "Chats" + folder that is read-only, but contains logs of the quick chats. (E. + Ryabinkin) +* Fix str.format() calls for Python 2.6 (D. Logie) +* Remove APPENDUID hack, previously introduced to fix Gmail, no longer + necessary, it might have been breaking things. (J. Wiegley) +* Improve regex that could lead to 'NoneType' object has no attribute + 'group' (D. Franke) +* Improved error throwing on repository misconfiguration + +### OfflineIMAP v6.5.4 (2012-06-02) + +* bump bundled imaplib2 library 2.29 --> 2.33 +* Actually perform the SSL fingerprint check (reported by J. Cook) +* Curses UI, don't use colors after we shut down curses already (C.Höger) +* Document that '%' needs encoding as '%%' in configuration files. +* Fix crash when IMAP.quickchanged() led to an Error (reported by sharat87) +* Implement the createfolders setting to disable folder propagation (see docs) + +### OfflineIMAP v6.5.3.1 (2012-04-03) + +* Don't fail if no dry-run setting exists in offlineimap.conf + (introduced in 6.5.3) + + +### OfflineIMAP v6.5.3 (2012-04-02) + +* --dry-run mode protects us from performing any actual action. It will + not precisely give the exact information what will happen. If e.g. it + would need to create a folder, it merely outputs "Would create folder + X", but not how many and which mails it would transfer. + +* internal code changes to prepare for Python3 + +* Improve user documentation of nametrans/folderfilter + +* Fixed some cases where invalid nametrans rules were not caught and + we would not propagate local folders to the remote repository. + (now tested in test03) + +* Revert "* Slight performance enhancement uploading mails to an IMAP + server in the common case." It might have led to instabilities. + +* Revamped documentation structure. `make` in the `docs` dir or `make + doc` in the root dir will now create the 1) man page and 2) the user + documentation using sphinx (requiring python-doctools, and + sphinx). The resulting user docs are in `docs/html`. You can also + only create the man pages with `make man` in the `docs` dir. + +* -f command line option only works on the untranslated remote + repository folder names now. Previously folderfilters had to match + both the local AND remote name which caused unwanted behavior in + combination with nametrans rules. Clarify in the help text. + +* Some better output when using nonsensical configuration settings + +* Improve compatability of the curses UI with python 2.6 + +### OfflineIMAP v6.5.2.1 (2012-04-04) + +* Fix python2.6 compatibility with the TTYUI backend (crash) + +* Fix TTYUI regression from 6.5.2 in refresh loop (crash) + +* Fix crashes related to UIDVALIDITY returning "None" + +* Beginning of a test suite. So far there is only one test. Configure + test/credentials.conf and invoke with "python setup.py test" + +* Make folders containing quotes work rather than crashing + (reported by Mark Eichin) + +* Improve delete msg performance with SQLITE backend + +* Enforce basic UI when using the --info switch + +* Remove the Gmail "realdelete" option, as it could lead to potential + data loss. + +### OfflineIMAP v6.5.2 (2012-01-17) + +* Gmail "realdelete" option is considered harmful and has the potential + for data loss. Analysis at + http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 + Warnings were added to offlineimap.conf + +* Rather than write out the nametrans'lated folder names for mbnames, we + now write out the local untransformed box names. This is generally + what we want. This became relevant since we support nametrans rules on + the local side since only a short time. Reported by Paul Collignan. + +* Some sanity checks and improved error messages. + +* Revert 6.5.1.1 change to use public imaplib2 function, it was reported to + not always work. + +* Don't fail when ~/netrc is not readable by us. + +* Don't emit noisy regular sleeping announcements in Basic UI. + +### OfflineIMAP v6.5.1.2 (2012-01-07) - "Baby steps" + +Smallish bug fixes that deserve to be put out. + +* Fix possible crash during --info run +* Fix reading in Maildirs, where we would attempt to create empty + directories on REMOTE. +* Do not attempt to sync lower case custom Maildir flags. We do not + support them (yet) (this prevents many scary bogus sync messages) +* Add filter information to the filter list in --info output + +### OfflineIMAP v6.5.1.1 (2012-01-07) - "Das machine control is nicht fur gerfinger-poken und mittengrabben" + +Blinkenlights UI 6.5.0 regression fixes only. + +* Sleep led to crash ('abort_signal' not existing) + +* Make exit via 'q' key work again cleanly + +### OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" + +* Fixed Maildir regression "flagmatchre" not found. (regressed in 6.5.0) + +* Have console output go by default to STDOUT and not STDERR (regression + in 6.5.0) + +* Fixed MachineUI to urlencode() output lines again, rather than + outputting multi-line items. It's ugly as hell, but it had been that + way for years. + +* Remove the old global locking system. We lock only the accounts that + we currently sync, so you can invoke OfflineImap multiple times now as + long as you sync different accounts. This system is compatible with + all releases >= 6.4.0, so don't run older releases simultanous to this + one. + +### OfflineIMAP v6.5.0 (2012-01-06) + +This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. +Please upgrade to avoid potential data loss! The version has been bumped to +6.5.0, please let everyone know that the 6.4.x series is problematic. + +* Uploading multiple emails to an IMAP server would lead to wrong UIDs + being returned (ie the same for all), which confused offlineimap and + led to recurrent upload/download loops and inconsistencies in the + IMAP<->IMAP uid mapping. + +* Uploading of Messages from Maildir and IMAP<->IMAP has been made more + efficient by renaming files/mapping entries, rather than actually + loading and saving the message under a new UID. + +* Fix regression that broke MachineUI + +### OfflineIMAP v6.4.4 (2012-01-06) + +This is a bugfix release, fixing regressions occurring in or since 6.4.0. + +* Fix the missing folder error that occured when a new remote folder was + detected (IMAP<->Maildir) + +* Possibly fixed bug that prevented us from ever re-reading Maildir + folders, so flag changes and deletions were not detected when running + in a refresh loop. This is a regression that was introduced in about + 6.4.0. + +* Never mangle maildir file names when using nonstandard Maildir flags + (such as 'a'), note that they will still be deleted as they are not + supported in the sync to an IMAP server. + +### OfflineIMAP v6.4.3 (2012-01-04) + +#### New Features + +* add a --info command line switch that outputs useful information about + the server and the configuration for all enabled accounts. + +#### Changes + +* Reworked logging which was reported to e.g. not flush output to files + often enough. User-visible changes: + a) console output goes to stderr (for now). + b) file output has timestamps and looks identical in the basic and + ttyui UIs. + c) File output should be flushed after logging by default (do + report if not). + +* Bumped bundled imaplib2 to release 2.29 + +* Make ctrl-c exit cleanly rather aborting brutally (which could leave + around temporary files, half-written cache files, etc). Exiting on + SIGTERM and CTRL-C can take a little longer, but will be clean. + + +### OfflineIMAP v6.4.2 (2011-12-01) + +* IMAP<->IMAP sync with a readonly local IMAP repository failed with a + rather mysterious "TypeError: expected a character buffer object" + error. Fix this my retrieving the list of folders early enough even + for readonly repositories. + +* Fix regression from 6.4.0. When using local Maildirs with "/" as a + folder separator, all folder names would get a trailing slash + appended, which is plain wrong. + +### OfflineIMAP v6.4.1 (2011-11-17) + +#### Changes + +* Indicate progress when copying many messages (slightly change log format) + +* Output how long an account sync took (min:sec). + +#### Bug Fixes + +* Syncing multiple accounts in single-threaded mode would fail as we try + to "register" a thread as belonging to two accounts which was + fatal. Make it non-fatal (it can be legitimate). + +* New folders on the remote would be skipped on the very sync run they + are created and only by synced in subsequent runs. Fixed. + +* a readonly parameter to select() was not always treated correctly, + which could result in some folders being opened read-only when we + really needed read-write. + +### OfflineIMAP v6.4.0 (2011-09-29) + +This is the first stable release to support the forward-compatible per-account +locks and remote folder creation that has been introduced in the 6.3.5 series. + +* Various regression and bug fixes from the last couple of RCs + +### OfflineIMAP v6.3.5-rc3 (2011-09-21) + +#### Changes + +* Refresh server capabilities after login, so we know that Gmail + supports UIDPLUS (it only announces that after login, not + before). This prevents us from adding custom headers to Gmail uploads. + +#### Bug Fixes + +* Fix the creation of folders on remote repositories, which was still + botched on rc2. + +### OfflineIMAP v6.3.5-rc2 (2011-09-19) + +#### New Features + +* Implement per-account locking, so that it will possible to sync + different accounts at the same time. The old global lock is still in + place for backward compatibility reasons (to be able to run old and + new versions of OfflineImap concurrently) and will be removed in the + future. Starting with this version, OfflineImap will be + forward-compatible with the per-account locking style. + +* Implement RFC 2595 LOGINDISABLED. Warn the user and abort when we + attempt a plaintext login but the server has explicitly disabled + plaintext logins rather than crashing. + +* Folders will now also be automatically created on the REMOTE side of + an account if they exist on the local side. Use the folderfilters + setting on the local side to prevent some folders from migrating to + the remote side. Also, if you have a nametrans setting on the remote + repository, you might need a nametrans setting on the local repository + that leads to the original name (reverse nametrans). + +#### Changes + +* Documentation improvements concerning 'restoreatime' and some code cleanup + +* Maildir repositories now also respond to folderfilter= configurations. + +#### Bug Fixes + +* New emails are not created with "-rwxr-xr-x" but as "-rw-r--r--" + anymore, fixing a regression in 6.3.4. + +### OfflineIMAP v6.3.5-rc1 (2011-09-12) + +#### Notes + +Idle feature and SQLite backend leave the experimental stage! ,-) + +#### New Features + +* When a message upload/download fails, we do not abort the whole folder + synchronization, but only skip that message, informing the user at the + end of the sync run. + +* If you connect via ssl and 'cert_fingerprint' is configured, we check + that the server certificate is actually known and identical by + comparing the stored sha1 fingerprint with the current one. + +#### Changes + +* Refactor our IMAPServer class. Background work without user-visible + changes. +* Remove the configurability of the Blinkenlights statuschar. It + cluttered the main configuration file for little gain. +* Updated bundled imaplib2 to version 2.28. + +#### Bug Fixes + +* We protect more robustly against asking for inexistent messages from the + IMAP server, when someone else deletes or moves messages while we sync. +* Selecting inexistent folders specified in folderincludes now throws + nice errors and continues to sync with all other folders rather than + exiting offlineimap with a traceback. + + + +### OfflineIMAP v6.3.4 (2011-08-10) + +#### Notes + +Here we are. A nice release since v6.3.3, I think. + +#### Changes + +* Handle when UID can't be found on saved messages. + + + +### OfflineIMAP v6.3.4-rc4 (2011-07-27) + +#### Notes + +There is nothing exciting in this release. This is somewhat expected due to the +late merge on -rc3. + +#### New Features + +* Support maildir for Windows. + +#### Changes + +* Manual improved. + + +### OfflineIMAP v6.3.4-rc3 (2011-07-07) + +#### Notes + +Here is a surprising release. :-) + +As expected we have a lot bug fixes in this round (see git log for details), +including a fix for a bug we had for ages (details below) which is a very good +news. + +What makes this cycle so unusual is that I merged a feature to support StartTLS +automatically (thanks Sebastian!). Another very good news. + +We usually don't do much changes so late in a cycle. Now, things are highly +calming down and I hope a lot of people will test this release. Next one could +be the stable! + +#### New Features + +* Added StartTLS support, it will automatically be used if the server + supports it. + +#### Bug Fixes + +* We protect more robustly against asking for inexistent messages from the + IMAP server, when someone else deletes or moves messages while we sync. + + +### OfflineIMAP v6.3.4-rc2 (2011-06-15) + +#### Notes + +This was a very active rc1 and we could expect a lot of new fixes for the next +release. + +The most important fix is about a bug that could lead to data loss. Find more +information about his bug here: + + http://permalink.gmane.org/gmane.mail.imap.offlineimap.general/3803 + +The IDLE support is merged as experimental feature. + +#### New Features + +* Implement experimental IDLE feature. + +#### Changes + +* Maildirs use less memory while syncing. + +#### Bug Fixes + +* Saving to Maildirs now checks for file existence without race conditions. +* A bug in the underlying imap library has been fixed that could + potentially lead to data loss if the server interrupted responses with + unexpected but legal server status responses. This would mainly occur + in folders with many thousands of emails. Upgrading from the previous + release is strongly recommended. + + +### OfflineIMAP v6.3.4-rc1 (2011-05-16) + +#### Notes + +Welcome to the v6.3.4 pre-release cycle. Your favorite IMAP tool wins 2 new +features which were asked for a long time: +* an experimental SQL-based backend for the local cache; +* one-way synchronization cabability. + +Logic synchronization is reviewed and simplified (from 4 to 3 passes) giving +improved performance. + +Lot of work was done to give OfflineIMAP a better code base. Raised errors can +now rely on a new error system and should become the default in the coming +releases. + +As usual, we ask our users to test this release as much as possible, especially +the SQL backend. Have fun! + +#### New Features + +* Begin sphinx-based documentation for the code. +* Enable 1-way synchronization by settting a [Repository ...] to + readonly = True. When e.g. using offlineimap for backup purposes you + can thus make sure that no changes in your backup trickle back into + the main IMAP server. +* Optional: experimental SQLite-based backend for the LocalStatus + cache. Plain text remains the default. + +#### Changes + +* Start a enhanced error handling background system. This is designed to not + stop a whole sync process on all errors (not much used, yet). +* Documentation improvements: the FAQ wins new entries and add a new HACKING + file for developers. +* Lot of code cleanups. +* Reduced our sync logic from 4 passes to 3 passes (integrating upload of + "new" and "existing" messages into one function). This should result in a + slight speedup. +* No whitespace is stripped from comma-separated arguments passed via + the -f option. +* Give more detailed error when encountering a corrupt UID mapping file. + +#### Bug Fixes + +* Drop connection if synchronization failed. This is needed if resuming the + system from suspend mode gives a wrong connection. +* Fix the offlineimap crash when invoking debug option 'thread'. +* Make 'thread' command line option work. + + +### OfflineIMAP v6.3.3 (2011-04-24) + +#### Notes + +Make this last candidate cycle short. It looks like we don't need more tests as +most issues were raised and solved in the second round. Also, we have huge work +to merge big and expected features into OfflineIMAP. + +Thanks to all contributors, again. With such a contribution rate, we can release +stable faster. I hope it will be confirmed in the longer run! + +#### Changes + +* Improved documentation for querying password. + + +### OfflineIMAP v6.3.3-rc3 (2011-04-19) + +#### Notes + +It's more than a week since the previous release. Most of the issues raised were +discussed and fixed since last release. I think we can be glad and confident for +the future while the project live his merry life. + +#### Changes + +* The -f option did not work with Folder names with spaces. It works + now, use with quoting e.g. -f "INBOX, Deleted Mails". +* Improved documentation. +* Bump from imaplib2 v2.20 to v2.22. +* Code refactoring. + +#### Bug Fixes + +* Fix IMAP4 tunnel with imaplib2. + + +### OfflineIMAP v6.3.3-rc2 (2011-04-07) + +#### Notes + +We are now at the third week of the -rc1 cycle. I think it's welcome to begin +the -rc2 cycle. Things are highly calming down in the code even if we had +much more feedbacks than usual. Keep going your effort! + +I'd like to thank reporters who involved in this cycle: + - Баталов Григорий + - Alexander Skwar + - Christoph Höger + - dtk + - Greg Grossmeier + - h2oz7v + - Iain Dalton + - Pan Tsu + - Vincent Beffara + - Will Styler + +(my apologies if I forget somebody) ...and all active developers, of course! + +The imaplib2 migration looks to go the right way to be definetly released but +still needs more tests. So, here we go... + +#### Changes + +* Increase compatability with Gmail servers which claim to not support + the UIDPLUS extension but in reality do. + +#### Bug Fixes + +* Fix hang when using Ctrl+C in some cases. + + +### OfflineIMAP v6.3.3-rc1 (2011-03-16) + +#### Notes + +Here is time to begin the tests cycle. If feature topics are sent, I may merge +or delay them until the next stable release. + +Main change comes from the migration from imaplib to imaplib2. It's internal +code changes and doesn't impact users. UIDPLUS and subjectAltName for SSL are +also great improvements. + +This release includes a hang fix due to infinite loop. Users seeing OfflineIMAP +hang and consuming a lot of CPU are asked to update. + +That beeing said, this is still an early release candidate you should use for +non-critical data only! + +#### New Features + +* Implement UIDPLUS extension support. OfflineIMAP will now not insert + an X-OfflineIMAP header if the mail server supports the UIDPLUS + extension. +* SSL: support subjectAltName. + +#### Changes + +* Use imaplib2 instead of imaplib. +* Makefile use magic to find the version number. +* Rework the repository module +* Change UI names to Blinkenlights,TTYUI,Basic,Quiet,MachineUI. + Old names will still work, but are deprecated. + Document that we don't accept a list of UIs anymore. +* Reworked the syncing strategy. The only user-visible change is that + blowing away LocalStatus will not require you to redownload ALL of + your mails if you still have the local Maildir. It will simply + recreate LocalStatus. +* TTYUI ouput improved. +* Code cleanups. + +#### Bug Fixes + +* Fix ignoring output while determining the rst2xxx command name to build + documentation. +* Fix hang because of infinite loop reading EOF. +* Allow SSL connections to send keep-alive messages. +* Fix regression (UIBase is no more). +* Make profiling mode really enforce single-threading +* Do not send localized date strings to the IMAP server as it will + either ignore or refuse them. + + +### OfflineIMAP v6.3.2 (2010-02-21) + +#### Notes + +First of all I'm really happy to announce our new official `website +`_. Most of the work started from the impulse +of Philippe LeCavalier with the help of Sebastian Spaeth and other +contributors. Thanks to everybody. + +In this release, we are still touched by the "SSL3 write pending" but I think +time was long enough to try to fix it. We have our first entry in the "KNOWN +BUG" section of the manual about that. I'm afraid it could impact a lot of users +if some distribution package any SSL library not having underlying (still +obscure) requirements. Distribution maintainers should be care of it. I hope +this release will help us to have more reports. + +This release will also be the root of our long maintenance support. + +Other bugs were fixed. + +#### Bug Fixes + +* Fix craches for getglobalui(). +* Fix documentation build. +* Restore compatibiliy with python 2.5. + + +### OfflineIMAP v6.3.2-rc3 (2010-02-06) + +#### Notes + +We are still touched by the "SSL3 write pending" bug it would be really nice to +fix before releasing the coming stable. In the worse case, we'll have to add the +first entry in the "KNOWN BUG" section of the manual. I'm afraid it could impact +a lot of users if some distribution package any SSL library not having +underlying (still obscure) requirements. + +The best news with this release are the Curse UI fixed and the better reports +on errors. + +In this release I won't merge any patch not fixing a bug or a security issue. + +More feedbacks on the main issue would be appreciated. + +#### Changes + +* Sample offlineimap.conf states it expects a PEM formatted certificat. +* Give better trace information if an error occurs. +* Have --version ONLY print the version number. +* Code cleanups. + +#### Bug Fixes + +* Fix Curses UI (simplified by moving from MultiLock to Rlock implementation). +* Makefile: docutils build work whether python extension command is stripped or not. +* Makefile: clean now removes HTML documentation files. + + +### OfflineIMAP v6.3.2-rc2 (2010-12-21) + +#### Notes + +We are beginning a new tests cycle. At this stage, I expect most people will try +to intensively stuck OfflineIMAP. :-) + +#### New Features + +* Makefile learn to build the package and make it the default. +* Introduce a Changelog to involve community in the releasing process. +* Migrate documentation to restructuredtext. + +#### Changes + +* Improve CustomConfig documentation. +* Imply single threading mode in debug mode exept for "-d thread". +* Code and import cleanups. +* Allow UI to have arbitrary names. +* Code refactoring around UI and UIBase. +* Improve version managment and make it easier. +* Introduce a true single threading mode. + +#### Bug Fixes + +* Understand multiple EXISTS replies from servers like Zimbra. +* Only verify hostname if we actually use CA cert. +* Fix ssl ca-cert in the sample configuration file. +* Fix 'Ctrl+C' interruptions in threads. +* Fix makefile clean for files having whitespaces. +* Fix makefile to not remove unrelated files. +* Fixes in README. +* Remove uneeded files. + + +### OfflineIMAP v6.3.2-rc1 (2010-12-19) + +#### Notes + +We are beginning a tests cycle. If feature topics are sent, I may merge or +delay them until the next stable release. + +#### New Features + +* Primitive implementation of SSL certificates check. + +#### Changes + +* Use OptionParser instead of getopts. +* Code cleanups. + +#### Bug Fixes + +* Fix reading password from UI. + + +### OfflineIMAP v6.3.1 (2010-12-11) + +#### Notes + +Yes, I know I've just annouced the v6.3.0 in the same week. As said, it +was not really a true release for the software. This last release +includes fixes and improvements it might be nice to update to. + +Thanks to every body who helped to make this release with patches and +tips through the mailing list. This is clearly a release they own. + +#### Changes + +* cProfile becomes the default profiler. Sebastian Spaeth did refactoring to + prepare to the coming unit test suites. +* UI output formating enhanced. +* Some code cleanups. + +#### Bug Fixes + +* Fix possible overflow while working with Exchange. +* Fix time sleep while exiting threads. + + +### OfflineIMAP v6.3.0 (2010-12-09) + +#### Notes + +This release is more "administrative" than anything else and mainly marks the +change of the maintainer. New workflow and policy for developers come in. BTW, +I don't think I'll maintain debian/changelog. At least, not in the debian way. + +Most users and maintainers may rather want to skip this release. + +#### Bug Fixes + +* Fix terminal display on exit. +* netrc password authentication. +* User name querying from netrc. diff --git a/Changelog.rst b/Changelog.rst deleted file mode 100644 index 78554cd..0000000 --- a/Changelog.rst +++ /dev/null @@ -1,190 +0,0 @@ -========= -ChangeLog -========= - -:website: http://offlineimap.org - - -**NOTE FROM THE MAINTAINER:** - Contributors should use the `WIP` section in Changelog.draft.rst in order to - add changes they are working on. I will use it to make the new changelog entry - on releases. And because I'm lazy, it will also be used as a draft for the - releases announces. - - -OfflineIMAP v6.3.2 (2010-02-21) -=============================== - -Notes ------ - -First of all I'm really happy to announce our new official `website`_. Most of -the work started from the impulse of Philippe LeCavalier with the help of -Sebastian Spaeth and other contributors. Thanks to everybody. - -In this release, we are still touched by the "SSL3 write pending" but I think -time was long enough to try to fix it. We have our first entry in the "KNOWN -BUG" section of the manual about that. I'm afraid it could impact a lot of users -if some distribution package any SSL library not having underlying (still -obscure) requirements. Distribution maintainers should be care of it. I hope -this release will help us to have more reports. - -This release will also be the root of our long maintenance support. - -Other bugs were fixed. - -Bug Fixes ---------- - -* Fix craches for getglobalui(). -* Fix documentation build. -* Restore compatibiliy with python 2.5. - - -OfflineIMAP v6.3.2-rc3 (2010-02-06) -=================================== - -Notes ------ - -We are still touched by the "SSL3 write pending" bug it would be really nice to -fix before releasing the coming stable. In the worse case, we'll have to add the -first entry in the "KNOWN BUG" section of the manual. I'm afraid it could impact -a lot of users if some distribution package any SSL library not having -underlying (still obscure) requirements. - -The best news with this release are the Curse UI fixed and the better reports -on errors. - -In this release I won't merge any patch not fixing a bug or a security issue. - -More feedbacks on the main issue would be appreciated. - -Changes -------- - -* Sample offlineimap.conf states it expects a PEM formatted certificat. -* Give better trace information if an error occurs. -* Have --version ONLY print the version number. -* Code cleanups. - -Bug Fixes ---------- - -* Fix Curses UI (simplified by moving from MultiLock to Rlock implementation). -* Makefile: docutils build work whether python extension command is stripped or not. -* Makefile: clean now removes HTML documentation files. - - -OfflineIMAP v6.3.2-rc2 (2010-12-21) -=================================== - -Notes ------ - -We are beginning a new tests cycle. At this stage, I expect most people will try -to intensively stuck OfflineIMAP. :-) - -New Features ------------- - -* Makefile learn to build the package and make it the default. -* Introduce a Changelog to involve community in the releasing process. -* Migrate documentation to restructuredtext. - -Changes -------- - -* Improve CustomConfig documentation. -* Imply single threading mode in debug mode exept for "-d thread". -* Code and import cleanups. -* Allow UI to have arbitrary names. -* Code refactoring around UI and UIBase. -* Improve version managment and make it easier. -* Introduce a true single threading mode. - -Bug Fixes ---------- - -* Understand multiple EXISTS replies from servers like Zimbra. -* Only verify hostname if we actually use CA cert. -* Fix ssl ca-cert in the sample configuration file. -* Fix 'Ctrl+C' interruptions in threads. -* Fix makefile clean for files having whitespaces. -* Fix makefile to not remove unrelated files. -* Fixes in README. -* Remove uneeded files. - - -OfflineIMAP v6.3.2-rc1 (2010-12-19) -=================================== - -Notes ------ - -We are beginning a tests cycle. If feature topics are sent, I may merge or -delay them until the next stable release. - -New Features ------------- - -* Primitive implementation of SSL certificates check. - -Changes -------- - -* Use OptionParser instead of getopts. -* Code cleanups. - -Bug Fixes ---------- - -* Fix reading password from UI. - - -OfflineIMAP v6.3.1 (2010-12-11) -=============================== - -Notes ------ - -Yes, I know I've just annouced the v6.3.0 in the same week. As said, it -was not really a true release for the software. This last release -includes fixes and improvements it might be nice to update to. - -Thanks to every body who helped to make this release with patches and -tips through the mailing list. This is clearly a release they own. - -Changes -------- - -* cProfile becomes the default profiler. Sebastian Spaeth did refactoring to - prepare to the coming unit test suites. -* UI output formating enhanced. -* Some code cleanups. - -Bug Fixes ---------- - -* Fix possible overflow while working with Exchange. -* Fix time sleep while exiting threads. - - -OfflineIMAP v6.3.0 (2010-12-09) -=============================== - -Notes ------ - -This release is more "administrative" than anything else and mainly marks the -change of the maintainer. New workflow and policy for developers come in. BTW, -I don't think I'll maintain debian/changelog. At least, not in the debian way. - -Most users and maintainers may rather want to skip this release. - -Bug Fixes ---------- - -* Fix terminal display on exit. -* netrc password authentication. -* User name querying from netrc. diff --git a/MAINTAINERS.rst b/MAINTAINERS.rst new file mode 100644 index 0000000..2cff161 --- /dev/null +++ b/MAINTAINERS.rst @@ -0,0 +1,29 @@ +.. -*- coding: utf-8 -*- + +Official maintainers +==================== + +Eygene Ryabinkin + email: rea at freebsd.org + github: konvpalto + +Sebastian Spaeth + email: sebastian at sspaeth.de + github: spaetz + +Nicolas Sebrecht + email: nicolas.s-dev at laposte.net + github: nicolas33 + +Mailing List maintainers +======================== + +Eygene Ryabinkin + email: rea at freebsd.org + +Sebastian Spaeth + email: sebastian at sspaeth.de + +Nicolas Sebrecht + email: nicolas.s-dev at laposte.net + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..3a403a3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,14 @@ +global-exclude .gitignore .git *.bak *.orig *.rej +include setup.py +include COPYING +include Changelog* +include MAINTAINERS +include MANIFEST.in +include Makefile +include README.md +include offlineimap.conf* +include offlineimap.py +recursive-include offlineimap *.py +recursive-include bin * +recursive-include docs * +recursive-include test * diff --git a/Makefile b/Makefile index 518261f..da5c660 100644 --- a/Makefile +++ b/Makefile @@ -15,10 +15,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -VERSION=6.3.2.3 +VERSION=`./offlineimap.py --version` TARGZ=offlineimap_$(VERSION).tar.gz SHELL=/bin/bash -RST2HTML=`type rst2html 2>/dev/null 2>&1 && echo rst2html || echo rst2html.py` +RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` all: build @@ -35,19 +35,15 @@ clean: -find . -name '*.pygc' -exec rm -f {} \; -find . -name '*.class' -exec rm -f {} \; -find . -name '.cache*' -exec rm -f {} \; - -find . -name '*.html' -exec rm -f {} \; -rm -f manpage.links manpage.refs -find . -name auth -exec rm -vf {}/password {}/username \; - @$(MAKE) -C docs clean + @$(MAKE) -C clean -man: - @$(MAKE) -C docs man - -doc: +docs: @$(MAKE) -C docs - $(RST2HTML) README.rst readme.html - $(RST2HTML) SubmittingPatches.rst SubmittingPatches.html - $(RST2HTML) Changelog.rst Changelog.html + +websitedoc: + @$(MAKE) -C websitedoc targz: ../$(TARGZ) ../$(TARGZ): @@ -55,7 +51,7 @@ targz: ../$(TARGZ) echo "Containing directory must be called offlineimap-$(VERSION)"; \ exit 1; \ fi; \ - pwd && cd .. && pwd && tar -zhcv --exclude '.git' -f $(TARGZ) offlineimap-$(VERSION) + pwd && cd .. && pwd && tar -zhcv --exclude '.git' --exclude 'website' --exclude 'wiki' -f $(TARGZ) offlineimap-$(VERSION) rpm: targz cd .. && sudo rpmbuild -ta $(TARGZ) diff --git a/README.md b/README.md new file mode 100644 index 0000000..ae9c6b6 --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +[offlineimap]: https://github.com/OfflineIMAP/offlineimap +[website]: http://offlineimap.org +[wiki]: http://github.com/OfflineIMAP/offlineimap/wiki + +# OfflineImap + +## Description + +OfflineIMAP is a software to dispose your e-mail mailbox(es) as a **local +Maildir**. OfflineIMAP will synchronize both sides via *IMAP*. + +The main downside about IMAP is that you have to **trust** your MAIL provider to +not loose your mails. This is not something impossible while not very common. +With OfflineIMAP, you can download your Mailboxes and make you own backups of +the Maildir. + +This allows reading your mails while offline without the need for the mail +reader (MUA) to support IMAP disconnected operations. Need an attachement from a +message without internet? It's fine, the message is still there. + + +## License + +GNU General Public License v2. + + +## Why should I use OfflineIMAP? + +* It is **fast**. +* It is **reliable**. +* It is **flexible**. +* It is **safe**. + + +## Downloads + +You should first check if your distribution already package OfflineIMAP for you. +Downloads releases as [tarball or zipball](https://github.com/OfflineIMAP/offlineimap/tags). + + +## Feedbacks and contributions + +**The user discussions, development, announces and all the exciting stuff take +place in the mailing list.** While not mandatory to send emails, you can +[subscribe here](http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project). + +Bugs, issues and contributions can be requested to both the mailing list or the +[official Github project][offlineimap]. + + +## The community + +* OfflineIMAP's main site is the [project page at Github][offlineimap]. +* There is the [OfflineIMAP community's website][website]. +* And finally, [the wiki][wiki]. + + +## Requirements + +* Python v2.7 +* Python SQlite (optional while recommended) + + +## Documentation + +All the current and updated documentation is at the [community's website][website]. + +### Dispose locally + +You might want to dispose the documentation locally. Get the sources of the website. +For the other documentations, run the approppriate make target: +``` +$ ./scripts/get-repository.sh website +$ cd docs +$ make html # Require rst2html +$ make man # Require a2x +$ make api # Require sphinx +``` diff --git a/README.rst b/README.rst deleted file mode 100644 index b4bdccc..0000000 --- a/README.rst +++ /dev/null @@ -1,296 +0,0 @@ -.. -*- coding: utf-8 -*- - -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project - -====== -README -====== - -.. contents:: -.. sectnum:: - - -Description -=========== - -Welcome to the official OfflineIMAP project. - -*NOTICE:* this software was written by John Goerzen, who retired from -maintaining. It is now maintained by Nicolas Sebrecht at -https://github.com/nicolas33/offlineimap. Thanks to John for his great job and -to have share this project with us. - - -OfflineIMAP is a tool to simplify your e-mail reading. With OfflineIMAP, you can -read the same mailbox from multiple computers. You get a current copy of your -messages on each computer, and changes you make one place will be visible on all -other systems. For instance, you can delete a message on your home computer, and -it will appear deleted on your work computer as well. OfflineIMAP is also useful -if you want to use a mail reader that does not have IMAP support, has poor IMAP -support, or does not provide disconnected operation. - -OfflineIMAP works on pretty much any POSIX operating system, such as Linux, BSD -operating systems, MacOS X, Solaris, etc. - -OfflineIMAP is a Free Software project licensed under the GNU General Public -License. You can download it for free, and you can modify it. In fact, you are -encouraged to contribute to OfflineIMAP, and doing so is fast and easy. - - -OfflineIMAP is FAST; it synchronizes my two accounts with over 50 folders in 3 -seconds. Other similar tools might take over a minute, and achieve a -less-reliable result. Some mail readers can take over 10 minutes to do the same -thing, and some don't even support it at all. Unlike other mail tools, -OfflineIMAP features a multi-threaded synchronization algorithm that can -dramatically speed up performance in many situations by synchronizing several -different things simultaneously. - -OfflineIMAP is FLEXIBLE; you can customize which folders are synced via regular -expressions, lists, or Python expressions; a versatile and comprehensive -configuration file is used to control behavior; two user interfaces are -built-in; fine-tuning of synchronization performance is possible; internal or -external automation is supported; SSL and PREAUTH tunnels are both supported; -offline (or "unplugged") reading is supported; and esoteric IMAP features are -supported to ensure compatibility with the widest variety of IMAP servers. - -OfflineIMAP is SAFE; it uses an algorithm designed to prevent mail loss at all -costs. Because of the design of this algorithm, even programming errors should -not result in loss of mail. I am so confident in the algorithm that I use my -own personal and work accounts for testing of OfflineIMAP pre-release, -development, and beta releases. Of course, legally speaking, OfflineIMAP comes -with no warranty, so I am not responsible if this turns out to be wrong. - - -Method of Operation -=================== - -OfflineIMAP traditionally operates by maintaining a hierarchy of mail folders in -Maildir format locally. Your own mail reader will read mail from this tree, and -need never know that the mail comes from IMAP. OfflineIMAP will detect changes -to the mail folders on your IMAP server and your own computer and -bi-directionally synchronize them, copying, marking, and deleting messages as -necessary. - -With OfflineIMAP 4.0, a powerful new ability has been introduced ― the program -can now synchronize two IMAP servers with each other, with no need to have a -Maildir layer in-between. Many people use this if they use a mail reader on -their local machine that does not support Maildirs. People may install an IMAP -server on their local machine, and point both OfflineIMAP and their mail reader -of choice at it. This is often preferable to the mail reader's own IMAP support -since OfflineIMAP supports many features (offline reading, for one) that most -IMAP-aware readers don't. However, this feature is not as time-tested as -traditional syncing, so my advice is to stick with normal methods of operation -for the time being. - - -Quick Start -=========== - -If you have already installed OfflineIMAP system-wide, or your system -administrator has done that for you, your task for setting up OfflineIMAP for -the first time is quite simple. You just need to set up your configuration -file, make your folder directory, and run it! - -You can quickly set up your configuration file. The distribution includes a -file offlineimap.conf.minimal (Debian users may find this at -``/usr/share/doc/offlineimap/examples/offlineimap.conf.minimal``) that is a -basic example of setting of OfflineIMAP. You can simply copy this file into -your home directory and name it ``.offlineimaprc`` (note the leading period). A -command such as ``cp offlineimap.conf.minimal ~/.offlineimaprc`` will do it. -Or, if you prefer, you can just copy this text to ``~/.offlineimaprc``:: - - [general] - accounts = Test - - [Account Test] - localrepository = Local - remoterepository = Remote - - [Repository Local] - type = Maildir - localfolders = ~/Test - - [Repository Remote] - type = IMAP - remotehost = examplehost - remoteuser = jgoerzen - - -Now, edit the ``~/.offlineimaprc`` file with your favorite editor. All you have -to do is specify a directory for your folders to be in (on the localfolders -line), the host name of your IMAP server (on the remotehost line), and your -login name on the remote (on the remoteuser line). That's it! - -To run OfflineIMAP, you just have to say offlineimap ― it will fire up, ask you -for a login password if necessary, synchronize your folders, and exit. See? - -You can just throw away the rest of this finely-crafted, perfectly-honed manual! -Of course, if you want to see how you can make OfflineIMAP FIVE TIMES FASTER FOR -JUST $19.95 (err, well, $0), you have to read on! - - -Documentation -============= - -If you are reading this file on github, you can find more documentations in the -`docs` directory. - -Using your git repository, you can generate documentation with:: - - $ make doc - - -Mailing list -============ - -The user discussion, development and all exciting stuff take place in the -`mailing list`_. You're *NOT* supposed to subscribe to send emails. - - -Reporting bugs and contributions -================================ - -Bugs ----- - -Bugs, issues and contributions should be reported to the `mailing list`_. -**Please, don't use the github features (messages, pull requests, etc) at all. -It would most likely be discarded or ignored.** - - -======== -Examples -======== - -Here are some example configurations for various situations. Please e-mail any -other examples you have that may be useful to me. - - -Multiple Accounts with Mutt -=========================== - -This example shows you how to set up OfflineIMAP to synchronize multiple -accounts with the mutt mail reader. - -Start by creating a directory to hold your folders by running ``mkdir ~/Mail``. -Then, in your ``~/.offlineimaprc``, specify:: - - accounts = Personal, Work - - -Make sure that you have both an [Account Personal] and an [Account Work] -section. The local repository for each account must have different localfolder -path names. Also, make sure to enable [mbnames]. - -In each local repository section, write something like this:: - - localfolders = ~/Mail/Personal - - -Finally, add these lines to your ``~/.muttrc``:: - - source ~/path-to-mbnames-muttrc-mailboxes - folder-hook Personal set from="youremail@personal.com" - folder-hook Work set from="youremail@work.com" - set mbox_type=Maildir - set folder=$HOME/Mail - spoolfile=+Personal/INBOX - - -That's it! - - -UW-IMAPD and References -======================= - -Some users with a UW-IMAPD server need to use OfflineIMAP's "reference" feature -to get at their mailboxes, specifying a reference of ``~/Mail`` or ``#mh/`` -depending on the configuration. The below configuration from (originally from -docwhat@gerf.org) shows using a reference of Mail, a nametrans that strips the -leading Mail/ off incoming folder names, and a folderfilter that limits the -folders synced to just three:: - - [Account Gerf] - localrepository = GerfLocal - remoterepository = GerfRemote - - [Repository GerfLocal] - type = Maildir - localfolders = ~/Mail - - [Repository GerfRemote] - type = IMAP - remotehost = gerf.org - ssl = yes - remoteuser = docwhat - reference = Mail - # Trims off the preceeding Mail on all the folder names. - nametrans = lambda foldername: \ - re.sub('^Mail/', '', foldername) - # Yeah, you have to mention the Mail dir, even though it - # would seem intuitive that reference would trim it. - folderfilter = lambda foldername: foldername in [ - 'Mail/INBOX', - 'Mail/list/zaurus-general', - 'Mail/list/zaurus-dev', - ] - maxconnections = 1 - holdconnectionopen = no - - -pythonfile Configuration File Option -==================================== - -You can have OfflineIMAP load up a Python file before evaluating the -configuration file options that are Python expressions. This example is based -on one supplied by Tommi Virtanen for this feature. - - -In ~/.offlineimaprc, he adds these options:: - - [general] - pythonfile=~/.offlineimap.py - [Repository foo] - foldersort=mycmp - -Then, the ~/.offlineimap.py file will contain:: - - prioritized = ['INBOX', 'personal', 'announce', 'list'] - - def mycmp(x, y): - for prefix in prioritized: - xsw = x.startswith(prefix) - ysw = y.startswith(prefix) - if xsw and ysw: - return cmp(x, y) - elif xsw: - return -1 - elif ysw: - return +1 - return cmp(x, y) - - def test_mycmp(): - import os, os.path - folders=os.listdir(os.path.expanduser('~/data/mail/tv@hq.yok.utu.fi')) - folders.sort(mycmp) - print folders - - -This code snippet illustrates how the foldersort option can be customized with a -Python function from the pythonfile to always synchronize certain folders first. - - -Signals -======= - -OfflineIMAP writes its current PID into ``~/.offlineimap/pid`` when it is -running. It is not guaranteed that this file will not exist when OfflineIMAP is -not running. - - diff --git a/SubmittingPatches.rst b/SubmittingPatches.rst deleted file mode 100644 index c200087..0000000 --- a/SubmittingPatches.rst +++ /dev/null @@ -1,596 +0,0 @@ -.. -*- coding: utf-8 -*- - -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project - -================================================= -Checklist (and a short version for the impatient) -================================================= - -Commits -======= - -* make commits of logical units -* check for unnecessary whitespace with ``git diff --check`` - before committing -* do not check in commented out code or unneeded files -* the first line of the commit message should be a short - description (50 characters is the soft limit, see DISCUSSION - in git-commit(1)), and should skip the full stop -* the body should provide a meaningful commit message, which: - * uses the imperative, present tense: **change**, - not **changed** or **changes**. - * includes motivation for the change, and contrasts - its implementation with previous behaviour -* add a ``Signed-off-by: Your Name `` line to the - commit message (or just use the option `-s` when committing) - to confirm that you agree to the **Developer's Certificate of Origin** -* make sure that you have tests for the bug you are fixing -* make sure that the test suite passes after your commit - -Patch -===== - -* use ``git format-patch -M`` to create the patch -* do not PGP sign your patch -* do not attach your patch, but read in the mail - body, unless you cannot teach your mailer to - leave the formatting of the patch alone. -* be careful doing cut & paste into your mailer, not to - corrupt whitespaces. -* provide additional information (which is unsuitable for - the commit message) between the ``---`` and the diffstat -* if you change, add, or remove a command line option or - make some other user interface change, the associated - documentation should be updated as well. -* if your name is not writable in ASCII, make sure that - you send off a message in the correct encoding. -* send the patch to the `mailing list`_ and the - maintainer (nicolas.s-dev@laposte.net) if (and only if) - the patch is ready for inclusion. If you use `git-send-email(1)`, - please test it first by sending email to yourself. -* see below for instructions specific to your mailer - - -============ -Long version -============ - -I started reading over the SubmittingPatches document for Git, primarily because -I wanted to have a document similar to it for OfflineIMAP to make sure people -understand what they are doing when they write `Signed-off-by` line. - -But the patch submission requirements are a lot more relaxed here on the -technical/contents front, because the OfflineIMAP is a lot smaller ;-). So here -is only the relevant bits. - -Decide what to base your work on -================================ - -In general, always base your work on the oldest branch that your -change is relevant to. - -* A bugfix should be based on 'maint' in general. If the bug is not - present in 'maint', base it on 'master'. For a bug that's not yet - in 'master', find the topic that introduces the regression, and - base your work on the tip of the topic. -* A new feature should be based on 'master' in general. If the new - feature depends on a topic that is in 'pu', but not in 'master', - base your work on the tip of that topic. -* Corrections and enhancements to a topic not yet in 'master' should - be based on the tip of that topic. If the topic has not been merged - to 'next', it's alright to add a note to squash minor corrections - into the series. -* In the exceptional case that a new feature depends on several topics - not in 'master', start working on 'next' or 'pu' privately and send - out patches for discussion. Before the final merge, you may have to - wait until some of the dependent topics graduate to 'master', and - rebase your work. - -To find the tip of a topic branch, run ``git log --first-parent -master..pu`` and look for the merge commit. The second parent of this -commit is the tip of the topic branch. - -Make separate commits for logically separate changes -==================================================== - -Unless your patch is really trivial, you should not be sending -out a patch that was generated between your working tree and -your commit head. Instead, always make a commit with complete -commit message and generate a series of patches from your -repository. It is a good discipline. - -Describe the technical detail of the change(s). - -If your description starts to get too long, that's a sign that you -probably need to split up your commit to finer grained pieces. -That being said, patches which plainly describe the things that -help reviewers check the patch, and future maintainers understand -the code, are the most beautiful patches. Descriptions that summarise -the point in the subject well, and describe the motivation for the -change, the approach taken by the change, and if relevant how this -differs substantially from the prior version, can be found on Usenet -archives back into the late 80's. Consider it like good Netiquette, -but for code. - - -Generate your patch using git tools out of your commits -------------------------------------------------------- - -git based diff tools (git, Cogito, and StGIT included) generate -unidiff which is the preferred format. - -You do not have to be afraid to use -M option to ``git diff`` or -``git format-patch``, if your patch involves file renames. The -receiving end can handle them just fine. - -Please make sure your patch does not include any extra files -which do not belong in a patch submission. Make sure to review -your patch after generating it, to ensure accuracy. Before -sending out, please make sure it cleanly applies to the "master" -branch head. If you are preparing a work based on "next" branch, -that is fine, but please mark it as such. - - -Sending your patches -==================== - -People on the mailing list need to be able to read and -comment on the changes you are submitting. It is important for -a developer to be able to "quote" your changes, using standard -e-mail tools, so that they may comment on specific portions of -your code. For this reason, all patches should be submitted -"inline". WARNING: Be wary of your MUAs word-wrap -corrupting your patch. Do not cut-n-paste your patch; you can -lose tabs that way if you are not careful. - -It is a common convention to prefix your subject line with -[PATCH]. This lets people easily distinguish patches from other -e-mail discussions. Use of additional markers after PATCH and -the closing bracket to mark the nature of the patch is also -encouraged. E.g. [PATCH/RFC] is often used when the patch is -not ready to be applied but it is for discussion, [PATCH v2], -[PATCH v3] etc. are often seen when you are sending an update to -what you have previously sent. - -``git format-patch`` command follows the best current practice to -format the body of an e-mail message. At the beginning of the -patch should come your commit message, ending with the -Signed-off-by: lines, and a line that consists of three dashes, -followed by the diffstat information and the patch itself. If -you are forwarding a patch from somebody else, optionally, at -the beginning of the e-mail message just before the commit -message starts, you can put a "From: " line to name that person. - -You often want to add additional explanation about the patch, -other than the commit message itself. Place such "cover letter" -material between the three dash lines and the diffstat. - -Do not attach the patch as a MIME attachment, compressed or not. -Do not let your e-mail client send quoted-printable. Do not let -your e-mail client send format=flowed which would destroy -whitespaces in your patches. Many -popular e-mail applications will not always transmit a MIME -attachment as plain text, making it impossible to comment on -your code. A MIME attachment also takes a bit more time to -process. This does not decrease the likelihood of your -MIME-attached change being accepted, but it makes it more likely -that it will be postponed. - -Exception: If your mailer is mangling patches then someone may ask -you to re-send them using MIME, that is OK. - -Do not PGP sign your patch, at least for now. Most likely, your -maintainer or other people on the list would not have your PGP -key and would not bother obtaining it anyway. Your patch is not -judged by who you are; a good patch from an unknown origin has a -far better chance of being accepted than a patch from a known, -respected origin that is done poorly or does incorrect things. - -If you really really really really want to do a PGP signed -patch, format it as "multipart/signed", not a text/plain message -that starts with '-----BEGIN PGP SIGNED MESSAGE-----'. That is -not a text/plain, it's something else. - -Unless your patch is a very trivial and an obviously correct one, -first send it with "To:" set to the mailing list, with "cc:" listing -people who are involved in the area you are touching (the output from -"git blame $path" and "git shortlog --no-merges $path" would help to -identify them), to solicit comments and reviews. After the list -reached a consensus that it is a good idea to apply the patch, re-send -it with "To:" set to the maintainer and optionally "cc:" the list for -inclusion. Do not forget to add trailers such as "Acked-by:", -"Reviewed-by:" and "Tested-by:" after your "Signed-off-by:" line as -necessary. - - -Sign your work -============== - -To improve tracking of who did what, we've borrowed the -"sign-off" procedure from the Linux kernel project on patches -that are being emailed around. Although OfflineIMAP is a lot -smaller project it is a good discipline to follow it. - -The sign-off is a simple line at the end of the explanation for -the patch, which **certifies that you wrote it or otherwise have -the right to pass it on as a open-source patch**. The rules are -pretty simple: if you can certify the below: - -**Developer's Certificate of Origin 1.1** ------------------------------------------ - - By making a contribution to this project, I certify that: - - (a) The contribution was created in whole or in part by me and I - have the right to submit it under the open source license - indicated in the file; or - - (b) The contribution is based upon previous work that, to the best - of my knowledge, is covered under an appropriate open source - license and I have the right under that license to submit that - work with modifications, whether created in whole or in part - by me, under the same open source license (unless I am - permitted to submit under a different license), as indicated - in the file; or - - (c) The contribution was provided directly to me by some other - person who certified (a), (b) or (c) and I have not modified - it. - - (d) I understand and agree that this project and the contribution - are public and that a record of the contribution (including all - personal information I submit with it, including my sign-off) is - maintained indefinitely and may be redistributed consistent with - this project or the open source license(s) involved. - -then you just add a line saying - - Signed-off-by: Random J Developer - -This line can be automatically added by git if you run the git-commit -command with the -s option. - -Notice that you can place your own Signed-off-by: line when -forwarding somebody else's patch with the above rules for -D-C-O. Indeed you are encouraged to do so. Do not forget to -place an in-body "From: " line at the beginning to properly attribute -the change to its true author (see above). - -Also notice that a real name is used in the Signed-off-by: line. Please -don't hide your real name. - -If you like, you can put extra tags at the end: - -* "Reported-by:" is used to to credit someone who found the bug that - the patch attempts to fix. -* "Acked-by:" says that the person who is more familiar with the area - the patch attempts to modify liked the patch. -* "Reviewed-by:", unlike the other tags, can only be offered by the - reviewer and means that she is completely satisfied that the patch - is ready for application. It is usually offered only after a - detailed review. -* "Tested-by:" is used to indicate that the person applied the patch - and found it to have the desired effect. - -You can also create your own tag or use one that's in common usage -such as "Thanks-to:", "Based-on-patch-by:", or "Mentored-by:". - -An ideal patch flow -=================== - -Here is an ideal patch flow for this project the current maintainer -suggests to the contributors: - - (0) You come up with an itch. You code it up. - - (1) Send it to the list and cc people who may need to know about - the change. - - The people who may need to know are the ones whose code you - are butchering. These people happen to be the ones who are - most likely to be knowledgeable enough to help you, but - they have no obligation to help you (i.e. you ask for help, - don't demand). ``git log -p -- $area_you_are_modifying`` would - help you find out who they are. - - (2) You get comments and suggestions for improvements. You may - even get them in a "on top of your change" patch form. - - (3) Polish, refine, and re-send to the list and the people who - spend their time to improve your patch. Go back to step (2). - - (4) The list forms consensus that the last round of your patch is - good. Send it to the list and cc the maintainer. - - (5) A topic branch is created with the patch and is merged to 'next', - and cooked further and eventually graduates to 'master'. - -In any time between the (2)-(3) cycle, the maintainer may pick it up -from the list and queue it to 'pu', in order to make it easier for -people play with it without having to pick up and apply the patch to -their trees themselves. - -Know the status of your patch after submission ----------------------------------------------- - -* You can use Git itself to find out when your patch is merged in - master. ``git pull --rebase`` will automatically skip already-applied - patches, and will let you know. This works only if you rebase on top - of the branch in which your patch has been merged (i.e. it will not - tell you if your patch is merged in pu if you rebase on top of - master). - -.. * Read the git mailing list, the maintainer regularly posts messages - entitled "What's cooking in git.git" and "What's in git.git" giving - the status of various proposed changes. - -MUA specific hints -================== - -Some of patches I receive or pick up from the list share common -patterns of breakage. Please make sure your MUA is set up -properly not to corrupt whitespaces. Here are two common ones -I have seen: - -* Empty context lines that do not have _any_ whitespace. - -* Non empty context lines that have one extra whitespace at the - beginning. - -One test you could do yourself if your MUA is set up correctly is: - -* Send the patch to yourself, exactly the way you would, except - To: and Cc: lines, which would not contain the list and - maintainer address. - -* Save that patch to a file in UNIX mailbox format. Call it say - a.patch. - -* Try to apply to the tip of the "master" branch from the - git.git public repository:: - - $ git fetch http://kernel.org/pub/scm/git/git.git master:test-apply - $ git checkout test-apply - $ git reset --hard - $ git am a.patch - -If it does not apply correctly, there can be various reasons. - -* Your patch itself does not apply cleanly. That is _bad_ but - does not have much to do with your MUA. Please rebase the - patch appropriately. - -* Your MUA corrupted your patch; "am" would complain that - the patch does not apply. Look at .git/rebase-apply/ subdirectory and - see what 'patch' file contains and check for the common - corruption patterns mentioned above. - -* While you are at it, check what are in 'info' and - 'final-commit' files as well. If what is in 'final-commit' is - not exactly what you would want to see in the commit log - message, it is very likely that your maintainer would end up - hand editing the log message when he applies your patch. - Things like "Hi, this is my first patch.\n", if you really - want to put in the patch e-mail, should come after the - three-dash line that signals the end of the commit message. - - -Pine ----- - -(Johannes Schindelin) - I don't know how many people still use pine, but for those poor souls it may - be good to mention that the quell-flowed-text is needed for recent versions. - - ... the "no-strip-whitespace-before-send" option, too. AFAIK it was introduced - in 4.60. - -(Linus Torvalds) - And 4.58 needs at least this - -:: - - --- - diff-tree 8326dd8350be64ac7fc805f6563a1d61ad10d32c (from e886a61f76edf5410573e92e38ce22974f9c40f1) - Author: Linus Torvalds - Date: Mon Aug 15 17:23:51 2005 -0700 - - Fix pine whitespace-corruption bug - - There's no excuse for unconditionally removing whitespace from - the pico buffers on close. - - diff --git a/pico/pico.c b/pico/pico.c - --- a/pico/pico.c - +++ b/pico/pico.c - @@ -219,7 +219,9 @@ PICO *pm; - switch(pico_all_done){ /* prepare for/handle final events */ - case COMP_EXIT : /* already confirmed */ - packheader(); - +#if 0 - stripwhitespace(); - +#endif - c |= COMP_EXIT; - break; - -(Daniel Barkalow) - > A patch to SubmittingPatches, MUA specific help section for - > users of Pine 4.63 would be very much appreciated. - - Ah, it looks like a recent version changed the default behavior to do the - right thing, and inverted the sense of the configuration option. (Either - that or Gentoo did it.) So you need to set the - "no-strip-whitespace-before-send" option, unless the option you have is - "strip-whitespace-before-send", in which case you should avoid checking - it. - - -Thunderbird ------------ - -(A Large Angry SCM) - By default, Thunderbird will both wrap emails as well as flag them as - being 'format=flowed', both of which will make the resulting email unusable - by git. - - Here are some hints on how to successfully submit patches inline using - Thunderbird. - - There are two different approaches. One approach is to configure - Thunderbird to not mangle patches. The second approach is to use - an external editor to keep Thunderbird from mangling the patches. - -**Approach #1 (configuration):** - - This recipe is current as of Thunderbird 2.0.0.19. Three steps: - - 1. Configure your mail server composition as plain text - Edit...Account Settings...Composition & Addressing, - uncheck 'Compose Messages in HTML'. - 2. Configure your general composition window to not wrap - Edit..Preferences..Composition, wrap plain text messages at 0 - 3. Disable the use of format=flowed - Edit..Preferences..Advanced..Config Editor. Search for: - mailnews.send_plaintext_flowed - toggle it to make sure it is set to 'false'. - - After that is done, you should be able to compose email as you - otherwise would (cut + paste, git-format-patch | git-imap-send, etc), - and the patches should not be mangled. - -**Approach #2 (external editor):** - -This recipe appears to work with the current [*1*] Thunderbird from Suse. - -The following Thunderbird extensions are needed: - AboutConfig 0.5 - http://aboutconfig.mozdev.org/ - External Editor 0.7.2 - http://globs.org/articles.php?lng=en&pg=8 - - -1) Prepare the patch as a text file using your method of choice. - -2) Before opening a compose window, use Edit->Account Settings to - uncheck the "Compose messages in HTML format" setting in the - "Composition & Addressing" panel of the account to be used to send the - patch. [*2*] - -3) In the main Thunderbird window, _before_ you open the compose window - for the patch, use Tools->about:config to set the following to the - indicated values:: - - mailnews.send_plaintext_flowed => false - mailnews.wraplength => 0 - -4) Open a compose window and click the external editor icon. - -5) In the external editor window, read in the patch file and exit the - editor normally. - -6) Back in the compose window: Add whatever other text you wish to the - message, complete the addressing and subject fields, and press send. - -7) Optionally, undo the about:config/account settings changes made in - steps 2 & 3. - - -[Footnotes] - -*1* Version 1.0 (20041207) from the MozillaThunderbird-1.0-5 rpm of Suse -9.3 professional updates. - -*2* It may be possible to do this with about:config and the following -settings but I haven't tried, yet:: - - mail.html_compose => false - mail.identity.default.compose_html => false - mail.identity.id?.compose_html => false - -(Lukas Sandström) - There is a script in contrib/thunderbird-patch-inline which can help you - include patches with Thunderbird in an easy way. To use it, do the steps above - and then use the script as the external editor. - -Gnus ----- - -'|' in the *Summary* buffer can be used to pipe the current -message to an external program, and this is a handy way to drive -"git am". However, if the message is MIME encoded, what is -piped into the program is the representation you see in your -*Article* buffer after unwrapping MIME. This is often not what -you would want for two reasons. It tends to screw up non ASCII -characters (most notably in people's names), and also -whitespaces (fatal in patches). Running 'C-u g' to display the -message in raw form before using '|' to run the pipe can work -this problem around. - - -KMail ------ - -This should help you to submit patches inline using KMail. - -1) Prepare the patch as a text file. - -2) Click on New Mail. - -3) Go under "Options" in the Composer window and be sure that - "Word wrap" is not set. - -4) Use Message -> Insert file... and insert the patch. - -5) Back in the compose window: add whatever other text you wish to the - message, complete the addressing and subject fields, and press send. - - -Gmail ------ - -GMail does not appear to have any way to turn off line wrapping in the web -interface, so this will mangle any emails that you send. You can however -use "git send-email" and send your patches through the GMail SMTP server, or -use any IMAP email client to connect to the google IMAP server and forward -the emails through that. - -To use ``git send-email`` and send your patches through the GMail SMTP server, -edit `~/.gitconfig` to specify your account settings:: - - [sendemail] - smtpencryption = tls - smtpserver = smtp.gmail.com - smtpuser = user@gmail.com - smtppass = p4ssw0rd - smtpserverport = 587 - -Once your commits are ready to be sent to the mailing list, run the -following commands:: - - $ git format-patch --cover-letter -M origin/master -o outgoing/ - $ edit outgoing/0000-* - $ git send-email outgoing/* - -To submit using the IMAP interface, first, edit your `~/.gitconfig` to specify your -account settings:: - - [imap] - folder = "[Gmail]/Drafts" - host = imaps://imap.gmail.com - user = user@gmail.com - pass = p4ssw0rd - port = 993 - sslverify = false - -You might need to instead use: folder = "[Google Mail]/Drafts" if you get an error -that the "Folder doesn't exist". - -Once your commits are ready to be sent to the mailing list, run the -following commands:: - - $ git format-patch --cover-letter -M --stdout origin/master | git imap-send - -Just make sure to disable line wrapping in the email client (GMail web -interface will line wrap no matter what, so you need to use a real -IMAP client). - diff --git a/TODO.rst b/TODO.rst new file mode 100644 index 0000000..73f80da --- /dev/null +++ b/TODO.rst @@ -0,0 +1,127 @@ +.. vim: spelllang=en ts=2 expandtab : + +.. _coding style: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/CodingGuidelines.rst + +============================ +TODO list by relevance order +============================ + +Should be the starting point to improve the `coding style`_. + +Write your WIP directly in this file. + +TODO list +--------- + +* Better names for variables, objects, etc. + + +* Improve comments. + + Most of the current comments assume a very good + knowledge of the internals. That sucks because I guess nobody is + anymore aware of ALL of them. Time when this was a one guy made + project has long passed. + + +* Better policy on objects. + + - Turn ALL attributes private and use accessors. This is not + "pythonic" but such pythonic thing turn the code into intricated + code. + + - Turn ALL methods not intended to be used outside, private. + + +* Revamp the factorization. + + It's not unusual to find "factorized" code + for bad reasons: because it made the code /look/ nicer, but the + factorized function/methods is actually called from ONE place. While it + might locally help, such practice globally defeat the purpose because + we lose the view of what is true factorized code and what is not. + + +* Namespace the factorized code. + + If a method require a local function, DON'T USE yet another method. Use a + local namespaced function.:: + + class BLah(object): + def _internal_method(self, arg): + def local_factorized(local_arg): + # local_factorized's code + # _internal_method's code. + + Python allows local namespaced functions for good reasons. + + +* Better inheritance policy. + + Take the sample of the folder/LocalStatus(SQlite) and folder/Base stuffs. It's + *nearly IMPOSSIBLE* to know and understand what parent method is used by what + child, for what purpose, etc. So, instead of (re)defining methods in the wild, + keep the well common NON-redefined stuff into the parent and define the + required methods in the childs. We really don't want anything like:: + + def method(self): + raise NotImplemented + + While this is common practice in Python, think about that again: how a + parent object should know all the expected methods/accessors of all the + possible kind of childs? + + Inheritance is about factorizing, certainly **NOT** about **defining the + interface** of the childs. + + +* Introduce as many as intermediate inherited objects as required. + + Keeping linear inheritance is good because Python sucks at playing + with multiple parents and it keeps things simple. But a parent should + have ALL its methods used in ALL the childs. If not, it's a good + sign that a new intermediate object should be introduced in the + inheritance line. + +* Don't blindly inherit from library objects. + + We do want **well defined interfaces**. For example, we do too much things + like imapobj.methodcall() while the imapobj is far inherited from imaplib2. + + We have NO clue about what we currently use from the library. + Having a dump wrappper for each call should be made mandatory for + objects inherited from a library. Using composed objects should be + seriously considered in this case, instead of using inheritance. + +* Use factories. + + Current objects do too much initialization stuff varying with the context it + is used. Move things like that into factories and keep the objects definitions + clean. + + +* Make it clear when we expect a composite object and what we expect + exactly. + + Even the more obvious composed objects are badly defined. For example, + the ``conf`` instances are spread across a lot of objects. Did you know + that such composed objects are sometimes restricted to the section the + object works on, and most of the time it's not restricted at all? + How many time it requires to find and understand on what we are + currently working? + + +* Seriously improve our debugging/hacking sessions (AGAIN). + + Until now, we have limited the improvements to allow better/full stack traces. + While this was actually required, we now hit some limitations of the whole + exception-based paradigm. For example, it's very HARD to follow an instance + during its life time. I have a good overview of what we could do in this area, + so don't matter much about that if you don't get the point or what could be + done. + + +* Support Python 3. + + +* Support Unicode. diff --git a/contrib/release.sh b/contrib/release.sh new file mode 100755 index 0000000..72e9936 --- /dev/null +++ b/contrib/release.sh @@ -0,0 +1,435 @@ +#!/bin/sh +# +# Put into Public Domain, by Nicolas Sebrecht +# +# Create new releases in OfflineIMAP. + +# TODO: https://developer.github.com/v3/repos/releases/#create-a-release +# https://developer.github.com/libraries/ +# https://github.com/turnkeylinux/octohub +# https://github.com/michaelliao/githubpy (onefile) +# https://github.com/sigmavirus24/github3.py +# https://github.com/copitux/python-github3 +# https://github.com/PyGithub/PyGithub +# https://github.com/micha/resty (curl) + +# TODO: move configuration out and source it. +# TODO: implement rollback. + +__VERSION__='v0.2' + +SPHINXBUILD=sphinx-build + +MAILING_LIST='offlineimap-project@lists.alioth.debian.org' + +DOCSDIR='docs' +ANNOUNCE_MAGIC='#### Notes ' +CHANGELOG_MAGIC='{:toc}' +CHANGELOG='Changelog.md' +CACHEDIR='.git/offlineimap-release' +WEBSITE='website' +WEBSITE_LATEST="${WEBSITE}/_data/latest.yml" + +TMP_CHANGELOG_EXCERPT="${CACHEDIR}/changelog.excerpt.md" +TMP_CHANGELOG_EXCERPT_OLD="${TMP_CHANGELOG_EXCERPT}.old" +TMP_CHANGELOG="${CACHEDIR}/changelog.md" +TMP_ANNOUNCE="${CACHEDIR}/announce.txt" + +True=0 +False=1 +Yes=$True +No=$False + +DEBUG=$True + +# +# $1: EXIT_CODE +# $2..: message +function die () { + n=$1 + shift + echo $* + exit $n +} + + +function debug () { + if test $DEBUG -eq $True + then + echo "DEBUG: $*" >&2 + fi +} + + + +# +# $1: question +# $2: message on abort +# +function ask () { + echo + echo -n "--- $1 " + read -r ans + test "n$ans" = 'n' -o "n$ans" = 'ny' && return $Yes + test "n$ans" = "ns" -o "n$ans" = 'nn' && return $No + die 1 "! $2" +} + + + +# +# $1: message +# $1: path to file +# +function edit_file () { + ask "Press Enter to $1" + test $? -eq $Yes && { + $EDITOR "$2" + reset + } +} + + + +function fix_pwd () { + debug 'in fix_pwd' + cd "$(git rev-parse --show-toplevel)" || \ + die 2 "cannot determine the root of the repository" +} + + +function prepare_env () { + debug 'in prepare_env' + mkdir "$CACHEDIR" 2>/dev/null + test ! -d "$CACHEDIR" && die 5 "Could not make cache directory $CACHEDIR" +} + + +function check_dirty () { + debug 'in check_dirty' + git diff --quiet 2>/dev/null && git diff --quiet --cached 2>/dev/null || { + die 4 "Commit all your changes first!" + } +} + + +function welcome () { + debug 'in welcome' +cat <' : yes, continue +- 'n' : no +- 's' : skip (ONLY where applicable, otherwise continue) + +Any other key will abort the program. +EOF + ask 'Ready?' +} + + +function checkout_next () { + debug 'in checkout_next' + git checkout --quiet next || { + die 6 "Could not checkout 'next' branch" + } +} + + +function get_version () { + debug 'in get_version' + echo "v$(./offlineimap.py --version)" +} + + +function update_offlineimap_version () { + debug 'in update_offlineimap_version' + edit_file 'update the version in __init__.py' offlineimap/__init__.py +} + + +# +# $1: previous version +# +function get_git_history () { + debug 'in get_git_history' + git log --oneline "${1}.." | sed -r -e 's,^(.),\- \1,' +} + + + +# +# $1: new version +# $2: shortlog +function changelog_template () { + debug 'in changelog_template' + cat < "$TMP_CHANGELOG_EXCERPT" + get_git_history "$2" >> "$TMP_CHANGELOG_EXCERPT" + edit_file "the Changelog excerpt" $TMP_CHANGELOG_EXCERPT + + # Remove comments. + grep -v '//' "$TMP_CHANGELOG_EXCERPT" > "${TMP_CHANGELOG_EXCERPT}.nocomment" + mv -f "${TMP_CHANGELOG_EXCERPT}.nocomment" "$TMP_CHANGELOG_EXCERPT" + fi + + # Write new Changelog. + cat "$CHANGELOG" > "$TMP_CHANGELOG" + debug "include excerpt $TMP_CHANGELOG_EXCERPT to $TMP_CHANGELOG" + sed -i -e "/${CHANGELOG_MAGIC}/ r ${TMP_CHANGELOG_EXCERPT}" "$TMP_CHANGELOG" + debug 'remove trailing whitespaces' + sed -i -r -e 's, +$,,' "$TMP_CHANGELOG" # Remove trailing whitespaces. + debug "copy to $TMP_CHANGELOG -> $CHANGELOG" + cp -f "$TMP_CHANGELOG" "$CHANGELOG" + + # Check and edit Changelog. + ask "Next step: you'll be asked to review the diff of $CHANGELOG" + action=$No + while test ! $action -eq $Yes + do + git diff -- "$CHANGELOG" | less + ask 'edit Changelog?' $CHANGELOG + action=$? + done +} + + +# +# $1: new version +# +function git_release () { + debug 'in git_release' + git commit -as -m"$1" + git tag -a "$1" -m"$1" + git checkout master + git merge next + git checkout next +} + + + +function get_last_rc () { + git tag | grep -E '^v([0-9][\.-]){3}rc' | sort -n | tail -n1 +} + +function get_last_stable () { + git tag | grep -E '^v([0-9][\.])+' | grep -v '\-rc' | sort -n | tail -n1 +} + +function update_website_releases_info() { + cat > "$WEBSITE_LATEST" < /dev/null 2>&1 + if test ! $? -eq 0 + then + echo "Oops! you don't have $SPHINXBUILD installed?" + echo "Cannot update the webite documentation..." + echo "You should install it and run:" + echo " $ cd docs" + echo " $ make websitedoc" + echo "Then, commit and push changes of the website." + ask 'continue' + return + fi + + # Check website sources are available. + cd website + if test ! $? -eq 0 + then + echo "ERROR: cannot go to the website sources" + ask 'continue' + return + fi + # Stash any WIP in the website sources. + git diff --quiet 2>/dev/null && git diff --quiet --cached 2>/dev/null || { + echo "There is WIP in the website repository, stashing" + echo "git stash create 'WIP during offlineimap API import'" + git stash create 'WIP during offlineimap API import' + ask 'continue' + } + + cd .. # Back to offlineimap.git. + update_website_releases_info + cd "./$DOCSDIR" # Enter the docs directory in offlineimap.git. + # Build the docs! + make websitedoc && { + # Commit changes in a branch. + cd ../website # Enter the website sources. + branch_name="import-$1" + git checkout -b "$branch_name" + git add '_doc/versions' + git commit -a -s -m"update for offlineimap $1" + echo "website: branch '$branch_name' ready for a merge in master!" + } + ask 'website updated locally; continue' + fi +} + + +function git_username () { + git config --get user.name +} +function git_usermail () { + git config --get user.email +} + +# +# $1: new version +# +function announce_header () { + cat < +Date: $(git log HEAD~1.. --oneline --pretty='%cD') +From: $(git_username) <$(git_usermail)> +To: $MAILING_LIST +Subject: [ANNOUNCE] OfflineIMAP $1 released + +OfflineIMAP $1 is out. + +Downloads: + http://github.com/OfflineIMAP/offlineimap/archive/${1}.tar.gz + http://github.com/OfflineIMAP/offlineimap/archive/${1}.zip +EOF +} + + +function announce_footer () { + cat < "$TMP_ANNOUNCE" + grep -v '^### OfflineIMAP' "$TMP_CHANGELOG_EXCERPT" | \ + grep -v '^#### Notes' >> "$TMP_ANNOUNCE" + sed -i -r -e "s,^$ANNOUNCE_MAGIC,," "$TMP_ANNOUNCE" + sed -i -r -e "s,^#### ,# ," "$TMP_ANNOUNCE" + announce_footer >> "$TMP_ANNOUNCE" +} + + +function edit_announce () { + edit_file 'edit announce' "$TMP_ANNOUNCE" +} + + + +# +# run +# +function run () { + debug 'in run' + fix_pwd + check_dirty + prepare_env + checkout_next + clear + welcome + + if test -f "$TMP_CHANGELOG_EXCERPT" + then + head "$TMP_CHANGELOG_EXCERPT" + ask "A previous Changelog excerpt (head above) was found, use it?" + if test ! $? -eq $Yes + then + mv -f "$TMP_CHANGELOG_EXCERPT" "$TMP_CHANGELOG_EXCERPT_OLD" + fi + fi + + previous_version="$(get_version)" + message="Safety check: release after version:" + ask "$message $previous_version ?" + update_offlineimap_version + new_version="$(get_version)" + ask "Safety check: make a new release with version: '$new_version'" "Clear changes and restart" + + update_changelog "$new_version" "$previous_version" + build_announce "$new_version" "$previous_version" + edit_announce + + git_release $new_version + + update_website $new_version +} + +run +cat < + + +## Systemd units + +These unit files are meant to be used in the user session. You may drop them into `/etc/systemd/user` or `${XDG_DATA_HOME}/systemd/user` followed by `systemctl --user daemon-reload` to have systemd aware of the unit files. + +These files are meant to be triggered either manually using `systemctl --user start offlineimap.service` or by enabling the timer unit using `systemctl --user enable offlineimap.timer`. Additionally, specific accounts may be triggered by using `offlineimap@myaccount.timer` or `offlineimap@myaccount.service`. + +These unit files are installed as being enabled via a `mail.target` unit which is intended to be a catch-all for mail-related unit files. A simple `mail.target` file is also provided. diff --git a/contrib/systemd/mail.target b/contrib/systemd/mail.target new file mode 100644 index 0000000..5a408b2 --- /dev/null +++ b/contrib/systemd/mail.target @@ -0,0 +1,5 @@ +[Unit] +Description=Mail Target + +[Install] +WantedBy=default.target diff --git a/contrib/systemd/offlineimap.service b/contrib/systemd/offlineimap.service new file mode 100644 index 0000000..f29f93c --- /dev/null +++ b/contrib/systemd/offlineimap.service @@ -0,0 +1,9 @@ +[Unit] +Description=Offlineimap Service + +[Service] +Type=oneshot +ExecStart=/usr/bin/offlineimap -o + +[Install] +WantedBy=mail.target diff --git a/contrib/systemd/offlineimap.timer b/contrib/systemd/offlineimap.timer new file mode 100644 index 0000000..ef84a3d --- /dev/null +++ b/contrib/systemd/offlineimap.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Offlineimap Query Timer + +[Timer] +OnUnitInactiveSec=15m +Unit=offlineimap.service + +[Install] +WantedBy=mail.target diff --git a/contrib/systemd/offlineimap@.service b/contrib/systemd/offlineimap@.service new file mode 100644 index 0000000..7be965a --- /dev/null +++ b/contrib/systemd/offlineimap@.service @@ -0,0 +1,9 @@ +[Unit] +Description=Offlineimap Service for account %i + +[Service] +Type=oneshot +ExecStart=/usr/bin/offlineimap -o -a %i + +[Install] +WantedBy=mail.target diff --git a/contrib/systemd/offlineimap@.timer b/contrib/systemd/offlineimap@.timer new file mode 100644 index 0000000..6cdbac4 --- /dev/null +++ b/contrib/systemd/offlineimap@.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Offlineimap Query Timer for account %i + +[Timer] +OnUnitInactiveSec=15m +Unit=offlineimap@%i.service + +[Install] +WantedBy=mail.target diff --git a/docs/FAQ.rst b/docs/FAQ.rst deleted file mode 100644 index 9b66e29..0000000 --- a/docs/FAQ.rst +++ /dev/null @@ -1,384 +0,0 @@ -.. -*- coding: utf-8 -*- - -.. NOTE TO MAINTAINERS: Please add new questions to the end of their - sections, so section/question numbers remain stable. - - -============================================= - OfflineIMAP FAQ (Frequently Asked Questions) -============================================= - -:Web site: https://github.com/nicolas33/offlineimap -:Copyright: This document is licensed under GPLv2. - -.. contents:: -.. sectnum:: - - -This is a work in progress. - -Please feel free to ask questions and/or provide answers; send email to the -`mailing list`_. - -.. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project -.. _OfflineIMAP: https://github.com/nicolas33/offlineimap - - -OfflineIMAP -=========== - -Where do I get OfflineIMAP? ---------------------------- - -See the information on the Home page `OfflineIMAP`_. - -How fast is it? ---------------- - -OfflineIMAP has a multithreaded sync, so it should have very nice performance. - -OfflineIMAP versions 2.0 and above contain a multithreaded system. A good way -to experiment is by setting maxsyncaccounts to 3 and maxconnections to 3 in -each account clause. - -This lets OfflineIMAP open up multiple connections simultaneously. That will -let it process multiple folders and messages at once. In most cases, this will -increase performance of the sync. - -Don’t set the number too high. If you do that, things might actually slow down -as your link gets saturated. Also, too many connections can cause mail servers -to have excessive load. Administrators might take unkindly to this, and the -server might bog down. There are many variables in the optimal setting; -experimentation may help. - -An informal benchmark yields these results for my setup:: - - * 10 minutes with MacOS X Mail.app “manual cache” - * 5 minutes with GNUS agent sync - * 20 seconds with OfflineIMAP 1.x - * 9 seconds with OfflineIMAP 2.x - * 3 seconds with OfflineIMAP 3.x “cold start” - * 2 seconds with OfflineIMAP 3.x “held connection” - -What platforms does OfflineIMAP support? ----------------------------------------- - -It should run on most platforms supported by Python, which are quite a few. I -do not support Windows myself, but some have made it work there. Use on -Windows - -These answers have been reported by OfflineIMAP users. I do not run OfflineIMAP -on Windows myself, so I can’t directly address their accuracy. - -The basic answer is that it’s possible and doesn’t require hacking OfflineIMAP -source code. However, it’s not necessarily trivial. The information below is -based in instructions submitted by Chris Walker. - -First, you must run OfflineIMAP in the Cygwin environment. The Windows -filesystem is not powerful enough to accomodate Maildir by itself. - -Next, you’ll need to mount your Maildir directory in a special way. There is -information for doing that at http://barnson.org/node/295. That site gives this -example:: - - mount -f -s -b -o managed "d:/tmp/mail" "/home/of/mail" - -That URL also has more details on making OfflineIMAP work with Windows. - - -Does OfflineIMAP support mbox, mh, or anything else other than Maildir? ------------------------------------------------------------------------ - -Not directly. Maildir was the easiest to implement. I’m not planning to write -mbox code for OfflineIMAP, though if someone sent me well-written mbox support -and pledged to support it, I’d commit it to the tree. - -However, OfflineIMAP can directly sync accounts on two different IMAP servers -together. So you could install an IMAP server on your local machine that -supports mbox, sync to it, and then instruct your mail readers to use the -mboxes. - -Or you could install whatever IMAP server you like on the local machine, and -point your mail readers to that IMAP server on localhost. - -What is the UID validity problem for folder? --------------------------------------------- - -IMAP servers use a unique ID (UID) to refer to a specific message. This number -is guaranteed to be unique to a particular message forever. No other message in -the same folder will ever get the same UID. UIDs are an integral part of -`OfflineIMAP`_'s synchronization scheme; they are used to match up messages on -your computer to messages on the server. - -Sometimes, the UIDs on the server might get reset. Usually this will happen if -you delete and then recreate a folder. When you create a folder, the server -will often start the UID back from 1. But `OfflineIMAP`_ might still have the -UIDs from the previous folder by the same name stored. `OfflineIMAP`_ will -detect this condition and skip the folder. This is GOOD, because it prevents -data loss. - -You can fix it by removing your local folder and cache data. For instance, if -your folders are under `~/Folders` and the folder with the problem is INBOX, -you'd type this:: - - rm -r ~/Folders/INBOX - rm -r ~/.offlineimap/Account-AccountName/LocalStatus/INBOX - rm -r ~/.offlineimap/Repository-RemoteRepositoryName/FolderValidity/INBOX - -(Of course, replace AccountName and RemoteRepositoryName with the names as -specified in `~/.offlineimaprc`). - -Next time you run `OfflineIMAP`_, it will re-download the folder with the new -UIDs. Note that the procedure specified above will lose any local changes made -to the folder. - -Some IMAP servers are broken and do not support UIDs properly. If you continue -to get this error for all your folders even after performing the above -procedure, it is likely that your IMAP server falls into this category. -`OfflineIMAP`_ is incompatible with such servers. Using `OfflineIMAP`_ with -them will not destroy any mail, but at the same time, it will not actually -synchronize it either. (`OfflineIMAP`_ will detect this condition and abort -prior to synchronization.) - - -This question comes up frequently on the `mailing list`_. You can find a detailed -discussion of the problem there -http://lists.complete.org/offlineimap@complete.org/2003/04/msg00012.html.gz. - -How do I add or delete a folder? --------------------------------- - -OfflineIMAP does not currently provide this feature. However, if you create a -new folder on the remote server, OfflineIMAP will detect this and create the -corresponding folder locally automatically. - -May I delete local folders? ---------------------------- - -`OfflineIMAP`_ does a two-way synchronization. That is, if you make a change -to the mail on the server, it will be propagated to your local copy, and -vise-versa. Some people might think that it would be wise to just delete all -their local mail folders periodically. If you do this with `OfflineIMAP`_, -remember to also remove your local status cache (`~/.offlineimap` by default). -Otherwise, `OfflineIMAP`_ will take this as an intentional deletion of many -messages and will interpret your action as requesting them to be deleted from -the server as well. (If you don't understand this, don't worry; you probably -won't encounter this situation.) - -Can I run multiple instances? ------------------------------ - -`OfflineIMAP`_ is not designed to have several instances (for instance, a cron -job and an interactive invocation) run over the same mailbox simultaneously. -It will perform a check on startup and abort if another `OfflineIMAP`_ is -already running. If you need to schedule synchronizations, you'll probably -find autorefresh settings more convenient than cron. Alternatively, you can -set a separate metadata directory for each instance. - -Can I copy messages between folders? ---------------------------------------- - -Normally, when you copy a message between folders or add a new message to a -folder locally, `OfflineIMAP`_ will just do the right thing. However, -sometimes this can be tricky ― if your IMAP server does not provide the SEARCH -command, or does not return something useful, `OfflineIMAP`_ cannot determine -the new UID of the message. So, in these rare instances, OfflineIMAP will -upload the message to the IMAP server and delete it from your local folder. -Then, on your next sync, the message will be re-downloaded with the proper UID. -`OfflineIMAP`_ makes sure that the message was properly uploaded before -deleting it, so there should be no risk of data loss. - -Does OfflineIMAP support POP? ------------------------------ - -No. POP is not robust enough to do a completely reliable multi-machine sync -like OfflineIMAP can do. - -OfflineIMAP will never support POP. - -How is OfflineIMAP conformance? -------------------------------- - -* Internet Message Access Protocol version 4rev1 (IMAP 4rev1) as specified in - `2060`:RFC: and `3501`:RFC: -* CRAM-MD5 as specified in `2195`:RFC: -* Maildir as specified in the Maildir manpage and the qmail website -* Standard Python 2.6 as implemented on POSIX-compliant systems - - -Configuration Questions -======================= - -Can I synchronize multiple accounts with OfflineIMAP? ------------------------------------------------------ - -Of course! - -Just name them all in the accounts line in the general section of the -configuration file, and add a per-account section for each one. - -You can also optionally use the -a option when you run OfflineIMAP to request -that it only operate upon a subset of the accounts for a particular run. - -How do I specify the names of folders? --------------------------------------- - -You do not need to. OfflineIMAP is smart enough to automatically figure out -what folders are present on the IMAP server and synchronize them. You can use -the folderfilter and nametrans configuration file options to request only -certain folders and rename them as they come in if you like. - -How do I prevent certain folders from being synced? ---------------------------------------------------- - -Use the folderfilter option. - -What is the mailbox name recorder (mbnames) for? ------------------------------------------------- - -Some mail readers, such as mutt, are not capable of automatically determining the names of your mailboxes. OfflineIMAP can help these programs by writing the names of the folders in a format you specify. See the example offlineimap.conf for details. - -IMAP Server Notes -================= - -In general, OfflineIMAP works with any IMAP server that provides compatibility -with the IMAP RFCs. Some servers provide imperfect compatibility that may be -good enough for general clients. OfflineIMAP needs more features, specifically -support for UIDs, in order to do its job accurately and completely. - -Microsoft Exchange ------------------- - -Several users have reported problems with Microsoft Exchange servers in -conjunction with OfflineIMAP. This generally seems to be related to the -Exchange servers not properly following the IMAP standards. - -Mark Biggers has posted some information to the OfflineIMAP `mailing list`_ -about how he made it work. - -Other users have indicated that older (5.5) releases of Exchange are so bad -that they will likely not work at all. - -I do not have access to Exchange servers for testing, so any problems with it, -if they can even be solved at all, will require help from OfflineIMAP users to -find and fix. - - -Client Notes -============ - -What clients does OfflineIMAP work with? ----------------------------------------- - -Any client that supports Maildir. Popular ones include mutt, Evolution and -KMail. Thunderbird does not have maildir suppport. - -With OfflineIMAP’s IMAP-to-IMAP syncing, this can be even wider; see the next -question. - -Evolution ---------- - -OfflineIMAP can work with Evolution. To do so, first configure your OfflineIMAP -account to have sep = / in its configuration. Then, configure Evolution with -the “Maildir-format mail directories” server type. For the path, you will need -to specify the name of the top-level folder inside your OfflineIMAP storage -location. You’re now set! - -KMail ------ - -At this time, I believe that OfflineIMAP with Maildirs is not compatible with -KMail. KMail cannot work in any mode other than to move all messages out of all -folders immediately, which (besides being annoying and fundamentally broken) is -incompatible with OfflineIMAP. - -However, I have made KMail version 3 work well with OfflineIMAP by installing -an IMAP server on my local machine, having OfflineIMAP sync to that, and -pointing KMail at the same server. - -Another way to see mails downloaded with offlineimap in KMail (KDE4) is to -create a local folder (e.g. Backup) and then use ``ln -s -localfolders_in_offlineimaprc ~/.kde/share/apps/kmail/mail/.Backup.directory``. -Maybe you have to rebuild the index of the new folder. Works well with KMail -1.11.4 (KDE4.x), offlineimap 6.1.2 and ArchLinux and sep = / in .offlineimaprc. - -Mutt ----- - -* Do I need to use set maildir_trash? - -Other IMAP sync programs require you to do this. OfflineIMAP does not. You’ll -get the best results without it, in fact, though turning it on won’t hurt -anything. - -* How do I set up mbnames with mutt? - -The example offlineimap.conf file has this example. In your offlineimap.conf, -you’ll list this:: - - [mbnames] - enabled = yes - filename = ~/Mutt/muttrc.mailboxes - header = "mailboxes " - peritem = "+%(accountname)s/%(foldername)s" - sep = " " - footer = "\n" - -Then in your ``.muttrc``:: - - source ~/Mutt/muttrc.mailboxes - - -You might also want to set:: - - set mbox_type=Maildir - set folder=$HOME/Maildirpath - -The OfflineIMAP manual has a more detailed example for doing this for multiple -accounts. - -Miscellaneous Questions -======================= - -Why are your Maildir message filenames so long? ------------------------------------------------ - -OfflineIMAP has two relevant principles: 1) never modifying your messages in -any way and 2) ensuring 100% reliable synchronizations. In order to do a -reliable sync, OfflineIMAP must have a way to uniquely identify each e-mail. -Three pieces of information are required to do this: your account name, the -folder name, and the message UID. The account name can be calculated from the -path in which your messages are. The folder name can usually be as well, BUT -some mail clients move messages between folders by simply moving the file, -leaving the name intact. - -So, OfflineIMAP must store both a message UID and a folder ID. The -folder ID is necessary so OfflineIMAP can detect a message being moved -to a different folder. OfflineIMAP stores the UID (U= number) and an -md5sum of the foldername (FMD5= number) to facilitate this. - - -What can I do to ensure OfflineIMAP is still running and hasn’t crashed? ------------------------------------------------------------------------- - -This shell script will restart OfflineIMAP if it has crashed. Sorry, its -written in Korn, so you’ll need ksh, pdksh, or mksh to run it:: - - #!/bin/ksh - # remove any old instances of this shell script or offlineimap - for pid in $(pgrep offlineimap) - do - if $pid -ne $$ - then - kill $pid - fi - done - - # wait for compiz (or whatever) to start and setup wifi - sleep 20 - # If offlineimap exits, restart it - while true - do - ( exec /usr/bin/offlineimap -u Noninteractive.Quiet ) - sleep 60 # prevents extended failure condition diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst deleted file mode 100644 index bf0fa80..0000000 --- a/docs/INSTALL.rst +++ /dev/null @@ -1,97 +0,0 @@ -.. -*- coding: utf-8 -*- - -.. _OfflineIMAP: https://github.com/nicolas33/offlineimap - -.. contents:: -.. sectnum:: - -============= -Prerequisites -============= - -In order to use `OfflineIMAP`_, you need to have these conditions satisfied: - -1. Your mail server must support IMAP. Most Internet Service Providers and - corporate networks do, and most operating systems have an IMAP implementation - readily available. A special Gmail mailbox type is available to interface with - Gmail's IMAP front-end. - -2. You must have Python version 2.6 or above installed. If you are running on - Debian GNU/Linux, this requirement will automatically be taken care of for you. - If you do not have Python already, check with your system administrator or - operating system vendor; or, download it from the Python website. If you intend - to use the SSL interface, your Python must have been built with SSL support. - -3. Have a mail reader that supports the Maildir mailbox format. Most modern - mail readers have this support built-in, so you can choose from a wide variety - of mail servers. This format is also known as the "qmail" format, so any mail - reader compatible with it will work with `OfflineIMAP`_. If you do not have a - mail reader that supports Maildir, you can often install a local IMAP server and - point both `OfflineIMAP`_ and your mail reader at it. - - -============ -Installation -============ - -You have three options: - -1. a system-wide installation with Debian -2. a system-wide installation with other systems -3. a single-user installation. You can checkout the latest version of - `OfflineIMAP`_ from official `OfflineIMAP`_ repository. - - -System-Wide Installation, Debian -================================ - -If you are tracking Debian unstable, you may install `OfflineIMAP`_ by simply -running the following command as root:: - - apt-get install offlineimap - -If you are not tracking Debian unstable, download the Debian `.deb` package from -the `OfflineIMAP`_ website and then run ``dpkg -i`` to install the downloaded -package. Then, skip to below. You will type offlineimap to invoke the -program. - -System-Wide Installation, Other -=============================== - -Download the tar.gz version of the package from the website. Then run these -commands, making sure that you are the "root" user first:: - - tar -zxvf offlineimap_x.y.z.tar.gz - cd offlineimap-x.y.z - python2.2 setup.py install - -On some systems, you will need to use python instead of python2.6. Next, -proceed to below. You will type offlineimap to invoke the program. - -Single-Account Installation -=========================== - -Download the tar.gz version of the package from the website. Then run these -commands:: - - tar -zxvf offlineimap_x.y.z.tar.gz - cd offlineimap-x.y.z - -When you want to run `OfflineIMAP`_, you will issue the cd command as above and -then type `./offlineimap.py`; there is no installation step necessary. - -============= -Configuration -============= - -`OfflineIMAP`_ is regulated by a configuration file that is normally stored in -`~/.offlineimaprc`. `OfflineIMAP`_ ships with a file named `offlineimap.conf` -that you should copy to that location and then edit. This file is vital to -proper operation of the system; it sets everything you need to run -`OfflineIMAP`_. Full documentation for the configuration file is included -within the sample file. - - -`OfflineIMAP`_ also ships a file named `offlineimap.conf.minimal` that you can -also try. It's useful if you want to get started with the most basic feature -set, and you can read about other features later with `offlineimap.conf`. diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst deleted file mode 100644 index 67fa383..0000000 --- a/docs/MANUAL.rst +++ /dev/null @@ -1,280 +0,0 @@ -==================== - OfflineIMAP Manual -==================== - --------------------------------------------------------- -Powerful IMAP/Maildir synchronization and reader support --------------------------------------------------------- - -:Author: John Goerzen -:Date: 2011-01-15 -:Copyright: GPL v2 -:Manual section: 1 - -.. TODO: :Manual group: - - -SYNOPSIS -======== - - offlineimap [-h|--help] - - offlineimap [OPTIONS] - -| -1 -| -P profiledir -| -a accountlist -| -c configfile -| -d debugtype[,...] -| -f foldername[,...] -| -k [section:]option=value -| -l filename -| -o -| -u interface - - -DESCRIPTION -=========== - -Most configuration is done via the configuration file. Nevertheless, there are -a few command-line options that you may set for OfflineIMAP. - - -OPTIONS -======= - - - - - --1 Disable most multithreading operations - - Use solely a single-connection sync. This effectively sets the - maxsyncaccounts and all maxconnections configuration file variables to 1. - - --P profiledir - - Sets OfflineIMAP into profile mode. The program will create profiledir (it - must not already exist). As it runs, Python profiling information about each - thread is logged into profiledir. Please note: This option is present for - debugging and optimization only, and should NOT be used unless you have a - specific reason to do so. It will significantly slow program performance, may - reduce reliability, and can generate huge amounts of data. You must use the - -1 option when you use -P. - - --a accountlist - - Overrides the accounts option in the general section of the configuration - file. You might use this to exclude certain accounts, or to sync some - accounts that you normally prefer not to. Separate the accounts by commas, - and use no embedded spaces. - - --c configfile - - Specifies a configuration file to use in lieu of the default, - ``~/.offlineimaprc``. - - --d debugtype[,...] - - Enables debugging for OfflineIMAP. This is useful if you are trying to track - down a malfunction or figure out what is going on under the hood. I suggest - that you use this with -1 to make the results more sensible. - - -d requires one or more debugtypes, separated by commas. These define what - exactly will be debugged, and include three options: imap, maildir, and - thread. The imap option will enable IMAP protocol stream and parsing - debugging. Note that the output may contain passwords, so take care to remove - that from the debugging output before sending it to anyone else. The maildir - option will enable debugging for certain Maildir operations. And thread will - debug the threading model. - - --f foldername[,foldername] - - Only sync the specified folders. The foldernames are the untranslated - foldernames. This command-line option overrides any folderfilter and - folderincludes options in the configuration file. - - --k [section:]option=value - - Override configuration file option. If "section" is omitted, it defaults to - general. Any underscores "_" in the section name are replaced with spaces: - for instance, to override option autorefresh in the "[Account Personal]" - section in the config file one would use "-k Account_Personal:autorefresh=30". - You may give more than one -k on the command line if you wish. - - --l filename - - Enables logging to filename. This will log everything that goes to the screen - to the specified file. Additionally, if any debugging is specified with -d, - then debug messages will not go to the screen, but instead to the logfile - only. - - --o Run only once, - - ignoring all autorefresh settings in the configuration file. - - --q Run only quick synchronizations. - - Ignore any flag updates on IMAP servers. - - --h|--help Show summary of options. - - --u interface - - Specifies an alternative user interface module to use. This overrides the - default specified in the configuration file. The pre-defined options are - listed in the User Interfaces section. - - -User Interfaces -=============== - -OfflineIMAP has a pluggable user interface system that lets you choose how the -program communicates information to you. There are two graphical interfaces, -two terminal interfaces, and two noninteractive interfaces suitable for -scripting or logging purposes. The ui option in the configuration file -specifies user interface preferences. The -u command-line option can override -the configuration file setting. The available values for the configuration file -or command-line are described in this section. - - -Curses.Blinkenlights --------------------- - -Curses.Blinkenlights is an interface designed to be sleek, fun to watch, and -informative of the overall picture of what OfflineIMAP is doing. I consider it -to be the best general-purpose interface in OfflineIMAP. - - -Curses.Blinkenlights contains a row of "LEDs" with command buttons and a log. -The log shows more detail about what is happening and is color-coded to match -the color of the lights. - - -Each light in the Blinkenlights interface represents a thread of execution -- -that is, a particular task that OfflineIMAP is performing right now. The colors -indicate what task the particular thread is performing, and are as follows: - -* Black: - indicates that this light's thread has terminated; it will light up again - later when new threads start up. So, black indicates no activity. - -* Red (Meaning 1): - is the color of the main program's thread, which basically does nothing but - monitor the others. It might remind you of HAL 9000 in 2001. - -* Gray: - indicates that the thread is establishing a new connection to the IMAP - server. - -* Purple: - is the color of an account synchronization thread that is monitoring the - progress of the folders in that account (not generating any I/O). - -* Cyan: - indicates that the thread is syncing a folder. - -* Green: - means that a folder's message list is being loaded. - -* Blue: - is the color of a message synchronization controller thread. - -* Orange: - indicates that an actual message is being copied. (We use fuchsia for fake - messages.) - -* Red (meaning 2): - indicates that a message is being deleted. - -* Yellow / bright orange: - indicates that message flags are being added. - -* Pink / bright red: - indicates that message flags are being removed. - -* Red / Black Flashing: - corresponds to the countdown timer that runs between synchronizations. - - -The name of this interfaces derives from a bit of computer history. Eric -Raymond's Jargon File defines blinkenlights, in part, as: - - Front-panel diagnostic lights on a computer, esp. a dinosaur. Now that - dinosaurs are rare, this term usually refers to status lights on a modem, - network hub, or the like. - -This term derives from the last word of the famous blackletter-Gothic sign in -mangled pseudo-German that once graced about half the computer rooms in the -English-speaking world. One version ran in its entirety as follows: - -| ACHTUNG! ALLES LOOKENSPEEPERS! -| -| Das computermachine ist nicht fuer gefingerpoken und mittengrabben. -| Ist easy schnappen der springenwerk, blowenfusen und poppencorken -| mit spitzensparken. Ist nicht fuer gewerken bei das dumpkopfen. -| Das rubbernecken sichtseeren keepen das cotten-pickenen hans in das -| pockets muss; relaxen und watchen das blinkenlichten. - - -TTY.TTYUI ---------- - -TTY.TTYUI interface is for people running in basic, non-color terminals. It -prints out basic status messages and is generally friendly to use on a console -or xterm. - - -Noninteractive.Basic --------------------- - -Noninteractive.Basic is designed for situations in which OfflineIMAP will be run -non-attended and the status of its execution will be logged. You might use it, -for instance, to have the system run automatically and e-mail you the results of -the synchronization. This user interface is not capable of reading a password -from the keyboard; account passwords must be specified using one of the -configuration file options. - - -Noninteractive.Quiet --------------------- - -Noninteractive.Quiet is designed for non-attended running in situations where -normal status messages are not desired. It will output nothing except errors -and serious warnings. Like Noninteractive.Basic, this user interface is not -capable of reading a password from the keyboard; account passwords must be -specified using one of the configuration file options. - - -Machine.MachineUI ------------------ - -Machine.MachineUI generates output in a machine-parsable format. It is designed -for other programs that will interface to OfflineIMAP. - - -KNOWN BUGS -========== - -* SSL3 write pending: - users enabling SSL may hit a bug about "SSL3 write pending". If so, the - account(s) will stay unsynchronised from the time the bug appeared. Running - OfflineIMAP again can help. We are still working on this bug. Patches or - detailed bug reports would be appreciated. Please check you're running the - last stable version and send us a report to the mailing list including the - full log. - - -SEE ALSO -======== diff --git a/docs/Makefile b/docs/Makefile index fff639d..aadfd66 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,22 +5,38 @@ SOURCES = $(wildcard *.rst) HTML_TARGETS = $(patsubst %.rst,%.html,$(SOURCES)) RM = rm -RST2HTML=`type rst2html 2>/dev/null 2>&1 && echo rst2html || echo rst2html.py` -RST2MAN=`type rst2man 2>/dev/null 2>&1 && echo rst2man || echo rst2man.py` +RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` +RST2MAN=`type rst2man >/dev/null 2>&1 && echo rst2man || echo rst2man.py` +SPHINXBUILD = sphinx-build -all: html +docs: man api html: $(HTML_TARGETS) $(HTML_TARGETS): %.html : %.rst $(RST2HTML) $? $@ -man: offlineimap.1 +man: offlineimap.1 offlineimapui.7 -offlineimap.1: MANUAL.rst - $(RST2MAN) MANUAL.rst offlineimap.1 - cp -f offlineimap.1 .. +offlineimap.1: offlineimap.txt + a2x -v -f manpage $? + +offlineimapui.7: offlineimapui.txt + a2x -v -f manpage $? + +api: + $(SPHINXBUILD) -b html -d html/doctrees doc-src html + +websitedoc: + ./website-doc.sh releases + ./website-doc.sh api + ./website-doc.sh contrib clean: $(RM) -f $(HTML_TARGETS) - $(RM) -f offlineimap.1 ../offlineimap.1 + $(RM) -f offlineimap.1 + $(RM) -f offlineimap.7 + $(RM) -rf html/* + -find ./docs -name '*.html' -exec rm -f {} \; + +.PHONY: clean doc diff --git a/docs/UPGRADE.rst b/docs/UPGRADE.rst deleted file mode 100644 index 169ce36..0000000 --- a/docs/UPGRADE.rst +++ /dev/null @@ -1,26 +0,0 @@ - - -Upgrading to 4.0 ----------------- - -If you are upgrading from a version of OfflineIMAP prior to 3.99.12, you will -find that you will get errors when OfflineIMAP starts up (relating to -ConfigParser or AccountHashGenerator) and the configuration file. This is -because the config file format had to change to accommodate new features in 4.0. -Fortunately, it's not difficult to adjust it to suit. - - -First thing you need to do is stop any running OfflineIMAP instance, making sure -first that it's synced all your mail. Then, modify your `~/.offlineimaprc` file. -You'll need to split up each account section (make sure that it now starts with -"Account ") into two Repository sections (one for the local side and another for -the remote side.) See the files offlineimap.conf.minimal and offlineimap.conf -in the distribution if you need more assistance. - - -OfflineIMAP's status directory area has also changed. Therefore, you should -delete everything in `~/.offlineimap` as well as your local mail folders. - - -When you start up OfflineIMAP 4.0, it will re-download all your mail from the -server and then you can continue using it like normal. diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst new file mode 100644 index 0000000..de8b500 --- /dev/null +++ b/docs/doc-src/API.rst @@ -0,0 +1,86 @@ +.. OfflineImap API documentation + +.. currentmodule:: offlineimap + +.. _API docs: + +:mod:`offlineimap's` API documentation +====================================== + +Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the +high-level functionality. The rest of the classes should usually not needed to +be touched by the user. Email repositories are represented by a +:class:`offlineimap.repository.Base.BaseRepository` or derivatives (see +:mod:`offlineimap.repository` for details). A folder within a repository is +represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative +from :mod:`offlineimap.folder`. + +This page contains the main API overview of OfflineImap |release|. + +OfflineImap can be imported as:: + + from offlineimap import OfflineImap + + +:mod:`offlineimap` -- The OfflineImap module +============================================= + +.. module:: offlineimap + +.. autoclass:: offlineimap.OfflineImap(cmdline_opts = None) + :members: + :inherited-members: + :undoc-members: + :private-members: + + +:class:`offlineimap.account` +============================ + +An :class:`accounts.Account` connects two email repositories that are to be +synced. It comes in two flavors, normal and syncable. + +.. autoclass:: offlineimap.accounts.Account + +.. autoclass:: offlineimap.accounts.SyncableAccount + :members: + :inherited-members: + + .. autodata:: ui + + Contains the current :mod:`offlineimap.ui`, and can be used for logging etc. + +:exc:`OfflineImapError` -- A Notmuch execution error +-------------------------------------------------------- + +.. autoexception:: offlineimap.error.OfflineImapError + :members: + + This exception inherits directly from :exc:`Exception` and is raised + on errors during the offlineimap execution. It has an attribute + `severity` that denotes the severity level of the error. + + +:mod:`offlineimap.globals` -- module with global variables +========================================================== + +Module `offlineimap.globals` provides the read-only storage +for the global variables. + +All exported module attributes can be set manually, but this practice +is highly discouraged and shouldn't be used. +However, attributes of all stored variables can only be read, write +access to them is denied. + +Currently, we have only :attr:`options` attribute that holds +command-line options as returned by OptionParser. +The value of :attr:`options` must be set by :func:`set_options` +prior to its first use. + +.. automodule:: offlineimap.globals + :members: + + .. data:: options + + You can access the values of stored options using the usual + syntax, offlineimap.globals.options. diff --git a/docs/doc-src/conf.py b/docs/doc-src/conf.py new file mode 100644 index 0000000..0886d4a --- /dev/null +++ b/docs/doc-src/conf.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- +# +# pyDNS documentation build configuration file, created by +# sphinx-quickstart on Tue Feb 2 10:00:47 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../..')) + +from offlineimap import __version__, __bigversion__, __author__, __copyright__ +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] +autoclass_content = "both" + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'OfflineIMAP' +copyright = __copyright__ + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = __version__ +# The full version, including alpha/beta/rc tags. +release = __bigversion__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = False + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' +#html_style = '' +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['html'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +html_use_modindex = False + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'dev-doc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'offlineimap.tex', u'OfflineIMAP Documentation', + u'OfflineIMAP contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/doc-src/dco.rst b/docs/doc-src/dco.rst new file mode 100644 index 0000000..ad8a3b0 --- /dev/null +++ b/docs/doc-src/dco.rst @@ -0,0 +1,69 @@ +.. _dco + +Developer's Certificate of Origin +================================= + +v1.1:: + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. + + +Then, you just add a line saying:: + + Signed-off-by: Random J Developer + +This line can be automatically added by git if you run the git-commit command +with the ``-s`` option. Signing can made be afterword with ``--amend -s``. + +Notice that you can place your own ``Signed-off-by:`` line when forwarding +somebody else's patch with the above rules for D-C-O. Indeed you are encouraged +to do so. Do not forget to place an in-body ``From:`` line at the beginning to +properly attribute the change to its true author (see above). + +Also notice that a real name is used in the ``Signed-off-by:`` line. Please +don't hide your real name. + +If you like, you can put extra tags at the end: + +Reported-by + is used to to credit someone who found the bug that the patch attempts to fix. + +Acked-by + says that the person who is more familiar with the area the patch attempts to + modify liked the patch. + +Reviewed-by + unlike the other tags, can only be offered by the reviewer and means that she + is completely satisfied that the patch is ready for application. It is + usually offered only after a detailed review. + +Tested-by + is used to indicate that the person applied the patch and found it to have the + desired effect. + +You can also create your own tag or use one that's in common usage such as +``Thanks-to:``, ``Based-on-patch-by:``, or ``Mentored-by:``. + + diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst new file mode 100644 index 0000000..e923a64 --- /dev/null +++ b/docs/doc-src/index.rst @@ -0,0 +1,21 @@ +.. OfflineImap documentation master file +.. _OfflineIMAP: http://offlineimap.org + + +Welcome to OfflineIMAP's developer documentation +================================================ + +**License** + :doc:`dco` (dco) + +**Documented APIs** + +.. toctree:: + API + repository + ui + + +.. moduleauthor:: John Goerzen, and many others. See AUTHORS and the git history for a full list. + +:License: This module is covered under the GNU GPL v2 (or later). diff --git a/docs/doc-src/repository.rst b/docs/doc-src/repository.rst new file mode 100644 index 0000000..1594a68 --- /dev/null +++ b/docs/doc-src/repository.rst @@ -0,0 +1,68 @@ +.. currentmodule:: offlineimap.repository + +:mod:`offlineimap.repository` -- Email repositories +------------------------------------------------------------ + +A derivative of class +:class:`Base.BaseRepository` represents an email +repository depending on the type of storage, possible options are: + + * :class:`IMAPRepository`, + * :class:`MappedIMAPRepository` + * :class:`GmailRepository`, + * :class:`MaildirRepository`, or + * :class:`LocalStatusRepository`. + +Which class you need depends on your account +configuration. The helper class :class:`offlineimap.repository.Repository` is +an *autoloader*, that returns the correct class depending +on your configuration. So when you want to instanciate a new +:mod:`offlineimap.repository`, you will mostly do it through this class. + +.. autoclass:: offlineimap.repository.Repository + :members: + :inherited-members: + + + +:mod:`offlineimap.repository.Base.BaseRepository` -- Representation of a mail repository +------------------------------------------------------------------------------------------ +.. autoclass:: offlineimap.repository.Base.BaseRepository + :members: + :inherited-members: + :undoc-members: + +.. .. note:: :meth:`foo` +.. .. attribute:: Database.MODE + + Defines constants that are used as the mode in which to open a database. + + MODE.READ_ONLY + Open the database in read-only mode + + MODE.READ_WRITE + Open the database in read-write mode + +.. autoclass:: offlineimap.repository.IMAPRepository +.. autoclass:: offlineimap.repository.MappedIMAPRepository +.. autoclass:: offlineimap.repository.GmailRepository +.. autoclass:: offlineimap.repository.MaildirRepository +.. autoclass:: offlineimap.repository.LocalStatusRepository + +:mod:`offlineimap.folder` -- Basic representation of a local or remote Mail folder +--------------------------------------------------------------------------------------------------------- + +.. autoclass:: offlineimap.folder.Base.BaseFolder + :members: + :inherited-members: + :undoc-members: + +.. .. attribute:: Database.MODE + + Defines constants that are used as the mode in which to open a database. + + MODE.READ_ONLY + Open the database in read-only mode + + MODE.READ_WRITE + Open the database in read-write mode diff --git a/docs/doc-src/ui.rst b/docs/doc-src/ui.rst new file mode 100644 index 0000000..2931010 --- /dev/null +++ b/docs/doc-src/ui.rst @@ -0,0 +1,30 @@ +:mod:`offlineimap.ui` -- A flexible logging system +-------------------------------------------------------- + +.. currentmodule:: offlineimap.ui + +OfflineImap has various ui systems, that can be selected. They offer various +functionalities. They must implement all functions that the +:class:`offlineimap.ui.UIBase` offers. Early on, the ui must be set using +:meth:`getglobalui` + +.. automethod:: offlineimap.ui.setglobalui +.. automethod:: offlineimap.ui.getglobalui + +Base UI plugin +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: offlineimap.ui.UIBase.UIBase + :members: + :inherited-members: + +.. .. note:: :meth:`foo` +.. .. attribute:: Database.MODE + + Defines constants that are used as the mode in which to open a database. + + MODE.READ_ONLY + Open the database in read-only mode + + MODE.READ_WRITE + Open the database in read-write mode diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt new file mode 100644 index 0000000..618f2ab --- /dev/null +++ b/docs/offlineimap.txt @@ -0,0 +1,425 @@ + +offlineimap(1) +============== + +NAME +---- +offlineimap - Synchronize mailboxes and Maildirs + +SYNOPSIS +-------- +[verse] +'offlineimap' (options) + +DESCRIPTION +----------- + +Synchronize the accounts configured in the configuration file via IMAP. Each +account has two sides. + +One of the side must be an IMAP server. +The other side can either be a Maildir or another IMAP server. + + +OPTIONS +------- + +-h:: +--help:: + + Display summary of options. + +--version:: + + Output version. + +--dry-run:: + + Run in dry run mode. ++ +Do not actually modify any store but check and print what synchronization +actions would be taken if a sync would be performed. It will not precisely +give the exact information what will happen. If e.g. we need to create a +folder, it merely outputs 'Would create folder X', but not how many and which +mails it would transfer. + + +--info:: + + Output information on the configured email repositories. ++ +Useful for debugging and bug reporting. Use in conjunction with the -a option +to limit the output to a single account. This mode will prevent any actual +sync to occur and exits after it outp ut the debug information. + + +-1:: + + Limit multithreading operations and run solely a single-thread sync. ++ +This effectively sets the maxsyncaccounts and all maxconnections configuration +file variables to 1. This is 1, the number. + + +-P :: + + Set OfflineIMAP into profile mode. ++ +The program will create DIR (it must not already exist). As it runs, Python +profiling information about each thread is logged into profiledir. Please +note: This option is present for debugging and optimization only, and should +NOT be used unless you have a specific reason to do so. It will significantly +decrease program performance, may reduce reliability, and can generate huge +amounts of data. This option implies the -1 option. + + +-a :: + + Overrides the accounts section in the config file. ++ +Allows to specify a particular account or set of accounts to sync without +having to edit the config file. + + +-c :: + + Specifies a configuration file to use. + + +-d :: + + Enables debugging for OfflineIMAP. ++ +This is useful if you are to track down a malfunction or figure out what is +going on under the hood. This option requires one or more debugtypes, +separated by commas. These define what exactly will be debugged, and so far +include two options: imap, thread, maildir or ALL. The imap option will enable +IMAP protocol stream and parsing debugging. Note that the output may contain +passwords, so take care to remove that from the debugging output before +sending it to anyone else. The maildir option will enable debugging for +certain Maildir operations. The use of any debug option (unless 'thread' is +included), implies the single-thread option -1. + + +-l :: + + Send logs to . + + +-f :: + + Only sync the specified folders. ++ +The folder names are the untranslated foldernames of the remote repository. +This command-line option overrides any 'folderfilter' and 'folderincludes' +options in the configuration file. + +-k <[section:]option=value:: + + Override any configuration file option. ++ +If "section" is omitted, it defaults to "general". Any underscores in the +section name are replaced with spaces: for instance, to override option +"autorefresh" in the "[Account Personal]" section in the config file one would +use "-k Account_Personal:autorefresh=30". Repeat this option as much as +necessary to redefine multiple options. + +-o:: + + Run only once. ++ +Ignore any autorefresh setting in the configuration file. + +-q:: + + Run only quick synchronizations. ++ +Ignore any flag updates on IMAP servers. If a flag on the remote IMAP changes, +and we have the message locally, it will be left untouched in a quick run. This +option is ignored if maxage is set. + + +-u :: + + Specifies an alternative user interface to use. ++ +This overrides the default specified in the configuration file. The UI +specified with -u will be forced to be used, even if checks determine that it +is not usable. Possible interface choices are: quiet, basic, ttyui, +blinkenlights, machineui. + + + +--column[=]:: +--no-column:: + Display branch listing in columns. See configuration variable + column.branch for option syntax.`--column` and `--no-column` + without options are equivalent to 'always' and 'never' respectively. ++ +This option is only applicable in non-verbose mode. + + +Synchronization Performance +--------------------------- + +By default, we use fairly conservative settings that are safe for syncing but +that might not be the best performing one. Once you got everything set up and +running, you might want to look into speeding up your synchronization. Here +are a couple of hints and tips on how to achieve this. + +1. Use maxconnections > 1. ++ +By default we only use one connection to an IMAP server. Using 2 or even 3 +speeds things up considerably in most cases. This setting goes into the +[Repository XXX] section. + +2. Use folderfilters. ++ +The quickest sync is a sync that can ignore some folders. I sort my inbox into +monthly folders, and ignore every folder that is more than 2-3 months old, +this lets me only inspect a fraction of my Mails on every sync. If you haven't +done this yet, do it :). See the folderfilter section the example +offlineimap.conf. + +3. The cache. ++ +The default status cache is a plain text file that will write out the complete +file for each single new message (or even changed flag) to a temporary file. +If you have plenty of files in a folder, this is a few hundred kilo to +megabytes for each mail and is bound to make things slower. I recommend to use +the sqlite backend for that. See the status_backend = sqlite setting in the +example offlineimap.conf. You will need to have python-sqlite installed in +order to use this. This will save you plenty of disk activity. Do note that +the sqlite backend is still considered experimental as it has only been +included recently (although a loss of your status cache should not be a +tragedy as that file can be rebuilt automatically) + +4. Use quick sync. ++ +A regular sync will request all flags and all UIDs of all mails in each folder +which takes quite some time. A 'quick' sync only compares the number of +messages in a folder on the IMAP side (it will detect flag changes on the +Maildir side of things though). A quick sync on my smallish account will take +7 seconds rather than 40 seconds. Eg, I run a cron script that does a regular +sync once a day, and does quick syncs (-q) only synchronizing the "-f INBOX" +in between. + +5. Turn off fsync. ++ +In the [general] section you can set fsync to True or False. If you want to +play 110% safe and wait for all operations to hit the disk before continueing, +you can set this to True. If you set it to False, you lose some of that +safety, trading it for speed. + + +Upgrading from plain text to SQLite cache format +------------------------------------------------ + +OfflineImap uses a cache to store the last know status of mails (flags etc). + +Historically that has meant plain text files, but recently we introduced +sqlite-based cache, which helps with performance and CPU usage on large +folders. Here is how to upgrade existing plain text cache installations to +sqlite based one: + +1. Sync to make sure things are reasonably similar. + +2. Change the account section to "status_backend = sqlite". + +3. Run a new sync. ++ +This will convert your plain text cache to an sqlite cache (but leave +the old plain text cache around for easy reverting). This must be quick and +not involve any mail up/downloading. + +4. See if it works! :-) + +5. If it does not work, go back to the old version or set "status_backend = plain" + +6. Delete the old text cache files. + +Once you are sure it works, you can delete the +~/.offlineimap/Account-foo/LocalStatus folder (the new cache will be in the +LocalStatus-sqlite folder) + + +Security and SSL +---------------- + +By default, OfflineIMAP will connect using any method that 'openssl' supports, +that is SSLv2, SSLv3, or TLSv1. + +Do note that SSLv2 is notoriously insecure and deprecated. Unfortunately, +python2 does not offer easy ways to disable SSLv2. It is recommended you test +your setup and make sure that the mail server does not use an SSLv2 +connection. Use e.g. "openssl s_client -host mail.server -port 443" to find +out the connection that is used by default. + +* Certificate checking ++ +Unfortunately, by default we will not verify the certificate of an IMAP +TLS/SSL server we connect to, so connecting by SSL is no guarantee against +man-in-the-middle attacks. While verifying a server certificate fingerprint is +being planned, it is not implemented yet. There is currently only one safe way +to ensure that you connect to the correct server in an encrypted manner: you +can specify a 'sslcacertfile' setting in your repository section of +offlineimap.conf pointing to a file that contains (among others) a CA +Certificate in PEM format which validating your server certificate. In this +case, we will check that: + +1. The server SSL certificate is validated by the CA Certificate. + +2. The server host name matches the SSL certificate. + +3. The server certificate is not past its expiration date. + +The FAQ has an entry on how to create your own certificate and CA certificate. + +* StartTLS ++ +If you have not configured your account to connect via SSL anyway, OfflineImap +will still attempt to set up an SSL connection via the STARTTLS function, in +case the imap server supports it. ++ +There is no certificate or fingerprint checking involved at all, when using +STARTTLS (the underlying imaplib library does not support this yet). This +means that you will be protected against passively listening eavesdroppers and +they will not be able to see your password or email contents. However, this +will not protect you from active attacks, such as Man-In-The-Middle attacks +which cause you to connect to the wrong server and pretend to be your mail +server. ++ +DO NOT RELY ON STARTTLS AS A SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF +YOUR IMAP SERVER! + + +Unix Signals +------------ + +OfflineImap listens to the unix signals SIGUSR1, SIGUSR2, SIGTERM, SIGINT, +SIGHUP, SIGQUIT. + +* If sent a SIGUSR1 it will abort any current (or next future) sleep of all +accounts that are configured to "autorefresh". In effect, this will trigger a +full sync of all accounts to be performed as soon as possible. + +* If sent a SIGUSR2, it will stop "autorefresh mode" for all accounts. That +is, accounts will abort any current sleep and will exit after a currently +running synchronization has finished. This signal can be used to gracefully +exit out of a running offlineimap "daemon". + +* SIGTERM, SIGINT, SIGHUP are all treated to gracefully terminate as soon as +possible. This means it will finish syncing the current folder in each +account, close keep alive connections, remove locks on the accounts and exit. ++ +It may take up to 10 seconds, if autorefresh option is used. + +* If sent SIGQUIT, dumps stack traces for all threads and tries to dump +process core. + + +Known Issues +------------ + +* SSL3 write pending. ++ +Users enabling SSL may hit a bug about "SSL3 write pending". If so, the +account(s) will stay unsynchronised from the time the bug appeared. Running +OfflineIMAP again can help. We are still working on this bug. Patches or +detailed bug reports would be appreciated. Please check you're running the +last stable version and send us a report to the mailing list including the +full log. + +* IDLE support is incomplete and experimental. Bugs may be encountered. + + - No hook exists for "run after an IDLE response". ++ +Email will show up, but may not be processed until the next refresh cycle. + + - nametrans may not be supported correctly. + + - IMAP IDLE <-> IMAP IDLE doesn't work yet. + + - IDLE may only work "once" per refresh. ++ +If you encounter this bug, please send a report to the list! + +* Maildir support in Windows drive. ++ +Maildir uses colon caracter (:) in message file names. Colon is however +forbidden character in windows drives. There are several workarounds for that +situation: + + . Enable file name character translation in windows registry (not tested). + - + + . Use cygwin managed mount (not tested). + - not available anymore since cygwin 1.7 + + . Use "maildir-windows-compatible = yes" account OfflineIMAP configuration. + - That makes OfflineIMAP to use exclamation mark (!) instead of colon for + storing messages. Such files can be written to windows partitions. But + you will probably loose compatibility with other programs trying to + read the same Maildir. ++ + - Exclamation mark was chosen because of the note in + http://docs.python.org/library/mailbox.html ++ + - If you have some messages already stored without this option, you will + have to re-sync them again + +* OfflineIMAP confused after system suspend. ++ +When resuming a suspended session, OfflineIMAP does not cleanly handles the +broken socket(s) if socktimeout option is not set. +You should enable this option with a value like 10. + +* OfflineIMAP confused when mails change while in a sync. ++ +When OfflineIMAP is syncing, some events happening since the invokation on +remote or local side are badly handled. OfflineIMAP won't track for changes +during the sync. + + +* Sharing a maildir with multiple IMAP servers. ++ +Generally a word of caution mixing IMAP repositories on the same Maildir root. +You have to be careful that you *never* use the same maildir folder for 2 IMAP +servers. In the best case, the folder MD5 will be different, and you will get +a loop where it will upload your mails to both servers in turn (infinitely!) +as it thinks you have placed new mails in the local Maildir. In the worst +case, the MD5 is the same (likely) and mail UIDs overlap (likely too!) and it +will fail to sync some mails as it thinks they are already existent. ++ +I would create a new local Maildir Repository for the Personal Gmail and +use a different root to be on the safe side here. You could e.g. use + + `~/mail/Pro` as Maildir root for the ProGmail and + `~/mail/Personal` as root for the personal one. ++ +If you then point your local mutt, or whatever MUA you use to `~/mail/` +as root, it should still recognize all folders. + + +* Edge cases with maxage causing too many messages to be synced. ++ +All messages from at most maxage days ago (+/- a few hours, depending on +timezones) are synced, but there are cases in which older messages can also be +synced. This happens when a message's UID is significantly higher than those of +other messages with similar dates, e.g. when messages are added to the local +folder behind offlineimap's back, causing them to get assigned a new UID, or +when offlineimap first syncs a pre-existing Maildir. In the latter case, it +could appear as if a noticeable and random subset of old messages are synced. + + +Main authors +------------ + + John Goerzen, Sebastian Spaetz, Eygene Ryabinkin, Nicolas Sebrecht. + + +See Also +-------- + + offlineimapui(7), openssl(1), signal(7), sqlite3(1). + http://offlineimap.org diff --git a/docs/offlineimapui.txt b/docs/offlineimapui.txt new file mode 100644 index 0000000..c087ece --- /dev/null +++ b/docs/offlineimapui.txt @@ -0,0 +1,140 @@ + +offlineimapui(7) +================ + +NAME +---- +offlineimapui - The User Interfaces + +DESCRIPTION +----------- + +OfflineIMAP comes with differents UI, each aiming its own purpose. + + +TTYUI +------ + +TTYUI interface is for people running in terminals. It prints out basic +status messages and is generally friendly to use on a console or xterm. + + +Basic +------ + +Basic is designed for situations in which OfflineIMAP will be run non-attended +and the status of its execution will be logged. ++ +This user interface is not capable of reading a password from the keyboard; +account passwords must be specified using one of the configuration file +options. For example, it will not print periodic sleep announcements and tends +to be a tad less verbose, in general. + + +Blinkenlights +------------- + +Blinkenlights is an interface designed to be sleek, fun to watch, and +informative of the overall picture of what OfflineIMAP is doing. + +Blinkenlights contains a row of "LEDs" with command buttons and a log. The +log shows more detail about what is happening and is color-coded to match the +color of the lights. + +Each light in the Blinkenlights interface represents a thread of execution -- +that is, a particular task that OfflineIMAP is performing right now. The +colors indicate what task the particular thread is performing, and are as +follows: + +* Black + + indicates that this light's thread has terminated; it will light up again + later when new threads start up. So, black indicates no activity. + +* Red (Meaning 1) + + is the color of the main program's thread, which basically does nothing but + monitor the others. It might remind you of HAL 9000 in 2001. + +* Gray + + indicates that the thread is establishing a new connection to the IMAP + server. + +* Purple + + is the color of an account synchronization thread that is monitoring the + progress of the folders in that account (not generating any I/O). + +* Cyan + + indicates that the thread is syncing a folder. + +* Green + + means that a folder's message list is being loaded. + +* Blue + + is the color of a message synchronization controller thread. + +* Orange + + indicates that an actual message is being copied. (We use fuchsia for fake + messages.) + +* Red (meaning 2) + + indicates that a message is being deleted. + +* Yellow / bright orange + + indicates that message flags are being added. + +* Pink / bright red + + indicates that message flags are being removed. + +* Red / Black Flashing + + corresponds to the countdown timer that runs between synchronizations. + + +The name of this interfaces derives from a bit of computer history. Eric +Raymond's Jargon File defines blinkenlights, in part, as: + + Front-panel diagnostic lights on a computer, esp. a dinosaur. Now that + dinosaurs are rare, this term usually refers to status lights on a modem, + network hub, or the like. + +This term derives from the last word of the famous blackletter-Gothic sign in +mangled pseudo-German that once graced about half the computer rooms in the +English-speaking world. One version ran in its entirety as follows: + + ACHTUNG! ALLES LOOKENSPEEPERS! + + Das computermachine ist nicht fuer gefingerpoken und mittengrabben. + Ist easy schnappen der springenwerk, blowenfusen und poppencorken + mit spitzensparken. Ist nicht fuer gewerken bei das dumpkopfen. + Das rubbernecken sichtseeren keepen das cotten-pickenen hans in das + pockets muss; relaxen und watchen das blinkenlichten. + + +Quiet +----- + +It will output nothing except errors and serious warnings. Like Basic, this +user interface is not capable of reading a password from the keyboard; account +passwords must be specified using one of the configuration file options. + +MachineUI +--------- + +MachineUI generates output in a machine-parsable format. It is designed +for other programs that will interface to OfflineIMAP. + + +See Also +-------- + + offlineimap(1) diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md new file mode 100644 index 0000000..e011dbc --- /dev/null +++ b/docs/rfcs/README.md @@ -0,0 +1,4 @@ + +All RFCs related to IMAP. + +TODO: Add a brief introduction here to introduce the most important RFCs. diff --git a/docs/rfcs/rfc1731.IMAP4_auth.txt b/docs/rfcs/rfc1731.IMAP4_auth.txt new file mode 100644 index 0000000..9cced5d --- /dev/null +++ b/docs/rfcs/rfc1731.IMAP4_auth.txt @@ -0,0 +1,339 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 1731 Carnegie Mellon +Category: Standards Track December 1994 + + + IMAP4 Authentication Mechanisms + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + + +1. Introduction + + The Internet Message Access Protocol, Version 4 [IMAP4] contains the + AUTHENTICATE command, for identifying and authenticating a user to an + IMAP4 server and for optionally negotiating a protection mechanism + for subsequent protocol interactions. This document describes + several authentication mechanisms for use by the IMAP4 AUTHENTICATE + command. + + +2. Kerberos version 4 authentication mechanism + + The authentication type associated with Kerberos version 4 is + "KERBEROS_V4". + + The data encoded in the first ready response contains a random 32-bit + number in network byte order. The client should respond with a + Kerberos ticket and an authenticator for the principal + "imap.hostname@realm", where "hostname" is the first component of the + host name of the server with all letters in lower case and where + "realm" is the Kerberos realm of the server. The encrypted checksum + field included within the Kerberos authenticator should contain the + server provided 32-bit number in network byte order. + + Upon decrypting and verifying the ticket and authenticator, the + server should verify that the contained checksum field equals the + original server provided random 32-bit number. Should the + verification be successful, the server must add one to the checksum + and construct 8 octets of data, with the first four octets containing + the incremented checksum in network byte order, the fifth octet + containing a bit-mask specifying the protection mechanisms supported + by the server, and the sixth through eighth octets containing, in + + + +Myers [Page 1] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + + network byte order, the maximum cipher-text buffer size the server is + able to receive. The server must encrypt the 8 octets of data in the + session key and issue that encrypted data in a second ready response. + The client should consider the server authenticated if the first four + octets the un-encrypted data is equal to one plus the checksum it + previously sent. + + The client must construct data with the first four octets containing + the original server-issued checksum in network byte order, the fifth + octet containing the bit-mask specifying the selected protection + mechanism, the sixth through eighth octets containing in network byte + order the maximum cipher-text buffer size the client is able to + receive, and the following octets containing a user name string. The + client must then append from one to eight octets so that the length + of the data is a multiple of eight octets. The client must then PCBC + encrypt the data with the session key and respond to the second ready + response with the encrypted data. The server decrypts the data and + verifies the contained checksum. The username field identifies the + user for whom subsequent IMAP operations are to be performed; the + server must verify that the principal identified in the Kerberos + ticket is authorized to connect as that user. After these + verifications, the authentication process is complete. + + The protection mechanisms and their corresponding bit-masks are as + follows: + + 1 No protection mechanism + 2 Integrity (krb_mk_safe) protection + 4 Privacy (krb_mk_priv) protection + + + EXAMPLE: The following are two Kerberos version 4 login scenarios + (note that the line breaks in the sample authenticators are for + editorial clarity and are not in real authenticators) + + S: * OK IMAP4 Server + C: A001 AUTHENTICATE KERBEROS_V4 + S: + AmFYig== + C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT + +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd + WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh + S: + or//EoAADZI= + C: DiAF5A4gA+oOIALuBkAAmw== + S: A001 OK Kerberos V4 authentication successful + + + + + + + +Myers [Page 2] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + + S: * OK IMAP4 Server + C: A001 AUTHENTICATE KERBEROS_V4 + S: + gcfgCA== + C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT + +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd + WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh + S: A001 NO Kerberos V4 authentication failed + + +3. GSSAPI authentication mechanism + + The authentication type associated with all mechanisms employing the + GSSAPI [RFC1508] is "GSSAPI". + + The first ready response issued by the server contains no data. The + client should call GSS_Init_sec_context, passing in 0 for + input_context_handle (initially) and a targ_name equal to output_name + from GSS_Import_Name called with input_name_type of NULL and + input_name_string of "SERVICE:imap@hostname" where "hostname" is the + fully qualified host name of the server with all letters in lower + case. The client must then respond with the resulting output_token. + If GSS_Init_sec_context returns GSS_CONTINUE_NEEDED, then the client + should expect the server to issue a token in a subsequent ready + response. The client must pass the token to another call to + GSS_Init_sec_context. + + If GSS_Init_sec_context returns GSS_COMPLETE, then the client should + respond with any resulting output_token. If there is no + output_token, the client should respond with no data. The client + should then expect the server to issue a token in a subsequent ready + response. The client should pass this token to GSS_Unseal and + interpret the first octet of resulting cleartext as a bit-mask + specifying the protection mechanisms supported by the server and the + second through fourth octets as the maximum size output_message to + send to the server. The client should construct data, with the first + octet containing the bit-mask specifying the selected protection + mechanism, the second through fourth octets containing in network + byte order the maximum size output_message the client is able to + receive, and the remaining octets containing a user name string. The + client must pass the data to GSS_Seal with conf_flag set to FALSE, + and respond with the generated output_message. The client can then + consider the server authenticated. + + The server must issue a ready response with no data and pass the + resulting client supplied token to GSS_Accept_sec_context as + input_token, setting acceptor_cred_handle to NULL (for "use default + credentials"), and 0 for input_context_handle (initially). If + GSS_Accept_sec_context returns GSS_CONTINUE_NEEDED, the server should + + + +Myers [Page 3] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + + return the generated output_token to the client in a ready response + and pass the resulting client supplied token to another call to + GSS_Accept_sec_context. + + If GSS_Accept_sec_context returns GSS_COMPLETE, then if an + output_token is returned, the server should return it to the client + in a ready response and expect a reply from the client with no data. + Whether or not an output_token was returned, the server then should + then construct 4 octets of data, with the first octet containing a + bit-mask specifying the protection mechanisms supported by the server + and the second through fourth octets containing in network byte order + the maximum size output_token the server is able to receive. The + server must then pass the plaintext to GSS_Seal with conf_flag set to + FALSE and issue the generated output_message to the client in a ready + response. The server must then pass the resulting client supplied + token to GSS_Unseal and interpret the first octet of resulting + cleartext as the bit-mask for the selected protection mechanism, the + second through fourth octets as the maximum size output_message to + send to the client, and the remaining octets as the user name. Upon + verifying the src_name is authorized to authenticate as the user + name, The server should then consider the client authenticated. + + The protection mechanisms and their corresponding bit-masks are as + follows: + + 1 No protection mechanism + 2 Integrity protection. + Sender calls GSS_Seal with conf_flag set to FALSE + 4 Privacy protection. + Sender calls GSS_Seal with conf_flag set to TRUE + + +4. S/Key authentication mechanism + + The authentication type associated with S/Key [SKEY] is "SKEY". + + The first ready response issued by the server contains no data. The + client responds with the user name string. + + The data encoded in the second ready response contains the decimal + sequence number followed by a single space and the seed string for + the indicated user. The client responds with the one-time-password, + as either a 64-bit value in network byte order or encoded in the "six + English words" format. + + Upon successful verification of the one-time-password, the server + should consider the client authenticated. + + + + +Myers [Page 4] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + + S/Key authentication does not provide for any protection mechanisms. + + + EXAMPLE: The following are two S/Key login scenarios. + + S: * OK IMAP4 Server + C: A001 AUTHENTICATE SKEY + S: + + C: bW9yZ2Fu + S: + OTUgUWE1ODMwOA== + C: Rk9VUiBNQU5OIFNPT04gRklSIFZBUlkgTUFTSA== + S: A001 OK S/Key authentication successful + + + S: * OK IMAP4 Server + C: A001 AUTHENTICATE SKEY + S: + + C: c21pdGg= + S: + OTUgUWE1ODMwOA== + C: BsAY3g4gBNo= + S: A001 NO S/Key authentication failed + + +5. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", + RFC 1730, University of Washington, December 1994. + + [RFC1508] Linn, J., "Generic Security Service Application Program + Interface", RFC 1508, Geer Zolot Associates, September 1993. + + [SKEY] Haller, Neil M. "The S/Key One-Time Password System", + Bellcore, Morristown, New Jersey, October 1993, + thumper.bellcore.com:pub/nmh/docs/ISOC.symp.ps + + + + + + + + + + + + + + + + + +Myers [Page 5] + +RFC 1731 IMAP4 Authentication Mechanisms December 1994 + + +6. Security Considerations + + Security issues are discussed throughout this memo. + + +7. Author's Address + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave. + Pittsburgh PA, 15213-3890 + + EMail: jgm+@cmu.edu + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Myers [Page 6] + diff --git a/docs/rfcs/rfc1732.compatibiliy_IMAP2-IMAP2bis.txt b/docs/rfcs/rfc1732.compatibiliy_IMAP2-IMAP2bis.txt new file mode 100644 index 0000000..cfae89c --- /dev/null +++ b/docs/rfcs/rfc1732.compatibiliy_IMAP2-IMAP2bis.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 1732 University of Washington +Category: Informational December 1994 + + + IMAP4 COMPATIBILITY WITH IMAP2 AND IMAP2BIS + + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + +Introduction + + This is a summary of hints and recommendations to enable an IMAP4 + implementation to interoperate with implementations that conform to + earlier specifications. None of these hints and recommendations are + required by the IMAP4 specification; implementors must decide for + themselves whether they want their implementation to fail if it + encounters old software. + + IMAP4 has been designed to be upwards compatible with earlier + specifications. For the most part, IMAP4 facilities that were not in + earlier specifications should be invisible to clients unless the + client asks for the facility. + + In some cases, older servers may support some of the capabilities + listed as being "new in IMAP4" as experimental extensions to the + IMAP2 protocol described in RFC 1176. + + This information may not be complete; it reflects current knowledge + of server and client implementations as well as "folklore" acquired + in the evolution of the protocol. + + + + + + + + + + + + + + + + +Crispin [Page 1] + +RFC 1732 IMAP4 - Compatibility December 1994 + + +IMAP4 client interoperability with old servers + + In general, a client should be able to discover whether an IMAP2 + server supports a facility by trial-and-error; if an attempt to use a + facility generates a BAD response, the client can assume that the + server does not support the facility. + + A quick way to check whether a server implementation supports the + IMAP4 specification is to try the CAPABILITY command. An OK response + that includes the IMAP4 capability value indicates a server that + supports IMAP4; a BAD response or one without the IMAP4 capability + value indicates an older server. + + The following is a list of facilities that are only in IMAP4, and + suggestions for how new clients might interoperate with old servers: + + CAPABILITY command + A BAD response to this command indicates that the server + implements IMAP2 (or IMAP2bis) and not IMAP4. + + AUTHENTICATE command. + Use the LOGIN command. + + LSUB and LIST commands + Try the RFC 1176 FIND command. + + * in a sequence + Use the number of messages in the mailbox from the EXISTS + unsolicited response. + + SEARCH extensions (character set, additional criteria) + Reformulate the search request using only the searching + options listed in search_old in the IMAP4 grammar. This may + entail doing multiple searches to achieve the desired + results. + + BODYSTRUCTURE fetch data item + Try to fetch the non-extensible BODY data item. + + body section number 0 + Fetch the entire message and extract the header. + + RFC822.HEADER.LINES and RFC822.HEADER.LINES.NOT fetch data items + Use RFC822.HEADER and remove the unwanted information. + + BODY.PEEK[section], RFC822.PEEK, and RFC822.TEXT.PEEK fetch data + items Use the corresponding non-PEEK versions and manually + clear the \Seen flag as necessary. + + + +Crispin [Page 2] + +RFC 1732 IMAP4 - Compatibility December 1994 + + + UID fetch data item and the UID commands + No equivalent capabilitity exists in older servers. + + FLAGS.SILENT, +FLAGS.SILENT, and -FLAGS.SILENT store data items + Use the corresponding non-SILENT versions and ignore the + untagged FETCH responses which com eback. + + + The following IMAP4 facilities were introduced in the experimental + IMAP2bis revisions to RFC-1176, and may be present in a server that + does not support the CAPABILITY command: + + CREATE, DELETE, and RENAME commands + To test whether these commands are present, try a CREATE + INBOX command. If the response is NO, these commands are + supported by the server. If the response is BAD, they are + not. Older servers without the CREATE capability may sup- + port implicit creation of a mailbox by a COPY command with a + non-existant name as the destination. + + APPEND command + To test whether this command is present, try to append a + zero-length stream to a mailbox name that is known not to + exist (or at least, highly unlikely to exist) on the remote + system. + + SUBSCRIBE and UNSUBSCRIBE commands + Try the form of these commands with the optional MAILBOX + keyword. + + EXAMINE command + Use the SELECT command instead. + + flags and internal date argument to APPEND command + Try the APPEND without any flag list and internal date argu- + ments. + + BODY, BODY[section], and FULL fetch data items + Use RFC822.TEXT and ALL instead. Server does not support + MIME. + + PARTIAL command + Use the appropriate FETCH command and ignore the unwanted + data. + + + IMAP4 client implementations must accept all responses and data for- + mats documented in the IMAP4 specification, including those labeled + + + +Crispin [Page 3] + +RFC 1732 IMAP4 - Compatibility December 1994 + + + as obsolete. This includes the COPY and STORE unsolicited responses + and the old format of dates and times. In particular, client imple- + mentations must not treat a date/time as a fixed format string; nor + may they assume that the time begins at a particular octet. + + IMAP4 client implementations must not depend upon the presence of any + server extensions that are not in the base IMAP4 specification. + + The experimental IMAP2bis version specified that the TRYCREATE spe- + cial information token is sent as a separate unsolicited OK response + instead of inside the NO response. + + The FIND BBOARDS, FIND ALL.BBOARDS, and BBOARD commands of RFC 1176 + are removed from IMAP4. There is no equivalent to the bboard com- + mands, which provided a separate namespace with implicit restrictions + on what may be done in that namespace. + + Older server implementations may automatically create the destination + mailbox on COPY if that mailbox does not already exist. This was how + a new mailbox was created in older specifications. If the server + does not support the CREATE command (see above for how to test for + this), it will probably create a mailbox on COPY. + + Older server implementations may not preserve flags or internal dates + on COPY. Some server implementations may not permit the preservation + of certain flags on COPY or their setting with APPEND as site policy. + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin [Page 4] + +RFC 1732 IMAP4 - Compatibility December 1994 + + +IMAP4 server interoperability with old clients + + In general, there should be no interoperation problem between a + server conforming to the IMAP4 specification and a well-written + client that conforms to an earlier specification. Known problems are + noted below: + + Poor wording in the description of the CHECK command in earlier + specifications implied that a CHECK command is the way to get the + current number of messages in the mailbox. This is incorrect. A + CHECK command does not necessarily result in an EXISTS response. + Clients must remember the most recent EXISTS value sent from the + server, and should not generate unnecessary CHECK commands. + + An incompatibility exists with COPY in IMAP4. COPY in IMAP4 + servers does not automatically create the destination mailbox if + that mailbox does not already exist. This may cause problems with + old clients that expect automatic mailbox creation in COPY. + + The PREAUTH unsolicited response is new in IMAP4. It is highly + unlikely that an old client would ever see this response. + + The format of dates and times has changed due to the impending end + of the century. Clients that fail to accept a four-digit year or + a signed four-digit timezone value will not work properly with + IMAP4. + + An incompatibility exists with the use of "\" in quoted strings. + This is best avoided by using literals instead of quoted strings + if "\" or <"> is embedded in the string. + +Security Considerations + + Security issues are not discussed in this memo. + +Author's Address: + + Mark R. Crispin + Networks and Distributed Computing, JE-30 + University of Washington + Seattle, WA 98195 + + Phone: (206) 543-5762 + + EMail: MRC@CAC.Washington.EDU + + + + + + +Crispin [Page 5] + diff --git a/docs/rfcs/rfc1733.models_in_IMAP4.txt b/docs/rfcs/rfc1733.models_in_IMAP4.txt new file mode 100644 index 0000000..2ec385f --- /dev/null +++ b/docs/rfcs/rfc1733.models_in_IMAP4.txt @@ -0,0 +1,171 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 1733 University of Washington +Category: Informational December 1994 + + + DISTRIBUTED ELECTRONIC MAIL MODELS IN IMAP4 + + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + + +Distributed Electronic Mail Models + + There are three fundamental models of client/server email: offline, + online, and disconnected use. IMAP4 can be used in any one of these + three models. + + The offline model is the most familiar form of client/server email + today, and is used by protocols such as POP-3 (RFC 1225) and UUCP. + In this model, a client application periodically connects to a + server. It downloads all the pending messages to the client machine + and deletes these from the server. Thereafter, all mail processing + is local to the client. This model is store-and-forward; it moves + mail on demand from an intermediate server (maildrop) to a single + destination machine. + + The online model is most commonly used with remote filesystem + protocols such as NFS. In this model, a client application + manipulates mailbox data on a server machine. A connection to the + server is maintained throughout the session. No mailbox data are + kept on the client; the client retrieves data from the server as is + needed. IMAP4 introduces a form of the online model that requires + considerably less network bandwidth than a remote filesystem + protocol, and provides the opportunity for using the server for CPU + or I/O intensive functions such as parsing and searching. + + The disconnected use model is a hybrid of the offline and online + models, and is used by protocols such as PCMAIL (RFC 1056). In this + model, a client user downloads some set of messages from the server, + manipulates them offline, then at some later time uploads the + changes. The server remains the authoritative repository of the + messages. The problems of synchronization (particularly when + multiple clients are involved) are handled through the means of + unique identifiers for each message. + + + +Crispin [Page 1] + +RFC 1733 IMAP4 - Model December 1994 + + + Each of these models have their own strengths and weaknesses: + + Feature Offline Online Disc + ------- ------- ------ ---- + Can use multiple clients NO YES YES + Minimum use of server connect time YES NO YES + Minimum use of server resources YES NO NO + Minimum use of client disk resources NO YES NO + Multiple remote mailboxes NO YES YES + Fast startup NO YES NO + Mail processing when not online YES NO YES + + Although IMAP4 has its origins as a protocol designed to accommodate + the online model, it can support the other two models as well. This + makes possible the creation of clients that can be used in any of the + three models. For example, a user may wish to switch between the + online and disconnected models on a regular basis (e.g. owing to + travel). + + IMAP4 is designed to transmit message data on demand, and to provide + the facilities necessary for a client to decide what data it needs at + any particular time. There is generally no need to do a wholesale + transfer of an entire mailbox or even of the complete text of a + message. This makes a difference in situations where the mailbox is + large, or when the link to the server is slow. + + More specifically, IMAP4 supports server-based RFC 822 and MIME + processing. With this information, it is possible for a client to + determine in advance whether it wishes to retrieve a particular + message or part of a message. For example, a user connected to an + IMAP4 server via a dialup link can determine that a message has a + 2000 byte text segment and a 40 megabyte video segment, and elect to + fetch only the text segment. + + In IMAP4, the client/server relationship lasts only for the duration + of the TCP connection. There is no registration of clients. Except + for any unique identifiers used in disconnected use operation, the + client initially has no knowledge of mailbox state and learns it from + the IMAP4 server when a mailbox is selected. This initial transfer + is minimal; the client requests additional state data as it needs. + + As noted above, the choice for the location of mailbox data depends + upon the model chosen. The location of message state (e.g. whether + or not a message has been read or answered) is also determined by the + model, and is not necessarily the same as the location of the mailbox + data. For example, in the online model message state can be co- + located with mailbox data; it can also be located elsewhere (on the + client or on a third agent) using unique identifiers to achieve + + + +Crispin [Page 2] + +RFC 1733 IMAP4 - Model December 1994 + + + common reference across sessions. The latter is particularly useful + with a server that exports public data such as netnews and does not + maintain per-user state. + + The IMAP4 protocol provides the generality to implement these + different models. This is done by means of server and (especially) + client configuration, and not by requiring changes to the protocol or + the implementation of the protocol. + + +Security Considerations + + Security issues are not discussed in this memo. + + +Author's Address: + + Mark R. Crispin + Networks and Distributed Computing, JE-30 + University of Washington + Seattle, WA 98195 + + Phone: (206) 543-5762 + + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin [Page 3] + diff --git a/docs/rfcs/rfc1734.POP3_AUTHentication b/docs/rfcs/rfc1734.POP3_AUTHentication new file mode 100644 index 0000000..2e6c431 --- /dev/null +++ b/docs/rfcs/rfc1734.POP3_AUTHentication @@ -0,0 +1,439 @@ + + + + + + + + + + + + + + + + RFC 1734 - POP3 AUTHentication command + + + + + + + + +
+
+ +
+[Docs] [txt|pdf] [draft-myers-pop3-...] [Diff1] [Diff2]
+
+Obsoleted by: 5034 PROPOSED STANDARD
+
+
+Network Working Group                                           J. Myers
+Request for Comments: 1734                               Carnegie Mellon
+Category: Standards Track                                  December 1994
+
+
+                      POP3 AUTHentication command
+
+Status of this Memo
+
+   This document specifies an Internet standards track protocol for the
+   Internet community, and requests discussion and suggestions for
+   improvements.  Please refer to the current edition of the "Internet
+   Official Protocol Standards" (STD 1) for the standardization state
+   and status of this protocol.  Distribution of this memo is unlimited.
+
+
+1. Introduction
+
+   This document describes the optional AUTH command, for indicating an
+   authentication mechanism to the server, performing an authentication
+   protocol exchange, and optionally negotiating a protection mechanism
+   for subsequent protocol interactions.  The authentication and
+   protection mechanisms used by the POP3 AUTH command are those used by
+   IMAP4.
+
+
+2. The AUTH command
+
+   AUTH mechanism
+
+         Arguments:
+             a string identifying an IMAP4 authentication mechanism,
+             such as defined by [IMAP4-AUTH].  Any use of the string
+             "imap" used in a server authentication identity in the
+             definition of an authentication mechanism is replaced with
+             the string "pop".
+
+         Restrictions:
+             may only be given in the AUTHORIZATION state
+
+         Discussion:
+             The AUTH command indicates an authentication mechanism to
+             the server.  If the server supports the requested
+             authentication mechanism, it performs an authentication
+             protocol exchange to authenticate and identify the user.
+             Optionally, it also negotiates a protection mechanism for
+             subsequent protocol interactions.  If the requested
+             authentication mechanism is not supported, the server
+
+
+
+Myers                                                           [Page 1]
+

+RFC 1734                       POP3 AUTH                   December 1994
+
+
+             should reject the AUTH command by sending a negative
+             response.
+
+             The authentication protocol exchange consists of a series
+             of server challenges and client answers that are specific
+             to the authentication mechanism.  A server challenge,
+             otherwise known as a ready response, is a line consisting
+             of a "+" character followed by a single space and a BASE64
+             encoded string.  The client answer consists of a line
+             containing a BASE64 encoded string.  If the client wishes
+             to cancel an authentication exchange, it should issue a
+             line with a single "*".  If the server receives such an
+             answer, it must reject the AUTH command by sending a
+             negative response.
+
+             A protection mechanism provides integrity and privacy
+             protection to the protocol session.  If a protection
+             mechanism is negotiated, it is applied to all subsequent
+             data sent over the connection.  The protection mechanism
+             takes effect immediately following the CRLF that concludes
+             the authentication exchange for the client, and the CRLF of
+             the positive response for the server.  Once the protection
+             mechanism is in effect, the stream of command and response
+             octets is processed into buffers of ciphertext.  Each
+             buffer is transferred over the connection as a stream of
+             octets prepended with a four octet field in network byte
+             order that represents the length of the following data.
+             The maximum ciphertext buffer length is defined by the
+             protection mechanism.
+
+             The server is not required to support any particular
+             authentication mechanism, nor are authentication mechanisms
+             required to support any protection mechanisms.  If an AUTH
+             command fails with a negative response, the session remains
+             in the AUTHORIZATION state and client may try another
+             authentication mechanism by issuing another AUTH command,
+             or may attempt to authenticate by using the USER/PASS or
+             APOP commands.  In other words, the client may request
+             authentication types in decreasing order of preference,
+             with the USER/PASS or APOP command as a last resort.
+
+             Should the client successfully complete the authentication
+             exchange, the POP3 server issues a positive response and
+             the POP3 session enters the TRANSACTION state.
+
+         Possible Responses:
+             +OK maildrop locked and ready
+             -ERR authentication exchange failed
+
+
+
+Myers                                                           [Page 2]
+

+RFC 1734                       POP3 AUTH                   December 1994
+
+
+
+         Examples:
+             S: +OK POP3 server ready
+             C: AUTH KERBEROS_V4
+             S: + AmFYig==
+             C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT
+                +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd
+                WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh
+             S: + or//EoAADZI=
+             C: DiAF5A4gA+oOIALuBkAAmw==
+             S: +OK Kerberos V4 authentication successful
+                ...
+             C: AUTH FOOBAR
+             S: -ERR Unrecognized authentication type
+
+              Note: the line breaks in the first client answer  are
+              for editorial clarity and are not in real authentica-
+              tors.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Myers                                                           [Page 3]
+

+RFC 1734                       POP3 AUTH                   December 1994
+
+
+3. Formal Syntax
+
+   The following syntax specification uses the augmented Backus-Naur
+   Form (BNF) notation as specified in RFC 822.
+
+   Except as noted otherwise, all alphabetic characters are case-
+   insensitive.  The use of upper or lower case characters to define
+   token strings is for editorial clarity only.  Implementations MUST
+   accept these strings in a case-insensitive fashion.
+
+   ATOM_CHAR       ::= <any CHAR except atom_specials>
+
+   atom_specials   ::= "(" / ")" / "{" / SPACE / CTLs / "%" / "*" /
+                       <"> / "\"
+
+   auth            ::= "AUTH" 1*(SPACE / TAB) auth_type *(CRLF base64)
+                       CRLF
+
+   auth_type       ::= 1*ATOM_CHAR
+
+   base64          ::= *(4base64_CHAR) [base64_terminal]
+
+   base64_char     ::= "A" / "B" / "C" / "D" / "E" / "F" / "G" / "H" /
+           "I" / "J" / "K" / "L" / "M" / "N" / "O" / "P" /
+                       "Q" / "R" / "S" / "T" / "U" / "V" / "W" / "X" /
+                       "Y" / "Z" /
+                       "a" / "b" / "c" / "d" / "e" / "f" / "g" / "h" /
+                       "i" / "j" / "k" / "l" / "m" / "n" / "o" / "p" /
+                       "q" / "r" / "s" / "t" / "u" / "v" / "w" / "x" /
+                       "y" / "z" /
+                       "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7" /
+                       "8" / "9" / "+" / "/"
+                       ;; Case-sensitive
+
+   base64_terminal ::= (2base64_char "==") / (3base64_char "=")
+
+   CHAR            ::= <any 7-bit US-ASCII character except NUL,
+                        0x01 - 0x7f>
+
+   continue_req    ::= "+" SPACE base64 CRLF
+
+   CR              ::= <ASCII CR, carriage return, 0x0C>
+
+   CRLF            ::= CR LF
+
+   CTL             ::= <any ASCII control character and DEL,
+                        0x00 - 0x1f, 0x7f>
+
+
+
+
+Myers                                                           [Page 4]
+

+RFC 1734                       POP3 AUTH                   December 1994
+
+
+   LF              ::= <ASCII LF, line feed, 0x0A>
+
+   SPACE           ::= <ASCII SP, space, 0x20>
+
+   TAB             ::= <ASCII HT, tab, 0x09>
+
+
+
+4. References
+
+   [IMAP4-AUTH]  Myers, J., "IMAP4 Authentication Mechanisms", RFC 1731,
+   Carnegie Mellon, December 1994.
+
+
+
+5. Security Considerations
+
+   Security issues are discussed throughout this memo.
+
+
+
+6. Author's Address
+
+   John G. Myers
+   Carnegie-Mellon University
+   5000 Forbes Ave
+   Pittsburgh, PA 15213
+
+   EMail: jgm+@cmu.edu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Myers                                                           [Page 5]
+
+

+Html markup produced by rfcmarkup 1.111, available from +https://tools.ietf.org/tools/rfcmarkup/ + + diff --git a/docs/rfcs/rfc2061.compatibility_IMAP4-IMAP2bis.txt b/docs/rfcs/rfc2061.compatibility_IMAP4-IMAP2bis.txt new file mode 100644 index 0000000..7cb02bb --- /dev/null +++ b/docs/rfcs/rfc2061.compatibility_IMAP4-IMAP2bis.txt @@ -0,0 +1,171 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 2061 University of Washington +Category: Informational December 1996 + + + IMAP4 COMPATIBILITY WITH IMAP2BIS + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + +Introduction + + The Internet Message Access Protocol (IMAP) has been through several + revisions and variants in its 10-year history. Many of these are + either extinct or extremely rare; in particular, several undocumented + variants and the variants described in RFC 1064, RFC 1176, and RFC + 1203 fall into this category. + + One variant, IMAP2bis, is at the time of this writing very common and + has been widely distributed with the Pine mailer. Unfortunately, + there is no definite document describing IMAP2bis. This document is + intended to be read along with RFC 1176 and the most recent IMAP4 + specification (RFC 2060) to assist implementors in creating an IMAP4 + implementation to interoperate with implementations that conform to + earlier specifications. Nothing in this document is required by the + IMAP4 specification; implementors must decide for themselves whether + they want their implementation to fail if it encounters old software. + + At the time of this writing, IMAP4 has been updated from the version + described in RFC 1730. An implementor who wishes to interoperate + with both RFC 1730 and RFC 2060 should refer to both documents. + + This information is not complete; it reflects current knowledge of + server and client implementations as well as "folklore" acquired in + the evolution of the protocol. It is NOT a description of how to + interoperate with all variants of IMAP, but rather with the old + variant that is most likely to be encountered. For detailed + information on interoperating with other old variants, refer to RFC + 1732. + +IMAP4 client interoperability with IMAP2bis servers + + A quick way to check whether a server implementation supports the + IMAP4 specification is to try the CAPABILITY command. An OK response + will indicate which variant(s) of IMAP4 are supported by the server. + + + +Crispin Informational [Page 1] + +RFC 2061 IMAP4 Compatibility December 1996 + + + If the client does not find any of its known variant in the response, + it should treat the server as IMAP2bis. A BAD response indicates an + IMAP2bis or older server. + + Most IMAP4 facilities are in IMAP2bis. The following exceptions + exist: + + CAPABILITY command + The absense of this command indicates IMAP2bis (or older). + + AUTHENTICATE command. + Use the LOGIN command. + + LSUB, SUBSCRIBE, and UNSUBSCRIBE commands + No direct functional equivalent. IMAP2bis had a concept + called "bboards" which is not in IMAP4. RFC 1176 supported + these with the BBOARD and FIND BBOARDS commands. IMAP2bis + augmented these with the FIND ALL.BBOARDS, SUBSCRIBE BBOARD, + and UNSUBSCRIBE BBOARD commands. It is recommended that + none of these commands be implemented in new software, + including servers that support old clients. + + LIST command + Use the command FIND ALL.MAILBOXES, which has a similar syn- + tax and response to the FIND MAILBOXES command described in + RFC 1176. The FIND MAILBOXES command is unlikely to produce + useful information. + + * in a sequence + Use the number of messages in the mailbox from the EXISTS + unsolicited response. + + SEARCH extensions (character set, additional criteria) + Reformulate the search request using only the RFC 1176 syn- + tax. This may entail doing multiple searches to achieve the + desired results. + + BODYSTRUCTURE fetch data item + Use the non-extensible BODY data item. + + body sections HEADER, TEXT, MIME, HEADER.FIELDS, HEADER.FIELDS.NOT + Use body section numbers only. + + BODY.PEEK[section] + Use BODY[section] and manually clear the \Seen flag as + necessary. + + + + + +Crispin Informational [Page 2] + +RFC 2061 IMAP4 Compatibility December 1996 + + + FLAGS.SILENT, +FLAGS.SILENT, and -FLAGS.SILENT store data items + Use the corresponding non-SILENT versions and ignore the + untagged FETCH responses which come back. + + UID fetch data item and the UID commands + No functional equivalent. + + CLOSE command + No functional equivalent. + + + In IMAP2bis, the TRYCREATE special information token is sent as a + separate unsolicited OK response instead of inside the NO response. + + IMAP2bis is ambiguous about whether or not flags or internal dates + are preserved on COPY. It is impossible to know what behavior is + supported by the server. + +IMAP4 server interoperability with IMAP2bis clients + + The only interoperability problem between an IMAP4 server and a + well-written IMAP2bis client is an incompatibility with the use of + "\" in quoted strings. This is best avoided by using literals + instead of quoted strings if "\" or <"> is embedded in the string. + +Security Considerations + + Security issues are not discussed in this memo. + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Aveneue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + +Crispin Informational [Page 3] + diff --git a/docs/rfcs/rfc2086.IMAP4_ACL_extension.txt b/docs/rfcs/rfc2086.IMAP4_ACL_extension.txt new file mode 100644 index 0000000..b6a98b5 --- /dev/null +++ b/docs/rfcs/rfc2086.IMAP4_ACL_extension.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 2086 Carnegie Mellon +Category: Standards Track January 1997 + + + IMAP4 ACL extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + The ACL extension of the Internet Message Access Protocol [IMAP4] + permits access control lists to be manipulated through the IMAP + protocol. + +Table of Contents + + 1. Abstract............................................... 1 + 2. Conventions Used in this Document...................... 1 + 3. Introduction and Overview.............................. 2 + 4. Commands............................................... 3 + 4.1. SETACL................................................. 3 + 4.2. DELETEACL.............................................. 4 + 4.3. GETACL................................................. 4 + 4.4. LISTRIGHTS............................................. 4 + 4.5. MYRIGHTS............................................... 5 + 5. Responses.............................................. 5 + 5.1. ACL.................................................... 5 + 5.2. LISTRIGHTS............................................. 6 + 5.3. MYRIGHTS............................................... 6 + 6. Formal Syntax.......................................... 6 + 7. References............................................. 7 + 8. Security Considerations................................ 7 + 9. Author's Address....................................... 8 + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + + + + + +Myers Standards Track [Page 1] + +RFC 2086 ACL extension January 1997 + + +3. Introduction and Overview + + The ACL extension is present in any IMAP4 implementation which + returns "ACL" as one of the supported capabilities to the CAPABILITY + command. + + An access control list is a set of pairs. + + Identifier is a US-ASCII string. The identifier anyone is reserved + to refer to the universal identity (all authentications, including + anonymous). All user name strings accepted by the LOGIN or + AUTHENTICATE commands to authenticate to the IMAP server are reserved + as identifiers for the corresponding user. Identifiers starting with + a dash ("-") are reserved for "negative rights", described below. + All other identifier strings are interpreted in an implementation- + defined manner. + + Rights is a string listing a (possibly empty) set of alphanumeric + characters, each character listing a set of operations which is being + controlled. Letters are reserved for ``standard'' rights, listed + below. The set of standard rights may only be extended by a + standards-track document. Digits are reserved for implementation or + site defined rights. The currently defined standard rights are: + + l - lookup (mailbox is visible to LIST/LSUB commands) + r - read (SELECT the mailbox, perform CHECK, FETCH, PARTIAL, + SEARCH, COPY from mailbox) + s - keep seen/unseen information across sessions (STORE SEEN flag) + w - write (STORE flags other than SEEN and DELETED) + i - insert (perform APPEND, COPY into mailbox) + p - post (send mail to submission address for mailbox, + not enforced by IMAP4 itself) + c - create (CREATE new sub-mailboxes in any implementation-defined + hierarchy) + d - delete (STORE DELETED flag, perform EXPUNGE) + a - administer (perform SETACL) + + An implementation may tie rights together or may force rights to + always or never be granted to particular identifiers. For example, + in an implementation that uses unix mode bits, the rights "wisd" are + tied, the "a" right is always granted to the owner of a mailbox and + is never granted to another user. If rights are tied in an + implementation, the implementation must be conservative in granting + rights in response to SETACL commands--unless all rights in a tied + set are specified, none of that set should be included in the ACL + entry for that identifier. A client may discover the set of rights + which may be granted to a given identifier in the ACL for a given + mailbox by using the LISTRIGHTS command. + + + +Myers Standards Track [Page 2] + +RFC 2086 ACL extension January 1997 + + + It is possible for multiple identifiers in an access control list to + apply to a given user (or other authentication identity). For + example, an ACL may include rights to be granted to the identifier + matching the user, one or more implementation-defined identifiers + matching groups which include the user, and/or the identifier + "anyone". How these rights are combined to determine the user's + access is implementation-defined. An implementation may choose, for + example, to use the union of the rights granted to the applicable + identifiers. An implementation may instead choose, for example, to + only use those rights granted to the most specific identifier present + in the ACL. A client may determine the set of rights granted to the + logged-in user for a given mailbox by using the MYRIGHTS command. + + When an identifier in an ACL starts with a dash ("-"), that indicates + that associated rights are to be removed from the identifier that is + prefixed by the dash. For example, if the identifier "-fred" is + granted the "w" right, that indicates that the "w" right is to be + removed from users matching the identifier "fred". Implementations + need not support having identifiers which start with a dash in ACLs. + +4. Commands + +4.1. SETACL + + Arguments: mailbox name + authentication identifier + access right modification + + Data: no specific data for this command + + Result: OK - setacl completed + NO - setacl failure: can't set acl + BAD - command unknown or arguments invalid + + The SETACL command changes the access control list on the + specified mailbox so that the specified identifier is granted + permissions as specified in the third argument. + + The third argument is a string containing an optional plus ("+") + or minus ("-") prefix, followed by zero or more rights characters. + If the string starts with a plus, the following rights are added + to any existing rights for the identifier. If the string starts + with a minus, the following rights are removed from any existing + rights for the identifier. If the string does not start with a + plus or minus, the rights replace any existing rights for the + identifier. + + + + + +Myers Standards Track [Page 3] + +RFC 2086 ACL extension January 1997 + + +4.2. DELETEACL + + Arguments: mailbox name + authentication identifier + + Data: no specific data for this command + + Result: OK - deleteacl completed + NO - deleteacl failure: can't delete acl + BAD - command unknown or arguments invalid + + The DELETEACL command removes any pair for the + specified identifier from the access control list for the specified + mailbox. + +4.3. GETACL + + Arguments: mailbox name + + Data: untagged responses: ACL + + Result: OK - getacl completed + NO - getacl failure: can't get acl + BAD - command unknown or arguments invalid + + The GETACL command returns the access control list for mailbox in + an untagged ACL reply. + + Example: C: A002 GETACL INBOX + S: * ACL INBOX Fred rwipslda + S: A002 OK Getacl complete + +4.4. LISTRIGHTS + + Arguments: mailbox name + authentication identifier + + Data: untagged responses: LISTRIGHTS + + Result: OK - listrights completed + NO - listrights failure: can't get rights list + BAD - command unknown or arguments invalid + + The LISTRIGHTS command takes a mailbox name and an identifier and + returns information about what rights may be granted to the identifier + in the ACL for the mailbox. + + + + + +Myers Standards Track [Page 4] + +RFC 2086 ACL extension January 1997 + + + Example: C: a001 LISTRIGHTS ~/Mail/saved smith + S: * LISTRIGHTS ~/Mail/saved smith la r swicd + S: a001 OK Listrights completed + + + C: a005 LISTRIGHTS archive.imap anyone + S: * LISTRIGHTS archive.imap anyone "" l r s w i p c d a + 0 1 2 3 4 5 6 7 8 9 + +4.5. MYRIGHTS + + Arguments: mailbox name + + Data: untagged responses: MYRIGHTS + + Result: OK - myrights completed + NO - myrights failure: can't get rights + BAD - command unknown or arguments invalid + + The MYRIGHTS command returns the set of rights that the user has + to mailbox in an untagged MYRIGHTS reply. + + Example: C: A003 MYRIGHTS INBOX + S: * MYRIGHTS INBOX rwipslda + S: A003 OK Myrights complete + +5. Responses + +5.1. ACL + + Data: mailbox name + zero or more identifier rights pairs + + The ACL response occurs as a result of a GETACL command. The first + string is the mailbox name for which this ACL applies. This is + followed by zero or more pairs of strings, each pair contains the + identifier for which the entry applies followed by the set of + rights that the identifier has. + + + + + + + + + + + + + +Myers Standards Track [Page 5] + +RFC 2086 ACL extension January 1997 + + +5.2. LISTRIGHTS + + Data: mailbox name + identifier + required rights + list of optional rights + + The LISTRIGHTS response occurs as a result of a LISTRIGHTS + command. The first two strings are the mailbox name and identifier + for which this rights list applies. Following the identifier is a + string containing the (possibly empty) set of rights the + identifier will always be granted in the mailbox. + + Following this are zero or more strings each containing a set of + rights the identifier may be granted in the mailbox. Rights + mentioned in the same string are tied together--either all must be + granted to the identifier in the mailbox or none may be granted. + + The same right may not be listed more than once in the LISTRIGHTS + command. + +5.3. MYRIGHTS + + Data: mailbox name + rights + + The MYRIGHTS response occurs as a result of a MYRIGHTS command. + The first string is the mailbox name for which these rights apply. + The second string is the set of rights that the client has. + +6. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + + + + + + + + + +Myers Standards Track [Page 6] + +RFC 2086 ACL extension January 1997 + + + acl_data ::= "ACL" SPACE mailbox *(SPACE identifier SPACE + rights) + + deleteacl ::= "DELETEACL" SPACE mailbox SPACE identifier + + getacl ::= "GETACL" SPACE mailbox + + identifier ::= astring + + listrights ::= "LISTRIGHTS" SPACE mailbox SPACE identifier + + listrights_data ::= "LISTRIGHTS" SPACE mailbox SPACE identifier + SPACE rights *(SPACE rights) + + mod_rights ::= astring + ;; +rights to add, -rights to remove + ;; rights to replace + + myrights ::= "MYRIGHTS" SPACE mailbox + + myrights_data ::= "MYRIGHTS" SPACE mailbox SPACE rights + + rights ::= astring + + setacl ::= "SETACL" SPACE mailbox SPACE identifier + SPACE mod_rights + +7. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", + RFC 1730, University of Washington, December 1994. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text + Messages", STD 11, RFC 822. + +8. Security Considerations + + An implementation must make sure the ACL commands themselves do not + give information about mailboxes with appropriately restricted ACL's. + For example, a GETACL command on a mailbox for which the user has + insufficient rights should not admit the mailbox exists, much less + return the mailbox's ACL. + + + + + + + + + +Myers Standards Track [Page 7] + +RFC 2086 ACL extension January 1997 + + +9. Author's Address + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave. + Pittsburgh PA, 15213-3890 + + Email: jgm+@cmu.edu + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Myers Standards Track [Page 8] + diff --git a/docs/rfcs/rfc2087.IMAP4_QUOTA_extension.txt b/docs/rfcs/rfc2087.IMAP4_QUOTA_extension.txt new file mode 100644 index 0000000..1db5b57 --- /dev/null +++ b/docs/rfcs/rfc2087.IMAP4_QUOTA_extension.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 2087 Carnegie Mellon +Category: Standards Track January 1997 + + + IMAP4 QUOTA extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + The QUOTA extension of the Internet Message Access Protocol [IMAP4] + permits administrative limits on resource usage (quotas) to be + manipulated through the IMAP protocol. + +Table of Contents + + 1. Abstract........................................... 1 + 2. Conventions Used in this Document.................. 1 + 3. Introduction and Overview.......................... 2 + 4. Commands........................................... 2 + 4.1. SETQUOTA Command................................... 2 + 4.2. GETQUOTA Command................................... 2 + 4.3. GETQUOTAROOT Command............................... 3 + 5. Responses.......................................... 3 + 5.1. QUOTA Response..................................... 3 + 5.2. QUOTAROOT Response................................. 4 + 6. Formal syntax...................................... 4 + 7. References......................................... 5 + 8. Security Considerations............................ 5 + 9. Author's Address................................... 5 + + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + + + + + + + +Myers Standards Track [Page 1] + +RFC 2087 QUOTA January 1997 + + +3. Introduction and Overview + + The QUOTA extension is present in any IMAP4 implementation which + returns "QUOTA" as one of the supported capabilities to the + CAPABILITY command. + + An IMAP4 server which supports the QUOTA capability may support + limits on any number of resources. Each resource has an atom name + and an implementation-defined interpretation which evaluates to an + integer. Examples of such resources are: + + Name Interpretation + + STORAGE Sum of messages' RFC822.SIZE, in units of 1024 octets + MESSAGE Number of messages + + + Each mailbox has zero or more implementation-defined named "quota + roots". Each quota root has zero or more resource limits. All + mailboxes that share the same named quota root share the resource + limits of the quota root. + + Quota root names do not necessarily have to match the names of + existing mailboxes. + +4. Commands + +4.1. SETQUOTA Command + + Arguments: quota root + list of resource limits + + Data: untagged responses: QUOTA + + Result: OK - setquota completed + NO - setquota error: can't set that data + BAD - command unknown or arguments invalid + + The SETQUOTA command takes the name of a mailbox quota root and a + list of resource limits. The resource limits for the named quota root + are changed to be the specified limits. Any previous resource limits + for the named quota root are discarded. + + If the named quota root did not previously exist, an implementation + may optionally create it and change the quota roots for any number of + existing mailboxes in an implementation-defined manner. + + + + + +Myers Standards Track [Page 2] + +RFC 2087 QUOTA January 1997 + + + Example: C: A001 SETQUOTA "" (STORAGE 512) + S: * QUOTA "" (STORAGE 10 512) + S: A001 OK Setquota completed + +4.2. GETQUOTA Command + + Arguments: quota root + + Data: untagged responses: QUOTA + + Result: OK - getquota completed + NO - getquota error: no such quota root, permission + denied + BAD - command unknown or arguments invalid + + The GETQUOTA command takes the name of a quota root and returns the + quota root's resource usage and limits in an untagged QUOTA response. + + Example: C: A003 GETQUOTA "" + S: * QUOTA "" (STORAGE 10 512) + S: A003 OK Getquota completed + +4.3. GETQUOTAROOT Command + + Arguments: mailbox name + + Data: untagged responses: QUOTAROOT, QUOTA + + Result: OK - getquota completed + NO - getquota error: no such mailbox, permission denied + BAD - command unknown or arguments invalid + + The GETQUOTAROOT command takes the name of a mailbox and returns the + list of quota roots for the mailbox in an untagged QUOTAROOT + response. For each listed quota root, it also returns the quota + root's resource usage and limits in an untagged QUOTA response. + + Example: C: A003 GETQUOTAROOT INBOX + S: * QUOTAROOT INBOX "" + S: * QUOTA "" (STORAGE 10 512) + S: A003 OK Getquota completed + + + + + + + + + + +Myers Standards Track [Page 3] + +RFC 2087 QUOTA January 1997 + + +5. Responses + +5.1. QUOTA Response + + Data: quota root name + list of resource names, usages, and limits + + This response occurs as a result of a GETQUOTA or GETQUOTAROOT + command. The first string is the name of the quota root for which + this quota applies. + + The name is followed by a S-expression format list of the resource + usage and limits of the quota root. The list contains zero or + more triplets. Each triplet conatins a resource name, the current + usage of the resource, and the resource limit. + + Resources not named in the list are not limited in the quota root. + Thus, an empty list means there are no administrative resource + limits in the quota root. + + Example: S: * QUOTA "" (STORAGE 10 512) + +5.2. QUOTAROOT Response + + Data: mailbox name + zero or more quota root names + + This response occurs as a result of a GETQUOTAROOT command. The + first string is the mailbox and the remaining strings are the + names of the quota roots for the mailbox. + + Example: S: * QUOTAROOT INBOX "" + S: * QUOTAROOT comp.mail.mime + +6. Formal syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in RFC 822 with one exception; the + delimiter used with the "#" construct is a single space (SP) and not + one or more commas. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + + + + + +Myers Standards Track [Page 4] + +RFC 2087 QUOTA January 1997 + + + getquota ::= "GETQUOTA" SP astring + + getquotaroot ::= "GETQUOTAROOT" SP astring + + quota_list ::= "(" #quota_resource ")" + + quota_resource ::= atom SP number SP number + + quota_response ::= "QUOTA" SP astring SP quota_list + + quotaroot_response + ::= "QUOTAROOT" SP astring *(SP astring) + + setquota ::= "SETQUOTA" SP astring SP setquota_list + + setquota_list ::= "(" 0#setquota_resource ")" + + setquota_resource ::= atom SP number + +7. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", + RFC 1730, University of Washington, December 1994. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet + Text Messages", STD 11, RFC 822. + +8. Security Considerations + + Implementors should be careful to make sure the implementation of + these commands does not violate the site's security policy. The + resource usage of other users is likely to be considered confidential + information and should not be divulged to unauthorized persons. + +9. Author's Address + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave. + Pittsburgh PA, 15213-3890 + + EMail: jgm+@cmu.edu + + + + + + + + + +Myers Standards Track [Page 5] + diff --git a/docs/rfcs/rfc2088.IMAP4_non_synchronizing_literals.txt b/docs/rfcs/rfc2088.IMAP4_non_synchronizing_literals.txt new file mode 100644 index 0000000..f36cc76 --- /dev/null +++ b/docs/rfcs/rfc2088.IMAP4_non_synchronizing_literals.txt @@ -0,0 +1,115 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 2088 Carnegie Mellon +Cateogry: Standards Track January 1997 + + + IMAP4 non-synchronizing literals + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + The Internet Message Access Protocol [IMAP4] contains the "literal" + syntactic construct for communicating strings. When sending a + literal from client to server, IMAP4 requires the client to wait for + the server to send a command continuation request between sending the + octet count and the string data. This document specifies an + alternate form of literal which does not require this network round + trip. + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + +3. Specification + + The non-synchronizing literal is added an alternate form of literal, + and may appear in communication from client to server instead of the + IMAP4 form of literal. The IMAP4 form of literal, used in + communication from client to server, is referred to as a + synchronizing literal. + + Non-synchronizing literals may be used with any IMAP4 server + implementation which returns "LITERAL+" as one of the supported + capabilities to the CAPABILITY command. If the server does not + advertise the LITERAL+ capability, the client must use synchronizing + literals instead. + + The non-synchronizing literal is distinguished from the original + synchronizing literal by having a plus ('+') between the octet count + and the closing brace ('}'). The server does not generate a command + continuation request in response to a non-synchronizing literal, and + + + +Myers Standards Track [Page 1] + +RFC 2088 LITERAL January 1997 + + + clients are not required to wait before sending the octets of a non- + synchronizing literal. + + The protocol receiver of an IMAP4 server must check the end of every + received line for an open brace ('{') followed by an octet count, a + plus ('+'), and a close brace ('}') immediately preceeding the CRLF. + If it finds this sequence, it is the octet count of a non- + synchronizing literal and the server MUST treat the specified number + of following octets and the following line as part of the same + command. A server MAY still process commands and reject errors on a + line-by-line basis, as long as it checks for non-synchronizing + literals at the end of each line. + + Example: C: A001 LOGIN {11+} + C: FRED FOOBAR {7+} + C: fat man + S: A001 OK LOGIN completed + +4. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + literal ::= "{" number ["+"] "}" CRLF *CHAR8 + ;; Number represents the number of CHAR8 octets + +6. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", + draft-crispin-imap-base-XX.txt, University of Washington, April 1996. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text + Messages", STD 11, RFC 822. + +7. Security Considerations + + There are no known security issues with this extension. + +8. Author's Address + + John G. Myers + Carnegie-Mellon University + 5000 Forbes Ave. + Pittsburgh PA, 15213-3890 + + Email: jgm+@cmu.edu + + + +Myers Standards Track [Page 2] + diff --git a/docs/rfcs/rfc2095.IMAP-POP_AUTHorize_extension.txt b/docs/rfcs/rfc2095.IMAP-POP_AUTHorize_extension.txt new file mode 100644 index 0000000..f1167f6 --- /dev/null +++ b/docs/rfcs/rfc2095.IMAP-POP_AUTHorize_extension.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group J. Klensin +Request for Comments: 2095 R. Catoe +Category: Standards Track P. Krumviede + MCI + January 1997 + + + IMAP/POP AUTHorize Extension for Simple Challenge/Response + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + While IMAP4 supports a number of strong authentication mechanisms as + described in RFC 1731, it lacks any mechanism that neither passes + cleartext, reusable passwords across the network nor requires either + a significant security infrastructure or that the mail server update + a mail-system-wide user authentication file on each mail access. + This specification provides a simple challenge-response + authentication protocol that is suitable for use with IMAP4. Since + it utilizes Keyed-MD5 digests and does not require that the secret be + stored in the clear on the server, it may also constitute an + improvement on APOP for POP3 use as specified in RFC 1734. + +1. Introduction + + Existing Proposed Standards specify an AUTHENTICATE mechanism for the + IMAP4 protocol [IMAP, IMAP-AUTH] and a parallel AUTH mechanism for + the POP3 protocol [POP3-AUTH]. The AUTHENTICATE mechanism is + intended to be extensible; the four methods specified in [IMAP-AUTH] + are all fairly powerful and require some security infrastructure to + support. The base POP3 specification [POP3] also contains a + lightweight challenge-response mechanism called APOP. APOP is + associated with most of the risks associated with such protocols: in + particular, it requires that both the client and server machines have + access to the shared secret in cleartext form. CRAM offers a method + for avoiding such cleartext storage while retaining the algorithmic + simplicity of APOP in using only MD5, though in a "keyed" method. + + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 1] + +RFC 2095 IMAP/POP AUTHorize Extension January 1997 + + + At present, IMAP [IMAP] lacks any facility corresponding to APOP. + The only alternative to the strong mechanisms identified in [IMAP- + AUTH] is a presumably cleartext username and password, supported + through the LOGIN command in [IMAP]. This document describes a + simple challenge-response mechanism, similar to APOP and PPP CHAP + [PPP], that can be used with IMAP (and, in principle, with POP3). + + This mechanism also has the advantage over some possible alternatives + of not requiring that the server maintain information about email + "logins" on a per-login basis. While mechanisms that do require such + per-login history records may offer enhanced security, protocols such + as IMAP, which may have several connections between a given client + and server open more or less simultaneous, may make their + implementation particularly challenging. + +2. Challenge-Response Authentication Mechanism (CRAM) + + The authentication type associated with CRAM is "CRAM-MD5". + + The data encoded in the first ready response contains an + presumptively arbitrary string of random digits, a timestamp, and the + fully-qualified primary host name of the server. The syntax of the + unencoded form must correspond to that of an RFC 822 'msg-id' + [RFC822] as described in [POP3]. + + The client makes note of the data and then responds with a string + consisting of the user name, a space, and a 'digest'. The latter is + computed by applying the keyed MD5 algorithm from [KEYED-MD5] where + the key is a shared secret and the digested text is the timestamp + (including angle-brackets). + + This shared secret is a string known only to the client and server. + The `digest' parameter itself is a 16-octet value which is sent in + hexadecimal format, using lower-case ASCII characters. + + When the server receives this client response, it verifies the digest + provided. If the digest is correct, the server should consider the + client authenticated and respond appropriately. + + Keyed MD5 is chosen for this application because of the greater + security imparted to authentication of short messages. In addition, + the use of the techniques described in [KEYED-MD5] for precomputation + of intermediate results make it possible to avoid explicit cleartext + storage of the shared secret on the server system by instead storing + the intermediate results which are known as "contexts". + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 2] + +RFC 2095 IMAP/POP AUTHorize Extension January 1997 + + + CRAM does not support a protection mechanism. + + Example: + + The examples in this document show the use of the CRAM mechanism with + the IMAP4 AUTHENTICATE command [IMAP-AUTH]. The base64 encoding of + the challenges and responses is part of the IMAP4 AUTHENTICATE + command, not part of the CRAM specification itself. + + S: * OK IMAP4 Server + C: A0001 AUTHENTICATE CRAM-MD5 + S: + PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2UucmVzdG9uLm1jaS5uZXQ+ + C: dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw + S: A0001 OK CRAM authentication successful + + In this example, the shared secret is the string + 'tanstaaftanstaaf'. Hence, the Keyed MD5 digest is produced by + calculating + + MD5((tanstaaftanstaaf XOR opad), + MD5((tanstaaftanstaaf XOR ipad), + <1896.697170952@postoffice.reston.mci.net>)) + + where ipad and opad are as defined in the keyed-MD5 Work in + Progress [KEYED-MD5] and the string shown in the challenge is the + base64 encoding of <1896.697170952@postoffice.reston.mci.net>. The + shared secret is null-padded to a length of 64 bytes. If the + shared secret is longer than 64 bytes, the MD5 digest of the + shared secret is used as a 16 byte input to the keyed MD5 + calculation. + + This produces a digest value (in hexadecimal) of + + b913a602c7eda7a495b4e6e7334d3890 + + The user name is then prepended to it, forming + + tim b913a602c7eda7a495b4e6e7334d3890 + + Which is then base64 encoded to meet the requirements of the IMAP4 + AUTHENTICATE command (or the similar POP3 AUTH command), yielding + + dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw + + + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 3] + +RFC 2095 IMAP/POP AUTHorize Extension January 1997 + + +3. References + + [CHAP] Lloyd, B., and W. Simpson, "PPP Authentication Protocols", + RFC 1334, October 1992. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanisms", + RFC 1731, Carnegie Mellon, December 1994. + + [KEYED-MD5] Krawczyk, H., "HMAC-MD5: Keyed-MD5 for Message + Authentication", Work in Progess. + + [MD5] Rivest, R., "The MD5 Message Digest Algorithm", + RFC 1321, MIT Laboratory for Computer Science, April 1992. + + [POP3] Myers, J., and M. Rose, "Post Office Protocol - Version 3", + STD 53, RFC 1939, Carnegie Mellon, May 1996. + + [POP3-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, + Carnegie Mellon, December, 1994. + +4. Security Considerations + + It is conjectured that use of the CRAM authentication mechanism + provides origin identification and replay protection for a session. + Accordingly, a server that implements both a cleartext password + command and this authentication type should not allow both methods of + access for a given user. + + While the saving, on the server, of "contexts" (see section 2) is + marginally better than saving the shared secrets in cleartext as is + required by CHAP [CHAP] and APOP [POP3], it is not sufficient to + protect the secrets if the server itself is compromised. + Consequently, servers that store the secrets or contexts must both be + protected to a level appropriate to the potential information value + in user mailboxes and identities. + + As the length of the shared secret increases, so does the difficulty + of deriving it. + + While there are now suggestions in the literature that the use of MD5 + and keyed MD5 in authentication procedures probably has a limited + effective lifetime, the technique is now widely deployed and widely + understood. It is believed that this general understanding may + assist with the rapid replacement, by CRAM-MD5, of the current uses + of permanent cleartext passwords in IMAP. This document has been + + + +Klensin, Catoe & Krumviede Standards Track [Page 4] + +RFC 2095 IMAP/POP AUTHorize Extension January 1997 + + + deliberately written to permit easy upgrading to use SHA (or whatever + alternatives emerge) when they are considered to be widely available + and adequately safe. + + Even with the use of CRAM, users are still vulnerable to active + attacks. An example of an increasingly common active attack is 'TCP + Session Hijacking' as described in CERT Advisory CA-95:01 [CERT95]. + + See section 1 above for additional discussion. + +5. Acknowledgements + + This memo borrows ideas and some text liberally from [POP3] and + [RFC-1731] and thanks are due the authors of those documents. Ran + Atkinson made a number of valuable technical and editorial + contributions to the document. + +6. Authors' Addresses + + John C. Klensin + MCI Telecommunications + 800 Boylston St, 7th floor + Boston, MA 02199 + USA + + EMail: klensin@mci.net + Phone: +1 617 960 1011 + + Randy Catoe + MCI Telecommunications + 2100 Reston Parkway + Reston, VA 22091 + USA + + EMail: randy@mci.net + Phone: +1 703 715 7366 + + Paul Krumviede + MCI Telecommunications + 2100 Reston Parkway + Reston, VA 22091 + USA + + EMail: paul@mci.net + Phone: +1 703 715 7251 + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 5] + diff --git a/docs/rfcs/rfc2177.IMAP4_IDLE_command.txt b/docs/rfcs/rfc2177.IMAP4_IDLE_command.txt new file mode 100644 index 0000000..c11c765 --- /dev/null +++ b/docs/rfcs/rfc2177.IMAP4_IDLE_command.txt @@ -0,0 +1,227 @@ + + + + + + +Network Working Group B. Leiba +Request for Comments: 2177 IBM T.J. Watson Research Center +Category: Standards Track June 1997 + + + IMAP4 IDLE command + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + The Internet Message Access Protocol [IMAP4] requires a client to + poll the server for changes to the selected mailbox (new mail, + deletions). It's often more desirable to have the server transmit + updates to the client in real time. This allows a user to see new + mail immediately. It also helps some real-time applications based on + IMAP, which might otherwise need to poll extremely often (such as + every few seconds). (While the spec actually does allow a server to + push EXISTS responses aysynchronously, a client can't expect this + behaviour and must poll.) + + This document specifies the syntax of an IDLE command, which will + allow a client to tell the server that it's ready to accept such + real-time updates. + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as described in RFC 2060 + [IMAP4]. + +3. Specification + + IDLE Command + + Arguments: none + + Responses: continuation data will be requested; the client sends + the continuation data "DONE" to end the command + + + +Leiba Standards Track [Page 1] + +RFC 2177 IMAP4 IDLE command June 1997 + + + + Result: OK - IDLE completed after client sent "DONE" + NO - failure: the server will not allow the IDLE + command at this time + BAD - command unknown or arguments invalid + + The IDLE command may be used with any IMAP4 server implementation + that returns "IDLE" as one of the supported capabilities to the + CAPABILITY command. If the server does not advertise the IDLE + capability, the client MUST NOT use the IDLE command and must poll + for mailbox updates. In particular, the client MUST continue to be + able to accept unsolicited untagged responses to ANY command, as + specified in the base IMAP specification. + + The IDLE command is sent from the client to the server when the + client is ready to accept unsolicited mailbox update messages. The + server requests a response to the IDLE command using the continuation + ("+") response. The IDLE command remains active until the client + responds to the continuation, and as long as an IDLE command is + active, the server is now free to send untagged EXISTS, EXPUNGE, and + other messages at any time. + + The IDLE command is terminated by the receipt of a "DONE" + continuation from the client; such response satisfies the server's + continuation request. At that point, the server MAY send any + remaining queued untagged responses and then MUST immediately send + the tagged response to the IDLE command and prepare to process other + commands. As in the base specification, the processing of any new + command may cause the sending of unsolicited untagged responses, + subject to the ambiguity limitations. The client MUST NOT send a + command while the server is waiting for the DONE, since the server + will not be able to distinguish a command from a continuation. + + The server MAY consider a client inactive if it has an IDLE command + running, and if such a server has an inactivity timeout it MAY log + the client off implicitly at the end of its timeout period. Because + of that, clients using IDLE are advised to terminate the IDLE and + re-issue it at least every 29 minutes to avoid being logged off. + This still allows a client to receive immediate mailbox updates even + though it need only "poll" at half hour intervals. + + + + + + + + + + + +Leiba Standards Track [Page 2] + +RFC 2177 IMAP4 IDLE command June 1997 + + + Example: C: A001 SELECT INBOX + S: * FLAGS (Deleted Seen) + S: * 3 EXISTS + S: * 0 RECENT + S: * OK [UIDVALIDITY 1] + S: A001 OK SELECT completed + C: A002 IDLE + S: + idling + ...time passes; new mail arrives... + S: * 4 EXISTS + C: DONE + S: A002 OK IDLE terminated + ...another client expunges message 2 now... + C: A003 FETCH 4 ALL + S: * 4 FETCH (...) + S: A003 OK FETCH completed + C: A004 IDLE + S: * 2 EXPUNGE + S: * 3 EXISTS + S: + idling + ...time passes; another client expunges message 3... + S: * 3 EXPUNGE + S: * 2 EXISTS + ...time passes; new mail arrives... + S: * 3 EXISTS + C: DONE + S: A004 OK IDLE terminated + C: A005 FETCH 3 ALL + S: * 3 FETCH (...) + S: A005 OK FETCH completed + C: A006 IDLE + +4. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + command_auth ::= append / create / delete / examine / list / lsub / + rename / select / status / subscribe / unsubscribe + / idle + ;; Valid only in Authenticated or Selected state + + idle ::= "IDLE" CRLF "DONE" + + + + + + +Leiba Standards Track [Page 3] + +RFC 2177 IMAP4 IDLE command June 1997 + + +5. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + +6. Security Considerations + + There are no known security issues with this extension. + +7. Author's Address + + Barry Leiba + IBM T.J. Watson Research Center + 30 Saw Mill River Road + Hawthorne, NY 10532 + + Email: leiba@watson.ibm.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Leiba Standards Track [Page 4] + diff --git a/docs/rfcs/rfc2180.IMAP4_multi-accessed_Mailbox_practice.txt b/docs/rfcs/rfc2180.IMAP4_multi-accessed_Mailbox_practice.txt new file mode 100644 index 0000000..5760700 --- /dev/null +++ b/docs/rfcs/rfc2180.IMAP4_multi-accessed_Mailbox_practice.txt @@ -0,0 +1,787 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 2180 Microsoft +Category: Informational July 1997 + + + IMAP4 Multi-Accessed Mailbox Practice + +Status of this Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + +1. Abstract + + IMAP4[RFC-2060] is rich client/server protocol that allows a client + to access and manipulate electronic mail messages on a server. + Within the protocol framework, it is possible to have differing + results for particular client/server interactions. If a protocol does + not allow for this, it is often unduly restrictive. + + For example, when multiple clients are accessing a mailbox and one + attempts to delete the mailbox, an IMAP4 server may choose to + implement a solution based upon server architectural constraints or + individual preference. + + With this flexibility comes greater client responsibility. It is not + sufficient for a client to be written based upon the behavior of a + particular IMAP server. Rather the client must be based upon the + behavior allowed by the protocol. + + By documenting common IMAP4 server practice for the case of + simultaneous client access to a mailbox, we hope to ensure the widest + amount of inter-operation between IMAP4 clients and servers. + + The behavior described in this document reflects the practice of some + existing servers or behavior that the consensus of the IMAP mailing + list has deemed to be reasonable. The behavior described within this + document is believed to be [RFC-2060] compliant. However, this + document is not meant to define IMAP4 compliance, nor is it an + exhaustive list of valid IMAP4 behavior. [RFC-2060] must always be + consulted to determine IMAP4 compliance, especially for server + behavior not described within this document. + + + + + + + + +Gahrns Informational [Page 1] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +2. Conventions used in this document + + In examples,"C1:", "C2:" and "C3:" indicate lines sent by 3 different + clients (client #1, client #2 and client #3) that are connected to a + server. "S1:", "S2:" and "S3:" indicated lines sent by the server to + client #1, client #2 and client #3 respectively. + + A shared mailbox, is a mailbox that can be used by multiple users. + + A multi-accessed mailbox, is a mailbox that has multiple clients + simultaneously accessing it. + + A client is said to have accessed a mailbox after a successful SELECT + or EXAMINE command. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + + +3. Deletion/Renaming of a multi-accessed mailbox + + If an external agent or multiple clients are accessing a mailbox, + care must be taken when handling the deletion or renaming of the + mailbox. Following are some strategies an IMAP server may choose to + use when dealing with this situation. + + +3.1. The server MAY fail the DELETE/RENAME command of a multi-accessed + mailbox + + In some cases, this behavior may not be practical. For example, if a + large number of clients are accessing a shared mailbox, the window in + which no clients have the mailbox accessed may be small or non- + existent, effectively rendering the mailbox undeletable or + unrenamable. + + Example: + + + + C1: A001 DELETE FOO + S1: A001 NO Mailbox FOO is in use by another user. + + + + + + + +Gahrns Informational [Page 2] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +3.2. The server MAY allow the DELETE command of a multi-accessed + mailbox, but keep the information in the mailbox available for + those clients that currently have access to the mailbox. + + When all clients have finished accessing the mailbox, it is + permanently removed. For clients that do not already have access to + the mailbox, the 'ghosted' mailbox would not be available. For + example, it would not be returned to these clients in a subsequent + LIST or LSUB command and would not be a valid mailbox argument to any + other IMAP command until the reference count of clients accessing the + mailbox reached 0. + + In some cases, this behavior may not be desirable. For example if + someone created a mailbox with offensive or sensitive information, + one might prefer to have the mailbox deleted and all access to the + information contained within removed immediately, rather than + continuing to allow access until the client closes the mailbox. + + Furthermore, this behavior, may prevent 'recycling' of the same + mailbox name until all clients have finished accessing the original + mailbox. + + Example: + + + + C1: A001 DELETE FOO + S1: A001 OK Mailbox FOO is deleted. + + + + C2: B001 STORE 1 +FLAGS (\Seen) + S2: * 1 FETCH FLAGS (\Seen) + S2: B001 OK STORE completed + + + + C3: C001 STATUS FOO (MESSAGES) + S3: C001 NO Mailbox does not exist + + + + C3: C002 CREATE FOO + S3: C002 NO Mailbox FOO is still in use. Try again later. + + + + +Gahrns Informational [Page 3] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + + + C2: B002 CLOSE + S2: B002 OK CLOSE Completed + + + + C3: C003 CREATE FOO + S3: C003 OK CREATE Completed + + +3.3. The server MAY allow the DELETE/RENAME of a multi-accessed + mailbox, but disconnect all other clients who have the mailbox + accessed by sending a untagged BYE response. + + A server may often choose to disconnect clients in the DELETE case, + but may choose to implement a "friendlier" method for the RENAME + case. + + Example: + + + + C1: A002 DELETE FOO + S1: A002 OK DELETE completed. + + + S2: * BYE Mailbox FOO has been deleted. + + +3.4. The server MAY allow the RENAME of a multi-accessed mailbox by + simply changing the name attribute on the mailbox. + + Other clients that have access to the mailbox can continue issuing + commands such as FETCH that do not reference the mailbox name. + Clients would discover the renaming the next time they referred to + the old mailbox name. Some servers MAY choose to include the + [NEWNAME] response code in their tagged NO response to a command that + contained the old mailbox name, as a hint to the client that the + operation can succeed if the command is issued with the new mailbox + name. + + + + + + + +Gahrns Informational [Page 4] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + Example: + + + + C1: A001 RENAME FOO BAR + S1: A001 OK RENAME completed. + + + + C2: B001 FETCH 2:4 (FLAGS) + S2: * 2 FETCH . . . + S2: * 3 FETCH . . . + S2: * 4 FETCH . . . + S2: B001 OK FETCH completed + + + + C2: B002 APPEND FOO {300} C2: Date: Mon, 7 Feb 1994 + 21:52:25 0800 (PST) C2: . . . S2: B002 NO [NEWNAME FOO + BAR] Mailbox has been renamed + + +4. Expunging of messages on a multi-accessed mailbox + + If an external agent or multiple clients are accessing a mailbox, + care must be taken when handling the EXPUNGE of messages. Other + clients accessing the mailbox may be in the midst of issuing a + command that depends upon message sequence numbers. Because an + EXPUNGE response can not be sent while responding to a FETCH, STORE + or SEARCH command, it is not possible to immediately notify the + client of the EXPUNGE. This can result in ambiguity if the client + issues a FETCH, STORE or SEARCH operation on a message that has been + EXPUNGED. + + +4.1. Fetching of expunged messages + + Following are some strategies an IMAP server may choose to use when + dealing with a FETCH command on expunged messages. + + + + + + + + + +Gahrns Informational [Page 5] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + Consider the following scenario: + + - Client #1 and Client #2 have mailbox FOO selected. + - There are 7 messages in the mailbox. + - Messages 4:7 are marked for deletion. + - Client #1 issues an EXPUNGE, to expunge messages 4:7 + + +4.1.1. The server MAY allow the EXPUNGE of a multi-accessed mailbox but + keep the messages available to satisfy subsequent FETCH commands + until it is able to send an EXPUNGE response to each client. + + In some cases, the behavior of keeping "ghosted" messages may not be + desirable. For example if a message contained offensive or sensitive + information, one might prefer to instantaneously remove all access to + the information, regardless of whether another client is in the midst + of accessing it. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 FETCH 4:7 RFC822 + S2: * 4 FETCH RFC822 . . . (RFC822 info returned) + S2: * 5 FETCH RFC822 . . . (RFC822 info returned) + S2: * 6 FETCH RFC822 . . . (RFC822 info returned) + S2: * 7 FETCH RFC822 . . . (RFC822 info returned) + S2: B001 OK FETCH Completed + + + + C2: B002 NOOP + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 3 EXISTS + S2: B002 OK NOOP Complete + + + + C2: B003 FETCH 4:7 RFC822 + S2: B003 NO Messages 4:7 are no longer available. + + + + + + +Gahrns Informational [Page 6] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.1.2 The server MAY allow the EXPUNGE of a multi-accessed mailbox, + and on subsequent FETCH commands return FETCH responses only for + non-expunged messages and a tagged NO. + + After receiving a tagged NO FETCH response, the client SHOULD issue a + NOOP command so that it will be informed of any pending EXPUNGE + responses. The client may then either reissue the failed FETCH + command, or by examining the EXPUNGE response from the NOOP and the + FETCH response from the FETCH, determine that the FETCH failed + because of pending expunges. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 FETCH 3:5 ENVELOPE + S2: * 3 FETCH ENVELOPE . . . (ENVELOPE info returned) + S2: B001 NO Some of the requested messages no longer exist + + + + C2: B002 NOOP + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 3 EXISTS + S2: B002 OK NOOP Completed. + + + + + + + + + + + + + + + + + + +Gahrns Informational [Page 7] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.1.3 The server MAY allow the EXPUNGE of a multi-accessed mailbox, and + on subsequent FETCH commands return the usual FETCH responses for + non-expunged messages, "NIL FETCH Responses" for expunged + messages, and a tagged OK response. + + If all of the messages in the subsequent FETCH command have been + expunged, the server SHOULD return only a tagged NO. In this case, + the client SHOULD issue a NOOP command so that it will be informed of + any pending EXPUNGE responses. The client may then either reissue + the failed FETCH command, or by examining the EXPUNGE response from + the NOOP, determine that the FETCH failed because of pending + expunges. + + "NIL FETCH responses" are a representation of empty data as + appropriate for the FETCH argument specified. + + Example: + + * 1 FETCH (ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL)) + * 1 FETCH (FLAGS ()) + * 1 FETCH (INTERNALDATE "00-Jan-0000 00:00:00 +0000") + * 1 FETCH (RFC822 "") + * 1 FETCH (RFC822.HEADER "") + * 1 FETCH (RFC822.TEXT "") + * 1 FETCH (RFC822.SIZE 0) + * 1 FETCH (BODY ("TEXT" "PLAIN" NIL NIL NIL "7BIT" 0 0) + * 1 FETCH (BODYSTRUCTURE ("TEXT" "PLAIN" NIL NIL NIL "7BIT" 0 0) + * 1 FETCH (BODY[
] "") + * 1 FETCH (BODY[
] "") + + In some cases, a client may not be able to distinguish between "NIL + FETCH responses" received because a message was expunged and those + received because the data actually was NIL. For example, a * 5 + FETCH (FLAGS ()) response could be received if no flags were set on + message 5, or because message 5 was expunged. In a case of potential + ambiguity, the client SHOULD issue a command such as NOOP to force + the sending of the EXPUNGE responses to resolve any ambiguity. + + Example: (Building upon the scenario outlined in 4.1.) + + + + + + + + + + +Gahrns Informational [Page 8] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + C2: B002 FETCH 3:5 ENVELOPE + S2: * 3 FETCH ENVELOPE . . . (ENVELOPE info returned) + S2: * 4 FETCH ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL + NIL NIL) + S2: * 5 FETCH ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL + NIL NIL) + S2: B002 OK FETCH Completed + + + + C2: B002 FETCH 4:7 ENVELOPE + S2: B002 NO Messages 4:7 have been expunged. + + +4.1.4 To avoid the situation altogether, the server MAY fail the + EXPUNGE of a multi-accessed mailbox + + In some cases, this behavior may not be practical. For example, if a + large number of clients are accessing a shared mailbox, the window in + which no clients have the mailbox accessed may be small or non- + existent, effectively rendering the message unexpungeable. + + +4.2. Storing of expunged messages + + Following are some strategies an IMAP server may choose to use when + dealing with a STORE command on expunged messages. + + +4.2.1 If the ".SILENT" suffix is used, and the STORE completed + successfully for all the non-expunged messages, the server SHOULD + return a tagged OK. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 STORE 1:7 +FLAGS.SILENT (\SEEN) + S2: B001 OK + + + + + + + + + +Gahrns Informational [Page 9] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.2.2. If the ".SILENT" suffix is not used, and only expunged messages + are referenced, the server SHOULD return only a tagged NO. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 STORE 5:7 +FLAGS (\SEEN) + S2: B001 NO Messages have been expunged + + +4.2.3. If the ".SILENT" suffix is not used, and a mixture of expunged + and non-expunged messages are referenced, the server MAY set the + flags and return a FETCH response for the non-expunged messages + along with a tagged NO. + + After receiving a tagged NO STORE response, the client SHOULD issue a + NOOP command so that it will be informed of any pending EXPUNGE + responses. The client may then either reissue the failed STORE + command, or by examining the EXPUNGE responses from the NOOP and + FETCH responses from the STORE, determine that the STORE failed + because of pending expunges. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 STORE 1:7 +FLAGS (\SEEN) + S2: * FETCH 1 FLAGS (\SEEN) + S2: * FETCH 2 FLAGS (\SEEN) + S2: * FETCH 3 FLAGS (\SEEN) + S2: B001 NO Some of the messages no longer exist. + + C2: B002 NOOP + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 3 EXISTS + S2: B002 OK NOOP Completed. + + + + + + + + +Gahrns Informational [Page 10] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.2.4. If the ".SILENT" suffix is not used, and a mixture of expunged + and non-expunged messages are referenced, the server MAY return + an untagged NO and not set any flags. + + After receiving a tagged NO STORE response, the client SHOULD issue a + NOOP command so that it will be informed of any pending EXPUNGE + responses. The client would then re-issue the STORE command after + updating its message list per any EXPUNGE response. + + If a large number of clients are accessing a shared mailbox, the + window in which there are no pending expunges may be small or non- + existent, effectively disallowing a client from setting the flags on + all messages at once. + + Example: (Building upon the scenario outlined in 4.1.) + + + + C2: B001 STORE 1:7 +FLAGS (\SEEN) + S2: B001 NO Some of the messages no longer exist. + + + + C2: B002 NOOP + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 4 EXPUNGE + S2: * 3 EXISTS + S2: B002 OK NOOP Completed. + + + + C2: B003 STORE 1:3 +FLAGS (\SEEN) S2: * FETCH 1 FLAGS + (\SEEN) S2: * FETCH 2 FLAGS (\SEEN) S2: * FETCH 3 FLAGS + (\SEEN) S2: B003 OK STORE Completed + + +4.3. Searching of expunged messages + + A server MAY simply not return a search response for messages that + have been expunged and it has not been able to inform the client + about. If a client was expecting a particular message to be returned + in a search result, and it was not, the client SHOULD issue a NOOP + command to see if the message was expunged by another client. + + + + +Gahrns Informational [Page 11] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +4.4 Copying of expunged messages + + COPY is the only IMAP4 sequence number command that is safe to allow + an EXPUNGE response on. This is because a client is not permitted to + cascade several COPY commands together. A client is required to wait + and confirm that the copy worked before issuing another one. + +4.4.1 The server MAY disallow the COPY of messages in a multi-access + mailbox that contains expunged messages. + + Pending EXPUNGE response(s) MUST be returned to the COPY command. + + Example: + + C: A001 COPY 2,4,6,8 FRED + S: * 4 EXPUNGE + S: A001 NO COPY rejected, because some of the requested + messages were expunged + + Note: Non of the above messages are copied because if a COPY command + is unsuccessful, the server MUST restore the destination mailbox to + its state before the COPY attempt. + + +4.4.2 The server MAY allow the COPY of messages in a multi-access + mailbox that contains expunged messages. + + Pending EXPUNGE response(s) MUST be returned to the COPY command. + Messages that are copied are messages corresponding to sequence + numbers before any EXPUNGE response. + + Example: + + C: A001 COPY 2,4,6,8 FRED + S: * 3 EXPUNGE + S: A001 OK COPY completed + + In the above example, the messages that are copied to FRED are + messages 2,4,6,8 at the start of the COPY command. These are + equivalent to messages 2,3,5,7 at the end of the COPY command. The + EXPUNGE response can't take place until after the messages from the + COPY command are identified (because of the "no expunge while no + commands in progress" rule). + + + + + + + + +Gahrns Informational [Page 12] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + + Example: + + C: A001 COPY 2,4,6,8 FRED + S: * 4 EXPUNGE + S: A001 OK COPY completed + + In the above example, message 4 was copied before it was expunged, + and MUST appear in the destination mailbox FRED. + + +5. Security Considerations + + This document describes behavior of servers that use the IMAP4 + protocol, and as such, has the same security considerations as + described in [RFC-2060]. + + In particular, some described server behavior does not allow for the + immediate deletion of information when a mailbox is accessed by + multiple clients. This may be a consideration when dealing with + sensitive information where immediate deletion would be preferred. + + +6. References + + [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, Harvard University, March 1997. + + +7. Acknowledgments + + This document is the result of discussions on the IMAP4 mailing list + and is meant to reflect consensus of this group. In particular, + Raymond Cheng, Mark Crispin, Jim Evans, Erik Forsberg, Steve Hole, + Mark Keasling, Barry Leiba, Syd Logan, John Mani, Pat Moran, Larry + Osterman, Chris Newman, Bart Schaefer, Vladimir Vulovic, and Jack De + Winter were active participants in this discussion or made + suggestions to this document. + + + + + + + + + + + +Gahrns Informational [Page 13] + +RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 + + +8. Author's Address + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98072 + + Phone: (206) 936-9833 + EMail: mikega@microsoft.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gahrns Informational [Page 14] + diff --git a/docs/rfcs/rfc2192.IMAP_URL_scheme.txt b/docs/rfcs/rfc2192.IMAP_URL_scheme.txt new file mode 100644 index 0000000..1b5a1d4 --- /dev/null +++ b/docs/rfcs/rfc2192.IMAP_URL_scheme.txt @@ -0,0 +1,899 @@ + + + + + + +Network Working Group C. Newman +Request for Comments: 2192 Innosoft +Category: Standards Track September 1997 + + + IMAP URL Scheme + + +Status of this memo + + This document specifies an Internet standards track protocol for + the Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is + unlimited. + + +Abstract + + IMAP [IMAP4] is a rich protocol for accessing remote message + stores. It provides an ideal mechanism for accessing public + mailing list archives as well as private and shared message stores. + This document defines a URL scheme for referencing objects on an + IMAP server. + + +1. Conventions used in this document + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in "Key words for + use in RFCs to Indicate Requirement Levels" [KEYWORDS]. + + +2. IMAP scheme + + The IMAP URL scheme is used to designate IMAP servers, mailboxes, + messages, MIME bodies [MIME], and search programs on Internet hosts + accessible using the IMAP protocol. + + The IMAP URL follows the common Internet scheme syntax as defined + in RFC 1738 [BASIC-URL] except that clear text passwords are not + permitted. If : is omitted, the port defaults to 143. + + + + + + + + +Newman Standards Track [Page 1] + +RFC 2192 IMAP URL Scheme September 1997 + + + An IMAP URL takes one of the following forms: + + imap:/// + imap:///;TYPE= + imap:///[uidvalidity][?] + imap:///[uidvalidity][isection] + + The first form is used to refer to an IMAP server, the second form + refers to a list of mailboxes, the third form refers to the + contents of a mailbox or a set of messages resulting from a search, + and the final form refers to a specific message or message part. + Note that the syntax here is informal. The authoritative formal + syntax for IMAP URLs is defined in section 11. + + +3. IMAP User Name and Authentication Mechanism + + A user name and/or authentication mechanism may be supplied. They + are used in the "LOGIN" or "AUTHENTICATE" commands after making the + connection to the IMAP server. If no user name or authentication + mechanism is supplied, the user name "anonymous" is used with the + "LOGIN" command and the password is supplied as the Internet e-mail + address of the end user accessing the resource. If the URL doesn't + supply a user name, the program interpreting the IMAP URL SHOULD + request one from the user if necessary. + + An authentication mechanism can be expressed by adding + ";AUTH=" to the end of the user name. When such an + is indicated, the client SHOULD request appropriate + credentials from that mechanism and use the "AUTHENTICATE" command + instead of the "LOGIN" command. If no user name is specified, one + SHOULD be obtained from the mechanism or requested from the user as + appropriate. + + The string ";AUTH=*" indicates that the client SHOULD select an + appropriate authentication mechanism. It MAY use any mechanism + listed in the CAPABILITY command or use an out of band security + service resulting in a PREAUTH connection. If no user name is + specified and no appropriate authentication mechanisms are + available, the client SHOULD fall back to anonymous login as + described above. This allows a URL which grants read-write access + to authorized users, and read-only anonymous access to other users. + + If a user name is included with no authentication mechanism, then + ";AUTH=*" is assumed. + + + + + + +Newman Standards Track [Page 2] + +RFC 2192 IMAP URL Scheme September 1997 + + + Since URLs can easily come from untrusted sources, care must be + taken when resolving a URL which requires or requests any sort of + authentication. If authentication credentials are supplied to the + wrong server, it may compromise the security of the user's account. + The program resolving the URL should make sure it meets at least + one of the following criteria in this case: + + (1) The URL comes from a trusted source, such as a referral server + which the client has validated and trusts according to site policy. + Note that user entry of the URL may or may not count as a trusted + source, depending on the experience level of the user and site + policy. + (2) Explicit local site policy permits the client to connect to the + server in the URL. For example, if the client knows the site + domain name, site policy may dictate that any hostname ending in + that domain is trusted. + (3) The user confirms that connecting to that domain name with the + specified credentials and/or mechanism is permitted. + (4) A mechanism is used which validates the server before passing + potentially compromising client credentials. + (5) An authentication mechanism is used which will not reveal + information to the server which could be used to compromise future + connections. + + URLs which do not include a user name must be treated with extra + care, since they are more likely to compromise the user's primary + account. A URL containing ";AUTH=*" must also be treated with + extra care since it might fall back on a weaker security mechanism. + Finally, clients are discouraged from using a plain text password + as a fallback with ";AUTH=*" unless the connection has strong + encryption (e.g. a key length of greater than 56 bits). + + A program interpreting IMAP URLs MAY cache open connections to an + IMAP server for later re-use. If a URL contains a user name, only + connections authenticated as that user may be re-used. If a URL + does not contain a user name or authentication mechanism, then only + an anonymous connection may be re-used. If a URL contains an + authentication mechanism without a user name, then any non- + anonymous connection may be re-used. + + Note that if unsafe or reserved characters such as " " or ";" are + present in the user name or authentication mechanism, they MUST be + encoded as described in RFC 1738 [BASIC-URL]. + + + + + + + + +Newman Standards Track [Page 3] + +RFC 2192 IMAP URL Scheme September 1997 + + +4. IMAP server + + An IMAP URL referring to an IMAP server has the following form: + + imap:/// + + A program interpreting this URL would issue the standard set of + commands it uses to present a view of the contents of an IMAP + server. This is likely to be semanticly equivalent to one of the + following URLs: + + imap:///;TYPE=LIST + imap:///;TYPE=LSUB + + The program interpreting this URL SHOULD use the LSUB form if it + supports mailbox subscriptions. + + +5. Lists of mailboxes + + An IMAP URL referring to a list of mailboxes has the following + form: + + imap:///;TYPE= + + The may be either "LIST" or "LSUB", and is case + insensitive. The field ";TYPE=" MUST be included. + + The is any argument suitable for the + list_mailbox field of the IMAP [IMAP4] LIST or LSUB commands. The + field may be omitted, in which case the program + interpreting the IMAP URL may use "*" or "%" as the + . The program SHOULD use "%" if it supports a + hierarchical view, otherwise it SHOULD use "*". + + Note that if unsafe or reserved characters such as " " or "%" are + present in they MUST be encoded as described in + RFC 1738 [BASIC-URL]. If the character "/" is present in + enc_list_mailbox, it SHOULD NOT be encoded. + + +6. Lists of messages + + An IMAP URL referring to a list of messages has the following form: + + imap:///[uidvalidity][?] + + + + + +Newman Standards Track [Page 4] + +RFC 2192 IMAP URL Scheme September 1997 + + + The field is used as the argument to the IMAP4 + "SELECT" command. Note that if unsafe or reserved characters such + as " ", ";", or "?" are present in they MUST be + encoded as described in RFC 1738 [BASIC-URL]. If the character "/" + is present in enc_mailbox, it SHOULD NOT be encoded. + + The [uidvalidity] field is optional. If it is present, it MUST be + the argument to the IMAP4 UIDVALIDITY status response at the time + the URL was created. This SHOULD be used by the program + interpreting the IMAP URL to determine if the URL is stale. + + The [?] field is optional. If it is not present, the + contents of the mailbox SHOULD be presented by the program + interpreting the URL. If it is present, it SHOULD be used as the + arguments following an IMAP4 SEARCH command with unsafe characters + such as " " (which are likely to be present in the ) + encoded as described in RFC 1738 [BASIC-URL]. + + +7. A specific message or message part + + An IMAP URL referring to a specific message or message part has the + following form: + + imap:///[uidvalidity][isection] + + The and [uidvalidity] are as defined above. + + If [uidvalidity] is present in this form, it SHOULD be used by the + program interpreting the URL to determine if the URL is stale. + + The refers to an IMAP4 message UID, and SHOULD be used as + the argument to the IMAP4 "UID FETCH" command. + + The [isection] field is optional. If not present, the URL refers + to the entire Internet message as returned by the IMAP command "UID + FETCH BODY.PEEK[]". If present, the URL refers to the object + returned by a "UID FETCH BODY.PEEK[
]" command. The + type of the object may be determined with a "UID FETCH + BODYSTRUCTURE" command and locating the appropriate part in the + resulting BODYSTRUCTURE. Note that unsafe characters in [isection] + MUST be encoded as described in [BASIC-URL]. + + + + + + + + + +Newman Standards Track [Page 5] + +RFC 2192 IMAP URL Scheme September 1997 + + +8. Relative IMAP URLs + + Relative IMAP URLs are permitted and are resolved according to the + rules defined in RFC 1808 [REL-URL] with one exception. In IMAP + URLs, parameters are treated as part of the normal path with + respect to relative URL resolution. This is believed to be the + behavior of the installed base and is likely to be documented in a + future revision of the relative URL specification. + + The following observations are also important: + + The grammar element is considered part of the user name for + purposes of resolving relative IMAP URLs. This means that unless a + new login/server specification is included in the relative URL, the + authentication mechanism is inherited from a base IMAP URL. + + URLs always use "/" as the hierarchy delimiter for the purpose of + resolving paths in relative URLs. IMAP4 permits the use of any + hierarchy delimiter in mailbox names. For this reason, relative + mailbox paths will only work if the mailbox uses "/" as the + hierarchy delimiter. Relative URLs may be used on mailboxes which + use other delimiters, but in that case, the entire mailbox name + MUST be specified in the relative URL or inherited as a whole from + the base URL. + + The base URL for a list of mailboxes or messages which was referred + to by an IMAP URL is always the referring IMAP URL itself. The + base URL for a message or message part which was referred to by an + IMAP URL may be more complicated to determine. The program + interpreting the relative URL will have to check the headers of the + MIME entity and any enclosing MIME entities in order to locate the + "Content-Base" and "Content-Location" headers. These headers are + used to determine the base URL as defined in [HTTP]. For example, + if the referring IMAP URL contains a "/;SECTION=1.2" parameter, + then the MIME headers for section 1.2, for section 1, and for the + enclosing message itself SHOULD be checked in that order for + "Content-Base" or "Content-Location" headers. + + +9. Multinational Considerations + + IMAP4 [IMAP4] section 5.1.3 includes a convention for encoding + non-US-ASCII characters in IMAP mailbox names. Because this + convention is private to IMAP, it is necessary to convert IMAP's + encoding to one that can be more easily interpreted by a URL + display program. For this reason, IMAP's modified UTF-7 encoding + for mailboxes MUST be converted to UTF-8 [UTF8]. Since 8-bit + characters are not permitted in URLs, the UTF-8 characters are + + + +Newman Standards Track [Page 6] + +RFC 2192 IMAP URL Scheme September 1997 + + + encoded as required by the URL specification [BASIC-URL]. Sample + code is included in Appendix A to demonstrate this conversion. + + +10. Examples + + The following examples demonstrate how an IMAP4 client program + might translate various IMAP4 URLs into a series of IMAP4 commands. + Commands sent from the client to the server are prefixed with "C:", + and responses sent from the server to the client are prefixed with + "S:". + + The URL: + + + + Results in the following client commands: + + + C: A001 LOGIN ANONYMOUS sheridan@babylon5.org + C: A002 SELECT gray-council + + C: A003 UID FETCH 20 BODY.PEEK[] + + The URL: + + + + Results in the following client commands: + + + + C: A001 LOGIN MICHAEL zipper + C: A002 LIST "" users.* + + The URL: + + + + Results in the following client commands: + + + C: A001 LOGIN ANONYMOUS bester@psycop.psicorp.org + C: A002 SELECT ~peter/&ZeVnLIqe-/&U,BTFw- + + + + + + +Newman Standards Track [Page 7] + +RFC 2192 IMAP URL Scheme September 1997 + + + The URL: + + + + Results in the following client commands: + + + C: A001 AUTHENTICATE KERBEROS_V4 + + C: A002 SELECT gray-council + C: A003 UID FETCH 20 BODY.PEEK[1.2] + + If the following relative URL is located in that body part: + + <;section=1.4> + + This could result in the following client commands: + + C: A004 UID FETCH 20 (BODY.PEEK[1.2.MIME] + BODY.PEEK[1.MIME] + BODY.PEEK[HEADER.FIELDS (Content-Base Content-Location)]) + + C: A005 UID FETCH 20 BODY.PEEK[1.4] + + The URL: + + + + Could result in the following: + + + C: A001 CAPABILITY + S: * CAPABILITY IMAP4rev1 AUTH=GSSAPI + S: A001 OK + C: A002 AUTHENTICATE GSSAPI + + S: A002 OK user lennier authenticated + C: A003 SELECT "gray council" + ... + C: A004 SEARCH SUBJECT shadows + S: * SEARCH 8 10 13 14 15 16 + S: A004 OK SEARCH completed + C: A005 FETCH 8,10,13:16 ALL + ... + + + + + +Newman Standards Track [Page 8] + +RFC 2192 IMAP URL Scheme September 1997 + + + NOTE: In this final example, the client has implementation + dependent choices. The authentication mechanism could be anything, + including PREAUTH. And the final FETCH command could fetch more or + less information about the messages, depending on what it wishes to + display to the user. + + +11. Security Considerations + + Security considerations discussed in the IMAP specification [IMAP4] + and the URL specification [BASIC-URL] are relevant. Security + considerations related to authenticated URLs are discussed in + section 3 of this document. + + Many email clients store the plain text password for later use + after logging into an IMAP server. Such clients MUST NOT use a + stored password in response to an IMAP URL without explicit + permission from the user to supply that password to the specified + host name. + + +12. ABNF for IMAP URL scheme + + This uses ABNF as defined in RFC 822 [IMAIL]. Terminals from the + BNF for IMAP [IMAP4] and URLs [BASIC-URL] are also used. Strings + are not case sensitive and free insertion of linear-white-space is + not permitted. + + achar = uchar / "&" / "=" / "~" + ; see [BASIC-URL] for "uchar" definition + + bchar = achar / ":" / "@" / "/" + + enc_auth_type = 1*achar + ; encoded version of [IMAP-AUTH] "auth_type" + + enc_list_mailbox = 1*bchar + ; encoded version of [IMAP4] "list_mailbox" + + enc_mailbox = 1*bchar + ; encoded version of [IMAP4] "mailbox" + + enc_search = 1*bchar + ; encoded version of search_program below + + enc_section = 1*bchar + ; encoded version of section below + + + + +Newman Standards Track [Page 9] + +RFC 2192 IMAP URL Scheme September 1997 + + + enc_user = 1*achar + ; encoded version of [IMAP4] "userid" + + imapurl = "imap://" iserver "/" [ icommand ] + + iauth = ";AUTH=" ( "*" / enc_auth_type ) + + icommand = imailboxlist / imessagelist / imessagepart + + imailboxlist = [enc_list_mailbox] ";TYPE=" list_type + + imessagelist = enc_mailbox [ "?" enc_search ] [uidvalidity] + + imessagepart = enc_mailbox [uidvalidity] iuid [isection] + + isection = "/;SECTION=" enc_section + + iserver = [iuserauth "@"] hostport + ; See [BASIC-URL] for "hostport" definition + + iuid = "/;UID=" nz_number + ; See [IMAP4] for "nz_number" definition + + iuserauth = enc_user [iauth] / [enc_user] iauth + + list_type = "LIST" / "LSUB" + + search_program = ["CHARSET" SPACE astring SPACE] + search_key *(SPACE search_key) + ; IMAP4 literals may not be used + ; See [IMAP4] for "astring" and "search_key" + + section = section_text / (nz_number *["." nz_number] + ["." (section_text / "MIME")]) + ; See [IMAP4] for "section_text" and "nz_number" + + uidvalidity = ";UIDVALIDITY=" nz_number + ; See [IMAP4] for "nz_number" definition + +13. References + + [BASIC-URL] Berners-Lee, Masinter, McCahill, "Uniform Resource + Locators (URL)", RFC 1738, CERN, Xerox Corporation, University of + Minnesota, December 1994. + + + + + + + +Newman Standards Track [Page 10] + +RFC 2192 IMAP URL Scheme September 1997 + + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + + + [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanism", RFC 1731, + Carnegie-Mellon University, December 1994. + + + + [HTTP] Fielding, Gettys, Mogul, Frystyk, Berners-Lee, "Hypertext + Transfer Protocol -- HTTP/1.1", RFC 2068, UC Irvine, DEC, MIT/LCS, + January 1997. + + + + [IMAIL] Crocker, "Standard for the Format of ARPA Internet Text + Messages", STD 11, RFC 822, University of Delaware, August 1982. + + + + [KEYWORDS] Bradner, "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, Harvard University, March 1997. + + + + [MIME] Freed, N., Borenstein, N., "Multipurpose Internet Mail + Extensions", RFC 2045, Innosoft, First Virtual, November 1996. + + + + [REL-URL] Fielding, "Relative Uniform Resource Locators", RFC 1808, + UC Irvine, June 1995. + + + + [UTF8] Yergeau, F. "UTF-8, a transformation format of Unicode and + ISO 10646", RFC 2044, Alis Technologies, October 1996. + + + +14. Author's Address + + Chris Newman + Innosoft International, Inc. + 1050 Lakes Drive + West Covina, CA 91790 USA + EMail: chris.newman@innosoft.com + + + +Newman Standards Track [Page 11] + +RFC 2192 IMAP URL Scheme September 1997 + + +Appendix A. Sample code + +Here is sample C source code to convert between URL paths and IMAP +mailbox names, taking into account mapping between IMAP's modified UTF-7 +[IMAP4] and hex-encoded UTF-8 which is more appropriate for URLs. This +code has not been rigorously tested nor does it necessarily behave +reasonably with invalid input, but it should serve as a useful example. +This code just converts the mailbox portion of the URL and does not deal +with parameters, query or server components of the URL. + +#include +#include + +/* hexadecimal lookup table */ +static char hex[] = "0123456789ABCDEF"; + +/* URL unsafe printable characters */ +static char urlunsafe[] = " \"#%&+:;<=>?@[\\]^`{|}"; + +/* UTF7 modified base64 alphabet */ +static char base64chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; +#define UNDEFINED 64 + +/* UTF16 definitions */ +#define UTF16MASK 0x03FFUL +#define UTF16SHIFT 10 +#define UTF16BASE 0x10000UL +#define UTF16HIGHSTART 0xD800UL +#define UTF16HIGHEND 0xDBFFUL +#define UTF16LOSTART 0xDC00UL +#define UTF16LOEND 0xDFFFUL + +/* Convert an IMAP mailbox to a URL path + * dst needs to have roughly 4 times the storage space of src + * Hex encoding can triple the size of the input + * UTF-7 can be slightly denser than UTF-8 + * (worst case: 8 octets UTF-7 becomes 9 octets UTF-8) + */ +void MailboxToURL(char *dst, char *src) +{ + unsigned char c, i, bitcount; + unsigned long ucs4, utf16, bitbuf; + unsigned char base64[256], utf8[6]; + + + + + + + +Newman Standards Track [Page 12] + +RFC 2192 IMAP URL Scheme September 1997 + + + /* initialize modified base64 decoding table */ + memset(base64, UNDEFINED, sizeof (base64)); + for (i = 0; i < sizeof (base64chars); ++i) { + base64[base64chars[i]] = i; + } + + /* loop until end of string */ + while (*src != '\0') { + c = *src++; + /* deal with literal characters and &- */ + if (c != '&' || *src == '-') { + if (c < ' ' || c > '~' || strchr(urlunsafe, c) != NULL) { + /* hex encode if necessary */ + dst[0] = '%'; + dst[1] = hex[c >> 4]; + dst[2] = hex[c & 0x0f]; + dst += 3; + } else { + /* encode literally */ + *dst++ = c; + } + /* skip over the '-' if this is an &- sequence */ + if (c == '&') ++src; + } else { + /* convert modified UTF-7 -> UTF-16 -> UCS-4 -> UTF-8 -> HEX */ + bitbuf = 0; + bitcount = 0; + ucs4 = 0; + while ((c = base64[(unsigned char) *src]) != UNDEFINED) { + ++src; + bitbuf = (bitbuf << 6) | c; + bitcount += 6; + /* enough bits for a UTF-16 character? */ + if (bitcount >= 16) { + bitcount -= 16; + utf16 = (bitcount ? bitbuf >> bitcount + : bitbuf) & 0xffff; + /* convert UTF16 to UCS4 */ + if + (utf16 >= UTF16HIGHSTART && utf16 <= UTF16HIGHEND) { + ucs4 = (utf16 - UTF16HIGHSTART) << UTF16SHIFT; + continue; + } else if + (utf16 >= UTF16LOSTART && utf16 <= UTF16LOEND) { + ucs4 += utf16 - UTF16LOSTART + UTF16BASE; + } else { + ucs4 = utf16; + } + + + +Newman Standards Track [Page 13] + +RFC 2192 IMAP URL Scheme September 1997 + + + /* convert UTF-16 range of UCS4 to UTF-8 */ + if (ucs4 <= 0x7fUL) { + utf8[0] = ucs4; + i = 1; + } else if (ucs4 <= 0x7ffUL) { + utf8[0] = 0xc0 | (ucs4 >> 6); + utf8[1] = 0x80 | (ucs4 & 0x3f); + i = 2; + } else if (ucs4 <= 0xffffUL) { + utf8[0] = 0xe0 | (ucs4 >> 12); + utf8[1] = 0x80 | ((ucs4 >> 6) & 0x3f); + utf8[2] = 0x80 | (ucs4 & 0x3f); + i = 3; + } else { + utf8[0] = 0xf0 | (ucs4 >> 18); + utf8[1] = 0x80 | ((ucs4 >> 12) & 0x3f); + utf8[2] = 0x80 | ((ucs4 >> 6) & 0x3f); + utf8[3] = 0x80 | (ucs4 & 0x3f); + i = 4; + } + /* convert utf8 to hex */ + for (c = 0; c < i; ++c) { + dst[0] = '%'; + dst[1] = hex[utf8[c] >> 4]; + dst[2] = hex[utf8[c] & 0x0f]; + dst += 3; + } + } + } + /* skip over trailing '-' in modified UTF-7 encoding */ + if (*src == '-') ++src; + } + } + /* terminate destination string */ + *dst = '\0'; +} + +/* Convert hex coded UTF-8 URL path to modified UTF-7 IMAP mailbox + * dst should be about twice the length of src to deal with non-hex + * coded URLs + */ +void URLtoMailbox(char *dst, char *src) +{ + unsigned int utf8pos, utf8total, i, c, utf7mode, bitstogo, utf16flag; + unsigned long ucs4, bitbuf; + unsigned char hextab[256]; + + /* initialize hex lookup table */ + + + +Newman Standards Track [Page 14] + +RFC 2192 IMAP URL Scheme September 1997 + + + memset(hextab, 0, sizeof (hextab)); + for (i = 0; i < sizeof (hex); ++i) { + hextab[hex[i]] = i; + if (isupper(hex[i])) hextab[tolower(hex[i])] = i; + } + + utf7mode = 0; + utf8total = 0; + bitstogo = 0; + while ((c = *src) != '\0') { + ++src; + /* undo hex-encoding */ + if (c == '%' && src[0] != '\0' && src[1] != '\0') { + c = (hextab[src[0]] << 4) | hextab[src[1]]; + src += 2; + } + /* normal character? */ + if (c >= ' ' && c <= '~') { + /* switch out of UTF-7 mode */ + if (utf7mode) { + if (bitstogo) { + *dst++ = base64chars[(bitbuf << (6 - bitstogo)) & 0x3F]; + } + *dst++ = '-'; + utf7mode = 0; + } + *dst++ = c; + /* encode '&' as '&-' */ + if (c == '&') { + *dst++ = '-'; + } + continue; + } + /* switch to UTF-7 mode */ + if (!utf7mode) { + *dst++ = '&'; + utf7mode = 1; + } + /* Encode US-ASCII characters as themselves */ + if (c < 0x80) { + ucs4 = c; + utf8total = 1; + } else if (utf8total) { + /* save UTF8 bits into UCS4 */ + ucs4 = (ucs4 << 6) | (c & 0x3FUL); + if (++utf8pos < utf8total) { + continue; + } + + + +Newman Standards Track [Page 15] + +RFC 2192 IMAP URL Scheme September 1997 + + + } else { + utf8pos = 1; + if (c < 0xE0) { + utf8total = 2; + ucs4 = c & 0x1F; + } else if (c < 0xF0) { + utf8total = 3; + ucs4 = c & 0x0F; + } else { + /* NOTE: can't convert UTF8 sequences longer than 4 */ + utf8total = 4; + ucs4 = c & 0x03; + } + continue; + } + /* loop to split ucs4 into two utf16 chars if necessary */ + utf8total = 0; + do { + if (ucs4 >= UTF16BASE) { + ucs4 -= UTF16BASE; + bitbuf = (bitbuf << 16) | ((ucs4 >> UTF16SHIFT) + + UTF16HIGHSTART); + ucs4 = (ucs4 & UTF16MASK) + UTF16LOSTART; + utf16flag = 1; + } else { + bitbuf = (bitbuf << 16) | ucs4; + utf16flag = 0; + } + bitstogo += 16; + /* spew out base64 */ + while (bitstogo >= 6) { + bitstogo -= 6; + *dst++ = base64chars[(bitstogo ? (bitbuf >> bitstogo) + : bitbuf) + & 0x3F]; + } + } while (utf16flag); + } + /* if in UTF-7 mode, finish in ASCII */ + if (utf7mode) { + if (bitstogo) { + *dst++ = base64chars[(bitbuf << (6 - bitstogo)) & 0x3F]; + } + *dst++ = '-'; + } + /* tie off string */ + *dst = '\0'; +} + + + +Newman Standards Track [Page 16] + diff --git a/docs/rfcs/rfc2193.IMAP4_Mailbox_referrals.txt b/docs/rfcs/rfc2193.IMAP4_Mailbox_referrals.txt new file mode 100644 index 0000000..2fec58d --- /dev/null +++ b/docs/rfcs/rfc2193.IMAP4_Mailbox_referrals.txt @@ -0,0 +1,507 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 2193 Microsoft +Category: Standards Track September 1997 + + + IMAP4 Mailbox Referrals + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +1. Abstract + + When dealing with large amounts of users, messages and geographically + dispersed IMAP4 [RFC-2060] servers, it is often desirable to + distribute messages amongst different servers within an organization. + For example an administrator may choose to store user's personal + mailboxes on a local IMAP4 server, while storing shared mailboxes + remotely on another server. This type of configuration is common + when it is uneconomical to store all data centrally due to limited + bandwidth or disk resources. + + Mailbox referrals allow clients to seamlessly access mailboxes that + are distributed across several IMAP4 servers. + + A referral mechanism can provide efficiencies over the alternative + "proxy method", in which the local IMAP4 server contacts the remote + server on behalf of the client, and then transfers the data from the + remote server to itself, and then on to the client. The referral + mechanism's direct client connection to the remote server is often a + more efficient use of bandwidth, and does not require the local + server to impersonate the client when authenticating to the remote + server. + +2. Conventions used in this document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + A home server, is an IMAP4 server that contains the user's inbox. + + A remote mailbox is a mailbox that is not hosted on the user's home + server. + + + + +Gahrns Standards Track [Page 1] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + A remote server is a server that contains remote mailboxes. + + A shared mailbox, is a mailbox that multiple users have access to. + + An IMAP mailbox referral is when the server directs the client to + another IMAP mailbox. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + +3. Introduction and Overview + + IMAP4 servers that support this extension MUST list the keyword + MAILBOX-REFERRALS in their CAPABILITY response. No client action is + needed to invoke the MAILBOX-REFERRALS capability in a server. + + A MAILBOX-REFERRALS capable IMAP4 server MUST NOT return referrals + that result in a referrals loop. + + A referral response consists of a tagged NO response and a REFERRAL + response code. The REFERRAL response code MUST contain as an + argument a one or more valid URLs separated by a space as defined in + [RFC-1738]. If a server replies with multiple URLs for a particular + object, they MUST all be of the same type. In this case, the URL MUST + be an IMAP URL as defined in [RFC-2192]. A client that supports the + REFERRALS extension MUST be prepared for a URL of any type, but it + need only be able to process IMAP URLs. + + A server MAY respond with multiple IMAP mailbox referrals if there is + more than one replica of the mailbox. This allows the implementation + of a load balancing or failover scheme. How a server keeps multiple + replicas of a mailbox in sync is not addressed by this document. + + If the server has a preferred order in which the client should + attempt to access the URLs, the preferred URL SHOULD be listed in the + first, with the remaining URLs presented in descending order of + preference. If multiple referrals are given for a mailbox, a server + should be aware that there are synchronization issues for a client if + the UIDVALIDITY of the referred mailboxes are different. + + An IMAP mailbox referral may be given in response to an IMAP command + that specifies a mailbox as an argument. + + + + + + + + +Gahrns Standards Track [Page 2] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + Example: + + A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/REMOTE]Remote Mailbox + + NOTE: user;AUTH=* is specified as required by [RFC-2192] to avoid a + client falling back to anonymous login. + + Remote mailboxes and their inferiors, that are accessible only via + referrals SHOULD NOT appear in LIST and LSUB responses issued against + the user's home server. They MUST appear in RLIST and RLSUB + responses issued against the user's home server. Hierarchy referrals, + in which a client would be required to connect to the remote server + to issue a LIST to discover the inferiors of a mailbox are not + addressed in this document. + + For example, if shared mailboxes were only accessible via referrals + on a remote server, a RLIST "" "#SHARED/%" command would return the + same response if issued against the user's home server or the remote + server. + + Note: Mailboxes that are available on the user's home server do not + need to be available on the remote server. In addition, there may be + additional mailboxes available on the remote server, but they will + not accessible to the client via referrals unless they appear in the + LIST response to the RLIST command against the user's home server. + + A MAILBOX-REFERRALS capable client will issue the RLIST and RLSUB + commands in lieu of LIST and LSUB. The RLIST and RLSUB commands + behave identically to their LIST and LSUB counterparts, except remote + mailboxes are returned in addition to local mailboxes in the LIST and + LSUB responses. This avoids displaying to a non MAILBOX-REFERRALS + enabled client inaccessible remote mailboxes. + +4.1. SELECT, EXAMINE, DELETE, SUBSCRIBE, UNSUBSCRIBE, STATUS and APPEND + Referrals + + An IMAP4 server MAY respond to the SELECT, EXAMINE, DELETE, + SUBSCRIBE, UNSUBSCRIBE, STATUS or APPEND command with one or more + IMAP mailbox referrals to indicate to the client that the mailbox is + hosted on a remote server. + + When a client processes an IMAP mailbox referral, it will open a new + connection or use an existing connection to the remote server so that + it is able to issue the commands necessary to process the remote + mailbox. + + + + + + +Gahrns Standards Track [Page 3] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + Example: + + C: A001 DELETE "SHARED/FOO" + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] + Remote mailbox. Try SERVER2. + + + + S: * OK IMAP4rev1 server ready + C: B001 AUTHENTICATE KERBEROS_V4 + + S: B001 OK user is authenticated + + C: B002 DELETE "SHARED/FOO" + S: B002 OK DELETE completed + + Example: + + C: A001 SELECT REMOTE + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/REMOTE + IMAP://user;AUTH=*@SERVER3/REMOTE] Remote mailbox. + Try SERVER2 or SERVER3. + + + + S: * OK IMAP4rev1 server ready + C: B001 AUTHENTICATE KERBEROS_V4 + + S: B001 OK user is authenticated + + C: B002 SELECT REMOTE + S: * 12 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 10] Message 10 is first unseen + S: * OK [UIDVALIDITY 123456789] + S: * FLAGS (Answered Flagged Deleted Seen Draft) + S: * OK [PERMANENTFLAGS (Answered Deleted Seen ] + S: B002 OK [READ-WRITE] Selected completed + + C: B003 FETCH 10:12 RFC822 + S: * 10 FETCH . . . + S: * 11 FETCH . . . + S: * 12 FETCH . . . + S: B003 OK FETCH Completed + + + + +Gahrns Standards Track [Page 4] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + + + C: B004 LOGOUT + S: * BYE IMAP4rev1 server logging out + S: B004 OK LOGOUT Completed + + + + C: A002 SELECT INBOX + S: * 16 EXISTS + S: * 2 RECENT + S: * OK [UNSEEN 10] Message 10 is first unseen + S: * OK [UIDVALIDITY 123456789] + S: * FLAGS (Answered Flagged Deleted Seen Draft) + S: * OK [PERMANENTFLAGS (Answered Deleted Seen ] + S: A002 OK [READ-WRITE] Selected completed + +4.2. CREATE Referrals + + An IMAP4 server MAY respond to the CREATE command with one or more + IMAP mailbox referrals, if it wishes to direct the client to issue + the CREATE against another server. The server can employ any means, + such as examining the hierarchy of the specified mailbox name, in + determining which server the mailbox should be created on. + + Example: + + C: A001 CREATE "SHARED/FOO" + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] + Mailbox should be created on remote server + + Alternatively, because a home server is required to maintain a + listing of referred remote mailboxes, a server MAY allow the creation + of a mailbox that will ultimately reside on a remote server against + the home server, and provide referrals on subsequent commands that + manipulate the mailbox. + + Example: + + C: A001 CREATE "SHARED/FOO" + S: A001 OK CREATE succeeded + + C: A002 SELECT "SHARED/FOO" + S: A002 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] + Remote mailbox. Try SERVER2 + + + + + +Gahrns Standards Track [Page 5] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + +4.3. RENAME Referrals + + An IMAP4 server MAY respond to the RENAME command with one or more + pairs of IMAP mailbox referrals. In each pair of IMAP mailbox + referrals, the first one is an URL to the existing mailbox name and + the second is an URL to the requested new mailbox name. + + If within an IMAP mailbox referral pair, the existing and new mailbox + URLs are on different servers, the remote servers are unable to + perform the RENAME operation. To achieve the same behavior of + server RENAME, the client MAY issue the constituent CREATE, FETCH, + APPEND, and DELETE commands against both servers. + + If within an IMAP mailbox referral pair, the existing and new mailbox + URLs are on the same server it is an indication that the currently + connected server is unable to perform the operation. The client can + simply re-issue the RENAME command on the remote server. + + Example: + + C: A001 RENAME FOO BAR + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER1/FOO + IMAP://user;AUTH=*@SERVER2/BAR] Unable to rename mailbox + across servers + + Since the existing and new mailbox names are on different servers, + the client would be required to make a connection to both servers and + issue the constituent commands require to achieve the RENAME. + + Example: + + C: A001 RENAME FOO BAR + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/FOO + IMAP://user;AUTH=*@SERVER2/BAR] Unable to rename mailbox + located on SERVER2 + + Since both the existing and new mailbox are on the same remote + server, the client can simply make a connection to the remote server + and re-issue the RENAME command. + +4.4. COPY Referrals + + An IMAP4 server MAY respond to the COPY command with one or more IMAP + mailbox referrals. This indicates that the destination mailbox is on + a remote server. To achieve the same behavior of a server COPY, the + client MAY issue the constituent FETCH and APPEND commands against + both servers. + + + + +Gahrns Standards Track [Page 6] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + Example: + + C: A001 COPY 1 "SHARED/STUFF" + S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/STUFF] + Unable to copy message(s) to SERVER2. + +5.1 RLIST command + + Arguments: reference name + mailbox name with possible wildcards + + Responses: untagged responses: LIST + + Result: OK - RLIST Completed + NO - RLIST Failure + BAD - command unknown or arguments invalid + + The RLIST command behaves identically to its LIST counterpart, except + remote mailboxes are returned in addition to local mailboxes in the + LIST responses. + +5.2 RLSUB Command + + Arguments: reference name + mailbox name with possible wildcards + + Responses: untagged responses: LSUB + + Result: OK - RLSUB Completed + NO - RLSUB Failure + BAD - command unknown or arguments invalid + + The RLSUB command behaves identically to its LSUB counterpart, except + remote mailboxes are returned in addition to local mailboxes in the + LSUB responses. + +6. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. + + list_mailbox = as defined in [RFC-2060] + + mailbox = as defined in [RFC-2060] + + mailbox_referral = SPACE "NO" SPACE + (text / text_mime2) + ; See [RFC-2060] for , text and text_mime2 definition + + + +Gahrns Standards Track [Page 7] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + + referral_response_code = "[" "REFERRAL" 1*(SPACE ) "]" + ; See [RFC-1738] for definition + + rlist = "RLIST" SPACE mailbox SPACE list_mailbox + + rlsub = "RLSUB" SPACE mailbox SPACE list_mailbox + +6. Security Considerations + + The IMAP4 referral mechanism makes use of IMAP URLs, and as such, + have the same security considerations as general internet URLs [RFC- + 1738], and in particular IMAP URLs [RFC-2192]. + + With the MAILBOX-REFERRALS capability, it is potentially easier to + write a rogue server that injects a bogus referral response that + directs a user to an incorrect mailbox. Although referrals reduce + the effort to write such a server, the referral response makes + detection of the intrusion easier. + +7. References + + [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + [RFC-2192], Newman, C., "IMAP URL Scheme", RFC 2192, Innosoft, + September 1997. + + [RFC-1738], Berners-Lee, T., Masinter, L., and M. McCahill, "Uniform + Resource Locators (URL)", RFC 1738, CERN, Xerox Corporation, + University of Minnesota, December 1994. + + [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, Harvard University, March 1997. + + [ABNF], DRUMS working group, Dave Crocker Editor, "Augmented BNF for + Syntax Specifications: ABNF", Work in Progress, Internet Mail + Consortium, April 1997. + +8. Acknowledgments + + Many valuable suggestions were received from private discussions and + the IMAP4 mailing list. In particular, Raymond Cheng, Mark Crispin, + Mark Keasling, Chris Newman and Larry Osterman made significant + contributions to this document. + + + + + + + +Gahrns Standards Track [Page 8] + +RFC 2193 IMAP4 Mailbox Referrals September 1997 + + +9. Author's Address + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98072 + + Phone: (206) 936-9833 + EMail: mikega@microsoft.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gahrns Standards Track [Page 9] + diff --git a/docs/rfcs/rfc2195.IMAP-POP_AUTHorize_extension.txt b/docs/rfcs/rfc2195.IMAP-POP_AUTHorize_extension.txt new file mode 100644 index 0000000..4a2725b --- /dev/null +++ b/docs/rfcs/rfc2195.IMAP-POP_AUTHorize_extension.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group J. Klensin +Request for Comments: 2195 R. Catoe +Category: Standards Track P. Krumviede +Obsoletes: 2095 MCI + September 1997 + + + IMAP/POP AUTHorize Extension for Simple Challenge/Response + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + While IMAP4 supports a number of strong authentication mechanisms as + described in RFC 1731, it lacks any mechanism that neither passes + cleartext, reusable passwords across the network nor requires either + a significant security infrastructure or that the mail server update + a mail-system-wide user authentication file on each mail access. + This specification provides a simple challenge-response + authentication protocol that is suitable for use with IMAP4. Since + it utilizes Keyed-MD5 digests and does not require that the secret be + stored in the clear on the server, it may also constitute an + improvement on APOP for POP3 use as specified in RFC 1734. + +1. Introduction + + Existing Proposed Standards specify an AUTHENTICATE mechanism for the + IMAP4 protocol [IMAP, IMAP-AUTH] and a parallel AUTH mechanism for + the POP3 protocol [POP3-AUTH]. The AUTHENTICATE mechanism is + intended to be extensible; the four methods specified in [IMAP-AUTH] + are all fairly powerful and require some security infrastructure to + support. The base POP3 specification [POP3] also contains a + lightweight challenge-response mechanism called APOP. APOP is + associated with most of the risks associated with such protocols: in + particular, it requires that both the client and server machines have + access to the shared secret in cleartext form. CRAM offers a method + for avoiding such cleartext storage while retaining the algorithmic + simplicity of APOP in using only MD5, though in a "keyed" method. + + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 1] + +RFC 2195 IMAP/POP AUTHorize Extension September 1997 + + + At present, IMAP [IMAP] lacks any facility corresponding to APOP. + The only alternative to the strong mechanisms identified in [IMAP- + AUTH] is a presumably cleartext username and password, supported + through the LOGIN command in [IMAP]. This document describes a + simple challenge-response mechanism, similar to APOP and PPP CHAP + [PPP], that can be used with IMAP (and, in principle, with POP3). + + This mechanism also has the advantage over some possible alternatives + of not requiring that the server maintain information about email + "logins" on a per-login basis. While mechanisms that do require such + per-login history records may offer enhanced security, protocols such + as IMAP, which may have several connections between a given client + and server open more or less simultaneous, may make their + implementation particularly challenging. + +2. Challenge-Response Authentication Mechanism (CRAM) + + The authentication type associated with CRAM is "CRAM-MD5". + + The data encoded in the first ready response contains an + presumptively arbitrary string of random digits, a timestamp, and the + fully-qualified primary host name of the server. The syntax of the + unencoded form must correspond to that of an RFC 822 'msg-id' + [RFC822] as described in [POP3]. + + The client makes note of the data and then responds with a string + consisting of the user name, a space, and a 'digest'. The latter is + computed by applying the keyed MD5 algorithm from [KEYED-MD5] where + the key is a shared secret and the digested text is the timestamp + (including angle-brackets). + + This shared secret is a string known only to the client and server. + The `digest' parameter itself is a 16-octet value which is sent in + hexadecimal format, using lower-case ASCII characters. + + When the server receives this client response, it verifies the digest + provided. If the digest is correct, the server should consider the + client authenticated and respond appropriately. + + Keyed MD5 is chosen for this application because of the greater + security imparted to authentication of short messages. In addition, + the use of the techniques described in [KEYED-MD5] for precomputation + of intermediate results make it possible to avoid explicit cleartext + storage of the shared secret on the server system by instead storing + the intermediate results which are known as "contexts". + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 2] + +RFC 2195 IMAP/POP AUTHorize Extension September 1997 + + + CRAM does not support a protection mechanism. + + Example: + + The examples in this document show the use of the CRAM mechanism with + the IMAP4 AUTHENTICATE command [IMAP-AUTH]. The base64 encoding of + the challenges and responses is part of the IMAP4 AUTHENTICATE + command, not part of the CRAM specification itself. + + S: * OK IMAP4 Server + C: A0001 AUTHENTICATE CRAM-MD5 + S: + PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2UucmVzdG9uLm1jaS5uZXQ+ + C: dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw + S: A0001 OK CRAM authentication successful + + In this example, the shared secret is the string + 'tanstaaftanstaaf'. Hence, the Keyed MD5 digest is produced by + calculating + + MD5((tanstaaftanstaaf XOR opad), + MD5((tanstaaftanstaaf XOR ipad), + <1896.697170952@postoffice.reston.mci.net>)) + + where ipad and opad are as defined in the keyed-MD5 Work in + Progress [KEYED-MD5] and the string shown in the challenge is the + base64 encoding of <1896.697170952@postoffice.reston.mci.net>. The + shared secret is null-padded to a length of 64 bytes. If the + shared secret is longer than 64 bytes, the MD5 digest of the + shared secret is used as a 16 byte input to the keyed MD5 + calculation. + + This produces a digest value (in hexadecimal) of + + b913a602c7eda7a495b4e6e7334d3890 + + The user name is then prepended to it, forming + + tim b913a602c7eda7a495b4e6e7334d3890 + + Which is then base64 encoded to meet the requirements of the IMAP4 + AUTHENTICATE command (or the similar POP3 AUTH command), yielding + + dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw + + + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 3] + +RFC 2195 IMAP/POP AUTHorize Extension September 1997 + + +3. References + + [CHAP] Lloyd, B., and W. Simpson, "PPP Authentication Protocols", + RFC 1334, October 1992. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanisms", + RFC 1731, Carnegie Mellon, December 1994. + + [KEYED-MD5] Krawczyk, Bellare, Canetti, "HMAC: Keyed-Hashing for + Message Authentication", RFC 2104, February 1997. + + [MD5] Rivest, R., "The MD5 Message Digest Algorithm", + RFC 1321, MIT Laboratory for Computer Science, April 1992. + + [POP3] Myers, J., and M. Rose, "Post Office Protocol - Version 3", + STD 53, RFC 1939, Carnegie Mellon, May 1996. + + [POP3-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, + Carnegie Mellon, December, 1994. + +4. Security Considerations + + It is conjectured that use of the CRAM authentication mechanism + provides origin identification and replay protection for a session. + Accordingly, a server that implements both a cleartext password + command and this authentication type should not allow both methods of + access for a given user. + + While the saving, on the server, of "contexts" (see section 2) is + marginally better than saving the shared secrets in cleartext as is + required by CHAP [CHAP] and APOP [POP3], it is not sufficient to + protect the secrets if the server itself is compromised. + Consequently, servers that store the secrets or contexts must both be + protected to a level appropriate to the potential information value + in user mailboxes and identities. + + As the length of the shared secret increases, so does the difficulty + of deriving it. + + While there are now suggestions in the literature that the use of MD5 + and keyed MD5 in authentication procedures probably has a limited + effective lifetime, the technique is now widely deployed and widely + understood. It is believed that this general understanding may + assist with the rapid replacement, by CRAM-MD5, of the current uses + of permanent cleartext passwords in IMAP. This document has been + + + +Klensin, Catoe & Krumviede Standards Track [Page 4] + +RFC 2195 IMAP/POP AUTHorize Extension September 1997 + + + deliberately written to permit easy upgrading to use SHA (or whatever + alternatives emerge) when they are considered to be widely available + and adequately safe. + + Even with the use of CRAM, users are still vulnerable to active + attacks. An example of an increasingly common active attack is 'TCP + Session Hijacking' as described in CERT Advisory CA-95:01 [CERT95]. + + See section 1 above for additional discussion. + +5. Acknowledgements + + This memo borrows ideas and some text liberally from [POP3] and + [RFC-1731] and thanks are due the authors of those documents. Ran + Atkinson made a number of valuable technical and editorial + contributions to the document. + +6. Authors' Addresses + + John C. Klensin + MCI Telecommunications + 800 Boylston St, 7th floor + Boston, MA 02199 + USA + + EMail: klensin@mci.net + Phone: +1 617 960 1011 + + Randy Catoe + MCI Telecommunications + 2100 Reston Parkway + Reston, VA 22091 + USA + + EMail: randy@mci.net + Phone: +1 703 715 7366 + + Paul Krumviede + MCI Telecommunications + 2100 Reston Parkway + Reston, VA 22091 + USA + + EMail: paul@mci.net + Phone: +1 703 715 7251 + + + + + + +Klensin, Catoe & Krumviede Standards Track [Page 5] + diff --git a/docs/rfcs/rfc2221.IMAP4_Login_referrals.txt b/docs/rfcs/rfc2221.IMAP4_Login_referrals.txt new file mode 100644 index 0000000..81d0062 --- /dev/null +++ b/docs/rfcs/rfc2221.IMAP4_Login_referrals.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 2221 Microsoft +Category: Standards Track October 1997 + + + IMAP4 Login Referrals + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1997). All Rights Reserved. + +1. Abstract + + When dealing with large amounts of users and many IMAP4 [RFC-2060] + servers, it is often necessary to move users from one IMAP4 server to + another. For example, hardware failures or organizational changes + may dictate such a move. + + Login referrals allow clients to transparently connect to an + alternate IMAP4 server, if their home IMAP4 server has changed. + + A referral mechanism can provide efficiencies over the alternative + 'proxy method', in which the local IMAP4 server contacts the remote + server on behalf of the client, and then transfers the data from the + remote server to itself, and then on to the client. The referral + mechanism's direct client connection to the remote server is often a + more efficient use of bandwidth, and does not require the local + server to impersonate the client when authenticating to the remote + server. + +2. Conventions used in this document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + A home server, is an IMAP4 server that contains the user's inbox. + + A remote server is a server that contains remote mailboxes. + + + + + +Gahrns Standards Track [Page 1] + +RFC 2221 IMAP4 Login Referrals October 1997 + + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + +3. Introduction and Overview + + IMAP4 servers that support this extension MUST list the keyword + LOGIN-REFERRALS in their CAPABILITY response. No client action is + needed to invoke the LOGIN-REFERRALS capability in a server. + + A LOGIN-REFERRALS capable IMAP4 server SHOULD NOT return a referral + to a server that will return a referral. A client MUST NOT follow + more than 10 levels of referral without consulting the user. + + A LOGIN-REFERRALS response code MUST contain as an argument a valid + IMAP server URL as defined in [IMAP-URL]. + + A home server referral consists of either a tagged NO or OK, or an + untagged BYE response that contains a LOGIN-REFERRALS response code. + + Example: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/] Remote Server + + NOTE: user;AUTH=* is specified as required by [IMAP-URL] to avoid a + client falling back to anonymous login. + +4. Home Server Referrals + + A home server referral may be returned in response to an AUTHENTICATE + or LOGIN command, or it may appear in the connection startup banner. + If a server returns a home server referral in a tagged NO response, + that server does not contain any mailboxes that are accessible to the + user. If a server returns a home server referral in a tagged OK + response, it indicates that the user's personal mailboxes are + elsewhere, but the server contains public mailboxes which are + readable by the user. After receiving a home server referral, the + client can not make any assumptions as to whether this was a + permanent or temporary move of the user. + +4.1. LOGIN and AUTHENTICATE Referrals + + An IMAP4 server MAY respond to a LOGIN or AUTHENTICATE command with a + home server referral if it wishes to direct the user to another IMAP4 + server. + + Example: C: A001 LOGIN MIKE PASSWORD + S: A001 NO [REFERRAL IMAP://MIKE@SERVER2/] Specified user + is invalid on this server. Try SERVER2. + + + + +Gahrns Standards Track [Page 2] + +RFC 2221 IMAP4 Login Referrals October 1997 + + + Example: C: A001 LOGIN MATTHEW PASSWORD + S: A001 OK [REFERRAL IMAP://MATTHEW@SERVER2/] Specified + user's personal mailboxes located on Server2, but + public mailboxes are available. + + Example: C: A001 AUTHENTICATE GSSAPI + + S: A001 NO [REFERRAL IMAP://user;AUTH=GSSAPI@SERVER2/] + Specified user is invalid on this server. Try + SERVER2. + +4.2. BYE at connection startup referral + + An IMAP4 server MAY respond with an untagged BYE and a REFERRAL + response code that contains an IMAP URL to a home server if it is not + willing to accept connections and wishes to direct the client to + another IMAP4 server. + + Example: S: * BYE [REFERRAL IMAP://user;AUTH=*@SERVER2/] Server not + accepting connections. Try SERVER2 + +5. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. + + This amends the "resp_text_code" element of the IMAP4 grammar + described in [RFC-2060] + + resp_text_code =/ "REFERRAL" SPACE + ; See [IMAP-URL] for definition of + ; See [RFC-2060] for base definition of resp_text_code + +6. Security Considerations + + The IMAP4 login referral mechanism makes use of IMAP URLs, and as + such, have the same security considerations as general internet URLs + [RFC-1738], and in particular IMAP URLs [IMAP-URL]. + + A server MUST NOT give a login referral if authentication for that + user fails. This is to avoid revealing information about the user's + account to an unauthorized user. + + With the LOGIN-REFERRALS capability, it is potentially easier to + write a rogue 'password catching' server that collects login data and + then refers the client to their actual IMAP4 server. Although + referrals reduce the effort to write such a server, the referral + response makes detection of the intrusion easier. + + + +Gahrns Standards Track [Page 3] + +RFC 2221 IMAP4 Login Referrals October 1997 + + +7. References + + [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [IMAP-URL], Newman, C., "IMAP URL Scheme", RFC 2192, Innosoft, + September 1997. + + [RFC-1738], Berners-Lee, T., Masinter, L. and M. McCahill, "Uniform + Resource Locators (URL)", RFC 1738, December 1994. + + [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, March 1997. + + [ABNF], DRUMS working group, Dave Crocker Editor, "Augmented BNF for + Syntax Specifications: ABNF", Work in Progress. + +8. Acknowledgments + + Many valuable suggestions were received from private discussions and + the IMAP4 mailing list. In particular, Raymond Cheng, Mark Crispin, + Mark Keasling Chris Newman and Larry Osterman made significant + contributions to this document. + +9. Author's Address + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98072 + + Phone: (206) 936-9833 + EMail: mikega@microsoft.com + + + + + + + + + + + + + + + + + + +Gahrns Standards Track [Page 4] + +RFC 2221 IMAP4 Login Referrals October 1997 + + +10. Full Copyright Statement + + Copyright (C) The Internet Society (1997). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implmentation may be prepared, copied, published + andand distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE." + + + + + + + + + + + + + + + + + + + + + + + + +Gahrns Standards Track [Page 5] + diff --git a/docs/rfcs/rfc2244.ACAP.txt b/docs/rfcs/rfc2244.ACAP.txt new file mode 100644 index 0000000..ecf9492 --- /dev/null +++ b/docs/rfcs/rfc2244.ACAP.txt @@ -0,0 +1,4035 @@ + + + + + + +Network Working Group C. Newman +Request for Comments: 2244 Innosoft +Category: Standards Track J. G. Myers + Netscape + November 1997 + + + ACAP -- Application Configuration Access Protocol + + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society 1997. All Rights Reserved. + +Abstract + + The Application Configuration Access Protocol (ACAP) is designed to + support remote storage and access of program option, configuration + and preference information. The data store model is designed to + allow a client relatively simple access to interesting data, to allow + new information to be easily added without server re-configuration, + and to promote the use of both standardized data and custom or + proprietary data. Key features include "inheritance" which can be + used to manage default values for configuration settings and access + control lists which allow interesting personal information to be + shared and group information to be restricted. + + + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page i] + +RFC 2244 ACAP November 1997 + + + + + + Table of Contents + + + +Status of this Memo ............................................... i +Copyright Notice .................................................. i +Abstract .......................................................... i +ACAP Protocol Specification ....................................... 1 +1. Introduction ............................................. 1 +1.1. Conventions Used in this Document ........................ 1 +1.2. ACAP Data Model .......................................... 1 +1.3. ACAP Design Goals ........................................ 1 +1.4. Validation ............................................... 2 +1.5. Definitions .............................................. 2 +1.6. ACAP Command Overview .................................... 4 +2. Protocol Framework ....................................... 4 +2.1. Link Level ............................................... 4 +2.2. Commands and Responses ................................... 4 +2.2.1. Client Protocol Sender and Server Protocol Receiver ...... 4 +2.2.2. Server Protocol Sender and Client Protocol Receiver ...... 5 +2.3. Server States ............................................ 6 +2.3.1. Non-Authenticated State .................................. 6 +2.3.2. Authenticated State ...................................... 6 +2.3.3. Logout State ............................................. 6 +2.4. Operational Considerations ............................... 7 +2.4.1. Untagged Status Updates .................................. 7 +2.4.2. Response when No Command in Progress ..................... 7 +2.4.3. Auto-logout Timer ........................................ 7 +2.4.4. Multiple Commands in Progress ............................ 8 +2.5. Server Command Continuation Request ...................... 8 +2.6. Data Formats ............................................. 8 +2.6.1. Atom ..................................................... 9 +2.6.2. Number ................................................... 9 +2.6.3. String ................................................... 9 +2.6.3.1. 8-bit and Binary Strings ................................. 10 +2.6.4. Parenthesized List ....................................... 10 +2.6.5. NIL ...................................................... 10 +3. Protocol Elements ........................................ 10 +3.1. Entries and Attributes ................................... 10 +3.1.1. Predefined Attributes .................................... 11 +3.1.2. Attribute Metadata ....................................... 12 +3.2. ACAP URL scheme .......................................... 13 +3.2.1. ACAP URL User Name and Authentication Mechanism .......... 13 +3.2.2. Relative ACAP URLs ....................................... 14 +3.3. Contexts ................................................. 14 + + + +Newman & Myers Standards Track [Page ii] + +RFC 2244 ACAP November 1997 + + +3.4. Comparators .............................................. 15 +3.5. Access Control Lists (ACLs) .............................. 17 +3.6. Server Response Codes .................................... 18 +4. Namespace Conventions .................................... 21 +4.1. Dataset Namespace ........................................ 21 +4.2. Attribute Namespace ...................................... 21 +4.3. Formal Syntax for Dataset and Attribute Namespace ........ 22 +5. Dataset Management ....................................... 23 +5.1. Dataset Inheritance ...................................... 23 +5.2. Dataset Attributes ....................................... 24 +5.3. Dataset Creation ......................................... 25 +5.4. Dataset Class Capabilities ............................... 25 +5.5. Dataset Quotas ........................................... 26 +6. Command and Response Specifications ...................... 26 +6.1. Initial Connection ....................................... 26 +6.1.1. ACAP Untagged Response ................................... 26 +6.2. Any State ................................................ 27 +6.2.1. NOOP Command ............................................. 27 +6.2.2. LANG Command ............................................. 28 +6.2.3. LANG Intermediate Response ............................... 28 +6.2.4. LOGOUT Command ........................................... 29 +6.2.5. OK Response .............................................. 29 +6.2.6. NO Response .............................................. 29 +6.2.7. BAD Response ............................................. 30 +6.2.8. BYE Untagged Response .................................... 30 +6.2.9. ALERT Untagged Response .................................. 31 +6.3. Non-Authenticated State .................................. 31 +6.3.1. AUTHENTICATE Command ..................................... 31 +6.4. Searching ................................................ 33 +6.4.1. SEARCH Command ........................................... 33 +6.4.2. ENTRY Intermediate Response .............................. 37 +6.4.3. MODTIME Intermediate Response ............................ 38 +6.4.4. REFER Intermediate Response .............................. 38 +6.4.5. Search Examples .......................................... 38 +6.5. Contexts ................................................. 39 +6.5.1. FREECONTEXT Command ...................................... 39 +6.5.2. UPDATECONTEXT Command .................................... 40 +6.5.3. ADDTO Untagged Response .................................. 40 +6.5.4. REMOVEFROM Untagged Response ............................. 41 +6.5.5. CHANGE Untagged Response ................................. 41 +6.5.6. MODTIME Untagged Response ................................ 42 +6.6. Dataset modification ..................................... 42 +6.6.1. STORE Command ............................................ 42 +6.6.2. DELETEDSINCE Command ..................................... 45 +6.6.3. DELETED Intermediate Response ............................ 45 +6.7. Access Control List Commands ............................. 45 +6.7.1. SETACL Command ........................................... 46 +6.7.2. DELETEACL Command ........................................ 46 + + + +Newman & Myers Standards Track [Page iii] + +RFC 2244 ACAP November 1997 + + +6.7.3. MYRIGHTS Command ......................................... 47 +6.7.4. MYRIGHTS Intermediate Response ........................... 47 +6.7.5. LISTRIGHTS Command ....................................... 47 +6.7.6. LISTRIGHTS Intermediate Response ......................... 48 +6.8. Quotas ................................................... 48 +6.8.1. GETQUOTA Command ......................................... 48 +6.8.3. QUOTA Untagged Response .................................. 49 +6.9. Extensions ............................................... 49 +7. Registration Procedures .................................. 49 +7.1. ACAP Capabilities ........................................ 50 +7.2. ACAP Response Codes ...................................... 50 +7.3. Dataset Classes .......................................... 51 +7.4. Vendor Subtree ........................................... 51 +8. Formal Syntax ............................................ 52 +9. Multi-lingual Considerations ............................. 61 +10. Security Considerations .................................. 62 +11. Acknowledgments .......................................... 63 +12. Authors' Addresses ....................................... 63 +Appendices ........................................................ 64 +A. References ............................................... 64 +B. ACAP Keyword Index ....................................... 66 +C. Full Copyright Statement + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page iv] + RFC 2244 ACAP November 1997 + + +ACAP Protocol Specification + +1. Introduction + +1.1. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If such lines are wrapped without a new "C:" or + "S:" label, then the wrapping is for editorial clarity and is not + part of the command. + + The key words "REQUIRED", "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", + and "MAY" in this document are to be interpreted as described in "Key + words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. + +1.2. ACAP Data Model + + An ACAP server exports a hierarchical tree of entries. Each level of + the tree is called a dataset, and each dataset is made up of a list + of entries. Each entry has a unique name and may contain any number + of named attributes. Each attribute within an entry may be single + valued or multi-valued and may have associated metadata to assist + access and interpretation of the value. + + The rules with which a client interprets the data within a portion of + ACAP's tree of entries are called a dataset class. + +1.3. ACAP Design Goals + + ACAP's primary purpose is to allow users access to their + configuration data from multiple network-connected computers. Users + can then sit down in front of any network-connected computer, run any + ACAP-enabled application and have access to their own configuration + data. Because it is hoped that many applications will become ACAP- + enabled, client simplicity was preferred to server or protocol + simplicity whenever reasonable. + + ACAP is designed to be easily manageable. For this reason, it + includes "inheritance" which allows one dataset to inherit default + attributes from another dataset. In addition, access control lists + are included to permit delegation of management and quotas are + included to control storage. Finally, an ACAP server which is + conformant to this base specification should be able to support most + dataset classes defined in the future without requiring a server + reconfiguration or upgrade. + + + + + + +Newman & Myers Standards Track [Page 1] + +RFC 2244 ACAP November 1997 + + + ACAP is designed to operate well with a client that only has + intermittent access to an ACAP server. For this reason, each entry + has a server maintained modification time so that the client may + detect changes. In addition, the client may ask the server for a + list of entries which have been removed since it last accessed the + server. + + ACAP presumes that a dataset may be potentially large and/or the + client's network connection may be slow, and thus offers server + sorting, selective fetching and change notification for entries + within a dataset. + + As required for most Internet protocols, security, scalability and + internationalization were important design goals. + + Given these design goals, an attempt was made to keep ACAP as simple + as possible. It is a traditional Internet text based protocol which + massively simplifies protocol debugging. It was designed based on + the successful IMAP [IMAP4] protocol framework, with a few + refinements. + +1.4. Validation + + By default, any value may be stored in any attribute for which the + user has appropriate permission and quota. This rule is necessary to + allow the addition of new simple dataset classes without + reconfiguring or upgrading the server. + + In some cases, such as when the value has special meaning to the + server, it is useful to have the server enforce validation by + returning the INVALID response code to a STORE command. These cases + MUST be explicitly identified in the dataset class specification + which SHOULD include specific fixed rules for validation. Since a + given ACAP server may be unaware of any particular dataset class + specification, clients MUST NOT depend on the presence of enforced + validation on the server. + +1.5. Definitions + + + access control list (ACL) + A set of identifier, rights pairs associated with an object. An + ACL is used to determine which operations a user is permitted to + perform on that object. See section 3.5. + + attribute + A named value within an entry. See section 3.1. + + + + +Newman & Myers Standards Track [Page 2] + +RFC 2244 ACAP November 1997 + + + comparator + A named function which can be used to perform one or more of + three comparison operations: ordering, equality and substring + matching. See section 3.4. + + context + An ordered subset of entries in a dataset, created by a SEARCH + command with a MAKECONTEXT modifier. See section 3.3. + + dataset + One level of hierarchy in ACAP's tree of entries. + + dataset class specification + The rules which allow a client to interpret the data within a + portion of ACAP's tree of entries. + + entry + A set of attributes with a unique entry name. See section 3.1. + + metadata + Information describing an attribute, its value and any access + controls associated with that attribute. See section 3.1.2. + + NIL This represents the non-existence of a particular data item. + + NUL A control character encoded as 0 in US-ASCII [US-ASCII]. + + octet + An 8-bit value. On most modern computer systems, an octet is + one byte. + + SASL Simple Authentication and Security Layer [SASL]. + + UTC Universal Coordinated Time as maintained by the Bureau + International des Poids et Mesures (BIPM). + + UTF-8 + An 8-bit transformation format of the Universal Character Set + [UTF8]. Note that an incompatible change was made to the coded + character set referenced by [UTF8], so for the purpose of this + document, UTF-8 refers to the UTF-8 encoding as defined by + version 2.0 of Unicode [UNICODE-2], or ISO 10646 [ISO-10646] + including amendments one through seven. + + + + + + + + +Newman & Myers Standards Track [Page 3] + +RFC 2244 ACAP November 1997 + + +1.6. ACAP Command Overview + + The AUTHENTICATE, NOOP, LANG and LOGOUT commands provide basic + protocol services. The SEARCH command is used to select, sort, fetch + and monitor changes to attribute values and metadata. The + UPDATECONTEXT and FREECONTEXT commands are also used to assist in + monitoring changes in attribute values and metadata. The STORE + command is used to add, modify and delete entries and attributes. + The DELETEDSINCE command is used to assist a client in + re-synchronizing a cache with the server. The GETQUOTA, SETACL, + DELETEACL, LISTRIGHTS and MYRIGHTS commands are used to examine + storage quotas and examine or modify access permissions. + +2. Protocol Framework + +2.1. Link Level + + The ACAP protocol assumes a reliable data stream such as provided by + TCP. When TCP is used, an ACAP server listens on port 674. + +2.2. Commands and Responses + + An ACAP session consists of the establishment of a client/server + connection, an initial greeting from the server, and client/server + interactions. These client/server interactions consist of a client + command, server data, and a server completion result. + + ACAP is a text-based line-oriented protocol. In general, + interactions transmitted by clients and servers are in the form of + lines; that is, sequences of characters that end with a CRLF. The + protocol receiver of an ACAP client or server is either reading a + line, or is reading a sequence of octets with a known count (a + literal) followed by a line. Both clients and servers must be + capable of handling lines of arbitrary length. + +2.2.1. Client Protocol Sender and Server Protocol Receiver + + The client command begins an operation. Each client command is + prefixed with a identifier (an alphanumeric string of no more than 32 + characters, e.g., A0001, A0002, etc.) called a "tag". A different + tag SHOULD be generated by the client for each command. + + There are two cases in which a line from the client does not + represent a complete command. In one case, a command argument is + quoted with an octet count (see the description of literal in section + 2.6.3); in the other case, the command arguments require server + + + + + +Newman & Myers Standards Track [Page 4] + +RFC 2244 ACAP November 1997 + + + feedback (see the AUTHENTICATE command). In some of these cases, the + server sends a command continuation request if it is ready for the + next part of the command. This response is prefixed with the token + "+". + + Note: If, instead, the server detected an error in a + command, it sends a BAD completion response with tag + matching the command (as described below) to reject the + command and prevent the client from sending any more of the + command. + + It is also possible for the server to send a completion or + intermediate response for some other command (if multiple + commands are in progress), or untagged data. In either + case, the command continuation request is still pending; + the client takes the appropriate action for the response, + and reads another response from the server. + + The ACAP server reads a command line from the client, parses the + command and its arguments, and transmits server data and a server + command completion result. + +2.2.2. Server Protocol Sender and Client Protocol Receiver + + Data transmitted by the server to the client come in four forms: + command continuation requests, command completion results, + intermediate responses, and untagged responses. + + A command continuation request is prefixed with the token "+". + + A command completion result indicates the success or failure of the + operation. It is tagged with the same tag as the client command + which began the operation. Thus, if more than one command is in + progress, the tag in a server completion response identifies the + command to which the response applies. There are three possible + server completion responses: OK (indicating success), NO (indicating + failure), or BAD (indicating protocol error such as unrecognized + command or command syntax error). + + An intermediate response returns data which can only be interpreted + within the context of a command in progress. It is tagged with the + same tag as the client command which began the operation. Thus, if + more than one command is in progress, the tag in an intermediate + response identifies the command to which the response applies. A + tagged response other than "OK", "NO", or "BAD" is an intermediate + response. + + + + + +Newman & Myers Standards Track [Page 5] + +RFC 2244 ACAP November 1997 + + + An untagged response returns data or status messages which may be + interpreted outside the context of a command in progress. It is + prefixed with the token "*". Untagged data may be sent as a result + of a client command, or may be sent unilaterally by the server. + There is no syntactic difference between untagged data that resulted + from a specific command and untagged data that were sent + unilaterally. + + The protocol receiver of an ACAP client reads a response line from + the server. It then takes action on the response based upon the + first token of the response, which may be a tag, a "*", or a "+" as + described above. + + A client MUST be prepared to accept any server response at all times. + This includes untagged data that it may not have requested. + + This topic is discussed in greater detail in the Server Responses + section. + +2.3. Server States + + An ACAP server is in one of three states. Most commands are valid in + only certain states. It is a protocol error for the client to + attempt a command while the server is in an inappropriate state for + that command. In this case, a server will respond with a BAD command + completion result. + +2.3.1. Non-Authenticated State + + In non-authenticated state, the user must supply authentication + credentials before most commands will be permitted. This state is + entered when a connection starts. + +2.3.2. Authenticated State + + In authenticated state, the user is authenticated and most commands + will be permitted. This state is entered when acceptable + authentication credentials have been provided. + +2.3.3. Logout State + + In logout state, the session is being terminated, and the server will + close the connection. This state can be entered as a result of a + client request or by unilateral server decision. + + + + + + + +Newman & Myers Standards Track [Page 6] + +RFC 2244 ACAP November 1997 + + + +--------------------------------------+ + |initial connection and server greeting| + +--------------------------------------+ + || (1) || (2) + VV || + +-----------------+ || + |non-authenticated| || + +-----------------+ || + || (4) || (3) || + || VV || + || +----------------+ || + || | authenticated | || + || +----------------+ || + || || (4) || + VV VV VV + +--------------------------------------+ + | logout and close connection | + +--------------------------------------+ + + (1) connection (ACAP greeting) + (2) rejected connection (BYE greeting) + (3) successful AUTHENTICATE command + (4) LOGOUT command, server shutdown, or connection closed + +2.4. Operational Considerations + +2.4.1. Untagged Status Updates + + At any time, a server can send data that the client did not request. + +2.4.2. Response when No Command in Progress + + Server implementations are permitted to send an untagged response + while there is no command in progress. Server implementations that + send such responses MUST deal with flow control considerations. + Specifically, they must either (1) verify that the size of the data + does not exceed the underlying transport's available window size, or + (2) use non-blocking writes. + +2.4.3. Auto-logout Timer + + If a server has an inactivity auto-logout timer, that timer MUST be + of at least 30 minutes duration. The receipt of ANY command from the + client during that interval MUST suffice to reset the auto-logout + timer. + + + + + + +Newman & Myers Standards Track [Page 7] + +RFC 2244 ACAP November 1997 + + +2.4.4. Multiple Commands in Progress + + The client is not required to wait for the completion result of a + command before sending another command, subject to flow control + constraints on the underlying data stream. Similarly, a server is + not required to process a command to completion before beginning + processing of the next command, unless an ambiguity would result + because of a command that would affect the results of other commands. + If there is such an ambiguity, the server executes commands to + completion in the order given by the client. + +2.5. Server Command Continuation Request + + The command continuation request is indicated by a "+" token instead + of a tag. This indicates that the server is ready to accept the + continuation of a command from the client. + + This response is used in the AUTHENTICATE command to transmit server + data to the client, and request additional client data. This + response is also used if an argument to any command is a + synchronizing literal (see section 2.6.3). + + The client is not permitted to send the octets of a synchronizing + literal unless the server indicates that it expects it. This permits + the server to process commands and reject errors on a line-by-line + basis, assuming it checks for non-synchronizing literals at the end + of each line. The remainder of the command, including the CRLF that + terminates a command, follows the octets of the literal. If there + are any additional command arguments the literal octets are followed + by a space and those arguments. + + Example: C: A099 FREECONTEXT {10} + S: + "Ready for additional command text" + C: FRED + C: FOOB + S: A099 OK "FREECONTEXT completed" + C: A044 BLURDYBLOOP {102856} + S: A044 BAD "No such command as 'BLURDYBLOOP'" + + +2.6. Data Formats + + ACAP uses textual commands and responses. Data in ACAP can be in one + of five forms: atom, number, string, parenthesized list or NIL. + + + + + + + +Newman & Myers Standards Track [Page 8] + +RFC 2244 ACAP November 1997 + + +2.6.1. Atom + + An atom consists of one to 1024 non-special characters. It must + begin with a letter. Atoms are used for protocol keywords. + +2.6.2. Number + + A number consists of one or more digit characters, and represents a + numeric value. Numbers are restricted to the range of an unsigned + 32-bit integer: 0 < number < 4,294,967,296. + +2.6.3. String + + A string is in one of two forms: literal and quoted string. The + literal form is the general form of string. The quoted string form + is an alternative that avoids the overhead of processing a literal at + the cost of restrictions of what may be in a quoted string. + + A literal is a sequence of zero or more octets (including CR and LF), + prefix-quoted with an octet count in the form of an open brace ("{"), + the number of octets, close brace ("}"), and CRLF. In the case of + literals transmitted from server to client, the CRLF is immediately + followed by the octet data. + + There are two forms of literals transmitted from client to server. + The form where the open brace ("{") and number of octets is + immediately followed by a close brace ("}") and CRLF is called a + synchronizing literal. When sending a synchronizing literal, the + client must wait to receive a command continuation request before + sending the octet data (and the remainder of the command). The other + form of literal, the non-synchronizing literal, is used to transmit a + string from client to server without waiting for a command + continuation request. The non-synchronizing literal differs from the + synchronizing literal by having a plus ("+") between the number of + octets and the close brace ("}") and by having the octet data + immediately following the CRLF. + + A quoted string is a sequence of zero to 1024 octets excluding NUL, + CR and LF, with double quote (<">) characters at each end. + + The empty string is represented as "" (a quoted string with zero + characters between double quotes), as {0} followed by CRLF (a + synchronizing literal with an octet count of 0), or as {0+} followed + by a CRLF (a non-synchronizing literal with an octet count of 0). + + Note: Even if the octet count is 0, a client transmitting a + synchronizing literal must wait to receive a command + continuation request. + + + +Newman & Myers Standards Track [Page 9] + +RFC 2244 ACAP November 1997 + + +2.6.3.1. 8-bit and Binary Strings + + Most strings in ACAP are restricted to UTF-8 characters and may not + contain NUL octets. Attribute values MAY contain any octets + including NUL. + +2.6.4. Parenthesized List + + Data structures are represented as a "parenthesized list"; a sequence + of data items, delimited by space, and bounded at each end by + parentheses. A parenthesized list can contain other parenthesized + lists, using multiple levels of parentheses to indicate nesting. + + The empty list is represented as () -- a parenthesized list with no + members. + +2.6.5. NIL + + The special atom "NIL" represents the non-existence of a particular + data item that is represented as a string or parenthesized list, as + distinct from the empty string "" or the empty parenthesized list (). + +3. Protocol Elements + + This section defines data formats and other protocol elements used + throughout the ACAP protocol. + +3.1. Entries and Attributes + + Within a dataset, each entry name is made up of zero or more UTF-8 + characters other than slash ("/"). A slash separated list of + entries, one at each level of the hierarchy, forms the full path to + an entry. + + Each entry is made up of a set of attributes. Each attribute has a + hierarchical name in UTF-8, with each component of the name separated + by a period ("."). + + The value of an attribute is either single or multi-valued. A single + value is NIL (has no value), or a string of zero or more octets. A + multi-value is a list of zero or more strings, each of zero or more + octets. + + Attribute names are not permitted to contain asterisk ("*") or + percent ("%") and MUST be valid UTF-8 strings which do not contain + NUL. Invalid attribute names result in a BAD response. Entry names + + + + + +Newman & Myers Standards Track [Page 10] + +RFC 2244 ACAP November 1997 + + + are not permitted to begin with "." or contain slash ("/") and MUST + be valid UTF-8 strings which do not contain NUL. Invalid entry names + in the entry field of a command result in a BAD response. + + Use of non-visible UTF-8 characters in attribute and entry names is + discouraged. + +3.1.1. Predefined Attributes + + Attribute names which do not contain a dot (".") are reserved for + standardized attributes which have meaning in any dataset. The + following attributes are defined by the ACAP protocol. + + entry + Contains the name of the entry. MUST be single valued. + Attempts to use illegal or multi-valued values for the entry + attribute are protocol errors and MUST result in a BAD + completion response. This is a special case. + + modtime + Contains the date and time any read-write metadata in the entry + was last modified. This value MUST be in UTC, MUST be + automatically updated by the server. + + The value consists of 14 or more US-ASCII digits. The first + four indicate the year, the next two indicate the month, the + next two indicate the day of month, the next two indicate the + hour (0 - 23), the next two indicate the minute, and the next + two indicate the second. Any further digits indicate fractions + of a second. + + The time, particularly fractions of a second, need not be + accurate. It is REQUIRED, however, that any two entries in a + dataset changed by successive modifications have strictly + ascending modtime values. In addition, each STORE command + within a dataset (including simultaneous stores from different + connections) MUST use different modtime values. + + This attribute has enforced validation, so any attempt to STORE + a value in this attribute MAY result in a NO response with an + INVALID response code. + + subdataset + If this attribute is set, it indicates the existence of a sub- + dataset of this entry. + + + + + + +Newman & Myers Standards Track [Page 11] + +RFC 2244 ACAP November 1997 + + + The value consists of a list of relative ACAP URLs (see section + 3.2) which may be used to locate the sub-dataset. The base URL + is the full path to the entry followed by a slash ("/"). The + value "." indicates a subdataset is located directly under this + one. Multiple values indicate replicated copies of the + subdataset. + + For example, if the dataset "/folder/site/" has an entry + "public-folder" with a subdataset attribute of ".", then there + exists a dataset "/folder/site/public-folder/". If the value of + the subdataset attribute was instead + "//other.acap.domain//folder/site/public-folder/", that would + indicate the dataset is actually located on a different ACAP + server. + + A dataset can be created by storing a "subdataset" attribute + including ".", and a sub-hierarchy of datasets is deleted by + storing a NIL value to the "subdataset" attribute on the entry + in the parent dataset. + + This attribute has enforced syntax validation. Specifically, if + an attempt is made to STORE a non-list value (other than NIL), + an empty list, or one of the values does not follow the URL + syntax rules [BASIC-URL, REL-URL], then this will result in a NO + response with an INVALID response code. + +3.1.2. Attribute Metadata + + Each attribute is made up of metadata items which describe that + attribute, its value and any associated access controls. Metadata + items may be either read-only, in which case the client is never + permitted to modify the item, or read-write, in which case the client + may modify the item if the access control list (ACL) permits. + + The following metadata items are defined in this specification: + + acl The access control list for the attribute, if one exists. If + the attribute does not have an ACL, NIL is returned. + Read-write. See section 3.5 for the contents of an ACL. + + attribute + The attribute name. Read-only. + + myrights + The set of rights that the client has to the attribute. + Read-only. See section 3.5 for the possible rights. + + + + + +Newman & Myers Standards Track [Page 12] + +RFC 2244 ACAP November 1997 + + + size This is the length of the value. In the case of a + multi-value, this is a list of lengths for each of the values. + Read-only. + + value The value. For a multi-value, this is a list of single + values. Read-write. + + Additional items of metadata may be defined in extensions to this + protocol. Servers MUST respond to unrecognized metadata by returning + a BAD command completion result. + +3.2. ACAP URL scheme + + ACAP URLs are used within the ACAP protocol for the "subdataset" + attribute, referrals and inheritance. They provide a convenient + syntax for referring to other ACAP datasets. The ACAP URL follows + the common Internet scheme syntax as defined in [BASIC-URL] except + that plaintext passwords are not permitted. If : is omitted, + the port defaults to 674. + + An ACAP URL has the following general form: + + url-acap = "acap://" url-server "/" url-enc-entry [url-filter] + [url-extension] + + The element includes the hostname, and optional user + name, authentication mechanism and port number. The + element contains the name of an entry path encoded according to the + rules in [BASIC-URL]. + + The element is an optional list of interesting attribute + names. If omitted, the URL refers to all attributes of the named + entry. The element is reserved for extensions to + this URL scheme. + + Note that unsafe or reserved characters such as " " or "?" MUST be + hex encoded as described in the URL specification [BASIC-URL]. Hex + encoded octets are interpreted according to UTF-8 [UTF8]. + +3.2.1. ACAP URL User Name and Authentication Mechanism + + A user name and/or authentication mechanism may be supplied. They + are used in the "AUTHENTICATE" command after making the connection to + the ACAP server. If no user name or authentication mechanism is + supplied, then the SASL ANONYMOUS [SASL-ANON] mechanism is used by + default. If an authentication mechanism is supplied without a user + + + + + +Newman & Myers Standards Track [Page 13] + +RFC 2244 ACAP November 1997 + + + name, then one SHOULD be obtained from the specified mechanism or + requested from the user as appropriate. If a user name is supplied + without an authentication mechanism then ";AUTH=*" is assumed. + + The ";AUTH=" authentication parameter is interpreted as described in + the IMAP URL Scheme [IMAP-URL]. + + Note that if unsafe or reserved characters such as " " or ";" are + present in the user name or authentication mechanism, they MUST be + encoded as described in the URL specification [BASIC-URL]. + +3.2.2. Relative ACAP URLs + + Because ACAP uses "/" as the hierarchy separator for dataset paths, + it works well with the relative URL rules defined in the relative URL + specification [REL-URL]. + + The grammar element is considered part of the user name for + purposes of resolving relative ACAP URLs. + + The base URL for a relative URL stored in an attribute's value is + formed by taking the path to the dataset containing that attribute, + appending a "/" followed by the entry name of the entry containing + that attribute followed by "/". + +3.3. Contexts + + A context is subset of entries in a dataset or datasets, created by a + SEARCH command with a MAKECONTEXT modifier. Context names are + client-generated strings and must not start with the slash ('/') + character. + + When a client creates a context, it may request automatic + notification of changes. A client may also request enumeration of + entries within a context. Enumeration simplifies the implementation + of a "virtual scrollbar" by the client. + + A context exists only within the ACAP session in which it was + created. When the connection is closed, all contexts associated with + that connection are automatically discarded. A server is required to + support at least 100 active contexts within a session. If the server + supports a larger limit it must advertise it in a CONTEXTLIMIT + capability. + + + + + + + + +Newman & Myers Standards Track [Page 14] + +RFC 2244 ACAP November 1997 + + +3.4. Comparators + + A comparator is a named function which takes two input values and can + be used to perform one or more of four comparison operations: + ordering, equality, prefix and substring matching. + + The ordering operation is used both for the SORT search modifier and + the COMPARE and COMPARESTRICT search keys. Ordering comparators can + determine the ordinal precedence of any two values. When used for + ordering, a comparator's name can be prefixed with "+" or "-" to + indicate that the ordering should be normal order or reversed order + respectively. If no prefix is included, "+" is assumed. + + For the purpose of ordering, a comparator may designate certain + values as having an undefined ordinal precedence. Such values always + collate with equal value after all other values regardless of whether + normal or reversed ordering is used. Unless the comparator + definition specifies otherwise, multi-values and NIL values have an + undefined ordinal precedence. + + The equality operation is used for the EQUAL search modifier, and + simply determines if the two values are considered equal under the + comparator function. When comparing a single value to a multi-value, + the two are considered equal if any one of the multiple values is + equal to the single value. + + The prefix match operation is used for the PREFIX search modifier, + and simply determines if the search value is a prefix of the item + being searched. In the case of prefix search on a multi-value, the + match is successful if the value is a prefix of any one of the + multiple values. + + The substring match operation is used for the SUBSTRING search + modifier, and simply determines if search value is a substring of the + item being searched. In the case of substring search on a multi- + value, the match is successful if the value is a substring of any one + of the multiple values. + + Rules for naming and registering comparators will be defined in a + future specification. Servers MUST respond to unknown or improperly + used comparators with a BAD command completion result. + + + + + + + + + + +Newman & Myers Standards Track [Page 15] + +RFC 2244 ACAP November 1997 + + + The following comparators are defined by this standard and MUST be + implemented: + + i;octet + Operations: Ordering, Equality, Prefix match, Substring match + + For collation, the i;octet comparator interprets the value of + an attribute as a series of unsigned octets with ordinal + values from 0 to 255. When ordering two strings, each octet + pair is compared in sequence until the octets are unequal or + the end of the string is reached. When collating two strings + where the shorter is a prefix of the longer, the shorter + string is interpreted as having a smaller ordinal value. The + "i;octet" or "+i;octet" forms collate smaller ordinal values + earlier, and the "-i;octet" form collates larger ordinal + values earlier. + + For the equality function, two strings are equal if they are + the same length and contain the same octets in the same + order. NIL is equal only to itself. + + For non-binary, non-nil single values, i;octet ordering is + equivalent to the ANSI C [ISO-C] strcmp() function applied to + C string representations of the values. For non-binary, + non-nil single values, i;octet substring match is equivalent + to the ANSI C strstr() function applied to the C string + representations of the values. + + i;ascii-casemap + Operations: Ordering, Equality, Prefix match, Substring match + + The i;ascii-casemap comparator first applies a mapping to the + attribute values which translates all US-ASCII letters to + uppercase (octet values 0x61 to 0x7A are translated to octet + values 0x41 to 0x5A respectively), then applies the i;octet + comparator as described above. With this function the values + "hello" and "HELLO" have the same ordinal value and are + considered equal. + + i;ascii-numeric + Operations: Ordering, Equality + + The i;ascii-numeric comparator interprets strings as decimal + positive integers represented as US-ASCII digits. All values + which do not begin with a US-ASCII digit are considered equal + with an ordinal value higher than all non-NIL single-valued + + + + + +Newman & Myers Standards Track [Page 16] + +RFC 2244 ACAP November 1997 + + + attributes. Otherwise, all US-ASCII digits (octet values + 0x30 to 0x39) are interpreted starting from the beginning of + the string to the first non-digit or the end of the string. + + +3.5. Access Control Lists (ACLs) + + An access control list is a set of identifier, rights pairs used to + restrict access to a given dataset, attribute or attribute within an + entry. An ACL is represented by a multi-value with each value + containing an identifier followed by a tab character followed by the + rights. The syntax is defined by the "acl" rule in the formal syntax + in section 8. + + Identifier is a UTF-8 string. The identifier "anyone" is reserved to + refer to the universal identity (all authentications, including + anonymous). All user name strings accepted by the AUTHENTICATE + command to authenticate to the ACAP server are reserved as + identifiers for the corresponding user. Identifiers starting with a + slash ("/") character are reserved for authorization groups which + will be defined in a future specification. Identifiers MAY be + prefixed with a dash ("-") to indicate a revocation of rights. All + other identifiers have implementation-defined meanings. + + Rights is a string listing a (possibly empty) set of alphanumeric + characters, each character listing a set of operations which is being + controlled. Letters are reserved for "standard" rights, listed + below. The set of standard rights may only be extended by a + standards-track or IESG approved experimental RFC. Digits are + reserved for implementation or site defined rights. The currently + defined standard rights are: + + x - search (use EQUAL search key with i;octet comparator) + r - read (access with SEARCH command) + w - write (modify with STORE command) + i - insert (perform STORE on a previously NIL value) + a - administer (perform SETACL or STORE on ACL attribute/metadata) + + An implementation may force rights to always or never be granted. In + particular, implementations are expected to grant implicit read and + administer rights to a user's personal dataset storage in order to + avoid denial of service problems. Rights are never tied, unlike the + IMAP ACL extension [IMAP-ACL]. + + It is possible for multiple identifiers in an access control list to + apply to a given user (or other authentication identity). For + example, an ACL may include rights to be granted to the identifier + matching the user, one or more implementation-defined identifiers + + + +Newman & Myers Standards Track [Page 17] + +RFC 2244 ACAP November 1997 + + + matching groups which include the user, and/or the identifier + "anyone". These rights are combined by taking the union of all + positive rights which apply to a given user and subtracting the union + of all negative rights which apply to that user. A client MAY avoid + this calculation by using the MYRIGHTS command and metadata items. + + Each attribute of each entry of a dataset may potentially have an + ACL. If an attribute in an entry does not have an ACL, then access + is controlled by a default ACL for that attribute in the dataset, if + it exists. If there is no default ACL for that attribute in the + dataset, access is controlled by a default ACL for that dataset. The + default ACL for a dataset must exist. + + In order to perform any access or manipulation on an entry in a + dataset, the client must have 'r' rights on the "entry" attribute of + the entry. Implementations should take care not to reveal via error + messages the existence of an entry for which the client does not have + 'r' rights. A client does not need access to the "subdataset" + attribute of the parent dataset in order to access the contents of a + dataset. + + Many of the ACL commands and responses include an "acl object" + parameter, for specifying what the ACL applies to. This is a + parenthesized list. The list contains just the dataset name when + referring to the default ACL for a dataset. The list contains a + dataset name and an attribute name when referring to the default ACL + for an attribute in a dataset. The list contains a dataset name, an + attribute name, and an entry name when referring to the ACL for an + attribute of an entry of a dataset. + + +3.6. Server Response Codes + + An OK, NO, BAD, ALERT or BYE response from the server MAY contain a + response code to describe the event in a more detailed machine + parsable fashion. A response code consists of data inside + parentheses in the form of an atom, possibly followed by a space and + arguments. Response codes are defined when there is a specific + action that a client can take based upon the additional information. + In order to support future extension, the response code is + represented as a slash-separated hierarchy with each level of + hierarchy representing increasing detail about the error. Clients + MUST tolerate additional hierarchical response code detail which they + don't understand. + + The currently defined response codes are: + + + + + +Newman & Myers Standards Track [Page 18] + +RFC 2244 ACAP November 1997 + + + AUTH-TOO-WEAK + This response code is returned on a tagged NO result from an + AUTHENTICATE command. It indicates that site security policy + forbids the use of the requested mechanism for the specified + authentication identity. + + ENCRYPT-NEEDED + This response code is returned on a tagged NO result from an + AUTHENTICATE command. It indicates that site security policy + requires the use of a strong encryption mechanism for the + specified authentication identity and mechanism. + + INVALID + This response code indicates that a STORE command included + data which the server implementation does not permit. It + MUST NOT be used unless the dataset class specification for + the attribute in question explicitly permits enforced server + validation. The argument is the attribute which was invalid. + + MODIFIED + This response code indicates that a conditional store failed + because the modtime on the entry is later than the modtime + specified with the STORE command UNCHANGEDSINCE modifier. + The argument is the entry which had been modified. + + NOEXIST + This response code indicates that a search or NOCREATE store + failed because a specified dataset did not exist. The + argument is the dataset which does not exist. + + PERMISSION + A command failed due to insufficient permission based on the + access control list or implicit rights. The argument is the + acl-object which caused the permission failure. + + QUOTA + A STORE or SETACL command which would have increased the size + of the dataset failed due to insufficient quota. + + REFER + This response code may be returned in a tagged NO response to + any command that takes a dataset name as a parameter. It has + one or more arguments with the syntax of relative URLs. It + is a referral, indicating that the command should be retried + using one of the relative URLs. + + + + + + +Newman & Myers Standards Track [Page 19] + +RFC 2244 ACAP November 1997 + + + SASL This response code can occur in the tagged OK response to a + successful AUTHENTICATE command and includes the optional + final server response data from the server as specified by + SASL [SASL]. + + TOOMANY + This response code may be returned in a tagged OK response to + a SEARCH command which includes the LIMIT modifier. The + argument returns the total number of matching entries. + + TOOOLD + The modtime specified in the DELETEDSINCE command is too old, + so deletedsince information is no longer available. + + TRANSITION-NEEDED + This response code occurs on a NO response to an AUTHENTICATE + command. It indicates that the user name is valid, but the + entry in the authentication database needs to be updated in + order to permit authentication with the specified mechanism. + This can happen if a user has an entry in a system + authentication database such as Unix /etc/passwd, but does + not have credentials suitable for use by the specified + mechanism. + + TRYLATER + A command failed due to a temporary server failure. The + client MAY continue using local information and try the + command later. + + TRYFREECONTEXT + This response code may be returned in a tagged NO response to + a SEARCH command which includes the MAKECONTEXT modifier. It + indicates that a new context may not be created due to the + server's limit on the number of existing contexts. + + WAYTOOMANY + This response code may be returned in a tagged NO response to + a SEARCH command which includes a HARDLIMIT search modifier. + It indicates that the SEARCH would have returned more entries + than the HARDLIMIT permitted. + + Additional response codes MUST be registered with IANA according + to the proceedures in section 7.2. Client implementations MUST + tolerate response codes that they do not recognize. + + + + + + + +Newman & Myers Standards Track [Page 20] + +RFC 2244 ACAP November 1997 + + +4. Namespace Conventions + +4.1. Dataset Namespace + + The dataset namespace is a slash-separated hierarchy. The first + component of the dataset namespace is a dataset class. Dataset + classes MUST have a vendor prefix (vendor.) or be + specified in a standards track or IESG approved experimental RFC. + See section 7.3 for the registration template. + + The second component of the dataset name is "site", "group", "host", + or "user" referring to server-wide data, administrative group data, + per-host data and per-user data respectively. + + For "group", "host", and "user" areas, the third component of the + path is the group name, the fully qualified host domain name, or the + user name. A path of the form "//~/" is a convenient + abbreviation for "//user//". + + Dataset names which begin with "/byowner/" are reserved as an + alternate view of the namespace. This provides a way to see all the + dataset classes which a particular owner uses. For example, + "/byowner/~//" is an alternate name for + "//~/". Byowner provides a way to view a list of + dataset classes owned by a given user; this is done using the dataset + "/byowner/user//" with the NOINHERIT SEARCH modifier. + + The dataset "/" may be used to find all dataset classes visible to + the current user. A dataset of the form "//user/" may + be used to find all users which have made a dataset or entry of that + class visible to the current user. + + The formal syntax for a dataset name is defined by the "dataset-name" + rule in section 4.3. + +4.2. Attribute Namespace + + Attribute names which do not contain a dot (".") are reserved for + standardized attributes which have meaning in any dataset. In order + to simplify client implementations, the attribute namespace is + intended to be unique across all datasets. To achieve this, + attribute names are prefixed with the dataset class name followed by + a dot ("."). Attributes which affect management of the dataset are + prefixed with "dataset.". In addition, a subtree of the "vendor." + attribute namespace may be registered with IANA according to the + rules in section 7.4. ACAP implementors are encouraged to help + define interoperable dataset classes specifications rather than using + the private attribute namespace. + + + +Newman & Myers Standards Track [Page 21] + +RFC 2244 ACAP November 1997 + + + Some users or sites may wish to add their own private attributes to + certain dataset classes. In order to enable this, the "user.." and "site." subtrees of the attribute namespace are reserved + for user-specific and site-specific attributes respectively and will + not be standardized. Such attributes are not interoperable so are + discouraged in favor of defining standard attributes. A future + extension is expected to permit discovery of syntax for user or + site-specific attributes. Clients wishing to support display of user + or site-specific attributes should display the value of any non-NIL + single-valued "user.." or "site." attribute which has + valid UTF-8 syntax. + + The formal syntax for an attribute name is defined by the + "attribute-name" rule in the next section. + +4.3. Formal Syntax for Dataset and Attribute Namespace + + The naming conventions for datasets and attributes are defined by the + following ABNF. Note that this grammar is not part of the ACAP + protocol syntax in section 8, as dataset names and attribute names + are encoded as strings within the ACAP protocol. + + attribute-dacl = "dataset.acl" *("." name-component) + + attribute-dset = dataset-std 1*("." name-component) + ;; MUST be defined in a dataset class specification + + attribute-name = attribute-std / attr-site / attr-user / vendor-name + + attribute-std = "entry" / "subdataset" / "modtime" / + "dataset.inherit" / attribute-dacl / attribute-dset + + attr-site = "site" 1*("." name-component) + + attr-user = "user." name-component 1*("." name-component) + + byowner = "/byowner/" owner "/" + [dataset-class "/" dataset-sub] + + dataset-class = dataset-std / vendor-name + + dataset-normal = "/" [dataset-class "/" + (owner-prefix / dataset-tail)] + + dataset-name = byowner / dataset-normal + + + + + + +Newman & Myers Standards Track [Page 22] + +RFC 2244 ACAP November 1997 + + + dataset-std = name-component + ;; MUST be registered with IANA and the spec MUST + ;; be published as a standards track or + ;; IESG-approved experimental RFC + + dataset-sub = *(dname-component "/") + ;; The rules for this portion of the namespace may + ;; be further restricted by the dataset class + ;; specification. + + dataset-tail = owner "/" dataset-sub + + dname-component = 1*UTF8-CHAR + ;; MUST NOT begin with "." or contain "/" + + name-component = 1*UTF8-CHAR + ;; MUST NOT contain ".", "/", "%", or "*" + + owner = "site" / owner-host / owner-group / + owner-user / "~" + + owner-group = "group/" dname-component + + owner-host = "host/" dname-component + + owner-prefix = "group/" / "host/" / "user/" + + owner-user = "user/" dname-component + + vendor-name = vendor-token *("." name-component) + + vendor-token = "vendor." name-component + ;; MUST be registered with IANA + +5. Dataset Management + + The entry with an empty name ("") in the dataset is used to hold + management information for the dataset as a whole. + +5.1. Dataset Inheritance + + It is possible for one dataset to inherit data from another. The + dataset from which the data is inherited is called the base dataset. + Data in the base dataset appears in the inheriting dataset, except + when overridden by a STORE to the inheriting dataset. + + + + + + +Newman & Myers Standards Track [Page 23] + +RFC 2244 ACAP November 1997 + + + The base dataset is usually a system-wide or group-wide set of + defaults. A system-wide dataset usually has one inheriting dataset + per user, allowing each user to add to or modify the defaults as + appropriate. + + An entry which exists in both the inheriting and base dataset + inherits a modtime equal to the greater of the two modtimes. An + attribute in such an entry is inherited from the base dataset if it + was never modified by a STORE command in the inheriting dataset or if + DEFAULT was stored to that attribute. This permits default entries + to be amended rather than replaced in the inheriting dataset. + + The "subdataset" attribute is not directly inherited. If the base + dataset includes a "subdataset" attribute and the inheriting dataset + does not, then the "subdataset" attribute will inherit a virtual + value of a list containing a ".". The subdataset at that node is + said to be a "virtual" dataset as it is simply a virtual copy of the + appropriate base dataset with all "subdataset" attributes changed to + a list containing a ".". A virtual dataset is not visible if + NOINHERIT is specified on the SEARCH command. + + Servers MUST support at least two levels of inheritance. This + permits a user's dataset such as "/options/user/fred/common" to + inherit from a group dataset such as "/options/group/dinosaur + operators/common" which in turn inherits from a server-wide dataset + such as "/options/site/common". + +5.2. Dataset Attributes + + The following attributes apply to management of the dataset when + stored in the "" entry of a dataset. These attributes are not + inherited. + + dataset.acl + This holds the default access control list for the dataset. + This attribute is validated, so an invalid access control list + in a STORE command will result in a NO response with an INVALID + response code. + + dataset.acl. + This holds the default access control list for an attribute + within the dataset. This attribute is validated, so an invalid + access control list in a STORE command will result in a NO + response with an INVALID response code. + + dataset.inherit + This holds the name of a dataset from which to inherit according + to the rules in the previous section. This attribute MAY refer + + + +Newman & Myers Standards Track [Page 24] + +RFC 2244 ACAP November 1997 + + + to a non-existent dataset, in which case nothing is inherited. + This attribute is validated, so illegal dataset syntax or an + attempt to store a multi-value will result in a NO response with + an INVALID response code. + +5.3. Dataset Creation + + When a dataset is first created (by storing a "." in the subdataset + attribute or storing an entry in a previously non-existent dataset), + the dataset attributes are initialized with the values from the + parent dataset in the "/byowner/" hierarchy. In the case of the + "dataset.inherit" attribute, the appropriate hierarchy component is + added. For example, given the following entry (note that \t refers + to the US-ASCII horizontal tab character): + + entry path "/byowner/user/joe/" + dataset.acl ("joe\txrwia" "fred\txr") + dataset.inherit "/byowner/site" + + If a new dataset class "/byowner/user/joe/new" is created, it will + have the following dataset attributes: + + entry path "/byowner/user/joe/new/" + dataset.acl ("joe\txrwia" "fred\txr") + dataset.inherit "/byowner/site/new" + + Note that the dataset "/byowner/user/joe/new/" is equivalent to + "/new/user/joe/". + +5.4. Dataset Class Capabilities + + Certain dataset classes or dataset class features may only be useful + if there is an active updating client or integrated server support + for the feature. The dataset class "capability" is reserved to allow + clients or servers to advertise such features. The "entry" attribute + within this dataset class is the name of the dataset class whose + features are being described. The attributes are prefixed with + "capability.." and are defined by the appropriate + dataset class specification. + + Since it is possible for an unprivileged user to run an active client + for himself, a per-user capability dataset is useful. The dataset + "/capability/~/" holds information about all features available to + the user (via inheritance), and the dataset "/capability/site/" holds + information about all features supported by the site. + + + + + + +Newman & Myers Standards Track [Page 25] + +RFC 2244 ACAP November 1997 + + +5.5. Dataset Quotas + + Management and scope of quotas is implementation dependent. Clients + can check the applicable quota limit and usage (in bytes) with the + GETQUOTA command. Servers can notify the client of a low quota + situation with the QUOTA untagged response. + +6. Command and Response Specifications + + ACAP commands and responses are described in this section. Commands + are organized first by the state in which the command is permitted, + then by a general category of command type. + + Command arguments, identified by "Arguments:" in the command + descriptions below, are described by function, not by syntax. The + precise syntax of command arguments is described in the Formal Syntax + section. + + Some commands cause specific server data to be returned; these are + identified by "Data:" in the command descriptions below. See the + response descriptions in the Responses section for information on + these responses, and the Formal Syntax section for the precise syntax + of these responses. It is possible for server data to be transmitted + as a result of any command; thus, commands that do not specifically + require server data specify "no specific data for this command" + instead of "none". + + The "Result:" in the command description refers to the possible + tagged status responses to a command, and any special interpretation + of these status responses. + +6.1. Initial Connection + + Upon session startup, the server sends one of two untagged responses: + ACAP or BYE. The untagged BYE response is described in section + 6.2.8. + +6.1.1. ACAP Untagged Response + + Data: capability list + + The untagged ACAP response indicates the session is ready to + accept commands and contains a space-separated listing of + capabilities that the server supports. Each capability is + represented by a list containing the capability name optionally + followed by capability specific string arguments. + + + + + +Newman & Myers Standards Track [Page 26] + +RFC 2244 ACAP November 1997 + + + ACAP capability names MUST be registered with IANA according to + the rules in section 7.1. + + Client implementations SHOULD NOT require any capability name + beyond those defined in this specification, and MUST tolerate any + unknown capability names. A client implementation MAY be + configurable to require SASL mechanisms other than CRAM-MD5 + [CRAM-MD5] for site security policy reasons. + + The following initial capabilities are defined: + + CONTEXTLIMIT + The CONTEXTLIMIT capability has one argument which is a + number describing the maximum number of contexts the server + supports per connection. The number 0 indicates the server + has no limit, otherwise this number MUST be greater than + 100. + + IMPLEMENTATION + The IMPLEMENTATION capability has one argument which is a + string describing the server implementation. ACAP clients + MUST NOT alter their behavior based on this value. It is + intended primarily for debugging purposes. + + SASL The SASL capability includes a list of the authentication + mechanisms supported by the server. See section 6.3.1. + + + Example: S: * ACAP (IMPLEMENTATION "ACME v3.5") + (SASL "CRAM-MD5") (CONTEXTLIMIT "200") + +6.2. Any State + + The following commands and responses are valid in any state. + +6.2.1. NOOP Command + + Arguments: none + + Data: no specific data for this command (but see below) + + Result: OK - noop completed + BAD - command unknown or arguments invalid + + The NOOP command always succeeds. It does nothing. It can be + used to reset any inactivity auto-logout timer on the server. + + Example: C: a002 NOOP + + + +Newman & Myers Standards Track [Page 27] + +RFC 2244 ACAP November 1997 + + + S: a002 OK "NOOP completed" + + +6.2.2. LANG Command + + Arguments: list of language preferences + + Data: intermediate response: LANG + + Result: OK - lang completed + NO - no matching language available + BAD - command unknown or arguments invalid + + One or more arguments are supplied to indicate the client's + preferred languages [LANG-TAGS] for error messages. The server + will match each client preference in order against its internal + table of available error string languages. For a client + preference to match a server language, the client's language tag + MUST be a prefix of the server's tag and match up to a "-" or the + end of string. If a match is found, the server returns an + intermediate LANG response and an OK response. The LANG response + indicates the actual language selected and appropriate comparators + for use with the languages listed in the LANG command. + + If no LANG command is issued, all error text strings MUST be in + the registered language "i-default" [CHARSET-LANG-POLICY], + intended for an international audience. + + Example: C: A003 LANG "fr-ca" "fr" "en-ca" "en-uk" + S: A003 LANG "fr-ca" "i;octet" "i;ascii-numeric" + "i;ascii-casemap" "en;primary" "fr;primary" + S: A003 OK "Bonjour" + + +6.2.3. LANG Intermediate Response + + Data: language for error responses + appropriate comparators + + The LANG response indicates the language which will be used for + error responses and the comparators which are appropriate for the + languages listed in the LANG command. The comparators SHOULD be + in approximate order from most efficient (usually "i;octet") to + most appropriate for human text in the preferred language. + + + + + + + +Newman & Myers Standards Track [Page 28] + +RFC 2244 ACAP November 1997 + + +6.2.4. LOGOUT Command + + Arguments: none + + Data: mandatory untagged response: BYE + + Result: OK - logout completed + BAD - command unknown or arguments invalid + + The LOGOUT command informs the server that the client is done with + the session. The server must send a BYE untagged response before + the (tagged) OK response, and then close the network connection. + + Example: C: A023 LOGOUT + S: * BYE "ACAP Server logging out" + S: A023 OK "LOGOUT completed" + (Server and client then close the connection) + + +6.2.5. OK Response + + Data: optional response code + human-readable text + + The OK response indicates an information message from the server. + When tagged, it indicates successful completion of the associated + command. The human-readable text may be presented to the user as + an information message. The untagged form indicates an + information-only message; the nature of the information MAY be + indicated by a response code. + + Example: S: * OK "Master ACAP server is back up" + + +6.2.6. NO Response + + Data: optional response code + human-readable text + + The NO response indicates an operational error message from the + server. When tagged, it indicates unsuccessful completion of the + associated command. The untagged form indicates a warning; the + command may still complete successfully. The human-readable text + describes the condition. + + Example: C: A010 SEARCH "/addressbook/" DEPTH 3 RETURN ("*") + EQUAL "entry" "+i;octet" "bozo" + S: * NO "Master ACAP server is down, your data may + + + +Newman & Myers Standards Track [Page 29] + +RFC 2244 ACAP November 1997 + + + be out of date." + S: A010 OK "search done" + ... + C: A222 STORE ("/folder/site/comp.mail.misc" + "folder.creation-time" "19951206103412") + S: A222 NO (PERMISSION ("/folder/site/")) "Permission + denied" + + +6.2.7. BAD Response + + Data: optional response code + human-readable text + + The BAD response indicates an error message from the server. When + tagged, it reports a protocol-level error in the client's command; + the tag indicates the command that caused the error. The untagged + form indicates a protocol-level error for which the associated + command can not be determined; it may also indicate an internal + server failure. The human-readable text describes the condition. + + Example: C: ...empty line... + S: * BAD "Empty command line" + C: A443 BLURDYBLOOP + S: A443 BAD "Unknown command" + C: A444 NOOP Hello + S: A444 BAD "invalid arguments" + + +6.2.8. BYE Untagged Response + + Data: optional response code + human-readable text + + The untagged BYE response indicates that the server is about to + close the connection. The human-readable text may be displayed to + the user in a status report by the client. The BYE response may + be sent as part of a normal logout sequence, or as a panic + shutdown announcement by the server. It is also used by some + server implementations as an announcement of an inactivity auto- + logout. + + This response is also used as one of two possible greetings at + session startup. It indicates that the server is not willing to + accept a session from this client. + + Example: S: * BYE "Auto-logout; idle for too long" + + + + +Newman & Myers Standards Track [Page 30] + +RFC 2244 ACAP November 1997 + + +6.2.9. ALERT Untagged Response + + Data: optional response code + human-readable text + + The human-readable text contains a special human generated alert + message that MUST be presented to the user in a fashion that calls + the user's attention to the message. This is intended to be used + for vital messages from the server administrator to the user, such + as a warning that the server will soon be shut down for + maintenance. + + Example: S: * ALERT "This ACAP server will be shut down in + 10 minutes for system maintenance." + + +6.3. Non-Authenticated State + + In non-authenticated state, the AUTHENTICATE command establishes + authentication and enters authenticated state. The AUTHENTICATE + command provides a general mechanism for a variety of authentication + techniques. + + Server implementations may allow non-authenticated access to certain + information by supporting the SASL ANONYMOUS [SASL-ANON] mechanism. + + Once authenticated (including as anonymous), it is not possible to + re-enter non-authenticated state. + + Only the any-state commands (NOOP, LANG and LOGOUT) and the + AUTHENTICATE command are valid in non-authenticated state. + + +6.3.1. AUTHENTICATE Command + + Arguments: SASL mechanism name + optional initial response + + Data: continuation data may be requested + + Result: OK - authenticate completed, now in authenticated state + NO - authenticate failure: unsupported authentication + mechanism, credentials rejected + BAD - command unknown or arguments invalid, + authentication exchange cancelled + + + + + + +Newman & Myers Standards Track [Page 31] + +RFC 2244 ACAP November 1997 + + + The AUTHENTICATE command indicates a SASL [SASL] authentication + mechanism to the server. If the server supports the requested + authentication mechanism, it performs an authentication protocol + exchange to authenticate and identify the user. Optionally, it + also negotiates a security layer for subsequent protocol + interactions. If the requested authentication mechanism is not + supported, the server rejects the AUTHENTICATE command by sending + a tagged NO response. + + The authentication protocol exchange consists of a series of + server challenges and client answers that are specific to the + authentication mechanism. A server challenge consists of a + command continuation request with the "+" token followed by a + string. The client answer consists of a line consisting of a + string. If the client wishes to cancel an authentication + exchange, it should issue a line with a single unquoted "*". If + the server receives such an answer, it must reject the + AUTHENTICATE command by sending a tagged BAD response. + + The optional initial-response argument to the AUTHENTICATE command + is used to save a round trip when using authentication mechanisms + that are defined to send no data in the initial challenge. When + the initial-response argument is used with such a mechanism, the + initial empty challenge is not sent to the client and the server + uses the data in the initial-response argument as if it were sent + in response to the empty challenge. If the initial-response + argument to the AUTHENTICATE command is used with a mechanism that + sends data in the initial challenge, the server rejects the + AUTHENTICATE command by sending a tagged NO response. + + The service name specified by this protocol's profile of SASL is + "acap". + + If a security layer is negotiated through the SASL authentication + exchange, it takes effect immediately following the CRLF that + concludes the authentication exchange for the client, and the CRLF + of the tagged OK response for the server. + + All ACAP implementations MUST implement the CRAM-MD5 SASL + mechanism [CRAM-MD5], although they MAY offer a configuration + option to disable it if site security policy dictates. The + example below is the same example described in the CRAM-MD5 + specification. + + If an AUTHENTICATE command fails with a NO response, the client + may try another authentication mechanism by issuing another + AUTHENTICATE command. In other words, the client may request + authentication types in decreasing order of preference. + + + +Newman & Myers Standards Track [Page 32] + +RFC 2244 ACAP November 1997 + + + Example: S: * ACAP (IMPLEMENTATION "Blorfysoft v3.5") + (SASL "CRAM-MD5" "KERBEROS_V4") + C: A001 AUTHENTICATE "CRAM-MD5" + S: + "<1896.697170952@postoffice.reston.mci.net>" + C: "tim b913a602c7eda7a495b4e6e7334d3890" + S: A001 OK "CRAM-MD5 authentication successful" + + +6.4. Searching + + This section describes the SEARCH command, for retrieving data from + datasets. + + +6.4.1. SEARCH Command + + Arguments: dataset or context name + optional list of modifiers + search criteria + + Data: intermediate responses: ENTRY, MODTIME, REFER + untagged responses: ADDTO, REMOVEFROM, CHANGE, MODTIME + + Result: OK - search completed + NO - search failure: can't perform search + BAD - command unknown or arguments invalid + + The SEARCH command identifies a subset of entries in a dataset and + returns information on that subset to the client. Inherited + entries and attributes are included in the search unless the + NOINHERIT search modifier is included or the user does not have + permission to read the attributes in the base dataset. + + The first argument to SEARCH identifies what is to be searched. + If the string begins with a slash ("/"), it is the name of a + dataset to be searched, otherwise it is a name of a context that + was created by a SEARCH command given previously in the session. + + A successful SEARCH command MAY result in intermediate ENTRY + responses and MUST result in a MODTIME intermediate response. + + Following that are zero or more modifiers to the search. Each + modifier may be specified at most once. The defined modifiers + are: + + + + + + + +Newman & Myers Standards Track [Page 33] + +RFC 2244 ACAP November 1997 + + + DEPTH number + The SEARCH command will traverse the dataset tree up to the + specified depth. ENTRY responses will include the full path + to the entry. A value of "0" indicates that the search + should traverse the entire tree. A value of "1" is the + default and indicates only the specified dataset should be + searched. If a dataset is traversed which is not located on + the current server, then a REFER intermediate response is + returned for that subtree and the search continues. + + HARDLIMIT number + If the SEARCH command would result in more than number + entries, the SEARCH fails with a NO completion result with a + WAYTOOMANY response code. + + LIMIT number number + Limits the number of intermediate ENTRY responses that the + search may generate. The first numeric argument specifies + the limit, the second number specifies the number of entries + to return if the number of matches exceeds the limit. If the + limit is exceeded, the SEARCH command still succeeds, + returning the total number of matches in a TOOMANY response + code in the tagged OK response. + + MAKECONTEXT [ENUMERATE] [NOTIFY] context + Causes the SEARCH command to create a context with the name + given in the argument to refer to the matching entries. If + the SEARCH is successful, the context name may then be given + as an argument to subsequent SEARCH commands to search the + set of matching entries. If a context with the specified + name already exists, it is first freed. If a new context may + not be created due to the server's limit on the number of + existing contexts, the command fails, returning a + TRYFREECONTEXT response code in the NO completion response. + + The optional "ENUMERATE" and "NOTIFY" arguments may be + included to request enumeration of the context (for virtual + scroll bars) or change notifications for the context. If + "NOTIFY" is not requested, the context represents a snapshot + of the entries at the time the SEARCH was issued. + + ENUMERATE requests that the contents of the context be + ordered according to the SORT modifier and that sequential + numbers, starting with one, be assigned to the entries in the + context. This permits the RANGE modifier to be used to fetch + portions of the ordered context. + + + + + +Newman & Myers Standards Track [Page 34] + +RFC 2244 ACAP November 1997 + + + NOTIFY requests that the server send untagged ADDTO, + REMOVEFROM, CHANGE, and MODTIME responses while the context + created by this SEARCH command exists. The server MAY issue + untagged ADDTO, REMOVEFROM, CHANGE and MODTIME notifications + for a context at any time between the issuing of the SEARCH + command with MAKECONTEXT NOTIFY and the completion of a + FREECONTEXT command for the context. Notifications are only + issued for changes which occur after the server receives the + SEARCH command which created the context. After issuing a + sequence of ADDTO, REMOVEFROM or CHANGE notifications, the + server MUST issue an untagged MODTIME notification indicating + that the client has all updates to the entries in the context + up to and including the given modtime value. Servers are + permitted a reasonable delay to batch change notifications + before sending them to the client. + + The position arguments of the ADDTO, REMOVEFROM and CHANGE + notifications are 0 if ENUMERATE is not requested. + + NOINHERIT + This causes the SEARCH command to operate without + inheritance. It can be used to tell which values are + explicit overrides. If MAKECONTEXT is also specified, the + created context is also not affected by inheritance. + + RETURN (metadata...) + Specifies what is to be returned in intermediate ENTRY + responses. If this modifier is not specified, no + intermediate ENTRY responses are returned. + + Inside the parentheses is an optional list of attributes, + each optionally followed by a parenthesized list of metadata. + If the parenthesized list of metadata is not specified, it + defaults to "(value)". + + An attribute name with a trailing "*" requests all attributes + with that prefix. A "*" by itself requests all attributes. + If the parenthesized list of metadata is not specified for an + attribute with a trailing "*", it defaults to "(attribute + value)". Results matching such an attribute pattern are + grouped in parentheses. + + Following the last intermediate ENTRY response, the server + returns a single intermediate MODTIME response. + + + + + + + +Newman & Myers Standards Track [Page 35] + +RFC 2244 ACAP November 1997 + + + SORT (attribute comparator ...) + Specifies the order in which any resulting ENTRY replies are + to be returned to the client. The SORT modifier takes as an + argument a parenthesized list of one or more + attribute/comparator pairs. Attribute lists the attribute to + sort on, comparator specifies the name of the collation rule + to apply to the values of the attribute. Successive + attribute/comparator pairs are used to order two entries only + when all preceding pairs indicate the two entries collate the + same. + + If the SORT modifier is used in conjunction with the + MAKECONTEXT modifier, the SORT modifier specifies the + ordering of entries in the created context. + + If no SORT modifier is specified, or none of the + attribute/comparator pairs indicates an order for the two + entries, the server uses the order of the entries that exists + in the context or dataset being searched. + + + Following the modifiers is the search criteria. Searching + criteria consist of one or more search keys. Search keys may be + combined using the AND, and OR search keys. For example, the + criteria (the newline is for readability and not part of the + criteria): + AND COMPARE "modtime" "+i;octet" "19951206103400" + COMPARE "modtime" "-i;octet" "19960112000000" + refers to all entries modified between 10:34 December 6 1995 and + midnight January 12, 1996 UTC. + + The currently defined search keys are as follows. + + ALL This matches all entries. + + AND search-key1 search-key2 + Entries that match both search keys. + + COMPARE attribute comparator value + Entries for which the value of the specified attribute + collates using the specified comparator the same or later + than the specified value. + + COMPARESTRICT attribute comparator value + Entries for which the specified attribute collates using the + specified comparator later than the specified value. + + + + + +Newman & Myers Standards Track [Page 36] + +RFC 2244 ACAP November 1997 + + + EQUAL attribute comparator value + Entries for which the value of the attribute is equal to the + specified value using the specified comparator. + + NOT search-key + Entries that do not match the specified search key. + + OR search-key1 search-key2 + Entries that match either search key. + + PREFIX attribute comparator value + Entries which begin with the specified value using the + specified comparator. If the specified comparator doesn't + support substring matching, a BAD response is returned. + + RANGE start end time + Entries which are within the specified range of the + enumerated context's ordering. The lowest-ordered entry in + the context is assigned number one, the next lowest entry is + assigned number two, and so on. The numeric arguments + specify the lowest and highest numbers to match. The time + specifies that the client has processed notifications for the + context up to the specified time. If the context has been + modified since then, the server MUST either return a NO with + a MODIFIED response code, or return the results that the + SEARCH would have returned if none of the changes since that + time had been made. + + RANGE is only permitted on enumerated contexts. If RANGE is + used with a dataset or non-enumerated context, the server + MUST return a BAD response. + + SUBSTRING attribute comparator value + Entries which contain the specified value, using the + specified comparator. If the specified comparator doesn't + support substring matching, a BAD response is returned. + + +6.4.2. ENTRY Intermediate Response + + Data: entry name + entry data + + The ENTRY intermediate response occurs as a result of a SEARCH or + STORE command. This is the means by which dataset entries are + returned to the client. + + + + + +Newman & Myers Standards Track [Page 37] + +RFC 2244 ACAP November 1997 + + + The ENTRY response begins with the entry name, if a SEARCH command + without the DEPTH modifier was issued, or the entry path in other + cases. This is followed by a set of zero or more items, one for + each metadata item in the RETURN search modifier. Results + matching an attribute pattern or returning multiple metadata items + are grouped in parentheses. + +6.4.3. MODTIME Intermediate Response + + Data: modtime value + + The MODTIME intermediate response occurs as a result of a SEARCH + command. It indicates that the just created context or the + previously returned ENTRY responses include all updates to the + returned entries up to and including the modtime value in the + argument. + +6.4.4. REFER Intermediate Response + + Data: dataset path + relative ACAP URLs + + The REFER intermediate response occurs as a result of a + multi-level SEARCH where one of the levels is located on a + different server. The response indicates the dataset which is not + located on the current server and one or more relative ACAP URLs + for where that dataset may be found. + +6.4.5. Search Examples + + Here are some SEARCH command exchanges between the client and server: + + C: A046 SEARCH "/addressbook/" DEPTH 3 RETURN ("addressbook.Alias" + "addressbook.Email" "addressbook.List") OR NOT EQUAL + "addressbook.Email" "i;octet" NIL NOT EQUAL + "addressbook.List" "i;octet" NIL + S: A046 ENTRY "/addressbook/user/joe/A0345" "fred" + "fred@stone.org" NIL + S: A046 ENTRY "/addressbook/user/fred/A0537" "joe" "joe@stone.org" + NIL + S: A046 ENTRY "/addressbook/group/Dinosaur Operators/A423" + "saurians" NIL "1" + S: A046 MODTIME "19970728105252" + S: A046 OK "SEARCH completed" + + C: A047 SEARCH "/addressbook/user/fred/" RETURN ("*") EQUAL "entry" + "i;octet" "A0345" + S: A047 ENTRY "A0345" (("modtime" "19970728102226") + + + +Newman & Myers Standards Track [Page 38] + +RFC 2244 ACAP November 1997 + + + ("addressbook.Alias" "fred") ("addressbook.Email" + "fred@stone.org") ("addressbook.CommonName" + "Fred Flintstone") ("addressbook.Surname" "Flintstone") + ("addressbook.GivenName" "Fred")) + S: A047 MODTIME "19970728105258" + S: A047 OK "SEARCH completed" + + C: A048 SEARCH "/options/~/vendor.example/" RETURN + ("option.value"("size" "value" "myrights")) + SORT ("entry" "i;octet") COMPARE "modtime" "i;octet" + "19970727123225" + S: A048 ENTRY "blurdybloop" (5 "ghoti" "rwia") + S: A048 ENTRY "buckybits" (2 "10" "rwia") + S: A048 ENTRY "windowSize" (7 "100x100" "rwia") + S: A048 MODTIME "19970728105304" + S: A048 OK "SEARCH completed" + + C: A049 SEARCH "/addressbook/~/public" RETURN ("addressbook.Alias" + "addressbook.Email") MAKECONTEXT ENUMERATE "blob" LIMIT 100 1 + SORT ("addressbook.Alias" "i;octet") NOT EQUAL + "addressbook.Email" NIL + S: A049 ENTRY "A437" "aaguy" "aaguy@stone.org" + S: A049 MODTIME "19970728105308" + S: A049 OK (TOOMANY 347) "Context 'blob' created" + + C: A050 SEARCH "blob" RANGE 2 2 "19970728105308" ALL + S: A050 ENTRY "A238" "abguy" "abguy@stone.org" + S: A050 MODTIME "19970728105310" + S: A050 OK "SEARCH Completed" + +6.5. Contexts + + The following commands use contexts created by a SEARCH command with + a MAKECONTEXT modifier. + + +6.5.1. FREECONTEXT Command + + Arguments: context name + + Data: no specific data for this command + + Result: OK - freecontext completed + NO - freecontext failure: no such context + BAD - command unknown or arguments invalid + + + + + + +Newman & Myers Standards Track [Page 39] + +RFC 2244 ACAP November 1997 + + + The FREECONTEXT command causes the server to free all state + associated with the named context. The context may no longer be + searched and the server will no longer issue any untagged + responses for the context. The context is no longer counted + against the server's limit on the number of contexts. + + Example: C: A683 FREECONTEXT "blurdybloop" + S: A683 OK "Freecontext completed" + + +6.5.2. UPDATECONTEXT Command + + Arguments: list of context names + + Data: untagged responses: ADDTO REMOVEFROM CHANGE MODTIME + + Result: OK - Updatecontext completed: all updates completed + NO - Updatecontext failed: no such context + not a notify context + BAD - command unknown or arguments invalid + + The UPDATECONTEXT command causes the server to ensure that the + client is notified of all changes known to the server for the + contexts listed as arguments up to the current time. The contexts + listed in the arguments must have been previously given to a + successful SEARCH command with a MAKECONTEXT NOTIFY modifier. A + MODTIME untagged response MUST be returned if any read-write + metadata in the context changed since the last MODTIME for that + context. This includes metadata which is not listed in the RETURN + modifier for the context. + + While a server may issue untagged ADDTO, REMOVEFROM, CHANGE, and + MODTIME at any time, the UPDATECONTEXT command is used to "prod" + the server to send any notifications it has not sent yet. + + The UPDATECONTEXT command SHOULD NOT be used to poll for updates. + + Example: C: Z4S9 UPDATECONTEXT "blurdybloop" "blarfl" + S: Z4S9 OK "client has been notified of all changes" + + +6.5.3. ADDTO Untagged Response + + Data: context name + entry name + position + metadata list + + + + +Newman & Myers Standards Track [Page 40] + +RFC 2244 ACAP November 1997 + + + The untagged ADDTO response informs the client that an entry has + been added to a context. The response includes the position + number of the added entry (the first entry in the context is + numbered 1) and those metadata contained in the entry which match + the RETURN statement when the context was created. + + For enumerated contexts, the ADDTO response implicitly adds one to + the position of all members of the context which had position + numbers that were greater than or equal to the ADDTO position + number. For non-enumerated contexts, the position field is always + 0. + + Example: S: * ADDTO "blurdybloop" "fred" 15 + ("addressbook.Email" "fred@stone.org") + + +6.5.4. REMOVEFROM Untagged Response + + Data: context name + entry name + old position + + The untagged REMOVEFROM response informs the client that an entry + has been removed from a context. The response includes the + position number that the removed entry used to have (the first + entry in the context is numbered 1). + + For enumerated contexts, the REMOVEFROM response implicitly + subtracts one from the position numbers of all members of the + context which had position numbers greater than the REMOVEFROM + position number. For non-enumerated contexts, the position field + is always 0. + + Example: S: * REMOVEFROM "blurdybloop" "fred" 15 + + +6.5.5. CHANGE Untagged Response + + Data: context name + entry name + old position + new position + metadata list + + The untagged CHANGE response informs the client that an entry in a + context has either changed position in the context or has changed + the values of one or more of the attributes specified in the + RETURN modifier when the context was created. + + + +Newman & Myers Standards Track [Page 41] + +RFC 2244 ACAP November 1997 + + + The response includes the previous and current position numbers of + the entry (which are 0 if ENUMERATE was not specified on the + context) and the attribute metadata requested in the RETURN + modifier when the context was created. + + For enumerated contexts, the CHANGE response implicitly changes + the position numbers of all entries which had position numbers + between the old and new position. If old position is less than + new position, than one is subtracted from all entries which had + position numbers in that range. Otherwise one is added to all + entries which had position numbers in that range. If the old + position and new position are the same, then no implicit position + renumbering occurs. + + CHANGE responses are not issued for entries which have changed + position implicitly due to another ADDTO, REMOVEFROM or CHANGE + response. + + Example: S: * CHANGE "blurdybloop" "fred" 15 10 + ("addressbook.Email" "fred@stone.org") + + +6.5.6. MODTIME Untagged Response + + Data: context name + modtime value + + The untagged MODTIME response informs the client that it has + received all updates to entries in the context which have modtime + values less than or equal to the modtime value in the argument. + + Example: S: * MODTIME mycontext "19970320162338" + +6.6. Dataset modification + + The following commands and responses handle modification of datasets. + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 42] + +RFC 2244 ACAP November 1997 + + +6.6.1. STORE Command + + Arguments: entry store list + + Data: intermediate responses: ENTRY + + Result: OK - store completed + NO - store failure: can't store that name + UNCHANGEDSINCE specified and entry changed + BAD - command unknown or arguments invalid + invalid UTF-8 syntax in attribute name + + + Creates, modifies, or deletes the named entries in the named + datasets. The values of metadata not specified in the command are + not changed. Setting the "value" metadata of an attribute to NIL + removes that attribute from the entry. Setting the "value" of the + "entry" attribute to NIL removes that entry from the dataset and + cancels inheritance for the entire entry. Setting the "value" of + the "entry" attribute to DEFAULT removes that entry from the + inheriting dataset and reverts the entry and its attributes to + inherited values, if any. Changing the value of the "entry" + attribute renames the entry. + + Storing DEFAULT to the "value" metadata of an attribute is + equivalent to storing NIL, except that inheritance is enabled for + that attribute. If a non-NIL value is inherited then an ENTRY + intermediate response is generated to notify the client of the + this change. The ENTRY response includes the entry-path and the + attribute name and value metadata for each attribute which + reverted to a non-NIL inherited setting. + + Storing NIL to the "value" metadata of an attribute MAY be treated + equivalent to storing DEFAULT to that "value" if there is a NIL + value in the base dataset. + + The STORE command is followed by one or more entry store lists. + Each entry store list begins with an entry path followed by STORE + modifiers, followed by zero or more attribute store items. Each + attribute store item is made up of the attribute name followed by + NIL (to remove the attribute's value), DEFAULT (to revert the item + to any inherited value), a single value (to set the attribute's + single value), or a list of metadata items to modify. The + following STORE modifiers may be specified: + + + + + + + +Newman & Myers Standards Track [Page 43] + +RFC 2244 ACAP November 1997 + + + NOCREATE + By default, the server MUST create any datasets necessary to + store the entry, including multiple hierarchy levels. If + NOCREATE is specified, the STORE command will fail with a + NOEXIST error unless the parent dataset already exists. + + UNCHANGEDSINCE + If the "modtime" of the entry is later than the + unchangedsince time, then the store fails with a MODIFIED + response code. Use of UNCHANGEDSINCE with a time of + "00000101000000" will always fail if the entry exists. + Clients writing to a shared dataset are encouraged to use + UNCHANGEDSINCE when modifying an existing entry. + + + The server MUST either make all the changes specified in a single + STORE command or make none of them. If successful, the server + MUST update the "modtime" attribute for every entry which was + changed. + + It is illegal to list any metadata item within an attribute twice, + any attribute within an entry twice or any entry path twice. The + server MUST return a BAD response if this happens. + + The server MAY re-order the strings in a multi-value on STORE and + MAY remove duplicate strings. However, SEARCH MUST return multi- + values and the associated size list metadata in a consistant + order. + + + Example: C: A342 STORE ("/addressbook/user/fred/ABC547" + "addressbook.TelephoneNumber" "555-1234" + "addressbook.CommonName" "Barney Rubble" + "addressbook.AlternateNames" ("value" + ("Barnacus Rubble" "Coco Puffs Thief")) + "addressbook.Email" NIL) + S: A342 OK "Store completed" + C: A343 STORE ("/addressbook/user/joe/ABD42" + UNCHANGEDSINCE "19970320162338" + "user.joe.hair-length" "10 inches") + S: A343 NO (MODIFIED) "'ABD42' has been changed + by somebody else." + C: A344 STORE ("/addressbook/group/Developers/ACD54" + "entry" NIL) + S: A344 OK "Store completed" + C: A345 STORE ("/option/~/common/SMTPserver" + "option.value" DEFAULT) + S: A345 ENTRY "/option/~/common/SMTPserver" + + + +Newman & Myers Standards Track [Page 44] + +RFC 2244 ACAP November 1997 + + + "option.value" "smtp.server.do.main" + S: A345 OK "Store completed" + C: A347 STORE ("/addressbook/~/" "dataset.inherit" + "/addressbook/group/Developers") + S: A347 OK "Store completed" + + +6.6.2. DELETEDSINCE Command + + Arguments: dataset name + time + + Data: intermediate response: DELETED + + Result: OK - DELETEDSINCE completed + NO - DELETEDSINCE failure: can't read dataset + date too far in the past + BAD - command unknown or arguments invalid + + The DELETEDSINCE command returns in intermediate DELETED replies + the names of entries that have been deleted from the named dataset + since the given time. + + Servers may impose a limit on the number or age of deleted entry + names they keep track of. If the server does not have information + going back to the specified time, the command fails, returning a + TOOOLD response code in the tagged NO response. + + Example: C: Z4S9 DELETEDSINCE "/folder/site/" 19951205103412 + S: Z4S9 DELETED "blurdybloop" + S: Z4S9 DELETED "anteaters" + S: Z4S9 OK "DELETEDSINCE completed" + C: Z4U3 DELETEDSINCE "/folder/site/" 19951009040854 + S: Z4U3 NO (TOOOLD) "Don't have that information" + + +6.6.3. DELETED Intermediate Response + + Data: entry name + + The intermediate DELETED response occurs as a result of a + DELETEDSINCE command. It returns an entry that has been deleted + from the dataset specified in the DELETEDSINCE command. + +6.7. Access Control List Commands + + The commands in this section are used to manage access control lists. + + + + +Newman & Myers Standards Track [Page 45] + +RFC 2244 ACAP November 1997 + + +6.7.1. SETACL Command + + Arguments: acl object + authentication identifier + access rights + + Data: no specific data for this command + + Result: OK - setacl completed + NO - setacl failure: can't set acl + BAD - command unknown or arguments invalid + + The SETACL command changes the access control list on the + specified object so that the specified identifier is granted the + permissions enumerated in rights. If the object did not + previously have an access control list, one is created. + + + Example: C: A123 SETACL ("/addressbook/~/public/") "anyone" "r" + S: A123 OK "Setacl complete" + C: A124 SETACL ("/folder/site/") "B1FF" "rwa" + S: A124 NO (PERMISSION ("/folder/site/")) "'B1FF' not + permitted to modify access rights + for '/folder/site/'" + + + +6.7.2. DELETEACL Command + + Arguments: acl object + optional authentication identifier + + Data: no specific data for this command + + Result: OK - deleteacl completed + NO - deleteacl failure: can't delete acl + BAD - command unknown or arguments invalid + + If given the optional identifier argument, the DELETEACL command + removes any portion of the access control list on the specified + object for the specified identifier. + + If not given the optional identifier argument, the DELETEACL + command removes the ACL from the object entirely, causing access + to be controlled by a higher-level default ACL. This form of the + DELETEACL command is not permitted on the default ACL for a + dataset and servers MUST return a BAD. + + + + +Newman & Myers Standards Track [Page 46] + +RFC 2244 ACAP November 1997 + + + Example: C: A223 DELETEACL ("/addressbook/~/public") "anyone" + S: A223 OK "Deleteacl complete" + C: A224 DELETEACL ("/folder/site") + S: A224 BAD "Can't delete ACL from dataset" + C: A225 DELETEACL ("/addressbook/user/fred" + "addressbook.Email" "barney") + S: A225 OK "Deleteacl complete" + + +6.7.3. MYRIGHTS Command + + Arguments: acl object + + Data: intermediate responses: MYRIGHTS + + Result: OK - myrights completed + NO - myrights failure: can't get rights + BAD - command unknown or arguments invalid + + The MYRIGHTS command returns the set of rights that the client has + to the given dataset or dataset attribute. + + + Example: C: A003 MYRIGHTS ("/folder/site") + S: A003 MYRIGHTS "r" + S: A003 OK "Myrights complete" + + +6.7.4. MYRIGHTS Intermediate Response + + Data: rights + + The MYRIGHTS response occurs as a result of a MYRIGHTS command. + The argument is the set of rights that the client has for the + object referred to in the MYRIGHTS command. + +6.7.5. LISTRIGHTS Command + + Arguments: acl object + authentication identifier + + Data: untagged responses: LISTRIGHTS + + Result: OK - listrights completed + NO - listrights failure: can't get rights list + BAD - command unknown or arguments invalid + + + + + +Newman & Myers Standards Track [Page 47] + +RFC 2244 ACAP November 1997 + + + The LISTRIGHTS command takes an object and an identifier and + returns information about what rights the current user may revoke + or grant to that identifier in the ACL for that object. + + Example: C: a001 LISTRIGHTS ("/folder/~/") "smith" + S: a001 LISTRIGHTS "xra" "w" "i" + S: a001 OK Listrights completed + C: a005 LISTRIGHTS ("/folder/site/archive/imap") "anyone" + S: a005 LISTRIGHTS "" "x" "r" "w" "i" + S: a005 OK Listrights completed + + + +6.7.6. LISTRIGHTS Intermediate Response + + Data: required rights + list of optional rights + + The LISTRIGHTS response occurs as a result of a LISTRIGHTS + command. The first argument is a string containing the (possibly + empty) set of rights the identifier will always be granted on the + dataset or attribute. + + Following this are zero or more strings each containing a single + right which the current user may revoke or grant to the identifier + in the dataset or attribute. + + The same right MUST NOT be listed more than once in the LISTRIGHTS + response. + + +6.8. Quotas + + The section defines the commands and responses relating to quotas. + + +6.8.1. GETQUOTA Command + + Arguments: dataset + + Data: untagged responses: QUOTA + + Result: OK - Quota information returned + NO - Quota failure: can't access resource limit + no resource limit + BAD - command unknown or arguments invalid + + + + + +Newman & Myers Standards Track [Page 48] + +RFC 2244 ACAP November 1997 + + + The GETQUOTA command takes the name of a dataset, and returns in + an untagged QUOTA response the name of the dataset, quota limit in + bytes that applies to that dataset and the quota usage within that + limit. The scope of a quota limit is implementation dependent. + + Example: C: A043 GETQUOTA "/option/user/fred/common" + S: * QUOTA "/option/user/fred/common" 1048576 2475 + S: A043 OK "Getquota completed" + + +6.8.3. QUOTA Untagged Response + + Data: dataset + quota limit in bytes + amount of quota limit used + extension data + + The QUOTA untagged response is generated as a result of a GETQUOTA + command or MAY be generated by the server in response to a SEARCH + or STORE command to warn about high usage of a quota. It includes + the name of the applicable dataset, the quota limit in bytes, the + quota usage and some optional extension data. Clients MUST + tolerate the extension data as its use is reserved for a future + extension. + +6.9. Extensions + + In order to simplify the process of extending the protocol, clients + MUST tolerate unknown server responses which meet the syntax of + response-extend. In addition, clients MUST tolerate unknown server + response codes which meet the syntax of resp-code-ext. Availability + of new commands MUST be announced via a capability on the initial + greeting line and such commands SHOULD meet the syntax of + command-extend. + + Servers MUST respond to unknown commands with a BAD command + completion result. Servers MUST skip over non-synchronizing literals + contained in an unknown command. This may be done by assuming the + unknown command matches the command-extend syntax, or by reading a + line at a time and checking for the non-synchronizing literal syntax + at the end of the line. + +7. Registration Procedures + + ACAP's usefulness comes from providing a structured storage model for + all sorts of configuration data. However, for its potential to be + achieved, it is important that the Internet community strives for the + following goals: + + + +Newman & Myers Standards Track [Page 49] + +RFC 2244 ACAP November 1997 + + + (1) Standardization. It is very important to standardize dataset + classes. The authors hope that ACAP achieves the success that SNMP + has seen with the definition of numerous standards track MIBs. + + (2) Community Review. In the absence of standardization, it is + important to get community review on a proposal to improve its + engineering quality. Community review is strongly recommended prior + to registration. The ACAP implementors mailing list + should be used for this purpose. + + (3) Registration. Registration serves a two-fold purpose. First it + prevents use of the same name for different purposes, and second it + provides a one-stop list which can be used to locate existing + extensions or dataset classes to prevent duplicate work. + + The following registration templates may be used to register ACAP + protocol elements with the Internet Assigned Numbers Authority + (IANA). + +7.1. ACAP Capabilities + + New ACAP capabilities MUST be registered prior to use. Careful + consideration should be made before extending the protocol, as it can + lead to complexity or interoperability problems. Review of proposals + on the acap implementors mailing list is strongly encouraged prior to + registration. + + To: iana@iana.org + Subject: Registration of ACAP capability + + Capability name: + + Capability keyword: + + Capability arguments: + + Published Specification(s): + + (Optional, but strongly encouraged) + + Person and email address to contact for further information: + +7.2. ACAP Response Codes + + ACAP response codes are registered on a first come, first served + basis. Review of proposals on the acap implementors mailing list is + strongly encouraged prior to registration. + + + + +Newman & Myers Standards Track [Page 50] + +RFC 2244 ACAP November 1997 + + + To: iana@iana.org + Subject: Registration of ACAP response code + + Response Code: + + Arguments (use ABNF to specify syntax): + + Purpose: + + Published Specification(s): + + (Optional, but strongly encouraged) + + Person and email address to contact for further information: + +7.3. Dataset Classes + + A dataset class provides a core set of attributes for use in a + specified hierarchy. It may also define rules for the dataset + hierarchy underneath that class. Dataset class specifications must + be standards track or IESG approved experimental RFCs. + + To: iana@iana.org + Subject: Registration of ACAP dataset class + + Dataset class name/attribute prefix: + + Purpose: + + Published Specification(s): + + (Standards track or IESG approved experimental RFC) + + Person and email address to contact for further information: + +7.4. Vendor Subtree + + Vendors may reserve a portion of the ACAP namespace for private use. + Dataset class names beginning with "vendor.." + are reserved for use by that company or product. In addition, all + attribute names beginning with "vendor.." are + reserved for use by that company or product once registered. + Registration is on a first come, first served basis. Whenever + possible, private attributes and dataset classes should be avoided in + favor of improving interoperable dataset class definitions. + + + + + + +Newman & Myers Standards Track [Page 51] + +RFC 2244 ACAP November 1997 + + + To: iana@iana.org + Subject: Registration of ACAP vendor subtree + + Private Prefix: vendor.. + + Person and email address to contact for further information: + + (company names and addresses should be included when appropriate) + +8. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [ABNF]. This uses the ABNF core + rules as specified in Appendix A of the ABNF specification [ABNF]. + + Except as noted otherwise, all alphabetic characters are + case-insensitive. The use of upper or lower case characters to + define token strings is for editorial clarity only. Implementations + MUST accept these strings in a case-insensitive fashion. + + The "initial-greeting" rule below defines the initial ACAP greeting + from the server. The "command" rule below defines the syntax for + commands sent by the client. The "response" rule below defines the + syntax for responses sent by the server. + + ATOM-CHAR = "!" / %x23-27 / %x2A-5B / %x5D-7A / %x7C-7E + ;; Any CHAR except ATOM-SPECIALS + + ATOM-SPECIALS = "(" / ")" / "{" / SP / CTL / QUOTED-SPECIALS + + CHAR = %x01-7F + + DIGIT-NZ = %x31-39 + ; non-zero digits ("1" - "9") + + QUOTED-CHAR = SAFE-UTF8-CHAR / "\" QUOTED-SPECIALS + + QUOTED-SPECIALS = <"> / "\" + + SAFE-CHAR = %x01-09 / %x0B-0C / %x0E-21 / + %x23-5B / %x5D-7F + ;; any TEXT-CHAR except QUOTED-SPECIALS + + SAFE-UTF8-CHAR = SAFE-CHAR / UTF8-2 / UTF8-3 / UTF8-4 / + UTF8-5 / UTF8-6 + + TAG-CHAR = %x21 / %x23-27 / %x2C-5B / %x5D-7A / %x7C-7E + ;; Any ATOM-CHAR except "*" or "+" + + + +Newman & Myers Standards Track [Page 52] + +RFC 2244 ACAP November 1997 + + + TEXT-CHAR = %x01-09 / %x0B-0C / %x0E-7F + ;; any CHAR except CR and LF + + TEXT-UTF8-CHAR = SAFE-UTF8-CHAR / QUOTED-SPECIALS + + UTF8-1 = %x80-BF + + UTF8-2 = %xC0-DF UTF8-1 + + UTF8-3 = %xE0-EF 2UTF8-1 + + UTF8-4 = %xF0-F7 3UTF8-1 + + UTF8-5 = %xF8-FB 4UTF8-1 + + UTF8-6 = %xFC-FD 5UTF8-1 + + UTF8-CHAR = TEXT-UTF8-CHAR / CR / LF + + acl = "(" [acl-identrights *(SP acl-identrights)] ")" + *(SPACE acl-identrights)] ")" + + acl-identifier = string-utf8 + ;; MUST NOT contain HTAB + + acl-identrights = string-utf8 + ;; The identifier followed by a HTAB, + ;; followed by the rights. + + acl-delobject = "(" dataset SP attribute [SP entry-name] ")" + + acl-object = "(" dataset [SP attribute [SP entry-name]] ")" + + acl-rights = quoted + + atom = ALPHA *1023ATOM-CHAR + + attribute = string-utf8 + ;; dot-separated attribute name + ;; MUST NOT contain "*" or "%" + + attribute-store = attribute SP (value-nildef / + "(" 1*(metadata-write-q SP value-store) ")") + ;; MUST NOT include the same metadata twice + + auth-type = <"> auth-type-name <"> + + + + + +Newman & Myers Standards Track [Page 53] + +RFC 2244 ACAP November 1997 + + + auth-type-name = iana-token + ;; as defined in SASL [SASL] + + command = tag SP (command-any / command-auth / + command-nonauth) CRLF + ;; Modal based on state + + command-authent = "AUTHENTICATE" SP auth-type + [SP string] *(CRLF string) + + command-any = "NOOP" / command-lang / "LOGOUT" / + command-extend + + command-auth = command-delacl / command-dsince / + command-freectx / command-getquota / + command-lrights / command-myrights / + command-search / command-setacl / + command-store + ;; only valid in authenticated state + + command-delacl = "DELETEACL" SP acl-delobject [SP acl-identifier] + + command-dsince = "DELETEDSINCE" SP dataset SP time + + command-extend = extend-token [SP extension-data] + + command-freectx = "FREECONTEXT" SP context + + command-getquota = "GETQUOTA" SP dataset + + command-lang = "LANG" *(SP lang-tag) + + command-lrights = "LISTRIGHTS" SP acl-object + + command-myrights = "MYRIGHTS" SP acl-object + + command-nonauth = command-authent + ;; only valid in non-authenticated state + + command-search = "SEARCH" SP (dataset / context) + *(SP search-modifier) SP search-criteria + ;; MUST NOT include same search-modifier twice + + command-setacl = "SETACL" SP acl-object SP acl-identifier + SP acl-rights + + command-store = "STORE" SP store-entry-list + + + + +Newman & Myers Standards Track [Page 54] + +RFC 2244 ACAP November 1997 + + + comparator = <"> comparator-name <"> + + comparator-name = ["+" / "-"] iana-token + + context = string-utf8 + ;; MUST NOT begin with slash ("/") + + dataset = string-utf8 + ;; slash-separated dataset name + ;; begins with slash + + entry = entry-name / entry-path + + entry-name = string-utf8 + ;; entry name MUST NOT contain slash + ;; MUST NOT begin with "." + + entry-path = string-utf8 + ;; slash-separated path to entry + ;; begins with slash + + entry-relative = string-utf8 + ;; potentially relative path to entry + + extend-token = atom + ;; MUST be defined by a standards track or + ;; IESG approved experimental protocol extension + + extension-data = extension-item *(SP extension-item) + + extension-item = extend-token / string / number / + "(" [extension-data] ")" + + iana-token = atom + ;; MUST be registered with IANA + + initial-greeting = "*" SP "ACAP" *(SP "(" init-capability ")") CRLF + + init-capability = init-cap-context / init-cap-extend / + init-cap-implem / init-cap-sasl + + init-cap-context = "CONTEXTLIMIT" SP string + + init-cap-extend = iana-token [SP string-list] + + init-cap-implem = "IMPLEMENTATION" SP string + + init-cap-sasl = "SASL" SP string-list + + + +Newman & Myers Standards Track [Page 55] + +RFC 2244 ACAP November 1997 + + + lang-tag = <"> Language-Tag <"> + ;; Language-Tag rule is defined in [LANG-TAGS] + + literal = "{" number [ "+" ] "}" CRLF *OCTET + ;; The number represents the number of octets + ;; MUST be literal-utf8 except for values + + literal-utf8 = "{" number [ "+" ] "}" CRLF *UTF8-CHAR + ;; The number represents the number of octets + ;; not the number of characters + + metadata = attribute [ "(" metadata-type-list ")" ] + ;; attribute MAY end in "*" as wildcard. + + metadata-list = metadata *(SP metadata) + + metadata-type = "attribute" / "myrights" / "size" / + "count" / metadata-write + + metadata-type-q = <"> metadata-type <"> + + metadata-type-list = metadata-type-q *(SP metadata-type-q) + + metadata-write = "value" / "acl" + + metadata-write-q = <"> metadata-write <"> + + nil = "NIL" + + number = *DIGIT + ;; A 32-bit unsigned number. + ;; (0 <= n < 4,294,967,296) + + nz-number = DIGIT-NZ *DIGIT + ;; A 32-bit unsigned non-zero number. + ;; (0 < n < 4,294,967,296) + + position = number + ;; "0" if context is not enumerated + ;; otherwise this is non-zero + + quota-limit = number + + quota-usage = number + + quoted = <"> *QUOTED-CHAR <"> + ;; limited to 1024 octets between the <">s + + + + +Newman & Myers Standards Track [Page 56] + +RFC 2244 ACAP November 1997 + + + response = response-addto / response-alert / response-bye / + response-change / response-cont / + response-deleted / response-done / + response-entry / response-extend / + response-listr / response-lang / + response-mtimei / response-mtimeu / + response-myright / response-quota / + response-refer / response-remove / response-stat + + response-addto = "*" SP "ADDTO" SP context SP entry-name + SP position SP return-data-list + + response-alert = "*" SP "ALERT" SP resp-body CRLF + ;; Client MUST display alert text to user + + response-bye = "*" SP "BYE" SP resp-body CRLF + ;; Server will disconnect condition + + response-change = "*" SP "CHANGE" SP context SP entry-name + SP position SP position SP return-data-list + + response-cont = "+" SP string + + response-deleted = tag SP "DELETED" SP entry-name + + response-done = tag SP resp-cond-state CRLF + + response-entry = tag SP "ENTRY" SP entry SP return-data-list + + response-extend = (tag / "*") SP extend-token [SP extension-data] + + response-lang = "*" SP "LANG" SP lang-tag 1*(SP comparator) + + response-listr = tag SP "LISTRIGHTS" SP acl-rights + *(SP acl-rights) + + response-mtimei = tag SP "MODTIME" SP time + + response-mtimeu = "*" SP "MODTIME" SP context SP time + + response-myright = tag SP "MYRIGHTS" SP acl-rights + + response-quota = "*" SP "QUOTA" SP dataset SP quota-limit + SP quota-usage [SP extension-data] + + response-refer = tag SP "REFER" SP dataset + 1*(SP <"> url-relative <">) + + + + +Newman & Myers Standards Track [Page 57] + +RFC 2244 ACAP November 1997 + + + response-remove = "*" SP "REMOVEFROM" SP context SP + entry-name SP position + + response-stat = "*" SP resp-cond-state CRLF + + resp-body = ["(" resp-code ")" SP] quoted + + resp-code = "AUTH-TOO-WEAK" / "ENCRYPT-NEEDED" / + resp-code-inval / resp-code-mod / + resp-code-noexist / resp-code-perm / "QUOTA" / + resp-code-refer / resp-code-sasl / + resp-code-toomany / "TOOOLD" / + "TRANSITION-NEEDED" / "TRYFREECONTEXT" / + "TRYLATER" / "WAYTOOMANY" / resp-code-ext + + resp-code-ext = iana-token [SP extension-data] + ;; unknown codes MUST be tolerated by the client + + resp-code-inval = "INVALID" 1*(SP entry-path SP attribute) + + resp-code-mod = "MODIFIED" SP entry-path + + resp-code-noexist = "NOEXIST" SP dataset + + resp-code-perm = "PERMISSION" SP acl-object + + resp-code-refer = "REFER" 1*(SP <"> url-relative <">) + + resp-code-sasl = "SASL" SP string + + resp-code-toomany = "TOOMANY" SP nz-number + + resp-cond-state = ("OK" / "NO" / "BAD") SP resp-body + ;; Status condition + + return-attr-list = "(" return-metalist *(SP return-metalist) ")" + ;; occurs when "*" in RETURN pattern on SEARCH + + return-data = return-metadata / return-metalist / + return-attr-list + + return-data-list = return-data *(SP return-data) + + return-metalist = "(" return-metadata *(SP return-metadata) ")" + ;; occurs when multiple metadata items requested + + return-metadata = nil / string / value-list / acl + + + + +Newman & Myers Standards Track [Page 58] + +RFC 2244 ACAP November 1997 + + + searchkey-equal = "EQUAL" SP attribute SP comparator SP value-nil + + searchkey-comp = "COMPARE" SP attribute SP comparator SP value + + searchkey-prefix = "PREFIX" SP attribute SP comparator SP value + + searchkey-range = "RANGE" SP nz-number SP nz-number SP time + + searchkey-strict = "COMPARESTRICT" SP attribute SP comparator + SP value + + searchkey-substr = "SUBSTRING" SP attribute SP comparator SP value + + searchmod-depth = "DEPTH" SP number + + searchmod-hard = "HARDLIMIT" SP nz-number + + searchmod-limit = "LIMIT" SP number SP number + + searchmod-make = "MAKECONTEXT" [SP "ENUMERATE"] + [SP "NOTIFY"] SP context + + searchmod-ninh = "NOINHERIT" + + searchmod-return = "RETURN" SP "(" [metadata-list] ")" + + searchmod-sort = "SORT" SP "(" sort-list ")" + + search-criteria = "ALL" / searchkey-equal / searchkey-comp / + searchkey-strict / searchkey-range / + searchkey-prefix / searchkey-substr / + "NOT" SP search-criteria / + "OR" SP search-criteria SP search-criteria / + "AND" SP search-criteria SP search-criteria + + search-modifier = searchmod-depth / searchmod-hard / + searchmod-limit / searchmod-make / + searchmod-ninh / searchmod-return / + searchmod-sort + + sort-list = sort-item *(SP sort-item) + + sort-item = attribute SP comparator + + store-entry = "(" entry-path *(SP store-modifier) + *(SP attribute-store) ")" + ;; MUST NOT include same store-modifier twice + ;; MUST NOT include same attribute twice + + + +Newman & Myers Standards Track [Page 59] + +RFC 2244 ACAP November 1997 + + + store-entry-list = store-entry *(SP store-entry) + ;; MUST NOT include same entry twice + + store-modifier = store-mod-unchang / store-mod-nocreate + + store-mod-nocreate = "NOCREATE" + + store-mod-unchang = "UNCHANGEDSINCE" SP time + + string = quoted / literal + + string-list = string *(SP string) + + string-utf8 = quoted / literal-utf8 + + tag = 1*32TAG-CHAR + + time = <"> time-year time-month time-day time-hour + time-minute time-second time-subsecond <"> + ;; Timestamp in UTC + + time-day = 2DIGIT ;; 01-31 + + time-hour = 2DIGIT ;; 00-23 + + time-minute = 2DIGIT ;; 00-59 + + time-month = 2DIGIT ;; 01-12 + + time-second = 2DIGIT ;; 00-60 + + time-subsecond = *DIGIT + + time-year = 4DIGIT + + value = string + + value-list = "(" [value *(SP value)] ")" + + value-nil = value / nil + + value-nildef = value-nil / "DEFAULT" + + value-store = value-nildef / value-list / acl + + url-acap = "acap://" url-server "/" url-enc-entry + [url-filter] [url-extension] + ;; url-enc-entry interpreted relative to "/" + + + +Newman & Myers Standards Track [Page 60] + +RFC 2244 ACAP November 1997 + + + url-attr-list = url-enc-attr *("&" url-enc-attr) + + url-auth = ";AUTH=" ("*" / url-enc-auth) + + url-achar = uchar / "&" / "=" / "~" + ;; See RFC 1738 for definition of "uchar" + + url-char = uchar / "=" / "~" / ":" / "@" / "/" + ;; See RFC 1738 for definition of "uchar" + + url-enc-attr = 1*url-char + ;; encoded version of attribute name + + url-enc-auth = 1*url-achar + ;; encoded version of auth-type-name above + + url-enc-entry = 1*url-char + ;; encoded version of entry-relative above + + url-enc-user = *url-achar + ;; encoded version of login userid + + url-extension = *("?" 1*url-char) + + url-filter = "?" url-attr-list + + url-relative = url-acap / [url-enc-entry] [url-filter] + ;; url-enc-entry is relative to base URL + + url-server = [url-enc-user [url-auth] "@"] hostport + ;; See RFC 1738 for definition of "hostport" + +9. Multi-lingual Considerations + + The IAB charset workshop [IAB-CHARSET] came to a number of + conclusions which influenced the design of ACAP. The decision to use + UTF-8 as the character encoding scheme was based on that work. The + LANG command to negotiate a language for error messages is also + included. + + Section 3.4.5 of the IAB charset workshop report states that there + should be a way to identify the natural language for human readable + strings. Several promising proposals have been made for use within + ACAP, but no clear consensus on a single method is apparent at this + stage. The following rules are likely to permit the addition of + multi-lingual support in the future: + + + + + +Newman & Myers Standards Track [Page 61] + +RFC 2244 ACAP November 1997 + + + (1) A work in progress called Multi-Lingual String Format (MLSF) + proposes a layer on top of UTF-8 which uses otherwise illegal UTF-8 + sequences to store language tags. In order to permit its addition to + a future version of this standard, client-side UTF-8 interpreters + MUST be able to silently ignore illegal multi-byte UTF-8 characters, + and treat illegal single-byte UTF-8 characters as end of string + markers. Servers, for the time being, MUST be able to silently + accept illegal UTF-8 characters, except in attribute names and entry + names. Clients MUST NOT send illegal UTF-8 characters to the server + unless a future standard changes this rule. + + (2) There is a proposal to add language tags to Unicode. To support + this, servers MUST be able to store UTF-8 characters of up to 20 bits + of data. + + (3) The metadata item "language" is reserved for future use. + +10. Security Considerations + + The AUTHENTICATE command uses SASL [SASL] to provide basic + authentication, authorization, integrity and privacy services. This + is described in section 6.3.1. + + When the CRAM-MD5 mechanism is used, the security considerations for + the CRAM-MD5 SASL mechanism [CRAM-MD5] apply. The CRAM-MD5 mechanism + is also susceptible to passive dictionary attacks. This means that + if an authentication session is recorded by a passive observer, that + observer can try common passwords through the CRAM-MD5 mechanism and + see if the results match. This attack is reduced by using hard to + guess passwords. Sites are encouraged to educate users and have the + password change service test candidate passwords against a + dictionary. ACAP implementations of CRAM-MD5 SHOULD permit passwords + of at least 64 characters in length. + + ACAP protocol transactions are susceptible to passive observers or + man in the middle attacks which alter the data, unless the optional + encryption and integrity services of the AUTHENTICATE command are + enabled, or an external security mechanism is used for protection. + It may be useful to allow configuration of both clients and servers + to refuse to transfer sensitive information in the absence of strong + encryption. + + ACAP access control lists provide fine grained authorization for + access to attributes. A number of related security issues are + described in section 3.5. + + ACAP URLs have the same security considerations as IMAP URLs + [IMAP-URL]. + + + +Newman & Myers Standards Track [Page 62] + +RFC 2244 ACAP November 1997 + + + ACAP clients are encouraged to consider the security problems + involved with a lab computer situation. Specifically, a client cache + of ACAP configuration information MUST NOT allow access by an + unauthorized user. One way to assure this is for an ACAP client to + be able to completely flush any non-public cached configuration data + when a user leaves. + + As laptop computers can be easily stolen and a cache of configuration + data may contain sensitive information, a disconnected mode ACAP + client may wish to encrypt and password protect cached configuration + information. + +11. Acknowledgments + + Many thanks to the follow people who have contributed to ACAP over + the past four years: Wallace Colyer, Mark Crispin, Jack DeWinter, Rob + Earhart, Ned Freed, Randy Gellens, Terry Gray, J. S. Greenfield, + Steve Dorner, Steve Hole, Steve Hubert, Dave Roberts, Bart Schaefer, + Matt Wall and other participants of the IETF ACAP working group. + +12. Authors' Addresses + + Chris Newman + Innosoft International, Inc. + 1050 Lakes Drive + West Covina, CA 91790 USA + + Email: chris.newman@innosoft.com + + + John Gardiner Myers + Netscape Communications + 501 East Middlefield Road + Mail Stop MV-029 + Mountain View, CA 94043 + + Email: jgmyers@netscape.com + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 63] + +RFC 2244 ACAP November 1997 + + +Appendices + +A. References + + [ABNF] Crocker, Overell, "Augmented BNF for Syntax Specifications: + ABNF", RFC 2234, Internet Mail Consortium, Demon Internet Ltd, + November 1997. + + + + [BASIC-URL] Berners-Lee, Masinter, McCahill, "Uniform Resource + Locators (URL)", RFC 1738, CERN, Xerox Coproration, University of + Minnesota, December 1994. + + + + [CHARSET-LANG-POLICY] Alvestrand, "IETF Policy on Character Sets and + Languages", work in progress. + + [CRAM-MD5] Klensin, Catoe, Krumviede, "IMAP/POP AUTHorize Extension + for Simple Challenge/Response", RFC 2195, MCI, September 1997. + + + + [IAB-CHARSET] Weider, Preston, Simonsen, Alvestrand, Atkinson, + Crispin, Svanberg, "The Report of the IAB Character Set Workshop held + 29 February - 1 March, 1996", RFC 2130, April 1997. + + + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, University of Washington, December 1996. + + + + [IMAP-ACL] Myers, J., "IMAP4 ACL extension", RFC 2086, Carnegie + Mellon, January 1997. + + + + [IMAP-URL] Newman, "IMAP URL Scheme", RFC 2192, Innosoft, July 1997. + + + + [ISO-10646] ISO/IEC 10646-1:1993(E) "Information Technology-- + Universal Multiple-octet Coded Character Set (UCS)." See also + amendments 1 through 7, plus editorial corrections. + + + + +Newman & Myers Standards Track [Page 64] + +RFC 2244 ACAP November 1997 + + + [ISO-C] "Programming languages -- C", ISO/IEC 9899:1990, + International Organization for Standardization. This is effectively + the same as ANSI C standard X3.159-1989. + + [KEYWORDS] Bradner, "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, Harvard University, March 1997. + + + + [LANG-TAGS] Alvestrand, H., "Tags for the Identification of + Languages", RFC 1766. + + + + [REL-URL] Fielding, "Relative Uniform Resource Locators", RFC 1808, + UC Irvine, June 1995. + + + + [SASL] Myers, J., "Simple Authentication and Security Layer (SASL)", + RFC 2222, Netscape Communications, October 1997. + + + + [SASL-ANON] Newman, C., "Anonymous SASL Mechanism", RFC 2245, + November 1997. + + [UNICODE-2] The Unicode Consortium, "The Unicode Standard, Version + 2.0", Addison-Wesley, 1996. ISBN 0-201-48345-9. + + [US-ASCII] "USA Standard Code for Information Interchange," X3.4. + American National Standards Institute: New York (1968). + + [UTF8] Yergeau, F. "UTF-8, a transformation format of Unicode and ISO + 10646", RFC 2044, Alis Technologies, October 1996. + + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 65] + +RFC 2244 ACAP November 1997 + + +B. ACAP Keyword Index + + + ACAP (untagged response) ................................... 26 + ADDTO (untagged response) .................................. 40 + ALERT (untagged response) .................................. 31 + ALL (search keyword) ....................................... 36 + AND (search keyword) ....................................... 36 + AUTH-TOO-WEAK (response code) .............................. 19 + AUTHENTICATE (command) ..................................... 31 + BAD (response) ............................................. 30 + BYE (untagged response) .................................... 30 + CHANGE (untagged response) ................................. 41 + COMPARE (search keyword) ................................... 36 + COMPARESTRICT (search keyword) ............................. 36 + CONTEXTLIMIT (ACAP capability) ............................. 27 + DELETEACL (command) ........................................ 46 + DELETED (intermediate response) ............................ 45 + DELETEDSINCE (command) ..................................... 45 + DEPTH (search modifier) .................................... 34 + ENCRYPT-NEEDED (response code) ............................. 19 + ENTRY (intermediate response) .............................. 37 + EQUAL (search keyword) ..................................... 37 + FREECONTEXT (command) ...................................... 39 + GETQUOTA (command) ......................................... 48 + HARDLIMIT (search modifier) ................................ 34 + IMPLEMENTATION (ACAP capability) ........................... 27 + INVALID (response code) .................................... 19 + LANG (command) ............................................. 28 + LANG (intermediate response) ............................... 28 + LIMIT (search modifier) .................................... 34 + LISTRIGHTS (command) ....................................... 47 + LISTRIGHTS (intermediate response) ......................... 48 + LOGOUT (command) ........................................... 29 + MAKECONTEXT (search modifier) .............................. 34 + MODIFIED (response code) ................................... 19 + MODTIME (intermediate response) ............................ 38 + MODTIME (untagged response) ................................ 42 + MYRIGHTS (command) ......................................... 47 + MYRIGHTS (intermediate response) ........................... 47 + NO (response) .............................................. 29 + NOCREATE (store modifier) .................................. 44 + NOEXIST (response code) .................................... 19 + NOINHERIT (search modifier) ................................ 35 + NOOP (command) ............................................. 27 + NOT (search keyword) ....................................... 37 + OK (response) .............................................. 29 + OR (search keyword) ........................................ 37 + PERMISSION (response code) ................................. 19 + + + +Newman & Myers Standards Track [Page 66] + +RFC 2244 ACAP November 1997 + + + + PREFIX (search keyword) .................................... 37 + QUOTA (response code) ...................................... 19 + QUOTA (untagged response) .................................. 49 + RANGE (search keyword) ..................................... 37 + REFER (intermediate response) .............................. 38 + REFER (response code) ...................................... 19 + REMOVEFROM (untagged response) ............................. 41 + RETURN (search modifier) ................................... 35 + SASL (ACAP capability) ..................................... 27 + SASL (response code) ....................................... 20 + SEARCH (command) ........................................... 33 + SETACL (command) ........................................... 46 + SORT (search modifier) ..................................... 36 + STORE (command) ............................................ 42 + SUBSTRING (search keyword) ................................. 37 + TOOMANY (response code) .................................... 20 + TOOOLD (response code) ..................................... 20 + TRANSITION-NEEDED (response code) .......................... 20 + TRYFREECONTEXT (response code) ............................. 20 + TRYLATER (response code) ................................... 20 + UNCHANGEDSINCE (store modifier) ............................ 44 + UPDATECONTEXT (command) .................................... 40 + WAYTOOMANY (response code) ................................. 20 + acl (attribute metadata) ................................... 12 + anyone (ACL identifier) .................................... 17 + attribute (attribute metadata) ............................. 12 + dataset.acl (dataset attribute) ............................ 24 + dataset.acl. (dataset attribute) ................ 24 + dataset.inherit (dataset attribute) ........................ 24 + entry (predefined attribute) ............................... 11 + i;ascii-casemap (comparator) ............................... 16 + i;ascii-numeric (comparator) ............................... 16 + i;octet (comparator) ....................................... 16 + modtime (predefined attribute) ............................. 11 + myrights (attribute metadata) .............................. 12 + size (attribute metadata) .................................. 13 + subdataset (predefined attribute) .......................... 11 + value (attribute metadata) ................................. 13 + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 67] + +RFC 2244 ACAP November 1997 + + +C. Full Copyright Statement + + Copyright (C) The Internet Society 1997. All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implmentation may be prepared, copied, published and + distributed, in whole or in part, without restriction of any kind, + provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of developing + Internet standards in which case the procedures for copyrights defined + in the Internet Standards process must be followed, or as required to + translate it into languages other than English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT + NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN + WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + + + + + + + + + + + + + + + + + + + + + + + + + +Newman & Myers Standards Track [Page 68] diff --git a/docs/rfcs/rfc2342.IMAP4_Namespace.txt b/docs/rfcs/rfc2342.IMAP4_Namespace.txt new file mode 100644 index 0000000..0926646 --- /dev/null +++ b/docs/rfcs/rfc2342.IMAP4_Namespace.txt @@ -0,0 +1,563 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 2342 Microsoft +Category: Standards Track C. Newman + Innosoft + May 1998 + + + IMAP4 Namespace + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1998). All Rights Reserved. + +1. Abstract + + IMAP4 [RFC-2060] does not define a default server namespace. As a + result, two common namespace models have evolved: + + The "Personal Mailbox" model, in which the default namespace that is + presented consists of only the user's personal mailboxes. To access + shared mailboxes, the user must use an escape mechanism to reach + another namespace. + + The "Complete Hierarchy" model, in which the default namespace that + is presented includes the user's personal mailboxes along with any + other mailboxes they have access to. + + These two models, create difficulties for certain client operations. + This document defines a NAMESPACE command that allows a client to + discover the prefixes of namespaces used by a server for personal + mailboxes, other users' mailboxes, and shared mailboxes. This allows + a client to avoid much of the manual user configuration that is now + necessary when mixing and matching IMAP4 clients and servers. + +2. Conventions used in this document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If such lines are wrapped without a new "C:" or + "S:" label, then the wrapping is for editorial clarity and is not + part of the command. + + + +Gahrns & Newman Standards Track [Page 1] + +RFC 2342 IMAP4 Namespace May 1998 + + + Personal Namespace: A namespace that the server considers within the + personal scope of the authenticated user on a particular connection. + Typically, only the authenticated user has access to mailboxes in + their Personal Namespace. It is the part of the namespace that + belongs to the user that is allocated for mailboxes. If an INBOX + exists for a user, it MUST appear within the user's personal + namespace. In the typical case, there SHOULD be only one Personal + Namespace on a server. + + Other Users' Namespace: A namespace that consists of mailboxes from + the Personal Namespaces of other users. To access mailboxes in the + Other Users' Namespace, the currently authenticated user MUST be + explicitly granted access rights. For example, it is common for a + manager to grant to their secretary access rights to their mailbox. + In the typical case, there SHOULD be only one Other Users' Namespace + on a server. + + Shared Namespace: A namespace that consists of mailboxes that are + intended to be shared amongst users and do not exist within a user's + Personal Namespace. + + The namespaces a server uses MAY differ on a per-user basis. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + +3. Introduction and Overview + + Clients often attempt to create mailboxes for such purposes as + maintaining a record of sent messages (e.g. "Sent Mail") or + temporarily saving messages being composed (e.g. "Drafts"). For + these clients to inter-operate correctly with the variety of IMAP4 + servers available, the user must enter the prefix of the Personal + Namespace used by the server. Using the NAMESPACE command, a client + is able to automatically discover this prefix without manual user + configuration. + + In addition, users are often required to manually enter the prefixes + of various namespaces in order to view the mailboxes located there. + For example, they might be required to enter the prefix of #shared to + view the shared mailboxes namespace. The NAMESPACE command allows a + client to automatically discover the namespaces that are available on + a server. This allows a client to present the available namespaces to + the user in what ever manner it deems appropriate. For example, a + + + + + + +Gahrns & Newman Standards Track [Page 2] + +RFC 2342 IMAP4 Namespace May 1998 + + + client could choose to initially display only personal mailboxes, or + it may choose to display the complete list of mailboxes available, + and initially position the user at the root of their Personal + Namespace. + + A server MAY choose to make available to the NAMESPACE command only a + subset of the complete set of namespaces the server supports. To + provide the ability to access these namespaces, a client SHOULD allow + the user the ability to manually enter a namespace prefix. + +4. Requirements + + IMAP4 servers that support this extension MUST list the keyword + NAMESPACE in their CAPABILITY response. + + The NAMESPACE command is valid in the Authenticated and Selected + state. + +5. NAMESPACE Command + + Arguments: none + + Response: an untagged NAMESPACE response that contains the prefix + and hierarchy delimiter to the server's Personal + Namespace(s), Other Users' Namespace(s), and Shared + Namespace(s) that the server wishes to expose. The + response will contain a NIL for any namespace class + that is not available. Namespace_Response_Extensions + MAY be included in the response. + Namespace_Response_Extensions which are not on the IETF + standards track, MUST be prefixed with an "X-". + + Result: OK - Command completed + NO - Error: Can't complete command + BAD - argument invalid + + Example 5.1: + =========== + + < A server that supports a single personal namespace. No leading + prefix is used on personal mailboxes and "/" is the hierarchy + delimiter.> + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) NIL NIL + S: A001 OK NAMESPACE command completed + + + + + +Gahrns & Newman Standards Track [Page 3] + +RFC 2342 IMAP4 Namespace May 1998 + + + Example 5.2: + =========== + + < A user logged on anonymously to a server. No personal mailboxes + are associated with the anonymous user and the user does not have + access to the Other Users' Namespace. No prefix is required to + access shared mailboxes and the hierarchy delimiter is "." > + + C: A001 NAMESPACE + S: * NAMESPACE NIL NIL (("" ".")) + S: A001 OK NAMESPACE command completed + + Example 5.3: + =========== + + < A server that contains a Personal Namespace and a single Shared + Namespace. > + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) NIL (("Public Folders/" "/")) + S: A001 OK NAMESPACE command completed + + Example 5.4: + =========== + + < A server that contains a Personal Namespace, Other Users' + Namespace and multiple Shared Namespaces. Note that the hierarchy + delimiter used within each namespace can be different. > + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) (("~" "/")) (("#shared/" "/") + ("#public/" "/")("#ftp/" "/")("#news." ".")) + S: A001 OK NAMESPACE command completed + + The prefix string allows a client to do things such as automatically + creating personal mailboxes or LISTing all available mailboxes within + a namespace. + + Example 5.5: + =========== + + < A server that supports only the Personal Namespace, with a + leading prefix of INBOX to personal mailboxes and a hierarchy + delimiter of "."> + + C: A001 NAMESPACE + S: * NAMESPACE (("INBOX." ".")) NIL NIL + S: A001 OK NAMESPACE command completed + + + +Gahrns & Newman Standards Track [Page 4] + +RFC 2342 IMAP4 Namespace May 1998 + + + < Automatically create a mailbox to store sent items.> + + C: A002 CREATE "INBOX.Sent Mail" + S: A002 OK CREATE command completed + + Although typically a server will support only a single Personal + Namespace, and a single Other User's Namespace, circumstances exist + where there MAY be multiples of these, and a client MUST be prepared + for them. If a client is configured such that it is required to + create a certain mailbox, there can be circumstances where it is + unclear which Personal Namespaces it should create the mailbox in. + In these situations a client SHOULD let the user select which + namespaces to create the mailbox in. + + Example 5.6: + =========== + + < In this example, a server supports 2 Personal Namespaces. In + addition to the regular Personal Namespace, the user has an + additional personal namespace to allow access to mailboxes in an + MH format mailstore. > + + < The client is configured to save a copy of all mail sent by the + user into a mailbox called 'Sent Mail'. Furthermore, after a + message is deleted from a mailbox, the client is configured to + move that message to a mailbox called 'Deleted Items'.> + + < Note that this example demonstrates how some extension flags can + be passed to further describe the #mh namespace. > + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")("#mh/" "/" "X-PARAM" ("FLAG1" "FLAG2"))) + NIL NIL + S: A001 OK NAMESPACE command completed + + < It is desired to keep only one copy of sent mail. It is unclear + which Personal Namespace the client should use to create the 'Sent + Mail' mailbox. The user is prompted to select a namespace and + only one 'Sent Mail' mailbox is created. > + + C: A002 CREATE "Sent Mail" + S: A002 OK CREATE command completed + + < The client is designed so that it keeps two 'Deleted Items' + mailboxes, one for each namespace. > + + C: A003 CREATE "Delete Items" + S: A003 OK CREATE command completed + + + +Gahrns & Newman Standards Track [Page 5] + +RFC 2342 IMAP4 Namespace May 1998 + + + C: A004 CREATE "#mh/Deleted Items" + S: A004 OK CREATE command completed + + The next level of hierarchy following the Other Users' Namespace + prefix SHOULD consist of , where is a user name + as per the IMAP4 LOGIN or AUTHENTICATE command. + + A client can construct a LIST command by appending a "%" to the Other + Users' Namespace prefix to discover the Personal Namespaces of other + users that are available to the currently authenticated user. + + In response to such a LIST command, a server SHOULD NOT return user + names that have not granted access to their personal mailboxes to the + user in question. + + A server MAY return a LIST response containing only the names of + users that have explicitly granted access to the user in question. + + Alternatively, a server MAY return NO to such a LIST command, + requiring that a user name be included with the Other Users' + Namespace prefix before listing any other user's mailboxes. + + Example 5.7: + =========== + + < A server that supports providing a list of other user's + mailboxes that are accessible to the currently logged on user. > + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) (("Other Users/" "/")) NIL + S: A001 OK NAMESPACE command completed + + C: A002 LIST "" "Other Users/%" + S: * LIST () "/" "Other Users/Mike" + S: * LIST () "/" "Other Users/Karen" + S: * LIST () "/" "Other Users/Matthew" + S: * LIST () "/" "Other Users/Tesa" + S: A002 OK LIST command completed + + Example 5.8: + =========== + + < A server that does not support providing a list of other user's + mailboxes that are accessible to the currently logged on user. + The mailboxes are listable if the client includes the name of the + other user with the Other Users' Namespace prefix. > + + + + + +Gahrns & Newman Standards Track [Page 6] + +RFC 2342 IMAP4 Namespace May 1998 + + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) (("#Users/" "/")) NIL + S: A001 OK NAMESPACE command completed + + < In this example, the currently logged on user has access to the + Personal Namespace of user Mike, but the server chose to suppress + this information in the LIST response. However, by appending the + user name Mike (received through user input) to the Other Users' + Namespace prefix, the client is able to get a listing of the + personal mailboxes of user Mike. > + + C: A002 LIST "" "#Users/%" + S: A002 NO The requested item could not be found. + + C: A003 LIST "" "#Users/Mike/%" + S: * LIST () "/" "#Users/Mike/INBOX" + S: * LIST () "/" "#Users/Mike/Foo" + S: A003 OK LIST command completed. + + A prefix string might not contain a hierarchy delimiter, because + in some cases it is not needed as part of the prefix. + + Example 5.9: + =========== + + < A server that allows access to the Other Users' Namespace by + prefixing the others' mailboxes with a '~' followed by , + where is a user name as per the IMAP4 LOGIN or + AUTHENTICATE command.> + + C: A001 NAMESPACE + S: * NAMESPACE (("" "/")) (("~" "/")) NIL + S: A001 OK NAMESPACE command completed + + < List the mailboxes for user mark > + + C: A002 LIST "" "~mark/%" + S: * LIST () "/" "~mark/INBOX" + S: * LIST () "/" "~mark/foo" + S: A002 OK LIST command completed + + Historical convention has been to start all namespaces with the "#" + character. Namespaces that include the "#" character are not IMAP + URL [IMAP-URL] friendly requiring the "#" character to be represented + as %23 when within URLs. As such, server implementers MAY instead + consider using namespace prefixes that do not contain the "#" + character. + + + + +Gahrns & Newman Standards Track [Page 7] + +RFC 2342 IMAP4 Namespace May 1998 + + +6. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. + + atom = + ; as defined in [RFC-2060] + + Namespace = nil / "(" 1*( "(" string SP (<"> QUOTED_CHAR <"> / + nil) *(Namespace_Response_Extension) ")" ) ")" + + Namespace_Command = "NAMESPACE" + + Namespace_Response_Extension = SP string SP "(" string *(SP string) + ")" + + Namespace_Response = "*" SP "NAMESPACE" SP Namespace SP Namespace SP + Namespace + + ; The first Namespace is the Personal Namespace(s) + ; The second Namespace is the Other Users' Namespace(s) + ; The third Namespace is the Shared Namespace(s) + + nil = + ; as defined in [RFC-2060] + + QUOTED_CHAR = + ; as defined in [RFC-2060] + + string = + ; as defined in [RFC-2060] + ; Note that the namespace prefix is to a mailbox and following + ; IMAP4 convention, any international string in the NAMESPACE + ; response MUST be of modified UTF-7 format as described in + ; [RFC-2060]. + +7. Security Considerations + + In response to a LIST command containing an argument of the Other + Users' Namespace prefix, a server SHOULD NOT list users that have not + granted list access to their personal mailboxes to the currently + authenticated user. Providing such a list, could compromise security + by potentially disclosing confidential information of who is located + on the server, or providing a starting point of a list of user + accounts to attack. + + + + + + +Gahrns & Newman Standards Track [Page 8] + +RFC 2342 IMAP4 Namespace May 1998 + + +8. References + + [RFC-2060], Crispin, M., "Internet Message Access Protocol Version + 4rev1", RFC 2060, December 1996. + + [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [ABNF] Crocker, D., Editor, and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [IMAP-URL], Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. + +9. Acknowledgments + + Many people have participated in the discussion of IMAP namespaces on + the IMAP mailing list. In particular, the authors would like to + thank Mark Crispin for many of the concepts relating to the Personal + Namespace and accessing the Personal Namespace of other users, Steve + Hole for summarizing the two namespace models, John Myers and Jack De + Winter for their work in a preceding effort trying to define a + standardized personal namespace, and Larry Osterman for his review + and collaboration on this document. + +11. Authors' Addresses + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98072, USA + + Phone: (425) 936-9833 + EMail: mikega@microsoft.com + + + Chris Newman + Innosoft International, Inc. + 1050 East Garvey Ave. South + West Covina, CA, 91790, USA + + EMail: chris.newman@innosoft.com + + + + + + + + + + +Gahrns & Newman Standards Track [Page 9] + +RFC 2342 IMAP4 Namespace May 1998 + + +12. Full Copyright Statement + + Copyright (C) The Internet Society (1998). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + + + + + + + + + + + + + + + + + + + + + + + + +Gahrns & Newman Standards Track [Page 10] + diff --git a/docs/rfcs/rfc2359.IMAP4_UIDPLUS_extension.txt b/docs/rfcs/rfc2359.IMAP4_UIDPLUS_extension.txt new file mode 100644 index 0000000..460ac4b --- /dev/null +++ b/docs/rfcs/rfc2359.IMAP4_UIDPLUS_extension.txt @@ -0,0 +1,339 @@ + + + + + + +Network Working Group J. Myers +Request for Comments: 2359 Netscape Communications +Category: Standards Track June 1998 + + + IMAP4 UIDPLUS extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1998). All Rights Reserved. + +IESG NOTE + + The IMAP extension described here assumes a particular means of using + IMAP to support disconnected operation. However, this means of + supporting disconnected operation is not yet documented. Also, there + are multiple theories about how best to do disconnected operation in + IMAP, and as yet, there is no consensus on which one should be + adopted as a standard. + + This document is being approved as a Proposed Standard because it + does not appear to have technical flaws in itelf. However, approval + of this document as a Proposed Standard should not be considered an + IETF endorsement of any particular means of doing disconnected + operation in IMAP. + +Table of Contents + + 1. Abstract .............................................. 2 + 2. Conventions Used in this Document ..................... 2 + 3. Introduction and Overview ............................. 2 + 4. Features .............................................. 2 + 4.1. UID EXPUNGE Command ................................... 2 + 4.2. APPENDUID response code ............................... 3 + 4.3. COPYUID response code ................................. 4 + 5. Formal Syntax ......................................... 4 + 6. References ............................................ 4 + 7. Security Considerations ............................... 5 + 8. Author's Address ...................................... 5 + 9. Full Copyright Statement .............................. 6 + + + +Myers Standards Track [Page 1] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + +1. Abstract + + The UIDPLUS extension of the Internet Message Access Protocol [IMAP4] + provides a set of features intended to reduce the amount of time and + resources used by some client operations. The features in UIDPLUS + are primarily intended for disconnected-use clients. + +2. Conventions Used in this Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in "Key words for + use in RFCs to Indicate Requirement Levels" [KEYWORDS]. + +3. Introduction and Overview + + The UIDPLUS extension is present in any IMAP4 server implementation + which returns "UIDPLUS" as one of the supported capabilities to the + CAPABILITY command. The UIDPLUS extension contains one additional + command and additional data returned with successful APPEND and COPY + commands. + + Clients that wish to use the new command in UIDPLUS must of course + first test for the presence of the extension by issuing a CAPABILITY + command. Each of the features in UIDPLUS are optimizations; clients + can provide the same functionality, albeit more slowly, by using + commands in the base protocol. With each feature, this document + recommends a fallback approach to take when the UIDPLUS extension is + not supported by the server. + +4. Features + +4.1. UID EXPUNGE Command + + Arguments: message set + + Data: untagged responses: EXPUNGE + + Result: OK - expunge completed + NO - expunge failure (e.g. permission denied) + BAD - command unknown or arguments invalid + + + + + + + + +Myers Standards Track [Page 2] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + + The UID EXPUNGE command permanently removes from the currently + selected mailbox all messages that both have the \Deleted flag set + and have a UID that is included in the specified message set. If + a message either does not have the \Deleted flag set or is has a + UID that is not included in the specified message set, it is not + affected. + + This command may be used to ensure that a replayed EXPUNGE command + does not remove any messages that have been marked as \Deleted + between the time that the user requested the expunge operation and + the time the server processes the command. + + If the server does not support the UIDPLUS capability, the client + should fall back to using the STORE command to temporarily remove + the \Deleted flag from messages it does not want to remove. The + client could alternatively fall back to using the EXPUNGE command, + risking the unintended removal of some messages. + + Example: C: A003 UID EXPUNGE 3000:3002 + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: A003 OK UID EXPUNGE completed + +4.2. APPENDUID response code + + Successful APPEND commands return an APPENDUID response code in the + tagged OK response. The APPENDUID response code contains as + arguments the UIDVALIDITY of the destination mailbox and the UID + assigned to the appended message. + + If the server does not support the UIDPLUS capability, the client can + only discover this information by selecting the destination mailbox + and issuing FETCH commands. + + Example: C: A003 APPEND saved-messages (\Seen) {310} + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 38505 3955] APPEND completed + + + + +Myers Standards Track [Page 3] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + +4.3. COPYUID response code + + Successful COPY and UID COPY commands return a COPYUID response code + in the tagged OK response whenever at least one message was copied. + The COPYUID response code contains as an argument the UIDVALIDITY of + the appended-to mailbox, a message set containing the UIDs of the + messages copied to the destination mailbox, in the order they were + copied, and a message containing the UIDs assigned to the copied + messages, in the order they were assigned. Neither of the message + sets may contain extraneous UIDs or the symbol '*'. + + If the server does not support the UIDPLUS capability, the client can + only discover this information by selecting the destination mailbox + and issuing FETCH commands. + + Example: C: A003 COPY 2:4 MEETING + S: A003 OK [COPYUID 38505 304,319:320 3956:3958] Done + C: A003 UID COPY 305:310 MEETING + S: A003 OK Done + +5. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + resp_code_apnd ::= "APPENDUID" SPACE nz_number SPACE uniqueid + + resp_code_copy ::= "COPYUID" SPACE nz_number SPACE set SPACE set + + uid_expunge ::= "UID" SPACE "EXPUNGE" SPACE set + +6. References + + [IMAP4] Crispin, M., "Internet Message Access Protocol - + Version 4rev1", RFC 2060, December 1996. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet + Text Messages", STD 11, RFC 822, August 1982. + + + +Myers Standards Track [Page 4] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + +7. Security Considerations + + There are no known security issues with this extension. + +8. Author's Address + + John Gardiner Myers + Netscape Communications + 501 East Middlefield Road + Mail Stop MV-029 + Mountain View, CA 94043 + + EMail: jgmyers@netscape.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Myers Standards Track [Page 5] + +RFC 2359 IMAP4 UIDPLUS extension June 1998 + + +9. Full Copyright Statement + + Copyright (C) The Internet Society (1998). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + + + + + + + + + + + + + + + + + + + + + + + + +Myers Standards Track [Page 6] + diff --git a/docs/rfcs/rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt b/docs/rfcs/rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt new file mode 100644 index 0000000..66897cd --- /dev/null +++ b/docs/rfcs/rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt @@ -0,0 +1,843 @@ + + + + + + +Network Working Group C. Newman +Request for Comments: 2595 Innosoft +Category: Standards Track June 1999 + + + Using TLS with IMAP, POP3 and ACAP + + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1999). All Rights Reserved. + +1. Motivation + + The TLS protocol (formerly known as SSL) provides a way to secure an + application protocol from tampering and eavesdropping. The option of + using such security is desirable for IMAP, POP and ACAP due to common + connection eavesdropping and hijacking attacks [AUTH]. Although + advanced SASL authentication mechanisms can provide a lightweight + version of this service, TLS is complimentary to simple + authentication-only SASL mechanisms or deployed clear-text password + login commands. + + Many sites have a high investment in authentication infrastructure + (e.g., a large database of a one-way-function applied to user + passwords), so a privacy layer which is not tightly bound to user + authentication can protect against network eavesdropping attacks + without requiring a new authentication infrastructure and/or forcing + all users to change their password. Recognizing that such sites will + desire simple password authentication in combination with TLS + encryption, this specification defines the PLAIN SASL mechanism for + use with protocols which lack a simple password authentication + command such as ACAP and SMTP. (Note there is a separate RFC for the + STARTTLS command in SMTP [SMTPTLS].) + + There is a strong desire in the IETF to eliminate the transmission of + clear-text passwords over unencrypted channels. While SASL can be + used for this purpose, TLS provides an additional tool with different + deployability characteristics. A server supporting both TLS with + + + + +Newman Standards Track [Page 1] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + simple passwords and a challenge/response SASL mechanism is likely to + interoperate with a wide variety of clients without resorting to + unencrypted clear-text passwords. + + The STARTTLS command rectifies a number of the problems with using a + separate port for a "secure" protocol variant. Some of these are + mentioned in section 7. + +1.1. Conventions Used in this Document + + The key words "REQUIRED", "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", + "MAY", and "OPTIONAL" in this document are to be interpreted as + described in "Key words for use in RFCs to Indicate Requirement + Levels" [KEYWORDS]. + + Terms related to authentication are defined in "On Internet + Authentication" [AUTH]. + + Formal syntax is defined using ABNF [ABNF]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + +2. Basic Interoperability and Security Requirements + + The following requirements apply to all implementations of the + STARTTLS extension for IMAP, POP3 and ACAP. + +2.1. Cipher Suite Requirements + + Implementation of the TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA [TLS] cipher + suite is REQUIRED. This is important as it assures that any two + compliant implementations can be configured to interoperate. + + All other cipher suites are OPTIONAL. + +2.2. Privacy Operational Mode Security Requirements + + Both clients and servers SHOULD have a privacy operational mode which + refuses authentication unless successful activation of an encryption + layer (such as that provided by TLS) occurs prior to or at the time + of authentication and which will terminate the connection if that + encryption layer is deactivated. Implementations are encouraged to + have flexability with respect to the minimal encryption strength or + cipher suites permitted. A minimalist approach to this + recommendation would be an operational mode where the + TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA cipher suite is mandatory prior to + permitting authentication. + + + +Newman Standards Track [Page 2] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + Clients MAY have an operational mode which uses encryption only when + it is advertised by the server, but authentication continues + regardless. For backwards compatibility, servers SHOULD have an + operational mode where only the authentication mechanisms required by + the relevant base protocol specification are needed to successfully + authenticate. + +2.3. Clear-Text Password Requirements + + Clients and servers which implement STARTTLS MUST be configurable to + refuse all clear-text login commands or mechanisms (including both + standards-track and nonstandard mechanisms) unless an encryption + layer of adequate strength is active. Servers which allow + unencrypted clear-text logins SHOULD be configurable to refuse + clear-text logins both for the entire server, and on a per-user + basis. + +2.4. Server Identity Check + + During the TLS negotiation, the client MUST check its understanding + of the server hostname against the server's identity as presented in + the server Certificate message, in order to prevent man-in-the-middle + attacks. Matching is performed according to these rules: + + - The client MUST use the server hostname it used to open the + connection as the value to compare against the server name as + expressed in the server certificate. The client MUST NOT use any + form of the server hostname derived from an insecure remote source + (e.g., insecure DNS lookup). CNAME canonicalization is not done. + + - If a subjectAltName extension of type dNSName is present in the + certificate, it SHOULD be used as the source of the server's + identity. + + - Matching is case-insensitive. + + - A "*" wildcard character MAY be used as the left-most name + component in the certificate. For example, *.example.com would + match a.example.com, foo.example.com, etc. but would not match + example.com. + + - If the certificate contains multiple names (e.g. more than one + dNSName field), then a match with any one of the fields is + considered acceptable. + + If the match fails, the client SHOULD either ask for explicit user + confirmation, or terminate the connection and indicate the server's + identity is suspect. + + + +Newman Standards Track [Page 3] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +2.5. TLS Security Policy Check + + Both the client and server MUST check the result of the STARTTLS + command and subsequent TLS negotiation to see whether acceptable + authentication or privacy was achieved. Ignoring this step + completely invalidates using TLS for security. The decision about + whether acceptable authentication or privacy was achieved is made + locally, is implementation-dependent, and is beyond the scope of this + document. + +3. IMAP STARTTLS extension + + When the TLS extension is present in IMAP, "STARTTLS" is listed as a + capability in response to the CAPABILITY command. This extension + adds a single command, "STARTTLS" to the IMAP protocol which is used + to begin a TLS negotiation. + +3.1. STARTTLS Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - begin TLS negotiation + BAD - command unknown or arguments invalid + + A TLS negotiation begins immediately after the CRLF at the end of + the tagged OK response from the server. Once a client issues a + STARTTLS command, it MUST NOT issue further commands until a + server response is seen and the TLS negotiation is complete. + + The STARTTLS command is only valid in non-authenticated state. + The server remains in non-authenticated state, even if client + credentials are supplied during the TLS negotiation. The SASL + [SASL] EXTERNAL mechanism MAY be used to authenticate once TLS + client credentials are successfully exchanged, but servers + supporting the STARTTLS command are not required to support the + EXTERNAL mechanism. + + Once TLS has been started, the client MUST discard cached + information about server capabilities and SHOULD re-issue the + CAPABILITY command. This is necessary to protect against + man-in-the-middle attacks which alter the capabilities list prior + to STARTTLS. The server MAY advertise different capabilities + after STARTTLS. + + The formal syntax for IMAP is amended as follows: + + + + +Newman Standards Track [Page 4] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + command_any =/ "STARTTLS" + + Example: C: a001 CAPABILITY + S: * CAPABILITY IMAP4rev1 STARTTLS LOGINDISABLED + S: a001 OK CAPABILITY completed + C: a002 STARTTLS + S: a002 OK Begin TLS negotiation now + + C: a003 CAPABILITY + S: * CAPABILITY IMAP4rev1 AUTH=EXTERNAL + S: a003 OK CAPABILITY completed + C: a004 LOGIN joe password + S: a004 OK LOGIN completed + +3.2. IMAP LOGINDISABLED capability + + The current IMAP protocol specification (RFC 2060) requires the + implementation of the LOGIN command which uses clear-text passwords. + Many sites may choose to disable this command unless encryption is + active for security reasons. An IMAP server MAY advertise that the + LOGIN command is disabled by including the LOGINDISABLED capability + in the capability response. Such a server will respond with a tagged + "NO" response to any attempt to use the LOGIN command. + + An IMAP server which implements STARTTLS MUST implement support for + the LOGINDISABLED capability on unencrypted connections. + + An IMAP client which complies with this specification MUST NOT issue + the LOGIN command if this capability is present. + + This capability is useful to prevent clients compliant with this + specification from sending an unencrypted password in an environment + subject to passive attacks. It has no impact on an environment + subject to active attacks as a man-in-the-middle attacker can remove + this capability. Therefore this does not relieve clients of the need + to follow the privacy mode recommendation in section 2.2. + + Servers advertising this capability will fail to interoperate with + many existing compliant IMAP clients and will be unable to prevent + those clients from disclosing the user's password. + +4. POP3 STARTTLS extension + + The POP3 STARTTLS extension adds the STLS command to POP3 servers. + If this is implemented, the POP3 extension mechanism [POP3EXT] MUST + also be implemented to avoid the need for client probing of multiple + commands. The capability name "STLS" indicates this command is + present and permitted in the current state. + + + +Newman Standards Track [Page 5] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + STLS + + Arguments: none + + Restrictions: + Only permitted in AUTHORIZATION state. + + Discussion: + A TLS negotiation begins immediately after the CRLF at the + end of the +OK response from the server. A -ERR response + MAY result if a security layer is already active. Once a + client issues a STLS command, it MUST NOT issue further + commands until a server response is seen and the TLS + negotiation is complete. + + The STLS command is only permitted in AUTHORIZATION state + and the server remains in AUTHORIZATION state, even if + client credentials are supplied during the TLS negotiation. + The AUTH command [POP-AUTH] with the EXTERNAL mechanism + [SASL] MAY be used to authenticate once TLS client + credentials are successfully exchanged, but servers + supporting the STLS command are not required to support the + EXTERNAL mechanism. + + Once TLS has been started, the client MUST discard cached + information about server capabilities and SHOULD re-issue + the CAPA command. This is necessary to protect against + man-in-the-middle attacks which alter the capabilities list + prior to STLS. The server MAY advertise different + capabilities after STLS. + + Possible Responses: + +OK -ERR + + Examples: + C: STLS + S: +OK Begin TLS negotiation + + ... + C: STLS + S: -ERR Command not permitted when TLS active + + + + + + + + + + +Newman Standards Track [Page 6] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +5. ACAP STARTTLS extension + + When the TLS extension is present in ACAP, "STARTTLS" is listed as a + capability in the ACAP greeting. No arguments to this capability are + defined at this time. This extension adds a single command, + "STARTTLS" to the ACAP protocol which is used to begin a TLS + negotiation. + +5.1. STARTTLS Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - begin TLS negotiation + BAD - command unknown or arguments invalid + + A TLS negotiation begins immediately after the CRLF at the end of + the tagged OK response from the server. Once a client issues a + STARTTLS command, it MUST NOT issue further commands until a + server response is seen and the TLS negotiation is complete. + + The STARTTLS command is only valid in non-authenticated state. + The server remains in non-authenticated state, even if client + credentials are supplied during the TLS negotiation. The SASL + [SASL] EXTERNAL mechanism MAY be used to authenticate once TLS + client credentials are successfully exchanged, but servers + supporting the STARTTLS command are not required to support the + EXTERNAL mechanism. + + After the TLS layer is established, the server MUST re-issue an + untagged ACAP greeting. This is necessary to protect against + man-in-the-middle attacks which alter the capabilities list prior + to STARTTLS. The client MUST discard cached capability + information and replace it with the information from the new ACAP + greeting. The server MAY advertise different capabilities after + STARTTLS. + + The formal syntax for ACAP is amended as follows: + + command_any =/ "STARTTLS" + + Example: S: * ACAP (SASL "CRAM-MD5") (STARTTLS) + C: a002 STARTTLS + S: a002 OK "Begin TLS negotiation now" + + S: * ACAP (SASL "CRAM-MD5" "PLAIN" "EXTERNAL") + + + + +Newman Standards Track [Page 7] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +6. PLAIN SASL mechanism + + Clear-text passwords are simple, interoperate with almost all + existing operating system authentication databases, and are useful + for a smooth transition to a more secure password-based + authentication mechanism. The drawback is that they are unacceptable + for use over an unencrypted network connection. + + This defines the "PLAIN" SASL mechanism for use with ACAP and other + protocols with no clear-text login command. The PLAIN SASL mechanism + MUST NOT be advertised or used unless a strong encryption layer (such + as the provided by TLS) is active or backwards compatibility dictates + otherwise. + + The mechanism consists of a single message from the client to the + server. The client sends the authorization identity (identity to + login as), followed by a US-ASCII NUL character, followed by the + authentication identity (identity whose password will be used), + followed by a US-ASCII NUL character, followed by the clear-text + password. The client may leave the authorization identity empty to + indicate that it is the same as the authentication identity. + + The server will verify the authentication identity and password with + the system authentication database and verify that the authentication + credentials permit the client to login as the authorization identity. + If both steps succeed, the user is logged in. + + The server MAY also use the password to initialize any new + authentication database, such as one suitable for CRAM-MD5 + [CRAM-MD5]. + + Non-US-ASCII characters are permitted as long as they are represented + in UTF-8 [UTF-8]. Use of non-visible characters or characters which + a user may be unable to enter on some keyboards is discouraged. + + The formal grammar for the client message using Augmented BNF [ABNF] + follows. + + message = [authorize-id] NUL authenticate-id NUL password + authenticate-id = 1*UTF8-SAFE ; MUST accept up to 255 octets + authorize-id = 1*UTF8-SAFE ; MUST accept up to 255 octets + password = 1*UTF8-SAFE ; MUST accept up to 255 octets + NUL = %x00 + UTF8-SAFE = %x01-09 / %x0B-0C / %x0E-7F / UTF8-2 / + UTF8-3 / UTF8-4 / UTF8-5 / UTF8-6 + UTF8-1 = %x80-BF + UTF8-2 = %xC0-DF UTF8-1 + UTF8-3 = %xE0-EF 2UTF8-1 + + + +Newman Standards Track [Page 8] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + UTF8-4 = %xF0-F7 3UTF8-1 + UTF8-5 = %xF8-FB 4UTF8-1 + UTF8-6 = %xFC-FD 5UTF8-1 + + Here is an example of how this might be used to initialize a CRAM-MD5 + authentication database for ACAP: + + Example: S: * ACAP (SASL "CRAM-MD5") (STARTTLS) + C: a001 AUTHENTICATE "CRAM-MD5" + S: + "<1896.697170952@postoffice.reston.mci.net>" + C: "tim b913a602c7eda7a495b4e6e7334d3890" + S: a001 NO (TRANSITION-NEEDED) + "Please change your password, or use TLS to login" + C: a002 STARTTLS + S: a002 OK "Begin TLS negotiation now" + + S: * ACAP (SASL "CRAM-MD5" "PLAIN" "EXTERNAL") + C: a003 AUTHENTICATE "PLAIN" {21+} + C: timtanstaaftanstaaf + S: a003 OK CRAM-MD5 password initialized + + Note: In this example, represents a single ASCII NUL octet. + +7. imaps and pop3s ports + + Separate "imaps" and "pop3s" ports were registered for use with SSL. + Use of these ports is discouraged in favor of the STARTTLS or STLS + commands. + + A number of problems have been observed with separate ports for + "secure" variants of protocols. This is an attempt to enumerate some + of those problems. + + - Separate ports lead to a separate URL scheme which intrudes into + the user interface in inappropriate ways. For example, many web + pages use language like "click here if your browser supports SSL." + This is a decision the browser is often more capable of making than + the user. + + - Separate ports imply a model of either "secure" or "not secure." + This can be misleading in a number of ways. First, the "secure" + port may not in fact be acceptably secure as an export-crippled + cipher suite might be in use. This can mislead users into a false + sense of security. Second, the normal port might in fact be + secured by using a SASL mechanism which includes a security layer. + Thus the separate port distinction makes the complex topic of + security policy even more confusing. One common result of this + confusion is that firewall administrators are often misled into + + + +Newman Standards Track [Page 9] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + permitting the "secure" port and blocking the standard port. This + could be a poor choice given the common use of SSL with a 40-bit + key encryption layer and plain-text password authentication is less + secure than strong SASL mechanisms such as GSSAPI with Kerberos 5. + + - Use of separate ports for SSL has caused clients to implement only + two security policies: use SSL or don't use SSL. The desirable + security policy "use TLS when available" would be cumbersome with + the separate port model, but is simple with STARTTLS. + + - Port numbers are a limited resource. While they are not yet in + short supply, it is unwise to set a precedent that could double (or + worse) the speed of their consumption. + + +8. IANA Considerations + + This constitutes registration of the "STARTTLS" and "LOGINDISABLED" + IMAP capabilities as required by section 7.2.1 of RFC 2060 [IMAP]. + + The registration for the POP3 "STLS" capability follows: + + CAPA tag: STLS + Arguments: none + Added commands: STLS + Standard commands affected: May enable USER/PASS as a side-effect. + CAPA command SHOULD be re-issued after successful completion. + Announced states/Valid states: AUTHORIZATION state only. + Specification reference: this memo + + The registration for the ACAP "STARTTLS" capability follows: + + Capability name: STARTTLS + Capability keyword: STARTTLS + Capability arguments: none + Published Specification(s): this memo + Person and email address for further information: + see author's address section below + + The registration for the PLAIN SASL mechanism follows: + + SASL mechanism name: PLAIN + Security Considerations: See section 9 of this memo + Published specification: this memo + Person & email address to contact for further information: + see author's address section below + Intended usage: COMMON + Author/Change controller: see author's address section below + + + +Newman Standards Track [Page 10] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +9. Security Considerations + + TLS only provides protection for data sent over a network connection. + Messages transferred over IMAP or POP3 are still available to server + administrators and usually subject to eavesdropping, tampering and + forgery when transmitted through SMTP or NNTP. TLS is no substitute + for an end-to-end message security mechanism using MIME security + multiparts [MIME-SEC]. + + A man-in-the-middle attacker can remove STARTTLS from the capability + list or generate a failure response to the STARTTLS command. In + order to detect such an attack, clients SHOULD warn the user when + session privacy is not active and/or be configurable to refuse to + proceed without an acceptable level of security. + + A man-in-the-middle attacker can always cause a down-negotiation to + the weakest authentication mechanism or cipher suite available. For + this reason, implementations SHOULD be configurable to refuse weak + mechanisms or cipher suites. + + Any protocol interactions prior to the TLS handshake are performed in + the clear and can be modified by a man-in-the-middle attacker. For + this reason, clients MUST discard cached information about server + capabilities advertised prior to the start of the TLS handshake. + + Clients are encouraged to clearly indicate when the level of + encryption active is known to be vulnerable to attack using modern + hardware (such as encryption keys with 56 bits of entropy or less). + + The LOGINDISABLED IMAP capability (discussed in section 3.2) only + reduces the potential for passive attacks, it provides no protection + against active attacks. The responsibility remains with the client + to avoid sending a password over a vulnerable channel. + + The PLAIN mechanism relies on the TLS encryption layer for security. + When used without TLS, it is vulnerable to a common network + eavesdropping attack. Therefore PLAIN MUST NOT be advertised or used + unless a suitable TLS encryption layer is active or backwards + compatibility dictates otherwise. + + When the PLAIN mechanism is used, the server gains the ability to + impersonate the user to all services with the same password + regardless of any encryption provided by TLS or other network privacy + mechanisms. While many other authentication mechanisms have similar + weaknesses, stronger SASL mechanisms such as Kerberos address this + issue. Clients are encouraged to have an operational mode where all + mechanisms which are likely to reveal the user's password to the + server are disabled. + + + +Newman Standards Track [Page 11] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + The security considerations for TLS apply to STARTTLS and the + security considerations for SASL apply to the PLAIN mechanism. + Additional security requirements are discussed in section 2. + +10. References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [AUTH] Haller, N. and R. Atkinson, "On Internet Authentication", + RFC 1704, October 1994. + + [CRAM-MD5] Klensin, J., Catoe, R. and P. Krumviede, "IMAP/POP + AUTHorize Extension for Simple Challenge/Response", RFC + 2195, September 1997. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MIME-SEC] Galvin, J., Murphy, S., Crocker, S. and N. Freed, + "Security Multiparts for MIME: Multipart/Signed and + Multipart/Encrypted", RFC 1847, October 1995. + + [POP3] Myers, J. and M. Rose, "Post Office Protocol - Version 3", + STD 53, RFC 1939, May 1996. + + [POP3EXT] Gellens, R., Newman, C. and L. Lundblade, "POP3 Extension + Mechanism", RFC 2449, November 1998. + + [POP-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, + December 1994. + + [SASL] Myers, J., "Simple Authentication and Security Layer + (SASL)", RFC 2222, October 1997. + + [SMTPTLS] Hoffman, P., "SMTP Service Extension for Secure SMTP over + TLS", RFC 2487, January 1999. + + [TLS] Dierks, T. and C. Allen, "The TLS Protocol Version 1.0", + RFC 2246, January 1999. + + + + + +Newman Standards Track [Page 12] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", RFC 2279, January 1998. + + +11. Author's Address + + Chris Newman + Innosoft International, Inc. + 1050 Lakes Drive + West Covina, CA 91790 USA + + EMail: chris.newman@innosoft.com + + +A. Appendix -- Compliance Checklist + + An implementation is not compliant if it fails to satisfy one or more + of the MUST requirements for the protocols it implements. An + implementation that satisfies all the MUST and all the SHOULD + requirements for its protocols is said to be "unconditionally + compliant"; one that satisfies all the MUST requirements but not all + the SHOULD requirements for its protocols is said to be + "conditionally compliant". + + Rules Section + ----- ------- + Mandatory-to-implement Cipher Suite 2.1 + SHOULD have mode where encryption required 2.2 + server SHOULD have mode where TLS not required 2.2 + MUST be configurable to refuse all clear-text login + commands or mechanisms 2.3 + server SHOULD be configurable to refuse clear-text + login commands on entire server and on per-user basis 2.3 + client MUST check server identity 2.4 + client MUST use hostname used to open connection 2.4 + client MUST NOT use hostname from insecure remote lookup 2.4 + client SHOULD support subjectAltName of dNSName type 2.4 + client SHOULD ask for confirmation or terminate on fail 2.4 + MUST check result of STARTTLS for acceptable privacy 2.5 + client MUST NOT issue commands after STARTTLS + until server response and negotiation done 3.1,4,5.1 + client MUST discard cached information 3.1,4,5.1,9 + client SHOULD re-issue CAPABILITY/CAPA command 3.1,4 + IMAP server with STARTTLS MUST implement LOGINDISABLED 3.2 + IMAP client MUST NOT issue LOGIN if LOGINDISABLED 3.2 + POP server MUST implement POP3 extensions 4 + ACAP server MUST re-issue ACAP greeting 5.1 + + + + +Newman Standards Track [Page 13] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + + client SHOULD warn when session privacy not active and/or + refuse to proceed without acceptable security level 9 + SHOULD be configurable to refuse weak mechanisms or + cipher suites 9 + + The PLAIN mechanism is an optional part of this specification. + However if it is implemented the following rules apply: + + Rules Section + ----- ------- + MUST NOT use PLAIN unless strong encryption active + or backwards compatibility dictates otherwise 6,9 + MUST use UTF-8 encoding for characters in PLAIN 6 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Newman Standards Track [Page 14] + +RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 + + +Full Copyright Statement + + Copyright (C) The Internet Society (1999). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Newman Standards Track [Page 15] + diff --git a/docs/rfcs/rfc2683.IMAP4_Implementation_recommendations.txt b/docs/rfcs/rfc2683.IMAP4_Implementation_recommendations.txt new file mode 100644 index 0000000..d92e340 --- /dev/null +++ b/docs/rfcs/rfc2683.IMAP4_Implementation_recommendations.txt @@ -0,0 +1,1291 @@ + + + + + + +Network Working Group B. Leiba +Request for Comments: 2683 IBM T.J. Watson Research Center +Category: Informational September 1999 + + + IMAP4 Implementation Recommendations + +Status of this Memo + + This memo provides information for the Internet community. It does + not specify an Internet standard of any kind. Distribution of this + memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (1999). All Rights Reserved. + +1. Abstract + + The IMAP4 specification [RFC-2060] describes a rich protocol for use + in building clients and servers for storage, retrieval, and + manipulation of electronic mail. Because the protocol is so rich and + has so many implementation choices, there are often trade-offs that + must be made and issues that must be considered when designing such + clients and servers. This document attempts to outline these issues + and to make recommendations in order to make the end products as + interoperable as possible. + +2. Conventions used in this document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The words "must", "must not", "should", "should not", and "may" are + used with specific meaning in this document; since their meaning is + somewhat different from that specified in RFC 2119, we do not put + them in all caps here. Their meaning is as follows: + + must -- This word means that the action described is necessary + to ensure interoperability. The recommendation should + not be ignored. + must not -- This phrase means that the action described will be + almost certain to hurt interoperability. The + recommendation should not be ignored. + + + + + + + +Leiba Informational [Page 1] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + should -- This word means that the action described is strongly + recommended and will enhance interoperability or + usability. The recommendation should not be ignored + without careful consideration. + should not -- This phrase means that the action described is strongly + recommended against, and might hurt interoperability or + usability. The recommendation should not be ignored + without careful consideration. + may -- This word means that the action described is an + acceptable implementation choice. No specific + recommendation is implied; this word is used to point + out a choice that might not be obvious, or to let + implementors know what choices have been made by + existing implementations. + +3. Interoperability Issues and Recommendations + +3.1. Accessibility + + This section describes the issues related to access to servers and + server resources. Concerns here include data sharing and maintenance + of client/server connections. + +3.1.1. Multiple Accesses of the Same Mailbox + + One strong point of IMAP4 is that, unlike POP3, it allows for + multiple simultaneous access to a single mailbox. A user can, thus, + read mail from a client at home while the client in the office is + still connected; or the help desk staff can all work out of the same + inbox, all seeing the same pool of questions. An important point + about this capability, though is that NO SERVER IS GUARANTEED TO + SUPPORT THIS. If you are selecting an IMAP server and this facility + is important to you, be sure that the server you choose to install, + in the configuration you choose to use, supports it. + + If you are designing a client, you must not assume that you can + access the same mailbox more than once at a time. That means + + 1. you must handle gracefully the failure of a SELECT command if the + server refuses the second SELECT, + 2. you must handle reasonably the severing of your connection (see + "Severed Connections", below) if the server chooses to allow the + second SELECT by forcing the first off, + 3. you must avoid making multiple connections to the same mailbox in + your own client (for load balancing or other such reasons), and + 4. you must avoid using the STATUS command on a mailbox that you have + selected (with some server implementations the STATUS command has + the same problems with multiple access as do the SELECT and + + + +Leiba Informational [Page 2] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + EXAMINE commands). + + A further note about STATUS: The STATUS command is sometimes used to + check a non-selected mailbox for new mail. This mechanism must not + be used to check for new mail in the selected mailbox; section 5.2 of + [RFC-2060] specifically forbids this in its last paragraph. Further, + since STATUS takes a mailbox name it is an independent operation, not + operating on the selected mailbox. Because of this, the information + it returns is not necessarily in synchronization with the selected + mailbox state. + +3.1.2. Severed Connections + + The client/server connection may be severed for one of three reasons: + the client severs the connection, the server severs the connection, + or the connection is severed by outside forces beyond the control of + the client and the server (a telephone line drops, for example). + Clients and servers must both deal with these situations. + + When the client wants to sever a connection, it's usually because it + has finished the work it needed to do on that connection. The client + should send a LOGOUT command, wait for the tagged response, and then + close the socket. But note that, while this is what's intended in + the protocol design, there isn't universal agreement here. Some + contend that sending the LOGOUT and waiting for the two responses + (untagged BYE and tagged OK) is wasteful and unnecessary, and that + the client can simply close the socket. The server should interpret + the closed socket as a log out by the client. The counterargument is + that it's useful from the standpoint of cleanup, problem + determination, and the like, to have an explicit client log out, + because otherwise there is no way for the server to tell the + difference between "closed socket because of log out" and "closed + socket because communication was disrupted". If there is a + client/server interaction problem, a client which routinely + terminates a session by breaking the connection without a LOGOUT will + make it much more difficult to determine the problem. + + Because of this disagreement, server designers must be aware that + some clients might close the socket without sending a LOGOUT. In any + case, whether or not a LOGOUT was sent, the server should not + implicitly expunge any messages from the selected mailbox. If a + client wants the server to do so, it must send a CLOSE or EXPUNGE + command explicitly. + + When the server wants to sever a connection it's usually due to an + inactivity timeout or is because a situation has arisen that has + changed the state of the mail store in a way that the server can not + communicate to the client. The server should send an untagged BYE + + + +Leiba Informational [Page 3] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + response to the client and then close the socket. Sending an + untagged BYE response before severing allows the server to send a + human-readable explanation of the problem to the client, which the + client may then log, display to the user, or both (see section 7.1.5 + of [RFC-2060]). + + Regarding inactivity timeouts, there is some controversy. Unlike + POP, for which the design is for a client to connect, retrieve mail, + and log out, IMAP's design encourages long-lived (and mostly + inactive) client/server sessions. As the number of users grows, this + can use up a lot of server resources, especially with clients that + are designed to maintain sessions for mailboxes that the user has + finished accessing. To alleviate this, a server may implement an + inactivity timeout, unilaterally closing a session (after first + sending an untagged BYE, as noted above). Some server operators have + reported dramatic improvements in server performance after doing + this. As specified in [RFC-2060], if such a timeout is done it must + not be until at least 30 minutes of inactivity. The reason for this + specification is to prevent clients from sending commands (such as + NOOP) to the server at frequent intervals simply to avert a too-early + timeout. If the client knows that the server may not time out the + session for at least 30 minutes, then the client need not poll at + intervals more frequent than, say, 25 minutes. + +3.2. Scaling + + IMAP4 has many features that allow for scalability, as mail stores + become larger and more numerous. Large numbers of users, mailboxes, + and messages, and very large messages require thought to handle + efficiently. This document will not address the administrative + issues involved in large numbers of users, but we will look at the + other items. + +3.2.1. Flood Control + + There are three situations when a client can make a request that will + result in a very large response - too large for the client reasonably + to deal with: there are a great many mailboxes available, there are a + great many messages in the selected mailbox, or there is a very large + message part. The danger here is that the end user will be stuck + waiting while the server sends (and the client processes) an enormous + response. In all of these cases there are things a client can do to + reduce that danger. + + There is also the case where a client can flood a server, by sending + an arbitratily long command. We'll discuss that issue, too, in this + section. + + + + +Leiba Informational [Page 4] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.2.1.1. Listing Mailboxes + + Some servers present Usenet newsgroups to IMAP users. Newsgroups, + and other such hierarchical mailbox structures, can be very numerous + but may have only a few entries at the top level of hierarchy. Also, + some servers are built against mail stores that can, unbeknownst to + the server, have circular hierarchies - that is, it's possible for + "a/b/c/d" to resolve to the same file structure as "a", which would + then mean that "a/b/c/d/b" is the same as "a/b", and the hierarchy + will never end. The LIST response in this case will be unlimited. + + Clients that will have trouble with this are those that use + + C: 001 LIST "" * + + to determine the mailbox list. Because of this, clients should not + use an unqualified "*" that way in the LIST command. A safer + approach is to list each level of hierarchy individually, allowing + the user to traverse the tree one limb at a time, thus: + + C: 001 LIST "" % + S: * LIST () "/" Banana + S: * LIST ...etc... + S: 001 OK done + + and then + + C: 002 LIST "" Banana/% + S: * LIST () "/" Banana/Apple + S: * LIST ...etc... + S: 002 OK done + + Using this technique the client's user interface can give the user + full flexibility without choking on the voluminous reply to "LIST *". + + Of course, it is still possible that the reply to + + C: 005 LIST "" alt.fan.celebrity.% + + may be thousands of entries long, and there is, unfortunately, + nothing the client can do to protect itself from that. This has not + yet been a notable problem. + + Servers that may export circular hierarchies (any server that + directly presents a UNIX file system, for instance) should limit the + hierarchy depth to prevent unlimited LIST responses. A suggested + depth limit is 20 hierarchy levels. + + + + +Leiba Informational [Page 5] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.2.1.2. Fetching the List of Messages + + When a client selects a mailbox, it is given a count, in the untagged + EXISTS response, of the messages in the mailbox. This number can be + very large. In such a case it might be unwise to use + + C: 004 FETCH 1:* ALL + + to populate the user's view of the mailbox. One good method to avoid + problems with this is to batch the requests, thus: + + C: 004 FETCH 1:50 ALL + S: * 1 FETCH ...etc... + S: 004 OK done + C: 005 FETCH 51:100 ALL + S: * 51 FETCH ...etc... + S: 005 OK done + C: 006 FETCH 101:150 ALL + ...etc... + + Using this method, another command, such as "FETCH 6 BODY[1]" can be + inserted as necessary, and the client will not have its access to the + server blocked by a storm of FETCH replies. (Such a method could be + reversed to fetch the LAST 50 messages first, then the 50 prior to + that, and so on.) + + As a smart extension of this, a well designed client, prepared for + very large mailboxes, will not automatically fetch data for all + messages AT ALL. Rather, the client will populate the user's view + only as the user sees it, possibly pre-fetching selected information, + and only fetching other information as the user scrolls to it. For + example, to select only those messages beginning with the first + unseen one: + + C: 003 SELECT INBOX + S: * 10000 EXISTS + S: * 80 RECENT + S: * FLAGS (\Answered \Flagged \Deleted \Draft \Seen) + S: * OK [UIDVALIDITY 824708485] UID validity status + S: * OK [UNSEEN 9921] First unseen message + S: 003 OK [READ-WRITE] SELECT completed + C: 004 FETCH 9921:* ALL + ... etc... + + If the server does not return an OK [UNSEEN] response, the client may + use SEARCH UNSEEN to obtain that value. + + + + + +Leiba Informational [Page 6] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + This mechanism is good as a default presentation method, but only + works well if the default message order is acceptable. A client may + want to present various sort orders to the user (by subject, by date + sent, by sender, and so on) and in that case (lacking a SORT + extension on the server side) the client WILL have to retrieve all + message descriptors. A client that provides this service should not + do it by default and should inform the user of the costs of choosing + this option for large mailboxes. + +3.2.1.3. Fetching a Large Body Part + + The issue here is similar to the one for a list of messages. In the + BODYSTRUCTURE response the client knows the size, in bytes, of the + body part it plans to fetch. Suppose this is a 70 MB video clip. The + client can use partial fetches to retrieve the body part in pieces, + avoiding the problem of an uninterruptible 70 MB literal coming back + from the server: + + C: 022 FETCH 3 BODY[1]<0.20000> + S: * 3 FETCH (FLAGS(\Seen) BODY[1]<0> {20000} + S: ...data...) + S: 022 OK done + C: 023 FETCH 3 BODY[1]<20001.20000> + S: * 3 FETCH (BODY[1]<20001> {20000} + S: ...data...) + S: 023 OK done + C: 024 FETCH 3 BODY[1]<40001.20000> + ...etc... + +3.2.1.4. BODYSTRUCTURE vs. Entire Messages + + Because FETCH BODYSTRUCTURE is necessary in order to determine the + number of body parts, and, thus, whether a message has "attachments", + clients often use FETCH FULL as their normal method of populating the + user's view of a mailbox. The benefit is that the client can display + a paperclip icon or some such indication along with the normal + message summary. However, this comes at a significant cost with some + server configurations. The parsing needed to generate the FETCH + BODYSTRUCTURE response may be time-consuming compared with that + needed for FETCH ENVELOPE. The client developer should consider this + issue when deciding whether the ability to add a paperclip icon is + worth the tradeoff in performance, especially with large mailboxes. + + Some clients, rather than using FETCH BODYSTRUCTURE, use FETCH BODY[] + (or the equivalent FETCH RFC822) to retrieve the entire message. + They then do the MIME parsing in the client. This may give the + client slightly more flexibility in some areas (access, for instance, + to header fields that aren't returned in the BODYSTRUCTURE and + + + +Leiba Informational [Page 7] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + ENVELOPE responses), but it can cause severe performance problems by + forcing the transfer of all body parts when the user might only want + to see some of them - a user logged on by modem and reading a small + text message with a large ZIP file attached may prefer to read the + text only and save the ZIP file for later. Therefore, a client + should not normally retrieve entire messages and should retrieve + message body parts selectively. + +3.2.1.5. Long Command Lines + + A client can wind up building a very long command line in an effort to + try to be efficient about requesting information from a server. This + can typically happen when a client builds a message set from selected + messages and doesn't recognise that contiguous blocks of messages may + be group in a range. Suppose a user selects all 10,000 messages in a + large mailbox and then unselects message 287. The client could build + that message set as "1:286,288:10000", but a client that doesn't + handle that might try to enumerate each message individually and build + "1,2,3,4, [and so on] ,9999,10000". Adding that to the fetch command + results in a command line that's almost 49,000 octets long, and, + clearly, one can construct a command line that's even longer. + + A client should limit the length of the command lines it generates to + approximately 1000 octets (including all quoted strings but not + including literals). If the client is unable to group things into + ranges so that the command line is within that length, it should + split the request into multiple commands. The client should use + literals instead of long quoted strings, in order to keep the command + length down. + + For its part, a server should allow for a command line of at least + 8000 octets. This provides plenty of leeway for accepting reasonable + length commands from clients. The server should send a BAD response + to a command that does not end within the server's maximum accepted + command length. + +3.2.2. Subscriptions + + The client isn't the only entity that can get flooded: the end user, + too, may need some flood control. The IMAP4 protocol provides such + control in the form of subscriptions. Most servers support the + SUBSCRIBE, UNSUBSCRIBE, and LSUB commands, and many users choose to + narrow down a large list of available mailboxes by subscribing to the + ones that they usually want to see. Clients, with this in mind, + should give the user a way to see only subscribed mailboxes. A + client that never uses the LSUB command takes a significant usability + feature away from the user. Of course, the client would not want to + hide the LIST command completely; the user needs to have a way to + + + +Leiba Informational [Page 8] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + choose between LIST and LSUB. The usual way to do this is to provide + a setting like "show which mailboxes?: [] all [] subscribed only". + +3.2.3. Searching + + IMAP SEARCH commands can become particularly troublesome (that is, + slow) on mailboxes containing a large number of messages. So let's + put a few things in perspective in that regard. + + The flag searches should be fast. The flag searches (ALL, [UN]SEEN, + [UN]ANSWERED, [UN]DELETED, [UN]DRAFT, [UN]FLAGGED, NEW, OLD, RECENT) + are known to be used by clients for the client's own use (for + instance, some clients use "SEARCH UNSEEN" to find unseen mail and + "SEARCH DELETED" to warn the user before expunging messages). + + Other searches, particularly the text searches (HEADER, TEXT, BODY) + are initiated by the user, rather than by the client itself, and + somewhat slower performance can be tolerated, since the user is aware + that the search is being done (and is probably aware that it might be + time-consuming). A smart server might use dynamic indexing to speed + commonly used text searches. + + The client may allow other commands to be sent to the server while a + SEARCH is in progress, but at the time of this writing there is + little or no server support for parallel processing of multiple + commands in the same session (and see "Multiple Accesses of the Same + Mailbox" above for a description of the dangers of trying to work + around this by doing your SEARCH in another session). + + Another word about text searches: some servers, built on database + back-ends with indexed search capabilities, may return search results + that do not match the IMAP spec's "case-insensitive substring" + requirements. While these servers are in violation of the protocol, + there is little harm in the violation as long as the search results + are used only in response to a user's request. Still, developers of + such servers should be aware that they ARE violating the protocol, + should think carefully about that behaviour, and must be certain that + their servers respond accurately to the flag searches for the reasons + outlined above. + + In addition, servers should support CHARSET UTF-8 [UTF-8] in + searches. + + + + + + + + + +Leiba Informational [Page 9] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.3 Avoiding Invalid Requests + + IMAP4 provides ways for a server to tell a client in advance what is + and isn't permitted in some circumstances. Clients should use these + features to avoid sending requests that a well designed client would + know to be invalid. This section explains this in more detail. + +3.3.1. The CAPABILITY Command + + All IMAP4 clients should use the CAPABILITY command to determine what + version of IMAP and what optional features a server supports. The + client should not send IMAP4rev1 commands and arguments to a server + that does not advertize IMAP4rev1 in its CAPABILITY response. + Similarly, the client should not send IMAP4 commands that no longer + exist in IMAP4rev1 to a server that does not advertize IMAP4 in its + CAPABILITY response. An IMAP4rev1 server is NOT required to support + obsolete IMAP4 or IMAP2bis commands (though some do; do not let this + fact lull you into thinking that it's valid to send such commands to + an IMAP4rev1 server). + + A client should not send commands to probe for the existance of + certain extensions. All standard and standards-track extensions + include CAPABILITY tokens indicating their presense. All private and + experimental extensions should do the same, and clients that take + advantage of them should use the CAPABILITY response to determine + whether they may be used or not. + +3.3.2. Don't Do What the Server Says You Can't + + In many cases, the server, in response to a command, will tell the + client something about what can and can't be done with a particular + mailbox. The client should pay attention to this information and + should not try to do things that it's been told it can't do. + + Examples: + + * Do not try to SELECT a mailbox that has the \Noselect flag set. + * Do not try to CREATE a sub-mailbox in a mailbox that has the + \Noinferiors flag set. + * Do not respond to a failing COPY or APPEND command by trying to + CREATE the target mailbox if the server does not respond with a + [TRYCREATE] response code. + * Do not try to expunge a mailbox that has been selected with the + [READ-ONLY] response code. + + + + + + + +Leiba Informational [Page 10] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.4. Miscellaneous Protocol Considerations + + We describe here a number of important protocol-related issues, the + misunderstanding of which has caused significant interoperability + problems in IMAP4 implementations. One general item is that every + implementer should be certain to take note of and to understand + section 2.2.2 and the preamble to section 7 of the IMAP4rev1 spec + [RFC-2060]. + +3.4.1. Well Formed Protocol + + We cannot stress enough the importance of adhering strictly to the + protocol grammar. The specification of the protocol is quite rigid; + do not assume that you can insert blank space for "readability" if + none is called for. Keep in mind that there are parsers out there + that will crash if there are protocol errors. There are clients that + will report every parser burp to the user. And in any case, + information that cannot be parsed is information that is lost. Be + careful in your protocol generation. And see "A Word About Testing", + below. + + In particular, note that the string in the INTERNALDATE response is + NOT an RFC-822 date string - that is, it is not in the same format as + the first string in the ENVELOPE response. Since most clients will, + in fact, accept an RFC-822 date string in the INTERNALDATE response, + it's easy to miss this in your interoperability testing. But it will + cause a problem with some client, so be sure to generate the correct + string for this field. + +3.4.2. Special Characters + + Certain characters, currently the double-quote and the backslash, may + not be sent as-is inside a quoted string. These characters must be + preceded by the escape character if they are in a quoted string, or + else the string must be sent as a literal. Both clients and servers + must handle this, both on output (they must send these characters + properly) and on input (they must be able to receive escaped + characters in quoted strings). Example: + + C: 001 LIST "" % + S: * LIST () "" INBOX + S: * LIST () "\\" TEST + S: * LIST () "\\" {12} + S: "My" mailbox + S: 001 OK done + C: 002 LIST "" "\"My\" mailbox\\%" + S: * LIST () "\\" {17} + S: "My" mailbox\Junk + + + +Leiba Informational [Page 11] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + S: 002 OK done + + Note that in the example the server sent the hierarchy delimiter as + an escaped character in the quoted string and sent the mailbox name + containing imbedded double-quotes as a literal. The client used only + quoted strings, escaping both the backslash and the double-quote + characters. + + The CR and LF characters may be sent ONLY in literals; they are not + allowed, even if escaped, inside quoted strings. + + And while we're talking about special characters: the IMAP spec, in + the section titled "Mailbox International Naming Convention", + describes how to encode mailbox names in modified UTF-7 [UTF-7 and + RFC-2060]. Implementations must adhere to this in order to be + interoperable in the international market, and servers should + validate mailbox names sent by client and reject names that do not + conform. + + As to special characters in userids and passwords: clients must not + restrict what a user may type in for a userid or a password. The + formal grammar specifies that these are "astrings", and an astring + can be a literal. A literal, in turn can contain any 8-bit + character, and clients must allow users to enter all 8-bit characters + here, and must pass them, unchanged, to the server (being careful to + send them as literals when necessary). In particular, some server + configurations use "@" in user names, and some clients do not allow + that character to be entered; this creates a severe interoperability + problem. + +3.4.3. UIDs and UIDVALIDITY + + Servers that support existing back-end mail stores often have no good + place to save UIDs for messages. Often the existing mail store will + not have the concept of UIDs in the sense that IMAP has: strictly + increasing, never re-issued, 32-bit integers. Some servers solve + this by storing the UIDs in a place that's accessible to end users, + allowing for the possibility that the users will delete them. Others + solve it by re-assigning UIDs every time a mailbox is selected. + + The server should maintain UIDs permanently for all messages if it + can. If that's not possible, the server must change the UIDVALIDITY + value for the mailbox whenever any of the UIDs may have become + invalid. Clients must recognize that the UIDVALIDITY has changed and + must respond to that condition by throwing away any information that + they have saved about UIDs in that mailbox. There have been many + problems in this area when clients have failed to do this; in the + worst case it will result in loss of mail when a client deletes the + + + +Leiba Informational [Page 12] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + wrong piece of mail by using a stale UID. + + It seems to be a common misunderstanding that "the UIDVALIDITY and + the UID, taken together, form a 64-bit identifier that uniquely + identifies a message on a server". This is absolutely NOT TRUE. + There is no assurance that the UIDVALIDITY values of two mailboxes be + different, so the UIDVALIDITY in no way identifies a mailbox. The + ONLY purpose of UIDVALIDITY is, as its name indicates, to give the + client a way to check the validity of the UIDs it has cached. While + it is a valid implementation choice to put these values together to + make a 64-bit identifier for the message, the important concept here + is that UIDs are not unique between mailboxes; they are only unique + WITHIN a given mailbox. + + Some server implementations have attempted to make UIDs unique across + the entire server. This is inadvisable, in that it limits the life + of UIDs unnecessarily. The UID is a 32-bit number and will run out + in reasonably finite time if it's global across the server. If you + assign UIDs sequentially in one mailbox, you will not have to start + re-using them until you have had, at one time or another, 2**32 + different messages in that mailbox. In the global case, you will + have to reuse them once you have had, at one time or another, 2**32 + different messages in the entire mail store. Suppose your server has + around 8000 users registered (2**13). That gives an average of 2**19 + UIDs per user. Suppose each user gets 32 messages (2**5) per day. + That gives you 2**14 days (16000+ days = about 45 years) before you + run out. That may seem like enough, but multiply the usage just a + little (a lot of spam, a lot of mailing list subscriptions, more + users) and you limit yourself too much. + + What's worse is that if you have to wrap the UIDs, and, thus, you + have to change UIDVALIDITY and invalidate the UIDs in the mailbox, + you have to do it for EVERY mailbox in the system, since they all + share the same UID pool. If you assign UIDs per mailbox and you have + a problem, you only have to kill the UIDs for that one mailbox. + + Under extreme circumstances (and this is extreme, indeed), the server + may have to invalidate UIDs while a mailbox is in use by a client - + that is, the UIDs that the client knows about in its active mailbox + are no longer valid. In that case, the server must immediately + change the UIDVALIDITY and must communicate this to the client. The + server may do this by sending an unsolicited UIDVALIDITY message, in + the same form as in response to the SELECT command. Clients must be + prepared to handle such a message and the possibly coincident failure + of the command in process. For example: + + + + + + +Leiba Informational [Page 13] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + C: 032 UID STORE 382 +Flags.silent \Deleted + S: * OK [UIDVALIDITY 12345] New UIDVALIDITY value! + S: 032 NO UID command rejected because UIDVALIDITY changed! + C: ...invalidates local information and re-fetches... + C: 033 FETCH 1:* UID + ...etc... + + At the time of the writing of this document, the only server known to + do this does so only under the following condition: the client + selects INBOX, but there is not yet a physical INBOX file created. + Nonetheless, the SELECT succeeds, exporting an empty INBOX with a + temporary UIDVALIDITY of 1. While the INBOX remains selected, mail + is delivered to the user, which creates the real INBOX file and + assigns a permanent UIDVALIDITY (that is likely not to be 1). The + server reports the change of UIDVALIDITY, but as there were no + messages before, so no UIDs have actually changed, all the client + must do is accept the change in UIDVALIDITY. + + Alternatively, a server may force the client to re-select the + mailbox, at which time it will obtain a new UIDVALIDITY value. To do + this, the server closes this client session (see "Severed + Connections" above) and the client then reconnects and gets back in + synch. Clients must be prepared for either of these behaviours. + + We do not know of, nor do we anticipate the future existance of, a + server that changes UIDVALIDITY while there are existing messages, + but clients must be prepared to handle this eventuality. + +3.4.4. FETCH Responses + + When a client asks for certain information in a FETCH command, the + server may return the requested information in any order, not + necessarily in the order that it was requested. Further, the server + may return the information in separate FETCH responses and may also + return information that was not explicitly requested (to reflect to + the client changes in the state of the subject message). Some + examples: + + C: 001 FETCH 1 UID FLAGS INTERNALDATE + S: * 5 FETCH (FLAGS (\Deleted)) + S: * 1 FETCH (FLAGS (\Seen) INTERNALDATE "..." UID 345) + S: 001 OK done + + (In this case, the responses are in a different order. Also, the + server returned a flag update for message 5, which wasn't part of the + client's request.) + + + + + +Leiba Informational [Page 14] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + C: 002 FETCH 2 UID FLAGS INTERNALDATE + S: * 2 FETCH (INTERNALDATE "...") + S: * 2 FETCH (UID 399) + S: * 2 FETCH (FLAGS ()) + S: 002 OK done + + (In this case, the responses are in a different order and were + returned in separate responses.) + + C: 003 FETCH 2 BODY[1] + S: * 2 FETCH (FLAGS (\Seen) BODY[1] {14} + S: Hello world! + S: ) + S: 003 OK done + + (In this case, the FLAGS response was added by the server, since + fetching the body part caused the server to set the \Seen flag.) + + Because of this characteristic a client must be ready to receive any + FETCH response at any time and should use that information to update + its local information about the message to which the FETCH response + refers. A client must not assume that any FETCH responses will come + in any particular order, or even that any will come at all. If after + receiving the tagged response for a FETCH command the client finds + that it did not get all of the information requested, the client + should send a NOOP command to the server to ensure that the server + has an opportunity to send any pending EXPUNGE responses to the + client (see [RFC-2180]). + +3.4.5. RFC822.SIZE + + Some back-end mail stores keep the mail in a canonical form, rather + than retaining the original MIME format of the messages. This means + that the server must reassemble the message to produce a MIME stream + when a client does a fetch such as RFC822 or BODY[], requesting the + entire message. It also may mean that the server has no convenient + way to know the RFC822.SIZE of the message. Often, such a server + will actually have to build the MIME stream to compute the size, only + to throw the stream away and report the size to the client. + + When this is the case, some servers have chosen to estimate the size, + rather than to compute it precisely. Such an estimate allows the + client to display an approximate size to the user and to use the + estimate in flood control considerations (q.v.), but requires that + the client not use the size for things such as allocation of buffers, + because those buffers might then be too small to hold the actual MIME + stream. Instead, a client should use the size that's returned in the + literal when you fetch the data. + + + +Leiba Informational [Page 15] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + The protocol requires that the RFC822.SIZE value returned by the + server be EXACT. Estimating the size is a protocol violation, and + server designers must be aware that, despite the performance savings + they might realize in using an estimate, this practice will cause + some clients to fail in various ways. If possible, the server should + compute the RFC822.SIZE for a particular message once, and then save + it for later retrieval. If that's not possible, the server must + compute the value exactly every time. Incorrect estimates do cause + severe interoperability problems with some clients. + +3.4.6. Expunged Messages + + If the server allows multiple connections to the same mailbox, it is + often possible for messages to be expunged in one client unbeknownst + to another client. Since the server is not allowed to tell the + client about these expunged messages in response to a FETCH command, + the server may have to deal with the issue of how to return + information about an expunged message. There was extensive + discussion about this issue, and the results of that discussion are + summarized in [RFC-2180]. See that reference for a detailed + explanation and for recommendations. + +3.4.7. The Namespace Issue + + Namespaces are a very muddy area in IMAP4 implementation right now + (see [NAMESPACE] for a proposal to clear the water a bit). Until the + issue is resolved, the important thing for client developers to + understand is that some servers provide access through IMAP to more + than just the user's personal mailboxes, and, in fact, the user's + personal mailboxes may be "hidden" somewhere in the user's default + hierarchy. The client, therefore, should provide a setting wherein + the user can specify a prefix to be used when accessing mailboxes. If + the user's mailboxes are all in "~/mail/", for instance, then the + user can put that string in the prefix. The client would then put + the prefix in front of any name pattern in the LIST and LSUB + commands: + + C: 001 LIST "" ~/mail/% + + (See also "Reference Names in the LIST Command" below.) + +3.4.8. Creating Special-Use Mailboxes + + It may seem at first that this is part of the namespace issue; it is + not, and is only indirectly related to it. A number of clients like + to create special-use mailboxes with particular names. Most + commonly, clients with a "trash folder" model of message deletion + want to create a mailbox with the name "Trash" or "Deleted". Some + + + +Leiba Informational [Page 16] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + clients want to create a "Drafts" mailbox, an "Outbox" mailbox, or a + "Sent Mail" mailbox. And so on. There are two major + interoperability problems with this practice: + + 1. different clients may use different names for mailboxes with + similar functions (such as "Trash" and "Deleted"), or may manage + the same mailboxes in different ways, causing problems if a user + switches between clients and + 2. there is no guarantee that the server will allow the creation of + the desired mailbox. + + The client developer is, therefore, well advised to consider + carefully the creation of any special-use mailboxes on the server, + and, further, the client must not require such mailbox creation - + that is, if you do decide to do this, you must handle gracefully the + failure of the CREATE command and behave reasonably when your + special-use mailboxes do not exist and can not be created. + + In addition, the client developer should provide a convenient way for + the user to select the names for any special-use mailboxes, allowing + the user to make these names the same in all clients used and to put + them where the user wants them. + +3.4.9. Reference Names in the LIST Command + + Many implementers of both clients and servers are confused by the + "reference name" on the LIST command. The reference name is intended + to be used in much the way a "cd" (change directory) command is used + on Unix, PC DOS, Windows, and OS/2 systems. That is, the mailbox + name is interpreted in much the same way as a file of that name would + be found if one had done a "cd" command into the directory specified + by the reference name. For example, in Unix we have the following: + + > cd /u/jones/junk + > vi banana [file is "/u/jones/junk/banana"] + > vi stuff/banana [file is "/u/jones/junk/stuff/banana"] + > vi /etc/hosts [file is "/etc/hosts"] + + In the past, there have been several interoperability problems with + this. First, while some IMAP servers are built on Unix or PC file + systems, many others are not, and the file system semantics do not + make sense in those configurations. Second, while some IMAP servers + expose the underlying file system to the clients, others allow access + only to the user's personal mailboxes, or to some other limited set + of files, making such file-system-like semantics less meaningful. + Third, because the IMAP spec leaves the interpretation of the + reference name as "implementation-dependent", in the past the various + server implementations handled it in vastly differing ways. + + + +Leiba Informational [Page 17] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + The following recommendations are the result of significant + operational experience, and are intended to maximize + interoperability. + + Server implementations must implement the reference argument in a way + that matches the intended "change directory" operation as closely as + possible. As a minimum implementation, the reference argument may be + prepended to the mailbox name (while suppressing double delimiters; + see the next paragraph). Even servers that do not provide a way to + break out of the current hierarchy (see "breakout facility" below) + must provide a reasonable implementation of the reference argument, + as described here, so that they will interoperate with clients that + use it. + + Server implementations that prepend the reference argument to the + mailbox name should insert a hierarchy delimiter between them, and + must not insert a second if one is already present: + + C: A001 LIST ABC DEF + S: * LIST () "/" ABC/DEF <=== should do this + S: A001 OK done + + C: A002 LIST ABC/ /DEF + S: * LIST () "/" ABC//DEF <=== must not do this + S: A002 OK done + + On clients, the reference argument is chiefly used to implement a + "breakout facility", wherein the user may directly access a mailbox + outside the "current directory" hierarchy. Client implementations + should have an operational mode that does not use the reference + argument. This is to interoperate with older servers that did not + implement the reference argument properly. While it's a good idea to + give the user access to a breakout facility, clients that do not + intend to do so should not use the reference argument at all. + + Client implementations should always place a trailing hierarchy + delimiter on the reference argument. This is because some servers + prepend the reference argument to the mailbox name without inserting + a hierarchy delimiter, while others do insert a hierarchy delimiter + if one is not already present. A client that puts the delimiter in + will work with both varieties of server. + + Client implementations that implement a breakout facility should + allow the user to choose whether or not to use a leading hierarchy + delimiter on the mailbox argument. This is because the handling of a + leading mailbox hierarchy delimiter also varies from server to + server, and even between different mailstores on the same server. In + some cases, a leading hierarchy delimiter means "discard the + + + +Leiba Informational [Page 18] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + reference argument" (implementing the intended breakout facility), + thus: + + C: A001 LIST ABC/ /DEF + S: * LIST () "/" /DEF + S: A001 OK done + + In other cases, however, the two are catenated and the extra + hierarchy delimiter is discarded, thus: + + C: A001 LIST ABC/ /DEF + S: * LIST () "/" ABC/DEF + S: A001 OK done + + Client implementations must not assume that the server supports a + breakout facility, but may provide a way for the user to use one if + it is available. Any breakout facility should be exported to the + user interface. Note that there may be other "breakout" characters + besides the hierarchy delimiter (for instance, UNIX filesystem + servers are likely to use a leading "~" as well), and that their + interpretation is server-dependent. + +3.4.10. Mailbox Hierarchy Delimiters + + The server's selection of what to use as a mailbox hierarchy + delimiter is a difficult one, involving several issues: What + characters do users expect to see? What characters can they enter + for a hierarchy delimiter if it is desired (or required) that the + user enter it? What character can be used for the hierarchy + delimiter, noting that the chosen character can not otherwise be used + in the mailbox name? + + Because some interfaces show users the hierarchy delimiters or allow + users to enter qualified mailbox names containing them, server + implementations should use delimiter characters that users generally + expect to see as name separators. The most common characters used + for this are "/" (as in Unix file names), "\" (as in OS/2 and Windows + file names), and "." (as in news groups). There is little to choose + among these apart from what users may expect or what is dictated by + the underlying file system, if any. One consideration about using + "\" is that it's also a special character in the IMAP protocol. While + the use of other hierarchy delimiter characters is permissible, A + DESIGNER IS WELL ADVISED TO STAY WITH ONE FROM THIS SET unless the + server is intended for special purposes only. Implementers might be + thinking about using characters such as "-", "_", ";", "&", "#", "@", + and "!", but they should be aware of the surprise to the user as well + as of the effect on URLs and other external specifications (since + some of these characters have special meanings there). Also, a + + + +Leiba Informational [Page 19] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + server that uses "\" (and clients of such a server) must remember to + escape that character in quoted strings or to send literals instead. + Literals are recommended over escaped characters in quoted strings in + order to maintain compatibility with older IMAP versions that did not + allow escaped characters in quoted strings (but check the grammar to + see where literals are allowed): + + C: 001 LIST "" {13} + S: + send literal + C: this\%\%\%\h* + S: * LIST () "\\" {27} + S: this\is\a\mailbox\hierarchy + S: 001 OK LIST complete + + In any case, a server should not use normal alpha-numeric characters + (such as "X" or "0") as delimiters; a user would be very surprised to + find that "EXPENDITURES" actually represented a two-level hierarchy. + And a server should not use characters that are non-printable or + difficult or impossible to enter on a standard US keyboard. Control + characters, box-drawing characters, and characters from non-US + alphabets fit into this category. Their use presents + interoperability problems that are best avoided. + + The UTF-7 encoding of mailbox names also raises questions about what + to do with the hierarchy delimiters in encoded names: do we encode + each hierarchy level and separate them with delimiters, or do we + encode the fully qualified name, delimiters and all? The answer for + IMAP is the former: encode each hierarchy level separately, and + insert delimiters between. This makes it particularly important not + to use as a hierarchy delimiter a character that might cause + confusion with IMAP's modified UTF-7 [UTF-7 and RFC-2060] encoding. + + To repeat: a server should use "/", "\", or "." as its hierarchy + delimiter. The use of any other character is likely to cause + problems and is STRONGLY DISCOURAGED. + +3.4.11. ALERT Response Codes + + The protocol spec is very clear on the matter of what to do with + ALERT response codes, and yet there are many clients that violate it + so it needs to be said anyway: "The human-readable text contains a + special alert that must be presented to the user in a fashion that + calls the user's attention to the message." That should be clear + enough, but I'll repeat it here: Clients must present ALERT text + clearly to the user. + + + + + + +Leiba Informational [Page 20] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +3.4.12. Deleting Mailboxes + + The protocol does not guarantee that a client may delete a mailbox + that is not empty, though on some servers it is permissible and is, + in fact, much faster than the alternative or deleting all the + messages from the client. If the client chooses to try to take + advantage of this possibility it must be prepared to use the other + method in the even that the more convenient one fails. Further, a + client should not try to delete the mailbox that it has selected, but + should first close that mailbox; some servers do not permit the + deletion of the selected mailbox. + + That said, a server should permit the deletion of a non-empty + mailbox; there's little reason to pass this work on to the client. + Moreover, forbidding this prevents the deletion of a mailbox that for + some reason can not be opened or expunged, leading to possible + denial-of-service problems. + + Example: + + [User tells the client to delete mailbox BANANA, which is + currently selected...] + C: 008 CLOSE + S: 008 OK done + C: 009 DELETE BANANA + S: 009 NO Delete failed; mailbox is not empty. + C: 010 SELECT BANANA + S: * ... untagged SELECT responses + S: 010 OK done + C: 011 STORE 1:* +FLAGS.SILENT \DELETED + S: 011 OK done + C: 012 CLOSE + S: 012 OK done + C: 013 DELETE BANANA + S: 013 OK done + +3.5. A Word About Testing + + Since the whole point of IMAP is interoperability, and since + interoperability can not be tested in a vacuum, the final + recommendation of this treatise is, "Test against EVERYTHING." Test + your client against every server you can get an account on. Test + your server with every client you can get your hands on. Many + clients make limited test versions available on the Web for the + downloading. Many server owners will give serious client developers + guest accounts for testing. Contact them and ask. NEVER assume that + because your client works with one or two servers, or because your + server does fine with one or two clients, you will interoperate well + + + +Leiba Informational [Page 21] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + + in general. + + In particular, in addition to everything else, be sure to test + against the reference implementations: the PINE client, the + University of Washington server, and the Cyrus server. + + See the following URLs on the web for more information here: + + IMAP Products and Sources: http://www.imap.org/products.html + IMC MailConnect: http://www.imc.org/imc-mailconnect + +4. Security Considerations + + This document describes behaviour of clients and servers that use the + IMAP4 protocol, and as such, has the same security considerations as + described in [RFC-2060]. + +5. References + + [RFC-2060] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [RFC-2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC-2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC + 2180, July 1997. + + [UTF-8] Yergeau, F., " UTF-8, a transformation format of Unicode + and ISO 10646", RFC 2044, October 1996. + + [UTF-7] Goldsmith, D. and M. Davis, "UTF-7, a Mail-Safe + Transformation Format of Unicode", RFC 2152, May 1997. + + [NAMESPACE] Gahrns, M. and C. Newman, "IMAP4 Namespace", Work in + Progress. + +6. Author's Address + + Barry Leiba + IBM T.J. Watson Research Center + 30 Saw Mill River Road + Hawthorne, NY 10532 + + Phone: 1-914-784-7941 + EMail: leiba@watson.ibm.com + + + + + +Leiba Informational [Page 22] + +RFC 2683 IMAP4 Implementation Recommendations September 1999 + + +7. Full Copyright Statement + + Copyright (C) The Internet Society (1999). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Leiba Informational [Page 23] + diff --git a/docs/rfcs/rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt b/docs/rfcs/rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt new file mode 100644 index 0000000..c1a54c4 --- /dev/null +++ b/docs/rfcs/rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt @@ -0,0 +1,1515 @@ + + + + + + +Network Working Group P. Leach +Request for Comments: 2831 Microsoft +Category: Standards Track C. Newman + Innosoft + May 2000 + + + Using Digest Authentication as a SASL Mechanism + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2000). All Rights Reserved. + +Abstract + + This specification defines how HTTP Digest Authentication [Digest] + can be used as a SASL [RFC 2222] mechanism for any protocol that has + a SASL profile. It is intended both as an improvement over CRAM-MD5 + [RFC 2195] and as a convenient way to support a single authentication + mechanism for web, mail, LDAP, and other protocols. + +Table of Contents + + 1 INTRODUCTION.....................................................2 + 1.1 CONVENTIONS AND NOTATION......................................2 + 1.2 REQUIREMENTS..................................................3 + 2 AUTHENTICATION...................................................3 + 2.1 INITIAL AUTHENTICATION........................................3 + 2.1.1 Step One...................................................3 + 2.1.2 Step Two...................................................6 + 2.1.3 Step Three................................................12 + 2.2 SUBSEQUENT AUTHENTICATION....................................12 + 2.2.1 Step one..................................................13 + 2.2.2 Step Two..................................................13 + 2.3 INTEGRITY PROTECTION.........................................13 + 2.4 CONFIDENTIALITY PROTECTION...................................14 + 3 SECURITY CONSIDERATIONS.........................................15 + 3.1 AUTHENTICATION OF CLIENTS USING DIGEST AUTHENTICATION........15 + 3.2 COMPARISON OF DIGEST WITH PLAINTEXT PASSWORDS................16 + 3.3 REPLAY ATTACKS...............................................16 + + + +Leach & Newman Standards Track [Page 1] + +RFC 2831 Digest SASL Mechanism May 2000 + + + 3.4 ONLINE DICTIONARY ATTACKS....................................16 + 3.5 OFFLINE DICTIONARY ATTACKS...................................16 + 3.6 MAN IN THE MIDDLE............................................17 + 3.7 CHOSEN PLAINTEXT ATTACKS.....................................17 + 3.8 SPOOFING BY COUNTERFEIT SERVERS..............................17 + 3.9 STORING PASSWORDS............................................17 + 3.10 MULTIPLE REALMS.............................................18 + 3.11 SUMMARY.....................................................18 + 4 EXAMPLE.........................................................18 + 5 REFERENCES......................................................20 + 6 AUTHORS' ADDRESSES..............................................21 + 7 ABNF............................................................21 + 7.1 AUGMENTED BNF................................................21 + 7.2 BASIC RULES..................................................23 + 8 SAMPLE CODE.....................................................25 + 9 FULL COPYRIGHT STATEMENT........................................27 + +1 Introduction + + This specification describes the use of HTTP Digest Access + Authentication as a SASL mechanism. The authentication type + associated with the Digest SASL mechanism is "DIGEST-MD5". + + This specification is intended to be upward compatible with the + "md5-sess" algorithm of HTTP/1.1 Digest Access Authentication + specified in [Digest]. The only difference in the "md5-sess" + algorithm is that some directives not needed in a SASL mechanism have + had their values defaulted. + + There is one new feature for use as a SASL mechanism: integrity + protection on application protocol messages after an authentication + exchange. + + Also, compared to CRAM-MD5, DIGEST-MD5 prevents chosen plaintext + attacks, and permits the use of third party authentication servers, + mutual authentication, and optimized reauthentication if a client has + recently authenticated to a server. + +1.1 Conventions and Notation + + This specification uses the same ABNF notation and lexical + conventions as HTTP/1.1 specification; see appendix A. + + Let { a, b, ... } be the concatenation of the octet strings a, b, ... + + Let H(s) be the 16 octet MD5 hash [RFC 1321] of the octet string s. + + + + + +Leach & Newman Standards Track [Page 2] + +RFC 2831 Digest SASL Mechanism May 2000 + + + Let KD(k, s) be H({k, ":", s}), i.e., the 16 octet hash of the string + k, a colon and the string s. + + Let HEX(n) be the representation of the 16 octet MD5 hash n as a + string of 32 hex digits (with alphabetic characters always in lower + case, since MD5 is case sensitive). + + Let HMAC(k, s) be the 16 octet HMAC-MD5 [RFC 2104] of the octet + string s using the octet string k as a key. + + The value of a quoted string constant as an octet string does not + include any terminating null character. + +1.2 Requirements + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC 2119]. + + An implementation is not compliant if it fails to satisfy one or more + of the MUST level requirements for the protocols it implements. An + implementation that satisfies all the MUST level and all the SHOULD + level requirements for its protocols is said to be "unconditionally + compliant"; one that satisfies all the MUST level requirements but + not all the SHOULD level requirements for its protocols is said to be + "conditionally compliant." + +2 Authentication + + The following sections describe how to use Digest as a SASL + authentication mechanism. + +2.1 Initial Authentication + + If the client has not recently authenticated to the server, then it + must perform "initial authentication", as defined in this section. If + it has recently authenticated, then a more efficient form is + available, defined in the next section. + +2.1.1 Step One + + The server starts by sending a challenge. The data encoded in the + challenge contains a string formatted according to the rules for a + "digest-challenge" defined as follows: + + + + + + + +Leach & Newman Standards Track [Page 3] + +RFC 2831 Digest SASL Mechanism May 2000 + + + digest-challenge = + 1#( realm | nonce | qop-options | stale | maxbuf | charset + algorithm | cipher-opts | auth-param ) + + realm = "realm" "=" <"> realm-value <"> + realm-value = qdstr-val + nonce = "nonce" "=" <"> nonce-value <"> + nonce-value = qdstr-val + qop-options = "qop" "=" <"> qop-list <"> + qop-list = 1#qop-value + qop-value = "auth" | "auth-int" | "auth-conf" | + token + stale = "stale" "=" "true" + maxbuf = "maxbuf" "=" maxbuf-value + maxbuf-value = 1*DIGIT + charset = "charset" "=" "utf-8" + algorithm = "algorithm" "=" "md5-sess" + cipher-opts = "cipher" "=" <"> 1#cipher-value <"> + cipher-value = "3des" | "des" | "rc4-40" | "rc4" | + "rc4-56" | token + auth-param = token "=" ( token | quoted-string ) + + The meanings of the values of the directives used above are as + follows: + + realm + Mechanistically, a string which can enable users to know which + username and password to use, in case they might have different + ones for different servers. Conceptually, it is the name of a + collection of accounts that might include the user's account. This + string should contain at least the name of the host performing the + authentication and might additionally indicate the collection of + users who might have access. An example might be + "registered_users@gotham.news.example.com". This directive is + optional; if not present, the client SHOULD solicit it from the + user or be able to compute a default; a plausible default might be + the realm supplied by the user when they logged in to the client + system. Multiple realm directives are allowed, in which case the + user or client must choose one as the realm for which to supply to + username and password. + + nonce + A server-specified data string which MUST be different each time a + digest-challenge is sent as part of initial authentication. It is + recommended that this string be base64 or hexadecimal data. Note + that since the string is passed as a quoted string, the + double-quote character is not allowed unless escaped (see section + 7.2). The contents of the nonce are implementation dependent. The + + + +Leach & Newman Standards Track [Page 4] + +RFC 2831 Digest SASL Mechanism May 2000 + + + security of the implementation depends on a good choice. It is + RECOMMENDED that it contain at least 64 bits of entropy. The nonce + is opaque to the client. This directive is required and MUST + appear exactly once; if not present, or if multiple instances are + present, the client should abort the authentication exchange. + + qop-options + A quoted string of one or more tokens indicating the "quality of + protection" values supported by the server. The value "auth" + indicates authentication; the value "auth-int" indicates + authentication with integrity protection; the value "auth-conf" + indicates authentication with integrity protection and encryption. + This directive is optional; if not present it defaults to "auth". + The client MUST ignore unrecognized options; if the client + recognizes no option, it should abort the authentication exchange. + + stale + The "stale" directive is not used in initial authentication. See + the next section for its use in subsequent authentications. This + directive may appear at most once; if multiple instances are + present, the client should abort the authentication exchange. + + maxbuf + A number indicating the size of the largest buffer the server is + able to receive when using "auth-int" or "auth-conf". If this + directive is missing, the default value is 65536. This directive + may appear at most once; if multiple instances are present, the + client should abort the authentication exchange. + + charset + This directive, if present, specifies that the server supports + UTF-8 encoding for the username and password. If not present, the + username and password must be encoded in ISO 8859-1 (of which + US-ASCII is a subset). The directive is needed for backwards + compatibility with HTTP Digest, which only supports ISO 8859-1. + This directive may appear at most once; if multiple instances are + present, the client should abort the authentication exchange. + + algorithm + This directive is required for backwards compatibility with HTTP + Digest., which supports other algorithms. . This directive is + required and MUST appear exactly once; if not present, or if + multiple instances are present, the client should abort the + authentication exchange. + + + + + + + +Leach & Newman Standards Track [Page 5] + +RFC 2831 Digest SASL Mechanism May 2000 + + + cipher-opts + A list of ciphers that the server supports. This directive must be + present exactly once if "auth-conf" is offered in the + "qop-options" directive, in which case the "3des" and "des" modes + are mandatory-to-implement. The client MUST ignore unrecognized + options; if the client recognizes no option, it should abort the + authentication exchange. + + des + the Data Encryption Standard (DES) cipher [FIPS] in cipher + block chaining (CBC) mode with a 56 bit key. + + 3des + the "triple DES" cipher in CBC mode with EDE with the same key + for each E stage (aka "two keys mode") for a total key length + of 112 bits. + + rc4, rc4-40, rc4-56 + the RC4 cipher with a 128 bit, 40 bit, and 56 bit key, + respectively. + + auth-param This construct allows for future extensions; it may appear + more than once. The client MUST ignore any unrecognized + directives. + + For use as a SASL mechanism, note that the following changes are made + to "digest-challenge" from HTTP: the following Digest options (called + "directives" in HTTP terminology) are unused (i.e., MUST NOT be sent, + and MUST be ignored if received): + + opaque + domain + + The size of a digest-challenge MUST be less than 2048 bytes. + +2.1.2 Step Two + + The client makes note of the "digest-challenge" and then responds + with a string formatted and computed according to the rules for a + "digest-response" defined as follows: + + + + + + + + + + + +Leach & Newman Standards Track [Page 6] + +RFC 2831 Digest SASL Mechanism May 2000 + + + digest-response = 1#( username | realm | nonce | cnonce | + nonce-count | qop | digest-uri | response | + maxbuf | charset | cipher | authzid | + auth-param ) + + username = "username" "=" <"> username-value <"> + username-value = qdstr-val + cnonce = "cnonce" "=" <"> cnonce-value <"> + cnonce-value = qdstr-val + nonce-count = "nc" "=" nc-value + nc-value = 8LHEX + qop = "qop" "=" qop-value + digest-uri = "digest-uri" "=" <"> digest-uri-value <"> + digest-uri-value = serv-type "/" host [ "/" serv-name ] + serv-type = 1*ALPHA + host = 1*( ALPHA | DIGIT | "-" | "." ) + serv-name = host + response = "response" "=" response-value + response-value = 32LHEX + LHEX = "0" | "1" | "2" | "3" | + "4" | "5" | "6" | "7" | + "8" | "9" | "a" | "b" | + "c" | "d" | "e" | "f" + cipher = "cipher" "=" cipher-value + authzid = "authzid" "=" <"> authzid-value <"> + authzid-value = qdstr-val + + + username + The user's name in the specified realm, encoded according to the + value of the "charset" directive. This directive is required and + MUST be present exactly once; otherwise, authentication fails. + + realm + The realm containing the user's account. This directive is + required if the server provided any realms in the + "digest-challenge", in which case it may appear exactly once and + its value SHOULD be one of those realms. If the directive is + missing, "realm-value" will set to the empty string when computing + A1 (see below for details). + + nonce + The server-specified data string received in the preceding + digest-challenge. This directive is required and MUST be present + exactly once; otherwise, authentication fails. + + + + + + +Leach & Newman Standards Track [Page 7] + +RFC 2831 Digest SASL Mechanism May 2000 + + + cnonce + A client-specified data string which MUST be different each time a + digest-response is sent as part of initial authentication. The + cnonce-value is an opaque quoted string value provided by the + client and used by both client and server to avoid chosen + plaintext attacks, and to provide mutual authentication. The + security of the implementation depends on a good choice. It is + RECOMMENDED that it contain at least 64 bits of entropy. This + directive is required and MUST be present exactly once; otherwise, + authentication fails. + + nonce-count + The nc-value is the hexadecimal count of the number of requests + (including the current request) that the client has sent with the + nonce value in this request. For example, in the first request + sent in response to a given nonce value, the client sends + "nc=00000001". The purpose of this directive is to allow the + server to detect request replays by maintaining its own copy of + this count - if the same nc-value is seen twice, then the request + is a replay. See the description below of the construction of + the response value. This directive may appear at most once; if + multiple instances are present, the client should abort the + authentication exchange. + + qop + Indicates what "quality of protection" the client accepted. If + present, it may appear exactly once and its value MUST be one of + the alternatives in qop-options. If not present, it defaults to + "auth". These values affect the computation of the response. Note + that this is a single token, not a quoted list of alternatives. + + serv-type + Indicates the type of service, such as "www" for web service, + "ftp" for FTP service, "smtp" for mail delivery service, etc. The + service name as defined in the SASL profile for the protocol see + section 4 of [RFC 2222], registered in the IANA registry of + "service" elements for the GSSAPI host-based service name form + [RFC 2078]. + + host + The DNS host name or IP address for the service requested. The + DNS host name must be the fully-qualified canonical name of the + host. The DNS host name is the preferred form; see notes on server + processing of the digest-uri. + + + + + + + +Leach & Newman Standards Track [Page 8] + +RFC 2831 Digest SASL Mechanism May 2000 + + + serv-name + Indicates the name of the service if it is replicated. The service + is considered to be replicated if the client's service-location + process involves resolution using standard DNS lookup operations, + and if these operations involve DNS records (such as SRV, or MX) + which resolve one DNS name into a set of other DNS names. In this + case, the initial name used by the client is the "serv-name", and + the final name is the "host" component. For example, the incoming + mail service for "example.com" may be replicated through the use + of MX records stored in the DNS, one of which points at an SMTP + server called "mail3.example.com"; it's "serv-name" would be + "example.com", it's "host" would be "mail3.example.com". If the + service is not replicated, or the serv-name is identical to the + host, then the serv-name component MUST be omitted. + + digest-uri + Indicates the principal name of the service with which the client + wishes to connect, formed from the serv-type, host, and serv-name. + For example, the FTP service on "ftp.example.com" would have a + "digest-uri" value of "ftp/ftp.example.com"; the SMTP server from + the example above would have a "digest-uri" value of + "smtp/mail3.example.com/example.com". + + Servers SHOULD check that the supplied value is correct. This will + detect accidental connection to the incorrect server. It is also so + that clients will be trained to provide values that will work with + implementations that use a shared back-end authentication service + that can provide server authentication. + + The serv-type component should match the service being offered. The + host component should match one of the host names of the host on + which the service is running, or it's IP address. Servers SHOULD NOT + normally support the IP address form, because server authentication + by IP address is not very useful; they should only do so if the DNS + is unavailable or unreliable. The serv-name component should match + one of the service's configured service names. + + This directive may appear at most once; if multiple instances are + present, the client should abort the authentication exchange. + + Note: In the HTTP use of Digest authentication, the digest-uri is the + URI (usually a URL) of the resource requested -- hence the name of + the directive. + + response + A string of 32 hex digits computed as defined below, which proves + that the user knows a password. This directive is required and + MUST be present exactly once; otherwise, authentication fails. + + + +Leach & Newman Standards Track [Page 9] + +RFC 2831 Digest SASL Mechanism May 2000 + + + maxbuf + A number indicating the size of the largest buffer the client is + able to receive. If this directive is missing, the default value + is 65536. This directive may appear at most once; if multiple + instances are present, the server should abort the authentication + exchange. + + charset + This directive, if present, specifies that the client has used + UTF-8 encoding for the username and password. If not present, the + username and password must be encoded in ISO 8859-1 (of which + US-ASCII is a subset). The client should send this directive only + if the server has indicated it supports UTF-8. The directive is + needed for backwards compatibility with HTTP Digest, which only + supports ISO 8859-1. + + LHEX + 32 hex digits, where the alphabetic characters MUST be lower case, + because MD5 is not case insensitive. + + cipher + The cipher chosen by the client. This directive MUST appear + exactly once if "auth-conf" is negotiated; if required and not + present, authentication fails. + + authzid + The "authorization ID" as per RFC 2222, encoded in UTF-8. This + directive is optional. If present, and the authenticating user has + sufficient privilege, and the server supports it, then after + authentication the server will use this identity for making all + accesses and access checks. If the client specifies it, and the + server does not support it, then the response-value will be + incorrect, and authentication will fail. + + The size of a digest-response MUST be less than 4096 bytes. + +2.1.2.1 Response-value + + The definition of "response-value" above indicates the encoding for + its value -- 32 lower case hex characters. The following definitions + show how the value is computed. + + Although qop-value and components of digest-uri-value may be + case-insensitive, the case which the client supplies in step two is + preserved for the purpose of computing and verifying the + response-value. + + response-value = + + + +Leach & Newman Standards Track [Page 10] + +RFC 2831 Digest SASL Mechanism May 2000 + + + HEX( KD ( HEX(H(A1)), + { nonce-value, ":" nc-value, ":", + cnonce-value, ":", qop-value, ":", HEX(H(A2)) })) + + If authzid is specified, then A1 is + + + A1 = { H( { username-value, ":", realm-value, ":", passwd } ), + ":", nonce-value, ":", cnonce-value, ":", authzid-value } + + If authzid is not specified, then A1 is + + + A1 = { H( { username-value, ":", realm-value, ":", passwd } ), + ":", nonce-value, ":", cnonce-value } + + where + + passwd = *OCTET + + The "username-value", "realm-value" and "passwd" are encoded + according to the value of the "charset" directive. If "charset=UTF-8" + is present, and all the characters of either "username-value" or + "passwd" are in the ISO 8859-1 character set, then it must be + converted to ISO 8859-1 before being hashed. This is so that + authentication databases that store the hashed username, realm and + password (which is common) can be shared compatibly with HTTP, which + specifies ISO 8859-1. A sample implementation of this conversion is + in section 8. + + If the "qop" directive's value is "auth", then A2 is: + + A2 = { "AUTHENTICATE:", digest-uri-value } + + If the "qop" value is "auth-int" or "auth-conf" then A2 is: + + A2 = { "AUTHENTICATE:", digest-uri-value, + ":00000000000000000000000000000000" } + + Note that "AUTHENTICATE:" must be in upper case, and the second + string constant is a string with a colon followed by 32 zeros. + + These apparently strange values of A2 are for compatibility with + HTTP; they were arrived at by setting "Method" to "AUTHENTICATE" and + the hash of the entity body to zero in the HTTP digest calculation of + A2. + + Also, in the HTTP usage of Digest, several directives in the + + + +Leach & Newman Standards Track [Page 11] + +RFC 2831 Digest SASL Mechanism May 2000 + + + "digest-challenge" sent by the server have to be returned by the + client in the "digest-response". These are: + + opaque + algorithm + + These directives are not needed when Digest is used as a SASL + mechanism (i.e., MUST NOT be sent, and MUST be ignored if received). + +2.1.3 Step Three + + The server receives and validates the "digest-response". The server + checks that the nonce-count is "00000001". If it supports subsequent + authentication (see section 2.2), it saves the value of the nonce and + the nonce-count. It sends a message formatted as follows: + + response-auth = "rspauth" "=" response-value + + where response-value is calculated as above, using the values sent in + step two, except that if qop is "auth", then A2 is + + A2 = { ":", digest-uri-value } + + And if qop is "auth-int" or "auth-conf" then A2 is + + A2 = { ":", digest-uri-value, ":00000000000000000000000000000000" } + + Compared to its use in HTTP, the following Digest directives in the + "digest-response" are unused: + + nextnonce + qop + cnonce + nonce-count + +2.2 Subsequent Authentication + + If the client has previously authenticated to the server, and + remembers the values of username, realm, nonce, nonce-count, cnonce, + and qop that it used in that authentication, and the SASL profile for + a protocol permits an initial client response, then it MAY perform + "subsequent authentication", as defined in this section. + + + + + + + + + +Leach & Newman Standards Track [Page 12] + +RFC 2831 Digest SASL Mechanism May 2000 + + +2.2.1 Step one + + The client uses the values from the previous authentication and sends + an initial response with a string formatted and computed according to + the rules for a "digest-response", as defined above, but with a + nonce-count one greater than used in the last "digest-response". + +2.2.2 Step Two + + The server receives the "digest-response". If the server does not + support subsequent authentication, then it sends a + "digest-challenge", and authentication proceeds as in initial + authentication. If the server has no saved nonce and nonce-count from + a previous authentication, then it sends a "digest-challenge", and + authentication proceeds as in initial authentication. Otherwise, the + server validates the "digest-response", checks that the nonce-count + is one greater than that used in the previous authentication using + that nonce, and saves the new value of nonce-count. + + If the response is invalid, then the server sends a + "digest-challenge", and authentication proceeds as in initial + authentication (and should be configurable to log an authentication + failure in some sort of security audit log, since the failure may be + a symptom of an attack). The nonce-count MUST NOT be incremented in + this case: to do so would allow a denial of service attack by sending + an out-of-order nonce-count. + + If the response is valid, the server MAY choose to deem that + authentication has succeeded. However, if it has been too long since + the previous authentication, or for any other reason, the server MAY + send a new "digest-challenge" with a new value for nonce. The + challenge MAY contain a "stale" directive with value "true", which + says that the client may respond to the challenge using the password + it used in the previous response; otherwise, the client must solicit + the password anew from the user. This permits the server to make sure + that the user has presented their password recently. (The directive + name refers to the previous nonce being stale, not to the last use of + the password.) Except for the handling of "stale", after sending the + "digest-challenge" authentication proceeds as in the case of initial + authentication. + +2.3 Integrity Protection + + If the server offered "qop=auth-int" and the client responded + "qop=auth-int", then subsequent messages, up to but not including the + next subsequent authentication, between the client and the server + + + + + +Leach & Newman Standards Track [Page 13] + +RFC 2831 Digest SASL Mechanism May 2000 + + + MUST be integrity protected. Using as a base session key the value of + H(A1) as defined above the client and server calculate a pair of + message integrity keys as follows. + + The key for integrity protecting messages from client to server is: + + Kic = MD5({H(A1), + "Digest session key to client-to-server signing key magic constant"}) + + The key for integrity protecting messages from server to client is: + + Kis = MD5({H(A1), + "Digest session key to server-to-client signing key magic constant"}) + + where MD5 is as specified in [RFC 1321]. If message integrity is + negotiated, a MAC block for each message is appended to the message. + The MAC block is 16 bytes: the first 10 bytes of the HMAC-MD5 [RFC + 2104] of the message, a 2-byte message type number in network byte + order with value 1, and the 4-byte sequence number in network byte + order. The message type is to allow for future extensions such as + rekeying. + + MAC(Ki, SeqNum, msg) = (HMAC(Ki, {SeqNum, msg})[0..9], 0x0001, + SeqNum) + + where Ki is Kic for messages sent by the client and Kis for those + sent by the server. The sequence number is initialized to zero, and + incremented by one for each message sent. + + Upon receipt, MAC(Ki, SeqNum, msg) is computed and compared with the + received value; the message is discarded if they differ. + +2.4 Confidentiality Protection + + If the server sent a "cipher-opts" directive and the client responded + with a "cipher" directive, then subsequent messages between the + client and the server MUST be confidentiality protected. Using as a + base session key the value of H(A1) as defined above the client and + server calculate a pair of message integrity keys as follows. + + The key for confidentiality protecting messages from client to server + is: + + Kcc = MD5({H(A1)[0..n], + "Digest H(A1) to client-to-server sealing key magic constant"}) + + The key for confidentiality protecting messages from server to client + is: + + + +Leach & Newman Standards Track [Page 14] + +RFC 2831 Digest SASL Mechanism May 2000 + + + Kcs = MD5({H(A1)[0..n], + "Digest H(A1) to server-to-client sealing key magic constant"}) + + where MD5 is as specified in [RFC 1321]. For cipher "rc4-40" n is 5; + for "rc4-56" n is 7; for the rest n is 16. The key for the "rc-*" + ciphers is all 16 bytes of Kcc or Kcs; the key for "des" is the first + 7 bytes; the key for "3des" is the first 14 bytes. The IV for "des" + and "3des" is the last 8 bytes of Kcc or Kcs. + + If message confidentiality is negotiated, each message is encrypted + with the chosen cipher and a MAC block is appended to the message. + + The MAC block is a variable length padding prefix followed by 16 + bytes formatted as follows: the first 10 bytes of the HMAC-MD5 [RFC + 2104] of the message, a 2-byte message type number in network byte + order with value 1, and the 4-byte sequence number in network byte + order. If the blocksize of the chosen cipher is not 1 byte, the + padding prefix is one or more octets each containing the number of + padding bytes, such that total length of the encrypted part of the + message is a multiple of the blocksize. The padding and first 10 + bytes of the MAC block are encrypted along with the message. + + SEAL(Ki, Kc, SeqNum, msg) = + {CIPHER(Kc, {msg, pad, HMAC(Ki, {SeqNum, msg})[0..9])}), 0x0001, + SeqNum} + + where CIPHER is the chosen cipher, Ki and Kc are Kic and Kcc for + messages sent by the client and Kis and Kcs for those sent by the + server. The sequence number is initialized to zero, and incremented + by one for each message sent. + + Upon receipt, the message is decrypted, HMAC(Ki, {SeqNum, msg}) is + computed and compared with the received value; the message is + discarded if they differ. + +3 Security Considerations + +3.1 Authentication of Clients using Digest Authentication + + Digest Authentication does not provide a strong authentication + mechanism, when compared to public key based mechanisms, for example. + However, since it prevents chosen plaintext attacks, it is stronger + than (e.g.) CRAM-MD5, which has been proposed for use with LDAP [10], + POP and IMAP (see RFC 2195 [9]). It is intended to replace the much + weaker and even more dangerous use of plaintext passwords; however, + since it is still a password based mechanism it avoids some of the + potential deployabilty issues with public-key, OTP or similar + mechanisms. + + + +Leach & Newman Standards Track [Page 15] + +RFC 2831 Digest SASL Mechanism May 2000 + + + Digest Authentication offers no confidentiality protection beyond + protecting the actual password. All of the rest of the challenge and + response are available to an eavesdropper, including the user's name + and authentication realm. + +3.2 Comparison of Digest with Plaintext Passwords + + The greatest threat to the type of transactions for which these + protocols are used is network snooping. This kind of transaction + might involve, for example, online access to a mail service whose use + is restricted to paying subscribers. With plaintext password + authentication an eavesdropper can obtain the password of the user. + This not only permits him to access anything in the database, but, + often worse, will permit access to anything else the user protects + with the same password. + +3.3 Replay Attacks + + Replay attacks are defeated if the client or the server chooses a + fresh nonce for each authentication, as this specification requires. + +3.4 Online dictionary attacks + + If the attacker can eavesdrop, then it can test any overheard + nonce/response pairs against a (potentially very large) list of + common words. Such a list is usually much smaller than the total + number of possible passwords. The cost of computing the response for + each password on the list is paid once for each challenge. + + The server can mitigate this attack by not allowing users to select + passwords that are in a dictionary. + +3.5 Offline dictionary attacks + + If the attacker can choose the challenge, then it can precompute the + possible responses to that challenge for a list of common words. Such + a list is usually much smaller than the total number of possible + passwords. The cost of computing the response for each password on + the list is paid just once. + + Offline dictionary attacks are defeated if the client chooses a fresh + nonce for each authentication, as this specification requires. + + + + + + + + + +Leach & Newman Standards Track [Page 16] + +RFC 2831 Digest SASL Mechanism May 2000 + + +3.6 Man in the Middle + + Digest authentication is vulnerable to "man in the middle" (MITM) + attacks. Clearly, a MITM would present all the problems of + eavesdropping. But it also offers some additional opportunities to + the attacker. + + A possible man-in-the-middle attack would be to substitute a weaker + qop scheme for the one(s) sent by the server; the server will not be + able to detect this attack. For this reason, the client should always + use the strongest scheme that it understands from the choices + offered, and should never choose a scheme that does not meet its + minimum requirements. + +3.7 Chosen plaintext attacks + + A chosen plaintext attack is where a MITM or a malicious server can + arbitrarily choose the challenge that the client will use to compute + the response. The ability to choose the challenge is known to make + cryptanalysis much easier [8]. + + However, Digest does not permit the attack to choose the challenge as + long as the client chooses a fresh nonce for each authentication, as + this specification requires. + +3.8 Spoofing by Counterfeit Servers + + If a user can be led to believe that she is connecting to a host + containing information protected by a password she knows, when in + fact she is connecting to a hostile server, then the hostile server + can obtain challenge/response pairs where it was able to partly + choose the challenge. There is no known way that this can be + exploited. + +3.9 Storing passwords + + Digest authentication requires that the authenticating agent (usually + the server) store some data derived from the user's name and password + in a "password file" associated with a given realm. Normally this + might contain pairs consisting of username and H({ username-value, + ":", realm-value, ":", passwd }), which is adequate to compute H(A1) + as described above without directly exposing the user's password. + + The security implications of this are that if this password file is + compromised, then an attacker gains immediate access to documents on + the server using this realm. Unlike, say a standard UNIX password + file, this information need not be decrypted in order to access + documents in the server realm associated with this file. On the other + + + +Leach & Newman Standards Track [Page 17] + +RFC 2831 Digest SASL Mechanism May 2000 + + + hand, decryption, or more likely a brute force attack, would be + necessary to obtain the user's password. This is the reason that the + realm is part of the digested data stored in the password file. It + means that if one Digest authentication password file is compromised, + it does not automatically compromise others with the same username + and password (though it does expose them to brute force attack). + + There are two important security consequences of this. First the + password file must be protected as if it contained plaintext + passwords, because for the purpose of accessing documents in its + realm, it effectively does. + + A second consequence of this is that the realm string should be + unique among all realms that any single user is likely to use. In + particular a realm string should include the name of the host doing + the authentication. + +3.10 Multiple realms + + Use of multiple realms may mean both that compromise of a the + security database for a single realm does not compromise all + security, and that there are more things to protect in order to keep + the whole system secure. + +3.11 Summary + + By modern cryptographic standards Digest Authentication is weak, + compared to (say) public key based mechanisms. But for a large range + of purposes it is valuable as a replacement for plaintext passwords. + Its strength may vary depending on the implementation. + +4 Example + + This example shows the use of the Digest SASL mechanism with the + IMAP4 AUTHENTICATE command [RFC 2060]. + + In this example, "C:" and "S:" represent a line sent by the client or + server respectively including a CRLF at the end. Linebreaks and + indentation within a "C:" or "S:" are editorial and not part of the + protocol. The password in this example was "secret". Note that the + base64 encoding of the challenges and responses is part of the IMAP4 + AUTHENTICATE command, not part of the Digest specification itself. + + S: * OK elwood.innosoft.com PMDF IMAP4rev1 V6.0-9 + C: c CAPABILITY + S: * CAPABILITY IMAP4 IMAP4rev1 ACL LITERAL+ NAMESPACE QUOTA + UIDPLUS AUTH=CRAM-MD5 AUTH=DIGEST-MD5 AUTH=PLAIN + S: c OK Completed + + + +Leach & Newman Standards Track [Page 18] + +RFC 2831 Digest SASL Mechanism May 2000 + + + C: a AUTHENTICATE DIGEST-MD5 + S: + cmVhbG09ImVsd29vZC5pbm5vc29mdC5jb20iLG5vbmNlPSJPQTZNRzl0 + RVFHbTJoaCIscW9wPSJhdXRoIixhbGdvcml0aG09bWQ1LXNlc3MsY2hh + cnNldD11dGYtOA== + C: Y2hhcnNldD11dGYtOCx1c2VybmFtZT0iY2hyaXMiLHJlYWxtPSJlbHdvb2 + QuaW5ub3NvZnQuY29tIixub25jZT0iT0E2TUc5dEVRR20yaGgiLG5jPTAw + MDAwMDAxLGNub25jZT0iT0E2TUhYaDZWcVRyUmsiLGRpZ2VzdC11cmk9Im + ltYXAvZWx3b29kLmlubm9zb2Z0LmNvbSIscmVzcG9uc2U9ZDM4OGRhZDkw + ZDRiYmQ3NjBhMTUyMzIxZjIxNDNhZjcscW9wPWF1dGg= + S: + cnNwYXV0aD1lYTQwZjYwMzM1YzQyN2I1NTI3Yjg0ZGJhYmNkZmZmZA== + C: + S: a OK User logged in + --- + + The base64-decoded version of the SASL exchange is: + + S: realm="elwood.innosoft.com",nonce="OA6MG9tEQGm2hh",qop="auth", + algorithm=md5-sess,charset=utf-8 + C: charset=utf-8,username="chris",realm="elwood.innosoft.com", + nonce="OA6MG9tEQGm2hh",nc=00000001,cnonce="OA6MHXh6VqTrRk", + digest-uri="imap/elwood.innosoft.com", + response=d388dad90d4bbd760a152321f2143af7,qop=auth + S: rspauth=ea40f60335c427b5527b84dbabcdfffd + + The password in this example was "secret". + + This example shows the use of the Digest SASL mechanism with the + ACAP, using the same notational conventions and password as in the + previous example. Note that ACAP does not base64 encode and uses + fewer round trips that IMAP4. + + S: * ACAP (IMPLEMENTATION "Test ACAP server") (SASL "CRAM-MD5" + "DIGEST-MD5" "PLAIN") + C: a AUTHENTICATE "DIGEST-MD5" + S: + {94} + S: realm="elwood.innosoft.com",nonce="OA9BSXrbuRhWay",qop="auth", + algorithm=md5-sess,charset=utf-8 + C: {206} + C: charset=utf-8,username="chris",realm="elwood.innosoft.com", + nonce="OA9BSXrbuRhWay",nc=00000001,cnonce="OA9BSuZWMSpW8m", + digest-uri="acap/elwood.innosoft.com", + response=6084c6db3fede7352c551284490fd0fc,qop=auth + S: a OK (SASL {40} + S: rspauth=2f0b3d7c3c2e486600ef710726aa2eae) "AUTHENTICATE + Completed" + --- + + + + + +Leach & Newman Standards Track [Page 19] + +RFC 2831 Digest SASL Mechanism May 2000 + + + The server uses the values of all the directives, plus knowledge of + the users password (or the hash of the user's name, server's realm + and the user's password) to verify the computations above. If they + check, then the user has authenticated. + +5 References + + [Digest] Franks, J., et al., "HTTP Authentication: Basic and Digest + Access Authentication", RFC 2617, June 1999. + + [ISO-8859] ISO-8859. International Standard--Information Processing-- + 8-bit Single-Byte Coded Graphic Character Sets -- + Part 1: Latin alphabet No. 1, ISO-8859-1:1987. + Part 2: Latin alphabet No. 2, ISO-8859-2, 1987. + Part 3: Latin alphabet No. 3, ISO-8859-3, 1988. + Part 4: Latin alphabet No. 4, ISO-8859-4, 1988. + Part 5: Latin/Cyrillic alphabet, ISO-8859-5, 1988. + Part 6: Latin/Arabic alphabet, ISO-8859-6, 1987. + Part 7: Latin/Greek alphabet, ISO-8859-7, 1987. + Part 8: Latin/Hebrew alphabet, ISO-8859-8, 1988. + Part 9: Latin alphabet No. 5, ISO-8859-9, 1990. + + [RFC 822] Crocker, D., "Standard for The Format of ARPA Internet + Text Messages," STD 11, RFC 822, August 1982. + + [RFC 1321] Rivest, R., "The MD5 Message-Digest Algorithm", RFC 1321, + April 1992. + + [RFC 2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII Text", + RFC 2047, November 1996. + + [RFC 2052] Gulbrandsen, A. and P. Vixie, "A DNS RR for specifying the + location of services (DNS SRV)", RFC 2052, October 1996. + + [RFC 2060] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [RFC 2104] Krawczyk, H., Bellare, M. and R. Canetti, "HMAC: Keyed- + Hashing for Message Authentication", RFC 2104, February + 1997. + + [RFC 2195] Klensin, J., Catoe, R. and P. Krumviede, "IMAP/POP + AUTHorize Extension for Simple Challenge/Response", RFC + 2195, September 1997. + + + + + + +Leach & Newman Standards Track [Page 20] + +RFC 2831 Digest SASL Mechanism May 2000 + + + [RFC 2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC 2222] Myers, J., "Simple Authentication and Security Layer + (SASL)", RFC 2222, October 1997. + + [USASCII] US-ASCII. Coded Character Set - 7-Bit American Standard + Code for Information Interchange. Standard ANSI X3.4-1986, + ANSI, 1986. + +6 Authors' Addresses + + Paul Leach + Microsoft + 1 Microsoft Way + Redmond, WA 98052 + + EMail: paulle@microsoft.com + + + Chris Newman + Innosoft International, Inc. + 1050 Lakes Drive + West Covina, CA 91790 USA + + EMail: chris.newman@innosoft.com + +7 ABNF + + What follows is the definition of the notation as is used in the + HTTP/1.1 specification (RFC 2616) and the HTTP authentication + specification (RFC 2617); it is reproduced here for ease of + reference. Since it is intended that a single Digest implementation + can support both HTTP and SASL-based protocols, the same notation is + used in both to facilitate comparison and prevention of unwanted + differences. Since it is cut-and-paste from the HTTP specifications, + not all productions may be used in this specification. It is also not + quite legal ABNF; again, the errors were copied from the HTTP + specifications. + +7.1 Augmented BNF + + All of the mechanisms specified in this document are described in + both prose and an augmented Backus-Naur Form (BNF) similar to that + used by RFC 822 [RFC 822]. Implementers will need to be familiar with + the notation in order to understand this specification. + + + + + +Leach & Newman Standards Track [Page 21] + +RFC 2831 Digest SASL Mechanism May 2000 + + + The augmented BNF includes the following constructs: + + name = definition + The name of a rule is simply the name itself (without any + enclosing "<" and ">") and is separated from its definition by the + equal "=" character. White space is only significant in that + indentation of continuation lines is used to indicate a rule + definition that spans more than one line. Certain basic rules are + in uppercase, such as SP, LWS, HT, CRLF, DIGIT, ALPHA, etc. Angle + brackets are used within definitions whenever their presence will + facilitate discerning the use of rule names. + + "literal" + Quotation marks surround literal text. Unless stated otherwise, + the text is case-insensitive. + + rule1 | rule2 + Elements separated by a bar ("|") are alternatives, e.g., "yes | + no" will accept yes or no. + + (rule1 rule2) + Elements enclosed in parentheses are treated as a single element. + Thus, "(elem (foo | bar) elem)" allows the token sequences + "elem foo elem" and "elem bar elem". + + *rule + The character "*" preceding an element indicates repetition. The + full form is "*element" indicating at least and at most + occurrences of element. Default values are 0 and infinity so + that "*(element)" allows any number, including zero; "1*element" + requires at least one; and "1*2element" allows one or two. + + [rule] + Square brackets enclose optional elements; "[foo bar]" is + equivalent to "*1(foo bar)". + + N rule + Specific repetition: "(element)" is equivalent to + "*(element)"; that is, exactly occurrences of (element). + Thus 2DIGIT is a 2-digit number, and 3ALPHA is a string of three + alphabetic characters. + + #rule + A construct "#" is defined, similar to "*", for defining lists of + elements. The full form is "#element" indicating at least + and at most elements, each separated by one or more commas + (",") and OPTIONAL linear white space (LWS). This makes the usual + form of lists very easy; a rule such as + + + +Leach & Newman Standards Track [Page 22] + +RFC 2831 Digest SASL Mechanism May 2000 + + + ( *LWS element *( *LWS "," *LWS element )) + can be shown as + 1#element + Wherever this construct is used, null elements are allowed, but do + not contribute to the count of elements present. That is, + "(element), , (element) " is permitted, but counts as only two + elements. Therefore, where at least one element is required, at + least one non-null element MUST be present. Default values are 0 + and infinity so that "#element" allows any number, including zero; + "1#element" requires at least one; and "1#2element" allows one or + two. + + ; comment + A semi-colon, set off some distance to the right of rule text, + starts a comment that continues to the end of line. This is a + simple way of including useful notes in parallel with the + specifications. + + implied *LWS + The grammar described by this specification is word-based. Except + where noted otherwise, linear white space (LWS) can be included + between any two adjacent words (token or quoted-string), and + between adjacent words and separators, without changing the + interpretation of a field. At least one delimiter (LWS and/or + separators) MUST exist between any two tokens (for the definition + of "token" below), since they would otherwise be interpreted as a + single token. + +7.2 Basic Rules + + The following rules are used throughout this specification to + describe basic parsing constructs. The US-ASCII coded character set + is defined by ANSI X3.4-1986 [USASCII]. + + OCTET = + CHAR = + UPALPHA = + LOALPHA = + ALPHA = UPALPHA | LOALPHA + DIGIT = + CTL = + CR = + LF = + SP = + HT = + <"> = + CRLF = CR LF + + + +Leach & Newman Standards Track [Page 23] + +RFC 2831 Digest SASL Mechanism May 2000 + + + + All linear white space, including folding, has the same semantics as + SP. A recipient MAY replace any linear white space with a single SP + before interpreting the field value or forwarding the message + downstream. + + LWS = [CRLF] 1*( SP | HT ) + + The TEXT rule is only used for descriptive field contents and values + that are not intended to be interpreted by the message parser. Words + of *TEXT MAY contain characters from character sets other than + ISO-8859-1 [ISO 8859] only when encoded according to the rules of RFC + 2047 [RFC 2047]. + + TEXT = + + A CRLF is allowed in the definition of TEXT only as part of a header + field continuation. It is expected that the folding LWS will be + replaced with a single SP before interpretation of the TEXT value. + + Hexadecimal numeric characters are used in several protocol elements. + + HEX = "A" | "B" | "C" | "D" | "E" | "F" + | "a" | "b" | "c" | "d" | "e" | "f" | DIGIT + + Many HTTP/1.1 header field values consist of words separated by LWS + or special characters. These special characters MUST be in a quoted + string to be used within a parameter value. + + token = 1* + separators = "(" | ")" | "<" | ">" | "@" + | "," | ";" | ":" | "\" | <"> + | "/" | "[" | "]" | "?" | "=" + | "{" | "}" | SP | HT + + A string of text is parsed as a single word if it is quoted using + double-quote marks. + + quoted-string = ( <"> qdstr-val <"> ) + qdstr-val = *( qdtext | quoted-pair ) + qdtext = > + + Note that LWS is NOT implicit between the double-quote marks (<">) + surrounding a qdstr-val and the qdstr-val; any LWS will be considered + part of the qdstr-val. This is also the case for quotation marks + surrounding any other construct. + + + + +Leach & Newman Standards Track [Page 24] + +RFC 2831 Digest SASL Mechanism May 2000 + + + The backslash character ("\") MAY be used as a single-character + quoting mechanism only within qdstr-val and comment constructs. + + quoted-pair = "\" CHAR + + The value of this construct is CHAR. Note that an effect of this rule + is that backslash must be quoted. + +8 Sample Code + + The sample implementation in [Digest] also applies to DIGEST-MD5. + + The following code implements the conversion from UTF-8 to 8859-1 if + necessary. + + /* if the string is entirely in the 8859-1 subset of UTF-8, then + * translate to 8859-1 prior to MD5 + */ + void MD5_UTF8_8859_1(MD5_CTX *ctx, const unsigned char *base, + int len) + { + const unsigned char *scan, *end; + unsigned char cbuf; + + end = base + len; + for (scan = base; scan < end; ++scan) { + if (*scan > 0xC3) break; /* abort if outside 8859-1 */ + if (*scan >= 0xC0 && *scan <= 0xC3) { + if (++scan == end || *scan < 0x80 || *scan > 0xBF) + break; + } + } + /* if we found a character outside 8859-1, don't alter string + */ + if (scan < end) { + MD5Update(ctx, base, len); + return; + } + + /* convert to 8859-1 prior to applying hash + */ + do { + for (scan = base; scan < end && *scan < 0xC0; ++scan) + ; + if (scan != base) MD5Update(ctx, base, scan - base); + if (scan + 1 >= end) break; + cbuf = ((scan[0] & 0x3) << 6) | (scan[1] & 0x3f); + MD5Update(ctx, &cbuf, 1); + + + +Leach & Newman Standards Track [Page 25] + +RFC 2831 Digest SASL Mechanism May 2000 + + + base = scan + 2; + } while (base < end); + } + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Leach & Newman Standards Track [Page 26] + +RFC 2831 Digest SASL Mechanism May 2000 + + +9 Full Copyright Statement + + Copyright (C) The Internet Society (2000). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Leach & Newman Standards Track [Page 27] + diff --git a/docs/rfcs/rfc2971.IMAP4_ID_extension.txt b/docs/rfcs/rfc2971.IMAP4_ID_extension.txt new file mode 100644 index 0000000..9e7264d --- /dev/null +++ b/docs/rfcs/rfc2971.IMAP4_ID_extension.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group T. Showalter +Request for Comments: 2971 Mirapoint, Inc. +Category: Standards Track October 2000 + + + IMAP4 ID extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2000). All Rights Reserved. + +Abstract + + The ID extension to the Internet Message Access Protocol - Version + 4rev1 (IMAP4rev1) protocol allows the server and client to exchange + identification information on their implementation in order to make + bug reports and usage statistics more complete. + +1. Introduction + + The IMAP4rev1 protocol described in [IMAP4rev1] provides a method for + accessing remote mail stores, but it provides no facility to + advertise what program a client or server uses to provide service. + This makes it difficult for implementors to get complete bug reports + from users, as it is frequently difficult to know what client or + server is in use. + + Additionally, some sites may wish to assemble usage statistics based + on what clients are used, but in an an environment where users are + permitted to obtain and maintain their own clients this is difficult + to accomplish. + + The ID command provides a facility to advertise information on what + programs are being used along with contact information (should bugs + ever occur). + + + + + + + + +Showalter Standards Track [Page 1] + +RFC 2971 IMAP4 ID extension October 2000 + + +2. Conventions Used in this Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [KEYWORDS]. + + The conventions used in this document are the same as specified in + [IMAP4rev1]. In examples, "C:" and "S:" indicate lines sent by the + client and server respectively. Line breaks have been inserted for + readability. + +3. Specification + + The sole purpose of the ID extension is to enable clients and servers + to exchange information on their implementations for the purposes of + statistical analysis and problem determination. + + This information is be submitted to a server by any client wishing to + provide information for statistical purposes, provided the server + advertises its willingness to take the information with the atom "ID" + included in the list of capabilities returned by the CAPABILITY + command. + + Implementations MUST NOT make operational changes based on the data + sent as part of the ID command or response. The ID command is for + human consumption only, and is not to be used in improving the + performance of clients or servers. + + This includes, but is not limited to, the following: + + Servers MUST NOT attempt to work around client bugs by using + information from the ID command. Clients MUST NOT attempt to work + around server bugs based on the ID response. + + Servers MUST NOT provide features to a client or otherwise + optimize for a particular client by using information from the ID + command. Clients MUST NOT provide features to a server or + otherwise optimize for a particular server based on the ID + response. + + Servers MUST NOT deny access to or refuse service for a client + based on information from the ID command. Clients MUST NOT refuse + to operate or limit their operation with a server based on the ID + response. + + + + + + + +Showalter Standards Track [Page 2] + +RFC 2971 IMAP4 ID extension October 2000 + + + Rationale: It is imperative that this extension not supplant IMAP's + CAPABILITY mechanism with a ad-hoc approach where implementations + guess each other's features based on who they claim to be. + + Implementations MUST NOT send false information in an ID command. + + Implementations MAY send less information than they have available or + no information at all. Such behavior may be useful to preserve user + privacy. See Security Considerations, section 7. + +3.1. ID Command + + Arguments: client parameter list or NIL + + Responses: OPTIONAL untagged response: ID + + Result: OK identification information accepted + BAD command unknown or arguments invalid + + Implementation identification information is sent by the client with + the ID command. + + This command is valid in any state. + + The information sent is in the form of a list of field/value pairs. + Fields are permitted to be any IMAP4 string, and values are permitted + to be any IMAP4 string or NIL. A value of NIL indicates that the + client can not or will not specify this information. The client may + also send NIL instead of the list, indicating that it wants to send + no information, but would still accept a server response. + + The available fields are defined in section 3.3. + + Example: C: a023 ID ("name" "sodr" "version" "19.34" "vendor" + "Pink Floyd Music Limited") + S: * ID NIL + S: a023 OK ID completed + +3.2. ID Response + + Contents: server parameter list + + In response to an ID command issued by the client, the server replies + with a tagged response containing information on its implementation. + The format is the same as the client list. + + + + + + +Showalter Standards Track [Page 3] + +RFC 2971 IMAP4 ID extension October 2000 + + + Example: C: a042 ID NIL + S: * ID ("name" "Cyrus" "version" "1.5" "os" "sunos" + "os-version" "5.5" "support-url" + "mailto:cyrus-bugs+@andrew.cmu.edu") + S: a042 OK ID command completed + + A server MUST send a tagged ID response to an ID command. However, a + server MAY send NIL in place of the list. + +3.3. Defined Field Values + + Any string may be sent as a field, but the following are defined to + describe certain values that might be sent. Implementations are free + to send none, any, or all of these. Strings are not case-sensitive. + Field strings MUST NOT be longer than 30 octets. Value strings MUST + NOT be longer than 1024 octets. Implementations MUST NOT send more + than 30 field-value pairs. + + name Name of the program + version Version number of the program + os Name of the operating system + os-version Version of the operating system + vendor Vendor of the client/server + support-url URL to contact for support + address Postal address of contact/vendor + date Date program was released, specified as a date-time + in IMAP4rev1 + command Command used to start the program + arguments Arguments supplied on the command line, if any + if any + environment Description of environment, i.e., UNIX environment + variables or Windows registry settings + + Implementations MUST NOT use contact information to submit automatic + bug reports. Implementations may include information from an ID + response in a report automatically prepared, but are prohibited from + sending the report without user authorization. + + It is preferable to find the name and version of the underlying + operating system at runtime in cases where this is possible. + + Information sent via an ID response may violate user privacy. See + Security Considerations, section 7. + + Implementations MUST NOT send the same field name more than once. + + + + + + +Showalter Standards Track [Page 4] + +RFC 2971 IMAP4 ID extension October 2000 + + +4. Formal Syntax + + This syntax is intended to augment the grammar specified in + [IMAP4rev1] in order to provide for the ID command. This + specification uses the augmented Backus-Naur Form (BNF) notation as + used in [IMAP4rev1]. + + command_any ::= "CAPABILITY" / "LOGOUT" / "NOOP" / x_command / id + ;; adds id command to command_any in [IMAP4rev1] + + id ::= "ID" SPACE id_params_list + + id_response ::= "ID" SPACE id_params_list + + id_params_list ::= "(" #(string SPACE nstring) ")" / nil + ;; list of field value pairs + + response_data ::= "*" SPACE (resp_cond_state / resp_cond_bye / + mailbox_data / message_data / capability_data / id_response) + +5. Use of the ID extension with Firewalls and Other Intermediaries + + There exist proxies, firewalls, and other intermediary systems that + can intercept an IMAP session and make changes to the data exchanged + in the session. Such intermediaries are not anticipated by the IMAP4 + protocol design and are not within the scope of the IMAP4 standard. + However, in order for the ID command to be useful in the presence of + such intermediaries, those intermediaries need to take special note + of the ID command and response. In particular, if an intermediary + changes any part of the IMAP session it must also change the ID + command to advertise its presence. + + A firewall MAY act to block transmission of specific information + fields in the ID command and response that it believes reveal + information that could expose a security vulnerability. However, a + firewall SHOULD NOT disable the extension, when present, entirely, + and SHOULD NOT unconditionally remove either the client or server + list. + + Finally, it should be noted that a firewall, when handling a + CAPABILITY response, MUST NOT allow the names of extensions to be + returned to the client that the firewall has no knowledge of. + + + + + + + + + +Showalter Standards Track [Page 5] + +RFC 2971 IMAP4 ID extension October 2000 + + +6. References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, March 1997. + + [IMAP4rev1] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, October 1996. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet + Text Messages", STD 11, RFC 822, August 1982. + +7. Security Considerations + + This extension has the danger of violating the privacy of users if + misused. Clients and servers should notify users that they implement + and enable the ID command. + + It is highly desirable that implementations provide a method of + disabling ID support, perhaps by not sending ID at all, or by sending + NIL as the argument to the ID command or response. + + Implementors must exercise extreme care in adding fields sent as part + of an ID command or response. Some fields, including a processor ID + number, Ethernet address, or other unique (or mostly unique) + identifier allow tracking of users in ways that violate user privacy + expectations. + + Having implementation information of a given client or server may + make it easier for an attacker to gain unauthorized access due to + security holes. + + Since this command includes arbitrary data and does not require the + user to authenticate, server implementations are cautioned to guard + against an attacker sending arbitrary garbage data in order to fill + up the ID log. In particular, if a server naively logs each ID + command to disk without inspecting it, an attacker can simply fire up + thousands of connections and send a few kilobytes of random data. + Servers have to guard against this. Methods include truncating + abnormally large responses; collating responses by storing only a + single copy, then keeping a counter of the number of times that + response has been seen; keeping only particularly interesting parts + of responses; and only logging responses of users who actually log + in. + + Security is affected by firewalls which modify the IMAP protocol + stream; see section 5, Use of the ID Extension with Firewalls and + Other Intermediaries, for more information. + + + + +Showalter Standards Track [Page 6] + +RFC 2971 IMAP4 ID extension October 2000 + + +8. Author's Address + + Tim Showalter + Mirapoint, Inc. + 909 Hermosa Ct. + Sunnyvale, CA 94095 + + EMail: tjs@mirapoint.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Showalter Standards Track [Page 7] + +RFC 2971 IMAP4 ID extension October 2000 + + +9. Full Copyright Statement + + Copyright (C) The Internet Society (2000). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Showalter Standards Track [Page 8] + diff --git a/docs/rfcs/rfc3028.Sieve_Mail_filtering_language.txt b/docs/rfcs/rfc3028.Sieve_Mail_filtering_language.txt new file mode 100644 index 0000000..d909a54 --- /dev/null +++ b/docs/rfcs/rfc3028.Sieve_Mail_filtering_language.txt @@ -0,0 +1,2019 @@ + + + + + + +Network Working Group T. Showalter +Request for Comments: 3028 Mirapoint, Inc. +Category: Standards Track January 2001 + + + Sieve: A Mail Filtering Language + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2001). All Rights Reserved. + +Abstract + + This document describes a language for filtering e-mail messages at + time of final delivery. It is designed to be implementable on either + a mail client or mail server. It is meant to be extensible, simple, + and independent of access protocol, mail architecture, and operating + system. It is suitable for running on a mail server where users may + not be allowed to execute arbitrary programs, such as on black box + Internet Message Access Protocol (IMAP) servers, as it has no + variables, loops, or ability to shell out to external programs. + +Table of Contents + + 1. Introduction ........................................... 3 + 1.1. Conventions Used in This Document ..................... 4 + 1.2. Example mail messages ................................. 4 + 2. Design ................................................. 5 + 2.1. Form of the Language .................................. 5 + 2.2. Whitespace ............................................ 5 + 2.3. Comments .............................................. 6 + 2.4. Literal Data .......................................... 6 + 2.4.1. Numbers ............................................... 6 + 2.4.2. Strings ............................................... 7 + 2.4.2.1. String Lists .......................................... 7 + 2.4.2.2. Headers ............................................... 8 + 2.4.2.3. Addresses ............................................. 8 + 2.4.2.4. MIME Parts ............................................ 9 + 2.5. Tests ................................................. 9 + 2.5.1. Test Lists ............................................ 9 + + + +Showalter Standards Track [Page 1] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + 2.6. Arguments ............................................. 9 + 2.6.1. Positional Arguments .................................. 9 + 2.6.2. Tagged Arguments ...................................... 10 + 2.6.3. Optional Arguments .................................... 10 + 2.6.4. Types of Arguments .................................... 10 + 2.7. String Comparison ..................................... 11 + 2.7.1. Match Type ............................................ 11 + 2.7.2. Comparisons Across Character Sets ..................... 12 + 2.7.3. Comparators ........................................... 12 + 2.7.4. Comparisons Against Addresses ......................... 13 + 2.8. Blocks ................................................ 14 + 2.9. Commands .............................................. 14 + 2.10. Evaluation ............................................ 15 + 2.10.1. Action Interaction .................................... 15 + 2.10.2. Implicit Keep ......................................... 15 + 2.10.3. Message Uniqueness in a Mailbox ....................... 15 + 2.10.4. Limits on Numbers of Actions .......................... 16 + 2.10.5. Extensions and Optional Features ...................... 16 + 2.10.6. Errors ................................................ 17 + 2.10.7. Limits on Execution ................................... 17 + 3. Control Commands ....................................... 17 + 3.1. Control Structure If .................................. 18 + 3.2. Control Structure Require ............................. 19 + 3.3. Control Structure Stop ................................ 19 + 4. Action Commands ........................................ 19 + 4.1. Action reject ......................................... 20 + 4.2. Action fileinto ....................................... 20 + 4.3. Action redirect ....................................... 21 + 4.4. Action keep ........................................... 21 + 4.5. Action discard ........................................ 22 + 5. Test Commands .......................................... 22 + 5.1. Test address .......................................... 23 + 5.2. Test allof ............................................ 23 + 5.3. Test anyof ............................................ 24 + 5.4. Test envelope ......................................... 24 + 5.5. Test exists ........................................... 25 + 5.6. Test false ............................................ 25 + 5.7. Test header ........................................... 25 + 5.8. Test not .............................................. 26 + 5.9. Test size ............................................. 26 + 5.10. Test true ............................................. 26 + 6. Extensibility .......................................... 26 + 6.1. Capability String ..................................... 27 + 6.2. IANA Considerations ................................... 28 + 6.2.1. Template for Capability Registrations ................. 28 + 6.2.2. Initial Capability Registrations ...................... 28 + 6.3. Capability Transport .................................. 29 + 7. Transmission ........................................... 29 + + + +Showalter Standards Track [Page 2] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + 8. Parsing ................................................ 30 + 8.1. Lexical Tokens ........................................ 30 + 8.2. Grammar ............................................... 31 + 9. Extended Example ....................................... 32 + 10. Security Considerations ................................ 34 + 11. Acknowledgments ........................................ 34 + 12. Author's Address ....................................... 34 + 13. References ............................................. 34 + 14. Full Copyright Statement ............................... 36 + +1. Introduction + + This memo documents a language that can be used to create filters for + electronic mail. It is not tied to any particular operating system or + mail architecture. It requires the use of [IMAIL]-compliant + messages, but should otherwise generalize to many systems. + + The language is powerful enough to be useful but limited in order to + allow for a safe server-side filtering system. The intention is to + make it impossible for users to do anything more complex (and + dangerous) than write simple mail filters, along with facilitating + the use of GUIs for filter creation and manipulation. The language is + not Turing-complete: it provides no way to write a loop or a function + and variables are not provided. + + Scripts written in Sieve are executed during final delivery, when the + message is moved to the user-accessible mailbox. In systems where + the MTA does final delivery, such as traditional Unix mail, it is + reasonable to sort when the MTA deposits mail into the user's + mailbox. + + There are a number of reasons to use a filtering system. Mail + traffic for most users has been increasing due to increased usage of + e-mail, the emergence of unsolicited email as a form of advertising, + and increased usage of mailing lists. + + Experience at Carnegie Mellon has shown that if a filtering system is + made available to users, many will make use of it in order to file + messages from specific users or mailing lists. However, many others + did not make use of the Andrew system's FLAMES filtering language + [FLAMES] due to difficulty in setting it up. + + Because of the expectation that users will make use of filtering if + it is offered and easy to use, this language has been made simple + enough to allow many users to make use of it, but rich enough that it + can be used productively. However, it is expected that GUI-based + editors will be the preferred way of editing filters for a large + number of users. + + + +Showalter Standards Track [Page 3] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +1.1. Conventions Used in This Document + + In the sections of this document that discuss the requirements of + various keywords and operators, the following conventions have been + adopted. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and + "MAY" in this document are to be interpreted as defined in + [KEYWORDS]. + + Each section on a command (test, action, or control structure) has a + line labeled "Syntax:". This line describes the syntax of the + command, including its name and its arguments. Required arguments + are listed inside angle brackets ("<" and ">"). Optional arguments + are listed inside square brackets ("[" and "]"). Each argument is + followed by its type, so "" represents an argument + called "key" that is a string. Literal strings are represented with + double-quoted strings. Alternatives are separated with slashes, and + parenthesis are used for grouping, similar to [ABNF]. + + In the "Syntax" line, there are three special pieces of syntax that + are frequently repeated, MATCH-TYPE, COMPARATOR, and ADDRESS-PART. + These are discussed in sections 2.7.1, 2.7.3, and 2.7.4, + respectively. + + The formal grammar for these commands in section 10 and is the + authoritative reference on how to construct commands, but the formal + grammar does not specify the order, semantics, number or types of + arguments to commands, nor the legal command names. The intent is to + allow for extension without changing the grammar. + +1.2. Example mail messages + + The following mail messages will be used throughout this document in + examples. + + Message A + ----------------------------------------------------------- + Date: Tue, 1 Apr 1997 09:06:31 -0800 (PST) + From: coyote@desert.example.org + To: roadrunner@acme.example.com + Subject: I have a present for you + + Look, I'm sorry about the whole anvil thing, and I really + didn't mean to try and drop it on you from the top of the + cliff. I want to try to make it up to you. I've got some + great birdseed over here at my place--top of the line + + + + +Showalter Standards Track [Page 4] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + stuff--and if you come by, I'll have it all wrapped up + for you. I'm really sorry for all the problems I've caused + for you over the years, but I know we can work this out. + -- + Wile E. Coyote "Super Genius" coyote@desert.example.org + ----------------------------------------------------------- + + Message B + ----------------------------------------------------------- + From: youcouldberich!@reply-by-postal-mail.invalid + Sender: b1ff@de.res.example.com + To: rube@landru.example.edu + Date: Mon, 31 Mar 1997 18:26:10 -0800 + Subject: $$$ YOU, TOO, CAN BE A MILLIONAIRE! $$$ + + YOU MAY HAVE ALREADY WON TEN MILLION DOLLARS, BUT I DOUBT + IT! SO JUST POST THIS TO SIX HUNDRED NEWSGROUPS! IT WILL + GUARANTEE THAT YOU GET AT LEAST FIVE RESPONSES WITH MONEY! + MONEY! MONEY! COLD HARD CASH! YOU WILL RECEIVE OVER + $20,000 IN LESS THAN TWO MONTHS! AND IT'S LEGAL!!!!!!!!! + !!!!!!!!!!!!!!!!!!111111111!!!!!!!11111111111!!1 JUST + SEND $5 IN SMALL, UNMARKED BILLS TO THE ADDRESSES BELOW! + ----------------------------------------------------------- + +2. Design + +2.1. Form of the Language + + The language consists of a set of commands. Each command consists of + a set of tokens delimited by whitespace. The command identifier is + the first token and it is followed by zero or more argument tokens. + Arguments may be literal data, tags, blocks of commands, or test + commands. + + The language is represented in UTF-8, as specified in [UTF-8]. + + Tokens in the ASCII range are considered case-insensitive. + +2.2. Whitespace + + Whitespace is used to separate tokens. Whitespace is made up of + tabs, newlines (CRLF, never just CR or LF), and the space character. + The amount of whitespace used is not significant. + + + + + + + + +Showalter Standards Track [Page 5] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.3. Comments + + Two types of comments are offered. Comments are semantically + equivalent to whitespace and can be used anyplace that whitespace is + (with one exception in multi-line strings, as described in the + grammar). + + Hash comments begin with a "#" character that is not contained within + a string and continue until the next CRLF. + + Example: if size :over 100K { # this is a comment + discard; + } + + Bracketed comments begin with the token "/*" and end with "*/" outside + of a string. Bracketed comments may span multiple lines. Bracketed + comments do not nest. + + Example: if size :over 100K { /* this is a comment + this is still a comment */ discard /* this is a comment + */ ; + } + +2.4. Literal Data + + Literal data means data that is not executed, merely evaluated "as + is", to be used as arguments to commands. Literal data is limited to + numbers and strings. + +2.4.1. Numbers + + Numbers are given as ordinary decimal numbers. However, those + numbers that have a tendency to be fairly large, such as message + sizes, MAY have a "K", "M", or "G" appended to indicate a multiple of + a power of two. To be comparable with the power-of-two-based + versions of SI units that computers frequently use, K specifies + kibi-, or 1,024 (2^10) times the value of the number; M specifies + mebi-, or 1,048,576 (2^20) times the value of the number; and G + specifies tebi-, or 1,073,741,824 (2^30) times the value of the + number [BINARY-SI]. + + Implementations MUST provide 31 bits of magnitude in numbers, but MAY + provide more. + + Only positive integers are permitted by this specification. + + + + + + +Showalter Standards Track [Page 6] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.4.2. Strings + + Scripts involve large numbers of strings as they are used for pattern + matching, addresses, textual bodies, etc. Typically, short quoted + strings suffice for most uses, but a more convenient form is provided + for longer strings such as bodies of messages. + + A quoted string starts and ends with a single double quote (the <"> + character, ASCII 34). A backslash ("\", ASCII 92) inside of a quoted + string is followed by either another backslash or a double quote. + This two-character sequence represents a single backslash or double- + quote within the string, respectively. + + No other characters should be escaped with a single backslash. + + An undefined escape sequence (such as "\a" in a context where "a" has + no special meaning) is interpreted as if there were no backslash (in + this case, "\a" is just "a"). + + Non-printing characters such as tabs, CR and LF, and control + characters are permitted in quoted strings. Quoted strings MAY span + multiple lines. NUL (ASCII 0) is not allowed in strings. + + For entering larger amounts of text, such as an email message, a + multi-line form is allowed. It starts with the keyword "text:", + followed by a CRLF, and ends with the sequence of a CRLF, a single + period, and another CRLF. In order to allow the message to contain + lines with a single-dot, lines are dot-stuffed. That is, when + composing a message body, an extra `.' is added before each line + which begins with a `.'. When the server interprets the script, + these extra dots are removed. Note that a line that begins with a + dot followed by a non-dot character is not interpreted dot-stuffed; + that is, ".foo" is interpreted as ".foo". However, because this is + potentially ambiguous, scripts SHOULD be properly dot-stuffed so such + lines do not appear. + + Note that a hashed comment or whitespace may occur in between the + "text:" and the CRLF, but not within the string itself. Bracketed + comments are not allowed here. + +2.4.2.1. String Lists + + When matching patterns, it is frequently convenient to match against + groups of strings instead of single strings. For this reason, a list + of strings is allowed in many tests, implying that if the test is + true using any one of the strings, then the test is true. + Implementations are encouraged to use short-circuit evaluation in + these cases. + + + +Showalter Standards Track [Page 7] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + For instance, the test `header :contains ["To", "Cc"] + ["me@example.com", "me00@landru.example.edu"]' is true if either the + To header or Cc header of the input message contains either of the + e-mail addresses "me@example.com" or "me00@landru.example.edu". + + Conversely, in any case where a list of strings is appropriate, a + single string is allowed without being a member of a list: it is + equivalent to a list with a single member. This means that the test + `exists "To"' is equivalent to the test `exists ["To"]'. + +2.4.2.2. Headers + + Headers are a subset of strings. In the Internet Message + Specification [IMAIL] [RFC1123], each header line is allowed to have + whitespace nearly anywhere in the line, including after the field + name and before the subsequent colon. Extra spaces between the + header name and the ":" in a header field are ignored. + + A header name never contains a colon. The "From" header refers to a + line beginning "From:" (or "From :", etc.). No header will match + the string "From:" due to the trailing colon. + + Folding of long header lines (as described in [IMAIL] 3.4.8) is + removed prior to interpretation of the data. The folding syntax (the + CRLF that ends a line plus any leading whitespace at the beginning of + the next line that indicates folding) are interpreted as if they were + a single space. + +2.4.2.3. Addresses + + A number of commands call for email addresses, which are also a + subset of strings. When these addresses are used in outbound + contexts, addresses must be compliant with [IMAIL], but are further + constrained. Using the symbols defined in [IMAIL], section 6.1, the + syntax of an address is: + + sieve-address = addr-spec ; simple address + / phrase "<" addr-spec ">" ; name & addr-spec + + That is, routes and group syntax are not permitted. If multiple + addresses are required, use a string list. Named groups are not used + here. + + Implementations MUST ensure that the addresses are syntactically + valid, but need not ensure that they actually identify an email + recipient. + + + + + +Showalter Standards Track [Page 8] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.4.2.4. MIME Parts + + In a few places, [MIME] body parts are represented as strings. These + parts include MIME headers and the body. This provides a way of + embedding typed data within a Sieve script so that, among other + things, character sets other than UTF-8 can be used for output + messages. + +2.5. Tests + + Tests are given as arguments to commands in order to control their + actions. In this document, tests are given to if/elsif/else to + decide which block of code is run. + + Tests MUST NOT have side effects. That is, a test cannot affect the + state of the filter or message. No tests in this specification have + side effects, and side effects are forbidden in extension tests as + well. + + The rationale for this is that tests with side effects impair + readability and maintainability and are difficult to represent in a + graphic interface for generating scripts. Side effects are confined + to actions where they are clearer. + +2.5.1. Test Lists + + Some tests ("allof" and "anyof", which implement logical "and" and + logical "or", respectively) may require more than a single test as an + argument. The test-list syntax element provides a way of grouping + tests. + + Example: if anyof (not exists ["From", "Date"], + header :contains "from" "fool@example.edu") { + discard; + } + +2.6. Arguments + + In order to specify what to do, most commands take arguments. There + are three types of arguments: positional, tagged, and optional. + +2.6.1. Positional Arguments + + Positional arguments are given to a command which discerns their + meaning based on their order. When a command takes positional + arguments, all positional arguments must be supplied and must be in + the order prescribed. + + + + +Showalter Standards Track [Page 9] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.6.2. Tagged Arguments + + This document provides for tagged arguments in the style of + CommonLISP. These are also similar to flags given to commands in + most command-line systems. + + A tagged argument is an argument for a command that begins with ":" + followed by a tag naming the argument, such as ":contains". This + argument means that zero or more of the next tokens have some + particular meaning depending on the argument. These next tokens may + be numbers or strings but they are never blocks. + + Tagged arguments are similar to positional arguments, except that + instead of the meaning being derived from the command, it is derived + from the tag. + + Tagged arguments must appear before positional arguments, but they + may appear in any order with other tagged arguments. For simplicity + of the specification, this is not expressed in the syntax definitions + with commands, but they still may be reordered arbitrarily provided + they appear before positional arguments. Tagged arguments may be + mixed with optional arguments. + + To simplify this specification, tagged arguments SHOULD NOT take + tagged arguments as arguments. + +2.6.3. Optional Arguments + + Optional arguments are exactly like tagged arguments except that they + may be left out, in which case a default value is implied. Because + optional arguments tend to result in shorter scripts, they have been + used far more than tagged arguments. + + One particularly noteworthy case is the ":comparator" argument, which + allows the user to specify which [ACAP] comparator will be used to + compare two strings, since different languages may impose different + orderings on UTF-8 [UTF-8] characters. + +2.6.4. Types of Arguments + + Abstractly, arguments may be literal data, tests, or blocks of + commands. In this way, an "if" control structure is merely a command + that happens to take a test and a block as arguments and may execute + the block of code. + + However, this abstraction is ambiguous from a parsing standpoint. + The grammar in section 9.2 presents a parsable version of this: + Arguments are string-lists, numbers, and tags, which may be followed + + + +Showalter Standards Track [Page 10] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + by a test or a test-list, which may be followed by a block of + commands. No more than one test or test list, nor more than one + block of commands, may be used, and commands that end with blocks of + commands do not end with semicolons. + +2.7. String Comparison + + When matching one string against another, there are a number of ways + of performing the match operation. These are accomplished with three + types of matches: an exact match, a substring match, and a wildcard + glob-style match. These are described below. + + In order to provide for matches between character sets and case + insensitivity, Sieve borrows ACAP's comparator registry. + + However, when a string represents the name of a header, the + comparator is never user-specified. Header comparisons are always + done with the "i;ascii-casemap" operator, i.e., case-insensitive + comparisons, because this is the way things are defined in the + message specification [IMAIL]. + +2.7.1. Match Type + + There are three match types describing the matching used in this + specification: ":is", ":contains", and ":matches". Match type + arguments are supplied to those commands which allow them to specify + what kind of match is to be performed. + + These are used as tagged arguments to tests that perform string + comparison. + + The ":contains" match type describes a substring match. If the value + argument contains the key argument as a substring, the match is true. + For instance, the string "frobnitzm" contains "frob" and "nit", but + not "fbm". The null key ("") is contained in all values. + + The ":is" match type describes an absolute match; if the contents of + the first string are absolutely the same as the contents of the + second string, they match. Only the string "frobnitzm" is the string + "frobnitzm". The null key ":is" and only ":is" the null value. + + The ":matches" version specifies a wildcard match using the + characters "*" and "?". "*" matches zero or more characters, and "?" + matches a single character. "?" and "*" may be escaped as "\\?" and + "\\*" in strings to match against themselves. The first backslash + escapes the second backslash; together, they escape the "*". This is + awkward, but it is commonplace in several programming languages that + use globs and regular expressions. + + + +Showalter Standards Track [Page 11] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + In order to specify what type of match is supposed to happen, + commands that support matching take optional tagged arguments + ":matches", ":is", and ":contains". Commands default to using ":is" + matching if no match type argument is supplied. Note that these + modifiers may interact with comparators; in particular, some + comparators are not suitable for matching with ":contains" or + ":matches". It is an error to use a comparator with ":contains" or + ":matches" that is not compatible with it. + + It is an error to give more than one of these arguments to a given + command. + + For convenience, the "MATCH-TYPE" syntax element is defined here as + follows: + + Syntax: ":is" / ":contains" / ":matches" + +2.7.2. Comparisons Across Character Sets + + All Sieve scripts are represented in UTF-8, but messages may involve + a number of character sets. In order for comparisons to work across + character sets, implementations SHOULD implement the following + behavior: + + Implementations decode header charsets to UTF-8. Two strings are + considered equal if their UTF-8 representations are identical. + Implementations should decode charsets represented in the forms + specified by [MIME] for both message headers and bodies. + Implementations must be capable of decoding US-ASCII, ISO-8859-1, + the ASCII subset of ISO-8859-* character sets, and UTF-8. + + If implementations fail to support the above behavior, they MUST + conform to the following: + + No two strings can be considered equal if one contains octets + greater than 127. + +2.7.3. Comparators + + In order to allow for language-independent, case-independent matches, + the match type may be coupled with a comparator name. Comparators + are described for [ACAP]; a registry is defined for ACAP, and this + specification uses that registry. + + ACAP defines multiple comparator types. Only equality types are used + in this specification. + + + + + +Showalter Standards Track [Page 12] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + All implementations MUST support the "i;octet" comparator (simply + compares octets) and the "i;ascii-casemap" comparator (which treats + uppercase and lowercase characters in the ASCII subset of UTF-8 as + the same). If left unspecified, the default is "i;ascii-casemap". + + Some comparators may not be usable with substring matches; that is, + they may only work with ":is". It is an error to try and use a + comparator with ":matches" or ":contains" that is not compatible with + it. + + A comparator is specified by the ":comparator" option with commands + that support matching. This option is followed by a string providing + the name of the comparator to be used. For convenience, the syntax + of a comparator is abbreviated to "COMPARATOR", and (repeated in + several tests) is as follows: + + Syntax: ":comparator" + + So in this example, + + Example: if header :contains :comparator "i;octet" "Subject" + "MAKE MONEY FAST" { + discard; + } + + would discard any message with subjects like "You can MAKE MONEY + FAST", but not "You can Make Money Fast", since the comparator used + is case-sensitive. + + Comparators other than i;octet and i;ascii-casemap must be declared + with require, as they are extensions. If a comparator declared with + require is not known, it is an error, and execution fails. If the + comparator is not declared with require, it is also an error, even if + the comparator is supported. (See 2.10.5.) + + Both ":matches" and ":contains" match types are compatible with the + "i;octet" and "i;ascii-casemap" comparators and may be used with + them. + + It is an error to give more than one of these arguments to a given + command. + +2.7.4. Comparisons Against Addresses + + Addresses are one of the most frequent things represented as strings. + These are structured, and being able to compare against the local- + part or the domain of an address is useful, so some tests that act + + + + +Showalter Standards Track [Page 13] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + exclusively on addresses take an additional optional argument that + specifies what the test acts on. + + These optional arguments are ":localpart", ":domain", and ":all", + which act on the local-part (left-side), the domain part (right- + side), and the whole address. + + The kind of comparison done, such as whether or not the test done is + case-insensitive, is specified as a comparator argument to the test. + + If an optional address-part is omitted, the default is ":all". + + It is an error to give more than one of these arguments to a given + command. + + For convenience, the "ADDRESS-PART" syntax element is defined here as + follows: + + Syntax: ":localpart" / ":domain" / ":all" + +2.8. Blocks + + Blocks are sets of commands enclosed within curly braces. Blocks are + supplied to commands so that the commands can implement control + commands. + + A control structure is a command that happens to take a test and a + block as one of its arguments; depending on the result of the test + supplied as another argument, it runs the code in the block some + number of times. + + With the commands supplied in this memo, there are no loops. The + control structures supplied--if, elsif, and else--run a block either + once or not at all. So there are two arguments, the test and the + block. + +2.9. Commands + + Sieve scripts are sequences of commands. Commands can take any of + the tokens above as arguments, and arguments may be either tagged or + positional arguments. Not all commands take all arguments. + + There are three kinds of commands: test commands, action commands, + and control commands. + + The simplest is an action command. An action command is an + identifier followed by zero or more arguments, terminated by a + semicolon. Action commands do not take tests or blocks as arguments. + + + +Showalter Standards Track [Page 14] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + A control command is similar, but it takes a test as an argument, and + ends with a block instead of a semicolon. + + A test command is used as part of a control command. It is used to + specify whether or not the block of code given to the control command + is executed. + +2.10. Evaluation + +2.10.1. Action Interaction + + Some actions cannot be used with other actions because the result + would be absurd. These restrictions are noted throughout this memo. + + Extension actions MUST state how they interact with actions defined + in this specification. + +2.10.2. Implicit Keep + + Previous experience with filtering systems suggests that cases tend + to be missed in scripts. To prevent errors, Sieve has an "implicit + keep". + + An implicit keep is a keep action (see 4.4) performed in absence of + any action that cancels the implicit keep. + + An implicit keep is performed if a message is not written to a + mailbox, redirected to a new address, or explicitly thrown out. That + is, if a fileinto, a keep, a redirect, or a discard is performed, an + implicit keep is not. + + Some actions may be defined to not cancel the implicit keep. These + actions may not directly affect the delivery of a message, and are + used for their side effects. None of the actions specified in this + document meet that criteria, but extension actions will. + + For instance, with any of the short messages offered above, the + following script produces no actions. + + Example: if size :over 500K { discard; } + + As a result, the implicit keep is taken. + +2.10.3. Message Uniqueness in a Mailbox + + Implementations SHOULD NOT deliver a message to the same folder more + than once, even if a script explicitly asks for a message to be + written to a mailbox twice. + + + +Showalter Standards Track [Page 15] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + The test for equality of two messages is implementation-defined. + + If a script asks for a message to be written to a mailbox twice, it + MUST NOT be treated as an error. + +2.10.4. Limits on Numbers of Actions + + Site policy MAY limit numbers of actions taken and MAY impose + restrictions on which actions can be used together. In the event + that a script hits a policy limit on the number of actions taken for + a particular message, an error occurs. + + Implementations MUST prohibit more than one reject. + + Implementations MUST allow at least one keep or one fileinto. If + fileinto is not implemented, implementations MUST allow at least one + keep. + + Implementations SHOULD prohibit reject when used with other actions. + +2.10.5. Extensions and Optional Features + + Because of the differing capabilities of many mail systems, several + features of this specification are optional. Before any of these + extensions can be executed, they must be declared with the "require" + action. + + If an extension is not enabled with "require", implementations MUST + treat it as if they did not support it at all. + + If a script does not understand an extension declared with require, + the script must not be used at all. Implementations MUST NOT execute + scripts which require unknown capability names. + + Note: The reason for this restriction is that prior experiences with + languages such as LISP and Tcl suggest that this is a workable + way of noting that a given script uses an extension. + + Experience with PostScript suggests that mechanisms that allow + a script to work around missing extensions are not used in + practice. + + Extensions which define actions MUST state how they interact with + actions discussed in the base specification. + + + + + + + +Showalter Standards Track [Page 16] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +2.10.6. Errors + + In any programming language, there are compile-time and run-time + errors. + + Compile-time errors are ones in syntax that are detectable if a + syntax check is done. + + Run-time errors are not detectable until the script is run. This + includes transient failures like disk full conditions, but also + includes issues like invalid combinations of actions. + + When an error occurs in a Sieve script, all processing stops. + + Implementations MAY choose to do a full parse, then evaluate the + script, then do all actions. Implementations might even go so far as + to ensure that execution is atomic (either all actions are executed + or none are executed). + + Other implementations may choose to parse and run at the same time. + Such implementations are simpler, but have issues with partial + failure (some actions happen, others don't). + + Implementations might even go so far as to ensure that scripts can + never execute an invalid set of actions (e.g., reject + fileinto) + before execution, although this could involve solving the Halting + Problem. + + This specification allows any of these approaches. Solving the + Halting Problem is considered extra credit. + + When an error happens, implementations MUST notify the user that an + error occurred, which actions (if any) were taken, and do an implicit + keep. + +2.10.7. Limits on Execution + + Implementations may limit certain constructs. However, this + specification places a lower bound on some of these limits. + + Implementations MUST support fifteen levels of nested blocks. + + Implementations MUST support fifteen levels of nested test lists. + +3. Control Commands + + Control structures are needed to allow for multiple and conditional + actions. + + + +Showalter Standards Track [Page 17] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +3.1. Control Structure If + + There are three pieces to if: "if", "elsif", and "else". Each is + actually a separate command in terms of the grammar. However, an + elsif MUST only follow an if, and an else MUST follow only either an + if or an elsif. An error occurs if these conditions are not met. + + Syntax: if + + Syntax: elsif + + Syntax: else + + The semantics are similar to those of any of the many other + programming languages these control commands appear in. When the + interpreter sees an "if", it evaluates the test associated with it. + If the test is true, it executes the block associated with it. + + If the test of the "if" is false, it evaluates the test of the first + "elsif" (if any). If the test of "elsif" is true, it runs the + elsif's block. An elsif may be followed by an elsif, in which case, + the interpreter repeats this process until it runs out of elsifs. + + When the interpreter runs out of elsifs, there may be an "else" case. + If there is, and none of the if or elsif tests were true, the + interpreter runs the else case. + + This provides a way of performing exactly one of the blocks in the + chain. + + In the following example, both Message A and B are dropped. + + Example: require "fileinto"; + if header :contains "from" "coyote" { + discard; + } elsif header :contains ["subject"] ["$$$"] { + discard; + } else { + fileinto "INBOX"; + } + + + When the script below is run over message A, it redirects the message + to acm@example.edu; message B, to postmaster@example.edu; any other + message is redirected to field@example.edu. + + + + + + +Showalter Standards Track [Page 18] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Example: if header :contains ["From"] ["coyote"] { + redirect "acm@example.edu"; + } elsif header :contains "Subject" "$$$" { + redirect "postmaster@example.edu"; + } else { + redirect "field@example.edu"; + } + + Note that this definition prohibits the "... else if ..." sequence + used by C. This is intentional, because this construct produces a + shift-reduce conflict. + +3.2. Control Structure Require + + Syntax: require + + The require action notes that a script makes use of a certain + extension. Such a declaration is required to use the extension, as + discussed in section 2.10.5. Multiple capabilities can be declared + with a single require. + + The require command, if present, MUST be used before anything other + than a require can be used. An error occurs if a require appears + after a command other than require. + + Example: require ["fileinto", "reject"]; + + Example: require "fileinto"; + require "vacation"; + +3.3. Control Structure Stop + + Syntax: stop + + The "stop" action ends all processing. If no actions have been + executed, then the keep action is taken. + +4. Action Commands + + This document supplies five actions that may be taken on a message: + keep, fileinto, redirect, reject, and discard. + + Implementations MUST support the "keep", "discard", and "redirect" + actions. + + Implementations SHOULD support "reject" and "fileinto". + + + + + +Showalter Standards Track [Page 19] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Implementations MAY limit the number of certain actions taken (see + section 2.10.4). + +4.1. Action reject + + Syntax: reject + + The optional "reject" action refuses delivery of a message by sending + back an [MDN] to the sender. It resends the message to the sender, + wrapping it in a "reject" form, noting that it was rejected by the + recipient. In the following script, message A is rejected and + returned to the sender. + + Example: if header :contains "from" "coyote@desert.example.org" { + reject "I am not taking mail from you, and I don't want + your birdseed, either!"; + } + + A reject message MUST take the form of a failure MDN as specified by + [MDN]. The human-readable portion of the message, the first + component of the MDN, contains the human readable message describing + the error, and it SHOULD contain additional text alerting the + original sender that mail was refused by a filter. This part of the + MDN might appear as follows: + + ------------------------------------------------------------ + Message was refused by recipient's mail filtering program. Reason + given was as follows: + + I am not taking mail from you, and I don't want your birdseed, + either! + ------------------------------------------------------------ + + The MDN action-value field as defined in the MDN specification MUST + be "deleted" and MUST have the MDN-sent-automatically and automatic- + action modes set. + + Because some implementations can not or will not implement the reject + command, it is optional. The capability string to be used with the + require command is "reject". + +4.2. Action fileinto + + Syntax: fileinto + + The "fileinto" action delivers the message into the specified folder. + Implementations SHOULD support fileinto, but in some environments + this may be impossible. + + + +Showalter Standards Track [Page 20] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + The capability string for use with the require command is "fileinto". + + In the following script, message A is filed into folder + "INBOX.harassment". + + Example: require "fileinto"; + if header :contains ["from"] "coyote" { + fileinto "INBOX.harassment"; + } + +4.3. Action redirect + + Syntax: redirect + + The "redirect" action is used to send the message to another user at + a supplied address, as a mail forwarding feature does. The + "redirect" action makes no changes to the message body or existing + headers, but it may add new headers. The "redirect" modifies the + envelope recipient. + + The redirect command performs an MTA-style "forward"--that is, what + you get from a .forward file using sendmail under UNIX. The address + on the SMTP envelope is replaced with the one on the redirect command + and the message is sent back out. (This is not an MUA-style forward, + which creates a new message with a different sender and message ID, + wrapping the old message in a new one.) + + A simple script can be used for redirecting all mail: + + Example: redirect "bart@example.edu"; + + Implementations SHOULD take measures to implement loop control, + possibly including adding headers to the message or counting received + headers. If an implementation detects a loop, it causes an error. + +4.4. Action keep + + Syntax: keep + + The "keep" action is whatever action is taken in lieu of all other + actions, if no filtering happens at all; generally, this simply means + to file the message into the user's main mailbox. This command + provides a way to execute this action without needing to know the + name of the user's main mailbox, providing a way to call it without + needing to understand the user's setup, or the underlying mail + system. + + + + + +Showalter Standards Track [Page 21] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + For instance, in an implementation where the IMAP server is running + scripts on behalf of the user at time of delivery, a keep command is + equivalent to a fileinto "INBOX". + + Example: if size :under 1M { keep; } else { discard; } + + Note that the above script is identical to the one below. + + Example: if not size :under 1M { discard; } + +4.5. Action discard + + Syntax: discard + + Discard is used to silently throw away the message. It does so by + simply canceling the implicit keep. If discard is used with other + actions, the other actions still happen. Discard is compatible with + all other actions. (For instance fileinto+discard is equivalent to + fileinto.) + + Discard MUST be silent; that is, it MUST NOT return a non-delivery + notification of any kind ([DSN], [MDN], or otherwise). + + In the following script, any mail from "idiot@example.edu" is thrown + out. + + Example: if header :contains ["from"] ["idiot@example.edu"] { + discard; + } + + While an important part of this language, "discard" has the potential + to create serious problems for users: Students who leave themselves + logged in to an unattended machine in a public computer lab may find + their script changed to just "discard". In order to protect users in + this situation (along with similar situations), implementations MAY + keep messages destroyed by a script for an indefinite period, and MAY + disallow scripts that throw out all mail. + +5. Test Commands + + Tests are used in conditionals to decide which part(s) of the + conditional to execute. + + Implementations MUST support these tests: "address", "allof", + "anyof", "exists", "false", "header", "not", "size", and "true". + + Implementations SHOULD support the "envelope" test. + + + + +Showalter Standards Track [Page 22] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +5.1. Test address + + Syntax: address [ADDRESS-PART] [COMPARATOR] [MATCH-TYPE] + + + The address test matches Internet addresses in structured headers + that contain addresses. It returns true if any header contains any + key in the specified part of the address, as modified by the + comparator and the match keyword. + + Like envelope and header, this test returns true if any combination + of the header-list and key-list arguments match. + + Internet email addresses [IMAIL] have the somewhat awkward + characteristic that the local-part to the left of the at-sign is + considered case sensitive, and the domain-part to the right of the + at-sign is case insensitive. The "address" command does not deal + with this itself, but provides the ADDRESS-PART argument for allowing + users to deal with it. + + The address primitive never acts on the phrase part of an email + address, nor on comments within that address. It also never acts on + group names, although it does act on the addresses within the group + construct. + + Implementations MUST restrict the address test to headers that + contain addresses, but MUST include at least From, To, Cc, Bcc, + Sender, Resent-From, Resent-To, and SHOULD include any other header + that utilizes an "address-list" structured header body. + + Example: if address :is :all "from" "tim@example.com" { + discard; + +5.2. Test allof + + Syntax: allof + + The allof test performs a logical AND on the tests supplied to it. + + Example: allof (false, false) => false + allof (false, true) => false + allof (true, true) => true + + The allof test takes as its argument a test-list. + + + + + + + +Showalter Standards Track [Page 23] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +5.3. Test anyof + + Syntax: anyof + + The anyof test performs a logical OR on the tests supplied to it. + + Example: anyof (false, false) => false + anyof (false, true) => true + anyof (true, true) => true + +5.4. Test envelope + + Syntax: envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] + + + The "envelope" test is true if the specified part of the SMTP (or + equivalent) envelope matches the specified key. + + If one of the envelope-part strings is (case insensitive) "from", + then matching occurs against the FROM address used in the SMTP MAIL + command. + + If one of the envelope-part strings is (case insensitive) "to", then + matching occurs against the TO address used in the SMTP RCPT command + that resulted in this message getting delivered to this user. Note + that only the most recent TO is available, and only the one relevant + to this user. + + The envelope-part is a string list and may contain more than one + parameter, in which case all of the strings specified in the key-list + are matched against all parts given in the envelope-part list. + + Like address and header, this test returns true if any combination of + the envelope-part and key-list arguments is true. + + All tests against envelopes MUST drop source routes. + + If the SMTP transaction involved several RCPT commands, only the data + from the RCPT command that caused delivery to this user is available + in the "to" part of the envelope. + + If a protocol other than SMTP is used for message transport, + implementations are expected to adapt this command appropriately. + + The envelope command is optional. Implementations SHOULD support it, + but the necessary information may not be available in all cases. + + + + + +Showalter Standards Track [Page 24] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Example: require "envelope"; + if envelope :all :is "from" "tim@example.com" { + discard; + } + +5.5. Test exists + + Syntax: exists + + The "exists" test is true if the headers listed in the header-names + argument exist within the message. All of the headers must exist or + the test is false. + + The following example throws out mail that doesn't have a From header + and a Date header. + + Example: if not exists ["From","Date"] { + discard; + } + +5.6. Test false + + Syntax: false + + The "false" test always evaluates to false. + +5.7. Test header + + Syntax: header [COMPARATOR] [MATCH-TYPE] + + + The "header" test evaluates to true if any header name matches any + key. The type of match is specified by the optional match argument, + which defaults to ":is" if not specified, as specified in section + 2.6. + + Like address and envelope, this test returns true if any combination + of the string-list and key-list arguments match. + + If a header listed in the header-names argument exists, it contains + the null key (""). However, if the named header is not present, it + does not contain the null key. So if a message contained the header + + X-Caffeine: C8H10N4O2 + + + + + + + +Showalter Standards Track [Page 25] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + these tests on that header evaluate as follows: + + header :is ["X-Caffeine"] [""] => false + header :contains ["X-Caffeine"] [""] => true + +5.8. Test not + + Syntax: not + + The "not" test takes some other test as an argument, and yields the + opposite result. "not false" evaluates to "true" and "not true" + evaluates to "false". + +5.9. Test size + + Syntax: size <":over" / ":under"> + + The "size" test deals with the size of a message. It takes either a + tagged argument of ":over" or ":under", followed by a number + representing the size of the message. + + If the argument is ":over", and the size of the message is greater + than the number provided, the test is true; otherwise, it is false. + + If the argument is ":under", and the size of the message is less than + the number provided, the test is true; otherwise, it is false. + + Exactly one of ":over" or ":under" must be specified, and anything + else is an error. + + The size of a message is defined to be the number of octets from the + initial header until the last character in the message body. + + Note that for a message that is exactly 4,000 octets, the message is + neither ":over" 4000 octets or ":under" 4000 octets. + +5.10. Test true + + Syntax: true + + The "true" test always evaluates to true. + +6. Extensibility + + New control structures, actions, and tests can be added to the + language. Sites must make these features known to their users; this + document does not define a way to discover the list of extensions + supported by the server. + + + +Showalter Standards Track [Page 26] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Any extensions to this language MUST define a capability string that + uniquely identifies that extension. If a new version of an extension + changes the functionality of a previously defined extension, it MUST + use a different name. + + In a situation where there is a submission protocol and an extension + advertisement mechanism aware of the details of this language, + scripts submitted can be checked against the mail server to prevent + use of an extension that the server does not support. + + Extensions MUST state how they interact with constraints defined in + section 2.10, e.g., whether they cancel the implicit keep, and which + actions they are compatible and incompatible with. + +6.1. Capability String + + Capability strings are typically short strings describing what + capabilities are supported by the server. + + Capability strings beginning with "vnd." represent vendor-defined + extensions. Such extensions are not defined by Internet standards or + RFCs, but are still registered with IANA in order to prevent + conflicts. Extensions starting with "vnd." SHOULD be followed by the + name of the vendor and product, such as "vnd.acme.rocket-sled". + + The following capability strings are defined by this document: + + envelope The string "envelope" indicates that the implementation + supports the "envelope" command. + + fileinto The string "fileinto" indicates that the implementation + supports the "fileinto" command. + + reject The string "reject" indicates that the implementation + supports the "reject" command. + + comparator- The string "comparator-elbonia" is provided if the + implementation supports the "elbonia" comparator. + Therefore, all implementations have at least the + "comparator-i;octet" and "comparator-i;ascii-casemap" + capabilities. However, these comparators may be used + without being declared with require. + + + + + + + + + +Showalter Standards Track [Page 27] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +6.2. IANA Considerations + + In order to provide a standard set of extensions, a registry is + provided by IANA. Capability names may be registered on a first- + come, first-served basis. Extensions designed for interoperable use + SHOULD be defined as standards track or IESG approved experimental + RFCs. + +6.2.1. Template for Capability Registrations + + The following template is to be used for registering new Sieve + extensions with IANA. + + To: iana@iana.org + Subject: Registration of new Sieve extension + + Capability name: + Capability keyword: + Capability arguments: + Standards Track/IESG-approved experimental RFC number: + Person and email address to contact for further information: + +6.2.2. Initial Capability Registrations + + The following are to be added to the IANA registry for Sieve + extensions as the initial contents of the capability registry. + + Capability name: fileinto + Capability keyword: fileinto + Capability arguments: fileinto + Standards Track/IESG-approved experimental RFC number: + RFC 3028 (Sieve base spec) + Person and email address to contact for further information: + Tim Showalter + tjs@mirapoint.com + + Capability name: reject + Capability keyword: reject + Capability arguments: reject + Standards Track/IESG-approved experimental RFC number: + RFC 3028 (Sieve base spec) + Person and email address to contact for further information: + Tim Showalter + tjs@mirapoint.com + + + + + + + +Showalter Standards Track [Page 28] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Capability name: envelope + Capability keyword: envelope + Capability arguments: + envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] + + Standards Track/IESG-approved experimental RFC number: + RFC 3028 (Sieve base spec) + Person and email address to contact for further information: + Tim Showalter + tjs@mirapoint.com + + Capability name: comparator-* + Capability keyword: + comparator-* (anything starting with "comparator-") + Capability arguments: (none) + Standards Track/IESG-approved experimental RFC number: + RFC 3028, Sieve, by reference of + RFC 2244, Application Configuration Access Protocol + Person and email address to contact for further information: + Tim Showalter + tjs@mirapoint.com + +6.3. Capability Transport + + As the range of mail systems that this document is intended to apply + to is quite varied, a method of advertising which capabilities an + implementation supports is difficult due to the wide range of + possible implementations. Such a mechanism, however, should have + property that the implementation can advertise the complete set of + extensions that it supports. + +7. Transmission + + The MIME type for a Sieve script is "application/sieve". + + The registration of this type for RFC 2048 requirements is as + follows: + + Subject: Registration of MIME media type application/sieve + + MIME media type name: application + MIME subtype name: sieve + Required parameters: none + Optional parameters: none + Encoding considerations: Most sieve scripts will be textual, + written in UTF-8. When non-7bit characters are used, + quoted-printable is appropriate for transport systems + that require 7bit encoding. + + + +Showalter Standards Track [Page 29] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + Security considerations: Discussed in section 10 of RFC 3028. + Interoperability considerations: Discussed in section 2.10.5 + of RFC 3028. + Published specification: RFC 3028. + Applications which use this media type: sieve-enabled mail servers + Additional information: + Magic number(s): + File extension(s): .siv + Macintosh File Type Code(s): + Person & email address to contact for further information: + See the discussion list at ietf-mta-filters@imc.org. + Intended usage: + COMMON + Author/Change controller: + See Author information in RFC 3028. + +8. Parsing + + The Sieve grammar is separated into tokens and a separate grammar as + most programming languages are. + +8.1. Lexical Tokens + + Sieve scripts are encoded in UTF-8. The following assumes a valid + UTF-8 encoding; special characters in Sieve scripts are all ASCII. + + The following are tokens in Sieve: + + - identifiers + - tags + - numbers + - quoted strings + - multi-line strings + - other separators + + Blanks, horizontal tabs, CRLFs, and comments ("white space") are + ignored except as they separate tokens. Some white space is required + to separate otherwise adjacent tokens and in specific places in the + multi-line strings. + + The other separators are single individual characters, and are + mentioned explicitly in the grammar. + + The lexical structure of sieve is defined in the following BNF (as + described in [ABNF]): + + + + + + +Showalter Standards Track [Page 30] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + bracket-comment = "/*" *(CHAR-NOT-STAR / ("*" CHAR-NOT-SLASH)) "*/" + ;; No */ allowed inside a comment. + ;; (No * is allowed unless it is the last character, + ;; or unless it is followed by a character that isn't a + ;; slash.) + + CHAR-NOT-DOT = (%x01-09 / %x0b-0c / %x0e-2d / %x2f-ff) + ;; no dots, no CRLFs + + CHAR-NOT-CRLF = (%x01-09 / %x0b-0c / %x0e-ff) + + CHAR-NOT-SLASH = (%x00-57 / %x58-ff) + + CHAR-NOT-STAR = (%x00-51 / %x53-ff) + + comment = bracket-comment / hash-comment + + hash-comment = ( "#" *CHAR-NOT-CRLF CRLF ) + + identifier = (ALPHA / "_") *(ALPHA DIGIT "_") + + tag = ":" identifier + + number = 1*DIGIT [QUANTIFIER] + + QUANTIFIER = "K" / "M" / "G" + + quoted-string = DQUOTE *CHAR DQUOTE + ;; in general, \ CHAR inside a string maps to CHAR + ;; so \" maps to " and \\ maps to \ + ;; note that newlines and other characters are all allowed + ;; strings + + multi-line = "text:" *(SP / HTAB) (hash-comment / CRLF) + *(multi-line-literal / multi-line-dotstuff) + "." CRLF + multi-line-literal = [CHAR-NOT-DOT *CHAR-NOT-CRLF] CRLF + multi-line-dotstuff = "." 1*CHAR-NOT-CRLF CRLF + ;; A line containing only "." ends the multi-line. + ;; Remove a leading '.' if followed by another '.'. + + white-space = 1*(SP / CRLF / HTAB) / comment + +8.2. Grammar + + The following is the grammar of Sieve after it has been lexically + interpreted. No white space or comments appear below. The start + symbol is "start". + + + +Showalter Standards Track [Page 31] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + argument = string-list / number / tag + + arguments = *argument [test / test-list] + + block = "{" commands "}" + + command = identifier arguments ( ";" / block ) + + commands = *command + + start = commands + + string = quoted-string / multi-line + + string-list = "[" string *("," string) "]" / string ;; if + there is only a single string, the brackets are optional + + test = identifier arguments + + test-list = "(" test *("," test) ")" + +9. Extended Example + + The following is an extended example of a Sieve script. Note that it + does not make use of the implicit keep. + + # + # Example Sieve Filter + # Declare any optional features or extension used by the script + # + require ["fileinto", "reject"]; + + # + # Reject any large messages (note that the four leading dots get + # "stuffed" to three) + # + if size :over 1M + { + reject text: + Please do not send me large attachments. + Put your file on a server and send me the URL. + Thank you. + .... Fred + . + ; + stop; + } + # + + + +Showalter Standards Track [Page 32] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + # Handle messages from known mailing lists + # Move messages from IETF filter discussion list to filter folder + # + if header :is "Sender" "owner-ietf-mta-filters@imc.org" + { + fileinto "filter"; # move to "filter" folder + } + # + # Keep all messages to or from people in my company + # + elsif address :domain :is ["From", "To"] "example.com" + { + keep; # keep in "In" folder + } + + # + # Try and catch unsolicited email. If a message is not to me, + # or it contains a subject known to be spam, file it away. + # + elsif anyof (not address :all :contains + ["To", "Cc", "Bcc"] "me@example.com", + header :matches "subject" + ["*make*money*fast*", "*university*dipl*mas*"]) + { + # If message header does not contain my address, + # it's from a list. + fileinto "spam"; # move to "spam" folder + } + else + { + # Move all other (non-company) mail to "personal" + # folder. + fileinto "personal"; + } + + + + + + + + + + + + + + + + + +Showalter Standards Track [Page 33] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +10. Security Considerations + + Users must get their mail. It is imperative that whatever method + implementations use to store the user-defined filtering scripts be + secure. + + It is equally important that implementations sanity-check the user's + scripts, and not allow users to create on-demand mailbombs. For + instance, an implementation that allows a user to reject or redirect + multiple times to a single message might also allow a user to create + a mailbomb triggered by mail from a specific user. Site- or + implementation-defined limits on actions are useful for this. + + Several commands, such as "discard", "redirect", and "fileinto" allow + for actions to be taken that are potentially very dangerous. + + Implementations SHOULD take measures to prevent languages from + looping. + +11. Acknowledgments + + I am very thankful to Chris Newman for his support and his ABNF + syntax checker, to John Myers and Steve Hole for outlining the + requirements for the original drafts, to Larry Greenfield for nagging + me about the grammar and finally fixing it, to Greg Sereda for + repeatedly fixing and providing examples, to Ned Freed for fixing + everything else, to Rob Earhart for an early implementation and a + great deal of help, and to Randall Gellens for endless amounts of + proofreading. I am grateful to Carnegie Mellon University where most + of the work on this document was done. I am also indebted to all of + the readers of the ietf-mta-filters@imc.org mailing list. + +12. Author's Address + + Tim Showalter + Mirapoint, Inc. + 909 Hermosa Court + Sunnyvale, CA 94085 + + EMail: tjs@mirapoint.com + +13. References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + + + + + +Showalter Standards Track [Page 34] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + + [ACAP] Newman, C. and J. G. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [BINARY-SI] "Standard IEC 60027-2: Letter symbols to be used in + electrical technology - Part 2: Telecommunications and + electronics", January 1999. + + [DSN] Moore, K. and G. Vaudreuil, "An Extensible Message Format + for Delivery Status Notifications", RFC 1894, January + 1996. + + [FLAMES] Borenstein, N, and C. Thyberg, "Power, Ease of Use, and + Cooperative Work in a Practical Multimedia Message + System", Int. J. of Man-Machine Studies, April, 1991. + Reprinted in Computer-Supported Cooperative Work and + Groupware, Saul Greenberg, editor, Harcourt Brace + Jovanovich, 1991. Reprinted in Readings in Groupware and + Computer-Supported Cooperative Work, Ronald Baecker, + editor, Morgan Kaufmann, 1993. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP] Crispin, M., "Internet Message Access Protocol - version + 4rev1", RFC 2060, December 1996. + + [IMAIL] Crocker, D., "Standard for the Format of ARPA Internet + Text Messages", STD 11, RFC 822, August 1982. + + [MIME] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [MDN] Fajman, R., "An Extensible Message Format for Message + Disposition Notifications", RFC 2298, March 1998. + + [RFC1123] Braden, R., "Requirements for Internet Hosts -- + Application and Support", STD 3, RFC 1123, November 1989. + + [SMTP] Postel, J., "Simple Mail Transfer Protocol", STD 10, RFC + 821, August 1982. + + [UTF-8] Yergeau, F., "UTF-8, a transformation format of Unicode + and ISO 10646", RFC 2044, October 1996. + + + + + + + +Showalter Standards Track [Page 35] + +RFC 3028 Sieve: A Mail Filtering Language January 2001 + + +14. Full Copyright Statement + + Copyright (C) The Internet Society (2001). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Showalter Standards Track [Page 36] + diff --git a/docs/rfcs/rfc3348.IMAP4_Child_Mailbox_extension.txt b/docs/rfcs/rfc3348.IMAP4_Child_Mailbox_extension.txt new file mode 100644 index 0000000..500871c --- /dev/null +++ b/docs/rfcs/rfc3348.IMAP4_Child_Mailbox_extension.txt @@ -0,0 +1,339 @@ + + + + + + +Network Working Group M. Gahrns +Request for Comments: 3348 R. Cheng +Category: Informational Microsoft + July 2002 + + + The Internet Message Action Protocol (IMAP4) + Child Mailbox Extension + +Status of this Memo + + This memo provides information for the Internet community. It does + not specify an Internet standard of any kind. Distribution of this + memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2002). All Rights Reserved. + +Abstract + + The Internet Message Action Protocol (IMAP4) CHILDREN extension + provides a mechanism for a client to efficiently determine if a + particular mailbox has children, without issuing a LIST "" * or a + LIST "" % for each mailbox. + +1. Conventions used in this document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If such lines are wrapped without a new "C:" or + "S:" label, then the wrapping is for editorial clarity and is not + part of the command. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC-2119]. + +2. Introduction and Overview + + Many IMAP4 [RFC-2060] clients present to the user a hierarchical view + of the mailboxes that a user has access to. Rather than initially + presenting to the user the entire mailbox hierarchy, it is often + preferable to show to the user a collapsed outline list of the + mailbox hierarchy (particularly if there is a large number of + mailboxes). The user can then expand the collapsed outline hierarchy + as needed. It is common to include within the collapsed hierarchy a + + + + + +Gahrns, et al. Informational [Page 1] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + + visual clue (such as a "+") to indicate that there are child + mailboxes under a particular mailbox. When the visual clue is + clicked the hierarchy list is expanded to show the child mailboxes. + + Several IMAP vendors implemented this proposal, and it is proposed to + document this behavior and functionality as an Informational RFC. + + There is interest in addressing the general extensibility of the IMAP + LIST command through an IMAP LIST Extension draft. Similar + functionality to the \HasChildren and \HasNoChildren flags could be + incorporated into this new LIST Extension. It is proposed that the + more general LIST Extension draft proceed on the standards track with + this proposal being relegated to informational status only. + + If the functionality of the \HasChildren and \HasNoChildren flags + were incorporated into a more general LIST extension, this would have + the advantage that a client could then have the opportunity to + request whether or not the server should return this information. + This would be an advantage over the current draft for servers where + this information is expensive to compute, since the server would only + need to compute the information when it knew that the client + requesting the information was able to consume it. + +3. Requirements + + IMAP4 servers that support this extension MUST list the keyword + CHILDREN in their CAPABILITY response. + + The CHILDREN extension defines two new attributes that MAY be + returned within a LIST response. + + \HasChildren - The presence of this attribute indicates that the + mailbox has child mailboxes. + + Servers SHOULD NOT return \HasChildren if child mailboxes exist, but + none will be displayed to the current user in a LIST response (as + should be the case where child mailboxes exist, but a client does not + have permissions to access them.) In this case, \HasNoChildren + SHOULD be used. + + In many cases, however, a server may not be able to efficiently + compute whether a user has access to all child mailboxes, or multiple + users may be accessing the same account and simultaneously changing + the mailbox hierarchy. As such a client MUST be prepared to accept + the \HasChildren attribute as a hint. That is, a mailbox MAY be + flagged with the \HasChildren attribute, but no child mailboxes will + appear in a subsequent LIST response. + + + + +Gahrns, et al. Informational [Page 2] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + + Example 3.1: + ============ + + /*** Consider a server that has the following mailbox hierarchy: + + INBOX + ITEM_1 + ITEM_1A + ITEM_2 + TOP_SECRET + + Where INBOX, ITEM_1 and ITEM_2 are top level mailboxes. ITEM_1A is a + child mailbox of ITEM_1 and TOP_SECRET is a child mailbox of ITEM_2 + that the currently logged on user does NOT have access to. + + Note that in this case, the server is not able to efficiently compute + access rights to child mailboxes and responds with a \HasChildren + attribute for mailbox ITEM_2, even though ITEM_2/TOP_SECRET does not + appear in the list response. ***/ + + C: A001 LIST "" * + S: * LIST (\HasNoChildren) "/" INBOX + S: * LIST (\HasChildren) "/" ITEM_1 + S: * LIST (\HasNoChildren) "/" ITEM_1/ITEM_1A + S: * LIST (\HasChildren) "/" ITEM_2 + S: A001 OK LIST Completed + + \HasNoChildren - The presence of this attribute indicates that the + mailbox has NO child mailboxes that are accessible to the currently + authenticated user. If a mailbox has the \Noinferiors attribute, the + \HasNoChildren attribute is redundant and SHOULD be omitted in the + LIST response. + + In some instances a server that supports the CHILDREN extension MAY + NOT be able to determine whether a mailbox has children. For example + it may have difficulty determining whether there are child mailboxes + when LISTing mailboxes while operating in a particular namespace. + + In these cases, a server MAY exclude both the \HasChildren and + \HasNoChildren attributes in the LIST response. As such, a client + can not make any assumptions about whether a mailbox has children + based upon the absence of a single attribute. + + It is an error for the server to return both a \HasChildren and a + \HasNoChildren attribute in a LIST response. + + + + + + +Gahrns, et al. Informational [Page 3] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + + It is an error for the server to return both a \HasChildren and a + \NoInferiors attribute in a LIST response. + + Note: the \HasNoChildren attribute should not be confused with the + IMAP4 [RFC-2060] defined attribute \Noinferiors which indicates that + no child mailboxes exist now and none can be created in the future. + + The \HasChildren and \HasNoChildren attributes might not be returned + in response to a LSUB response. Many servers maintain a simple + mailbox subscription list that is not updated when the underlying + mailbox structure is changed. A client MUST NOT assume that + hierarchy information will be maintained in the subscription list. + + RLIST is a command defined in [RFC-2193] that includes in a LIST + response mailboxes that are accessible only via referral. That is, a + client must explicitly issue an RLIST command to see a list of these + mailboxes. Thus in the case where a mailbox has child mailboxes that + are available only via referral, the mailboxes would appear as + \HasNoChildren in response to the LIST command, and \HasChildren in + response to the RLIST command. + +5. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. + + Two new mailbox attributes are defined as flag_extensions to the + IMAP4 mailbox_list response: + + HasChildren = "\HasChildren" + + HasNoChildren = "\HasNoChildren" + +6. Security Considerations + + This extension provides a client a more efficient means of + determining whether a particular mailbox has children. If a mailbox + has children, but the currently authenticated user does not have + access to any of them, the server SHOULD respond with a + \HasNoChildren attribute. In many cases, however, a server may not + be able to efficiently compute whether a user has access to all child + mailboxes. If such a server responds with a \HasChildren attribute, + when in fact the currently authenticated user does not have access to + any child mailboxes, potentially more information is conveyed about + the mailbox than intended. A server designed with such levels of + security in mind SHOULD NOT attach the \HasChildren attribute to a + mailbox unless the server is certain that the user has access to at + least one of the child mailboxes. + + + +Gahrns, et al. Informational [Page 4] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + +7. References + + [RFC-2060] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 2060, December 1996. + + [RFC-2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC-2234] Crocker, D. and P. Overell, Editors, "Augmented BNF for + Syntax Specifications: ABNF", RFC 2234, November 1997. + + [RFC-2193] Gahrns, M., "IMAP4 Mailbox Referrals", RFC 2193, September + 1997. + +8. Acknowledgments + + The authors would like to thank the participants of several IMC Mail + Connect events for their input when this idea was originally + presented and refined. + +9. Author's Address + + Mike Gahrns + Microsoft + One Microsoft Way + Redmond, WA, 98052 + Phone: (425) 936-9833 + EMail: mikega@microsoft.com + + Raymond Cheng + Microsoft + One Microsoft Way + Redmond, WA, 98052 + Phone: (425) 703-4913 + EMail: raych@microsoft.com + + + + + + + + + + + + + + + + +Gahrns, et al. Informational [Page 5] + +RFC 3348 IMAP4 Child Mailbox Extension July 2002 + + +10. Full Copyright Statement + + Copyright (C) The Internet Society (2002). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Gahrns, et al. Informational [Page 6] + diff --git a/docs/rfcs/rfc3501.IMAP4rev1.txt b/docs/rfcs/rfc3501.IMAP4rev1.txt new file mode 100644 index 0000000..5ab48e1 --- /dev/null +++ b/docs/rfcs/rfc3501.IMAP4rev1.txt @@ -0,0 +1,6051 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 3501 University of Washington +Obsoletes: 2060 March 2003 +Category: Standards Track + + + INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1 + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + The Internet Message Access Protocol, Version 4rev1 (IMAP4rev1) + allows a client to access and manipulate electronic mail messages on + a server. IMAP4rev1 permits manipulation of mailboxes (remote + message folders) in a way that is functionally equivalent to local + folders. IMAP4rev1 also provides the capability for an offline + client to resynchronize with the server. + + IMAP4rev1 includes operations for creating, deleting, and renaming + mailboxes, checking for new messages, permanently removing messages, + setting and clearing flags, RFC 2822 and RFC 2045 parsing, searching, + and selective fetching of message attributes, texts, and portions + thereof. Messages in IMAP4rev1 are accessed by the use of numbers. + These numbers are either message sequence numbers or unique + identifiers. + + IMAP4rev1 supports a single server. A mechanism for accessing + configuration information to support multiple IMAP4rev1 servers is + discussed in RFC 2244. + + IMAP4rev1 does not specify a means of posting mail; this function is + handled by a mail transfer protocol such as RFC 2821. + + + + + + + + +Crispin Standards Track [Page 1] + +RFC 3501 IMAPv4 March 2003 + + +Table of Contents + + IMAP4rev1 Protocol Specification ................................ 4 + 1. How to Read This Document ............................... 4 + 1.1. Organization of This Document ........................... 4 + 1.2. Conventions Used in This Document ....................... 4 + 1.3. Special Notes to Implementors ........................... 5 + 2. Protocol Overview ....................................... 6 + 2.1. Link Level .............................................. 6 + 2.2. Commands and Responses .................................. 6 + 2.2.1. Client Protocol Sender and Server Protocol Receiver ..... 6 + 2.2.2. Server Protocol Sender and Client Protocol Receiver ..... 7 + 2.3. Message Attributes ...................................... 8 + 2.3.1. Message Numbers ......................................... 8 + 2.3.1.1. Unique Identifier (UID) Message Attribute ....... 8 + 2.3.1.2. Message Sequence Number Message Attribute ....... 10 + 2.3.2. Flags Message Attribute ................................. 11 + 2.3.3. Internal Date Message Attribute ......................... 12 + 2.3.4. [RFC-2822] Size Message Attribute ....................... 12 + 2.3.5. Envelope Structure Message Attribute .................... 12 + 2.3.6. Body Structure Message Attribute ........................ 12 + 2.4. Message Texts ........................................... 13 + 3. State and Flow Diagram .................................. 13 + 3.1. Not Authenticated State ................................. 13 + 3.2. Authenticated State ..................................... 13 + 3.3. Selected State .......................................... 13 + 3.4. Logout State ............................................ 14 + 4. Data Formats ............................................ 16 + 4.1. Atom .................................................... 16 + 4.2. Number .................................................. 16 + 4.3. String .................................................. 16 + 4.3.1. 8-bit and Binary Strings ................................ 17 + 4.4. Parenthesized List ...................................... 17 + 4.5. NIL ..................................................... 17 + 5. Operational Considerations .............................. 18 + 5.1. Mailbox Naming .......................................... 18 + 5.1.1. Mailbox Hierarchy Naming ................................ 19 + 5.1.2. Mailbox Namespace Naming Convention ..................... 19 + 5.1.3. Mailbox International Naming Convention ................. 19 + 5.2. Mailbox Size and Message Status Updates ................. 21 + 5.3. Response when no Command in Progress .................... 21 + 5.4. Autologout Timer ........................................ 22 + 5.5. Multiple Commands in Progress ........................... 22 + 6. Client Commands ........................................ 23 + 6.1. Client Commands - Any State ............................ 24 + 6.1.1. CAPABILITY Command ..................................... 24 + 6.1.2. NOOP Command ........................................... 25 + 6.1.3. LOGOUT Command ......................................... 26 + + + +Crispin Standards Track [Page 2] + +RFC 3501 IMAPv4 March 2003 + + + 6.2. Client Commands - Not Authenticated State .............. 26 + 6.2.1. STARTTLS Command ....................................... 27 + 6.2.2. AUTHENTICATE Command ................................... 28 + 6.2.3. LOGIN Command .......................................... 30 + 6.3. Client Commands - Authenticated State .................. 31 + 6.3.1. SELECT Command ......................................... 32 + 6.3.2. EXAMINE Command ........................................ 34 + 6.3.3. CREATE Command ......................................... 34 + 6.3.4. DELETE Command ......................................... 35 + 6.3.5. RENAME Command ......................................... 37 + 6.3.6. SUBSCRIBE Command ...................................... 39 + 6.3.7. UNSUBSCRIBE Command .................................... 39 + 6.3.8. LIST Command ........................................... 40 + 6.3.9. LSUB Command ........................................... 43 + 6.3.10. STATUS Command ......................................... 44 + 6.3.11. APPEND Command ......................................... 46 + 6.4. Client Commands - Selected State ....................... 47 + 6.4.1. CHECK Command .......................................... 47 + 6.4.2. CLOSE Command .......................................... 48 + 6.4.3. EXPUNGE Command ........................................ 49 + 6.4.4. SEARCH Command ......................................... 49 + 6.4.5. FETCH Command .......................................... 54 + 6.4.6. STORE Command .......................................... 58 + 6.4.7. COPY Command ........................................... 59 + 6.4.8. UID Command ............................................ 60 + 6.5. Client Commands - Experimental/Expansion ............... 62 + 6.5.1. X Command ........................................ 62 + 7. Server Responses ....................................... 62 + 7.1. Server Responses - Status Responses .................... 63 + 7.1.1. OK Response ............................................ 65 + 7.1.2. NO Response ............................................ 66 + 7.1.3. BAD Response ........................................... 66 + 7.1.4. PREAUTH Response ....................................... 67 + 7.1.5. BYE Response ........................................... 67 + 7.2. Server Responses - Server and Mailbox Status ........... 68 + 7.2.1. CAPABILITY Response .................................... 68 + 7.2.2. LIST Response .......................................... 69 + 7.2.3. LSUB Response .......................................... 70 + 7.2.4 STATUS Response ........................................ 70 + 7.2.5. SEARCH Response ........................................ 71 + 7.2.6. FLAGS Response ......................................... 71 + 7.3. Server Responses - Mailbox Size ........................ 71 + 7.3.1. EXISTS Response ........................................ 71 + 7.3.2. RECENT Response ........................................ 72 + 7.4. Server Responses - Message Status ...................... 72 + 7.4.1. EXPUNGE Response ....................................... 72 + 7.4.2. FETCH Response ......................................... 73 + 7.5. Server Responses - Command Continuation Request ........ 79 + + + +Crispin Standards Track [Page 3] + +RFC 3501 IMAPv4 March 2003 + + + 8. Sample IMAP4rev1 connection ............................ 80 + 9. Formal Syntax .......................................... 81 + 10. Author's Note .......................................... 92 + 11. Security Considerations ................................ 92 + 11.1. STARTTLS Security Considerations ....................... 92 + 11.2. Other Security Considerations .......................... 93 + 12. IANA Considerations .................................... 94 + Appendices ..................................................... 95 + A. References ............................................. 95 + B. Changes from RFC 2060 .................................. 97 + C. Key Word Index ......................................... 103 + Author's Address ............................................... 107 + Full Copyright Statement ....................................... 108 + +IMAP4rev1 Protocol Specification + +1. How to Read This Document + +1.1. Organization of This Document + + This document is written from the point of view of the implementor of + an IMAP4rev1 client or server. Beyond the protocol overview in + section 2, it is not optimized for someone trying to understand the + operation of the protocol. The material in sections 3 through 5 + provides the general context and definitions with which IMAP4rev1 + operates. + + Sections 6, 7, and 9 describe the IMAP commands, responses, and + syntax, respectively. The relationships among these are such that it + is almost impossible to understand any of them separately. In + particular, do not attempt to deduce command syntax from the command + section alone; instead refer to the Formal Syntax section. + +1.2. Conventions Used in This Document + + "Conventions" are basic principles or procedures. Document + conventions are noted in this section. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to + be interpreted as described in [KEYWORDS]. + + The word "can" (not "may") is used to refer to a possible + circumstance or situation, as opposed to an optional facility of the + protocol. + + + +Crispin Standards Track [Page 4] + +RFC 3501 IMAPv4 March 2003 + + + "User" is used to refer to a human user, whereas "client" refers to + the software being run by the user. + + "Connection" refers to the entire sequence of client/server + interaction from the initial establishment of the network connection + until its termination. + + "Session" refers to the sequence of client/server interaction from + the time that a mailbox is selected (SELECT or EXAMINE command) until + the time that selection ends (SELECT or EXAMINE of another mailbox, + CLOSE command, or connection termination). + + Characters are 7-bit US-ASCII unless otherwise specified. Other + character sets are indicated using a "CHARSET", as described in + [MIME-IMT] and defined in [CHARSET]. CHARSETs have important + additional semantics in addition to defining character set; refer to + these documents for more detail. + + There are several protocol conventions in IMAP. These refer to + aspects of the specification which are not strictly part of the IMAP + protocol, but reflect generally-accepted practice. Implementations + need to be aware of these conventions, and avoid conflicts whether or + not they implement the convention. For example, "&" may not be used + as a hierarchy delimiter since it conflicts with the Mailbox + International Naming Convention, and other uses of "&" in mailbox + names are impacted as well. + +1.3. Special Notes to Implementors + + Implementors of the IMAP protocol are strongly encouraged to read the + IMAP implementation recommendations document [IMAP-IMPLEMENTATION] in + conjunction with this document, to help understand the intricacies of + this protocol and how best to build an interoperable product. + + IMAP4rev1 is designed to be upwards compatible from the [IMAP2] and + unpublished IMAP2bis protocols. IMAP4rev1 is largely compatible with + the IMAP4 protocol described in RFC 1730; the exception being in + certain facilities added in RFC 1730 that proved problematic and were + subsequently removed. In the course of the evolution of IMAP4rev1, + some aspects in the earlier protocols have become obsolete. Obsolete + commands, responses, and data formats which an IMAP4rev1 + implementation can encounter when used with an earlier implementation + are described in [IMAP-OBSOLETE]. + + Other compatibility issues with IMAP2bis, the most common variant of + the earlier protocol, are discussed in [IMAP-COMPAT]. A full + discussion of compatibility issues with rare (and presumed extinct) + + + + +Crispin Standards Track [Page 5] + +RFC 3501 IMAPv4 March 2003 + + + variants of [IMAP2] is in [IMAP-HISTORICAL]; this document is + primarily of historical interest. + + IMAP was originally developed for the older [RFC-822] standard, and + as a consequence several fetch items in IMAP incorporate "RFC822" in + their name. With the exception of RFC822.SIZE, there are more modern + replacements; for example, the modern version of RFC822.HEADER is + BODY.PEEK[HEADER]. In all cases, "RFC822" should be interpreted as a + reference to the updated [RFC-2822] standard. + +2. Protocol Overview + +2.1. Link Level + + The IMAP4rev1 protocol assumes a reliable data stream such as that + provided by TCP. When TCP is used, an IMAP4rev1 server listens on + port 143. + +2.2. Commands and Responses + + An IMAP4rev1 connection consists of the establishment of a + client/server network connection, an initial greeting from the + server, and client/server interactions. These client/server + interactions consist of a client command, server data, and a server + completion result response. + + All interactions transmitted by client and server are in the form of + lines, that is, strings that end with a CRLF. The protocol receiver + of an IMAP4rev1 client or server is either reading a line, or is + reading a sequence of octets with a known count followed by a line. + +2.2.1. Client Protocol Sender and Server Protocol Receiver + + The client command begins an operation. Each client command is + prefixed with an identifier (typically a short alphanumeric string, + e.g., A0001, A0002, etc.) called a "tag". A different tag is + generated by the client for each command. + + Clients MUST follow the syntax outlined in this specification + strictly. It is a syntax error to send a command with missing or + extraneous spaces or arguments. + + There are two cases in which a line from the client does not + represent a complete command. In one case, a command argument is + quoted with an octet count (see the description of literal in String + under Data Formats); in the other case, the command arguments require + server feedback (see the AUTHENTICATE command). In either case, the + + + + +Crispin Standards Track [Page 6] + +RFC 3501 IMAPv4 March 2003 + + + server sends a command continuation request response if it is ready + for the octets (if appropriate) and the remainder of the command. + This response is prefixed with the token "+". + + Note: If instead, the server detected an error in the + command, it sends a BAD completion response with a tag + matching the command (as described below) to reject the + command and prevent the client from sending any more of the + command. + + It is also possible for the server to send a completion + response for some other command (if multiple commands are + in progress), or untagged data. In either case, the + command continuation request is still pending; the client + takes the appropriate action for the response, and reads + another response from the server. In all cases, the client + MUST send a complete command (including receiving all + command continuation request responses and command + continuations for the command) before initiating a new + command. + + The protocol receiver of an IMAP4rev1 server reads a command line + from the client, parses the command and its arguments, and transmits + server data and a server command completion result response. + +2.2.2. Server Protocol Sender and Client Protocol Receiver + + Data transmitted by the server to the client and status responses + that do not indicate command completion are prefixed with the token + "*", and are called untagged responses. + + Server data MAY be sent as a result of a client command, or MAY be + sent unilaterally by the server. There is no syntactic difference + between server data that resulted from a specific command and server + data that were sent unilaterally. + + The server completion result response indicates the success or + failure of the operation. It is tagged with the same tag as the + client command which began the operation. Thus, if more than one + command is in progress, the tag in a server completion response + identifies the command to which the response applies. There are + three possible server completion responses: OK (indicating success), + NO (indicating failure), or BAD (indicating a protocol error such as + unrecognized command or command syntax error). + + Servers SHOULD enforce the syntax outlined in this specification + strictly. Any client command with a protocol syntax error, including + (but not limited to) missing or extraneous spaces or arguments, + + + +Crispin Standards Track [Page 7] + +RFC 3501 IMAPv4 March 2003 + + + SHOULD be rejected, and the client given a BAD server completion + response. + + The protocol receiver of an IMAP4rev1 client reads a response line + from the server. It then takes action on the response based upon the + first token of the response, which can be a tag, a "*", or a "+". + + A client MUST be prepared to accept any server response at all times. + This includes server data that was not requested. Server data SHOULD + be recorded, so that the client can reference its recorded copy + rather than sending a command to the server to request the data. In + the case of certain server data, the data MUST be recorded. + + This topic is discussed in greater detail in the Server Responses + section. + +2.3. Message Attributes + + In addition to message text, each message has several attributes + associated with it. These attributes can be retrieved individually + or in conjunction with other attributes or message texts. + +2.3.1. Message Numbers + + Messages in IMAP4rev1 are accessed by one of two numbers; the unique + identifier or the message sequence number. + + +2.3.1.1. Unique Identifier (UID) Message Attribute + + A 32-bit value assigned to each message, which when used with the + unique identifier validity value (see below) forms a 64-bit value + that MUST NOT refer to any other message in the mailbox or any + subsequent mailbox with the same name forever. Unique identifiers + are assigned in a strictly ascending fashion in the mailbox; as each + message is added to the mailbox it is assigned a higher UID than the + message(s) which were added previously. Unlike message sequence + numbers, unique identifiers are not necessarily contiguous. + + The unique identifier of a message MUST NOT change during the + session, and SHOULD NOT change between sessions. Any change of + unique identifiers between sessions MUST be detectable using the + UIDVALIDITY mechanism discussed below. Persistent unique identifiers + are required for a client to resynchronize its state from a previous + session with the server (e.g., disconnected or offline access + clients); this is discussed further in [IMAP-DISC]. + + + + + +Crispin Standards Track [Page 8] + +RFC 3501 IMAPv4 March 2003 + + + Associated with every mailbox are two values which aid in unique + identifier handling: the next unique identifier value and the unique + identifier validity value. + + The next unique identifier value is the predicted value that will be + assigned to a new message in the mailbox. Unless the unique + identifier validity also changes (see below), the next unique + identifier value MUST have the following two characteristics. First, + the next unique identifier value MUST NOT change unless new messages + are added to the mailbox; and second, the next unique identifier + value MUST change whenever new messages are added to the mailbox, + even if those new messages are subsequently expunged. + + Note: The next unique identifier value is intended to + provide a means for a client to determine whether any + messages have been delivered to the mailbox since the + previous time it checked this value. It is not intended to + provide any guarantee that any message will have this + unique identifier. A client can only assume, at the time + that it obtains the next unique identifier value, that + messages arriving after that time will have a UID greater + than or equal to that value. + + The unique identifier validity value is sent in a UIDVALIDITY + response code in an OK untagged response at mailbox selection time. + If unique identifiers from an earlier session fail to persist in this + session, the unique identifier validity value MUST be greater than + the one used in the earlier session. + + Note: Ideally, unique identifiers SHOULD persist at all + times. Although this specification recognizes that failure + to persist can be unavoidable in certain server + environments, it STRONGLY ENCOURAGES message store + implementation techniques that avoid this problem. For + example: + + 1) Unique identifiers MUST be strictly ascending in the + mailbox at all times. If the physical message store is + re-ordered by a non-IMAP agent, this requires that the + unique identifiers in the mailbox be regenerated, since + the former unique identifiers are no longer strictly + ascending as a result of the re-ordering. + + 2) If the message store has no mechanism to store unique + identifiers, it must regenerate unique identifiers at + each session, and each session must have a unique + UIDVALIDITY value. + + + + +Crispin Standards Track [Page 9] + +RFC 3501 IMAPv4 March 2003 + + + 3) If the mailbox is deleted and a new mailbox with the + same name is created at a later date, the server must + either keep track of unique identifiers from the + previous instance of the mailbox, or it must assign a + new UIDVALIDITY value to the new instance of the + mailbox. A good UIDVALIDITY value to use in this case + is a 32-bit representation of the creation date/time of + the mailbox. It is alright to use a constant such as + 1, but only if it guaranteed that unique identifiers + will never be reused, even in the case of a mailbox + being deleted (or renamed) and a new mailbox by the + same name created at some future time. + + 4) The combination of mailbox name, UIDVALIDITY, and UID + must refer to a single immutable message on that server + forever. In particular, the internal date, [RFC-2822] + size, envelope, body structure, and message texts + (RFC822, RFC822.HEADER, RFC822.TEXT, and all BODY[...] + fetch data items) must never change. This does not + include message numbers, nor does it include attributes + that can be set by a STORE command (e.g., FLAGS). + + +2.3.1.2. Message Sequence Number Message Attribute + + A relative position from 1 to the number of messages in the mailbox. + This position MUST be ordered by ascending unique identifier. As + each new message is added, it is assigned a message sequence number + that is 1 higher than the number of messages in the mailbox before + that new message was added. + + Message sequence numbers can be reassigned during the session. For + example, when a message is permanently removed (expunged) from the + mailbox, the message sequence number for all subsequent messages is + decremented. The number of messages in the mailbox is also + decremented. Similarly, a new message can be assigned a message + sequence number that was once held by some other message prior to an + expunge. + + In addition to accessing messages by relative position in the + mailbox, message sequence numbers can be used in mathematical + calculations. For example, if an untagged "11 EXISTS" is received, + and previously an untagged "8 EXISTS" was received, three new + messages have arrived with message sequence numbers of 9, 10, and 11. + Another example, if message 287 in a 523 message mailbox has UID + 12345, there are exactly 286 messages which have lesser UIDs and 236 + messages which have greater UIDs. + + + + +Crispin Standards Track [Page 10] + +RFC 3501 IMAPv4 March 2003 + + +2.3.2. Flags Message Attribute + + A list of zero or more named tokens associated with the message. A + flag is set by its addition to this list, and is cleared by its + removal. There are two types of flags in IMAP4rev1. A flag of + either type can be permanent or session-only. + + A system flag is a flag name that is pre-defined in this + specification. All system flags begin with "\". Certain system + flags (\Deleted and \Seen) have special semantics described + elsewhere. The currently-defined system flags are: + + \Seen + Message has been read + + \Answered + Message has been answered + + \Flagged + Message is "flagged" for urgent/special attention + + \Deleted + Message is "deleted" for removal by later EXPUNGE + + \Draft + Message has not completed composition (marked as a draft). + + \Recent + Message is "recently" arrived in this mailbox. This session + is the first session to have been notified about this + message; if the session is read-write, subsequent sessions + will not see \Recent set for this message. This flag can not + be altered by the client. + + If it is not possible to determine whether or not this + session is the first session to be notified about a message, + then that message SHOULD be considered recent. + + If multiple connections have the same mailbox selected + simultaneously, it is undefined which of these connections + will see newly-arrived messages with \Recent set and which + will see it without \Recent set. + + A keyword is defined by the server implementation. Keywords do not + begin with "\". Servers MAY permit the client to define new keywords + in the mailbox (see the description of the PERMANENTFLAGS response + code for more information). + + + + +Crispin Standards Track [Page 11] + +RFC 3501 IMAPv4 March 2003 + + + A flag can be permanent or session-only on a per-flag basis. + Permanent flags are those which the client can add or remove from the + message flags permanently; that is, concurrent and subsequent + sessions will see any change in permanent flags. Changes to session + flags are valid only in that session. + + Note: The \Recent system flag is a special case of a + session flag. \Recent can not be used as an argument in a + STORE or APPEND command, and thus can not be changed at + all. + +2.3.3. Internal Date Message Attribute + + The internal date and time of the message on the server. This + is not the date and time in the [RFC-2822] header, but rather a + date and time which reflects when the message was received. In + the case of messages delivered via [SMTP], this SHOULD be the + date and time of final delivery of the message as defined by + [SMTP]. In the case of messages delivered by the IMAP4rev1 COPY + command, this SHOULD be the internal date and time of the source + message. In the case of messages delivered by the IMAP4rev1 + APPEND command, this SHOULD be the date and time as specified in + the APPEND command description. All other cases are + implementation defined. + +2.3.4. [RFC-2822] Size Message Attribute + + The number of octets in the message, as expressed in [RFC-2822] + format. + +2.3.5. Envelope Structure Message Attribute + + A parsed representation of the [RFC-2822] header of the message. + Note that the IMAP Envelope structure is not the same as an + [SMTP] envelope. + +2.3.6. Body Structure Message Attribute + + A parsed representation of the [MIME-IMB] body structure + information of the message. + + + + + + + + + + + +Crispin Standards Track [Page 12] + +RFC 3501 IMAPv4 March 2003 + + +2.4. Message Texts + + In addition to being able to fetch the full [RFC-2822] text of a + message, IMAP4rev1 permits the fetching of portions of the full + message text. Specifically, it is possible to fetch the + [RFC-2822] message header, [RFC-2822] message body, a [MIME-IMB] + body part, or a [MIME-IMB] header. + +3. State and Flow Diagram + + Once the connection between client and server is established, an + IMAP4rev1 connection is in one of four states. The initial + state is identified in the server greeting. Most commands are + only valid in certain states. It is a protocol error for the + client to attempt a command while the connection is in an + inappropriate state, and the server will respond with a BAD or + NO (depending upon server implementation) command completion + result. + +3.1. Not Authenticated State + + In the not authenticated state, the client MUST supply + authentication credentials before most commands will be + permitted. This state is entered when a connection starts + unless the connection has been pre-authenticated. + +3.2. Authenticated State + + In the authenticated state, the client is authenticated and MUST + select a mailbox to access before commands that affect messages + will be permitted. This state is entered when a + pre-authenticated connection starts, when acceptable + authentication credentials have been provided, after an error in + selecting a mailbox, or after a successful CLOSE command. + +3.3. Selected State + + In a selected state, a mailbox has been selected to access. + This state is entered when a mailbox has been successfully + selected. + + + + + + + + + + + +Crispin Standards Track [Page 13] + +RFC 3501 IMAPv4 March 2003 + + +3.4. Logout State + + In the logout state, the connection is being terminated. This + state can be entered as a result of a client request (via the + LOGOUT command) or by unilateral action on the part of either + the client or server. + + If the client requests the logout state, the server MUST send an + untagged BYE response and a tagged OK response to the LOGOUT + command before the server closes the connection; and the client + MUST read the tagged OK response to the LOGOUT command before + the client closes the connection. + + A server MUST NOT unilaterally close the connection without + sending an untagged BYE response that contains the reason for + having done so. A client SHOULD NOT unilaterally close the + connection, and instead SHOULD issue a LOGOUT command. If the + server detects that the client has unilaterally closed the + connection, the server MAY omit the untagged BYE response and + simply close its connection. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 14] + +RFC 3501 IMAPv4 March 2003 + + + +----------------------+ + |connection established| + +----------------------+ + || + \/ + +--------------------------------------+ + | server greeting | + +--------------------------------------+ + || (1) || (2) || (3) + \/ || || + +-----------------+ || || + |Not Authenticated| || || + +-----------------+ || || + || (7) || (4) || || + || \/ \/ || + || +----------------+ || + || | Authenticated |<=++ || + || +----------------+ || || + || || (7) || (5) || (6) || + || || \/ || || + || || +--------+ || || + || || |Selected|==++ || + || || +--------+ || + || || || (7) || + \/ \/ \/ \/ + +--------------------------------------+ + | Logout | + +--------------------------------------+ + || + \/ + +-------------------------------+ + |both sides close the connection| + +-------------------------------+ + + (1) connection without pre-authentication (OK greeting) + (2) pre-authenticated connection (PREAUTH greeting) + (3) rejected connection (BYE greeting) + (4) successful LOGIN or AUTHENTICATE command + (5) successful SELECT or EXAMINE command + (6) CLOSE command, or failed SELECT or EXAMINE command + (7) LOGOUT command, server shutdown, or connection closed + + + + + + + + + + +Crispin Standards Track [Page 15] + +RFC 3501 IMAPv4 March 2003 + + +4. Data Formats + + IMAP4rev1 uses textual commands and responses. Data in + IMAP4rev1 can be in one of several forms: atom, number, string, + parenthesized list, or NIL. Note that a particular data item + may take more than one form; for example, a data item defined as + using "astring" syntax may be either an atom or a string. + +4.1. Atom + + An atom consists of one or more non-special characters. + +4.2. Number + + A number consists of one or more digit characters, and + represents a numeric value. + +4.3. String + + A string is in one of two forms: either literal or quoted + string. The literal form is the general form of string. The + quoted string form is an alternative that avoids the overhead of + processing a literal at the cost of limitations of characters + which may be used. + + A literal is a sequence of zero or more octets (including CR and + LF), prefix-quoted with an octet count in the form of an open + brace ("{"), the number of octets, close brace ("}"), and CRLF. + In the case of literals transmitted from server to client, the + CRLF is immediately followed by the octet data. In the case of + literals transmitted from client to server, the client MUST wait + to receive a command continuation request (described later in + this document) before sending the octet data (and the remainder + of the command). + + A quoted string is a sequence of zero or more 7-bit characters, + excluding CR and LF, with double quote (<">) characters at each + end. + + The empty string is represented as either "" (a quoted string + with zero characters between double quotes) or as {0} followed + by CRLF (a literal with an octet count of 0). + + Note: Even if the octet count is 0, a client transmitting a + literal MUST wait to receive a command continuation request. + + + + + + +Crispin Standards Track [Page 16] + +RFC 3501 IMAPv4 March 2003 + + +4.3.1. 8-bit and Binary Strings + + 8-bit textual and binary mail is supported through the use of a + [MIME-IMB] content transfer encoding. IMAP4rev1 implementations MAY + transmit 8-bit or multi-octet characters in literals, but SHOULD do + so only when the [CHARSET] is identified. + + Although a BINARY body encoding is defined, unencoded binary strings + are not permitted. A "binary string" is any string with NUL + characters. Implementations MUST encode binary data into a textual + form, such as BASE64, before transmitting the data. A string with an + excessive amount of CTL characters MAY also be considered to be + binary. + +4.4. Parenthesized List + + Data structures are represented as a "parenthesized list"; a sequence + of data items, delimited by space, and bounded at each end by + parentheses. A parenthesized list can contain other parenthesized + lists, using multiple levels of parentheses to indicate nesting. + + The empty list is represented as () -- a parenthesized list with no + members. + +4.5. NIL + + The special form "NIL" represents the non-existence of a particular + data item that is represented as a string or parenthesized list, as + distinct from the empty string "" or the empty parenthesized list (). + + Note: NIL is never used for any data item which takes the + form of an atom. For example, a mailbox name of "NIL" is a + mailbox named NIL as opposed to a non-existent mailbox + name. This is because mailbox uses "astring" syntax which + is an atom or a string. Conversely, an addr-name of NIL is + a non-existent personal name, because addr-name uses + "nstring" syntax which is NIL or a string, but never an + atom. + + + + + + + + + + + + + +Crispin Standards Track [Page 17] + +RFC 3501 IMAPv4 March 2003 + + +5. Operational Considerations + + The following rules are listed here to ensure that all IMAP4rev1 + implementations interoperate properly. + +5.1. Mailbox Naming + + Mailbox names are 7-bit. Client implementations MUST NOT attempt to + create 8-bit mailbox names, and SHOULD interpret any 8-bit mailbox + names returned by LIST or LSUB as UTF-8. Server implementations + SHOULD prohibit the creation of 8-bit mailbox names, and SHOULD NOT + return 8-bit mailbox names in LIST or LSUB. See section 5.1.3 for + more information on how to represent non-ASCII mailbox names. + + Note: 8-bit mailbox names were undefined in earlier + versions of this protocol. Some sites used a local 8-bit + character set to represent non-ASCII mailbox names. Such + usage is not interoperable, and is now formally deprecated. + + The case-insensitive mailbox name INBOX is a special name reserved to + mean "the primary mailbox for this user on this server". The + interpretation of all other names is implementation-dependent. + + In particular, this specification takes no position on case + sensitivity in non-INBOX mailbox names. Some server implementations + are fully case-sensitive; others preserve case of a newly-created + name but otherwise are case-insensitive; and yet others coerce names + to a particular case. Client implementations MUST interact with any + of these. If a server implementation interprets non-INBOX mailbox + names as case-insensitive, it MUST treat names using the + international naming convention specially as described in section + 5.1.3. + + There are certain client considerations when creating a new mailbox + name: + + 1) Any character which is one of the atom-specials (see the Formal + Syntax) will require that the mailbox name be represented as a + quoted string or literal. + + 2) CTL and other non-graphic characters are difficult to represent + in a user interface and are best avoided. + + 3) Although the list-wildcard characters ("%" and "*") are valid + in a mailbox name, it is difficult to use such mailbox names + with the LIST and LSUB commands due to the conflict with + wildcard interpretation. + + + + +Crispin Standards Track [Page 18] + +RFC 3501 IMAPv4 March 2003 + + + 4) Usually, a character (determined by the server implementation) + is reserved to delimit levels of hierarchy. + + 5) Two characters, "#" and "&", have meanings by convention, and + should be avoided except when used in that convention. + +5.1.1. Mailbox Hierarchy Naming + + If it is desired to export hierarchical mailbox names, mailbox names + MUST be left-to-right hierarchical using a single character to + separate levels of hierarchy. The same hierarchy separator character + is used for all levels of hierarchy within a single name. + +5.1.2. Mailbox Namespace Naming Convention + + By convention, the first hierarchical element of any mailbox name + which begins with "#" identifies the "namespace" of the remainder of + the name. This makes it possible to disambiguate between different + types of mailbox stores, each of which have their own namespaces. + + For example, implementations which offer access to USENET + newsgroups MAY use the "#news" namespace to partition the + USENET newsgroup namespace from that of other mailboxes. + Thus, the comp.mail.misc newsgroup would have a mailbox + name of "#news.comp.mail.misc", and the name + "comp.mail.misc" can refer to a different object (e.g., a + user's private mailbox). + +5.1.3. Mailbox International Naming Convention + + By convention, international mailbox names in IMAP4rev1 are specified + using a modified version of the UTF-7 encoding described in [UTF-7]. + Modified UTF-7 may also be usable in servers that implement an + earlier version of this protocol. + + In modified UTF-7, printable US-ASCII characters, except for "&", + represent themselves; that is, characters with octet values 0x20-0x25 + and 0x27-0x7e. The character "&" (0x26) is represented by the + two-octet sequence "&-". + + All other characters (octet values 0x00-0x1f and 0x7f-0xff) are + represented in modified BASE64, with a further modification from + [UTF-7] that "," is used instead of "/". Modified BASE64 MUST NOT be + used to represent any printing US-ASCII character which can represent + itself. + + + + + + +Crispin Standards Track [Page 19] + +RFC 3501 IMAPv4 March 2003 + + + "&" is used to shift to modified BASE64 and "-" to shift back to + US-ASCII. There is no implicit shift from BASE64 to US-ASCII, and + null shifts ("-&" while in BASE64; note that "&-" while in US-ASCII + means "&") are not permitted. However, all names start in US-ASCII, + and MUST end in US-ASCII; that is, a name that ends with a non-ASCII + ISO-10646 character MUST end with a "-"). + + The purpose of these modifications is to correct the following + problems with UTF-7: + + 1) UTF-7 uses the "+" character for shifting; this conflicts with + the common use of "+" in mailbox names, in particular USENET + newsgroup names. + + 2) UTF-7's encoding is BASE64 which uses the "/" character; this + conflicts with the use of "/" as a popular hierarchy delimiter. + + 3) UTF-7 prohibits the unencoded usage of "\"; this conflicts with + the use of "\" as a popular hierarchy delimiter. + + 4) UTF-7 prohibits the unencoded usage of "~"; this conflicts with + the use of "~" in some servers as a home directory indicator. + + 5) UTF-7 permits multiple alternate forms to represent the same + string; in particular, printable US-ASCII characters can be + represented in encoded form. + + Although modified UTF-7 is a convention, it establishes certain + requirements on server handling of any mailbox name with an + embedded "&" character. In particular, server implementations + MUST preserve the exact form of the modified BASE64 portion of a + modified UTF-7 name and treat that text as case-sensitive, even if + names are otherwise case-insensitive or case-folded. + + Server implementations SHOULD verify that any mailbox name with an + embedded "&" character, used as an argument to CREATE, is: in the + correctly modified UTF-7 syntax, has no superfluous shifts, and + has no encoding in modified BASE64 of any printing US-ASCII + character which can represent itself. However, client + implementations MUST NOT depend upon the server doing this, and + SHOULD NOT attempt to create a mailbox name with an embedded "&" + character unless it complies with the modified UTF-7 syntax. + + Server implementations which export a mail store that does not + follow the modified UTF-7 convention MUST convert to modified + UTF-7 any mailbox name that contains either non-ASCII characters + or the "&" character. + + + + +Crispin Standards Track [Page 20] + +RFC 3501 IMAPv4 March 2003 + + + For example, here is a mailbox name which mixes English, + Chinese, and Japanese text: + ~peter/mail/&U,BTFw-/&ZeVnLIqe- + + For example, the string "&Jjo!" is not a valid mailbox + name because it does not contain a shift to US-ASCII + before the "!". The correct form is "&Jjo-!". The + string "&U,BTFw-&ZeVnLIqe-" is not permitted because it + contains a superfluous shift. The correct form is + "&U,BTF2XlZyyKng-". + +5.2. Mailbox Size and Message Status Updates + + At any time, a server can send data that the client did not request. + Sometimes, such behavior is REQUIRED. For example, agents other than + the server MAY add messages to the mailbox (e.g., new message + delivery), change the flags of the messages in the mailbox (e.g., + simultaneous access to the same mailbox by multiple agents), or even + remove messages from the mailbox. A server MUST send mailbox size + updates automatically if a mailbox size change is observed during the + processing of a command. A server SHOULD send message flag updates + automatically, without requiring the client to request such updates + explicitly. + + Special rules exist for server notification of a client about the + removal of messages to prevent synchronization errors; see the + description of the EXPUNGE response for more detail. In particular, + it is NOT permitted to send an EXISTS response that would reduce the + number of messages in the mailbox; only the EXPUNGE response can do + this. + + Regardless of what implementation decisions a client makes on + remembering data from the server, a client implementation MUST record + mailbox size updates. It MUST NOT assume that any command after the + initial mailbox selection will return the size of the mailbox. + +5.3. Response when no Command in Progress + + Server implementations are permitted to send an untagged response + (except for EXPUNGE) while there is no command in progress. Server + implementations that send such responses MUST deal with flow control + considerations. Specifically, they MUST either (1) verify that the + size of the data does not exceed the underlying transport's available + window size, or (2) use non-blocking writes. + + + + + + + +Crispin Standards Track [Page 21] + +RFC 3501 IMAPv4 March 2003 + + +5.4. Autologout Timer + + If a server has an inactivity autologout timer, the duration of that + timer MUST be at least 30 minutes. The receipt of ANY command from + the client during that interval SHOULD suffice to reset the + autologout timer. + +5.5. Multiple Commands in Progress + + The client MAY send another command without waiting for the + completion result response of a command, subject to ambiguity rules + (see below) and flow control constraints on the underlying data + stream. Similarly, a server MAY begin processing another command + before processing the current command to completion, subject to + ambiguity rules. However, any command continuation request responses + and command continuations MUST be negotiated before any subsequent + command is initiated. + + The exception is if an ambiguity would result because of a command + that would affect the results of other commands. Clients MUST NOT + send multiple commands without waiting if an ambiguity would result. + If the server detects a possible ambiguity, it MUST execute commands + to completion in the order given by the client. + + The most obvious example of ambiguity is when a command would affect + the results of another command, e.g., a FETCH of a message's flags + and a STORE of that same message's flags. + + A non-obvious ambiguity occurs with commands that permit an untagged + EXPUNGE response (commands other than FETCH, STORE, and SEARCH), + since an untagged EXPUNGE response can invalidate sequence numbers in + a subsequent command. This is not a problem for FETCH, STORE, or + SEARCH commands because servers are prohibited from sending EXPUNGE + responses while any of those commands are in progress. Therefore, if + the client sends any command other than FETCH, STORE, or SEARCH, it + MUST wait for the completion result response before sending a command + with message sequence numbers. + + Note: UID FETCH, UID STORE, and UID SEARCH are different + commands from FETCH, STORE, and SEARCH. If the client + sends a UID command, it must wait for a completion result + response before sending a command with message sequence + numbers. + + + + + + + + +Crispin Standards Track [Page 22] + +RFC 3501 IMAPv4 March 2003 + + + For example, the following non-waiting command sequences are invalid: + + FETCH + NOOP + STORE + STORE + COPY + FETCH + COPY + COPY + CHECK + FETCH + + The following are examples of valid non-waiting command sequences: + + FETCH + STORE + SEARCH + CHECK + STORE + COPY + EXPUNGE + + UID SEARCH + UID SEARCH may be valid or invalid as a non-waiting + command sequence, depending upon whether or not the second UID + SEARCH contains message sequence numbers. + +6. Client Commands + + IMAP4rev1 commands are described in this section. Commands are + organized by the state in which the command is permitted. Commands + which are permitted in multiple states are listed in the minimum + permitted state (for example, commands valid in authenticated and + selected state are listed in the authenticated state commands). + + Command arguments, identified by "Arguments:" in the command + descriptions below, are described by function, not by syntax. The + precise syntax of command arguments is described in the Formal Syntax + section. + + Some commands cause specific server responses to be returned; these + are identified by "Responses:" in the command descriptions below. + See the response descriptions in the Responses section for + information on these responses, and the Formal Syntax section for the + precise syntax of these responses. It is possible for server data to + be transmitted as a result of any command. Thus, commands that do + not specifically require server data specify "no specific responses + for this command" instead of "none". + + The "Result:" in the command description refers to the possible + tagged status responses to a command, and any special interpretation + of these status responses. + + The state of a connection is only changed by successful commands + which are documented as changing state. A rejected command (BAD + response) never changes the state of the connection or of the + selected mailbox. A failed command (NO response) generally does not + change the state of the connection or of the selected mailbox; the + exception being the SELECT and EXAMINE commands. + + + +Crispin Standards Track [Page 23] + +RFC 3501 IMAPv4 March 2003 + + +6.1. Client Commands - Any State + + The following commands are valid in any state: CAPABILITY, NOOP, and + LOGOUT. + +6.1.1. CAPABILITY Command + + Arguments: none + + Responses: REQUIRED untagged response: CAPABILITY + + Result: OK - capability completed + BAD - command unknown or arguments invalid + + The CAPABILITY command requests a listing of capabilities that the + server supports. The server MUST send a single untagged + CAPABILITY response with "IMAP4rev1" as one of the listed + capabilities before the (tagged) OK response. + + A capability name which begins with "AUTH=" indicates that the + server supports that particular authentication mechanism. All + such names are, by definition, part of this specification. For + example, the authorization capability for an experimental + "blurdybloop" authenticator would be "AUTH=XBLURDYBLOOP" and not + "XAUTH=BLURDYBLOOP" or "XAUTH=XBLURDYBLOOP". + + Other capability names refer to extensions, revisions, or + amendments to this specification. See the documentation of the + CAPABILITY response for additional information. No capabilities, + beyond the base IMAP4rev1 set defined in this specification, are + enabled without explicit client action to invoke the capability. + + Client and server implementations MUST implement the STARTTLS, + LOGINDISABLED, and AUTH=PLAIN (described in [IMAP-TLS]) + capabilities. See the Security Considerations section for + important information. + + See the section entitled "Client Commands - + Experimental/Expansion" for information about the form of site or + implementation-specific capabilities. + + + + + + + + + + + +Crispin Standards Track [Page 24] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: abcd CAPABILITY + S: * CAPABILITY IMAP4rev1 STARTTLS AUTH=GSSAPI + LOGINDISABLED + S: abcd OK CAPABILITY completed + C: efgh STARTTLS + S: efgh OK STARTLS completed + + C: ijkl CAPABILITY + S: * CAPABILITY IMAP4rev1 AUTH=GSSAPI AUTH=PLAIN + S: ijkl OK CAPABILITY completed + + +6.1.2. NOOP Command + + Arguments: none + + Responses: no specific responses for this command (but see below) + + Result: OK - noop completed + BAD - command unknown or arguments invalid + + The NOOP command always succeeds. It does nothing. + + Since any command can return a status update as untagged data, the + NOOP command can be used as a periodic poll for new messages or + message status updates during a period of inactivity (this is the + preferred method to do this). The NOOP command can also be used + to reset any inactivity autologout timer on the server. + + Example: C: a002 NOOP + S: a002 OK NOOP completed + . . . + C: a047 NOOP + S: * 22 EXPUNGE + S: * 23 EXISTS + S: * 3 RECENT + S: * 14 FETCH (FLAGS (\Seen \Deleted)) + S: a047 OK NOOP completed + + + + + + + + + + + + + +Crispin Standards Track [Page 25] + +RFC 3501 IMAPv4 March 2003 + + +6.1.3. LOGOUT Command + + Arguments: none + + Responses: REQUIRED untagged response: BYE + + Result: OK - logout completed + BAD - command unknown or arguments invalid + + The LOGOUT command informs the server that the client is done with + the connection. The server MUST send a BYE untagged response + before the (tagged) OK response, and then close the network + connection. + + Example: C: A023 LOGOUT + S: * BYE IMAP4rev1 Server logging out + S: A023 OK LOGOUT completed + (Server and client then close the connection) + +6.2. Client Commands - Not Authenticated State + + In the not authenticated state, the AUTHENTICATE or LOGIN command + establishes authentication and enters the authenticated state. The + AUTHENTICATE command provides a general mechanism for a variety of + authentication techniques, privacy protection, and integrity + checking; whereas the LOGIN command uses a traditional user name and + plaintext password pair and has no means of establishing privacy + protection or integrity checking. + + The STARTTLS command is an alternate form of establishing session + privacy protection and integrity checking, but does not establish + authentication or enter the authenticated state. + + Server implementations MAY allow access to certain mailboxes without + establishing authentication. This can be done by means of the + ANONYMOUS [SASL] authenticator described in [ANONYMOUS]. An older + convention is a LOGIN command using the userid "anonymous"; in this + case, a password is required although the server may choose to accept + any password. The restrictions placed on anonymous users are + implementation-dependent. + + Once authenticated (including as anonymous), it is not possible to + re-enter not authenticated state. + + + + + + + + +Crispin Standards Track [Page 26] + +RFC 3501 IMAPv4 March 2003 + + + In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), + the following commands are valid in the not authenticated state: + STARTTLS, AUTHENTICATE and LOGIN. See the Security Considerations + section for important information about these commands. + +6.2.1. STARTTLS Command + + Arguments: none + + Responses: no specific response for this command + + Result: OK - starttls completed, begin TLS negotiation + BAD - command unknown or arguments invalid + + A [TLS] negotiation begins immediately after the CRLF at the end + of the tagged OK response from the server. Once a client issues a + STARTTLS command, it MUST NOT issue further commands until a + server response is seen and the [TLS] negotiation is complete. + + The server remains in the non-authenticated state, even if client + credentials are supplied during the [TLS] negotiation. This does + not preclude an authentication mechanism such as EXTERNAL (defined + in [SASL]) from using client identity determined by the [TLS] + negotiation. + + Once [TLS] has been started, the client MUST discard cached + information about server capabilities and SHOULD re-issue the + CAPABILITY command. This is necessary to protect against man-in- + the-middle attacks which alter the capabilities list prior to + STARTTLS. The server MAY advertise different capabilities after + STARTTLS. + + Example: C: a001 CAPABILITY + S: * CAPABILITY IMAP4rev1 STARTTLS LOGINDISABLED + S: a001 OK CAPABILITY completed + C: a002 STARTTLS + S: a002 OK Begin TLS negotiation now + + C: a003 CAPABILITY + S: * CAPABILITY IMAP4rev1 AUTH=PLAIN + S: a003 OK CAPABILITY completed + C: a004 LOGIN joe password + S: a004 OK LOGIN completed + + + + + + + + +Crispin Standards Track [Page 27] + +RFC 3501 IMAPv4 March 2003 + + +6.2.2. AUTHENTICATE Command + + Arguments: authentication mechanism name + + Responses: continuation data can be requested + + Result: OK - authenticate completed, now in authenticated state + NO - authenticate failure: unsupported authentication + mechanism, credentials rejected + BAD - command unknown or arguments invalid, + authentication exchange cancelled + + The AUTHENTICATE command indicates a [SASL] authentication + mechanism to the server. If the server supports the requested + authentication mechanism, it performs an authentication protocol + exchange to authenticate and identify the client. It MAY also + negotiate an OPTIONAL security layer for subsequent protocol + interactions. If the requested authentication mechanism is not + supported, the server SHOULD reject the AUTHENTICATE command by + sending a tagged NO response. + + The AUTHENTICATE command does not support the optional "initial + response" feature of [SASL]. Section 5.1 of [SASL] specifies how + to handle an authentication mechanism which uses an initial + response. + + The service name specified by this protocol's profile of [SASL] is + "imap". + + The authentication protocol exchange consists of a series of + server challenges and client responses that are specific to the + authentication mechanism. A server challenge consists of a + command continuation request response with the "+" token followed + by a BASE64 encoded string. The client response consists of a + single line consisting of a BASE64 encoded string. If the client + wishes to cancel an authentication exchange, it issues a line + consisting of a single "*". If the server receives such a + response, it MUST reject the AUTHENTICATE command by sending a + tagged BAD response. + + If a security layer is negotiated through the [SASL] + authentication exchange, it takes effect immediately following the + CRLF that concludes the authentication exchange for the client, + and the CRLF of the tagged OK response for the server. + + While client and server implementations MUST implement the + AUTHENTICATE command itself, it is not required to implement any + authentication mechanisms other than the PLAIN mechanism described + + + +Crispin Standards Track [Page 28] + +RFC 3501 IMAPv4 March 2003 + + + in [IMAP-TLS]. Also, an authentication mechanism is not required + to support any security layers. + + Note: a server implementation MUST implement a + configuration in which it does NOT permit any plaintext + password mechanisms, unless either the STARTTLS command + has been negotiated or some other mechanism that + protects the session from password snooping has been + provided. Server sites SHOULD NOT use any configuration + which permits a plaintext password mechanism without + such a protection mechanism against password snooping. + Client and server implementations SHOULD implement + additional [SASL] mechanisms that do not use plaintext + passwords, such the GSSAPI mechanism described in [SASL] + and/or the [DIGEST-MD5] mechanism. + + Servers and clients can support multiple authentication + mechanisms. The server SHOULD list its supported authentication + mechanisms in the response to the CAPABILITY command so that the + client knows which authentication mechanisms to use. + + A server MAY include a CAPABILITY response code in the tagged OK + response of a successful AUTHENTICATE command in order to send + capabilities automatically. It is unnecessary for a client to + send a separate CAPABILITY command if it recognizes these + automatic capabilities. This should only be done if a security + layer was not negotiated by the AUTHENTICATE command, because the + tagged OK response as part of an AUTHENTICATE command is not + protected by encryption/integrity checking. [SASL] requires the + client to re-issue a CAPABILITY command in this case. + + If an AUTHENTICATE command fails with a NO response, the client + MAY try another authentication mechanism by issuing another + AUTHENTICATE command. It MAY also attempt to authenticate by + using the LOGIN command (see section 6.2.3 for more detail). In + other words, the client MAY request authentication types in + decreasing order of preference, with the LOGIN command as a last + resort. + + The authorization identity passed from the client to the server + during the authentication exchange is interpreted by the server as + the user name whose privileges the client is requesting. + + + + + + + + + +Crispin Standards Track [Page 29] + +RFC 3501 IMAPv4 March 2003 + + + Example: S: * OK IMAP4rev1 Server + C: A001 AUTHENTICATE GSSAPI + S: + + C: YIIB+wYJKoZIhvcSAQICAQBuggHqMIIB5qADAgEFoQMCAQ6iBw + MFACAAAACjggEmYYIBIjCCAR6gAwIBBaESGxB1Lndhc2hpbmd0 + b24uZWR1oi0wK6ADAgEDoSQwIhsEaW1hcBsac2hpdmFtcy5jYW + Mud2FzaGluZ3Rvbi5lZHWjgdMwgdCgAwIBAaEDAgEDooHDBIHA + cS1GSa5b+fXnPZNmXB9SjL8Ollj2SKyb+3S0iXMljen/jNkpJX + AleKTz6BQPzj8duz8EtoOuNfKgweViyn/9B9bccy1uuAE2HI0y + C/PHXNNU9ZrBziJ8Lm0tTNc98kUpjXnHZhsMcz5Mx2GR6dGknb + I0iaGcRerMUsWOuBmKKKRmVMMdR9T3EZdpqsBd7jZCNMWotjhi + vd5zovQlFqQ2Wjc2+y46vKP/iXxWIuQJuDiisyXF0Y8+5GTpAL + pHDc1/pIGmMIGjoAMCAQGigZsEgZg2on5mSuxoDHEA1w9bcW9n + FdFxDKpdrQhVGVRDIzcCMCTzvUboqb5KjY1NJKJsfjRQiBYBdE + NKfzK+g5DlV8nrw81uOcP8NOQCLR5XkoMHC0Dr/80ziQzbNqhx + O6652Npft0LQwJvenwDI13YxpwOdMXzkWZN/XrEqOWp6GCgXTB + vCyLWLlWnbaUkZdEYbKHBPjd8t/1x5Yg== + S: + YGgGCSqGSIb3EgECAgIAb1kwV6ADAgEFoQMCAQ+iSzBJoAMC + AQGiQgRAtHTEuOP2BXb9sBYFR4SJlDZxmg39IxmRBOhXRKdDA0 + uHTCOT9Bq3OsUTXUlk0CsFLoa8j+gvGDlgHuqzWHPSQg== + C: + S: + YDMGCSqGSIb3EgECAgIBAAD/////6jcyG4GE3KkTzBeBiVHe + ceP2CWY0SR0fAQAgAAQEBAQ= + C: YDMGCSqGSIb3EgECAgIBAAD/////3LQBHXTpFfZgrejpLlLImP + wkhbfa2QteAQAgAG1yYwE= + S: A001 OK GSSAPI authentication successful + + Note: The line breaks within server challenges and client + responses are for editorial clarity and are not in real + authenticators. + + +6.2.3. LOGIN Command + + Arguments: user name + password + + Responses: no specific responses for this command + + Result: OK - login completed, now in authenticated state + NO - login failure: user name or password rejected + BAD - command unknown or arguments invalid + + The LOGIN command identifies the client to the server and carries + the plaintext password authenticating this user. + + + + + + +Crispin Standards Track [Page 30] + +RFC 3501 IMAPv4 March 2003 + + + A server MAY include a CAPABILITY response code in the tagged OK + response to a successful LOGIN command in order to send + capabilities automatically. It is unnecessary for a client to + send a separate CAPABILITY command if it recognizes these + automatic capabilities. + + Example: C: a001 LOGIN SMITH SESAME + S: a001 OK LOGIN completed + + Note: Use of the LOGIN command over an insecure network + (such as the Internet) is a security risk, because anyone + monitoring network traffic can obtain plaintext passwords. + The LOGIN command SHOULD NOT be used except as a last + resort, and it is recommended that client implementations + have a means to disable any automatic use of the LOGIN + command. + + Unless either the STARTTLS command has been negotiated or + some other mechanism that protects the session from + password snooping has been provided, a server + implementation MUST implement a configuration in which it + advertises the LOGINDISABLED capability and does NOT permit + the LOGIN command. Server sites SHOULD NOT use any + configuration which permits the LOGIN command without such + a protection mechanism against password snooping. A client + implementation MUST NOT send a LOGIN command if the + LOGINDISABLED capability is advertised. + +6.3. Client Commands - Authenticated State + + In the authenticated state, commands that manipulate mailboxes as + atomic entities are permitted. Of these commands, the SELECT and + EXAMINE commands will select a mailbox for access and enter the + selected state. + + In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), + the following commands are valid in the authenticated state: SELECT, + EXAMINE, CREATE, DELETE, RENAME, SUBSCRIBE, UNSUBSCRIBE, LIST, LSUB, + STATUS, and APPEND. + + + + + + + + + + + + +Crispin Standards Track [Page 31] + +RFC 3501 IMAPv4 March 2003 + + +6.3.1. SELECT Command + + Arguments: mailbox name + + Responses: REQUIRED untagged responses: FLAGS, EXISTS, RECENT + REQUIRED OK untagged responses: UNSEEN, PERMANENTFLAGS, + UIDNEXT, UIDVALIDITY + + Result: OK - select completed, now in selected state + NO - select failure, now in authenticated state: no + such mailbox, can't access mailbox + BAD - command unknown or arguments invalid + + The SELECT command selects a mailbox so that messages in the + mailbox can be accessed. Before returning an OK to the client, + the server MUST send the following untagged data to the client. + Note that earlier versions of this protocol only required the + FLAGS, EXISTS, and RECENT untagged data; consequently, client + implementations SHOULD implement default behavior for missing data + as discussed with the individual item. + + FLAGS Defined flags in the mailbox. See the description + of the FLAGS response for more detail. + + EXISTS The number of messages in the mailbox. See the + description of the EXISTS response for more detail. + + RECENT The number of messages with the \Recent flag set. + See the description of the RECENT response for more + detail. + + OK [UNSEEN ] + The message sequence number of the first unseen + message in the mailbox. If this is missing, the + client can not make any assumptions about the first + unseen message in the mailbox, and needs to issue a + SEARCH command if it wants to find it. + + OK [PERMANENTFLAGS ()] + A list of message flags that the client can change + permanently. If this is missing, the client should + assume that all flags can be changed permanently. + + OK [UIDNEXT ] + The next unique identifier value. Refer to section + 2.3.1.1 for more information. If this is missing, + the client can not make any assumptions about the + next unique identifier value. + + + +Crispin Standards Track [Page 32] + +RFC 3501 IMAPv4 March 2003 + + + OK [UIDVALIDITY ] + The unique identifier validity value. Refer to + section 2.3.1.1 for more information. If this is + missing, the server does not support unique + identifiers. + + Only one mailbox can be selected at a time in a connection; + simultaneous access to multiple mailboxes requires multiple + connections. The SELECT command automatically deselects any + currently selected mailbox before attempting the new selection. + Consequently, if a mailbox is selected and a SELECT command that + fails is attempted, no mailbox is selected. + + If the client is permitted to modify the mailbox, the server + SHOULD prefix the text of the tagged OK response with the + "[READ-WRITE]" response code. + + If the client is not permitted to modify the mailbox but is + permitted read access, the mailbox is selected as read-only, and + the server MUST prefix the text of the tagged OK response to + SELECT with the "[READ-ONLY]" response code. Read-only access + through SELECT differs from the EXAMINE command in that certain + read-only mailboxes MAY permit the change of permanent state on a + per-user (as opposed to global) basis. Netnews messages marked in + a server-based .newsrc file are an example of such per-user + permanent state that can be modified with read-only mailboxes. + + Example: C: A142 SELECT INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: A142 OK [READ-WRITE] SELECT completed + + + + + + + + + + + + + + + +Crispin Standards Track [Page 33] + +RFC 3501 IMAPv4 March 2003 + + +6.3.2. EXAMINE Command + + Arguments: mailbox name + + Responses: REQUIRED untagged responses: FLAGS, EXISTS, RECENT + REQUIRED OK untagged responses: UNSEEN, PERMANENTFLAGS, + UIDNEXT, UIDVALIDITY + + Result: OK - examine completed, now in selected state + NO - examine failure, now in authenticated state: no + such mailbox, can't access mailbox + BAD - command unknown or arguments invalid + + The EXAMINE command is identical to SELECT and returns the same + output; however, the selected mailbox is identified as read-only. + No changes to the permanent state of the mailbox, including + per-user state, are permitted; in particular, EXAMINE MUST NOT + cause messages to lose the \Recent flag. + + The text of the tagged OK response to the EXAMINE command MUST + begin with the "[READ-ONLY]" response code. + + Example: C: A932 EXAMINE blurdybloop + S: * 17 EXISTS + S: * 2 RECENT + S: * OK [UNSEEN 8] Message 8 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS ()] No permanent flags permitted + S: A932 OK [READ-ONLY] EXAMINE completed + + +6.3.3. CREATE Command + + Arguments: mailbox name + + Responses: no specific responses for this command + + Result: OK - create completed + NO - create failure: can't create mailbox with that name + BAD - command unknown or arguments invalid + + The CREATE command creates a mailbox with the given name. An OK + response is returned only if a new mailbox with that name has been + created. It is an error to attempt to create INBOX or a mailbox + with a name that refers to an extant mailbox. Any error in + creation will return a tagged NO response. + + + +Crispin Standards Track [Page 34] + +RFC 3501 IMAPv4 March 2003 + + + If the mailbox name is suffixed with the server's hierarchy + separator character (as returned from the server by a LIST + command), this is a declaration that the client intends to create + mailbox names under this name in the hierarchy. Server + implementations that do not require this declaration MUST ignore + the declaration. In any case, the name created is without the + trailing hierarchy delimiter. + + If the server's hierarchy separator character appears elsewhere in + the name, the server SHOULD create any superior hierarchical names + that are needed for the CREATE command to be successfully + completed. In other words, an attempt to create "foo/bar/zap" on + a server in which "/" is the hierarchy separator character SHOULD + create foo/ and foo/bar/ if they do not already exist. + + If a new mailbox is created with the same name as a mailbox which + was deleted, its unique identifiers MUST be greater than any + unique identifiers used in the previous incarnation of the mailbox + UNLESS the new incarnation has a different unique identifier + validity value. See the description of the UID command for more + detail. + + Example: C: A003 CREATE owatagusiam/ + S: A003 OK CREATE completed + C: A004 CREATE owatagusiam/blurdybloop + S: A004 OK CREATE completed + + Note: The interpretation of this example depends on whether + "/" was returned as the hierarchy separator from LIST. If + "/" is the hierarchy separator, a new level of hierarchy + named "owatagusiam" with a member called "blurdybloop" is + created. Otherwise, two mailboxes at the same hierarchy + level are created. + + +6.3.4. DELETE Command + + Arguments: mailbox name + + Responses: no specific responses for this command + + Result: OK - delete completed + NO - delete failure: can't delete mailbox with that name + BAD - command unknown or arguments invalid + + + + + + + +Crispin Standards Track [Page 35] + +RFC 3501 IMAPv4 March 2003 + + + The DELETE command permanently removes the mailbox with the given + name. A tagged OK response is returned only if the mailbox has + been deleted. It is an error to attempt to delete INBOX or a + mailbox name that does not exist. + + The DELETE command MUST NOT remove inferior hierarchical names. + For example, if a mailbox "foo" has an inferior "foo.bar" + (assuming "." is the hierarchy delimiter character), removing + "foo" MUST NOT remove "foo.bar". It is an error to attempt to + delete a name that has inferior hierarchical names and also has + the \Noselect mailbox name attribute (see the description of the + LIST response for more details). + + It is permitted to delete a name that has inferior hierarchical + names and does not have the \Noselect mailbox name attribute. In + this case, all messages in that mailbox are removed, and the name + will acquire the \Noselect mailbox name attribute. + + The value of the highest-used unique identifier of the deleted + mailbox MUST be preserved so that a new mailbox created with the + same name will not reuse the identifiers of the former + incarnation, UNLESS the new incarnation has a different unique + identifier validity value. See the description of the UID command + for more detail. + + Examples: C: A682 LIST "" * + S: * LIST () "/" blurdybloop + S: * LIST (\Noselect) "/" foo + S: * LIST () "/" foo/bar + S: A682 OK LIST completed + C: A683 DELETE blurdybloop + S: A683 OK DELETE completed + C: A684 DELETE foo + S: A684 NO Name "foo" has inferior hierarchical names + C: A685 DELETE foo/bar + S: A685 OK DELETE Completed + C: A686 LIST "" * + S: * LIST (\Noselect) "/" foo + S: A686 OK LIST completed + C: A687 DELETE foo + S: A687 OK DELETE Completed + + + + + + + + + + +Crispin Standards Track [Page 36] + +RFC 3501 IMAPv4 March 2003 + + + C: A82 LIST "" * + S: * LIST () "." blurdybloop + S: * LIST () "." foo + S: * LIST () "." foo.bar + S: A82 OK LIST completed + C: A83 DELETE blurdybloop + S: A83 OK DELETE completed + C: A84 DELETE foo + S: A84 OK DELETE Completed + C: A85 LIST "" * + S: * LIST () "." foo.bar + S: A85 OK LIST completed + C: A86 LIST "" % + S: * LIST (\Noselect) "." foo + S: A86 OK LIST completed + + +6.3.5. RENAME Command + + Arguments: existing mailbox name + new mailbox name + + Responses: no specific responses for this command + + Result: OK - rename completed + NO - rename failure: can't rename mailbox with that name, + can't rename to mailbox with that name + BAD - command unknown or arguments invalid + + The RENAME command changes the name of a mailbox. A tagged OK + response is returned only if the mailbox has been renamed. It is + an error to attempt to rename from a mailbox name that does not + exist or to a mailbox name that already exists. Any error in + renaming will return a tagged NO response. + + If the name has inferior hierarchical names, then the inferior + hierarchical names MUST also be renamed. For example, a rename of + "foo" to "zap" will rename "foo/bar" (assuming "/" is the + hierarchy delimiter character) to "zap/bar". + + If the server's hierarchy separator character appears in the name, + the server SHOULD create any superior hierarchical names that are + needed for the RENAME command to complete successfully. In other + words, an attempt to rename "foo/bar/zap" to baz/rag/zowie on a + server in which "/" is the hierarchy separator character SHOULD + create baz/ and baz/rag/ if they do not already exist. + + + + + +Crispin Standards Track [Page 37] + +RFC 3501 IMAPv4 March 2003 + + + The value of the highest-used unique identifier of the old mailbox + name MUST be preserved so that a new mailbox created with the same + name will not reuse the identifiers of the former incarnation, + UNLESS the new incarnation has a different unique identifier + validity value. See the description of the UID command for more + detail. + + Renaming INBOX is permitted, and has special behavior. It moves + all messages in INBOX to a new mailbox with the given name, + leaving INBOX empty. If the server implementation supports + inferior hierarchical names of INBOX, these are unaffected by a + rename of INBOX. + + Examples: C: A682 LIST "" * + S: * LIST () "/" blurdybloop + S: * LIST (\Noselect) "/" foo + S: * LIST () "/" foo/bar + S: A682 OK LIST completed + C: A683 RENAME blurdybloop sarasoop + S: A683 OK RENAME completed + C: A684 RENAME foo zowie + S: A684 OK RENAME Completed + C: A685 LIST "" * + S: * LIST () "/" sarasoop + S: * LIST (\Noselect) "/" zowie + S: * LIST () "/" zowie/bar + S: A685 OK LIST completed + + C: Z432 LIST "" * + S: * LIST () "." INBOX + S: * LIST () "." INBOX.bar + S: Z432 OK LIST completed + C: Z433 RENAME INBOX old-mail + S: Z433 OK RENAME completed + C: Z434 LIST "" * + S: * LIST () "." INBOX + S: * LIST () "." INBOX.bar + S: * LIST () "." old-mail + S: Z434 OK LIST completed + + + + + + + + + + + + +Crispin Standards Track [Page 38] + +RFC 3501 IMAPv4 March 2003 + + +6.3.6. SUBSCRIBE Command + + Arguments: mailbox + + Responses: no specific responses for this command + + Result: OK - subscribe completed + NO - subscribe failure: can't subscribe to that name + BAD - command unknown or arguments invalid + + The SUBSCRIBE command adds the specified mailbox name to the + server's set of "active" or "subscribed" mailboxes as returned by + the LSUB command. This command returns a tagged OK response only + if the subscription is successful. + + A server MAY validate the mailbox argument to SUBSCRIBE to verify + that it exists. However, it MUST NOT unilaterally remove an + existing mailbox name from the subscription list even if a mailbox + by that name no longer exists. + + Note: This requirement is because a server site can + choose to routinely remove a mailbox with a well-known + name (e.g., "system-alerts") after its contents expire, + with the intention of recreating it when new contents + are appropriate. + + + Example: C: A002 SUBSCRIBE #news.comp.mail.mime + S: A002 OK SUBSCRIBE completed + + +6.3.7. UNSUBSCRIBE Command + + Arguments: mailbox name + + Responses: no specific responses for this command + + Result: OK - unsubscribe completed + NO - unsubscribe failure: can't unsubscribe that name + BAD - command unknown or arguments invalid + + The UNSUBSCRIBE command removes the specified mailbox name from + the server's set of "active" or "subscribed" mailboxes as returned + by the LSUB command. This command returns a tagged OK response + only if the unsubscription is successful. + + Example: C: A002 UNSUBSCRIBE #news.comp.mail.mime + S: A002 OK UNSUBSCRIBE completed + + + +Crispin Standards Track [Page 39] + +RFC 3501 IMAPv4 March 2003 + + +6.3.8. LIST Command + + Arguments: reference name + mailbox name with possible wildcards + + Responses: untagged responses: LIST + + Result: OK - list completed + NO - list failure: can't list that reference or name + BAD - command unknown or arguments invalid + + The LIST command returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name; see the description of the LIST reply for + more detail. + + The LIST command SHOULD return its data quickly, without undue + delay. For example, it SHOULD NOT go to excess trouble to + calculate the \Marked or \Unmarked status or perform other + processing; if each name requires 1 second of processing, then a + list of 1200 names would take 20 minutes! + + An empty ("" string) reference name argument indicates that the + mailbox name is interpreted as by SELECT. The returned mailbox + names MUST match the supplied mailbox name pattern. A non-empty + reference name argument is the name of a mailbox or a level of + mailbox hierarchy, and indicates the context in which the mailbox + name is interpreted. + + An empty ("" string) mailbox name argument is a special request to + return the hierarchy delimiter and the root name of the name given + in the reference. The value returned as the root MAY be the empty + string if the reference is non-rooted or is an empty string. In + all cases, a hierarchy delimiter (or NIL if there is no hierarchy) + is returned. This permits a client to get the hierarchy delimiter + (or find out that the mailbox names are flat) even when no + mailboxes by that name currently exist. + + The reference and mailbox name arguments are interpreted into a + canonical form that represents an unambiguous left-to-right + hierarchy. The returned mailbox names will be in the interpreted + form. + + + + + + + + +Crispin Standards Track [Page 40] + +RFC 3501 IMAPv4 March 2003 + + + Note: The interpretation of the reference argument is + implementation-defined. It depends upon whether the + server implementation has a concept of the "current + working directory" and leading "break out characters", + which override the current working directory. + + For example, on a server which exports a UNIX or NT + filesystem, the reference argument contains the current + working directory, and the mailbox name argument would + contain the name as interpreted in the current working + directory. + + If a server implementation has no concept of break out + characters, the canonical form is normally the reference + name appended with the mailbox name. Note that if the + server implements the namespace convention (section + 5.1.2), "#" is a break out character and must be treated + as such. + + If the reference argument is not a level of mailbox + hierarchy (that is, it is a \NoInferiors name), and/or + the reference argument does not end with the hierarchy + delimiter, it is implementation-dependent how this is + interpreted. For example, a reference of "foo/bar" and + mailbox name of "rag/baz" could be interpreted as + "foo/bar/rag/baz", "foo/barrag/baz", or "foo/rag/baz". + A client SHOULD NOT use such a reference argument except + at the explicit request of the user. A hierarchical + browser MUST NOT make any assumptions about server + interpretation of the reference unless the reference is + a level of mailbox hierarchy AND ends with the hierarchy + delimiter. + + Any part of the reference argument that is included in the + interpreted form SHOULD prefix the interpreted form. It SHOULD + also be in the same form as the reference name argument. This + rule permits the client to determine if the returned mailbox name + is in the context of the reference argument, or if something about + the mailbox argument overrode the reference argument. Without + this rule, the client would have to have knowledge of the server's + naming semantics including what characters are "breakouts" that + override a naming context. + + + + + + + + + +Crispin Standards Track [Page 41] + +RFC 3501 IMAPv4 March 2003 + + + For example, here are some examples of how references + and mailbox names might be interpreted on a UNIX-based + server: + + Reference Mailbox Name Interpretation + ------------ ------------ -------------- + ~smith/Mail/ foo.* ~smith/Mail/foo.* + archive/ % archive/% + #news. comp.mail.* #news.comp.mail.* + ~smith/Mail/ /usr/doc/foo /usr/doc/foo + archive/ ~fred/Mail/* ~fred/Mail/* + + The first three examples demonstrate interpretations in + the context of the reference argument. Note that + "~smith/Mail" SHOULD NOT be transformed into something + like "/u2/users/smith/Mail", or it would be impossible + for the client to determine that the interpretation was + in the context of the reference. + + The character "*" is a wildcard, and matches zero or more + characters at this position. The character "%" is similar to "*", + but it does not match a hierarchy delimiter. If the "%" wildcard + is the last character of a mailbox name argument, matching levels + of hierarchy are also returned. If these levels of hierarchy are + not also selectable mailboxes, they are returned with the + \Noselect mailbox name attribute (see the description of the LIST + response for more details). + + Server implementations are permitted to "hide" otherwise + accessible mailboxes from the wildcard characters, by preventing + certain characters or names from matching a wildcard in certain + situations. For example, a UNIX-based server might restrict the + interpretation of "*" so that an initial "/" character does not + match. + + The special name INBOX is included in the output from LIST, if + INBOX is supported by this server for this user and if the + uppercase string "INBOX" matches the interpreted reference and + mailbox name arguments with wildcards as described above. The + criteria for omitting INBOX is whether SELECT INBOX will return + failure; it is not relevant whether the user's real INBOX resides + on this or some other server. + + + + + + + + + +Crispin Standards Track [Page 42] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: A101 LIST "" "" + S: * LIST (\Noselect) "/" "" + S: A101 OK LIST Completed + C: A102 LIST #news.comp.mail.misc "" + S: * LIST (\Noselect) "." #news. + S: A102 OK LIST Completed + C: A103 LIST /usr/staff/jones "" + S: * LIST (\Noselect) "/" / + S: A103 OK LIST Completed + C: A202 LIST ~/Mail/ % + S: * LIST (\Noselect) "/" ~/Mail/foo + S: * LIST () "/" ~/Mail/meetings + S: A202 OK LIST completed + + +6.3.9. LSUB Command + + Arguments: reference name + mailbox name with possible wildcards + + Responses: untagged responses: LSUB + + Result: OK - lsub completed + NO - lsub failure: can't list that reference or name + BAD - command unknown or arguments invalid + + The LSUB command returns a subset of names from the set of names + that the user has declared as being "active" or "subscribed". + Zero or more untagged LSUB replies are returned. The arguments to + LSUB are in the same form as those for LIST. + + The returned untagged LSUB response MAY contain different mailbox + flags from a LIST untagged response. If this should happen, the + flags in the untagged LIST are considered more authoritative. + + A special situation occurs when using LSUB with the % wildcard. + Consider what happens if "foo/bar" (with a hierarchy delimiter of + "/") is subscribed but "foo" is not. A "%" wildcard to LSUB must + return foo, not foo/bar, in the LSUB response, and it MUST be + flagged with the \Noselect attribute. + + The server MUST NOT unilaterally remove an existing mailbox name + from the subscription list even if a mailbox by that name no + longer exists. + + + + + + + +Crispin Standards Track [Page 43] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: A002 LSUB "#news." "comp.mail.*" + S: * LSUB () "." #news.comp.mail.mime + S: * LSUB () "." #news.comp.mail.misc + S: A002 OK LSUB completed + C: A003 LSUB "#news." "comp.%" + S: * LSUB (\NoSelect) "." #news.comp.mail + S: A003 OK LSUB completed + + +6.3.10. STATUS Command + + Arguments: mailbox name + status data item names + + Responses: untagged responses: STATUS + + Result: OK - status completed + NO - status failure: no status for that name + BAD - command unknown or arguments invalid + + The STATUS command requests the status of the indicated mailbox. + It does not change the currently selected mailbox, nor does it + affect the state of any messages in the queried mailbox (in + particular, STATUS MUST NOT cause messages to lose the \Recent + flag). + + The STATUS command provides an alternative to opening a second + IMAP4rev1 connection and doing an EXAMINE command on a mailbox to + query that mailbox's status without deselecting the current + mailbox in the first IMAP4rev1 connection. + + Unlike the LIST command, the STATUS command is not guaranteed to + be fast in its response. Under certain circumstances, it can be + quite slow. In some implementations, the server is obliged to + open the mailbox read-only internally to obtain certain status + information. Also unlike the LIST command, the STATUS command + does not accept wildcards. + + Note: The STATUS command is intended to access the + status of mailboxes other than the currently selected + mailbox. Because the STATUS command can cause the + mailbox to be opened internally, and because this + information is available by other means on the selected + mailbox, the STATUS command SHOULD NOT be used on the + currently selected mailbox. + + + + + + +Crispin Standards Track [Page 44] + +RFC 3501 IMAPv4 March 2003 + + + The STATUS command MUST NOT be used as a "check for new + messages in the selected mailbox" operation (refer to + sections 7, 7.3.1, and 7.3.2 for more information about + the proper method for new message checking). + + Because the STATUS command is not guaranteed to be fast + in its results, clients SHOULD NOT expect to be able to + issue many consecutive STATUS commands and obtain + reasonable performance. + + The currently defined status data items that can be requested are: + + MESSAGES + The number of messages in the mailbox. + + RECENT + The number of messages with the \Recent flag set. + + UIDNEXT + The next unique identifier value of the mailbox. Refer to + section 2.3.1.1 for more information. + + UIDVALIDITY + The unique identifier validity value of the mailbox. Refer to + section 2.3.1.1 for more information. + + UNSEEN + The number of messages which do not have the \Seen flag set. + + + Example: C: A042 STATUS blurdybloop (UIDNEXT MESSAGES) + S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292) + S: A042 OK STATUS completed + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 45] + +RFC 3501 IMAPv4 March 2003 + + +6.3.11. APPEND Command + + Arguments: mailbox name + OPTIONAL flag parenthesized list + OPTIONAL date/time string + message literal + + Responses: no specific responses for this command + + Result: OK - append completed + NO - append error: can't append to that mailbox, error + in flags or date/time or message text + BAD - command unknown or arguments invalid + + The APPEND command appends the literal argument as a new message + to the end of the specified destination mailbox. This argument + SHOULD be in the format of an [RFC-2822] message. 8-bit + characters are permitted in the message. A server implementation + that is unable to preserve 8-bit data properly MUST be able to + reversibly convert 8-bit APPEND data to 7-bit using a [MIME-IMB] + content transfer encoding. + + Note: There MAY be exceptions, e.g., draft messages, in + which required [RFC-2822] header lines are omitted in + the message literal argument to APPEND. The full + implications of doing so MUST be understood and + carefully weighed. + + If a flag parenthesized list is specified, the flags SHOULD be set + in the resulting message; otherwise, the flag list of the + resulting message is set to empty by default. In either case, the + Recent flag is also set. + + If a date-time is specified, the internal date SHOULD be set in + the resulting message; otherwise, the internal date of the + resulting message is set to the current date and time by default. + + If the append is unsuccessful for any reason, the mailbox MUST be + restored to its state before the APPEND attempt; no partial + appending is permitted. + + If the destination mailbox does not exist, a server MUST return an + error, and MUST NOT automatically create the mailbox. Unless it + is certain that the destination mailbox can not be created, the + server MUST send the response code "[TRYCREATE]" as the prefix of + the text of the tagged NO response. This gives a hint to the + client that it can attempt a CREATE command and retry the APPEND + if the CREATE is successful. + + + +Crispin Standards Track [Page 46] + +RFC 3501 IMAPv4 March 2003 + + + If the mailbox is currently selected, the normal new message + actions SHOULD occur. Specifically, the server SHOULD notify the + client immediately via an untagged EXISTS response. If the server + does not do so, the client MAY issue a NOOP command (or failing + that, a CHECK command) after one or more APPEND commands. + + Example: C: A003 APPEND saved-messages (\Seen) {310} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK APPEND completed + + Note: The APPEND command is not used for message delivery, + because it does not provide a mechanism to transfer [SMTP] + envelope information. + +6.4. Client Commands - Selected State + + In the selected state, commands that manipulate messages in a mailbox + are permitted. + + In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), + and the authenticated state commands (SELECT, EXAMINE, CREATE, + DELETE, RENAME, SUBSCRIBE, UNSUBSCRIBE, LIST, LSUB, STATUS, and + APPEND), the following commands are valid in the selected state: + CHECK, CLOSE, EXPUNGE, SEARCH, FETCH, STORE, COPY, and UID. + +6.4.1. CHECK Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - check completed + BAD - command unknown or arguments invalid + + The CHECK command requests a checkpoint of the currently selected + mailbox. A checkpoint refers to any implementation-dependent + housekeeping associated with the mailbox (e.g., resolving the + server's in-memory state of the mailbox with the state on its + + + +Crispin Standards Track [Page 47] + +RFC 3501 IMAPv4 March 2003 + + + disk) that is not normally executed as part of each command. A + checkpoint MAY take a non-instantaneous amount of real time to + complete. If a server implementation has no such housekeeping + considerations, CHECK is equivalent to NOOP. + + There is no guarantee that an EXISTS untagged response will happen + as a result of CHECK. NOOP, not CHECK, SHOULD be used for new + message polling. + + Example: C: FXXZ CHECK + S: FXXZ OK CHECK Completed + + +6.4.2. CLOSE Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - close completed, now in authenticated state + BAD - command unknown or arguments invalid + + The CLOSE command permanently removes all messages that have the + \Deleted flag set from the currently selected mailbox, and returns + to the authenticated state from the selected state. No untagged + EXPUNGE responses are sent. + + No messages are removed, and no error is given, if the mailbox is + selected by an EXAMINE command or is otherwise selected read-only. + + Even if a mailbox is selected, a SELECT, EXAMINE, or LOGOUT + command MAY be issued without previously issuing a CLOSE command. + The SELECT, EXAMINE, and LOGOUT commands implicitly close the + currently selected mailbox without doing an expunge. However, + when many messages are deleted, a CLOSE-LOGOUT or CLOSE-SELECT + sequence is considerably faster than an EXPUNGE-LOGOUT or + EXPUNGE-SELECT because no untagged EXPUNGE responses (which the + client would probably ignore) are sent. + + Example: C: A341 CLOSE + S: A341 OK CLOSE completed + + + + + + + + + + +Crispin Standards Track [Page 48] + +RFC 3501 IMAPv4 March 2003 + + +6.4.3. EXPUNGE Command + + Arguments: none + + Responses: untagged responses: EXPUNGE + + Result: OK - expunge completed + NO - expunge failure: can't expunge (e.g., permission + denied) + BAD - command unknown or arguments invalid + + The EXPUNGE command permanently removes all messages that have the + \Deleted flag set from the currently selected mailbox. Before + returning an OK to the client, an untagged EXPUNGE response is + sent for each message that is removed. + + Example: C: A202 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 5 EXPUNGE + S: * 8 EXPUNGE + S: A202 OK EXPUNGE completed + + Note: In this example, messages 3, 4, 7, and 11 had the + \Deleted flag set. See the description of the EXPUNGE + response for further explanation. + + +6.4.4. SEARCH Command + + Arguments: OPTIONAL [CHARSET] specification + searching criteria (one or more) + + Responses: REQUIRED untagged response: SEARCH + + Result: OK - search completed + NO - search error: can't search that [CHARSET] or + criteria + BAD - command unknown or arguments invalid + + The SEARCH command searches the mailbox for messages that match + the given searching criteria. Searching criteria consist of one + or more search keys. The untagged SEARCH response from the server + contains a listing of message sequence numbers corresponding to + those messages that match the searching criteria. + + + + + + +Crispin Standards Track [Page 49] + +RFC 3501 IMAPv4 March 2003 + + + When multiple keys are specified, the result is the intersection + (AND function) of all the messages that match those keys. For + example, the criteria DELETED FROM "SMITH" SINCE 1-Feb-1994 refers + to all deleted messages from Smith that were placed in the mailbox + since February 1, 1994. A search key can also be a parenthesized + list of one or more search keys (e.g., for use with the OR and NOT + keys). + + Server implementations MAY exclude [MIME-IMB] body parts with + terminal content media types other than TEXT and MESSAGE from + consideration in SEARCH matching. + + The OPTIONAL [CHARSET] specification consists of the word + "CHARSET" followed by a registered [CHARSET]. It indicates the + [CHARSET] of the strings that appear in the search criteria. + [MIME-IMB] content transfer encodings, and [MIME-HDRS] strings in + [RFC-2822]/[MIME-IMB] headers, MUST be decoded before comparing + text in a [CHARSET] other than US-ASCII. US-ASCII MUST be + supported; other [CHARSET]s MAY be supported. + + If the server does not support the specified [CHARSET], it MUST + return a tagged NO response (not a BAD). This response SHOULD + contain the BADCHARSET response code, which MAY list the + [CHARSET]s supported by the server. + + In all search keys that use strings, a message matches the key if + the string is a substring of the field. The matching is + case-insensitive. + + The defined search keys are as follows. Refer to the Formal + Syntax section for the precise syntactic definitions of the + arguments. + + + Messages with message sequence numbers corresponding to the + specified message sequence number set. + + ALL + All messages in the mailbox; the default initial key for + ANDing. + + ANSWERED + Messages with the \Answered flag set. + + + + + + + + +Crispin Standards Track [Page 50] + +RFC 3501 IMAPv4 March 2003 + + + BCC + Messages that contain the specified string in the envelope + structure's BCC field. + + BEFORE + Messages whose internal date (disregarding time and timezone) + is earlier than the specified date. + + BODY + Messages that contain the specified string in the body of the + message. + + CC + Messages that contain the specified string in the envelope + structure's CC field. + + DELETED + Messages with the \Deleted flag set. + + DRAFT + Messages with the \Draft flag set. + + FLAGGED + Messages with the \Flagged flag set. + + FROM + Messages that contain the specified string in the envelope + structure's FROM field. + + HEADER + Messages that have a header with the specified field-name (as + defined in [RFC-2822]) and that contains the specified string + in the text of the header (what comes after the colon). If the + string to search is zero-length, this matches all messages that + have a header line with the specified field-name regardless of + the contents. + + KEYWORD + Messages with the specified keyword flag set. + + LARGER + Messages with an [RFC-2822] size larger than the specified + number of octets. + + NEW + Messages that have the \Recent flag set but not the \Seen flag. + This is functionally equivalent to "(RECENT UNSEEN)". + + + + +Crispin Standards Track [Page 51] + +RFC 3501 IMAPv4 March 2003 + + + NOT + Messages that do not match the specified search key. + + OLD + Messages that do not have the \Recent flag set. This is + functionally equivalent to "NOT RECENT" (as opposed to "NOT + NEW"). + + ON + Messages whose internal date (disregarding time and timezone) + is within the specified date. + + OR + Messages that match either search key. + + RECENT + Messages that have the \Recent flag set. + + SEEN + Messages that have the \Seen flag set. + + SENTBEFORE + Messages whose [RFC-2822] Date: header (disregarding time and + timezone) is earlier than the specified date. + + SENTON + Messages whose [RFC-2822] Date: header (disregarding time and + timezone) is within the specified date. + + SENTSINCE + Messages whose [RFC-2822] Date: header (disregarding time and + timezone) is within or later than the specified date. + + SINCE + Messages whose internal date (disregarding time and timezone) + is within or later than the specified date. + + SMALLER + Messages with an [RFC-2822] size smaller than the specified + number of octets. + + + + + + + + + + + +Crispin Standards Track [Page 52] + +RFC 3501 IMAPv4 March 2003 + + + SUBJECT + Messages that contain the specified string in the envelope + structure's SUBJECT field. + + TEXT + Messages that contain the specified string in the header or + body of the message. + + TO + Messages that contain the specified string in the envelope + structure's TO field. + + UID + Messages with unique identifiers corresponding to the specified + unique identifier set. Sequence set ranges are permitted. + + UNANSWERED + Messages that do not have the \Answered flag set. + + UNDELETED + Messages that do not have the \Deleted flag set. + + UNDRAFT + Messages that do not have the \Draft flag set. + + UNFLAGGED + Messages that do not have the \Flagged flag set. + + UNKEYWORD + Messages that do not have the specified keyword flag set. + + UNSEEN + Messages that do not have the \Seen flag set. + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 53] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: A282 SEARCH FLAGGED SINCE 1-Feb-1994 NOT FROM "Smith" + S: * SEARCH 2 84 882 + S: A282 OK SEARCH completed + C: A283 SEARCH TEXT "string not in mailbox" + S: * SEARCH + S: A283 OK SEARCH completed + C: A284 SEARCH CHARSET UTF-8 TEXT {6} + C: XXXXXX + S: * SEARCH 43 + S: A284 OK SEARCH completed + + Note: Since this document is restricted to 7-bit ASCII + text, it is not possible to show actual UTF-8 data. The + "XXXXXX" is a placeholder for what would be 6 octets of + 8-bit data in an actual transaction. + + +6.4.5. FETCH Command + + Arguments: sequence set + message data item names or macro + + Responses: untagged responses: FETCH + + Result: OK - fetch completed + NO - fetch error: can't fetch that data + BAD - command unknown or arguments invalid + + The FETCH command retrieves data associated with a message in the + mailbox. The data items to be fetched can be either a single atom + or a parenthesized list. + + Most data items, identified in the formal syntax under the + msg-att-static rule, are static and MUST NOT change for any + particular message. Other data items, identified in the formal + syntax under the msg-att-dynamic rule, MAY change, either as a + result of a STORE command or due to external events. + + For example, if a client receives an ENVELOPE for a + message when it already knows the envelope, it can + safely ignore the newly transmitted envelope. + + There are three macros which specify commonly-used sets of data + items, and can be used instead of data items. A macro must be + used by itself, and not in conjunction with other macros or data + items. + + + + + +Crispin Standards Track [Page 54] + +RFC 3501 IMAPv4 March 2003 + + + ALL + Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE ENVELOPE) + + FAST + Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE) + + FULL + Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE ENVELOPE + BODY) + + The currently defined data items that can be fetched are: + + BODY + Non-extensible form of BODYSTRUCTURE. + + BODY[
]<> + The text of a particular body section. The section + specification is a set of zero or more part specifiers + delimited by periods. A part specifier is either a part number + or one of the following: HEADER, HEADER.FIELDS, + HEADER.FIELDS.NOT, MIME, and TEXT. An empty section + specification refers to the entire message, including the + header. + + Every message has at least one part number. Non-[MIME-IMB] + messages, and non-multipart [MIME-IMB] messages with no + encapsulated message, only have a part 1. + + Multipart messages are assigned consecutive part numbers, as + they occur in the message. If a particular part is of type + message or multipart, its parts MUST be indicated by a period + followed by the part number within that nested multipart part. + + A part of type MESSAGE/RFC822 also has nested part numbers, + referring to parts of the MESSAGE part's body. + + The HEADER, HEADER.FIELDS, HEADER.FIELDS.NOT, and TEXT part + specifiers can be the sole part specifier or can be prefixed by + one or more numeric part specifiers, provided that the numeric + part specifier refers to a part of type MESSAGE/RFC822. The + MIME part specifier MUST be prefixed by one or more numeric + part specifiers. + + The HEADER, HEADER.FIELDS, and HEADER.FIELDS.NOT part + specifiers refer to the [RFC-2822] header of the message or of + an encapsulated [MIME-IMT] MESSAGE/RFC822 message. + HEADER.FIELDS and HEADER.FIELDS.NOT are followed by a list of + field-name (as defined in [RFC-2822]) names, and return a + + + +Crispin Standards Track [Page 55] + +RFC 3501 IMAPv4 March 2003 + + + subset of the header. The subset returned by HEADER.FIELDS + contains only those header fields with a field-name that + matches one of the names in the list; similarly, the subset + returned by HEADER.FIELDS.NOT contains only the header fields + with a non-matching field-name. The field-matching is + case-insensitive but otherwise exact. Subsetting does not + exclude the [RFC-2822] delimiting blank line between the header + and the body; the blank line is included in all header fetches, + except in the case of a message which has no body and no blank + line. + + The MIME part specifier refers to the [MIME-IMB] header for + this part. + + The TEXT part specifier refers to the text body of the message, + omitting the [RFC-2822] header. + + Here is an example of a complex message with some of its + part specifiers: + + HEADER ([RFC-2822] header of the message) + TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED + 1 TEXT/PLAIN + 2 APPLICATION/OCTET-STREAM + 3 MESSAGE/RFC822 + 3.HEADER ([RFC-2822] header of the message) + 3.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED + 3.1 TEXT/PLAIN + 3.2 APPLICATION/OCTET-STREAM + 4 MULTIPART/MIXED + 4.1 IMAGE/GIF + 4.1.MIME ([MIME-IMB] header for the IMAGE/GIF) + 4.2 MESSAGE/RFC822 + 4.2.HEADER ([RFC-2822] header of the message) + 4.2.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED + 4.2.1 TEXT/PLAIN + 4.2.2 MULTIPART/ALTERNATIVE + 4.2.2.1 TEXT/PLAIN + 4.2.2.2 TEXT/RICHTEXT + + + It is possible to fetch a substring of the designated text. + This is done by appending an open angle bracket ("<"), the + octet position of the first desired octet, a period, the + maximum number of octets desired, and a close angle bracket + (">") to the part specifier. If the starting octet is beyond + the end of the text, an empty string is returned. + + + + +Crispin Standards Track [Page 56] + +RFC 3501 IMAPv4 March 2003 + + + Any partial fetch that attempts to read beyond the end of the + text is truncated as appropriate. A partial fetch that starts + at octet 0 is returned as a partial fetch, even if this + truncation happened. + + Note: This means that BODY[]<0.2048> of a 1500-octet message + will return BODY[]<0> with a literal of size 1500, not + BODY[]. + + Note: A substring fetch of a HEADER.FIELDS or + HEADER.FIELDS.NOT part specifier is calculated after + subsetting the header. + + The \Seen flag is implicitly set; if this causes the flags to + change, they SHOULD be included as part of the FETCH responses. + + BODY.PEEK[
]<> + An alternate form of BODY[
] that does not implicitly + set the \Seen flag. + + BODYSTRUCTURE + The [MIME-IMB] body structure of the message. This is computed + by the server by parsing the [MIME-IMB] header fields in the + [RFC-2822] header and [MIME-IMB] headers. + + ENVELOPE + The envelope structure of the message. This is computed by the + server by parsing the [RFC-2822] header into the component + parts, defaulting various fields as necessary. + + FLAGS + The flags that are set for this message. + + INTERNALDATE + The internal date of the message. + + RFC822 + Functionally equivalent to BODY[], differing in the syntax of + the resulting untagged FETCH data (RFC822 is returned). + + RFC822.HEADER + Functionally equivalent to BODY.PEEK[HEADER], differing in the + syntax of the resulting untagged FETCH data (RFC822.HEADER is + returned). + + RFC822.SIZE + The [RFC-2822] size of the message. + + + + +Crispin Standards Track [Page 57] + +RFC 3501 IMAPv4 March 2003 + + + RFC822.TEXT + Functionally equivalent to BODY[TEXT], differing in the syntax + of the resulting untagged FETCH data (RFC822.TEXT is returned). + + UID + The unique identifier for the message. + + + Example: C: A654 FETCH 2:4 (FLAGS BODY[HEADER.FIELDS (DATE FROM)]) + S: * 2 FETCH .... + S: * 3 FETCH .... + S: * 4 FETCH .... + S: A654 OK FETCH completed + + +6.4.6. STORE Command + + Arguments: sequence set + message data item name + value for message data item + + Responses: untagged responses: FETCH + + Result: OK - store completed + NO - store error: can't store that data + BAD - command unknown or arguments invalid + + The STORE command alters data associated with a message in the + mailbox. Normally, STORE will return the updated value of the + data with an untagged FETCH response. A suffix of ".SILENT" in + the data item name prevents the untagged FETCH, and the server + SHOULD assume that the client has determined the updated value + itself or does not care about the updated value. + + Note: Regardless of whether or not the ".SILENT" suffix + was used, the server SHOULD send an untagged FETCH + response if a change to a message's flags from an + external source is observed. The intent is that the + status of the flags is determinate without a race + condition. + + + + + + + + + + + +Crispin Standards Track [Page 58] + +RFC 3501 IMAPv4 March 2003 + + + The currently defined data items that can be stored are: + + FLAGS + Replace the flags for the message (other than \Recent) with the + argument. The new value of the flags is returned as if a FETCH + of those flags was done. + + FLAGS.SILENT + Equivalent to FLAGS, but without returning a new value. + + +FLAGS + Add the argument to the flags for the message. The new value + of the flags is returned as if a FETCH of those flags was done. + + +FLAGS.SILENT + Equivalent to +FLAGS, but without returning a new value. + + -FLAGS + Remove the argument from the flags for the message. The new + value of the flags is returned as if a FETCH of those flags was + done. + + -FLAGS.SILENT + Equivalent to -FLAGS, but without returning a new value. + + + Example: C: A003 STORE 2:4 +FLAGS (\Deleted) + S: * 2 FETCH (FLAGS (\Deleted \Seen)) + S: * 3 FETCH (FLAGS (\Deleted)) + S: * 4 FETCH (FLAGS (\Deleted \Flagged \Seen)) + S: A003 OK STORE completed + + +6.4.7. COPY Command + + Arguments: sequence set + mailbox name + + Responses: no specific responses for this command + + Result: OK - copy completed + NO - copy error: can't copy those messages or to that + name + BAD - command unknown or arguments invalid + + + + + + + +Crispin Standards Track [Page 59] + +RFC 3501 IMAPv4 March 2003 + + + The COPY command copies the specified message(s) to the end of the + specified destination mailbox. The flags and internal date of the + message(s) SHOULD be preserved, and the Recent flag SHOULD be set, + in the copy. + + If the destination mailbox does not exist, a server SHOULD return + an error. It SHOULD NOT automatically create the mailbox. Unless + it is certain that the destination mailbox can not be created, the + server MUST send the response code "[TRYCREATE]" as the prefix of + the text of the tagged NO response. This gives a hint to the + client that it can attempt a CREATE command and retry the COPY if + the CREATE is successful. + + If the COPY command is unsuccessful for any reason, server + implementations MUST restore the destination mailbox to its state + before the COPY attempt. + + Example: C: A003 COPY 2:4 MEETING + S: A003 OK COPY completed + + +6.4.8. UID Command + + Arguments: command name + command arguments + + Responses: untagged responses: FETCH, SEARCH + + Result: OK - UID command completed + NO - UID command error + BAD - command unknown or arguments invalid + + The UID command has two forms. In the first form, it takes as its + arguments a COPY, FETCH, or STORE command with arguments + appropriate for the associated command. However, the numbers in + the sequence set argument are unique identifiers instead of + message sequence numbers. Sequence set ranges are permitted, but + there is no guarantee that unique identifiers will be contiguous. + + A non-existent unique identifier is ignored without any error + message generated. Thus, it is possible for a UID FETCH command + to return an OK without any data or a UID COPY or UID STORE to + return an OK without performing any operations. + + In the second form, the UID command takes a SEARCH command with + SEARCH command arguments. The interpretation of the arguments is + the same as with SEARCH; however, the numbers returned in a SEARCH + response for a UID SEARCH command are unique identifiers instead + + + +Crispin Standards Track [Page 60] + +RFC 3501 IMAPv4 March 2003 + + + of message sequence numbers. For example, the command UID SEARCH + 1:100 UID 443:557 returns the unique identifiers corresponding to + the intersection of two sequence sets, the message sequence number + range 1:100 and the UID range 443:557. + + Note: in the above example, the UID range 443:557 + appears. The same comment about a non-existent unique + identifier being ignored without any error message also + applies here. Hence, even if neither UID 443 or 557 + exist, this range is valid and would include an existing + UID 495. + + Also note that a UID range of 559:* always includes the + UID of the last message in the mailbox, even if 559 is + higher than any assigned UID value. This is because the + contents of a range are independent of the order of the + range endpoints. Thus, any UID range with * as one of + the endpoints indicates at least one message (the + message with the highest numbered UID), unless the + mailbox is empty. + + The number after the "*" in an untagged FETCH response is always a + message sequence number, not a unique identifier, even for a UID + command response. However, server implementations MUST implicitly + include the UID message data item as part of any FETCH response + caused by a UID command, regardless of whether a UID was specified + as a message data item to the FETCH. + + + Note: The rule about including the UID message data item as part + of a FETCH response primarily applies to the UID FETCH and UID + STORE commands, including a UID FETCH command that does not + include UID as a message data item. Although it is unlikely that + the other UID commands will cause an untagged FETCH, this rule + applies to these commands as well. + + Example: C: A999 UID FETCH 4827313:4828442 FLAGS + S: * 23 FETCH (FLAGS (\Seen) UID 4827313) + S: * 24 FETCH (FLAGS (\Seen) UID 4827943) + S: * 25 FETCH (FLAGS (\Seen) UID 4828442) + S: A999 OK UID FETCH completed + + + + + + + + + + +Crispin Standards Track [Page 61] + +RFC 3501 IMAPv4 March 2003 + + +6.5. Client Commands - Experimental/Expansion + + +6.5.1. X Command + + Arguments: implementation defined + + Responses: implementation defined + + Result: OK - command completed + NO - failure + BAD - command unknown or arguments invalid + + Any command prefixed with an X is an experimental command. + Commands which are not part of this specification, a standard or + standards-track revision of this specification, or an + IESG-approved experimental protocol, MUST use the X prefix. + + Any added untagged responses issued by an experimental command + MUST also be prefixed with an X. Server implementations MUST NOT + send any such untagged responses, unless the client requested it + by issuing the associated experimental command. + + Example: C: a441 CAPABILITY + S: * CAPABILITY IMAP4rev1 XPIG-LATIN + S: a441 OK CAPABILITY completed + C: A442 XPIG-LATIN + S: * XPIG-LATIN ow-nay eaking-spay ig-pay atin-lay + S: A442 OK XPIG-LATIN ompleted-cay + +7. Server Responses + + Server responses are in three forms: status responses, server data, + and command continuation request. The information contained in a + server response, identified by "Contents:" in the response + descriptions below, is described by function, not by syntax. The + precise syntax of server responses is described in the Formal Syntax + section. + + The client MUST be prepared to accept any response at all times. + + Status responses can be tagged or untagged. Tagged status responses + indicate the completion result (OK, NO, or BAD status) of a client + command, and have a tag matching the command. + + Some status responses, and all server data, are untagged. An + untagged response is indicated by the token "*" instead of a tag. + Untagged status responses indicate server greeting, or server status + + + +Crispin Standards Track [Page 62] + +RFC 3501 IMAPv4 March 2003 + + + that does not indicate the completion of a command (for example, an + impending system shutdown alert). For historical reasons, untagged + server data responses are also called "unsolicited data", although + strictly speaking, only unilateral server data is truly + "unsolicited". + + Certain server data MUST be recorded by the client when it is + received; this is noted in the description of that data. Such data + conveys critical information which affects the interpretation of all + subsequent commands and responses (e.g., updates reflecting the + creation or destruction of messages). + + Other server data SHOULD be recorded for later reference; if the + client does not need to record the data, or if recording the data has + no obvious purpose (e.g., a SEARCH response when no SEARCH command is + in progress), the data SHOULD be ignored. + + An example of unilateral untagged server data occurs when the IMAP + connection is in the selected state. In the selected state, the + server checks the mailbox for new messages as part of command + execution. Normally, this is part of the execution of every command; + hence, a NOOP command suffices to check for new messages. If new + messages are found, the server sends untagged EXISTS and RECENT + responses reflecting the new size of the mailbox. Server + implementations that offer multiple simultaneous access to the same + mailbox SHOULD also send appropriate unilateral untagged FETCH and + EXPUNGE responses if another agent changes the state of any message + flags or expunges any messages. + + Command continuation request responses use the token "+" instead of a + tag. These responses are sent by the server to indicate acceptance + of an incomplete client command and readiness for the remainder of + the command. + +7.1. Server Responses - Status Responses + + Status responses are OK, NO, BAD, PREAUTH and BYE. OK, NO, and BAD + can be tagged or untagged. PREAUTH and BYE are always untagged. + + Status responses MAY include an OPTIONAL "response code". A response + code consists of data inside square brackets in the form of an atom, + possibly followed by a space and arguments. The response code + contains additional information or status codes for client software + beyond the OK/NO/BAD condition, and are defined when there is a + specific action that a client can take based upon the additional + information. + + + + + +Crispin Standards Track [Page 63] + +RFC 3501 IMAPv4 March 2003 + + + The currently defined response codes are: + + ALERT + + The human-readable text contains a special alert that MUST be + presented to the user in a fashion that calls the user's + attention to the message. + + BADCHARSET + + Optionally followed by a parenthesized list of charsets. A + SEARCH failed because the given charset is not supported by + this implementation. If the optional list of charsets is + given, this lists the charsets that are supported by this + implementation. + + CAPABILITY + + Followed by a list of capabilities. This can appear in the + initial OK or PREAUTH response to transmit an initial + capabilities list. This makes it unnecessary for a client to + send a separate CAPABILITY command if it recognizes this + response. + + PARSE + + The human-readable text represents an error in parsing the + [RFC-2822] header or [MIME-IMB] headers of a message in the + mailbox. + + PERMANENTFLAGS + + Followed by a parenthesized list of flags, indicates which of + the known flags the client can change permanently. Any flags + that are in the FLAGS untagged response, but not the + PERMANENTFLAGS list, can not be set permanently. If the client + attempts to STORE a flag that is not in the PERMANENTFLAGS + list, the server will either ignore the change or store the + state change for the remainder of the current session only. + The PERMANENTFLAGS list can also include the special flag \*, + which indicates that it is possible to create new keywords by + attempting to store those flags in the mailbox. + + + + + + + + + +Crispin Standards Track [Page 64] + +RFC 3501 IMAPv4 March 2003 + + + READ-ONLY + + The mailbox is selected read-only, or its access while selected + has changed from read-write to read-only. + + READ-WRITE + + The mailbox is selected read-write, or its access while + selected has changed from read-only to read-write. + + TRYCREATE + + An APPEND or COPY attempt is failing because the target mailbox + does not exist (as opposed to some other reason). This is a + hint to the client that the operation can succeed if the + mailbox is first created by the CREATE command. + + UIDNEXT + + Followed by a decimal number, indicates the next unique + identifier value. Refer to section 2.3.1.1 for more + information. + + UIDVALIDITY + + Followed by a decimal number, indicates the unique identifier + validity value. Refer to section 2.3.1.1 for more information. + + UNSEEN + + Followed by a decimal number, indicates the number of the first + message without the \Seen flag set. + + Additional response codes defined by particular client or server + implementations SHOULD be prefixed with an "X" until they are + added to a revision of this protocol. Client implementations + SHOULD ignore response codes that they do not recognize. + +7.1.1. OK Response + + Contents: OPTIONAL response code + human-readable text + + The OK response indicates an information message from the server. + When tagged, it indicates successful completion of the associated + command. The human-readable text MAY be presented to the user as + an information message. The untagged form indicates an + + + + +Crispin Standards Track [Page 65] + +RFC 3501 IMAPv4 March 2003 + + + information-only message; the nature of the information MAY be + indicated by a response code. + + The untagged form is also used as one of three possible greetings + at connection startup. It indicates that the connection is not + yet authenticated and that a LOGIN command is needed. + + Example: S: * OK IMAP4rev1 server ready + C: A001 LOGIN fred blurdybloop + S: * OK [ALERT] System shutdown in 10 minutes + S: A001 OK LOGIN Completed + + +7.1.2. NO Response + + Contents: OPTIONAL response code + human-readable text + + The NO response indicates an operational error message from the + server. When tagged, it indicates unsuccessful completion of the + associated command. The untagged form indicates a warning; the + command can still complete successfully. The human-readable text + describes the condition. + + Example: C: A222 COPY 1:2 owatagusiam + S: * NO Disk is 98% full, please delete unnecessary data + S: A222 OK COPY completed + C: A223 COPY 3:200 blurdybloop + S: * NO Disk is 98% full, please delete unnecessary data + S: * NO Disk is 99% full, please delete unnecessary data + S: A223 NO COPY failed: disk is full + + +7.1.3. BAD Response + + Contents: OPTIONAL response code + human-readable text + + The BAD response indicates an error message from the server. When + tagged, it reports a protocol-level error in the client's command; + the tag indicates the command that caused the error. The untagged + form indicates a protocol-level error for which the associated + command can not be determined; it can also indicate an internal + server failure. The human-readable text describes the condition. + + + + + + + +Crispin Standards Track [Page 66] + +RFC 3501 IMAPv4 March 2003 + + + Example: C: ...very long command line... + S: * BAD Command line too long + C: ...empty line... + S: * BAD Empty command line + C: A443 EXPUNGE + S: * BAD Disk crash, attempting salvage to a new disk! + S: * OK Salvage successful, no data lost + S: A443 OK Expunge completed + + +7.1.4. PREAUTH Response + + Contents: OPTIONAL response code + human-readable text + + The PREAUTH response is always untagged, and is one of three + possible greetings at connection startup. It indicates that the + connection has already been authenticated by external means; thus + no LOGIN command is needed. + + Example: S: * PREAUTH IMAP4rev1 server logged in as Smith + + +7.1.5. BYE Response + + Contents: OPTIONAL response code + human-readable text + + The BYE response is always untagged, and indicates that the server + is about to close the connection. The human-readable text MAY be + displayed to the user in a status report by the client. The BYE + response is sent under one of four conditions: + + 1) as part of a normal logout sequence. The server will close + the connection after sending the tagged OK response to the + LOGOUT command. + + 2) as a panic shutdown announcement. The server closes the + connection immediately. + + 3) as an announcement of an inactivity autologout. The server + closes the connection immediately. + + 4) as one of three possible greetings at connection startup, + indicating that the server is not willing to accept a + connection from this client. The server closes the + connection immediately. + + + + +Crispin Standards Track [Page 67] + +RFC 3501 IMAPv4 March 2003 + + + The difference between a BYE that occurs as part of a normal + LOGOUT sequence (the first case) and a BYE that occurs because of + a failure (the other three cases) is that the connection closes + immediately in the failure case. In all cases the client SHOULD + continue to read response data from the server until the + connection is closed; this will ensure that any pending untagged + or completion responses are read and processed. + + Example: S: * BYE Autologout; idle for too long + +7.2. Server Responses - Server and Mailbox Status + + These responses are always untagged. This is how server and mailbox + status data are transmitted from the server to the client. Many of + these responses typically result from a command with the same name. + +7.2.1. CAPABILITY Response + + Contents: capability listing + + The CAPABILITY response occurs as a result of a CAPABILITY + command. The capability listing contains a space-separated + listing of capability names that the server supports. The + capability listing MUST include the atom "IMAP4rev1". + + In addition, client and server implementations MUST implement the + STARTTLS, LOGINDISABLED, and AUTH=PLAIN (described in [IMAP-TLS]) + capabilities. See the Security Considerations section for + important information. + + A capability name which begins with "AUTH=" indicates that the + server supports that particular authentication mechanism. + + The LOGINDISABLED capability indicates that the LOGIN command is + disabled, and that the server will respond with a tagged NO + response to any attempt to use the LOGIN command even if the user + name and password are valid. An IMAP client MUST NOT issue the + LOGIN command if the server advertises the LOGINDISABLED + capability. + + Other capability names indicate that the server supports an + extension, revision, or amendment to the IMAP4rev1 protocol. + Server responses MUST conform to this document until the client + issues a command that uses the associated capability. + + Capability names MUST either begin with "X" or be standard or + standards-track IMAP4rev1 extensions, revisions, or amendments + registered with IANA. A server MUST NOT offer unregistered or + + + +Crispin Standards Track [Page 68] + +RFC 3501 IMAPv4 March 2003 + + + non-standard capability names, unless such names are prefixed with + an "X". + + Client implementations SHOULD NOT require any capability name + other than "IMAP4rev1", and MUST ignore any unknown capability + names. + + A server MAY send capabilities automatically, by using the + CAPABILITY response code in the initial PREAUTH or OK responses, + and by sending an updated CAPABILITY response code in the tagged + OK response as part of a successful authentication. It is + unnecessary for a client to send a separate CAPABILITY command if + it recognizes these automatic capabilities. + + Example: S: * CAPABILITY IMAP4rev1 STARTTLS AUTH=GSSAPI XPIG-LATIN + + +7.2.2. LIST Response + + Contents: name attributes + hierarchy delimiter + name + + The LIST response occurs as a result of a LIST command. It + returns a single name that matches the LIST specification. There + can be multiple LIST responses for a single LIST command. + + Four name attributes are defined: + + \Noinferiors + It is not possible for any child levels of hierarchy to exist + under this name; no child levels exist now and none can be + created in the future. + + \Noselect + It is not possible to use this name as a selectable mailbox. + + \Marked + The mailbox has been marked "interesting" by the server; the + mailbox probably contains messages that have been added since + the last time the mailbox was selected. + + \Unmarked + The mailbox does not contain any additional messages since the + last time the mailbox was selected. + + + + + + +Crispin Standards Track [Page 69] + +RFC 3501 IMAPv4 March 2003 + + + If it is not feasible for the server to determine whether or not + the mailbox is "interesting", or if the name is a \Noselect name, + the server SHOULD NOT send either \Marked or \Unmarked. + + The hierarchy delimiter is a character used to delimit levels of + hierarchy in a mailbox name. A client can use it to create child + mailboxes, and to search higher or lower levels of naming + hierarchy. All children of a top-level hierarchy node MUST use + the same separator character. A NIL hierarchy delimiter means + that no hierarchy exists; the name is a "flat" name. + + The name represents an unambiguous left-to-right hierarchy, and + MUST be valid for use as a reference in LIST and LSUB commands. + Unless \Noselect is indicated, the name MUST also be valid as an + argument for commands, such as SELECT, that accept mailbox names. + + Example: S: * LIST (\Noselect) "/" ~/Mail/foo + + +7.2.3. LSUB Response + + Contents: name attributes + hierarchy delimiter + name + + The LSUB response occurs as a result of an LSUB command. It + returns a single name that matches the LSUB specification. There + can be multiple LSUB responses for a single LSUB command. The + data is identical in format to the LIST response. + + Example: S: * LSUB () "." #news.comp.mail.misc + + +7.2.4 STATUS Response + + Contents: name + status parenthesized list + + The STATUS response occurs as a result of an STATUS command. It + returns the mailbox name that matches the STATUS specification and + the requested mailbox status information. + + Example: S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292) + + + + + + + + +Crispin Standards Track [Page 70] + +RFC 3501 IMAPv4 March 2003 + + +7.2.5. SEARCH Response + + Contents: zero or more numbers + + The SEARCH response occurs as a result of a SEARCH or UID SEARCH + command. The number(s) refer to those messages that match the + search criteria. For SEARCH, these are message sequence numbers; + for UID SEARCH, these are unique identifiers. Each number is + delimited by a space. + + Example: S: * SEARCH 2 3 6 + + +7.2.6. FLAGS Response + + Contents: flag parenthesized list + + The FLAGS response occurs as a result of a SELECT or EXAMINE + command. The flag parenthesized list identifies the flags (at a + minimum, the system-defined flags) that are applicable for this + mailbox. Flags other than the system flags can also exist, + depending on server implementation. + + The update from the FLAGS response MUST be recorded by the client. + + Example: S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + + +7.3. Server Responses - Mailbox Size + + These responses are always untagged. This is how changes in the size + of the mailbox are transmitted from the server to the client. + Immediately following the "*" token is a number that represents a + message count. + +7.3.1. EXISTS Response + + Contents: none + + The EXISTS response reports the number of messages in the mailbox. + This response occurs as a result of a SELECT or EXAMINE command, + and if the size of the mailbox changes (e.g., new messages). + + The update from the EXISTS response MUST be recorded by the + client. + + Example: S: * 23 EXISTS + + + + +Crispin Standards Track [Page 71] + +RFC 3501 IMAPv4 March 2003 + + +7.3.2. RECENT Response + + Contents: none + + The RECENT response reports the number of messages with the + \Recent flag set. This response occurs as a result of a SELECT or + EXAMINE command, and if the size of the mailbox changes (e.g., new + messages). + + Note: It is not guaranteed that the message sequence + numbers of recent messages will be a contiguous range of + the highest n messages in the mailbox (where n is the + value reported by the RECENT response). Examples of + situations in which this is not the case are: multiple + clients having the same mailbox open (the first session + to be notified will see it as recent, others will + probably see it as non-recent), and when the mailbox is + re-ordered by a non-IMAP agent. + + The only reliable way to identify recent messages is to + look at message flags to see which have the \Recent flag + set, or to do a SEARCH RECENT. + + The update from the RECENT response MUST be recorded by the + client. + + Example: S: * 5 RECENT + + +7.4. Server Responses - Message Status + + These responses are always untagged. This is how message data are + transmitted from the server to the client, often as a result of a + command with the same name. Immediately following the "*" token is a + number that represents a message sequence number. + +7.4.1. EXPUNGE Response + + Contents: none + + The EXPUNGE response reports that the specified message sequence + number has been permanently removed from the mailbox. The message + sequence number for each successive message in the mailbox is + immediately decremented by 1, and this decrement is reflected in + message sequence numbers in subsequent responses (including other + untagged EXPUNGE responses). + + + + + +Crispin Standards Track [Page 72] + +RFC 3501 IMAPv4 March 2003 + + + The EXPUNGE response also decrements the number of messages in the + mailbox; it is not necessary to send an EXISTS response with the + new value. + + As a result of the immediate decrement rule, message sequence + numbers that appear in a set of successive EXPUNGE responses + depend upon whether the messages are removed starting from lower + numbers to higher numbers, or from higher numbers to lower + numbers. For example, if the last 5 messages in a 9-message + mailbox are expunged, a "lower to higher" server will send five + untagged EXPUNGE responses for message sequence number 5, whereas + a "higher to lower server" will send successive untagged EXPUNGE + responses for message sequence numbers 9, 8, 7, 6, and 5. + + An EXPUNGE response MUST NOT be sent when no command is in + progress, nor while responding to a FETCH, STORE, or SEARCH + command. This rule is necessary to prevent a loss of + synchronization of message sequence numbers between client and + server. A command is not "in progress" until the complete command + has been received; in particular, a command is not "in progress" + during the negotiation of command continuation. + + Note: UID FETCH, UID STORE, and UID SEARCH are different + commands from FETCH, STORE, and SEARCH. An EXPUNGE + response MAY be sent during a UID command. + + The update from the EXPUNGE response MUST be recorded by the + client. + + Example: S: * 44 EXPUNGE + + +7.4.2. FETCH Response + + Contents: message data + + The FETCH response returns data about a message to the client. + The data are pairs of data item names and their values in + parentheses. This response occurs as the result of a FETCH or + STORE command, as well as by unilateral server decision (e.g., + flag updates). + + The current data items are: + + BODY + A form of BODYSTRUCTURE without extension data. + + + + + +Crispin Standards Track [Page 73] + +RFC 3501 IMAPv4 March 2003 + + + BODY[
]<> + A string expressing the body contents of the specified section. + The string SHOULD be interpreted by the client according to the + content transfer encoding, body type, and subtype. + + If the origin octet is specified, this string is a substring of + the entire body contents, starting at that origin octet. This + means that BODY[]<0> MAY be truncated, but BODY[] is NEVER + truncated. + + Note: The origin octet facility MUST NOT be used by a server + in a FETCH response unless the client specifically requested + it by means of a FETCH of a BODY[
]<> data + item. + + 8-bit textual data is permitted if a [CHARSET] identifier is + part of the body parameter parenthesized list for this section. + Note that headers (part specifiers HEADER or MIME, or the + header portion of a MESSAGE/RFC822 part), MUST be 7-bit; 8-bit + characters are not permitted in headers. Note also that the + [RFC-2822] delimiting blank line between the header and the + body is not affected by header line subsetting; the blank line + is always included as part of header data, except in the case + of a message which has no body and no blank line. + + Non-textual data such as binary data MUST be transfer encoded + into a textual form, such as BASE64, prior to being sent to the + client. To derive the original binary data, the client MUST + decode the transfer encoded string. + + BODYSTRUCTURE + A parenthesized list that describes the [MIME-IMB] body + structure of a message. This is computed by the server by + parsing the [MIME-IMB] header fields, defaulting various fields + as necessary. + + For example, a simple text message of 48 lines and 2279 octets + can have a body structure of: ("TEXT" "PLAIN" ("CHARSET" + "US-ASCII") NIL NIL "7BIT" 2279 48) + + Multiple parts are indicated by parenthesis nesting. Instead + of a body type as the first element of the parenthesized list, + there is a sequence of one or more nested body structures. The + second element of the parenthesized list is the multipart + subtype (mixed, digest, parallel, alternative, etc.). + + + + + + +Crispin Standards Track [Page 74] + +RFC 3501 IMAPv4 March 2003 + + + For example, a two part message consisting of a text and a + BASE64-encoded text attachment can have a body structure of: + (("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 1152 + 23)("TEXT" "PLAIN" ("CHARSET" "US-ASCII" "NAME" "cc.diff") + "<960723163407.20117h@cac.washington.edu>" "Compiler diff" + "BASE64" 4554 73) "MIXED") + + Extension data follows the multipart subtype. Extension data + is never returned with the BODY fetch, but can be returned with + a BODYSTRUCTURE fetch. Extension data, if present, MUST be in + the defined order. The extension data of a multipart body part + are in the following order: + + body parameter parenthesized list + A parenthesized list of attribute/value pairs [e.g., ("foo" + "bar" "baz" "rag") where "bar" is the value of "foo", and + "rag" is the value of "baz"] as defined in [MIME-IMB]. + + body disposition + A parenthesized list, consisting of a disposition type + string, followed by a parenthesized list of disposition + attribute/value pairs as defined in [DISPOSITION]. + + body language + A string or parenthesized list giving the body language + value as defined in [LANGUAGE-TAGS]. + + body location + A string list giving the body content URI as defined in + [LOCATION]. + + Any following extension data are not yet defined in this + version of the protocol. Such extension data can consist of + zero or more NILs, strings, numbers, or potentially nested + parenthesized lists of such data. Client implementations that + do a BODYSTRUCTURE fetch MUST be prepared to accept such + extension data. Server implementations MUST NOT send such + extension data until it has been defined by a revision of this + protocol. + + The basic fields of a non-multipart body part are in the + following order: + + body type + A string giving the content media type name as defined in + [MIME-IMB]. + + + + + +Crispin Standards Track [Page 75] + +RFC 3501 IMAPv4 March 2003 + + + body subtype + A string giving the content subtype name as defined in + [MIME-IMB]. + + body parameter parenthesized list + A parenthesized list of attribute/value pairs [e.g., ("foo" + "bar" "baz" "rag") where "bar" is the value of "foo" and + "rag" is the value of "baz"] as defined in [MIME-IMB]. + + body id + A string giving the content id as defined in [MIME-IMB]. + + body description + A string giving the content description as defined in + [MIME-IMB]. + + body encoding + A string giving the content transfer encoding as defined in + [MIME-IMB]. + + body size + A number giving the size of the body in octets. Note that + this size is the size in its transfer encoding and not the + resulting size after any decoding. + + A body type of type MESSAGE and subtype RFC822 contains, + immediately after the basic fields, the envelope structure, + body structure, and size in text lines of the encapsulated + message. + + A body type of type TEXT contains, immediately after the basic + fields, the size of the body in text lines. Note that this + size is the size in its content transfer encoding and not the + resulting size after any decoding. + + Extension data follows the basic fields and the type-specific + fields listed above. Extension data is never returned with the + BODY fetch, but can be returned with a BODYSTRUCTURE fetch. + Extension data, if present, MUST be in the defined order. + + The extension data of a non-multipart body part are in the + following order: + + body MD5 + A string giving the body MD5 value as defined in [MD5]. + + + + + + +Crispin Standards Track [Page 76] + +RFC 3501 IMAPv4 March 2003 + + + body disposition + A parenthesized list with the same content and function as + the body disposition for a multipart body part. + + body language + A string or parenthesized list giving the body language + value as defined in [LANGUAGE-TAGS]. + + body location + A string list giving the body content URI as defined in + [LOCATION]. + + Any following extension data are not yet defined in this + version of the protocol, and would be as described above under + multipart extension data. + + ENVELOPE + A parenthesized list that describes the envelope structure of a + message. This is computed by the server by parsing the + [RFC-2822] header into the component parts, defaulting various + fields as necessary. + + The fields of the envelope structure are in the following + order: date, subject, from, sender, reply-to, to, cc, bcc, + in-reply-to, and message-id. The date, subject, in-reply-to, + and message-id fields are strings. The from, sender, reply-to, + to, cc, and bcc fields are parenthesized lists of address + structures. + + An address structure is a parenthesized list that describes an + electronic mail address. The fields of an address structure + are in the following order: personal name, [SMTP] + at-domain-list (source route), mailbox name, and host name. + + [RFC-2822] group syntax is indicated by a special form of + address structure in which the host name field is NIL. If the + mailbox name field is also NIL, this is an end of group marker + (semi-colon in RFC 822 syntax). If the mailbox name field is + non-NIL, this is a start of group marker, and the mailbox name + field holds the group name phrase. + + If the Date, Subject, In-Reply-To, and Message-ID header lines + are absent in the [RFC-2822] header, the corresponding member + of the envelope is NIL; if these header lines are present but + empty the corresponding member of the envelope is the empty + string. + + + + + +Crispin Standards Track [Page 77] + +RFC 3501 IMAPv4 March 2003 + + + Note: some servers may return a NIL envelope member in the + "present but empty" case. Clients SHOULD treat NIL and + empty string as identical. + + Note: [RFC-2822] requires that all messages have a valid + Date header. Therefore, the date member in the envelope can + not be NIL or the empty string. + + Note: [RFC-2822] requires that the In-Reply-To and + Message-ID headers, if present, have non-empty content. + Therefore, the in-reply-to and message-id members in the + envelope can not be the empty string. + + If the From, To, cc, and bcc header lines are absent in the + [RFC-2822] header, or are present but empty, the corresponding + member of the envelope is NIL. + + If the Sender or Reply-To lines are absent in the [RFC-2822] + header, or are present but empty, the server sets the + corresponding member of the envelope to be the same value as + the from member (the client is not expected to know to do + this). + + Note: [RFC-2822] requires that all messages have a valid + From header. Therefore, the from, sender, and reply-to + members in the envelope can not be NIL. + + FLAGS + A parenthesized list of flags that are set for this message. + + INTERNALDATE + A string representing the internal date of the message. + + RFC822 + Equivalent to BODY[]. + + RFC822.HEADER + Equivalent to BODY[HEADER]. Note that this did not result in + \Seen being set, because RFC822.HEADER response data occurs as + a result of a FETCH of RFC822.HEADER. BODY[HEADER] response + data occurs as a result of a FETCH of BODY[HEADER] (which sets + \Seen) or BODY.PEEK[HEADER] (which does not set \Seen). + + RFC822.SIZE + A number expressing the [RFC-2822] size of the message. + + + + + + +Crispin Standards Track [Page 78] + +RFC 3501 IMAPv4 March 2003 + + + RFC822.TEXT + Equivalent to BODY[TEXT]. + + UID + A number expressing the unique identifier of the message. + + + Example: S: * 23 FETCH (FLAGS (\Seen) RFC822.SIZE 44827) + + +7.5. Server Responses - Command Continuation Request + + The command continuation request response is indicated by a "+" token + instead of a tag. This form of response indicates that the server is + ready to accept the continuation of a command from the client. The + remainder of this response is a line of text. + + This response is used in the AUTHENTICATE command to transmit server + data to the client, and request additional client data. This + response is also used if an argument to any command is a literal. + + The client is not permitted to send the octets of the literal unless + the server indicates that it is expected. This permits the server to + process commands and reject errors on a line-by-line basis. The + remainder of the command, including the CRLF that terminates a + command, follows the octets of the literal. If there are any + additional command arguments, the literal octets are followed by a + space and those arguments. + + Example: C: A001 LOGIN {11} + S: + Ready for additional command text + C: FRED FOOBAR {7} + S: + Ready for additional command text + C: fat man + S: A001 OK LOGIN completed + C: A044 BLURDYBLOOP {102856} + S: A044 BAD No such command as "BLURDYBLOOP" + + + + + + + + + + + + + + +Crispin Standards Track [Page 79] + +RFC 3501 IMAPv4 March 2003 + + +8. Sample IMAP4rev1 connection + + The following is a transcript of an IMAP4rev1 connection. A long + line in this sample is broken for editorial clarity. + +S: * OK IMAP4rev1 Service Ready +C: a001 login mrc secret +S: a001 OK LOGIN completed +C: a002 select inbox +S: * 18 EXISTS +S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) +S: * 2 RECENT +S: * OK [UNSEEN 17] Message 17 is the first unseen message +S: * OK [UIDVALIDITY 3857529045] UIDs valid +S: a002 OK [READ-WRITE] SELECT completed +C: a003 fetch 12 full +S: * 12 FETCH (FLAGS (\Seen) INTERNALDATE "17-Jul-1996 02:44:25 -0700" + RFC822.SIZE 4286 ENVELOPE ("Wed, 17 Jul 1996 02:23:25 -0700 (PDT)" + "IMAP4rev1 WG mtg summary and minutes" + (("Terry Gray" NIL "gray" "cac.washington.edu")) + (("Terry Gray" NIL "gray" "cac.washington.edu")) + (("Terry Gray" NIL "gray" "cac.washington.edu")) + ((NIL NIL "imap" "cac.washington.edu")) + ((NIL NIL "minutes" "CNRI.Reston.VA.US") + ("John Klensin" NIL "KLENSIN" "MIT.EDU")) NIL NIL + "") + BODY ("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 3028 + 92)) +S: a003 OK FETCH completed +C: a004 fetch 12 body[header] +S: * 12 FETCH (BODY[HEADER] {342} +S: Date: Wed, 17 Jul 1996 02:23:25 -0700 (PDT) +S: From: Terry Gray +S: Subject: IMAP4rev1 WG mtg summary and minutes +S: To: imap@cac.washington.edu +S: cc: minutes@CNRI.Reston.VA.US, John Klensin +S: Message-Id: +S: MIME-Version: 1.0 +S: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII +S: +S: ) +S: a004 OK FETCH completed +C: a005 store 12 +flags \deleted +S: * 12 FETCH (FLAGS (\Seen \Deleted)) +S: a005 OK +FLAGS completed +C: a006 logout +S: * BYE IMAP4rev1 server terminating connection +S: a006 OK LOGOUT completed + + + +Crispin Standards Track [Page 80] + +RFC 3501 IMAPv4 March 2003 + + +9. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + In the case of alternative or optional rules in which a later rule + overlaps an earlier rule, the rule which is listed earlier MUST take + priority. For example, "\Seen" when parsed as a flag is the \Seen + flag name and not a flag-extension, even though "\Seen" can be parsed + as a flag-extension. Some, but not all, instances of this rule are + noted below. + + Note: [ABNF] rules MUST be followed strictly; in + particular: + + (1) Except as noted otherwise, all alphabetic characters + are case-insensitive. The use of upper or lower case + characters to define token strings is for editorial clarity + only. Implementations MUST accept these strings in a + case-insensitive fashion. + + (2) In all cases, SP refers to exactly one space. It is + NOT permitted to substitute TAB, insert additional spaces, + or otherwise treat SP as being equivalent to LWSP. + + (3) The ASCII NUL character, %x00, MUST NOT be used at any + time. + +address = "(" addr-name SP addr-adl SP addr-mailbox SP + addr-host ")" + +addr-adl = nstring + ; Holds route from [RFC-2822] route-addr if + ; non-NIL + +addr-host = nstring + ; NIL indicates [RFC-2822] group syntax. + ; Otherwise, holds [RFC-2822] domain name + +addr-mailbox = nstring + ; NIL indicates end of [RFC-2822] group; if + ; non-NIL and addr-host is NIL, holds + ; [RFC-2822] group name. + ; Otherwise, holds [RFC-2822] local-part + ; after removing [RFC-2822] quoting + + + + + + +Crispin Standards Track [Page 81] + +RFC 3501 IMAPv4 March 2003 + + +addr-name = nstring + ; If non-NIL, holds phrase from [RFC-2822] + ; mailbox after removing [RFC-2822] quoting + +append = "APPEND" SP mailbox [SP flag-list] [SP date-time] SP + literal + +astring = 1*ASTRING-CHAR / string + +ASTRING-CHAR = ATOM-CHAR / resp-specials + +atom = 1*ATOM-CHAR + +ATOM-CHAR = + +atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / + quoted-specials / resp-specials + +authenticate = "AUTHENTICATE" SP auth-type *(CRLF base64) + +auth-type = atom + ; Defined by [SASL] + +base64 = *(4base64-char) [base64-terminal] + +base64-char = ALPHA / DIGIT / "+" / "/" + ; Case-sensitive + +base64-terminal = (2base64-char "==") / (3base64-char "=") + +body = "(" (body-type-1part / body-type-mpart) ")" + +body-extension = nstring / number / + "(" body-extension *(SP body-extension) ")" + ; Future expansion. Client implementations + ; MUST accept body-extension fields. Server + ; implementations MUST NOT generate + ; body-extension fields except as defined by + ; future standard or standards-track + ; revisions of this specification. + +body-ext-1part = body-fld-md5 [SP body-fld-dsp [SP body-fld-lang + [SP body-fld-loc *(SP body-extension)]]] + ; MUST NOT be returned on non-extensible + ; "BODY" fetch + + + + + + +Crispin Standards Track [Page 82] + +RFC 3501 IMAPv4 March 2003 + + +body-ext-mpart = body-fld-param [SP body-fld-dsp [SP body-fld-lang + [SP body-fld-loc *(SP body-extension)]]] + ; MUST NOT be returned on non-extensible + ; "BODY" fetch + +body-fields = body-fld-param SP body-fld-id SP body-fld-desc SP + body-fld-enc SP body-fld-octets + +body-fld-desc = nstring + +body-fld-dsp = "(" string SP body-fld-param ")" / nil + +body-fld-enc = (DQUOTE ("7BIT" / "8BIT" / "BINARY" / "BASE64"/ + "QUOTED-PRINTABLE") DQUOTE) / string + +body-fld-id = nstring + +body-fld-lang = nstring / "(" string *(SP string) ")" + +body-fld-loc = nstring + +body-fld-lines = number + +body-fld-md5 = nstring + +body-fld-octets = number + +body-fld-param = "(" string SP string *(SP string SP string) ")" / nil + +body-type-1part = (body-type-basic / body-type-msg / body-type-text) + [SP body-ext-1part] + +body-type-basic = media-basic SP body-fields + ; MESSAGE subtype MUST NOT be "RFC822" + +body-type-mpart = 1*body SP media-subtype + [SP body-ext-mpart] + +body-type-msg = media-message SP body-fields SP envelope + SP body SP body-fld-lines + +body-type-text = media-text SP body-fields SP body-fld-lines + +capability = ("AUTH=" auth-type) / atom + ; New capabilities MUST begin with "X" or be + ; registered with IANA as standard or + ; standards-track + + + + +Crispin Standards Track [Page 83] + +RFC 3501 IMAPv4 March 2003 + + +capability-data = "CAPABILITY" *(SP capability) SP "IMAP4rev1" + *(SP capability) + ; Servers MUST implement the STARTTLS, AUTH=PLAIN, + ; and LOGINDISABLED capabilities + ; Servers which offer RFC 1730 compatibility MUST + ; list "IMAP4" as the first capability. + +CHAR8 = %x01-ff + ; any OCTET except NUL, %x00 + +command = tag SP (command-any / command-auth / command-nonauth / + command-select) CRLF + ; Modal based on state + +command-any = "CAPABILITY" / "LOGOUT" / "NOOP" / x-command + ; Valid in all states + +command-auth = append / create / delete / examine / list / lsub / + rename / select / status / subscribe / unsubscribe + ; Valid only in Authenticated or Selected state + +command-nonauth = login / authenticate / "STARTTLS" + ; Valid only when in Not Authenticated state + +command-select = "CHECK" / "CLOSE" / "EXPUNGE" / copy / fetch / store / + uid / search + ; Valid only when in Selected state + +continue-req = "+" SP (resp-text / base64) CRLF + +copy = "COPY" SP sequence-set SP mailbox + +create = "CREATE" SP mailbox + ; Use of INBOX gives a NO error + +date = date-text / DQUOTE date-text DQUOTE + +date-day = 1*2DIGIT + ; Day of month + +date-day-fixed = (SP DIGIT) / 2DIGIT + ; Fixed-format version of date-day + +date-month = "Jan" / "Feb" / "Mar" / "Apr" / "May" / "Jun" / + "Jul" / "Aug" / "Sep" / "Oct" / "Nov" / "Dec" + +date-text = date-day "-" date-month "-" date-year + + + + +Crispin Standards Track [Page 84] + +RFC 3501 IMAPv4 March 2003 + + +date-year = 4DIGIT + +date-time = DQUOTE date-day-fixed "-" date-month "-" date-year + SP time SP zone DQUOTE + +delete = "DELETE" SP mailbox + ; Use of INBOX gives a NO error + +digit-nz = %x31-39 + ; 1-9 + +envelope = "(" env-date SP env-subject SP env-from SP + env-sender SP env-reply-to SP env-to SP env-cc SP + env-bcc SP env-in-reply-to SP env-message-id ")" + +env-bcc = "(" 1*address ")" / nil + +env-cc = "(" 1*address ")" / nil + +env-date = nstring + +env-from = "(" 1*address ")" / nil + +env-in-reply-to = nstring + +env-message-id = nstring + +env-reply-to = "(" 1*address ")" / nil + +env-sender = "(" 1*address ")" / nil + +env-subject = nstring + +env-to = "(" 1*address ")" / nil + +examine = "EXAMINE" SP mailbox + +fetch = "FETCH" SP sequence-set SP ("ALL" / "FULL" / "FAST" / + fetch-att / "(" fetch-att *(SP fetch-att) ")") + +fetch-att = "ENVELOPE" / "FLAGS" / "INTERNALDATE" / + "RFC822" [".HEADER" / ".SIZE" / ".TEXT"] / + "BODY" ["STRUCTURE"] / "UID" / + "BODY" section ["<" number "." nz-number ">"] / + "BODY.PEEK" section ["<" number "." nz-number ">"] + + + + + + +Crispin Standards Track [Page 85] + +RFC 3501 IMAPv4 March 2003 + + +flag = "\Answered" / "\Flagged" / "\Deleted" / + "\Seen" / "\Draft" / flag-keyword / flag-extension + ; Does not include "\Recent" + +flag-extension = "\" atom + ; Future expansion. Client implementations + ; MUST accept flag-extension flags. Server + ; implementations MUST NOT generate + ; flag-extension flags except as defined by + ; future standard or standards-track + ; revisions of this specification. + +flag-fetch = flag / "\Recent" + +flag-keyword = atom + +flag-list = "(" [flag *(SP flag)] ")" + +flag-perm = flag / "\*" + +greeting = "*" SP (resp-cond-auth / resp-cond-bye) CRLF + +header-fld-name = astring + +header-list = "(" header-fld-name *(SP header-fld-name) ")" + +list = "LIST" SP mailbox SP list-mailbox + +list-mailbox = 1*list-char / string + +list-char = ATOM-CHAR / list-wildcards / resp-specials + +list-wildcards = "%" / "*" + +literal = "{" number "}" CRLF *CHAR8 + ; Number represents the number of CHAR8s + +login = "LOGIN" SP userid SP password + +lsub = "LSUB" SP mailbox SP list-mailbox + + + + + + + + + + + +Crispin Standards Track [Page 86] + +RFC 3501 IMAPv4 March 2003 + + +mailbox = "INBOX" / astring + ; INBOX is case-insensitive. All case variants of + ; INBOX (e.g., "iNbOx") MUST be interpreted as INBOX + ; not as an astring. An astring which consists of + ; the case-insensitive sequence "I" "N" "B" "O" "X" + ; is considered to be INBOX and not an astring. + ; Refer to section 5.1 for further + ; semantic details of mailbox names. + +mailbox-data = "FLAGS" SP flag-list / "LIST" SP mailbox-list / + "LSUB" SP mailbox-list / "SEARCH" *(SP nz-number) / + "STATUS" SP mailbox SP "(" [status-att-list] ")" / + number SP "EXISTS" / number SP "RECENT" + +mailbox-list = "(" [mbx-list-flags] ")" SP + (DQUOTE QUOTED-CHAR DQUOTE / nil) SP mailbox + +mbx-list-flags = *(mbx-list-oflag SP) mbx-list-sflag + *(SP mbx-list-oflag) / + mbx-list-oflag *(SP mbx-list-oflag) + +mbx-list-oflag = "\Noinferiors" / flag-extension + ; Other flags; multiple possible per LIST response + +mbx-list-sflag = "\Noselect" / "\Marked" / "\Unmarked" + ; Selectability flags; only one per LIST response + +media-basic = ((DQUOTE ("APPLICATION" / "AUDIO" / "IMAGE" / + "MESSAGE" / "VIDEO") DQUOTE) / string) SP + media-subtype + ; Defined in [MIME-IMT] + +media-message = DQUOTE "MESSAGE" DQUOTE SP DQUOTE "RFC822" DQUOTE + ; Defined in [MIME-IMT] + +media-subtype = string + ; Defined in [MIME-IMT] + +media-text = DQUOTE "TEXT" DQUOTE SP media-subtype + ; Defined in [MIME-IMT] + +message-data = nz-number SP ("EXPUNGE" / ("FETCH" SP msg-att)) + +msg-att = "(" (msg-att-dynamic / msg-att-static) + *(SP (msg-att-dynamic / msg-att-static)) ")" + +msg-att-dynamic = "FLAGS" SP "(" [flag-fetch *(SP flag-fetch)] ")" + ; MAY change for a message + + + +Crispin Standards Track [Page 87] + +RFC 3501 IMAPv4 March 2003 + + +msg-att-static = "ENVELOPE" SP envelope / "INTERNALDATE" SP date-time / + "RFC822" [".HEADER" / ".TEXT"] SP nstring / + "RFC822.SIZE" SP number / + "BODY" ["STRUCTURE"] SP body / + "BODY" section ["<" number ">"] SP nstring / + "UID" SP uniqueid + ; MUST NOT change for a message + +nil = "NIL" + +nstring = string / nil + +number = 1*DIGIT + ; Unsigned 32-bit integer + ; (0 <= n < 4,294,967,296) + +nz-number = digit-nz *DIGIT + ; Non-zero unsigned 32-bit integer + ; (0 < n < 4,294,967,296) + +password = astring + +quoted = DQUOTE *QUOTED-CHAR DQUOTE + +QUOTED-CHAR = / + "\" quoted-specials + +quoted-specials = DQUOTE / "\" + +rename = "RENAME" SP mailbox SP mailbox + ; Use of INBOX as a destination gives a NO error + +response = *(continue-req / response-data) response-done + +response-data = "*" SP (resp-cond-state / resp-cond-bye / + mailbox-data / message-data / capability-data) CRLF + +response-done = response-tagged / response-fatal + +response-fatal = "*" SP resp-cond-bye CRLF + ; Server closes connection immediately + +response-tagged = tag SP resp-cond-state CRLF + +resp-cond-auth = ("OK" / "PREAUTH") SP resp-text + ; Authentication condition + + + + + +Crispin Standards Track [Page 88] + +RFC 3501 IMAPv4 March 2003 + + +resp-cond-bye = "BYE" SP resp-text + +resp-cond-state = ("OK" / "NO" / "BAD") SP resp-text + ; Status condition + +resp-specials = "]" + +resp-text = ["[" resp-text-code "]" SP] text + +resp-text-code = "ALERT" / + "BADCHARSET" [SP "(" astring *(SP astring) ")" ] / + capability-data / "PARSE" / + "PERMANENTFLAGS" SP "(" + [flag-perm *(SP flag-perm)] ")" / + "READ-ONLY" / "READ-WRITE" / "TRYCREATE" / + "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number / + "UNSEEN" SP nz-number / + atom [SP 1*] + +search = "SEARCH" [SP "CHARSET" SP astring] 1*(SP search-key) + ; CHARSET argument to MUST be registered with IANA + +search-key = "ALL" / "ANSWERED" / "BCC" SP astring / + "BEFORE" SP date / "BODY" SP astring / + "CC" SP astring / "DELETED" / "FLAGGED" / + "FROM" SP astring / "KEYWORD" SP flag-keyword / + "NEW" / "OLD" / "ON" SP date / "RECENT" / "SEEN" / + "SINCE" SP date / "SUBJECT" SP astring / + "TEXT" SP astring / "TO" SP astring / + "UNANSWERED" / "UNDELETED" / "UNFLAGGED" / + "UNKEYWORD" SP flag-keyword / "UNSEEN" / + ; Above this line were in [IMAP2] + "DRAFT" / "HEADER" SP header-fld-name SP astring / + "LARGER" SP number / "NOT" SP search-key / + "OR" SP search-key SP search-key / + "SENTBEFORE" SP date / "SENTON" SP date / + "SENTSINCE" SP date / "SMALLER" SP number / + "UID" SP sequence-set / "UNDRAFT" / sequence-set / + "(" search-key *(SP search-key) ")" + +section = "[" [section-spec] "]" + +section-msgtext = "HEADER" / "HEADER.FIELDS" [".NOT"] SP header-list / + "TEXT" + ; top-level or MESSAGE/RFC822 part + +section-part = nz-number *("." nz-number) + ; body part nesting + + + +Crispin Standards Track [Page 89] + +RFC 3501 IMAPv4 March 2003 + + +section-spec = section-msgtext / (section-part ["." section-text]) + +section-text = section-msgtext / "MIME" + ; text other than actual body part (headers, etc.) + +select = "SELECT" SP mailbox + +seq-number = nz-number / "*" + ; message sequence number (COPY, FETCH, STORE + ; commands) or unique identifier (UID COPY, + ; UID FETCH, UID STORE commands). + ; * represents the largest number in use. In + ; the case of message sequence numbers, it is + ; the number of messages in a non-empty mailbox. + ; In the case of unique identifiers, it is the + ; unique identifier of the last message in the + ; mailbox or, if the mailbox is empty, the + ; mailbox's current UIDNEXT value. + ; The server should respond with a tagged BAD + ; response to a command that uses a message + ; sequence number greater than the number of + ; messages in the selected mailbox. This + ; includes "*" if the selected mailbox is empty. + +seq-range = seq-number ":" seq-number + ; two seq-number values and all values between + ; these two regardless of order. + ; Example: 2:4 and 4:2 are equivalent and indicate + ; values 2, 3, and 4. + ; Example: a unique identifier sequence range of + ; 3291:* includes the UID of the last message in + ; the mailbox, even if that value is less than 3291. + +sequence-set = (seq-number / seq-range) *("," sequence-set) + ; set of seq-number values, regardless of order. + ; Servers MAY coalesce overlaps and/or execute the + ; sequence in any order. + ; Example: a message sequence number set of + ; 2,4:7,9,12:* for a mailbox with 15 messages is + ; equivalent to 2,4,5,6,7,9,12,13,14,15 + ; Example: a message sequence number set of *:4,5:7 + ; for a mailbox with 10 messages is equivalent to + ; 10,9,8,7,6,5,4,5,6,7 and MAY be reordered and + ; overlap coalesced to be 4,5,6,7,8,9,10. + +status = "STATUS" SP mailbox SP + "(" status-att *(SP status-att) ")" + + + + +Crispin Standards Track [Page 90] + +RFC 3501 IMAPv4 March 2003 + + +status-att = "MESSAGES" / "RECENT" / "UIDNEXT" / "UIDVALIDITY" / + "UNSEEN" + +status-att-list = status-att SP number *(SP status-att SP number) + +store = "STORE" SP sequence-set SP store-att-flags + +store-att-flags = (["+" / "-"] "FLAGS" [".SILENT"]) SP + (flag-list / (flag *(SP flag))) + +string = quoted / literal + +subscribe = "SUBSCRIBE" SP mailbox + +tag = 1* + +text = 1*TEXT-CHAR + +TEXT-CHAR = + +time = 2DIGIT ":" 2DIGIT ":" 2DIGIT + ; Hours minutes seconds + +uid = "UID" SP (copy / fetch / search / store) + ; Unique identifiers used instead of message + ; sequence numbers + +uniqueid = nz-number + ; Strictly ascending + +unsubscribe = "UNSUBSCRIBE" SP mailbox + +userid = astring + +x-command = "X" atom + +zone = ("+" / "-") 4DIGIT + ; Signed four-digit value of hhmm representing + ; hours and minutes east of Greenwich (that is, + ; the amount that the given time differs from + ; Universal Time). Subtracting the timezone + ; from the given time will give the UT form. + ; The Universal Time zone is "+0000". + + + + + + + + +Crispin Standards Track [Page 91] + +RFC 3501 IMAPv4 March 2003 + + +10. Author's Note + + This document is a revision or rewrite of earlier documents, and + supercedes the protocol specification in those documents: RFC 2060, + RFC 1730, unpublished IMAP2bis.TXT document, RFC 1176, and RFC 1064. + +11. Security Considerations + + IMAP4rev1 protocol transactions, including electronic mail data, are + sent in the clear over the network unless protection from snooping is + negotiated. This can be accomplished either by the use of STARTTLS, + negotiated privacy protection in the AUTHENTICATE command, or some + other protection mechanism. + +11.1. STARTTLS Security Considerations + + The specification of the STARTTLS command and LOGINDISABLED + capability in this document replaces that in [IMAP-TLS]. [IMAP-TLS] + remains normative for the PLAIN [SASL] authenticator. + + IMAP client and server implementations MUST implement the + TLS_RSA_WITH_RC4_128_MD5 [TLS] cipher suite, and SHOULD implement the + TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA [TLS] cipher suite. This is + important as it assures that any two compliant implementations can be + configured to interoperate. All other cipher suites are OPTIONAL. + Note that this is a change from section 2.1 of [IMAP-TLS]. + + During the [TLS] negotiation, the client MUST check its understanding + of the server hostname against the server's identity as presented in + the server Certificate message, in order to prevent man-in-the-middle + attacks. If the match fails, the client SHOULD either ask for + explicit user confirmation, or terminate the connection and indicate + that the server's identity is suspect. Matching is performed + according to these rules: + + The client MUST use the server hostname it used to open the + connection as the value to compare against the server name + as expressed in the server certificate. The client MUST + NOT use any form of the server hostname derived from an + insecure remote source (e.g., insecure DNS lookup). CNAME + canonicalization is not done. + + If a subjectAltName extension of type dNSName is present in + the certificate, it SHOULD be used as the source of the + server's identity. + + Matching is case-insensitive. + + + + +Crispin Standards Track [Page 92] + +RFC 3501 IMAPv4 March 2003 + + + A "*" wildcard character MAY be used as the left-most name + component in the certificate. For example, *.example.com + would match a.example.com, foo.example.com, etc. but would + not match example.com. + + If the certificate contains multiple names (e.g., more than + one dNSName field), then a match with any one of the fields + is considered acceptable. + + Both the client and server MUST check the result of the STARTTLS + command and subsequent [TLS] negotiation to see whether acceptable + authentication or privacy was achieved. + +11.2. Other Security Considerations + + A server error message for an AUTHENTICATE command which fails due to + invalid credentials SHOULD NOT detail why the credentials are + invalid. + + Use of the LOGIN command sends passwords in the clear. This can be + avoided by using the AUTHENTICATE command with a [SASL] mechanism + that does not use plaintext passwords, by first negotiating + encryption via STARTTLS or some other protection mechanism. + + A server implementation MUST implement a configuration that, at the + time of authentication, requires: + (1) The STARTTLS command has been negotiated. + OR + (2) Some other mechanism that protects the session from password + snooping has been provided. + OR + (3) The following measures are in place: + (a) The LOGINDISABLED capability is advertised, and [SASL] + mechanisms (such as PLAIN) using plaintext passwords are NOT + advertised in the CAPABILITY list. + AND + (b) The LOGIN command returns an error even if the password is + correct. + AND + (c) The AUTHENTICATE command returns an error with all [SASL] + mechanisms that use plaintext passwords, even if the password + is correct. + + A server error message for a failing LOGIN command SHOULD NOT specify + that the user name, as opposed to the password, is invalid. + + A server SHOULD have mechanisms in place to limit or delay failed + AUTHENTICATE/LOGIN attempts. + + + +Crispin Standards Track [Page 93] + +RFC 3501 IMAPv4 March 2003 + + + Additional security considerations are discussed in the section + discussing the AUTHENTICATE and LOGIN commands. + +12. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + As this specification revises the STARTTLS and LOGINDISABLED + extensions previously defined in [IMAP-TLS], the registry will be + updated accordingly. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 94] + +RFC 3501 IMAPv4 March 2003 + + +Appendices + +A. Normative References + + The following documents contain definitions or specifications that + are necessary to understand this document properly: + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 2234, + November 1997. + + [ANONYMOUS] Newman, C., "Anonymous SASL Mechanism", RFC + 2245, November 1997. + + [CHARSET] Freed, N. and J. Postel, "IANA Character Set + Registration Procedures", RFC 2978, October + 2000. + + [DIGEST-MD5] Leach, P. and C. Newman, "Using Digest + Authentication as a SASL Mechanism", RFC 2831, + May 2000. + + [DISPOSITION] Troost, R., Dorner, S. and K. Moore, + "Communicating Presentation Information in + Internet Messages: The Content-Disposition + Header", RFC 2183, August 1997. + + [IMAP-TLS] Newman, C., "Using TLS with IMAP, POP3 and + ACAP", RFC 2595, June 1999. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to + Indicate Requirement Levels", BCP 14, RFC 2119, + March 1997. + + [LANGUAGE-TAGS] Alvestrand, H., "Tags for the Identification of + Languages", BCP 47, RFC 3066, January 2001. + + [LOCATION] Palme, J., Hopmann, A. and N. Shelness, "MIME + Encapsulation of Aggregate Documents, such as + HTML (MHTML)", RFC 2557, March 1999. + + [MD5] Myers, J. and M. Rose, "The Content-MD5 Header + Field", RFC 1864, October 1995. + + + + + + + + + +Crispin Standards Track [Page 95] + +RFC 3501 IMAPv4 March 2003 + + + [MIME-HDRS] Moore, K., "MIME (Multipurpose Internet Mail + Extensions) Part Three: Message Header + Extensions for Non-ASCII Text", RFC 2047, + November 1996. + + [MIME-IMB] Freed, N. and N. Borenstein, "MIME + (Multipurpose Internet Mail Extensions) Part + One: Format of Internet Message Bodies", RFC + 2045, November 1996. + + [MIME-IMT] Freed, N. and N. Borenstein, "MIME + (Multipurpose Internet Mail Extensions) Part + Two: Media Types", RFC 2046, November 1996. + + [RFC-2822] Resnick, P., "Internet Message Format", RFC + 2822, April 2001. + + [SASL] Myers, J., "Simple Authentication and Security + Layer (SASL)", RFC 2222, October 1997. + + [TLS] Dierks, T. and C. Allen, "The TLS Protocol + Version 1.0", RFC 2246, January 1999. + + [UTF-7] Goldsmith, D. and M. Davis, "UTF-7: A Mail-Safe + Transformation Format of Unicode", RFC 2152, + May 1997. + + The following documents describe quality-of-implementation issues + that should be carefully considered when implementing this protocol: + + [IMAP-IMPLEMENTATION] Leiba, B., "IMAP Implementation + Recommendations", RFC 2683, September 1999. + + [IMAP-MULTIACCESS] Gahrns, M., "IMAP4 Multi-Accessed Mailbox + Practice", RFC 2180, July 1997. + +A.1 Informative References + + The following documents describe related protocols: + + [IMAP-DISC] Austein, R., "Synchronization Operations for + Disconnected IMAP4 Clients", Work in Progress. + + [IMAP-MODEL] Crispin, M., "Distributed Electronic Mail + Models in IMAP4", RFC 1733, December 1994. + + + + + + +Crispin Standards Track [Page 96] + +RFC 3501 IMAPv4 March 2003 + + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, + November 1997. + + [SMTP] Klensin, J., "Simple Mail Transfer Protocol", + STD 10, RFC 2821, April 2001. + + The following documents are historical or describe historical aspects + of this protocol: + + [IMAP-COMPAT] Crispin, M., "IMAP4 Compatibility with + IMAP2bis", RFC 2061, December 1996. + + [IMAP-HISTORICAL] Crispin, M., "IMAP4 Compatibility with IMAP2 + and IMAP2bis", RFC 1732, December 1994. + + [IMAP-OBSOLETE] Crispin, M., "Internet Message Access Protocol + - Obsolete Syntax", RFC 2062, December 1996. + + [IMAP2] Crispin, M., "Interactive Mail Access Protocol + - Version 2", RFC 1176, August 1990. + + [RFC-822] Crocker, D., "Standard for the Format of ARPA + Internet Text Messages", STD 11, RFC 822, + August 1982. + + [RFC-821] Postel, J., "Simple Mail Transfer Protocol", + STD 10, RFC 821, August 1982. + +B. Changes from RFC 2060 + + 1) Clarify description of unique identifiers and their semantics. + + 2) Fix the SELECT description to clarify that UIDVALIDITY is required + in the SELECT and EXAMINE responses. + + 3) Added an example of a failing search. + + 4) Correct store-att-flags: "#flag" should be "1#flag". + + 5) Made search and section rules clearer. + + 6) Correct the STORE example. + + 7) Correct "BASE645" misspelling. + + 8) Remove extraneous close parenthesis in example of two-part message + with text and BASE64 attachment. + + + +Crispin Standards Track [Page 97] + +RFC 3501 IMAPv4 March 2003 + + + 9) Remove obsolete "MAILBOX" response from mailbox-data. + + 10) A spurious "<" in the rule for mailbox-data was removed. + + 11) Add CRLF to continue-req. + + 12) Specifically exclude "]" from the atom in resp-text-code. + + 13) Clarify that clients and servers should adhere strictly to the + protocol syntax. + + 14) Emphasize in 5.2 that EXISTS can not be used to shrink a mailbox. + + 15) Add NEWNAME to resp-text-code. + + 16) Clarify that the empty string, not NIL, is used as arguments to + LIST. + + 17) Clarify that NIL can be returned as a hierarchy delimiter for the + empty string mailbox name argument if the mailbox namespace is flat. + + 18) Clarify that addr-mailbox and addr-name have RFC-2822 quoting + removed. + + 19) Update UTF-7 reference. + + 20) Fix example in 6.3.11. + + 21) Clarify that non-existent UIDs are ignored. + + 22) Update DISPOSITION reference. + + 23) Expand state diagram. + + 24) Clarify that partial fetch responses are only returned in + response to a partial fetch command. + + 25) Add UIDNEXT response code. Correct UIDVALIDITY definition + reference. + + 26) Further clarification of "can" vs. "MAY". + + 27) Reference RFC-2119. + + 28) Clarify that superfluous shifts are not permitted in modified + UTF-7. + + 29) Clarify that there are no implicit shifts in modified UTF-7. + + + +Crispin Standards Track [Page 98] + +RFC 3501 IMAPv4 March 2003 + + + 30) Clarify that "INBOX" in a mailbox name is always INBOX, even if + it is given as a string. + + 31) Add missing open parenthesis in media-basic grammar rule. + + 32) Correct attribute syntax in mailbox-data. + + 33) Add UIDNEXT to EXAMINE responses. + + 34) Clarify UNSEEN, PERMANENTFLAGS, UIDVALIDITY, and UIDNEXT + responses in SELECT and EXAMINE. They are required now, but weren't + in older versions. + + 35) Update references with RFC numbers. + + 36) Flush text-mime2. + + 37) Clarify that modified UTF-7 names must be case-sensitive and that + violating the convention should be avoided. + + 38) Correct UID FETCH example. + + 39) Clarify UID FETCH, UID STORE, and UID SEARCH vs. untagged EXPUNGE + responses. + + 40) Clarify the use of the word "convention". + + 41) Clarify that a command is not "in progress" until it has been + fully received (specifically, that a command is not "in progress" + during command continuation negotiation). + + 42) Clarify envelope defaulting. + + 43) Clarify that SP means one and only one space character. + + 44) Forbid silly states in LIST response. + + 45) Clarify that the ENVELOPE, INTERNALDATE, RFC822*, BODY*, and UID + for a message is static. + + 46) Add BADCHARSET response code. + + 47) Update formal syntax to [ABNF] conventions. + + 48) Clarify trailing hierarchy delimiter in CREATE semantics. + + 49) Clarify that the "blank line" is the [RFC-2822] delimiting blank + line. + + + +Crispin Standards Track [Page 99] + +RFC 3501 IMAPv4 March 2003 + + + 50) Clarify that RENAME should also create hierarchy as needed for + the command to complete. + + 51) Fix body-ext-mpart to not require language if disposition + present. + + 52) Clarify the RFC822.HEADER response. + + 53) Correct missing space after charset astring in search. + + 54) Correct missing quote for BADCHARSET in resp-text-code. + + 55) Clarify that ALL, FAST, and FULL preclude any other data items + appearing. + + 56) Clarify semantics of reference argument in LIST. + + 57) Clarify that a null string for SEARCH HEADER X-FOO means any + message with a header line with a field-name of X-FOO regardless of + the text of the header. + + 58) Specifically reserve 8-bit mailbox names for future use as UTF-8. + + 59) It is not an error for the client to store a flag that is not in + the PERMANENTFLAGS list; however, the server will either ignore the + change or make the change in the session only. + + 60) Correct/clarify the text regarding superfluous shifts. + + 61) Correct typographic errors in the "Changes" section. + + 62) Clarify that STATUS must not be used to check for new messages in + the selected mailbox + + 63) Clarify LSUB behavior with "%" wildcard. + + 64) Change AUTHORIZATION to AUTHENTICATE in section 7.5. + + 65) Clarify description of multipart body type. + + 66) Clarify that STORE FLAGS does not affect \Recent. + + 67) Change "west" to "east" in description of timezone. + + 68) Clarify that commands which break command pipelining must wait + for a completion result response. + + 69) Clarify that EXAMINE does not affect \Recent. + + + +Crispin Standards Track [Page 100] + +RFC 3501 IMAPv4 March 2003 + + + 70) Make description of MIME structure consistent. + + 71) Clarify that date searches disregard the time and timezone of the + INTERNALDATE or Date: header. In other words, "ON 13-APR-2000" means + messages with an INTERNALDATE text which starts with "13-APR-2000", + even if timezone differential from the local timezone is sufficient + to move that INTERNALDATE into the previous or next day. + + 72) Clarify that the header fetches don't add a blank line if one + isn't in the [RFC-2822] message. + + 73) Clarify (in discussion of UIDs) that messages are immutable. + + 74) Add an example of CHARSET searching. + + 75) Clarify in SEARCH that keywords are a type of flag. + + 76) Clarify the mandatory nature of the SELECT data responses. + + 77) Add optional CAPABILITY response code in the initial OK or + PREAUTH. + + 78) Add note that server can send an untagged CAPABILITY command as + part of the responses to AUTHENTICATE and LOGIN. + + 79) Remove statement about it being unnecessary to issue a CAPABILITY + command more than once in a connection. That statement is no longer + true. + + 80) Clarify that untagged EXPUNGE decrements the number of messages + in the mailbox. + + 81) Fix definition of "body" (concatenation has tighter binding than + alternation). + + 82) Add a new "Special Notes to Implementors" section with reference + to [IMAP-IMPLEMENTATION]. + + 83) Clarify that an untagged CAPABILITY response to an AUTHENTICATE + command should only be done if a security layer was not negotiated. + + 84) Change the definition of atom to exclude "]". Update astring to + include "]" for compatibility with the past. Remove resp-text-atom. + + 85) Remove NEWNAME. It can't work because mailbox names can be + literals and can include "]". Functionality can be addressed via + referrals. + + + + +Crispin Standards Track [Page 101] + +RFC 3501 IMAPv4 March 2003 + + + 86) Move modified UTF-7 rationale in order to have more logical + paragraph flow. + + 87) Clarify UID uniqueness guarantees with the use of MUST. + + 88) Note that clients should read response data until the connection + is closed instead of immediately closing on a BYE. + + 89) Change RFC-822 references to RFC-2822. + + 90) Clarify that RFC-2822 should be followed instead of RFC-822. + + 91) Change recommendation of optional automatic capabilities in LOGIN + and AUTHENTICATE to use the CAPABILITY response code in the tagged + OK. This is more interoperable than an unsolicited untagged + CAPABILITY response. + + 92) STARTTLS and AUTH=PLAIN are mandatory to implement; add + recommendations for other [SASL] mechanisms. + + 93) Clarify that a "connection" (as opposed to "server" or "command") + is in one of the four states. + + 94) Clarify that a failed or rejected command does not change state. + + 95) Split references between normative and informative. + + 96) Discuss authentication failure issues in security section. + + 97) Clarify that a data item is not necessarily of only one data + type. + + 98) Clarify that sequence ranges are independent of order. + + 99) Change an example to clarify that superfluous shifts in + Modified-UTF7 can not be fixed just by omitting the shift. The + entire string must be recalculated. + + 100) Change Envelope Structure definition since [RFC-2822] uses + "envelope" to refer to the [SMTP] envelope and not the envelope data + that appears in the [RFC-2822] header. + + 101) Expand on RFC822.HEADER response data vs. BODY[HEADER]. + + 102) Clarify Logout state semantics, change ASCII art. + + 103) Security changes to comply with IESG requirements. + + + + +Crispin Standards Track [Page 102] + +RFC 3501 IMAPv4 March 2003 + + + 104) Add definition for body URI. + + 105) Break sequence range definition into three rules, with rewritten + descriptions for each. + + 106) Move STARTTLS and LOGINDISABLED here from [IMAP-TLS]. + + 107) Add IANA Considerations section. + + 108) Clarify valid client assumptions for new message UIDs vs. + UIDNEXT. + + 109) Clarify that changes to permanentflags affect concurrent + sessions as well as subsequent sessions. + + 110) Clarify that authenticated state can be entered by the CLOSE + command. + + 111) Emphasize that SELECT and EXAMINE are the exceptions to the rule + that a failing command does not change state. + + 112) Clarify that newly-appended messages have the Recent flag set. + + 113) Clarify that newly-copied messages SHOULD have the Recent flag + set. + + 114) Clarify that UID commands always return the UID in FETCH + responses. + +C. Key Word Index + + +FLAGS (store command data item) ............... 59 + +FLAGS.SILENT (store command data item) ........ 59 + -FLAGS (store command data item) ............... 59 + -FLAGS.SILENT (store command data item) ........ 59 + ALERT (response code) ...................................... 64 + ALL (fetch item) ........................................... 55 + ALL (search key) ........................................... 50 + ANSWERED (search key) ...................................... 50 + APPEND (command) ........................................... 45 + AUTHENTICATE (command) ..................................... 27 + BAD (response) ............................................. 66 + BADCHARSET (response code) ................................. 64 + BCC (search key) .................................. 51 + BEFORE (search key) ................................. 51 + BODY (fetch item) .......................................... 55 + BODY (fetch result) ........................................ 73 + BODY (search key) ................................. 51 + + + +Crispin Standards Track [Page 103] + +RFC 3501 IMAPv4 March 2003 + + + BODY.PEEK[
]<> (fetch item) ............... 57 + BODYSTRUCTURE (fetch item) ................................. 57 + BODYSTRUCTURE (fetch result) ............................... 74 + BODY[
]<> (fetch result) ............. 74 + BODY[
]<> (fetch item) .................... 55 + BYE (response) ............................................. 67 + Body Structure (message attribute) ......................... 12 + CAPABILITY (command) ....................................... 24 + CAPABILITY (response code) ................................. 64 + CAPABILITY (response) ...................................... 68 + CC (search key) ................................... 51 + CHECK (command) ............................................ 47 + CLOSE (command) ............................................ 48 + COPY (command) ............................................. 59 + CREATE (command) ........................................... 34 + DELETE (command) ........................................... 35 + DELETED (search key) ....................................... 51 + DRAFT (search key) ......................................... 51 + ENVELOPE (fetch item) ...................................... 57 + ENVELOPE (fetch result) .................................... 77 + EXAMINE (command) .......................................... 33 + EXISTS (response) .......................................... 71 + EXPUNGE (command) .......................................... 48 + EXPUNGE (response) ......................................... 72 + Envelope Structure (message attribute) ..................... 12 + FAST (fetch item) .......................................... 55 + FETCH (command) ............................................ 54 + FETCH (response) ........................................... 73 + FLAGGED (search key) ....................................... 51 + FLAGS (fetch item) ......................................... 57 + FLAGS (fetch result) ....................................... 78 + FLAGS (response) ........................................... 71 + FLAGS (store command data item) ................ 59 + FLAGS.SILENT (store command data item) ......... 59 + FROM (search key) ................................. 51 + FULL (fetch item) .......................................... 55 + Flags (message attribute) .................................. 11 + HEADER (part specifier) .................................... 55 + HEADER (search key) .................. 51 + HEADER.FIELDS (part specifier) ............... 55 + HEADER.FIELDS.NOT (part specifier) ........... 55 + INTERNALDATE (fetch item) .................................. 57 + INTERNALDATE (fetch result) ................................ 78 + Internal Date (message attribute) .......................... 12 + KEYWORD (search key) ................................ 51 + Keyword (type of flag) ..................................... 11 + LARGER (search key) .................................... 51 + LIST (command) ............................................. 40 + + + +Crispin Standards Track [Page 104] + +RFC 3501 IMAPv4 March 2003 + + + LIST (response) ............................................ 69 + LOGIN (command) ............................................ 30 + LOGOUT (command) ........................................... 25 + LSUB (command) ............................................. 43 + LSUB (response) ............................................ 70 + MAY (specification requirement term) ....................... 4 + MESSAGES (status item) ..................................... 45 + MIME (part specifier) ...................................... 56 + MUST (specification requirement term) ...................... 4 + MUST NOT (specification requirement term) .................. 4 + Message Sequence Number (message attribute) ................ 10 + NEW (search key) ........................................... 51 + NO (response) .............................................. 66 + NOOP (command) ............................................. 25 + NOT (search key) .............................. 52 + OK (response) .............................................. 65 + OLD (search key) ........................................... 52 + ON (search key) ..................................... 52 + OPTIONAL (specification requirement term) .................. 4 + OR (search key) ................ 52 + PARSE (response code) ...................................... 64 + PERMANENTFLAGS (response code) ............................. 64 + PREAUTH (response) ......................................... 67 + Permanent Flag (class of flag) ............................. 12 + READ-ONLY (response code) .................................. 65 + READ-WRITE (response code) ................................. 65 + RECENT (response) .......................................... 72 + RECENT (search key) ........................................ 52 + RECENT (status item) ....................................... 45 + RENAME (command) ........................................... 37 + REQUIRED (specification requirement term) .................. 4 + RFC822 (fetch item) ........................................ 57 + RFC822 (fetch result) ...................................... 78 + RFC822.HEADER (fetch item) ................................. 57 + RFC822.HEADER (fetch result) ............................... 78 + RFC822.SIZE (fetch item) ................................... 57 + RFC822.SIZE (fetch result) ................................. 78 + RFC822.TEXT (fetch item) ................................... 58 + RFC822.TEXT (fetch result) ................................. 79 + SEARCH (command) ........................................... 49 + SEARCH (response) .......................................... 71 + SEEN (search key) .......................................... 52 + SELECT (command) ........................................... 31 + SENTBEFORE (search key) ............................. 52 + SENTON (search key) ................................. 52 + SENTSINCE (search key) .............................. 52 + SHOULD (specification requirement term) .................... 4 + SHOULD NOT (specification requirement term) ................ 4 + + + +Crispin Standards Track [Page 105] + +RFC 3501 IMAPv4 March 2003 + + + SINCE (search key) .................................. 52 + SMALLER (search key) ................................... 52 + STARTTLS (command) ......................................... 27 + STATUS (command) ........................................... 44 + STATUS (response) .......................................... 70 + STORE (command) ............................................ 58 + SUBJECT (search key) .............................. 53 + SUBSCRIBE (command) ........................................ 38 + Session Flag (class of flag) ............................... 12 + System Flag (type of flag) ................................. 11 + TEXT (part specifier) ...................................... 56 + TEXT (search key) ................................. 53 + TO (search key) ................................... 53 + TRYCREATE (response code) .................................. 65 + UID (command) .............................................. 60 + UID (fetch item) ........................................... 58 + UID (fetch result) ......................................... 79 + UID (search key) ............................ 53 + UIDNEXT (response code) .................................... 65 + UIDNEXT (status item) ...................................... 45 + UIDVALIDITY (response code) ................................ 65 + UIDVALIDITY (status item) .................................. 45 + UNANSWERED (search key) .................................... 53 + UNDELETED (search key) ..................................... 53 + UNDRAFT (search key) ....................................... 53 + UNFLAGGED (search key) ..................................... 53 + UNKEYWORD (search key) .............................. 53 + UNSEEN (response code) ..................................... 65 + UNSEEN (search key) ........................................ 53 + UNSEEN (status item) ....................................... 45 + UNSUBSCRIBE (command) ...................................... 39 + Unique Identifier (UID) (message attribute) ................ 8 + X (command) .......................................... 62 + [RFC-2822] Size (message attribute) ........................ 12 + \Answered (system flag) .................................... 11 + \Deleted (system flag) ..................................... 11 + \Draft (system flag) ....................................... 11 + \Flagged (system flag) ..................................... 11 + \Marked (mailbox name attribute) ........................... 69 + \Noinferiors (mailbox name attribute) ...................... 69 + \Noselect (mailbox name attribute) ......................... 69 + \Recent (system flag) ...................................... 11 + \Seen (system flag) ........................................ 11 + \Unmarked (mailbox name attribute) ......................... 69 + + + + + + + +Crispin Standards Track [Page 106] + +RFC 3501 IMAPv4 March 2003 + + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Avenue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 107] + +RFC 3501 IMAPv4 March 2003 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. v This + document and the information contained herein is provided on an "AS + IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK + FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT + LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL + NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY + OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 108] + diff --git a/docs/rfcs/rfc3502.MULTIAPPEND_extension.txt b/docs/rfcs/rfc3502.MULTIAPPEND_extension.txt new file mode 100644 index 0000000..f6b61a4 --- /dev/null +++ b/docs/rfcs/rfc3502.MULTIAPPEND_extension.txt @@ -0,0 +1,395 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 3502 University of Washington +Category: Standards Track March 2003 + + + Internet Message Access Protocol (IMAP) - MULTIAPPEND Extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + This document describes the multiappending extension to the Internet + Message Access Protocol (IMAP) (RFC 3501). This extension provides + substantial performance improvements for IMAP clients which upload + multiple messages at a time to a mailbox on the server. + + A server which supports this extension indicates this with a + capability name of "MULTIAPPEND". + +Terminology + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to + be interpreted as described in [KEYWORDS]. + +Introduction + + The MULTIAPPEND extension permits uploading of multiple messages with + a single command. When used in conjunction with the [LITERAL+] + extension, the entire upload is accomplished in a single + command/response round trip. + + A MULTIAPPEND APPEND operation is atomic; either all messages are + successfully appended, or no messages are appended. + + In the base IMAP specification, each message must be appended in a + separate command, and there is no mechanism to "unappend" messages if + an error occurs while appending. Also, some mail stores may require + + + +Crispin Standards Track [Page 1] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + + an expensive "open/lock + sync/unlock/close" operation as part of + appending; this can be quite expensive if it must be done on a + per-message basis. + + If the server supports both LITERAL+ and pipelining but not + MULTIAPPEND, it may be possible to get some of the performance + advantages of MULTIAPPEND by doing a pipelined "batch" append. + However, it will not work as well as MULTIAPPEND for the following + reasons: + + 1) Multiple APPEND commands, even as part of a pipelined batch, + are non-atomic by definition. There is no way to revert the + mailbox to the state before the batch append in the event of an + error. + + 2) It may not be feasible for the server to coalesce pipelined + APPEND operations so as to avoid the "open/lock + + sync/unlock/close" overhead described above. In any case, such + coalescing would be timing dependent and thus potentially + unreliable. In particular, with traditional UNIX mailbox files, + it is assumed that a lock is held only for a single atomic + operation, and many applications disregard any lock that is + older than 5 minutes. + + 3) If an error occurs, depending upon the nature of the error, + it is possible for additional messages to be appended after the + error. For example, the user wants to append 5 messages, but a + disk quota error occurs with the third message because of its + size. However, the fourth and fifth messages have already been + sent in the pipeline, so the mailbox ends up with the first, + second, fourth, and fifth messages of the batch appended. + +6.3.11. APPEND Command + + Arguments: mailbox name + one or more messages to upload, specified as: + OPTIONAL flag parenthesized list + OPTIONAL date/time string + message literal + + Data: no specific responses for this command + + Result: OK - append completed + NO - append error: can't append to that mailbox, error + in flags or date/time or message text, + append cancelled + BAD - command unknown or arguments invalid + + + + +Crispin Standards Track [Page 2] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + + The APPEND command appends the literal arguments as new messages + to the end of the specified destination mailbox. This argument + SHOULD be in the format of an [RFC-2822] message. 8-bit + characters are permitted in the message. A server implementation + that is unable to preserve 8-bit data properly MUST be able to + reversibly convert 8-bit APPEND data to 7-bit using a [MIME-IMB] + content transfer encoding. + + Note: There MAY be exceptions, e.g., draft messages, in + which required [RFC-2822] header lines are omitted in the + message literal argument to APPEND. The full implications + of doing so MUST be understood and carefully weighed. + + If a flag parenthesized list is specified, the flags SHOULD be set + in the resulting message; otherwise, the flag list of the + resulting message is set empty by default. + + If a date-time is specified, the internal date SHOULD be set in + the resulting message; otherwise, the internal date of the + resulting message is set to the current date and time by default. + + A zero-length message literal argument is an error, and MUST + return a NO. This can be used to cancel the append. + + If the append is unsuccessful for any reason (including being + cancelled), the mailbox MUST be restored to its state before the + APPEND attempt; no partial appending is permitted. The server MAY + return an error before processing all the message arguments. + + If the destination mailbox does not exist, a server MUST return an + error, and MUST NOT automatically create the mailbox. Unless it + is certain that the destination mailbox can not be created, the + server MUST send the response code "[TRYCREATE]" as the prefix of + the text of the tagged NO response. This gives a hint to the + client that it can attempt a CREATE command and retry the APPEND + if the CREATE is successful. + + If the mailbox is currently selected, the normal new message + actions SHOULD occur. Specifically, the server SHOULD notify the + client immediately via an untagged EXISTS response. If the server + does not do so, the client MAY issue a NOOP command (or failing + that, a CHECK command) after one or more APPEND commands. + + + + + + + + + +Crispin Standards Track [Page 3] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + + Example: C: A003 APPEND saved-messages (\Seen) {329} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.example.net + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: (\Seen) " 7-Feb-1994 22:43:04 -0800" {295} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) + C: From: Joe Mooch + C: Subject: Re: afternoon meeting + C: To: foobar@blurdybloop.example.com + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: 3:30 is fine with me. + C: + S: A003 OK APPEND completed + C: A004 APPEND bogusname (\Flagged) {1023} + S: A004 NO [TRYCREATE] No such mailbox as bogusname + C: A005 APPEND test (\Flagged) {99} + S: + Ready for literal data + C: Date: Mon, 7 Feb 2000 22:43:04 -0800 (PST) + C: From: Fred Foobar + C: Subject: hmm... + C: {35403} + S: A005 NO APPEND failed: Disk quota exceeded + + Note: The APPEND command is not used for message delivery, + because it does not provide a mechanism to transfer [SMTP] + envelope information. + +Modification to IMAP4rev1 Base Protocol Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + append = "APPEND" SP mailbox 1*append-message + + append-message = [SP flag-list] [SP date-time] SP literal + + + + + +Crispin Standards Track [Page 4] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + +MULTIAPPEND Interaction with UIDPLUS Extension + + Servers which support both MULTIAPPEND and [UIDPLUS] will have the + "resp-code-apnd" rule modified as follows: + + resp-code-apnd = "APPENDUID" SP nz-number SP set + + That is, the APPENDUID response code returns as many UIDs as there + were messages appended in the multiple append. The UIDs returned + should be in the order the articles where appended. The message set + may not contain extraneous UIDs or the symbol "*". + +Security Considerations + + The MULTIAPPEND extension does not raise any security considerations + that are not present in the base [IMAP] protocol, and these issues + are discussed in [IMAP]. Nevertheless, it is important to remember + that IMAP4rev1 protocol transactions, including electronic mail data, + are sent in the clear over the network unless protection from + snooping is negotiated, either by the use of STARTTLS, privacy + protection is negotiated in the AUTHENTICATE command, or some other + protection mechanism is in effect. + +Normative References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MIME-IMB] Freed, N. and N. Borenstein, "MIME (Multipurpose Internet + Mail Extensions) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [RFC-2822] Resnick, P., "Internet Message Format", RFC 2822, April + 2001. + + + + + + + + + + + +Crispin Standards Track [Page 5] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + +Informative References + + [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC 2088, + January 1997. + + [UIDPLUS] Myers, J., "IMAP4 UIDPLUS extension", RFC 2359, June 1988. + + [SMTP] Klensin, J., Editor, "Simple Mail Transfer Protocol", RFC + 2821, April 2001. + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Avenue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 6] + +RFC 3502 IMAP MULTIAPPEND March 2003 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 7] + diff --git a/docs/rfcs/rfc3503.Message_Disposition_Notification.txt b/docs/rfcs/rfc3503.Message_Disposition_Notification.txt new file mode 100644 index 0000000..5b82fb0 --- /dev/null +++ b/docs/rfcs/rfc3503.Message_Disposition_Notification.txt @@ -0,0 +1,507 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 3503 ACI Worldwide/MessagingDirect +Category: Standards Track March 2003 + + + Message Disposition Notification (MDN) profile for + Internet Message Access Protocol (IMAP) + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + The Message Disposition Notification (MDN) facility defined in RFC + 2298 provides a means by which a message can request that message + processing by the recipient be acknowledged as well as a format to be + used for such acknowledgements. However, it doesn't describe how + multiple Mail User Agents (MUAs) should handle the generation of MDNs + in an Internet Message Access Protocol (IMAP4) environment. + + This document describes how to handle MDNs in such an environment and + provides guidelines for implementers of IMAP4 that want to add MDN + support to their products. + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 1] + +RFC 3503 MDN profile for IMAP March 2003 + + +Table of Contents + + 1. Conventions Used in this Document............................. 2 + 2. Introduction and Overview..................................... 2 + 3. Client behavior............................................... 3 + 3.1. Client behavior when receiving a message................. 5 + 3.2. Client behavior when copying a message................... 5 + 3.3. Client behavior when sending a message................... 5 + 3.4. Client behavior when saving a temporary message.......... 5 + 4. Server behavior............................................... 5 + 4.1. Server that supports arbitrary keywords.................. 5 + 4.2. Server that supports only $MDNSent keyword............... 5 + 4.3. Interaction with IMAP ACL extension...................... 6 + 5. Examples...................................................... 6 + 6. Security Considerations....................................... 7 + 7. Formal Syntax................................................. 7 + 8. Acknowledgments............................................... 7 + 9. Normative References.......................................... 8 + 10. Author's Address.............................................. 8 + 11. Full Copyright Statement...................................... 9 + +1. Conventions Used in this Document + + "C:" and "S:" in examples show lines sent by the client and server + respectively. + + The keywords "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in + this document when typed in uppercase are to be interpreted as + defined in "Key words for use in RFCs to Indicate Requirement Levels" + [KEYWORDS]. + +2. Introduction and Overview + + This memo defines an additional [IMAP4] mailbox keyword that allows + multiple Mail User Agents (MUAs) to know if a requested receipt + notification was sent. + + Message Disposition Notification [MDN] does not require any special + support of IMAP in the case where a user has access to the mailstore + from only one computer and is using a single MUA. In this case, the + MUA behaves as described in [MDN], i.e., the MUA performs automatic + processing and generates corresponding MDNs, it performs requested + action and, with the user's permission, sends appropriate MDNs. The + MUA will not send MDN twice because the MUA keeps track of sent + notifications in a local configuration. However, that does not work + when IMAP is used to access the same mailstore from different + locations or is using different MUAs. + + + + +Melnikov Standards Track [Page 2] + +RFC 3503 MDN profile for IMAP March 2003 + + + This document defines a new special purpose mailbox keyword $MDNSent + that must be used by MUAs. It does not define any new command or + response for IMAP, but describes a technique that MUAs should use to + achieve interoperability. + + When a client opens a mailbox for the first time, it verifies that + the server is capable of storing the $MDNSent keyword by examining + the PERMANENTFLAGS response code. In order to support MDN in IMAP, a + server MUST support either the $MDNSent keyword, or arbitrary message + keywords. + +3. Client behavior + + The use of IMAP requires few additional steps in mail processing on + the client side. The following timeline modifies the timeline found + in Section 4 of [MDN]. + + -- User composes message. + + -- User tells MUA to send message. + + -- MUA passes message to MSA (original recipient information passed + along). MUA [optionally] saves message to a folder for sent mail + with $MDNSent flag set. + + -- MSA sends message to MTA. + + -- Final MTA receives message. + + -- Final MTA delivers message to MUA (possibly generating DSN). + + -- MUA logs into IMAP server, opens mailbox, verifies if mailbox can + store $MDNSent keyword by examining PERMANENTFLAGS response. + + -- MUA performs automatic processing and generates corresponding MDNs + ("dispatched", "processed", "deleted", "denied" or "failed" + disposition type with "automatic-action" and "MDN-sent- + automatically" disposition modes) for messages that do not have + $MDNSent keyword, or \Draft flag set. (*) + + -- MUA sets the $MDNSent keyword for every message that required an + automatic MDN to be sent, whether or not the MDN was sent. + + -- MUA displays a list of messages to user. + + -- User selects a message and requests that some action be performed + on it. + + + + +Melnikov Standards Track [Page 3] + +RFC 3503 MDN profile for IMAP March 2003 + + + -- MUA performs requested action and, with user's permission, sends + appropriate MDN ("displayed", "dispatched", "processed", + "deleted", "denied" or "failed" disposition type with "manual- + action" and "MDN-sent-manually" or "MDN-sent-automatically" + disposition mode). If the generated MDN is saved to a mailbox + with the APPEND command, the client MUST specify the $MDNSent + keyword in the APPEND. + + -- MUA sets the $MDNSent keyword for all messages for which the user + confirmed the dispatching of disposition (or was explicitly + prohibited to do so). + + -- User possibly performs other actions on message, but no further + MDNs are generated. + + (*) Note: MUA MUST NOT use \Recent flag as an indicator that it + should send MDN, because according to [IMAP4], "If multiple + connections have the same mailbox selected simultaneously, it is + undefined which of these connections will see newly-arrived + messages with \Recent set and which will see it without \Recent + set". Thus, using \Recent as an indicator will cause + unpredictable client behavior with different IMAP4 servers. + However, the client MAY use \Seen flag as one of the indicators + that MDN must not be sent. The client MUST NOT use any other + standard flags, like \Draft or \Answered, to indicate that MDN + was previously sent, because they have different well known + meaning. In any case, in the presence of the $MDNSent keyword, + the client MUST ignore all other flags or keywords for the + purpose of generating an MDN and MUST NOT send the MDN. + + When the client opens a mailbox for the first time, it must verify + that the server supports the $MDNSent keyword, or arbitrary message + keywords by examining PERMANENTFLAGS response code. + + The client MUST NOT try to set the $MDNSent keyword if the server is + incapable of storing it permanently. + + The client MUST be prepared to receive NO from the server as the + result of STORE $MDNSent when the server advertises the support of + storing arbitrary keywords, because the server may limit the number + of message keywords it can store in a particular mailbox. A client + SHOULD NOT send MDN if it fails to store the $MDNSent keyword. + + Once the $MDNSent keyword is set, it MUST NOT be unset by a client. + The client MAY set the $MDNSent keyword when a user denies sending + the notification. This prohibits all other MUAs from sending MDN for + this message. + + + + +Melnikov Standards Track [Page 4] + +RFC 3503 MDN profile for IMAP March 2003 + + +3.1. Client behavior when receiving a message + + The client MUST NOT send MDN if a message has the $MDNSent keyword + set. It also MUST NOT send MDN if a message has \Draft flag, because + some clients use this flag to mark a message as incomplete. + + See the timeline in section 3 for details on client behavior when + receiving a message. + +3.2. Client behavior when copying a message + + The client SHOULD verify that $MDNSent is preserved on a COPY + operation. Furthermore, when a message is copied between servers + with the APPEND command, the client MUST set the $MDNSent keyword + correctly. + +3.3. Client behavior when sending a message + + When saving a sent message to any folder, the client MUST set the + $MDNSent keyword to prevent another client from sending MDN for the + message. + +3.4. Client behavior when saving a temporary message + + When saving an unfinished message to any folder client MUST set + $MDNSent keyword to prevent another client from sending MDN for the + message. + +4. Server behavior + + Server implementors that want to follow this specification must + insure that their server complies with either section 4.1 or section + 4.2. If the server also supports the IMAP [ACL] extension, it MUST + also comply with the section 4.3. + +4.1. Server that supports arbitrary keywords + + No changes are required from the server to make it compatible with + the extension described in this document if it supports arbitrary + keywords. + +4.2. Server that supports only $MDNSent keyword + + Servers that support only the $MDNSent keyword MUST preserve it on + the COPY operation. It is also expected that a server that supports + SEARCH will also support the SEARCH KEYWORD $MDNSent. + + + + + +Melnikov Standards Track [Page 5] + +RFC 3503 MDN profile for IMAP March 2003 + + +4.3. Interaction with IMAP ACL extension + + Any server that conforms to either 4.1 or 4.2 and also supports the + IMAP [ACL] extension, SHOULD preserve the $MDNSent keyword on COPY + even if the client does not have 'w' right. This will prevent the + generation of a duplicated MDN for the same message. Note that the + server MUST still check if the client has rights to perform the COPY + operation on a message according to [ACL]. + +5. Examples + + 1) MUA opens mailbox for the first time. + + a) The server supports storing of arbitrary keywords + + C: a100 select INBOX + S: * FLAGS (\Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Flagged \Draft \Deleted \Seen \*)] + S: * 5 EXISTS + S: * 3 RECENT + S: * OK [UIDVALIDITY 894294713] + S: a100 OK [READ-WRITE] Completed + + b) The server supports storing of the $MDNSent keyword + + C: a100 select INBOX + S: * FLAGS (\Flagged \Draft \Deleted \Seen $MDNSent) + S: * OK [PERMANENTFLAGS (\Flagged \Draft \Deleted \Seen $MDNSent)] + S: * 5 EXISTS + S: * 3 RECENT + S: * OK [UIDVALIDITY 894294713] + S: a100 OK [READ-WRITE] Completed + + 2) The MUA successfully sets the $MDNSent keyword + + C: a200 STORE 4 +FLAGS ($MDNSent) + S: * 4 FETCH (FLAGS (\Flagged \Seen $MDNSent)) + S: * FLAGS ($MDNSent \Flagged \Deleted \Draft \Seen) + S: * OK [PERMANENTFLAGS ($MDNSent \Flagged \Deleted \Draft \Seen \*)] + S: a200 OK STORE completed + + 3) The server refuses to store the $MDNSent keyword + + C: a200 STORE 4 +FLAGS ($MDNSent) + S: a200 NO STORE failed : no space left to store $MDNSent keyword + + + + + + +Melnikov Standards Track [Page 6] + +RFC 3503 MDN profile for IMAP March 2003 + + + 4) All clients and servers MUST treat the $MDNSent keyword as case + insensitive in all operations, as stated in [IMAP]. + + C: a300 FETCH 1:* FLAGS + S: * 1 FETCH (FLAGS (\Seen)) + S: * 2 FETCH (FLAGS (\Answered \Seen $MdnSENt)) + S: * 3 FETCH (FLAGS ()) + S: * 4 FETCH (FLAGS (\Flagged \Seen $MdnSENT)) + S: * 5 FETCH (FLAGS ($MDNSent)) + S: * 6 FETCH (FLAGS (\Recent)) + S: a300 OK FETCH completed + C: a400 SEARCH KEYWORDS $mdnsent + S: * SEARCH 2 4 5 + S: a400 OK SEARCH completed + +6. Security Considerations + + There are no known security issues with this extension, not found in + [MDN] and/or [IMAP4]. + + Section 4.3 changes ACL checking requirements on an IMAP server that + implements IMAP [ACL] extension. + +7. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) notation as specified in [RFC-822], as modified by + [IMAP4]. Non-terminals referenced, but not defined below, are as + defined by [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + flag_keyword ::= "$MDNSent" / other_keywords + + other_keywords ::= atom + +8. Acknowledgments + + This document is the product of discussions that took place on the + IMAP mailing list. Special gratitude to Cyrus Daboo and Randall + Gellens for reviewing the document. + + Thank you to my father who as he has helped to make me what I am. I + miss you terribly. + + + + +Melnikov Standards Track [Page 7] + +RFC 3503 MDN profile for IMAP March 2003 + + +9. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MDN] Fajman, R., "An Extensible Message Format for Message + Disposition Notifications", RFC 2298, March 1998. + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [ACL] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. + +10. Author's Address + + Alexey Melnikov + ACI Worldwide/MessagingDirect + 59 Clarendon Road + Watford, Hertfordshire + United Kingdom, WD17 1FQ + + Phone: +44 1923 81 2877 + EMail: mel@messagingdirect.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 8] + +RFC 3503 MDN profile for IMAP March 2003 + + +11. Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 9] + diff --git a/docs/rfcs/rfc3516.IMAP4_Binary_content_extension.txt b/docs/rfcs/rfc3516.IMAP4_Binary_content_extension.txt new file mode 100644 index 0000000..4d02197 --- /dev/null +++ b/docs/rfcs/rfc3516.IMAP4_Binary_content_extension.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group L. Nerenberg +Request for Comments: 3516 Orthanc Systems +Category: Standards Track April 2003 + + + IMAP4 Binary Content Extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2003). All Rights Reserved. + +Abstract + + This memo defines the Binary extension to the Internet Message Access + Protocol (IMAP4). It provides a mechanism for IMAP4 clients and + servers to exchange message body data without using a MIME content- + transfer-encoding. + +1. Conventions Used in this Document + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as described in [KEYWORD]. + + The abbreviation "CTE" means content-transfer-encoding. + +2. Introduction + + The MIME extensions to Internet messaging allow for the transmission + of non-textual (binary) message content [MIME-IMB]. Since the + traditional transports for messaging are not always capable of + passing binary data transparently, MIME provides encoding schemes + that allow binary content to be transmitted over transports that are + not otherwise able to do so. + + The overhead of MIME-encoding this content can be considerable in + some contexts (e.g., slow radio links, streaming multimedia). + Reducing the overhead associated with CTE schemes such as base64 + + + + + + +Nerenberg Standards Track [Page 1] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + + can give a noticeable reduction in resource consumption. The Binary + extension lets the server perform CTE decoding prior to transmitting + message data to the client. + +3. Content-Transfer-Encoding Considerations + + Every IMAP4 body section has a MIME content-transfer-encoding. + (Those without an explicit Content-Transfer-Encoding header are + implicitly labeled as "7bit" content.) In the terminology of [MIME- + IMB], the CTE specifies both a decoding algorithm and the domain of + the decoded data. In this memo, "decoding" refers to the CTE + decoding step described in [MIME-IMB]. + + Certain CTEs use an identity encoding transformation. For these CTEs + there is no decoding required, however the domain of the underlying + data may not be expressible in the IMAP4 protocol (e.g., MIME + "binary" content containing NUL octets). To accommodate these cases + the Binary extension introduces a new type of literal protocol + element that is fully eight bit transparent. + + Thus, server processing of the FETCH BINARY command involves two + logical steps: + + 1) perform any CTE-related decoding + + 2) determine the domain of the decoded data + + Step 2 is necessary to determine which protocol element should be + used to transmit the decoded data. (See FETCH Response Extensions + for further details.) + +4. Framework for the IMAP4 Binary Extension + + This memo defines the following extensions to [IMAP4rev1]. + +4.1. CAPABILITY Identification + + IMAP4 servers that support this extension MUST include "BINARY" in + the response list to the CAPABILITY command. + +4.2. FETCH Command Extensions + + This extension defines three new FETCH command data items. + + BINARY[] + + Requests that the specified section be transmitted after + performing CTE-related decoding. + + + +Nerenberg Standards Track [Page 2] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + + The argument, if present, requests that a subset of + the data be returned. The semantics of a partial FETCH BINARY + command are the same as for a partial FETCH BODY command, with + the exception that the arguments refer to the DECODED + section data. + + BINARY.PEEK[] + + An alternate form of FETCH BINARY that does not implicitly set + the \Seen flag. + + BINARY.SIZE + + Requests the decoded size of the section (i.e., the size to + expect in response to the corresponding FETCH BINARY request). + + Note: client authors are cautioned that this might be an + expensive operation for some server implementations. + Needlessly issuing this request could result in degraded + performance due to servers having to calculate the value every + time the request is issued. + +4.3. FETCH Response Extensions + + This extension defines two new FETCH response data items. + + BINARY[<>] + + An or expressing the content of the + specified section after removing any CTE-related encoding. If + is present it refers to the offset within the DECODED + section data. + + If the domain of the decoded data is "8bit" and the data does + not contain the NUL octet, the server SHOULD return the data in + a instead of a ; this allows the client to + determine if the "8bit" data contains the NUL octet without + having to explicitly scan the data stream for for NULs. + + If the server does not know how to decode the section's CTE, it + MUST fail the request and issue a "NO" response that contains + the "UNKNOWN-CTE" extended response code. + + + + + + + + + +Nerenberg Standards Track [Page 3] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + + BINARY.SIZE + + The size of the section after removing any CTE-related + encoding. The value returned MUST match the size of the + or that will be returned by the + corresponding FETCH BINARY request. + + If the server does not know how to decode the section's CTE, it + MUST fail the request and issue a "NO" response that contains + the "UNKNOWN-CTE" extended response code. + +4.4. APPEND Command Extensions + + The APPEND command is extended to allow the client to append data + containing NULs by using the syntax. The server MAY + modify the CTE of the appended data, however any such transformation + MUST NOT result in a loss of data. + + If the destination mailbox does not support the storage of binary + content, the server MUST fail the request and issue a "NO" response + that contains the "UNKNOWN-CTE" extended response code. + +5. MIME Encoded Headers + + [MIME-MHE] defines an encoding that allows for non-US-ASCII text in + message headers. This encoding is not the same as the content- + transfer-encoding applied to message bodies, and the decoding + transformations described in this memo do not apply to [MIME-MHE] + encoded header text. A server MUST NOT perform any conversion of + [MIME-MHE] encoded header text in response to any binary FETCH or + APPEND request. + +6. Implementation Considerations + + Messaging clients and servers have been notoriously lax in their + adherence to the Internet CRLF convention for terminating lines of + textual data in Internet protocols. When sending data using the + Binary extension, servers MUST ensure that textual line-oriented + sections are always transmitted using the IMAP4 CRLF line termination + syntax, regardless of the underlying storage representation of the + data on the server. + + A server may choose to store message body binary content in a non- + encoded format. Regardless of the internal storage representation + used, the server MUST issue BODYSTRUCTURE responses that describe the + message as though the binary-encoded sections are encoded in a CTE + + + + + +Nerenberg Standards Track [Page 4] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + + acceptable to the IMAP4 base specification. Furthermore, the results + of a FETCH BODY MUST return the message body content in the format + described by the corresponding FETCH BODYSTRUCTURE response. + + While the server is allowed to modify the CTE of APPENDed + data, this should only be done when it is absolutely necessary. + Gratuitous encoding changes will render useless most cryptographic + operations that have been performed on the message. + + This extension provides an optimization that is useful in certain + specific situations. It does not absolve clients from providing + basic functionality (content transfer decoding) that should be + available in all messaging clients. Clients supporting this + extension SHOULD be prepared to perform their own CTE decoding + operations. + +7. Formal Protocol Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (ABNF) notation as used in [ABNF], and incorporates by reference + the Core Rules defined in that document. + + This syntax augments the grammar specified in [IMAP4rev1]. + + append =/ "APPEND" SP mailbox [SP flag-list] + [SP date-time] SP literal8 + + fetch-att =/ "BINARY" [".PEEK"] section-binary [partial] + / "BINARY.SIZE" section-binary + + literal8 = "~{" number "}" CRLF *OCTET + ; represents the number of OCTETs + ; in the response string. + + msg-att-static =/ "BINARY" section-binary SP (nstring / literal8) + / "BINARY.SIZE" section-binary SP number + + partial = "<" number "." nz-number ">" + + resp-text-code =/ "UNKNOWN-CTE" + + section-binary = "[" [section-part] "]" + + + + + + + + + +Nerenberg Standards Track [Page 5] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + +8. Normative References + + [ABNF] Crocker, D., Editor, and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 2234, November 1997. + + [IMAP4rev1] Crispin, M., "Internet Message Access Protocol Version + 4rev1", RFC 3501, March 2003. + + [KEYWORD] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MIME-IMB] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [MIME-MHE] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII + Text", RFC 2047, November 1996. + +9. Security Considerations + + There are no known additional security issues with this extension + beyond those described in the base protocol described in [IMAP4rev1]. + +10. Intellectual Property + + The IETF takes no position regarding the validity or scope of any + intellectual property or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; neither does it represent that it + has made any effort to identify any such rights. Information on the + IETF's procedures with respect to rights in standards-track and + standards-related documentation can be found in BCP-11. Copies of + claims of rights made available for publication and any assurances of + licenses to be made available, or the result of an attempt made to + obtain a general license or permission for the use of such + proprietary rights by implementors or users of this specification can + be obtained from the IETF Secretariat. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights which may cover technology that may be required to practice + this standard. Please address the information to the IETF Executive + Director. + + + + + + +Nerenberg Standards Track [Page 6] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + +11. Author's Address + + Lyndon Nerenberg + Orthanc Systems + 1606 - 10770 Winterburn Road + Edmonton, Alberta + Canada T5S 1T6 + + EMail: lyndon@orthanc.ab.ca + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Nerenberg Standards Track [Page 7] + +RFC 3516 IMAP4 Binary Content Extension April 2003 + + +12. Full Copyright Statement + + Copyright (C) The Internet Society (2003). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Nerenberg Standards Track [Page 8] + diff --git a/docs/rfcs/rfc3691.IMAP_UNSELECT_command.txt b/docs/rfcs/rfc3691.IMAP_UNSELECT_command.txt new file mode 100644 index 0000000..2f4e9b4 --- /dev/null +++ b/docs/rfcs/rfc3691.IMAP_UNSELECT_command.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 3691 Isode Ltd. +Category: Standards Track February 2004 + + + Internet Message Access Protocol (IMAP) UNSELECT command + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2004). All Rights Reserved. + +Abstract + + This document defines an UNSELECT command that can be used to close + the current mailbox in an Internet Message Access Protocol - version + 4 (IMAP4) session without expunging it. Certain types of IMAP + clients need to release resources associated with the selected + mailbox without selecting a different mailbox. While IMAP4 provides + this functionality (via a SELECT command with a nonexistent mailbox + name or reselecting the same mailbox with EXAMINE command), a more + clean solution is desirable. + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 2. UNSELECT command . . . . . . . . . . . . . . . . . . . . . . . 2 + 3. Security Considerations. . . . . . . . . . . . . . . . . . . . 3 + 4. Formal Syntax. . . . . . . . . . . . . . . . . . . . . . . . . 3 + 5. IANA Considerations. . . . . . . . . . . . . . . . . . . . . . 3 + 6. Acknowledgments. . . . . . . . . . . . . . . . . . . . . . . . 3 + 7. Normative References . . . . . . . . . . . . . . . . . . . . . 4 + 8. Author's Address . . . . . . . . . . . . . . . . . . . . . . . 4 + 9. Full Copyright Statement . . . . . . . . . . . . . . . . . . . 5 + + + + + + + + + + +Melnikov Standards Track [Page 1] + +RFC 3691 IMAP UNSELECT command February 2004 + + +1. Introduction + + Certain types of IMAP clients need to release resources associated + with the selected mailbox without selecting a different mailbox. + While [IMAP4] provides this functionality (via a SELECT command with + a nonexistent mailbox name or reselecting the same mailbox with + EXAMINE command), a more clean solution is desirable. + + [IMAP4] defines the CLOSE command that closes the selected mailbox as + well as permanently removes all messages with the \Deleted flag set. + + However [IMAP4] lacks a command that simply closes the mailbox + without expunging it. This document defines the UNSELECT command for + this purpose. + + A server which supports this extension indicates this with a + capability name of "UNSELECT". + + "C:" and "S:" in examples show lines sent by the client and server + respectively. + + The keywords "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in + this document when typed in uppercase are to be interpreted as + defined in "Key words for use in RFCs to Indicate Requirement Levels" + [KEYWORDS]. + +2. UNSELECT Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - unselect completed, now in authenticated state + BAD - no mailbox selected, or argument supplied but + none permitted + + The UNSELECT command frees server's resources associated with the + selected mailbox and returns the server to the authenticated + state. This command performs the same actions as CLOSE, except + that no messages are permanently removed from the currently + selected mailbox. + + Example: C: A341 UNSELECT + S: A341 OK Unselect completed + + + + + + + +Melnikov Standards Track [Page 2] + +RFC 3691 IMAP UNSELECT command February 2004 + + +3. Security Considerations + + It is believed that this extension doesn't raise any additional + security concerns not already discussed in [IMAP4]. + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. Non-terminals + referenced but not defined below are as defined by [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + command-select /= "UNSELECT" + +5. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + This document defines the UNSELECT IMAP capabilities. IANA has added + this capability to the registry. + +6. Acknowledgments + + UNSELECT command was originally implemented by Tim Showalter in Cyrus + IMAP server. + + Also, the author of the document would like to thank Vladimir Butenko + and Mark Crispin for reminding that UNSELECT has to be documented. + Also thanks to Simon Josefsson for pointing out that there are + multiple ways to implement UNSELECT. + + + + + + + + + + + + + +Melnikov Standards Track [Page 3] + +RFC 3691 IMAP UNSELECT command February 2004 + + +7. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + +8. Author's Address + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + Hampton, Middlesex TW12 2BX + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 4] + +RFC 3691 IMAP UNSELECT command February 2004 + + +9. Full Copyright Statement + + Copyright (C) The Internet Society (2004). This document is subject + to the rights, licenses and restrictions contained in BCP 78 and + except as set forth therein, the authors retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE + REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE + INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed + to pertain to the implementation or use of the technology + described in this document or the extent to which any license + under such rights might or might not be available; nor does it + represent that it has made any independent effort to identify any + such rights. Information on the procedures with respect to + rights in RFC documents can be found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use + of such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository + at http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention + any copyrights, patents or patent applications, or other + proprietary rights that may cover technology that may be required + to implement this standard. Please address the information to the + IETF at ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + +Melnikov Standards Track [Page 5] + diff --git a/docs/rfcs/rfc4314.IMAP4_ACL_extension.txt b/docs/rfcs/rfc4314.IMAP4_ACL_extension.txt new file mode 100644 index 0000000..e73a56f --- /dev/null +++ b/docs/rfcs/rfc4314.IMAP4_ACL_extension.txt @@ -0,0 +1,1515 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 4314 Isode Ltd. +Obsoletes: 2086 December 2005 +Category: Standards Track + + + IMAP4 Access Control List (ACL) Extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2005). + +Abstract + + The Access Control List (ACL) extension (RFC 2086) of the Internet + Message Access Protocol (IMAP) permits mailbox access control lists + to be retrieved and manipulated through the IMAP protocol. + + This document is a revision of RFC 2086. It defines several new + access control rights and clarifies which rights are required for + different IMAP commands. + + + + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 1] + +RFC 4314 IMAP ACL December 2005 + + +Table of Contents + + 1. Introduction and Overview .......................................3 + 1.1. Conventions Used in This Document ..........................3 + 2. Access Control ..................................................3 + 2.1. Standard Rights ............................................5 + 2.1.1. Obsolete Rights .....................................5 + 2.2. Rights Defined in RFC 2086 .................................8 + 3. Access control management commands and responses ................8 + 3.1. SETACL Command .............................................8 + 3.2. DELETEACL Command ..........................................9 + 3.3. GETACL Command ............................................10 + 3.4. LISTRIGHTS Command ........................................10 + 3.5. MYRIGHTS Command ..........................................11 + 3.6. ACL Response ..............................................11 + 3.7. LISTRIGHTS Response .......................................12 + 3.8. MYRIGHTS Response .........................................12 + 4. Rights Required to Perform Different IMAP4rev1 Commands ........12 + 5. Other Considerations ...........................................17 + 5.1. Additional Requirements and Implementation Notes ..........17 + 5.1.1. Servers ............................................17 + 5.1.2. Clients ............................................18 + 5.2. Mapping of ACL Rights to READ-WRITE and READ-ONLY + Response Codes ............................................19 + 6. Security Considerations ........................................20 + 7. Formal Syntax ..................................................21 + 8. IANA Considerations ............................................22 + 9. Internationalization Considerations ............................22 + Appendix A. Changes since RFC 2086 ................................23 + Appendix B. Compatibility with RFC 2086 ...........................24 + Appendix C. Known Deficiencies ....................................24 + Appendix D. Acknowledgements ......................................25 + Normative References ..............................................25 + Informative References ............................................25 + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 2] + +RFC 4314 IMAP ACL December 2005 + + +1. Introduction and Overview + + The ACL (Access Control List) extension of the Internet Message + Access Protocol [IMAP4] permits mailbox access control lists to be + retrieved and manipulated through the IMAP protocol. + + This document is a revision of RFC 2086 [RFC2086]. It tries to + clarify different ambiguities in RFC 2086, in particular, the use of + UTF-8 [UTF-8] in access identifiers, which rights are required for + different IMAP4 commands, and how READ-WRITE/READ-ONLY response codes + are related to ACL. + +1.1. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. + + In all examples "/" character is used as hierarchy separator. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + + The phrase "ACL server" is just a shortcut for saying "IMAP server + that supports ACL extension as defined in this document". + +2. Access Control + + The ACL extension is present in any IMAP4 implementation that returns + "ACL" as one of the supported capabilities to the CAPABILITY command. + + A server implementation conformant to this document MUST also return + rights (see below) not defined in Section 2.2 in the "RIGHTS=" + capability. + + An access control list is a set of pairs. + An ACL applies to a mailbox name. + + Access identifier (or just "identifier") is a UTF-8 [UTF-8] string. + The identifier "anyone" is reserved to refer to the universal + identity (all authentications, including anonymous). All user name + strings accepted by the LOGIN or AUTHENTICATE commands to + authenticate to the IMAP server are reserved as identifiers for the + corresponding users. Identifiers starting with a dash ("-") are + reserved for "negative rights", described below. All other + identifier strings are interpreted in an implementation-defined + manner. + + + + +Melnikov Standards Track [Page 3] + +RFC 4314 IMAP ACL December 2005 + + + Rights is a string listing a (possibly empty) set of alphanumeric + characters, each character listing a set of operations that is being + controlled. Lowercase letters are reserved for "standard" rights, + listed in Section 2.1. (Note that for compatibility with deployed + clients and servers uppercase rights are not allowed.) The set of + standard rights can only be extended by a standards-track document. + Digits are reserved for implementation- or site-defined rights. + + An implementation MAY tie rights together or MAY force rights to + always or never be granted to particular identifiers. For example, + in an implementation that uses UNIX mode bits, the rights "swite" are + tied, the "a" right is always granted to the owner of a mailbox and + is never granted to another user. If rights are tied in an + implementation, the implementation must be conservative in granting + rights in response to SETACL commands--unless all rights in a tied + set are specified, none of that set should be included in the ACL + entry for that identifier. A client can discover the set of rights + that may be granted to a given identifier in the ACL for a given + mailbox name by using the LISTRIGHTS command. + + It is possible for multiple identifiers in an access control list to + apply to a given user. For example, an ACL may include rights to be + granted to the identifier matching the user, one or more + implementation-defined identifiers matching groups that include the + user, and/or the identifier "anyone". How these rights are combined + to determine the user's access is implementation defined. An + implementation may choose, for example, to use the union of the + rights granted to the applicable identifiers. An implementation may + instead choose, for example, to use only those rights granted to the + most specific identifier present in the ACL. A client can determine + the set of rights granted to the logged-in user for a given mailbox + name by using the MYRIGHTS command. + + When an identifier in an ACL starts with a dash ("-"), that indicates + that associated rights are to be removed from the identifier prefixed + by the dash. This is referred to as a "negative right". This + differs from DELETEACL in that a negative right is added to the ACL + and is a part of the calculation of the rights. + + Let's assume that an identifier "fred" refers to a user with login + "fred". If the identifier "-fred" is granted the "w" right, that + indicates that the "w" right is to be removed from users matching the + identifier "fred", even though the user "fred" might have the "w" + right as a consequence of some other identifier in the ACL. A + DELETEACL of "fred" simply deletes the identifier "fred" from the + ACL; it does not affect any rights that the user "fred" may get from + another entry in the ACL, in particular it doesn't affect rights + granted to the identifier "-fred". + + + +Melnikov Standards Track [Page 4] + +RFC 4314 IMAP ACL December 2005 + + + Server implementations are not required to support "negative right" + identifiers. + +2.1. Standard Rights + + The currently defined standard rights are (note that the list below + doesn't list all commands that use a particular right): + + l - lookup (mailbox is visible to LIST/LSUB commands, SUBSCRIBE + mailbox) + r - read (SELECT the mailbox, perform STATUS) + s - keep seen/unseen information across sessions (set or clear + \SEEN flag via STORE, also set \SEEN during APPEND/COPY/ + FETCH BODY[...]) + w - write (set or clear flags other than \SEEN and \DELETED via + STORE, also set them during APPEND/COPY) + i - insert (perform APPEND, COPY into mailbox) + p - post (send mail to submission address for mailbox, + not enforced by IMAP4 itself) + k - create mailboxes (CREATE new sub-mailboxes in any + implementation-defined hierarchy, parent mailbox for the new + mailbox name in RENAME) + x - delete mailbox (DELETE mailbox, old mailbox name in RENAME) + t - delete messages (set or clear \DELETED flag via STORE, set + \DELETED flag during APPEND/COPY) + e - perform EXPUNGE and expunge as a part of CLOSE + a - administer (perform SETACL/DELETEACL/GETACL/LISTRIGHTS) + +2.1.1. Obsolete Rights + + Due to ambiguity in RFC 2086, some existing RFC 2086 server + implementations use the "c" right to control the DELETE command. + Others chose to use the "d" right to control the DELETE command. For + the former group, let's define the "create" right as union of the "k" + and "x" rights, and the "delete" right as union of the "e" and "t" + rights. For the latter group, let's define the "create" rights as a + synonym to the "k" right, and the "delete" right as union of the "e", + "t", and "x" rights. + + For compatibility with RFC 2086, this section defines two virtual + rights "d" and "c". + + If a client includes the "d" right in a rights list, then it MUST be + treated as if the client had included every member of the "delete" + right. (It is not an error for a client to specify both the "d" + right and one or more members of the "delete" right, but the effect + is no different than if just the "d" right or all members of the + "delete" right had been specified.) + + + +Melnikov Standards Track [Page 5] + +RFC 4314 IMAP ACL December 2005 + + + When any of the "delete" member rights is set in a list of rights, + the server MUST also include the "d" right when returning the list in + a MYRIGHTS or ACL response. This is to enable older clients + conforming to RFC 2086 to work with newer servers. (*) + + Example: C: A001 SeTacl INBOX/Drafts David lrswida + S: A001 OK Setacl complete + + The client has specified the "d" right in the SETACL command above + and it expands to "et" on the server: + + C: A002 getacl INBOX/Drafts + S: * ACL INBOX Fred rwipslxcetda David lrswideta + S: A002 OK Getacl complete + + If the identifier specified in the LISTRIGHTS command can be granted + any of the "delete" member rights on a mailbox, then the server MUST + include the "d" right in the corresponding LISTRIGHTS response. (*) + If the member rights aren't tied to non-member rights, then the "d" + right is returned by itself in the LISTRIGHTS response. If any of + the member rights needs to be tied to one (or more) non-member right, + then the "d" right and all of the member rights need to be tied to + the same non-member right(s) (**). + + If a client includes the "c" right in a rights list, then it MUST be + treated as if the client had included every member of the "create" + right. (It is not an error for a client to specify both the "c" + right and one or more members of the "create" right, but the effect + is no different than if just the "c" right or all members of the + "create" right had been specified.) + + When any of the "create" member rights is set in a list of rights, + the server MUST also include the "c" right when returning the list in + a MYRIGHTS or ACL response. This is to enable older clients + conforming to RFC 2086 to work with newer servers. (*) + + Example: C: A003 Setacl INBOX/Drafts Byron lrswikda + S: A001 OK Setacl complete + C: A002 getAcl INBOX/Drafts + S: * ACL INBOX Fred rwipslxcetda Byron lrswikcdeta + S: A002 OK Getacl complete + + The client has specified the "d" right in the SETACL command above + and it expands to "et" on the server: As the client has specified the + "k" right (which is a member of the "c" right), the server also + returns the "c" right. + + + + + +Melnikov Standards Track [Page 6] + +RFC 4314 IMAP ACL December 2005 + + + If the identifier specified in the LISTRIGHTS command can be granted + any of the "create" member rights on a mailbox, then the server MUST + include the "c" right in the corresponding LISTRIGHTS response. (*) + If the member rights aren't tied to non-member rights, then the "c" + right is returned by itself in the LISTRIGHTS response. If any of + the member rights needs to be tied to one (or more) non-member right, + then the "c" right and all of the member rights need to be tied to + the same non-member right(s) (**). + + Example: The server that ties the rights as follows: + + lr s w i p k x t + + and c=k + + will return: + + S: * LISTRIGHTS archive/imap anyone "" + lr s w i p k x t c d + + Example: The server that ties the rights as follows: + + lr s w i p k xte + + and c=k + + will return: + + S: * LISTRIGHTS archive/imap anyone "" + lr s w i p k xte c d + + Example: The server that ties the rights as follows: + + lr s w i p k x te + + and c=k + + will return: + + S: * LISTRIGHTS archive/imap anyone "" + lr s w i p k c x te d + + Example: The server that ties the rights as follows: + + lr swte i p k x + + and c=kx + + + + +Melnikov Standards Track [Page 7] + +RFC 4314 IMAP ACL December 2005 + + + will return: + + S: * LISTRIGHTS archive/imap anyone "" + lr swted i p k x c + + (*) Clients conforming to this document MUST ignore the virtual "d" + and "c" rights in MYRIGHTS, ACL, and LISTRIGHTS responses. + + (**) The IMAPEXT Working Group has debated this issue in great length + and after reviewing existing ACL implementations concluded that + this is a reasonable restriction. + +2.2. Rights Defined in RFC 2086 + + The "RIGHTS=" capability MUST NOT include any of the rights defined + in RFC 2086: "l", "r", "s", "w", "i", "p", "a", "c", "d", and the + digits ("0" .. "9"). + +3. Access control management commands and responses + + Servers, when processing a command that has an identifier as a + parameter (i.e., any of SETACL, DELETEACL, and LISTRIGHTS commands), + SHOULD first prepare the received identifier using "SASLprep" profile + [SASLprep] of the "stringprep" algorithm [Stringprep]. If the + preparation of the identifier fails or results in an empty string, + the server MUST refuse to perform the command with a BAD response. + Note that Section 6 recommends additional identifier's verification + steps. + +3.1. SETACL Command + + Arguments: mailbox name + identifier + access right modification + + Data: no specific data for this command + + Result: OK - setacl completed + NO - setacl failure: can't set acl + BAD - arguments invalid + + The SETACL command changes the access control list on the specified + mailbox so that the specified identifier is granted permissions as + specified in the third argument. + + The third argument is a string containing an optional plus ("+") or + minus ("-") prefix, followed by zero or more rights characters. If + the string starts with a plus, the following rights are added to any + + + +Melnikov Standards Track [Page 8] + +RFC 4314 IMAP ACL December 2005 + + + existing rights for the identifier. If the string starts with a + minus, the following rights are removed from any existing rights for + the identifier. If the string does not start with a plus or minus, + the rights replace any existing rights for the identifier. + + Note that an unrecognized right MUST cause the command to return the + BAD response. In particular, the server MUST NOT silently ignore + unrecognized rights. + + Example: C: A001 GETACL INBOX/Drafts + S: * ACL INBOX/Drafts Fred rwipslxetad Chris lrswi + S: A001 OK Getacl complete + C: A002 SETACL INBOX/Drafts Chris +cda + S: A002 OK Setacl complete + C: A003 GETACL INBOX/Drafts + S: * ACL INBOX/Drafts Fred rwipslxetad Chris lrswicdakxet + S: A003 OK Getacl complete + + + C: A035 SETACL INBOX/Drafts John lrQswicda + S: A035 BAD Uppercase rights are not allowed + + + C: A036 SETACL INBOX/Drafts John lrqswicda + S: A036 BAD The q right is not supported + +3.2. DELETEACL Command + + Arguments: mailbox name + identifier + + Data: no specific data for this command + + Result: OK - deleteacl completed + NO - deleteacl failure: can't delete acl + BAD - arguments invalid + + The DELETEACL command removes any pair for the + specified identifier from the access control list for the specified + mailbox. + + Example: C: B001 getacl INBOX + S: * ACL INBOX Fred rwipslxetad -Fred wetd $team w + S: B001 OK Getacl complete + C: B002 DeleteAcl INBOX Fred + S: B002 OK Deleteacl complete + + + + + +Melnikov Standards Track [Page 9] + +RFC 4314 IMAP ACL December 2005 + + + C: B003 GETACL INBOX + S: * ACL INBOX -Fred wetd $team w + S: B003 OK Getacl complete + +3.3. GETACL Command + + Arguments: mailbox name + + Data: untagged responses: ACL + + Result: OK - getacl completed + NO - getacl failure: can't get acl + BAD - arguments invalid + + The GETACL command returns the access control list for mailbox in an + untagged ACL response. + + Some implementations MAY permit multiple forms of an identifier to + reference the same IMAP account. Usually, such implementations will + have a canonical form that is stored internally. An ACL response + caused by a GETACL command MAY include a canonicalized form of the + identifier that might be different from the one used in the + corresponding SETACL command. + + Example: C: A002 GETACL INBOX + S: * ACL INBOX Fred rwipsldexta + S: A002 OK Getacl complete + +3.4. LISTRIGHTS Command + + Arguments: mailbox name + identifier + + Data: untagged responses: LISTRIGHTS + + Result: OK - listrights completed + NO - listrights failure: can't get rights list + BAD - arguments invalid + + The LISTRIGHTS command takes a mailbox name and an identifier and + returns information about what rights can be granted to the + identifier in the ACL for the mailbox. + + Some implementations MAY permit multiple forms of an identifier to + reference the same IMAP account. Usually, such implementations will + have a canonical form that is stored internally. A LISTRIGHTS + + + + + +Melnikov Standards Track [Page 10] + +RFC 4314 IMAP ACL December 2005 + + + response caused by a LISTRIGHTS command MUST always return the same + form of an identifier as specified by the client. This is to allow + the client to correlate the response with the command. + + Example: C: a001 LISTRIGHTS ~/Mail/saved smith + S: * LISTRIGHTS ~/Mail/saved smith la r swicdkxte + S: a001 OK Listrights completed + + Example: C: a005 listrights archive/imap anyone + S: * LISTRIGHTS archive.imap anyone "" + l r s w i p k x t e c d a 0 1 2 3 4 5 6 7 8 9 + S: a005 Listrights successful + +3.5. MYRIGHTS Command + + Arguments: mailbox name + + Data: untagged responses: MYRIGHTS + + Result: OK - myrights completed + NO - myrights failure: can't get rights + BAD - arguments invalid + + The MYRIGHTS command returns the set of rights that the user has to + mailbox in an untagged MYRIGHTS reply. + + Example: C: A003 MYRIGHTS INBOX + S: * MYRIGHTS INBOX rwiptsldaex + S: A003 OK Myrights complete + +3.6. ACL Response + + Data: mailbox name + zero or more identifier rights pairs + + The ACL response occurs as a result of a GETACL command. The first + string is the mailbox name for which this ACL applies. This is + followed by zero or more pairs of strings; each pair contains the + identifier for which the entry applies followed by the set of rights + that the identifier has. + + Section 2.1.1 details additional server requirements related to + handling of the virtual "d" and "c" rights. + + + + + + + + +Melnikov Standards Track [Page 11] + +RFC 4314 IMAP ACL December 2005 + + +3.7. LISTRIGHTS Response + + Data: mailbox name + identifier + required rights + list of optional rights + + The LISTRIGHTS response occurs as a result of a LISTRIGHTS command. + The first two strings are the mailbox name and identifier for which + this rights list applies. Following the identifier is a string + containing the (possibly empty) set of rights the identifier will + always be granted in the mailbox. + + Following this are zero or more strings each containing a set of + rights the identifier can be granted in the mailbox. Rights + mentioned in the same string are tied together. The server MUST + either grant all tied rights to the identifier in the mailbox or + grant none. Section 2.1.1 details additional server requirements + related to handling of the virtual "d" and "c" rights. + + The same right MUST NOT be listed more than once in the LISTRIGHTS + command. + +3.8. MYRIGHTS Response + + Data: mailbox name + rights + + The MYRIGHTS response occurs as a result of a MYRIGHTS command. The + first string is the mailbox name for which these rights apply. The + second string is the set of rights that the client has. + + Section 2.1.1 details additional server requirements related to + handling of the virtual "d" and "c" rights. + +4. Rights Required to Perform Different IMAP4rev1 Commands + + Before executing a command, an ACL-compliant server MUST check which + rights are required to perform it. This section groups command by + functions they perform and list the rights required. It also gives + the detailed description of any special processing required. + + For the purpose of this section the UID counterpart of a command is + considered to be the same command, e.g., both UID COPY and COPY + commands require the same set of rights. + + + + + + +Melnikov Standards Track [Page 12] + +RFC 4314 IMAP ACL December 2005 + + + The table below summarizes different rights or their combinations + that are required in order to perform different IMAP operations. As + it is not always possible to express complex right checking and + interactions, the description after the table should be used as the + primary reference. + + +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ + |Operations\Rights | l | r | s | w | i | k | x | t | e | a |Any|Non| + +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ + | commands in authenticated state | + +-------------------------------------------------------------------+ + | LIST | + | | | | | | | | | | | | + | SUBSCRIBE | * | | | | | | | | | | | * | + | UNSUBSCRIBE | | | | | | | | | | | | + | + | LSUB | * | | | | | | | | | | | * | + |CREATE (for parent)| | | | | | + | | | | | | | + | DELETE | | ? | | | | | + | ? | ? | | | | + | RENAME | | | | | | + | + | | | | | | + | SELECT/EXAMINE | | + | | | | | | | | | | | + | STATUS | | + | | | | | | | | | | | + | SETACL/DELETEACL | | | | | | | | | | + | | | + | GETACL/LISTRIGHTS | | | | | | | | | | + | | | + | MYRIGHTS | | | | | | | | | | | + | | + | APPEND | | | ? | ? | + | | | ? | | | | | + +-------------------------------------------------------------------+ + | commands in selected state | + +-------------------------------------------------------------------+ + | COPY | | | ? | ? | + | | | ? | | | | | + | EXPUNGE | | | | | | | | | + | | | | + | CLOSE | | | | | | | | | ? | | | | + | FETCH | | | ? | | | | | | | | | | + | STORE flags | | | ? | ? | | | | ? | | | | | + +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ + + Note: for all commands in the selected state, the "r" is implied, + because it is required to SELECT/EXAMINE a mailbox. Servers are not + required to check presence of the "r" right once a mailbox is + successfully selected. + + Legend: + + - The right is required + * - Only one of the rights marked with * is required + (see description below) + ? - The right is OPTIONAL (see description below) + "Any" - at least one of the "l", "r", "i", "k", "x", "a" rights is + required + "Non" - No rights required to perform the command + + + + +Melnikov Standards Track [Page 13] + +RFC 4314 IMAP ACL December 2005 + + + Listing and subscribing/unsubscribing mailboxes: + LIST - "l" right is required. However, unlike other commands + (e.g., SELECT) the server MUST NOT return a NO response if it + can't list a mailbox. + Note that if the user has "l" right to a mailbox "A/B", but not to + its parent mailbox "A", the LIST command should behave as if the + mailbox "A" doesn't exist, for example: + + C: A777 LIST "" * + S: * LIST (\NoInferiors) "/" "A/B" + S: * LIST () "/" "C" + S: * LIST (\NoInferiors) "/" "C/D" + S: A777 OK LIST completed + + + SUBSCRIBE - "l" right is required only if the server checks for + mailbox existence when performing SUBSCRIBE. + + UNSUBSCRIBE - no rights required to perform this operation. + + LSUB - "l" right is required only if the server checks for mailbox + existence when performing SUBSCRIBE. However, unlike other + commands (e.g., SELECT) the server MUST NOT return a NO response + if it can't list a subscribed mailbox. + + Mailbox management: + CREATE - "k" right on a nearest existing parent mailbox. When a + new mailbox is created, it SHOULD inherit the ACL from the parent + mailbox (if one exists) in the defined hierarchy. + + DELETE - "x" right on the mailbox. Note that some servers don't + allow to delete a non-empty mailbox. If this is the case, the + user would also need "r", "e", and "t" rights, in order to open + the mailbox and empty it. + + The DELETE command MUST delete the ACL associated with the deleted + mailbox. + + RENAME - Moving a mailbox from one parent to another requires the + "x" right on the mailbox itself and the "k" right for the new + parent. For example, if the user wants to rename the mailbox + named "A/B/C" to "D/E", the user must have the "x" right for the + mailbox "A/B/C" and the "k" right for the mailbox "D". + The RENAME command SHOULD NOT change the ACLs on the renamed + mailbox and submailboxes. + + + + + + +Melnikov Standards Track [Page 14] + +RFC 4314 IMAP ACL December 2005 + + + Copying or appending messages: + Before performing a COPY/APPEND command, the server MUST check if + the user has "i" right for the target mailbox. If the user + doesn't have "i" right, the operation fails. Otherwise for each + copied/appended message the server MUST check if the user has + "t" right - when the message has \Deleted flag set + "s" right - when the message has \Seen flag set + "w" right - for all other message flags. + Only when the user has a particular right are the corresponding + flags stored for the newly created message. The server MUST NOT + fail a COPY/APPEND if the user has no rights to set a particular + flag. + + Example: C: A003 MYRIGHTS TargetMailbox + S: * MYRIGHTS TargetMailbox rwis + S: A003 OK Myrights complete + + C: A004 FETCH 1:3 (FLAGS) + S: * 1 FETCH (FLAGS (\Draft \Deleted) + S: * 2 FETCH (FLAGS (\Answered) + S: * 3 FETCH (FLAGS ($Forwarded \Seen) + S: A004 OK Fetch Completed + + C: A005 COPY 1:3 TargetMailbox + S: A005 OK Copy completed + + C: A006 SELECT TargetMailbox + ... + S: A006 Select Completed + + Let's assume that the copied messages received message numbers + 77:79. + + C: A007 FETCH 77:79 (FLAGS) + S: * 77 FETCH (FLAGS (\Draft)) + S: * 78 FETCH (FLAGS (\Answered)) + S: * 79 FETCH (FLAGS ($Forwarded \Seen)) + S: A007 OK Fetch Completed + + \Deleted flag was lost on COPY, as the user has no "t" right in + the target mailbox. + If the MYRIGHTS command with the tag A003 would have returned: + + S: * MYRIGHTS TargetMailbox rsti + + the response from the FETCH with the tag A007 would have been: + + C: A007 FETCH 77:79 (FLAGS) + + + +Melnikov Standards Track [Page 15] + +RFC 4314 IMAP ACL December 2005 + + + S: * 77 FETCH (FLAGS (\Deleted)) + S: * 78 FETCH (FLAGS ()) + S: * 79 FETCH (FLAGS (\Seen)) + S: A007 OK Fetch Completed + + In the latter case, \Answered, $Forwarded, and \Draft flags were + lost on COPY, as the user has no "w" right in the target mailbox. + + Expunging the selected mailbox: + EXPUNGE - "e" right on the selected mailbox. + + CLOSE - "e" right on the selected mailbox. If the server is + unable to expunge the mailbox because the user doesn't have the + "e" right, the server MUST ignore the expunge request, close the + mailbox, and return the tagged OK response. + + Fetch information about a mailbox and its messages: + SELECT/EXAMINE/STATUS - "r" right on the mailbox. + + FETCH - A FETCH request that implies setting \Seen flag MUST NOT + set it, if the current user doesn't have "s" right. + + Changing flags: + STORE - the server MUST check if the user has + "t" right - when the user modifies \Deleted flag + "s" right - when the user modifies \Seen flag + "w" right - for all other message flags. + STORE operation SHOULD NOT fail if the user has rights to modify + at least one flag specified in the STORE, as the tagged NO + response to a STORE command is not handled very well by deployed + clients. + + Changing ACLs: + SETACL/DELETEACL - "a" right on the mailbox. + + Reading ACLs: + GETACL - "a" right on the mailbox. + + MYRIGHTS - any of the following rights is required to perform the + operation: "l", "r", "i", "k", "x", "a". + + LISTRIGHTS - "a" right on the mailbox. + + + + + + + + + +Melnikov Standards Track [Page 16] + +RFC 4314 IMAP ACL December 2005 + + +5. Other Considerations + +5.1. Additional Requirements and Implementation Notes + +5.1.1. Servers + + This document defines an additional capability that is used to + announce the list of extra rights (excluding the ones defined in RFC + 2086) supported by the server. The set of rights MUST include "t", + "e", "x", and "k". Note that the extra rights can appear in any + order. + + Example: C: 1 capability + S: * CAPABILITY IMAP4REV1 STARTTLS LITERAL+ + ACL RIGHTS=texk + S: 1 OK completed + + Any server implementing an ACL extension MUST accurately reflect the + current user's rights in FLAGS and PERMANENTFLAGS responses. + + Example: C: A142 SELECT INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Seen \Answered \Flagged \*)] L + S: A142 OK [READ-WRITE] SELECT completed + C: A143 MYRIGHTS INBOX + S: * MYRIGHTS INBOX lrwis + S: A143 OK completed + + Note that in order to get better performance the client MAY pipeline + SELECT and MYRIGHTS commands: + + C: A142 SELECT INBOX + C: A143 MYRIGHTS INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Seen \Answered \Flagged \*)] L + S: A142 OK [READ-WRITE] SELECT completed + S: * MYRIGHTS INBOX lrwis + S: A143 OK completed + + + +Melnikov Standards Track [Page 17] + +RFC 4314 IMAP ACL December 2005 + + + Servers MAY cache the rights a user has on a mailbox when the mailbox + is selected, so that if a client's rights on a mailbox are changed + with SETACL or DELETEACL, commands specific to the selected state + (e.g., STORE, EXPUNGE) might not reflect the changed rights until the + mailbox is re-selected. If the server checks the rights on each + command, then it SHOULD send FLAGS and PERMANENTFLAGS responses if + they have changed. If such server detects that the user no longer + has read access to the mailbox, it MAY send an untagged BYE response + and close connection. It MAY also refuse to execute all commands + specific to the selected state until the mailbox is closed; however, + server implementors should note that most clients don't handle NO + responses very well. + + An ACL server MAY modify one or more ACLs for one or more identifiers + as a side effect of modifying the ACL specified in a + SETACL/DELETEACL. If the server does that, it MUST send untagged ACL + response(s) to notify the client about the changes made. + + An ACL server implementation MUST treat received ACL modification + commands as a possible ambiguity with respect to subsequent commands + affected by the ACL, as described in Section 5.5 of [IMAP4]. Hence a + pipeline SETACL + MYRIGHTS is an ambiguity with respect to the + server, meaning that the server must execute the SETACL command to + completion before the MYRIGHTS. However, clients are permitted to + send such a pipeline. + +5.1.2. Clients + + The following requirement is put on clients in order to allow for + future extensibility. A client implementation that allows a user to + read and update ACLs MUST preserve unrecognized rights that it + doesn't allow the user to change. That is, if the client + + 1) can read ACLs + and + 2) can update ACLs + but + 3) doesn't allow the user to change the rights the client doesn't + recognize, then it MUST preserve unrecognized rights. + + Otherwise the client could risk unintentionally removing permissions + it doesn't understand. + + + + + + + + + +Melnikov Standards Track [Page 18] + +RFC 4314 IMAP ACL December 2005 + + +5.2. Mapping of ACL Rights to READ-WRITE and READ-ONLY Response Codes + + A particular ACL server implementation MAY allow "shared multiuser + access" to some mailboxes. "Shared multiuser access" to a mailbox + means that multiple different users are able to access the same + mailbox, if they have proper access rights. "Shared multiuser + access" to the mailbox doesn't mean that the ACL for the mailbox is + currently set to allow access by multiple users. Let's denote a + "shared multiuser write access" as a "shared multiuser access" when a + user can be granted flag modification rights (any of "w", "s", or + "t"). + + Section 4 describes which rights are required for modifying different + flags. + + If the ACL server implements some flags as shared for a mailbox + (i.e., the ACL for the mailbox MAY be set up so that changes to those + flags are visible to another user), let's call the set of rights + associated with these flags (as described in Section 4) for that + mailbox collectively as "shared flag rights". Note that the "shared + flag rights" set MAY be different for different mailboxes. + + If the server doesn't support "shared multiuser write access" to a + mailbox or doesn't implement shared flags on the mailbox, "shared + flag rights" for the mailbox is defined to be the empty set. + + Example 1: Mailbox "banan" allows "shared multiuser write access" and + implements flags \Deleted, \Answered, and $MDNSent as + shared flags. "Shared flag rights" for the mailbox "banan" + is a set containing flags "t" (because system flag + \Deleted requires "t" right) and "w" (because both + \Answered and $MDNSent require "w" right). + + Example 2: Mailbox "apple" allows "shared multiuser write access" and + implements \Seen system flag as shared flag. "Shared flag + rights" for the mailbox "apple" contains "s" right + because system flag \Seen requires "s" right. + + Example 3: Mailbox "pear" allows "shared multiuser write access" and + implements flags \Seen, \Draft as shared flags. "Shared + flag rights" for the mailbox "apple" is a set containing + flags "s" (because system flag \Seen requires "s" right) + and "w" (because system flag \Draft requires "w" right). + + The server MUST include a READ-ONLY response code in the tagged OK + response to a SELECT command if none of the following rights is + granted to the current user: + + + + +Melnikov Standards Track [Page 19] + +RFC 4314 IMAP ACL December 2005 + + + "i", "e", and "shared flag rights"(***). + + The server SHOULD include a READ-WRITE response code in the tagged OK + response if at least one of the "i", "e", or "shared flag + rights"(***) is granted to the current user. + + (***) Note that a future extension to this document can extend the + list of rights that causes the server to return the READ-WRITE + response code. + + Example 1 (continued): The user that has "lrs" rights for the mailbox + "banan". The server returns READ-ONLY + response code on SELECT, as none of "iewt" + rights is granted to the user. + + Example 2 (continued): The user that has "rit" rights for the mailbox + "apple". The server returns READ-WRITE + response code on SELECT, as the user has "i" + right. + + Example 3 (continued): The user that has "rset" rights for the + mailbox "pear". The server returns READ-WRITE + response code on SELECT, as the user has "e" + and "s" rights. + +6. Security Considerations + + An implementation MUST make sure the ACL commands themselves do not + give information about mailboxes with appropriately restricted ACLs. + For example, when a user agent executes a GETACL command on a mailbox + that the user has no permission to LIST, the server would respond to + that request with the same error that would be used if the mailbox + did not exist, thus revealing no existence information, much less the + mailbox's ACL. + + IMAP clients implementing ACL that are able to modify ACLs SHOULD + warn a user that wants to give full access (or even just the "a" + right) to the special identifier "anyone". + + This document relies on [SASLprep] to describe steps required to + perform identifier canonicalization (preparation). The preparation + algorithm in SASLprep was specifically designed such that its output + is canonical, and it is well-formed. However, due to an anomaly + [PR29] in the specification of Unicode normalization, canonical + equivalence is not guaranteed for a select few character sequences. + Identifiers prepared with SASLprep can be stored and returned by an + ACL server. The anomaly affects ACL manipulation and evaluation of + identifiers containing the selected character sequences. These + + + +Melnikov Standards Track [Page 20] + +RFC 4314 IMAP ACL December 2005 + + + sequences, however, do not appear in well-formed text. In order to + address this problem, an ACL server MAY reject identifiers containing + sequences described in [PR29] by sending the tagged BAD response. + This is in addition to the requirement to reject identifiers that + fail SASLprep preparation as described in Section 3. + + Other security considerations described in [IMAP4] are relevant to + this document. In particular, ACL information is sent in the clear + over the network unless confidentiality protection is negotiated. + + This can be accomplished either by the use of STARTTLS, negotiated + privacy protection in the AUTHENTICATE command, or some other + protection mechanism. + +7. Formal Syntax + + Formal syntax is defined using ABNF [ABNF], extending the ABNF rules + in Section 9 of [IMAP4]. Elements not defined here can be found in + [ABNF] and [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case + insensitive. The use of uppercase or lowercase characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + LOWER-ALPHA = %x61-7A ;; a-z + + acl-data = "ACL" SP mailbox *(SP identifier SP + rights) + + capability =/ rights-capa + ;;capability is defined in [IMAP4] + + command-auth =/ setacl / deleteacl / getacl / + listrights / myrights + ;;command-auth is defined in [IMAP4] + + deleteacl = "DELETEACL" SP mailbox SP identifier + + getacl = "GETACL" SP mailbox + + identifier = astring + + listrights = "LISTRIGHTS" SP mailbox SP identifier + + listrights-data = "LISTRIGHTS" SP mailbox SP identifier + SP rights *(SP rights) + + + + +Melnikov Standards Track [Page 21] + +RFC 4314 IMAP ACL December 2005 + + + mailbox-data =/ acl-data / listrights-data / myrights-data + ;;mailbox-data is defined in [IMAP4] + + mod-rights = astring + ;; +rights to add, -rights to remove + ;; rights to replace + + myrights = "MYRIGHTS" SP mailbox + + myrights-data = "MYRIGHTS" SP mailbox SP rights + + new-rights = 1*LOWER-ALPHA + ;; MUST include "t", "e", "x", and "k". + ;; MUST NOT include standard rights listed + ;; in section 2.2 + + rights = astring + ;; only lowercase ASCII letters and digits + ;; are allowed. + + rights-capa = "RIGHTS=" new-rights + ;; RIGHTS=... capability + + setacl = "SETACL" SP mailbox SP identifier + SP mod-rights + +8. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards-track or + IESG-approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + This document defines the RIGHTS= IMAP capability. IANA has added + this capability to the registry. + +9. Internationalization Considerations + + Section 3 states requirements on servers regarding + internationalization of identifiers. + + + + + + + + + + +Melnikov Standards Track [Page 22] + +RFC 4314 IMAP ACL December 2005 + + +Appendix A. Changes since RFC 2086 + + 1. Changed the charset of "identifier" from US-ASCII to UTF-8. + 2. Specified that mailbox deletion is controlled by the "x" right + and EXPUNGE is controlled by the "e" right. + 3. Added the "t" right that controls STORE \Deleted. Redefined the + "d" right to be a macro for "e", "t", and possibly "x". + 4. Added the "k" right that controls CREATE. Redefined the "c" + right to be a macro for "k" and possibly "x". + 5. Specified that the "a" right also controls DELETEACL. + 6. Specified that the "r" right also controls STATUS. + 7. Removed the requirement to check the "r" right for CHECK, SEARCH + and FETCH, as this is required for SELECT/EXAMINE to be + successful. + 8. LISTRIGHTS requires the "a" right on the mailbox (same as + SETACL). + 9. Deleted "PARTIAL", this is a deprecated feature of RFC 1730. + 10. Specified that the "w" right controls setting flags other than + \Seen and \Deleted on APPEND. Also specified that the "s" right + controls the \Seen flag and that the "t" right controls the + \Deleted flag. + 11. Specified that SUBSCRIBE is NOT allowed with the "r" right. + 12. Specified that the "l" right controls SUBSCRIBE. + 13. GETACL is NOT allowed with the "r" right, even though there are + several implementations that allows that. If a user only has + "r" right, GETACL can disclose information about identifiers + existing on the mail system. + 14. Clarified that RENAME requires the "k" right for the new parent + and the "x" right for the old name. + 15. Added new section that describes which rights are required + and/or checked when performing various IMAP commands. + 16. Added mail client security considerations when dealing with + special identifier "anyone". + 17. Clarified that negative rights are not the same as DELETEACL. + 18. Added "Compatibility with RFC 2086" section. + 19. Added section about mapping of ACL rights to READ-WRITE and + READ-ONLY response codes. + 20. Changed BNF to ABNF. + 21. Added "Implementation Notes" section. + 22. Updated "References" section. + 23. Added more examples. + 24. Clarified when the virtual "c" and "d" rights are returned in + ACL, MYRIGHTS, and LISTRIGHTS responses. + + + + + + + + +Melnikov Standards Track [Page 23] + +RFC 4314 IMAP ACL December 2005 + + +Appendix B. Compatibility with RFC 2086 + + This non-normative section gives guidelines as to how an existing RFC + 2086 server implementation may be updated to comply with this + document. + + This document splits the "d" right into several new different rights: + "t", "e", and possibly "x" (see Section 2.1.1 for more details). The + "d" right remains for backward-compatibility, but it is a virtual + right. There are two approaches for RFC 2086 server implementors to + handle the "d" right and the new rights that have replaced it: + + a. Tie "t", "e" (and possibly "x) together - almost no changes. + b. Implement separate "x", "t" and "e". Return the "d" right in a + MYRIGHTS response or an ACL response containing ACL information + when any of the "t", "e" (and "x") is granted. + + In a similar manner this document splits the "c" right into several + new different rights: "k" and possibly "x" (see Section 2.1.1 for + more details). The "c" right remains for backwards-compatibility but + it is a virtual right. Again, RFC 2086 server implementors can + choose to tie rights or to implement separate rights, as described + above. + + Also check Sections 5.1.1 and 5.1.2, as well as Appendix A, to see + other changes required. Server implementors should check which + rights are required to invoke different IMAP4 commands as described + in Section 4. + +Appendix C. Known Deficiencies + + This specification has some known deficiencies including: + + 1. This is inadequate to provide complete read-write access to + mailboxes protected by Unix-style rights bits because there is no + equivalent to "chown" and "chgrp" commands nor is there a good + way to discover such limitations are present. + 2. Because this extension leaves the specific semantics of how + rights are combined by the server as implementation defined, the + ability to build a user-friendly interface is limited. + 3. Users, groups, and special identifiers (e.g., anyone) exist in + the same namespace. + + The work-in-progress "ACL2" extension is intended to redesign this + extension to address these deficiencies without the constraint of + backward-compatibility and may eventually supercede this facility. + + + + + +Melnikov Standards Track [Page 24] + +RFC 4314 IMAP ACL December 2005 + + + However, RFC 2086 is deployed in multiple implementations so this + intermediate step, which fixes the straightforward deficiencies in a + backward-compatible fashion, is considered worthwhile. + +Appendix D. Acknowledgements + + This document is a revision of RFC 2086 written by John G. Myers. + + Editor appreciates comments received from Mark Crispin, Chris Newman, + Cyrus Daboo, John G. Myers, Dave Cridland, Ken Murchison, Steve Hole, + Vladimir Butenko, Larry Greenfield, Robert Siemborski, Harrie + Hazewinkel, Philip Guenther, Brian Candler, Curtis King, Lyndon + Nerenberg, Lisa Dusseault, Arnt Gulbrandsen, and other participants + of the IMAPEXT working group. + +Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [Stringprep] Hoffman, P. and M. Blanchet, "Preparation of + Internationalized Strings ("stringprep")", RFC 3454, + December 2002. + + [SASLprep] Zeilenga, K., "SASLprep: Stringprep Profile for User + Names and Passwords", RFC 4013, February 2005. + +Informative References + + [RFC2086] Myers, J., "IMAP4 ACL extension", RFC 2086, + January 1997. + + [PR29] "Public Review Issue #29: Normalization Issue", + February 2004, + . + + + + + + + +Melnikov Standards Track [Page 25] + +RFC 4314 IMAP ACL December 2005 + + +Author's Address + + Alexey Melnikov + Isode Ltd. + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + GB + + EMail: alexey.melnikov@isode.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 26] + +RFC 4314 IMAP ACL December 2005 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2005). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at ietf- + ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + +Melnikov Standards Track [Page 27] + diff --git a/docs/rfcs/rfc4315.IMAP_UIDPLUS_extension.txt b/docs/rfcs/rfc4315.IMAP_UIDPLUS_extension.txt new file mode 100644 index 0000000..c026f42 --- /dev/null +++ b/docs/rfcs/rfc4315.IMAP_UIDPLUS_extension.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 4315 December 2005 +Obsoletes: 2359 +Category: Standards Track + + + Internet Message Access Protocol (IMAP) - UIDPLUS extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2005). + +Abstract + + The UIDPLUS extension of the Internet Message Access Protocol (IMAP) + provides a set of features intended to reduce the amount of time and + resources used by some client operations. The features in UIDPLUS + are primarily intended for disconnected-use clients. + +1. Introduction and Overview + + The UIDPLUS extension is present in any IMAP server implementation + that returns "UIDPLUS" as one of the supported capabilities to the + CAPABILITY command. + + The UIDPLUS extension defines an additional command. In addition, + this document recommends new status response codes in IMAP that + SHOULD be returned by all server implementations, regardless of + whether or not the UIDPLUS extension is implemented. + + The added facilities of the features in UIDPLUS are optimizations; + clients can provide equivalent functionality, albeit less + efficiently, by using facilities in the base protocol. + +1.1. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + + + + +Crispin Standards Track [Page 1] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to + be interpreted as described in [KEYWORDS]. + + A "UID set" is similar to the [IMAP] sequence set; however, the "*" + value for a sequence number is not permitted. + +2. Additional Commands + + The following command definition is an extension to [IMAP] section + 6.4. + +2.1. UID EXPUNGE Command + + Arguments: sequence set + + Data: untagged responses: EXPUNGE + + Result: OK - expunge completed + NO - expunge failure (e.g., permission denied) + BAD - command unknown or arguments invalid + + The UID EXPUNGE command permanently removes all messages that both + have the \Deleted flag set and have a UID that is included in the + specified sequence set from the currently selected mailbox. If a + message either does not have the \Deleted flag set or has a UID + that is not included in the specified sequence set, it is not + affected. + + This command is particularly useful for disconnected use clients. + By using UID EXPUNGE instead of EXPUNGE when resynchronizing with + the server, the client can ensure that it does not inadvertantly + remove any messages that have been marked as \Deleted by other + clients between the time that the client was last connected and + the time the client resynchronizes. + + If the server does not support the UIDPLUS capability, the client + should fall back to using the STORE command to temporarily remove + the \Deleted flag from messages it does not want to remove, then + issuing the EXPUNGE command. Finally, the client should use the + STORE command to restore the \Deleted flag on the messages in + which it was temporarily removed. + + Alternatively, the client may fall back to using just the EXPUNGE + command, risking the unintended removal of some messages. + + + + + + +Crispin Standards Track [Page 2] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + Example: C: A003 UID EXPUNGE 3000:3002 + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: A003 OK UID EXPUNGE completed + +3. Additional Response Codes + + The following response codes are extensions to the response codes + defined in [IMAP] section 7.1. With limited exceptions, discussed + below, server implementations that advertise the UIDPLUS extension + SHOULD return these response codes. + + In the case of a mailbox that has permissions set so that the client + can COPY or APPEND to the mailbox, but not SELECT or EXAMINE it, the + server SHOULD NOT send an APPENDUID or COPYUID response code as it + would disclose information about the mailbox. + + In the case of a mailbox that has UIDNOTSTICKY status (as defined + below), the server MAY omit the APPENDUID or COPYUID response code as + it is not meaningful. + + If the server does not return the APPENDUID or COPYUID response + codes, the client can discover this information by selecting the + destination mailbox. The location of messages placed in the + destination mailbox by COPY or APPEND can be determined by using + FETCH and/or SEARCH commands (e.g., for Message-ID or some unique + marker placed in the message in an APPEND). + + APPENDUID + + Followed by the UIDVALIDITY of the destination mailbox and the UID + assigned to the appended message in the destination mailbox, + indicates that the message has been appended to the destination + mailbox with that UID. + + If the server also supports the [MULTIAPPEND] extension, and if + multiple messages were appended in the APPEND command, then the + second value is a UID set containing the UIDs assigned to the + appended messages, in the order they were transmitted in the + APPEND command. This UID set may not contain extraneous UIDs or + the symbol "*". + + Note: the UID set form of the APPENDUID response code MUST NOT + be used if only a single message was appended. In particular, + a server MUST NOT send a range such as 123:123. This is + because a client that does not support [MULTIAPPEND] expects + only a single UID and not a UID set. + + + +Crispin Standards Track [Page 3] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + UIDs are assigned in strictly ascending order in the mailbox + (refer to [IMAP], section 2.3.1.1) and UID ranges are as in + [IMAP]; in particular, note that a range of 12:10 is exactly + equivalent to 10:12 and refers to the sequence 10,11,12. + + This response code is returned in a tagged OK response to the + APPEND command. + + COPYUID + + Followed by the UIDVALIDITY of the destination mailbox, a UID set + containing the UIDs of the message(s) in the source mailbox that + were copied to the destination mailbox and containing the UIDs + assigned to the copied message(s) in the destination mailbox, + indicates that the message(s) have been copied to the destination + mailbox with the stated UID(s). + + The source UID set is in the order the message(s) were copied; the + destination UID set corresponds to the source UID set and is in + the same order. Neither of the UID sets may contain extraneous + UIDs or the symbol "*". + + UIDs are assigned in strictly ascending order in the mailbox + (refer to [IMAP], section 2.3.1.1) and UID ranges are as in + [IMAP]; in particular, note that a range of 12:10 is exactly + equivalent to 10:12 and refers to the sequence 10,11,12. + + This response code is returned in a tagged OK response to the COPY + command. + + UIDNOTSTICKY + + The selected mailbox is supported by a mail store that does not + support persistent UIDs; that is, UIDVALIDITY will be different + each time the mailbox is selected. Consequently, APPEND or COPY + to this mailbox will not return an APPENDUID or COPYUID response + code. + + This response code is returned in an untagged NO response to the + SELECT command. + + Note: servers SHOULD NOT have any UIDNOTSTICKY mail stores. + This facility exists to support legacy mail stores in which it + is technically infeasible to support persistent UIDs. This + should be avoided when designing new mail stores. + + + + + + +Crispin Standards Track [Page 4] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + Example: C: A003 APPEND saved-messages (\Seen) {297} + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@example.com + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 38505 3955] APPEND completed + C: A004 COPY 2:4 meeting + S: A004 OK [COPYUID 38505 304,319:320 3956:3958] Done + C: A005 UID COPY 305:310 meeting + S: A005 OK No matching messages, so nothing copied + C: A006 COPY 2 funny + S: A006 OK Done + C: A007 SELECT funny + S: * 1 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 1] Message 1 is first unseen + S: * OK [UIDVALIDITY 3857529045] Validity session-only + S: * OK [UIDNEXT 2] Predicted next UID + S: * NO [UIDNOTSTICKY] Non-persistent UIDs + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen)] Limited + S: A007 OK [READ-WRITE] SELECT completed + + In this example, A003 and A004 demonstrate successful appending and + copying to a mailbox that returns the UIDs assigned to the messages. + A005 is an example in which no messages were copied; this is because + in A003, we see that message 2 had UID 304, and message 3 had UID + 319; therefore, UIDs 305 through 310 do not exist (refer to section + 2.3.1.1 of [IMAP] for further explanation). A006 is an example of a + message being copied that did not return a COPYUID; and, as expected, + A007 shows that the mail store containing that mailbox does not + support persistent UIDs. + +4. Formal Syntax + + Formal syntax is defined using ABNF [ABNF], which extends the ABNF + rules defined in [IMAP]. The IMAP4 ABNF should be imported before + attempting to validate these rules. + + append-uid = uniqueid + + capability =/ "UIDPLUS" + + + +Crispin Standards Track [Page 5] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + command-select =/ uid-expunge + + resp-code-apnd = "APPENDUID" SP nz-number SP append-uid + + resp-code-copy = "COPYUID" SP nz-number SP uid-set SP uid-set + + resp-text-code =/ resp-code-apnd / resp-code-copy / "UIDNOTSTICKY" + ; incorporated before the expansion rule of + ; atom [SP 1*] + ; that appears in [IMAP] + + uid-expunge = "UID" SP "EXPUNGE" SP sequence-set + + uid-set = (uniqueid / uid-range) *("," uid-set) + + uid-range = (uniqueid ":" uniqueid) + ; two uniqueid values and all values + ; between these two regards of order. + ; Example: 2:4 and 4:2 are equivalent. + + Servers that support [MULTIAPPEND] will have the following extension + to the above rules: + + append-uid =/ uid-set + ; only permitted if client uses [MULTIAPPEND] + ; to append multiple messages. + +5. Security Considerations + + The COPYUID and APPENDUID response codes return information about the + mailbox, which may be considered sensitive if the mailbox has + permissions set that permit the client to COPY or APPEND to the + mailbox, but not SELECT or EXAMINE it. + + Consequently, these response codes SHOULD NOT be issued if the client + does not have access to SELECT or EXAMINE the mailbox. + +6. IANA Considerations + + This document constitutes registration of the UIDPLUS capability in + the imap4-capabilities registry, replacing [RFC2359]. + +7. Normative References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + + + + +Crispin Standards Track [Page 6] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + + [IMAP] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - + VERSION 4rev1", RFC 3501, March 2003. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + +8. Informative References + + [RFC2359] Myers, J., "IMAP4 UIDPLUS extension", RFC 2359, June + 1998. + +9. Changes from RFC 2359 + + This document obsoletes [RFC2359]. However, it is based upon that + document, and takes substantial text from it (albeit with numerous + clarifications in wording). + + [RFC2359] implied that a server must always return COPYUID/APPENDUID + data; thus suggesting that in such cases the server should return + arbitrary data if the destination mailbox did not support persistent + UIDs. This document adds the UIDNOTSTICKY response code to indicate + that a mailbox does not support persistent UIDs, and stipulates that + a UIDPLUS server does not return COPYUID/APPENDUID data when the COPY + (or APPEND) destination mailbox has UIDNOTSTICKY status. + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Avenue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + + +Crispin Standards Track [Page 7] + +RFC 4315 IMAP - UIDPLUS Extension December 2005 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2005). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at ietf- + ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + +Crispin Standards Track [Page 8] + diff --git a/docs/rfcs/rfc4466.Collected_extensions_to_IMAP4_ABNF.txt b/docs/rfcs/rfc4466.Collected_extensions_to_IMAP4_ABNF.txt new file mode 100644 index 0000000..dfde168 --- /dev/null +++ b/docs/rfcs/rfc4466.Collected_extensions_to_IMAP4_ABNF.txt @@ -0,0 +1,955 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 4466 Isode Ltd. +Updates: 2088, 2342, 3501, 3502, 3516 C. Daboo +Category: Standards Track April 2006 + + + Collected Extensions to IMAP4 ABNF + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + Over the years, many documents from IMAPEXT and LEMONADE working + groups, as well as many individual documents, have added syntactic + extensions to many base IMAP commands described in RFC 3501. For + ease of reference, this document collects most of such ABNF changes + in one place. + + This document also suggests a set of standard patterns for adding + options and extensions to several existing IMAP commands defined in + RFC 3501. The patterns provide for compatibility between existing + and future extensions. + + This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. + It also includes part of the errata to RFC 3501. This document + doesn't specify any semantic changes to the listed RFCs. + + + + + + + + + + + + + + + +Melnikov & Daboo Standards Track [Page 1] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +Table of Contents + + 1. Introduction ....................................................2 + 1.1. Purpose of This Document ...................................2 + 1.2. Conventions Used in This Document ..........................3 + 2. IMAP ABNF Extensions ............................................3 + 2.1. Optional Parameters with the SELECT/EXAMINE Commands .......3 + 2.2. Extended CREATE Command ....................................4 + 2.3. Extended RENAME Command ....................................5 + 2.4. Extensions to FETCH and UID FETCH Commands .................6 + 2.5. Extensions to STORE and UID STORE Commands .................6 + 2.6. Extensions to SEARCH Command ...............................7 + 2.6.1. Extended SEARCH Command .............................7 + 2.6.2. ESEARCH untagged response ...........................8 + 2.7. Extensions to APPEND Command ...............................8 + 3. Formal Syntax ...................................................9 + 4. Security Considerations ........................................14 + 5. Normative References ...........................................15 + 6. Acknowledgements ...............................................15 + +1. Introduction + +1.1. Purpose of This Document + + This document serves several purposes: + + 1. rationalize and generalize ABNF for some existing IMAP + extensions; + 2. collect the ABNF in one place in order to minimize cross + references between documents; + 3. define building blocks for future extensions so that they can + be used together in a compatible way. + + It is expected that a future revision of this document will be + incorporated into a revision of RFC 3501. + + This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. + It also includes part of the errata to RFC 3501. This document + doesn't specify any semantic changes to the listed RFCs. + + The ABNF in section 6 of RFC 2342 got rewritten to conform to the + ABNF syntax as defined in RFC 4234 and to reference new non-terminals + from RFC 3501. It was also restructured to allow for better + readability. There were no changes "on the wire". + + Section 2 extends ABNF for SELECT, EXAMINE, CREATE, RENAME, FETCH/UID + FETCH, STORE/UID STORE, SEARCH, and APPEND commands in a consistent + manner. Extensions to all the commands but APPEND have the same + + + +Melnikov & Daboo Standards Track [Page 2] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + structure. Extensibility for the APPEND command was done slightly + differently in order to preserve backward compatibility with existing + extensions. + + Section 2 also defines a new ESEARCH response, whose purpose is to + define a better version of the SEARCH response defined in RFC 3501. + + Section 3 defines the collected ABNF that replaces pieces of ABNF in + the aforementioned RFCs. The collected ABNF got generalized to allow + for easier future extensibility. + +1.2. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in "Key words for + use in RFCs to Indicate Requirement Levels" [KEYWORDS]. + +2. IMAP ABNF Extensions + + This section is not normative. It provides some background on the + intended use of different extensions and it gives some guidance about + how future extensions should extend the described commands. + +2.1. Optional Parameters with the SELECT/EXAMINE Commands + + This document adds the ability to include one or more parameters with + the IMAP SELECT (section 6.3.1 of [IMAP4]) or EXAMINE (section 6.3.2 + of [IMAP4]) commands, to turn on or off certain standard behaviors, + or to add new optional behaviors required for a particular extension. + + There are two possible modes of operation: + + o A global state change where a single use of the optional parameter + will affect the session state from that time on, irrespective of + subsequent SELECT/EXAMINE commands. + + o A per-mailbox state change that will affect the session only for + the duration of the new selected state. A subsequent + SELECT/EXAMINE without the optional parameter will cancel its + effect for the newly selected mailbox. + + Optional parameters to the SELECT or EXAMINE commands are added as a + parenthesized list of attribute/value pairs, and appear after the + mailbox name in the standard SELECT or EXAMINE command. The order of + individual parameters is arbitrary. A parameter value is optional + + + +Melnikov & Daboo Standards Track [Page 3] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + and may consist of atoms, strings, or lists in a specific order. If + the parameter value is present, it always appears in parentheses (*). + Any parameter not defined by extensions that the server supports must + be rejected with a BAD response. + + Example: + + C: a SELECT INBOX (ANNOTATE) + S: ... + S: a OK SELECT complete + + In the above example, a single parameter is used with the SELECT + command. + + Example: + + C: a EXAMINE INBOX (ANNOTATE RESPONSES ("UID Responses") + CONDSTORE) + S: ... + S: a OK EXAMINE complete + + In the above example, three parameters are used with the EXAMINE + command. The second parameter consists of two items: an atom + "RESPONSES" followed by a quoted string. + + Example: + + C: a SELECT INBOX (BLURDYBLOOP) + S: a BAD Unknown parameter in SELECT command + + In the above example, a parameter not supported by the server is + used. This results in the BAD response from the server. + + (*) - if a parameter has a mandatory value, which can always be + represented as a number or a sequence-set, the parameter value does + not need the enclosing (). See ABNF for more details. + +2.2. Extended CREATE Command + + Arguments: mailbox name + OPTIONAL list of CREATE parameters + + Responses: no specific responses for this command + + Result: OK - create completed + NO - create failure: cannot create mailbox with + that name + BAD - argument(s) invalid + + + +Melnikov & Daboo Standards Track [Page 4] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + This document adds the ability to include one or more parameters with + the IMAP CREATE command (see section 6.3.3 of [IMAP4]), to turn on or + off certain standard behaviors, or to add new optional behaviors + required for a particular extension. No CREATE parameters are + defined in this document. + + Optional parameters to the CREATE command are added as a + parenthesized list of attribute/value pairs after the mailbox name. + The order of individual parameters is arbitrary. A parameter value + is optional and may consist of atoms, strings, or lists in a specific + order. If the parameter value is present, it always appears in + parentheses (*). Any parameter not defined by extensions that the + server supports must be rejected with a BAD response. + + (*) - if a parameter has a mandatory value, which can always be + represented as a number or a sequence-set, the parameter value does + not need the enclosing (). See ABNF for more details. + +2.3. Extended RENAME Command + + Arguments: existing mailbox name + new mailbox name + OPTIONAL list of RENAME parameters + + Responses: no specific responses for this command + + Result: OK - rename completed + NO - rename failure: cannot rename mailbox with + that name, cannot rename to mailbox with + that name, etc. + BAD - argument(s) invalid + + This document adds the ability to include one or more parameters with + the IMAP RENAME command (see section 6.3.5 of [IMAP4]), to turn on or + off certain standard behaviors, or to add new optional behaviors + required for a particular extension. No RENAME parameters are + defined in this document. + + Optional parameters to the RENAME command are added as a + parenthesized list of attribute/value pairs after the new mailbox + name. The order of individual parameters is arbitrary. A parameter + value is optional and may consist of atoms, strings, or lists in a + specific order. If the parameter value is present, it always appears + in parentheses (*). Any parameter not defined by extensions that the + server supports must be rejected with a BAD response. + + + + + + +Melnikov & Daboo Standards Track [Page 5] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + (*) - if a parameter has a mandatory value, which can always be + represented as a number or a sequence-set, the parameter value does + not need the enclosing (). See ABNF for more details. + +2.4. Extensions to FETCH and UID FETCH Commands + + Arguments: sequence set + message data item names or macro + OPTIONAL fetch modifiers + + Responses: untagged responses: FETCH + + Result: OK - fetch completed + NO - fetch error: cannot fetch that data + BAD - command unknown or arguments invalid + + This document extends the syntax of the FETCH and UID FETCH commands + (see section 6.4.5 of [IMAP4]) to include optional FETCH modifiers. + No fetch modifiers are defined in this document. + + The order of individual modifiers is arbitrary. Each modifier is an + attribute/value pair. A modifier value is optional and may consist + of atoms and/or strings and/or lists in a specific order. If the + modifier value is present, it always appears in parentheses (*). Any + modifiers not defined by extensions that the server supports must be + rejected with a BAD response. + + (*) - if a modifier has a mandatory value, which can always be + represented as a number or a sequence-set, the modifier value does + not need the enclosing (). See ABNF for more details. + +2.5. Extensions to STORE and UID STORE Commands + + Arguments: message set + OPTIONAL store modifiers + message data item name + value for message data item + + Responses: untagged responses: FETCH + + Result: OK - store completed + NO - store error: cannot store that data + BAD - command unknown or arguments invalid + + This document extends the syntax of the STORE and UID STORE commands + (see section 6.4.6 of [IMAP4]) to include optional STORE modifiers. + No store modifiers are defined in this document. + + + + +Melnikov & Daboo Standards Track [Page 6] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + The order of individual modifiers is arbitrary. Each modifier is an + attribute/value pair. A modifier value is optional and may consist + of atoms and/or strings and/or lists in a specific order. If the + modifier value is present, it always appears in parentheses (*). Any + modifiers not defined by extensions that the server supports must be + rejected with a BAD response. + + (*) - if a modifier has a mandatory value, which can always be + represented as a number or a sequence-set, the modifier value does + not need the enclosing (). See ABNF for more details. + +2.6. Extensions to SEARCH Command + +2.6.1. Extended SEARCH Command + + Arguments: OPTIONAL result specifier + OPTIONAL [CHARSET] specification + searching criteria (one or more) + + Responses: REQUIRED untagged response: SEARCH (*) + + Result: OK - search completed + NO - search error: cannot search that [CHARSET] or + criteria + BAD - command unknown or arguments invalid + + This section updates definition of the SEARCH command described in + section 6.4.4 of [IMAP4]. + + The SEARCH command is extended to allow for result options. This + document does not define any result options. + + The order of individual options is arbitrary. Individual options may + contain parameters enclosed in parentheses (**). If an option has + parameters, they consist of atoms and/or strings and/or lists in a + specific order. Any options not defined by extensions that the + server supports must be rejected with a BAD response. + + (*) - An extension to the SEARCH command may require another untagged + response, or no untagged response to be returned. Section 2.6.2 + defines a new ESEARCH untagged response that replaces the SEARCH + untagged response. Note that for a given extended SEARCH command the + SEARCH and ESEARCH responses SHOULD be mutually exclusive, i.e., only + one of them should be returned. + + (**) - if an option has a mandatory parameter, which can always be + represented as a number or a sequence-set, the option parameter does + not need the enclosing (). See ABNF for more details. + + + +Melnikov & Daboo Standards Track [Page 7] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +2.6.2. ESEARCH untagged response + + Contents: one or more search-return-data pairs + + The ESEARCH response SHOULD be sent as a result of an extended SEARCH + or UID SEARCH command specified in section 2.6.1. + + The ESEARCH response starts with an optional search correlator. If + it is missing, then the response was not caused by a particular IMAP + command, whereas if it is present, it contains the tag of the command + that caused the response to be returned. + + The search correlator is followed by an optional UID indicator. If + this indicator is present, all data in the ESEARCH response refers to + UIDs, otherwise all returned data refers to message numbers. + + The rest of the ESEARCH response contains one or more search data + pairs. Each pair starts with unique return item name, followed by a + space and the corresponding data. Search data pairs may be returned + in any order. Unless specified otherwise by an extension, any return + item name SHOULD appear only once in an ESEARCH response. + + Example: S: * ESEARCH UID COUNT 5 ALL 4:19,21,28 + + Example: S: * ESEARCH (TAG "a567") UID COUNT 5 ALL 4:19,21,28 + + Example: S: * ESEARCH COUNT 5 ALL 1:17,21 + +2.7. Extensions to APPEND Command + + The IMAP BINARY extension [BINARY] extends the APPEND command to + allow a client to append data containing NULs by using the + syntax. The ABNF was rewritten to allow for easier extensibility by + IMAP extensions. This document hasn't specified any semantical + changes to the [BINARY] extension. + + In addition, the non-terminal "literal8" defined in [BINARY] got + extended to allow for non-synchronizing literals if both [BINARY] and + [LITERAL+] extensions are supported by the server. + + The IMAP MULTIAPPEND extension [MULTIAPPEND] extends the APPEND + command to allow a client to append multiple messages atomically. + This document defines a common syntax for the APPEND command that + takes into consideration syntactic extensions defined by both + [BINARY] and [MULTIAPPEND] extensions. + + + + + + +Melnikov & Daboo Standards Track [Page 8] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +3. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + Non-terminals referenced but not defined below are as defined by + [IMAP4]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of uppercase or lowercase characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + append = "APPEND" SP mailbox 1*append-message + ;; only a single append-message may appear + ;; if MULTIAPPEND [MULTIAPPEND] capability + ;; is not present + + append-message = append-opts SP append-data + + append-ext = append-ext-name SP append-ext-value + ;; This non-terminal define extensions to + ;; to message metadata. + + append-ext-name = tagged-ext-label + + append-ext-value= tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + append-data = literal / literal8 / append-data-ext + + append-data-ext = tagged-ext + ;; This non-terminal shows recommended syntax + ;; for future extensions, + ;; i.e., a mandatory label followed + ;; by parameters. + + append-opts = [SP flag-list] [SP date-time] *(SP append-ext) + ;; message metadata + + charset = atom / quoted + ;; Exact syntax is defined in [CHARSET]. + + create = "CREATE" SP mailbox + [create-params] + ;; Use of INBOX gives a NO error. + + + +Melnikov & Daboo Standards Track [Page 9] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + create-params = SP "(" create-param *( SP create-param) ")" + + create-param-name = tagged-ext-label + + create-param = create-param-name [SP create-param-value] + + create-param-value= tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + esearch-response = "ESEARCH" [search-correlator] [SP "UID"] + *(SP search-return-data) + ;; Note that SEARCH and ESEARCH responses + ;; SHOULD be mutually exclusive, + ;; i.e., only one of the response types + ;; should be + ;; returned as a result of a command. + + + examine = "EXAMINE" SP mailbox [select-params] + ;; modifies the original IMAP EXAMINE command + ;; to accept optional parameters + + fetch = "FETCH" SP sequence-set SP ("ALL" / "FULL" / + "FAST" / fetch-att / + "(" fetch-att *(SP fetch-att) ")") + [fetch-modifiers] + ;; modifies the original IMAP4 FETCH command to + ;; accept optional modifiers + + fetch-modifiers = SP "(" fetch-modifier *(SP fetch-modifier) ")" + + fetch-modifier = fetch-modifier-name [ SP fetch-modif-params ] + + fetch-modif-params = tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + fetch-modifier-name = tagged-ext-label + + literal8 = "~{" number ["+"] "}" CRLF *OCTET + ;; A string that might contain NULs. + ;; represents the number of OCTETs + ;; in the response string. + ;; The "+" is only allowed when both LITERAL+ and + ;; BINARY extensions are supported by the server. + + + + +Melnikov & Daboo Standards Track [Page 10] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + mailbox-data =/ Namespace-Response / + esearch-response + + Namespace = nil / "(" 1*Namespace-Descr ")" + + Namespace-Command = "NAMESPACE" + + Namespace-Descr = "(" string SP + (DQUOTE QUOTED-CHAR DQUOTE / nil) + *(Namespace-Response-Extension) ")" + + Namespace-Response-Extension = SP string SP + "(" string *(SP string) ")" + + Namespace-Response = "NAMESPACE" SP Namespace + SP Namespace SP Namespace + ;; This response is currently only allowed + ;; if the IMAP server supports [NAMESPACE]. + ;; The first Namespace is the Personal Namespace(s) + ;; The second Namespace is the Other Users' Namespace(s) + ;; The third Namespace is the Shared Namespace(s) + + rename = "RENAME" SP mailbox SP mailbox + [rename-params] + ;; Use of INBOX as a destination gives + ;; a NO error, unless rename-params + ;; is not empty. + + rename-params = SP "(" rename-param *( SP rename-param) ")" + + rename-param = rename-param-name [SP rename-param-value] + + rename-param-name = tagged-ext-label + + rename-param-value= tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + response-data = "*" SP response-payload CRLF + + response-payload= resp-cond-state / resp-cond-bye / + mailbox-data / message-data / capability-data + + search = "SEARCH" [search-return-opts] + SP search-program + + search-correlator = SP "(" "TAG" SP tag-string ")" + + + +Melnikov & Daboo Standards Track [Page 11] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + search-program = ["CHARSET" SP charset SP] + search-key *(SP search-key) + ;; CHARSET argument to SEARCH MUST be + ;; registered with IANA. + + search-return-data = search-modifier-name SP search-return-value + ;; Note that not every SEARCH return option + ;; is required to have the corresponding + ;; ESEARCH return data. + + search-return-opts = SP "RETURN" SP "(" [search-return-opt + *(SP search-return-opt)] ")" + + search-return-opt = search-modifier-name [SP search-mod-params] + + search-return-value = tagged-ext-val + ;; Data for the returned search option. + ;; A single "nz-number"/"number" value + ;; can be returned as an atom (i.e., without + ;; quoting). A sequence-set can be returned + ;; as an atom as well. + + search-modifier-name = tagged-ext-label + + search-mod-params = tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + select = "SELECT" SP mailbox [select-params] + ;; modifies the original IMAP SELECT command to + ;; accept optional parameters + + select-params = SP "(" select-param *(SP select-param) ")" + + select-param = select-param-name [SP select-param-value] + ;; a parameter to SELECT may contain one or + ;; more atoms and/or strings and/or lists. + + select-param-name= tagged-ext-label + + select-param-value= tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + + status-att-list = status-att-val *(SP status-att-val) + ;; Redefines status-att-list from RFC 3501. + + + +Melnikov & Daboo Standards Track [Page 12] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + ;; status-att-val is defined in RFC 3501 errata + + status-att-val = ("MESSAGES" SP number) / + ("RECENT" SP number) / + ("UIDNEXT" SP nz-number) / + ("UIDVALIDITY" SP nz-number) / + ("UNSEEN" SP number) + ;; Extensions to the STATUS responses + ;; should extend this production. + ;; Extensions should use the generic + ;; syntax defined by tagged-ext. + + store = "STORE" SP sequence-set [store-modifiers] + SP store-att-flags + ;; extend [IMAP4] STORE command syntax + ;; to allow for optional store-modifiers + + store-modifiers = SP "(" store-modifier *(SP store-modifier) + ")" + + store-modifier = store-modifier-name [SP store-modif-params] + + store-modif-params = tagged-ext-val + ;; This non-terminal shows recommended syntax + ;; for future extensions. + + store-modifier-name = tagged-ext-label + + tag-string = string + ;; tag of the command that caused + ;; the ESEARCH response, sent as + ;; a string. + + tagged-ext = tagged-ext-label SP tagged-ext-val + ;; recommended overarching syntax for + ;; extensions + + tagged-ext-label = tagged-label-fchar *tagged-label-char + ;; Is a valid RFC 3501 "atom". + + tagged-label-fchar = ALPHA / "-" / "_" / "." + + tagged-label-char = tagged-label-fchar / DIGIT / ":" + + + + + + + + +Melnikov & Daboo Standards Track [Page 13] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + + tagged-ext-comp = astring / + tagged-ext-comp *(SP tagged-ext-comp) / + "(" tagged-ext-comp ")" + ;; Extensions that follow this general + ;; syntax should use nstring instead of + ;; astring when appropriate in the context + ;; of the extension. + ;; Note that a message set or a "number" + ;; can always be represented as an "atom". + ;; An URL should be represented as + ;; a "quoted" string. + + tagged-ext-simple = sequence-set / number + + tagged-ext-val = tagged-ext-simple / + "(" [tagged-ext-comp] ")" + +4. Security Considerations + + This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. + The updated documents must be consulted for security considerations + for the extensions that they define. + + As a protocol gets more complex, parser bugs become more common + including buffer overflow, denial of service, and other common + security coding errors. To the extent that this document makes the + parser more complex, it makes this situation worse. To the extent + that this document makes the parser more consistent and thus simpler, + the situation is improved. The impact will depend on how many + deployed IMAP extensions are consistent with this document. + Implementers are encouraged to take care of these issues when + extending existing implementations. Future IMAP extensions should + strive for consistency and simplicity to the greatest extent + possible. + + Extensions to IMAP commands that are permitted in NOT AUTHENTICATED + state are more sensitive to these security issues due to the larger + possible attacker community prior to authentication, and the fact + that some IMAP servers run with elevated privileges in that state. + This document does not extend any commands permitted in NOT + AUTHENTICATED state. Future IMAP extensions to commands permitted in + NOT AUTHENTICATED state should favor simplicity over consistency or + extensibility. + + + + + + + + +Melnikov & Daboo Standards Track [Page 14] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +5. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - + VERSION 4rev1", RFC 3501, March 2003. + + [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 4234, October 2005. + + [CHARSET] Freed, N. and J. Postel, "IANA Charset Registration + Procedures", BCP 19, RFC 2978, October 2000. + + [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + + [NAMESPACE] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, + May 1998. + + [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC + 2088, January 1997. + + [BINARY] Nerenberg, L., "IMAP4 Binary Content Extension", RFC + 3516, April 2003. + +6. Acknowledgements + + This documents is based on ideas proposed by Pete Resnick, Mark + Crispin, Ken Murchison, Philip Guenther, Randall Gellens, and Lyndon + Nerenberg. + + However, all errors and omissions must be attributed to the authors + of the document. + + Thanks to Philip Guenther, Dave Cridland, Mark Crispin, Chris Newman, + Elwyn Davies, and Barry Leiba for comments and corrections. + + literal8 syntax was taken from RFC 3516. + + + + + + + + + + + + +Melnikov & Daboo Standards Track [Page 15] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex, TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + Cyrus Daboo + + EMail: cyrus@daboo.name + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Daboo Standards Track [Page 16] + +RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Melnikov & Daboo Standards Track [Page 17] + diff --git a/docs/rfcs/rfc4467.IMAP_URLAUTH_extension.txt b/docs/rfcs/rfc4467.IMAP_URLAUTH_extension.txt new file mode 100644 index 0000000..83b6516 --- /dev/null +++ b/docs/rfcs/rfc4467.IMAP_URLAUTH_extension.txt @@ -0,0 +1,1011 @@ + + + + + + +Network Working Group M. Crispin +Request for Comments: 4467 University of Washington +Updates: 3501 May 2006 +Category: Standards Track + + + Internet Message Access Protocol (IMAP) - URLAUTH Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + This document describes the URLAUTH extension to the Internet Message + Access Protocol (IMAP) (RFC 3501) and the IMAP URL Scheme (IMAPURL) + (RFC 2192). This extension provides a means by which an IMAP client + can use URLs carrying authorization to access limited message data on + the IMAP server. + + An IMAP server that supports this extension indicates this with a + capability name of "URLAUTH". + +1. Introduction + + In [IMAPURL], a URL of the form imap://fred@example.com/INBOX/;uid=20 + requires authorization as userid "fred". However, [IMAPURL] implies + that it only supports authentication and confuses the concepts of + authentication and authorization. + + The URLAUTH extension defines an authorization mechanism for IMAP + URLs to replace [IMAPURL]'s authentication-only mechanism. URLAUTH + conveys authorization in the URL string itself and reuses a portion + of the syntax of the [IMAPURL] authentication mechanism to convey the + authorization identity (which also defines the default namespace in + [IMAP]). + + The URLAUTH extension provides a means by which an authorized user of + an IMAP server can create URLAUTH-authorized IMAP URLs. A URLAUTH- + authorized URL conveys authorization (not authentication) to the data + + + +Crispin Standards Track [Page 1] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + addressed by that URL. This URL can be used in another IMAP session + to access specific content on the IMAP server, without otherwise + providing authorization to any other data (such as other data in the + mailbox specified in the URL) owned by the authorizing user. + + Conceptually, a URLAUTH-authorized URL can be thought of as a "pawn + ticket" that carries no authentication information and can be + redeemed by whomever presents it. However, unlike a pawn ticket, + URLAUTH has optional mechanisms to restrict the usage of a URLAUTH- + authorized URL. Using these mechanisms, URLAUTH-authorized URLs can + be usable by: + + . anonymous (the "pawn ticket" model) + . authenticated users only + . a specific authenticated user only + . message submission acting on behalf of a specific user only + + There is also a mechanism for expiration. + + A URLAUTH-authorized URL can be used in the argument to the BURL + command in message composition, as described in [BURL], for such + purposes as allowing a client (with limited memory or other + resources) to submit a message forward or to resend from an IMAP + mailbox without requiring the client to fetch that message data. + + The URLAUTH is generated using an authorization mechanism name and an + authorization token, which is generated using a secret mailbox access + key. An IMAP client can request that the server generate and assign + a new mailbox access key (thus effectively revoking all current URLs + using URLAUTH with the old mailbox access key) but cannot set the + mailbox access key to a key of its own choosing. + +1.1. Conventions Used in this Document + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in [KEYWORDS]. + + The formal syntax uses the Augmented Backus-Naur Form (ABNF) notation + including the core rules defined in Appendix A of [ABNF]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. If a single "C:" or "S:" label applies to + multiple lines, then the line breaks between those lines are for + editorial clarity only and are not part of the actual protocol + exchange. + + + + + + +Crispin Standards Track [Page 2] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +2. Concepts + +2.1. URLAUTH + + The URLAUTH is a component, appended at the end of a URL, that + conveys authorization to access the data addressed by that URL. It + contains an authorized access identifier, an authorization mechanism + name, and an authorization token. The authorization token is + generated from the URL, the authorized access identifier, the + authorization mechanism name, and a mailbox access key. + +2.2. Mailbox Access Key + + The mailbox access key is a random string with at least 128 bits of + entropy. It is generated by software (not by the human user) and + MUST be unpredictable. + + Each user has a table of mailboxes and an associated mailbox access + key for each mailbox. Consequently, the mailbox access key is per- + user and per-mailbox. In other words, two users sharing the same + mailbox each have a different mailbox access key for that mailbox, + and each mailbox accessed by a single user also has a different + mailbox access key. + +2.3. Authorized Access Identifier + + The authorized access identifier restricts use of the URLAUTH + authorized URL to certain users authorized on the server, as + described in section 3. + +2.4. Authorization Mechanism + + The authorization mechanism is the algorithm by which the URLAUTH is + generated and subsequently verified, using the mailbox access key. + +2.4.1. INTERNAL Authorization Mechanism + + This specification defines the INTERNAL mechanism, which uses a token + generation algorithm of the server's choosing and does not involve + disclosure of the mailbox access key to the client. + + Note: The token generation algorithm chosen by the server + implementation should be modern and reasonably secure. At the + time of the writing of this document, an [HMAC] such as HMAC-SHA1 + is recommended. + + + + + + +Crispin Standards Track [Page 3] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + If it becomes necessary to change the token generation algorithm + of the INTERNAL mechanism (e.g., because an attack against the + current algorithm has been discovered), all currently existing + URLAUTH-authorized URLs are invalidated by the change in + algorithm. Since this would be an unpleasant surprise to + applications that depend upon the validity of a URLAUTH-authorized + URL, and there is no good way to do a bulk update of existing + deployed URLs, it is best to avoid this situation by using a + secure algorithm as opposed to one that is "good enough". + + Server implementations SHOULD consider the possibility of changing + the algorithm. In some cases, it may be desirable to implement + the change of algorithm in a way that newly-generated tokens use + the new algorithm, but that for a limited period of time tokens + using either the new or old algorithm can be validated. + Consequently, the server SHOULD incorporate some means of + identifying the token generation algorithm within the token. + + Although this specification is extensible for other mechanisms, none + are defined in this document. In addition to the mechanism name + itself, other mechanisms may have mechanism-specific data, which is + to be interpreted according to the definition of that mechanism. + +2.5. Authorization Token + + The authorization token is a deterministic string of at least 128 + bits that an entity with knowledge of the secret mailbox access key + and URL authorization mechanism can use to verify the URL. + +3. IMAP URL Extensions + + [IMAPURL] is extended by allowing the addition of + ";EXPIRE=" and ";URLAUTH=::" to IMAP + URLs that refer to a specific message or message parts. + + The URLAUTH is comprised of ";URLAUTH=::" and + MUST be at the end of the URL. + + URLAUTH does not apply to, and MUST NOT be used with, any IMAP URL + that refers to an entire IMAP server, a list of mailboxes, an entire + IMAP mailbox, or IMAP search results. + + When ";EXPIRE=" is used, this indicates the latest date and + time that the URL is valid. After that date and time, the URL has + expired, and server implementations MUST reject the URL. If + ";EXPIRE=" is not used, the URL has no expiration, but + still can be revoked as discussed below. + + + + +Crispin Standards Track [Page 4] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + The URLAUTH takes the form ";URLAUTH=::". It is + composed of three parts. The portion provides the + authorized access identifiers, which may constrain the operations and + users that are permitted to use this URL. The portion + provides the authorization mechanism used by the IMAP server to + generate the authorization token that follows. The portion + provides the authorization token. + + The "submit+" access identifier prefix, followed by a userid, + indicates that only a userid authorized as a message submission + entity on behalf of the specified userid is permitted to use this + URL. The IMAP server does not validate the specified userid but does + validate that the IMAP session has an authorization identity that is + authorized as a message submission entity. The authorized message + submission entity MUST validate the userid prior to contacting the + IMAP server. + + The "user+" access identifier prefix, followed by a userid, indicates + that use of this URL is limited to IMAP sessions that are logged in + as the specified userid (that is, have authorization identity as that + userid). + + Note: If a SASL mechanism that provides both authorization and + authentication identifiers is used to authenticate to the IMAP + server, the "user+" access identifier MUST match the authorization + identifier. + + The "authuser" access identifier indicates that use of this URL is + limited to IMAP sessions that are logged in as an authorized user + (that is, have authorization identity as an authorized user) of that + IMAP server. Use of this URL is prohibited to anonymous IMAP + sessions. + + The "anonymous" access identifier indicates that use of this URL is + not restricted by session authorization identity; that is, any IMAP + session in authenticated or selected state (as defined in [IMAP]), + including anonymous sessions, may issue a URLFETCH using this URL. + + The authorization token is represented as an ASCII-encoded + hexadecimal string, which is used to authorize the URL. The length + and the calculation of the authorization token depends upon the + mechanism used; but, in all cases, the authorization token is at + least 128 bits (and therefore at least 32 hexadecimal digits). + + + + + + + + +Crispin Standards Track [Page 5] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +4. Discussion of URLAUTH Authorization Issues + + In [IMAPURL], the userid before the "@" in the URL has two purposes: + + 1) It provides context for user-specific mailbox paths such as + "INBOX". + + 2) It specifies that resolution of the URL requires logging in as + that user and limits use of that URL to only that user. + + An obvious limitation of using the same field for both purposes is + that the URL can only be resolved by the mailbox owner. + + URLAUTH overrides the second purpose of the userid in the IMAP URL + and by default permits the URL to be resolved by any user permitted + by the access identifier. + + The "user+" access identifier limits resolution of that URL + to a particular userid, whereas the "submit+" access + identifier is more general and simply requires that the session be + authorized by a user that has been granted a "submit" role within the + authentication system. Use of either of these access identifiers + makes it impossible for an attacker, spying on the session, to use + the same URL, either directly or by submission to a message + submission entity. + + The "authuser" and "anonymous" access identifiers do not have this + level of protection and should be used with caution. These access + identifiers are primarily useful for public export of data from an + IMAP server, without requiring that it be copied to a web or + anonymous FTP server. Refer to the Security Considerations for more + details. + +5. Generation of URLAUTH-Authorized URLs + + A URLAUTH-authorized URL is generated from an initial URL as follows: + + An initial URL is built, ending with ";URLAUTH=" but without + the "::" components. An authorization mechanism is + selected and used to calculate the authorization token, with the + initial URL as the data and a secret known to the IMAP server as the + key. The URLAUTH-authorized URL is generated by taking the initial + URL and appending ":", the URL authorization mechanism name, ":", and + the ASCII-encoded hexadecimal representation of the authorization + token. + + + + + + +Crispin Standards Track [Page 6] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + Note: ASCII-encoded hexadecimal is used instead of BASE64 because + a BASE64 representation may have "=" padding characters, which + would be problematic in a URL. + + In the INTERNAL mechanism, the mailbox access key for that mailbox is + the secret known to the IMAP server, and a server-selected algorithm + is used as described in section 2.4.1. + +6. Validation of URLAUTH-authorized URLs + + A URLAUTH-authorized URL is validated as follows: + + The URL is split at the ":" that separates "" from + ":" in the ";URLAUTH=::" portion of + the URL. The ":" portion is first parsed and saved as + the authorization mechanism and the authorization token. The URL is + truncated, discarding the ":" described above, to create a "rump URL" + (the URL minus the ":" and the ":" portion). The rump + URL is then analyzed to identify the mailbox. + + If the mailbox cannot be identified, an authorization token is + calculated on the rump URL, using random "plausible" keys (selected + by the server) as needed, before returning a validation failure. + This prevents timing attacks aimed at identifying mailbox names. + + If the mailbox can be identified, the authorization token is + calculated on the rump URL and a secret known to the IMAP server + using the given URL authorization mechanism. Validation is + successful if, and only if, the calculated authorization token for + that mechanism matches the authorization token supplied in + ";URLAUTH=::". + + Removal of the "::" portion of the URL MUST be the only + operation applied to the URLAUTH-authorized URL to get the rump URL. + In particular, URL percent escape decoding and case-folding + (including to the domain part of the URL) MUST NOT occur. + + In the INTERNAL mechanism, the mailbox access key for that mailbox is + used as the secret known to the IMAP server, and the same server- + selected algorithm used for generating URLs is used to calculate the + authorization token for verification. + + + + + + + + + + +Crispin Standards Track [Page 7] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +7. Additional Commands + + These commands are extensions to the [IMAP] base protocol. + + The section headings of these commands are intended to correspond + with where they would be located in the base protocol document if + they were part of that document. + +BASE.6.3.RESETKEY. RESETKEY Command + + Arguments: optional mailbox name + optional mechanism name(s) + + Responses: none other than in result + + Result: OK - RESETKEY completed, URLMECH containing new data + NO - RESETKEY error: can't change key of that mailbox + BAD - command unknown or arguments invalid + + The RESETKEY command has two forms. + + The first form accepts a mailbox name as an argument and generates a + new mailbox access key for the given mailbox in the user's mailbox + access key table, replacing any previous mailbox access key (and + revoking any URLs that were authorized with a URLAUTH using that key) + in that table. By default, the mailbox access key is generated for + the INTERNAL mechanism; other mechanisms can be specified with the + optional mechanism argument. + + The second form, with no arguments, removes all mailbox access keys + in the user's mailbox access key table, revoking all URLs currently + authorized using URLAUTH by the user. + + Any current IMAP session logged in as the user that has the mailbox + selected will receive an untagged OK response with the URLMECH status + response code (see section BASE.7.1.URLMECH for more details about + the URLMECH status response code). + + Example: + + C: a31 RESETKEY + S: a31 OK All keys removed + C: a32 RESETKEY INBOX + S: a32 OK [URLMECH INTERNAL] mechs + C: a33 RESETKEY INBOX XSAMPLE + S: a33 OK [URLMECH INTERNAL XSAMPLE=P34OKhO7VEkCbsiYY8rGEg==] done + + + + + +Crispin Standards Track [Page 8] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +BASE.6.3.GENURLAUTH. GENURLAUTH Command + + Argument: one or more URL/mechanism pairs + + Response: untagged response: GENURLAUTH + + Result: OK - GENURLAUTH completed + NO - GENURLAUTH error: can't generate a URLAUTH + BAD - command unknown or arguments invalid + + The GENURLAUTH command requests that the server generate a URLAUTH- + authorized URL for each of the given URLs using the given URL + authorization mechanism. + + The server MUST validate each supplied URL as follows: + + (1) The mailbox component of the URL MUST refer to an existing + mailbox. + + (2) The server component of the URL MUST contain a valid userid + that identifies the owner of the mailbox access key table that + will be used to generate the URLAUTH-authorized URL. As a + consequence, the iserver rule of [IMAPURL] is modified so that + iuserauth is mandatory. + + Note: the server component of the URL is generally the + logged in userid and server. If not, then the logged in + userid and server MUST have owner-type access to the + mailbox access key table owned by the userid and server + indicated by the server component of the URL. + + (3) There is a valid access identifier that, in the case of + "submit+" and "user+", will contain a valid userid. This + userid is not necessarily the same as the owner userid + described in (2). + + (4) The server MAY also verify that the iuid and/or isection + components (if present) are valid. + + If any of the above checks fail, the server MUST return a tagged BAD + response with the following exception. If an invalid userid is + supplied as the mailbox access key owner and/or as part of the access + identifier, the server MAY issue a tagged OK response with a + generated mailbox key that always fails validation when used with a + URLFETCH command. This exception prevents an attacker from + validating userids. + + + + + +Crispin Standards Track [Page 9] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + If there is currently no mailbox access key for the given mailbox in + the owner's mailbox access key table, one is automatically generated. + That is, it is not necessary to use RESETKEY prior to first-time use + of GENURLAUTH. + + If the command is successful, a GENURLAUTH response code is returned + listing the requested URLs as URLAUTH-authorized URLs. + + Examples: + + C: a775 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2" INTERNAL + S: a775 BAD missing access identifier in supplied URL + C: a776 GENURLAUTH "imap://example.com/Shared/;uid=20/ + ;section=1.2;urlauth=submit+fred" INTERNAL + S: a776 BAD missing owner username in supplied URL + C: a777 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2;urlauth=submit+fred" INTERNAL + S: * GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 + ;urlauth=submit+fred:internal:91354a473744909de610943775f92038" + S: a777 OK GENURLAUTH completed + +BASE.6.3.URLFETCH. URLFETCH Command + + Argument: one or more URLs + + Response: untagged response: URLFETCH + + Result: OK - urlfetch completed + NO - urlfetch failed due to server internal error + BAD - command unknown or arguments invalid + + The URLFETCH command requests that the server return the text data + associated with the specified IMAP URLs, as described in [IMAPURL] + and extended by this document. The data is returned for all + validated URLs, regardless of whether or not the session would + otherwise be able to access the mailbox containing that data via + SELECT or EXAMINE. + + Note: This command does not require that the URL refer to the + selected mailbox; nor does it require that any mailbox be + selected. It also does not in any way interfere with any selected + mailbox. + + + + + + + + +Crispin Standards Track [Page 10] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + The URLFETCH command effectively executes with the access of the + userid in the server component of the URL (which is generally the + userid that issued the GENURLAUTH). By itself, the URLAUTH does NOT + grant access to the data; once validated, it grants whatever access + to the data is held by the userid in the server component of the URL. + That access may have changed since the GENURLAUTH was done. + + The URLFETCH command MUST return an untagged URLFETCH response and a + tagged OK response to any URLFETCH command that is syntactically + valid. A NO response indicates a server internal failure that may be + resolved on later retry. + + Note: The possibility of a NO response is to accommodate + implementations that would otherwise have to issue an untagged BYE + with a fatal error due to an inability to respond to a valid + request. In an ideal world, a server SHOULD NOT issue a NO + response. + + The server MUST return NIL for any IMAP URL that references an entire + IMAP server, a list of mailboxes, an entire IMAP mailbox, or IMAP + search results. + + Example: + + Note: For clarity, this example uses the LOGIN command, which + SHOULD NOT be used over a non-encrypted communication path. + + This example is of a submit server, obtaining a message segment + for a message that it has already validated was submitted by + "fred". + + S: * OK [CAPABILITY IMAP4REV1 URLAUTH] example.com IMAP server + C: a001 LOGIN submitserver secret + S: a001 OK submitserver logged in + C: a002 URLFETCH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2;urlauth=submit+fred:internal + :91354a473744909de610943775f92038" + S: * URLFETCH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 + ;urlauth=submit+fred:internal + :91354a473744909de610943775f92038" {28} + S: Si vis pacem, para bellum. + S: + S: a002 OK URLFETCH completed + + + + + + + + +Crispin Standards Track [Page 11] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +8. Additional Responses + + These responses are extensions to the [IMAP] base protocol. + + The section headings of these responses are intended to correspond + with where they would be located in the base protocol document if + they were part of that document. + +BASE.7.1.URLMECH. URLMECH Status Response Code + + The URLMECH status response code is followed by a list of URL + authorization mechanism names. Mechanism names other than INTERNAL + may be appended with an "=" and BASE64-encoded form of mechanism- + specific data. + + This status response code is returned in an untagged OK response in + response to a RESETKEY, SELECT, or EXAMINE command. In the case of + the RESETKEY command, this status response code can be sent in the + tagged OK response instead of requiring a separate untagged OK + response. + + Example: + + C: a33 RESETKEY INBOX XSAMPLE + S: a33 OK [URLMECH INTERNAL XSAMPLE=P34OKhO7VEkCbsiYY8rGEg==] done + + In this example, the server supports the INTERNAL mechanism and an + experimental mechanism called XSAMPLE, which also holds some + mechanism-specific data (the name "XSAMPLE" is for illustrative + purposes only). + +BASE.7.4.GENURLAUTH. GENURLAUTH Response + + Contents: One or more URLs + + The GENURLAUTH response returns the URLAUTH-authorized URL(s) + requested by a GENURLAUTH command. + + Example: + + C: a777 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2;urlauth=submit+fred" INTERNAL + S: * GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 + ;urlauth=submit+fred:internal:91354a473744909de610943775f92038" + S: a777 OK GENURLAUTH completed + + + + + + +Crispin Standards Track [Page 12] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +BASE.7.4.URLFETCH. URLFETCH Response + + Contents: One or more URL/nstring pairs + + The URLFETCH response returns the message text data associated with + one or more IMAP URLs, as described in [IMAPURL] and extended by this + document. This response occurs as the result of a URLFETCH command. + + The returned data string is NIL if the URL is invalid for any reason + (including validation failure). If the URL is valid, but the IMAP + fetch of the body part returned NIL (this should not happen), the + returned data string should be the empty string ("") and not NIL. + + Note: This command does not require that the URL refer to the + selected mailbox; nor does it require that any mailbox be + selected. It also does not in any way interfere with any selected + mailbox. + + Example: + + C: a002 URLFETCH "imap://joe@example.com/INBOX/;uid=20/ + ;section=1.2;urlauth=submit+fred:internal + :91354a473744909de610943775f92038" + S: * URLFETCH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 + ;urlauth=submit+fred:internal + :91354a473744909de610943775f92038" {28} + S: Si vis pacem, para bellum. + S: + S: a002 OK URLFETCH completed + +9. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + The following modifications are made to the Formal Syntax in [IMAP]: + +resetkey = "RESETKEY" [SP mailbox *(SP mechanism)] + +capability =/ "URLAUTH" + +command-auth =/ resetkey / genurlauth / urlfetch + +resp-text-code =/ "URLMECH" SP "INTERNAL" *(SP mechanism ["=" base64]) + +genurlauth = "GENURLAUTH" 1*(SP url-rump SP mechanism) + +genurlauth-data = "*" SP "GENURLAUTH" 1*(SP url-full) + + + +Crispin Standards Track [Page 13] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +url-full = astring + ; contains authimapurlfull as defined below + +url-rump = astring + ; contains authimapurlrump as defined below + +urlfetch = "URLFETCH" 1*(SP url-full) + +urlfetch-data = "*" SP "URLFETCH" 1*(SP url-full SP nstring) + + The following extensions are made to the Formal Syntax in [IMAPURL]: + +authimapurl = "imap://" enc-user [iauth] "@" hostport "/" + imessagepart + ; replaces "imapurl" and "iserver" rules for + ; URLAUTH authorized URLs + +authimapurlfull = authimapurl iurlauth + +authimapurlrump = authimapurl iurlauth-rump + +enc-urlauth = 32*HEXDIG + +enc-user = 1*achar + ; same as "enc_user" in RFC 2192 + +iurlauth = iurlauth-rump ":" mechanism ":" enc-urlauth + +iurlauth-rump = [expire] ";URLAUTH=" access + +access = ("submit+" enc-user) / ("user+" enc-user) / + "authuser" / "anonymous" + +expire = ";EXPIRE=" date-time + ; date-time defined in [DATETIME] + +mechanism = "INTERNAL" / 1*(ALPHA / DIGIT / "-" / ".") + ; case-insensitive + ; new mechanisms MUST be registered with IANA + + + + + + + + + + + + +Crispin Standards Track [Page 14] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +10. Security Considerations + + Security considerations are discussed throughout this memo. + + The mailbox access key SHOULD have at least 128 bits of entropy + (refer to [RANDOM] for more details) and MUST be unpredictable. + + The server implementation of the INTERNAL mechanism SHOULD consider + the possibility of needing to change the token generation algorithm, + and SHOULD incorporate some means of identifying the token generation + algorithm within the token. + + The URLMECH status response code may expose sensitive data in the + mechanism-specific data for mechanisms other than INTERNAL. A server + implementation MUST implement a configuration that will not return a + URLMECH status response code unless some mechanism is provided that + protects the session from snooping, such as a TLS or SASL security + layer that provides confidentiality protection. + + The calculation of an authorization token with a "plausible" key if + the mailbox can not be identified is necessary to avoid attacks in + which the server is probed to see if a particular mailbox exists on + the server by measuring the amount of time taken to reject a known + bad name versus some other name. + + To protect against a computational denial-of-service attack, a server + MAY impose progressively longer delays on multiple URL requests that + fail validation. + + The decision to use the "authuser" access identifier should be made + with caution. An "authuser" access identifier can be used by any + authorized user of the IMAP server; therefore, use of this access + identifier should be limited to content that may be disclosed to any + authorized user of the IMAP server. + + The decision to use the "anonymous" access identifier should be made + with extreme caution. An "anonymous" access identifier can be used + by anyone; therefore, use of this access identifier should be limited + to content that may be disclosed to anyone. Many IMAP servers do not + permit anonymous access; in this case, the "anonymous" access + identifier is equivalent to "authuser", but this MUST NOT be relied + upon. + + Although this specification does not prohibit the theoretical + capability to generate a URL with a server component other than the + logged in userid and server, this capability should only be provided + + + + + +Crispin Standards Track [Page 15] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + + when the logged in userid/server has been authorized as equivalent to + the server component userid/server, or otherwise has access to that + userid/server mailbox access key table. + +11. IANA Considerations + + This document constitutes registration of the URLAUTH capability in + the imap4-capabilities registry. + + URLAUTH authorization mechanisms are registered by publishing a + standards track or IESG-approved experimental RFC. The registry is + currently located at: + +http://www.iana.org/assignments/urlauth-authorization-mechanism-registry + + This registry is case-insensitive. + + This document constitutes registration of the INTERNAL URLAUTH + authorization mechanism. + + IMAP URLAUTH Authorization Mechanism Registry + + Mechanism Name Reference + -------------- --------- + INTERNAL [RFC4467] + + + + + + + + + + + + + + + + + + + + + + + + + + +Crispin Standards Track [Page 16] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +12. Normative References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + [BURL] Newman, C., "Message Submission BURL Extension", RFC 4468, + May 2006. + + [DATETIME] Klyne, G. and C. Newman, "Date and Time on the Internet: + Timestamps", RFC 3339, July 2002. + + [IMAP] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [IMAPURL] Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + +13. Informative References + + [HMAC] Krawczyk, H., Bellare, M., and R. Canetti, "HMAC: Keyed- + Hashing for Message Authentication", RFC 2104, February + 1997. + + [RANDOM] Eastlake, D., 3rd, Schiller, J., and S. Crocker, + "Randomness Requirements for Security", BCP 106, RFC 4086, + June 2005. + +Author's Address + + Mark R. Crispin + Networks and Distributed Computing + University of Washington + 4545 15th Avenue NE + Seattle, WA 98105-4527 + + Phone: (206) 543-5762 + EMail: MRC@CAC.Washington.EDU + + + + + + + + + + + + +Crispin Standards Track [Page 17] + +RFC 4467 IMAP - URLAUTH Extension May 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Crispin Standards Track [Page 18] + diff --git a/docs/rfcs/rfc4469.IMAP_CATENATE_extension.txt b/docs/rfcs/rfc4469.IMAP_CATENATE_extension.txt new file mode 100644 index 0000000..da36551 --- /dev/null +++ b/docs/rfcs/rfc4469.IMAP_CATENATE_extension.txt @@ -0,0 +1,731 @@ + + + + + + +Network Working Group P. Resnick +Request for Comments: 4469 QUALCOMM Incorporated +Updates: 3501, 3502 April 2006 +Category: Standards Track + + + Internet Message Access Protocol (IMAP) CATENATE Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + The CATENATE extension to the Internet Message Access Protocol (IMAP) + extends the APPEND command to allow clients to create messages on the + IMAP server that may contain a combination of new data along with + parts of (or entire) messages already on the server. Using this + extension, the client can catenate parts of an already existing + message onto a new message without having to first download the data + and then upload it back to the server. + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 1] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +1. Introduction + + The CATENATE extension to the Internet Message Access Protocol (IMAP) + [1] allows the client to create a message on the server that can + include the text of messages (or parts of messages) that already + exist on the server without having to FETCH them and APPEND them back + to the server. The CATENATE extension extends the APPEND command so + that, instead of a single message literal, the command can take as + arguments any combination of message literals (as described in IMAP + [1]) and message URLs (as described in the IMAP URL Scheme [2] + specification). The server takes all the pieces and catenates them + into the output message. The CATENATE extension can also coexist + with the MULTIAPPEND extension [3] to APPEND multiple messages in a + single command. + + There are some obvious uses for the CATENATE extension. The + motivating use case was to provide a way for a resource-constrained + client to compose a message for subsequent submission that contains + data that already exists in that client's IMAP store. Because the + client does not have to download and re-upload potentially large + message parts, bandwidth and processing limitations do not have as + much impact. In addition, since the client can create a message in + its own IMAP store, the command also addresses the desire of the + client to archive a copy of a sent message without having to upload + the message twice. (Mechanisms for sending the message are outside + the scope of this document.) + + The extended APPEND command can also be used to copy parts of a + message to another mailbox for archival purposes while getting rid of + undesired parts. In environments where server storage is limited, a + client could get rid of large message parts by copying over only the + necessary parts and then deleting the original message. The + mechanism could also be used to add data to a message (such as + prepending message header fields) or to include other data by making + a copy of the original and catenating the new data. + +2. The CATENATE Capability + + A server that supports this extension returns "CATENATE" as one of + the responses to the CAPABILITY command. + + + + + + + + + + + +Resnick Standards Track [Page 2] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +3. The APPEND Command + + Arguments: mailbox name + (The following can be repeated in the presence of the + MULTIAPPEND extension [3]) + OPTIONAL flag parenthesized list + OPTIONAL date/time string + a single message literal or one or more message parts to + catenate, specified as: + message literal + or + message (or message part) URL + + Responses: OPTIONAL NO responses: BADURL, TOOBIG + + Result: OK - append completed + NO - append error: can't append to that mailbox, error + in flags or date/time or message text, or can't + fetch that data + BAD - command unknown or arguments invalid + + The APPEND command concatenates all the message parts and appends + them as a new message to the end of the specified mailbox. The + parenthesized flag list and date/time string set the flags and the + internal date, respectively, as described in IMAP [1]. The + subsequent command parameters specify the message parts that are + appended sequentially to the output message. + + If the original form of APPEND is used, a message literal follows the + optional flag list and date/time string, which is appended as + described in IMAP [1]. If the extended form is used, "CATENATE" and + a parenthesized list of message literals and message URLs follows, + each of which is appended to the new message. If a message literal + is specified (indicated by "TEXT"), the octets following the count + are appended. If a message URL is specified (indicated by "URL"), + the octets of the body part pointed to by that URL are appended, as + if the literal returned in a FETCH BODY response were put in place of + the message part specifier. The APPEND command does not cause the + \Seen flag to be set for any catenated body part. The APPEND command + does not change the selected mailbox. + + In the extended APPEND command, the string following "URL" is an IMAP + URL [2] and is interpreted according to the rules of [2]. The + present document only describes the behavior of the command using + IMAP URLs that refer to specific messages or message parts on the + current IMAP server from the current authenticated IMAP session. + Because of that, only relative IMAP message or message part URLs + (i.e., those having no scheme or ) are used. The base URL + + + +Resnick Standards Track [Page 3] + +RFC 4469 IMAP CATENATE Extension April 2006 + + + for evaluating the relative URL is considered "imap://user@server/", + where "user" is the user name of the currently authenticated user and + "server" is the domain name of the current server. When in the + selected state, the base URL is considered + "imap://user@server/mailbox", where "mailbox" is the encoded name of + the currently selected mailbox. Additionally, since the APPEND + command is valid in the authenticated state of an IMAP session, no + further LOGIN or AUTHENTICATE command is performed for URLs specified + in the extended APPEND command. + + Note: Use of an absolute IMAP URL or any URL that refers to + anything other than a message or message part from the current + authenticated IMAP session is outside the scope of this document + and would require an extension to this specification, and a server + implementing only this specification would return NO to such a + request. + + The client is responsible for making sure that the catenated message + is in the format of an Internet Message Format (RFC 2822) [4] or + Multipurpose Internet Mail Extension (MIME) [5] message. In + particular, when a URL is catenated, the server copies octets, + unchanged, from the indicated message or message part to the + catenated message. It does no data conversion (e.g., MIME transfer + encodings) nor any verification that the data is appropriate for the + MIME part of the message into which it is inserted. The client is + also responsible for inserting appropriate MIME boundaries between + body parts, and writing MIME Content-Type and Content-Transfer- + Encoding lines as needed in the appropriate places. + + Responses behave just as the original APPEND command described in + IMAP [1]. If the server implements the IMAP UIDPLUS extension [6], + it will also return an APPENDUID response code in the tagged OK + response. Two response codes are provided in Section 4 that can be + used in the tagged NO response if the APPEND command fails. + +4. Response Codes + + When a APPEND command fails, it may return a response code that + describes a reason for the failure. + +4.1. BADURL Response + + The BADURL response code is returned if the APPEND fails to process + one of the specified URLs. Possible reasons for this are bad URL + syntax, unrecognized URL schema, invalid message UID, or invalid body + part. The BADURL response code contains the first URL specified as a + parameter to the APPEND command that has caused the operation to + fail. + + + +Resnick Standards Track [Page 4] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +4.2. TOOBIG Response + + The TOOBIG response code is returned if the resulting message will + exceed the 4-GB IMAP message limit. This might happen, for example, + if the client specifies 3 URLs for 2-GB messages. Note that even if + the server doesn't return TOOBIG, it still has to be defensive + against misbehaving or malicious clients that try to construct a + message over the 4-GB limit. The server may also wish to return the + TOOBIG response code if the resulting message exceeds a server- + specific message size limit. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) [7] notation. Elements not defined here can be found in + the formal syntax of the ABNF [7], IMAP [1], and IMAP ABNF extensions + [8] specifications. Note that capability and resp-text-code are + extended from the IMAP [1] specification and append-data is extended + from the IMAP ABNF extensions [8] specification. + + append-data =/ "CATENATE" SP "(" cat-part *(SP cat-part) ")" + + cat-part = text-literal / url + + text-literal = "TEXT" SP literal + + url = "URL" SP astring + + resp-text-code =/ toobig-response-code / badurl-response-code + + toobig-response-code = "TOOBIG" + + badurl-response-code = "BADURL" SP url-resp-text + + url-resp-text = 1*(%x01-09 / + %x0B-0C / + %x0E-5B / + %x5D-FE) ; Any TEXT-CHAR except "]" + + capability =/ "CATENATE" + + The astring in the definition of url and the url-resp-text in the + definition of badurl-response-code each contain an imapurl as defined + by [2]. + + + + + + + +Resnick Standards Track [Page 5] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +6. Acknowledgements + + Thanks to the members of the LEMONADE working group for their input. + Special thanks to Alexey Melnikov for the examples. + +7. Security Considerations + + The CATENATE extension does not raise any security considerations + that are not present for the base protocol or in the use of IMAP + URLs, and these issues are discussed in the IMAP [1] and IMAP URL [2] + documents. + +8. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located at + . This document + defines the CATENATE IMAP capability. The IANA has added this + capability to the registry. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 6] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +Appendix A. Examples + + Lines not starting with "C: " or "S: " are continuations of the + previous lines. + + The original message in examples 1 and 2 below (UID = 20) has the + following structure: + + + multipart/mixed MIME message with two body parts: + + 1. text/plain + + 2. application/x-zip-compressed + + Example 1: The following example demonstrates how a CATENATE client + can replace an attachment in a draft message, without the need to + download it to the client and upload it back. + + C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE + (URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=HEADER" + TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050907 + C: URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=1.MIME" + URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=1" TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050907 + C: URL "/Drafts;UIDVALIDITY=385759045/;UID=30" TEXT {44} + S: + Ready for literal data + C: + C: --------------030308070208000400050907-- + C: ) + S: A003 OK catenate append completed + + + + + + + + + + + + + + + +Resnick Standards Track [Page 7] + +RFC 4469 IMAP CATENATE Extension April 2006 + + + Example 2: The following example demonstrates how the CATENATE + extension can be used to replace edited text in a draft message, as + well as header fields for the top level message part (e.g., Subject + has changed). The previous version of the draft is marked as + \Deleted. Note that the server also supports the UIDPLUS extension, + so the APPENDUID response code is returned in the successful OK + response to the APPEND command. + + C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE (TEXT {738} + S: + Ready for literal data + C: Return-Path: + C: Received: from [127.0.0.2] + C: by rufus.example.org via TCP (internal) with ESMTPA; + C: Thu, 11 Nov 2004 16:57:07 +0000 + C: Message-ID: <419399E1.6000505@example.org> + C: Date: Thu, 12 Nov 2004 16:57:05 +0000 + C: From: Bob Ar + C: X-Accept-Language: en-us, en + C: MIME-Version: 1.0 + C: To: foo@example.net + C: Subject: About our holiday trip + C: Content-Type: multipart/mixed; + C: boundary="------------030308070208000400050907" + C: + C: --------------030308070208000400050907 + C: Content-Type: text/plain; charset=us-ascii; format=flowed + C: Content-Transfer-Encoding: 7bit + C: + C: Our travel agent has sent the updated schedule. + C: + C: Cheers, + C: Bob + C: --------------030308070208000400050907 + C: URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;Section=2.MIME" + URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;Section=2" TEXT {44} + S: + Ready for literal data + C: + C: --------------030308070208000400050907-- + C: ) + S: A003 OK [APPENDUID 385759045 45] append Completed + C: A004 UID STORE 20 +FLAGS.SILENT (\Deleted) + S: A004 OK STORE completed + + + + + + + + + +Resnick Standards Track [Page 8] + +RFC 4469 IMAP CATENATE Extension April 2006 + + + Example 3: The following example demonstrates how the CATENATE + extension can be used to strip attachments. Below, a PowerPoint + attachment was replaced by a small text part explaining that the + attachment was stripped. + + C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE + (URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=HEADER" + TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050903 + C: URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=1.MIME" + URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=1" TEXT {255} + S: + Ready for literal data + C: + C: --------------030308070208000400050903 + C: Content-type: text/plain; charset="us-ascii" + C: Content-transfer-encoding: 7bit + C: + C: This body part contained a Power Point presentation that was + C: deleted upon your request. + C: --------------030308070208000400050903-- + C: ) + S: A003 OK append Completed + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 9] + +RFC 4469 IMAP CATENATE Extension April 2006 + + + Example 4: The following example demonstrates a failed APPEND + command. The server returns the BADURL response code to indicate + that one of the provided URLs is invalid. This example also + demonstrates how the CATENATE extension can be used to construct a + digest of several messages. + + C: A003 APPEND Sent (\Seen $MDNSent) CATENATE (TEXT {541} + S: + Ready for literal data + C: Return-Path: + C: Received: from [127.0.0.2] + C: by rufus.example.org via TCP (internal) with ESMTPA; + C: Thu, 11 Nov 2004 16:57:07 +0000 + C: Message-ID: <419399E1.6000505@example.org> + C: Date: Thu, 21 Nov 2004 16:57:05 +0000 + C: From: Farren Oo + C: X-Accept-Language: en-us, en + C: MIME-Version: 1.0 + C: To: bar@example.org + C: Subject: Digest of the mailing list for today + C: Content-Type: multipart/digest; + C: boundary="------------030308070208000400050904" + C: + C: --------------030308070208000400050904 + C: URL "/INBOX;UIDVALIDITY=785799047/;UID=11467" TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050904 + C: URL "/INBOX;UIDVALIDITY=785799047/;UID=113330/;section=1.5.9" + TEXT {42} + S: + Ready for literal data + C: + C: --------------030308070208000400050904 + C: URL "/INBOX;UIDVALIDITY=785799047/;UID=11916" TEXT {44} + S: + Ready for literal data + C: + C: --------------030308070208000400050904-- + C: ) + S: A003 NO [BADURL "/INBOX;UIDVALIDITY=785799047/;UID=113330; + section=1.5.9"] CATENATE append has failed, one message expunged + + Note that the server could have validated the URLs as they were + received and therefore could have returned the tagged NO response + with BADURL response-code in place of any continuation request after + the URL was received. + + + + + + + +Resnick Standards Track [Page 10] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +9. Normative References + + [1] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", + RFC 3501, March 2003. + + [2] Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. + + [3] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + + [4] Resnick, P., "Internet Message Format", RFC 2822, April 2001. + + [5] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message Bodies", + RFC 2045, November 1996. + + [6] Crispin, M., "Internet Message Access Protocol (IMAP) - UIDPLUS + extension", RFC 4315, December 2005. + + [7] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + [8] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", + RFC 4466, April 2006. + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 11] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +Author's Address + + Peter W. Resnick + QUALCOMM Incorporated + 5775 Morehouse Drive + San Diego, CA 92121-1714 + US + + Phone: +1 858 651 4478 + EMail: presnick@qualcomm.com + URI: http://www.qualcomm.com/~presnick/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick Standards Track [Page 12] + +RFC 4469 IMAP CATENATE Extension April 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Resnick Standards Track [Page 13] + diff --git a/docs/rfcs/rfc4549.Sync_operations_for_disconnected_IMAP4_Clients.txt b/docs/rfcs/rfc4549.Sync_operations_for_disconnected_IMAP4_Clients.txt new file mode 100644 index 0000000..8430ee1 --- /dev/null +++ b/docs/rfcs/rfc4549.Sync_operations_for_disconnected_IMAP4_Clients.txt @@ -0,0 +1,1963 @@ + + + + + + +Network Working Group A. Melnikov, Ed. +Request for Comments: 4549 Isode Ltd. +Category: Informational June 2006 + + + Synchronization Operations for Disconnected IMAP4 Clients + +Status of This Memo + + This memo provides information for the Internet community. It does + not specify an Internet standard of any kind. Distribution of this + memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + This document attempts to address some of the issues involved in + building a disconnected IMAP4 client. In particular, it deals with + the issues of what might be called the "driver" portion of the + synchronization tool: the portion of the code responsible for issuing + the correct set of IMAP4 commands to synchronize the disconnected + client in the way that is most likely to make the human who uses the + disconnected client happy. + + This note describes different strategies that can be used by + disconnected clients and shows how to use IMAP protocol in order to + minimize the time of the synchronization process. + + This note also lists IMAP extensions that a server should implement + in order to provide better synchronization facilities to disconnected + clients. + + + + + + + + + + + + + + + + + +Melnikov Informational [Page 1] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +Table of Contents + + 1. Introduction ....................................................3 + 1.1. Conventions Used in This Document ..........................3 + 2. Design Principles ...............................................3 + 3. Overall Picture of Synchronization ..............................4 + 4. Mailbox Synchronization Steps and Strategies ....................7 + 4.1. Checking UID Validity ......................................7 + 4.2. Synchronizing Local Changes with the Server ................8 + 4.2.1. Uploading Messages to the Mailbox ...................8 + 4.2.2. Optimizing "move" and "copy" Operations .............9 + 4.2.3. Replaying Local Flag Changes .......................14 + 4.2.4. Processing Mailbox Compression (EXPUNGE) Requests ..15 + 4.2.5. Closing a Mailbox ..................................17 + 4.3. Details of "Normal" Synchronization of a Single Mailbox ...18 + 4.3.1. Discovering New Messages and Changes to Old + Messages ...........................................18 + 4.3.2. Searching for "Interesting" Messages. ..............20 + 4.3.3. Populating Cache with "Interesting" Messages. ......21 + 4.3.4. User-Initiated Synchronization .....................22 + 4.4. Special Case: Descriptor-Only Synchronization .............22 + 4.5. Special Case: Fast New-Only Synchronization ...............23 + 4.6. Special Case: Blind FETCH .................................23 + 5. Implementation Considerations ..................................24 + 5.1. Error Recovery during Playback ............................26 + 5.2. Quality of Implementation Issues ..........................28 + 5.3. Optimizations .............................................28 + 6. IMAP Extensions That May Help ..................................30 + 6.1. CONDSTORE Extension .......................................30 + 7. Security Considerations ........................................33 + 8. References .....................................................33 + 8.1. Normative References ......................................33 + 8.2. Informative References ....................................34 + 9. Acknowledgements ...............................................34 + + + + + + + + + + + + + + + + + +Melnikov Informational [Page 2] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +1. Introduction + + Several recommendations presented in this document are generally + applicable to all types of IMAP clients. However, this document + tries to concentrate on disconnected mail clients [IMAP-MODEL]. It + also suggests some IMAP extensions* that should be implemented by + IMAP servers in order to make the life of disconnected clients + easier. In particular, the [UIDPLUS] extension was specifically + designed to streamline certain disconnected operations, like + expunging, uploading, and copying messages (see Sections 4.2.1, + 4.2.2.1, and 4.2.4). + + Readers of this document are also strongly advised to read RFC 2683 + [RFC2683]. + + * Note that the functionality provided by the base IMAP protocol + [IMAP4] is sufficient to perform basic synchronization. + +1.1. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. Long lines in examples are broken for + editorial clarity. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + + Let's call an IMAP command idempotent if the result of executing the + command twice sequentially is the same as the result of executing the + command just once. + +2. Design Principles + + All mailbox state or content information stored on the disconnected + client should be viewed strictly as a cache of the state of the + server. The "master" state remains on the server, just as it would + with an interactive IMAP4 client. The one exception to this rule is + that information about the state of the disconnected client's cache + (the state includes flag changes while offline and during scheduled + message uploads) remains on the disconnected client: that is, the + IMAP4 server is not responsible for remembering the state of the + disconnected IMAP4 client. + + We assume that a disconnected client is a client that, for whatever + reason, wants to minimize the length of time that it is "on the + phone" to the IMAP4 server. Often this will be because the client is + using a dialup connection, possibly with very low bandwidth, but + + + +Melnikov Informational [Page 3] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + sometimes it might just be that the human is in a hurry to catch an + airplane, or some other event beyond our control. Whatever the + reason, we assume that we must make efficient use of the network + connection, both in the usual sense (not generating spurious traffic) + and in the sense that we would prefer not to have the connection + sitting idle while the client and/or the server is performing + strictly local computation or I/O. Another, perhaps simpler way of + stating this is that we assume that network connections are + "expensive". + + Practical experience with disconnected mail systems has shown that + there is no single synchronization strategy that is appropriate for + all cases. Different humans have different preferences, and the same + human's preference will vary depending both on external circumstance + (how much of a hurry the human is in today) and on the value that the + human places on the messages being transferred. The point here is + that there is no way that the synchronization program can guess + exactly what the human wants to do, so the human will have to provide + some guidance. + + Taken together, the preceding two principles lead to the conclusion + that the synchronization program must make its decisions based on + some kind of guidance provided by the human, by selecting the + appropriate options in the user interface or through some sort of + configuration file. Almost certainly, it should not pause for I/O + with the human in the middle of the synchronization process. The + human will almost certainly have several different configurations for + the synchronization program, for different circumstances. + + Since a disconnected client has no way of knowing what changes might + have occurred to the mailbox while it was disconnected, message + numbers are not useful to a disconnected client. All disconnected + client operations should be performed using UIDs, so that the client + can be sure that it and the server are talking about the same + messages during the synchronization process. + +3. Overall Picture of Synchronization + + The basic strategy for synchronization is outlined below. Note that + the real strategy may vary from one application to another or may + depend on a synchronization mode. + + a) Process any "actions" that were pending on the client that were + not associated with any mailbox. (In particular sending messages + composed offline with SMTP. This is not part of IMAP + synchronization, but it is mentioned here for completeness.) + + + + + +Melnikov Informational [Page 4] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + b) Fetch the current list of "interesting" mailboxes. (The + disconnected client should allow the user to skip this step + completely.) + + c) "Client-to-server synchronization": for each IMAP "action" that + was pending on the client, do the following: + + 1) If the action implies opening a new mailbox (any operation that + operates on messages), open the mailbox. Check its UID + validity value (see Section 4.1 for more details) returned in + the UIDVALIDITY response code. If the UIDVALIDITY value + returned by the server differs, the client MUST empty the local + cache of the mailbox and remove any pending "actions" that + refer to UIDs in that mailbox (and consider them failed). Note + that this doesn't affect actions performed on client-generated + fake UIDs (see Section 5). + + 2) Perform the action. If the action is to delete a mailbox + (DELETE), make sure that the mailbox is closed first (see also + Section 3.4.12 of [RFC2683]). + + d) "Server-to-client synchronization": for each mailbox that requires + synchronization, do the following: + + 1) Check the mailbox UIDVALIDITY (see Section 4.1 for more + details) with SELECT/EXAMINE/STATUS. + + If UIDVALIDITY value returned by the server differs, the client + MUST + + * empty the local cache of that mailbox; + * remove any pending "actions" that refer to UIDs in that + mailbox and consider them failed; and + * skip step 2-II. + + 2) Fetch the current "descriptors"; + + I) Discover new messages. + + II) Discover changes to old messages. + + 3) Fetch the bodies of any "interesting" messages that the client + doesn't already have. + + e) Close all open mailboxes not required for further operations (if + staying online) or disconnect all open connections (if going + offline). + + + + +Melnikov Informational [Page 5] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Terms used: + + "Actions" are queued requests that were made by the human to the + client's Mail User Agent (MUA) software while the client was + disconnected. + + We define "descriptors" as a set of IMAP4 FETCH data items. + Conceptually, a message's descriptor is that set of information that + allows the synchronization program to decide what protocol actions + are necessary to bring the local cache to the desired state for this + message; since this decision is really up to the human, this + information probably includes at least a few header fields intended + for human consumption. Exactly what will constitute a descriptor + depends on the client implementation. At a minimum, the descriptor + contains the message's UID and FLAGS. Other likely candidates are + the RFC822.SIZE, RFC822.HEADER, BODYSTRUCTURE, or ENVELOPE data + items. + + Comments: + + 1) The list of actions should be ordered. For example, if the human + deletes message A1 in mailbox A, then expunges mailbox A, and then + deletes message A2 in mailbox A, the human will expect that + message A1 is gone and that message A2 is still present but is now + deleted. + + By processing all the actions before proceeding with + synchronization, we avoid having to compensate for the local MUA's + changes to the server's state. That is, once we have processed + all the pending actions, the steps that the client must take to + synchronize itself will be the same no matter where the changes to + the server's state originated. + + 2) Steps a and b can be performed in parallel. Alternatively, step a + can be performed after d. + + 3) On step b, the set of "interesting" mailboxes pretty much has to + be determined by the human. What mailboxes belong to this set may + vary between different IMAP4 sessions with the same server, + client, and human. An interesting mailbox can be a mailbox + returned by LSUB command (see Section 6.3.9 of [IMAP4]). The + special mailbox "INBOX" SHOULD be in the default set of mailboxes + that the client considers interesting. However, providing the + ability to ignore INBOX for a particular session or client may be + valuable for some mail filtering strategies. + + + + + + +Melnikov Informational [Page 6] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 4) On step d-2-II, the client also finds out about changes to the + flags of messages that the client already has in its local cache, + and about messages in the local cache that no longer exist on the + server (i.e., messages that have been expunged). + + 5) "Interesting" messages are those messages that the synchronization + program thinks the human wants to have cached locally, based on + the configuration and the data retrieved in step b. + + 6) A disconnected IMAP client is a special case of an IMAP client, so + it MUST be able to handle any "unexpected" unsolicited responses, + like EXISTS and EXPUNGE, at any time. The disconnected client MAY + ignore EXPUNGE response during "client-to-server" synchronization + phase (step c). + + The rest of this discussion will focus primarily on the + synchronization issues for a single mailbox. + +4. Mailbox Synchronization Steps and Strategies + +4.1. Checking UID Validity + + The "UID validity" of a mailbox is a number returned in an + UIDVALIDITY response code in an OK untagged response at mailbox + selection time. The UID validity value changes between sessions when + UIDs fail to persist between sessions. + + Whenever the client selects a mailbox, the client must compare the + returned UID validity value with the value stored in the local cache. + If the UID validity values differ, the UIDs in the client's cache are + no longer valid. The client MUST then empty the local cache of that + mailbox and remove any pending "actions" that refer to UIDs in that + mailbox. The client MAY also issue a warning to the human. The + client MUST NOT cancel any scheduled uploads (i.e., APPENDs) for the + mailbox. + + Note that UIDVALIDITY is not only returned on a mailbox selection. + The COPYUID and APPENDUID response codes defined in the [UIDPLUS] + extension (see also 4.2.2) and the UIDVALIDITY STATUS response data + item also contain a UIDVALIDITY value for some other mailbox. The + client SHOULD behave as described in the previous paragraph (but it + should act on the other mailbox's cache), no matter how it obtained + the UIDVALIDITY value. + + + + + + + + +Melnikov Informational [Page 7] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +4.2. Synchronizing Local Changes with the Server + +4.2.1. Uploading Messages to the Mailbox + + Two of the most common examples of operations resulting in message + uploads are: + + 1) Saving a draft message + + 2) Copying a message between remote mailboxes on two different IMAP + servers or a local mailbox and a remote mailbox. + + Message upload is performed with the APPEND command. A message + scheduled to be uploaded has no UID associated with it, as all UIDs + are assigned by the server. The APPEND command will effectively + associate a UID with the uploaded message that can be stored in the + local cache for future reference. However, [IMAP4] doesn't describe + a simple mechanism to discover the message UID by just performing the + APPEND command. In order to discover the UID, the client can do one + of the following: + + 1) Remove the uploaded message from cache. Then, use the mechanism + described in 4.3 to fetch the information about the uploaded + message as if it had been uploaded by some other client. + + 2) Try to fetch header information as described in 4.2.2 in order to + find a message that corresponds to the uploaded message. One + strategy for doing this is described in 4.2.2. + + Case 1 describes a not particularly smart client. + + C: A003 APPEND Drafts (\Seen $MDNSent) {310} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK APPEND Completed + + Fortunately, there is a simpler way to discover the message UID in + the presence of the [UIDPLUS] extension: + + + + +Melnikov Informational [Page 8] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + C: A003 APPEND Drafts (\Seen $MDNSent) {310} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 1022843275 77712] APPEND completed + + The UID of the appended message is the second parameter of APPENDUID + response code. + +4.2.2. Optimizing "move" and "copy" Operations + + Practical experience with IMAP and other mailbox access protocols + that support multiple mailboxes suggests that moving a message from + one mailbox to another is an extremely common operation. + +4.2.2.1. Moving a Message between Two Mailboxes on the Same Server + + In IMAP4, a "move" operation between two mailboxes on the same server + is really a combination of a COPY operation and a STORE +FLAGS + (\Deleted) operation. This makes good protocol sense for IMAP, but + it leaves a simple-minded disconnected client in the silly position + of deleting and possibly expunging its cached copy of a message, then + fetching an identical copy via the network. + + However, the presence of the UIDPLUS extension in the server can + help: + + C: A001 UID COPY 567,414 "Interesting Messages" + S: A001 OK [COPYUID 1022843275 414,567 5:6] Completed + + This tells the client that the message with UID 414 in the current + mailbox was successfully copied to the mailbox "Interesting Messages" + and was given the UID 5, and that the message with UID 567 was given + the UID 6. + + In the absence of UIDPLUS extension support in the server, the + following trick can be used. By including the Message-ID: header and + the INTERNALDATE data item as part of the descriptor, the client can + check the descriptor of a "new" message against messages that are + already in its cache and avoid fetching the extra copy. Of course, + + + +Melnikov Informational [Page 9] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + it's possible that the cost of checking to see if the message is + already in the local cache may exceed the cost of just fetching it, + so this technique should not be used blindly. If the MUA implements + a "move" command, it makes special provisions to use this technique + when it knows that a copy/delete sequence is the result of a "move" + command. + + Note that servers are not required (although they are strongly + encouraged with "SHOULD language") to preserve INTERNALDATE when + copying messages. + + Also note that since it's theoretically possible for this algorithm + to find the wrong message (given sufficiently malignant Message-ID + headers), implementers should provide a way to disable this + optimization, both permanently and on a message-by-message basis. + + Example 1: Copying a message in the absence of UIDPLUS extension. + + At some point in time the client has fetched the source message and + some information was cached: + + C: C021 UID FETCH (BODY.PEEK[] INTERNALDATE FLAGS) + ... + S: * 27 FETCH (UID 123 INTERNALDATE "31-May-2002 05:26:59 -0600" + FLAGS (\Draft $MDNSent) BODY[] {1036} + S: ... + S: Message-Id: <20040903110856.22a127cd@chardonnay> + S: ... + S: ...message body... + S: ) + ... + S: C021 OK fetch completed + + Later on, the client decides to copy the message: + + C: C035 UID COPY 123 "Interesting Messages" + S: C035 OK Completed + + As the server hasn't provided the COPYUID response code, the client + tries the optimization described above: + + C: C036 SELECT "Interesting Messages" + ... + C: C037 UID SEARCH ON 31-May-2002 HEADER + "Message-Id" "20040903110856.22a127cd@chardonnay" + S: SEARCH 12368 + S: C037 OK completed + + + + +Melnikov Informational [Page 10] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Note that if the server has returned multiple UIDs in the SEARCH + response, the client MUST NOT use any of the returned UID. + +4.2.2.2. Moving a Message from a Remote Mailbox to a Local + + Moving a message from a remote mailbox to a local is done with FETCH + (that includes FLAGS and INTERNALDATE) followed by UID STORE + +FLAGS.SILENT (\Deleted): + + C: A003 UID FETCH 123 (BODY.PEEK[] INTERNALDATE FLAGS) + S: * 27 FETCH (UID 123 INTERNALDATE "31-May-2002 05:26:59 -0600" + FLAGS (\Seen $MDNSent) BODY[] + S: ...message body... + S: ) + S: A003 OK UID FETCH completed + C: A004 UID STORE +FLAGS.SILENT (\Deleted) + S: A004 STORE completed + + Note that there is no reason to fetch the message during + synchronization if it's already in the client's cache. Also, the + client SHOULD preserve delivery date in the local cache. + +4.2.2.3. Moving a Message from a Local Mailbox to a Remote + + Moving a message from a local mailbox to a remote is done with + APPEND: + + C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" + {310} + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 1022843275 77712] completed + + The client SHOULD specify the delivery date from the local cache in + the APPEND. + + If the [LITERAL+] extension is available, the client can save a + round-trip*: + + + + +Melnikov Informational [Page 11] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" + {310+} + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: + S: A003 OK [APPENDUID 1022843275 77712] completed + + * Note that there is a risk that the server will reject the message + due to its size. If this happens, the client will waste bandwidth + transferring the whole message. If the client wouldn't have used + the LITERAL+, this could have been avoided: + + C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2004 05:26:59 -0600" + {16777215} + S: A003 NO Sorry, message is too big + +4.2.2.4. Moving a Message between Two Mailboxes on Different Servers + + Moving a message between two mailbox on two different servers is a + combination of the operations described in 4.2.2.2 followed by the + operations described in 4.2.2.3. + +4.2.2.5. Uploading Multiple Messages to a Remote Mailbox with + MULTIAPPEND + + When there is a need to upload multiple messages to a remote mailbox + (e.g., as per 4.2.2.3), the presence of certain IMAP extensions may + significantly improve performance. One of them is [MULTIAPPEND]. + + For some mail stores, opening a mailbox for appending might be + expensive. [MULTIAPPEND] tells the server to open the mailbox once + (instead of opening and closing it "n" times per "n" messages to be + uploaded) and to keep it open while a group of messages is being + uploaded to the server. + + Also, if the server supports both [MULTIAPPEND] and [LITERAL+] + extensions, the entire upload is accomplished in a single + command/response round-trip. + + + + + + +Melnikov Informational [Page 12] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Note: Client implementers should be aware that [MULTIAPPEND] performs + append of multiple messages atomically. This means, for example, if + there is not enough space to save "n"-th message (or the message has + invalid structure and is rejected by the server) after successful + upload of "n-1" messages, the whole upload operation fails, and no + message will be saved in the mailbox. Although this behavior might + be desirable in certain situations, it might not be what you want. + Otherwise, the client should use the regular APPEND command (Section + 4.2.2.3), possibly utilizing the [LITERAL+] extension. See also + Section 5.1 for discussions about error recovery. + + Note: MULTIAPPEND can be used together with the UIDPLUS extension in + a way similar to what was described in Section 4.2.1. [MULTIAPPEND] + extends the syntax of the APPENDUID response code to allow for + multiple message UIDs in the second parameter. + + Example 2: + + This example demonstrates the use of MULTIAPPEND together with + UIDPLUS (synchronization points where the client waits for + confirmations from the server are marked with "<--->"): + + C: A003 APPEND Jan-2002 (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" + {310} + <---> + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: (\Seen) " 1-Jun-2002 22:43:04 -0800" {286} + <---> + S: + Ready for literal data + C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) + C: From: Joe Mooch + C: Subject: Re: afternoon meeting + C: To: foobar@blt.example.com + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: 3:30 is fine with me. + C: + + + +Melnikov Informational [Page 13] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + S: A003 OK [APPENDUID 1022843275 77712,77713] completed + + The upload takes 3 round-trips. + + Example 3: + + In this example, Example 2 was modified for the case when the server + supports MULTIAPPEND, LITERAL+, and UIDPLUS. The upload takes only 1 + round-trip. + + C: A003 APPEND Jan-2002 (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" + {310+} + C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) + C: From: Fred Foobar + C: Subject: afternoon meeting + C: To: mooch@owatagu.siam.edu + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: Hello Joe, do you think we can meet at 3:30 tomorrow? + C: (\Seen) " 1-Jun-2002 22:43:04 -0800" {286+} + C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) + C: From: Joe Mooch + C: Subject: Re: afternoon meeting + C: To: foobar@blt.example.com + C: Message-Id: + C: MIME-Version: 1.0 + C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII + C: + C: 3:30 is fine with me. + C: + S: A003 OK [APPENDUID 1022843275 77712,77713] completed + +4.2.3. Replaying Local Flag Changes + + The disconnected client uses the STORE command to synchronize local + flag state with the server. The disconnected client SHOULD use + +FLAGS.SILENT or -FLAGS.SILENT in order to set or unset flags + modified by the user while offline. The FLAGS form MUST NOT be used, + as there is a risk that this will overwrite flags on the server that + have been changed by some other client. + + Example 4: + + For the message with UID 15, the disconnected client stores the + following flags \Seen and $Highest. The flags were modified on the + server by some other client: \Seen, \Answered, and $Highest. While + + + +Melnikov Informational [Page 14] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + offline, the user requested that the $Highest flags be removed and + that the \Deleted flag be added. The flag synchronization sequence + for the message should look like: + + C: A001 UID STORE 15 +FLAGS.SILENT (\Deleted) + S: A001 STORE completed + C: A002 UID STORE 15 -FLAGS.SILENT ($Highest) + S: A002 STORE completed + + If the disconnected client is able to store an additional binary + state information (or a piece of information that can take a value + from a predefined set of values) in the local cache of an IMAP + mailbox or in a local mailbox (e.g., message priority), and if the + server supports storing of arbitrary keywords, the client MUST use + keywords to store this state on the server. + + Example 5: + + Imagine a speculative mail client that can mark a message as one of + work-related ($Work), personal ($Personal), or spam ($Spam). In + order to mark a message as personal, the client issues: + + C: A001 UID STORE 15 +FLAGS.SILENT ($Personal) + S: A001 STORE completed + C: A002 UID STORE 15 -FLAGS.SILENT ($Work $Spam) + S: A002 STORE completed + + In order to mark the message as not work, not personal and not spam, + the client issues: + + C: A003 UID STORE 15 -FLAGS.SILENT ($Personal $Work $Spam) + S: A003 STORE completed + +4.2.4. Processing Mailbox Compression (EXPUNGE) Requests + + A naive disconnected client implementation that supports compressing + a mailbox while offline may decide to issue an EXPUNGE command to the + server in order to expunge messages marked \Deleted. The problem + with this command during synchronization is that it permanently + erases all messages with the \Deleted flag set, i.e., even those + messages that were marked as \Deleted on the server while the user + was offline. Doing this might result in an unpleasant surprise for + the user. + + Fortunately the [UIDPLUS] extension can help in this case as well. + The extension introduces UID EXPUNGE command, that, unlike EXPUNGE, + takes a UID set parameter, that lists UIDs of all messages that can + be expunged. When processing this command the server erases only + + + +Melnikov Informational [Page 15] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + messages with \Deleted flag listed in the UID list. Thus, messages + not listed in the UID set will not be expunged even if they have the + \Deleted flag set. + + Example 6: + + While the user was offline, 3 messages with UIDs 7, 27, and 65 were + marked \Deleted when the user requested to compress the open mailbox. + Another client marked a message \Deleted on the server (UID 34). + During synchronization, the disconnected client issues: + + C: A001 UID EXPUNGE 7,27,65 + S: * ... EXPUNGE + S: * ... EXPUNGE + S: * ... EXPUNGE + S: A001 UID EXPUNGE completed + + If another client issues UID SEARCH DELETED command (to find all + messages with the \Deleted flag) before and after the UID EXPUNGE, it + will get: + + Before: + + C: B001 UID SEARCH DELETED + S: * SEARCH 65 34 27 7 + S: B001 UID SEARCH completed + + After: + + C: B002 UID SEARCH DELETED + S: * SEARCH 34 + S: B002 UID SEARCH completed + + In the absence of the [UIDPLUS] extension, the following sequence of + commands can be used as an approximation. Note: It's possible for + another client to mark additional messages as deleted while this + sequence is being performed. In this case, these additional messages + will be expunged as well. + + 1) Find all messages marked \Deleted on the server. + + C: A001 UID SEARCH DELETED + S: * SEARCH 65 34 27 7 + S: A001 UID SEARCH completed + + 2) Find all messages that must not be erased (for the previous + example the list will consist of the message with UID 34). + + + + +Melnikov Informational [Page 16] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 3) Temporarily remove \Deleted flag on all messages found in step 2. + + C: A002 UID STORE 34 -FLAGS.SILENT (\Deleted) + S: A002 UID STORE completed + + 4) Expunge the mailbox. + + C: A003 EXPUNGE + S: * 20 EXPUNGE + S: * 7 EXPUNGE + S: * 1 EXPUNGE + S: A003 EXPUNGE completed + + Here, the message with UID 7 has message number 1, with UID 27 has + message number 7, and with UID 65 has message number 20. + + 5) Restore \Deleted flag on all messages found when performing step + 2. + + C: A004 UID STORE 34 +FLAGS.SILENT (\Deleted) + S: A004 UID STORE completed + +4.2.5. Closing a Mailbox + + When the disconnected client has to close a mailbox, it should not + use the CLOSE command, because CLOSE does a silent EXPUNGE. (Section + 4.2.4 explains why EXPUNGE should not be used by a disconnected + client.) It is safe to use CLOSE only if the mailbox was opened with + EXAMINE. + + If the mailbox was opened with SELECT, the client can use one of the + following commands to implicitly close the mailbox and prevent the + silent expunge: + + 1) UNSELECT - This is a command described in [UNSELECT] that works as + CLOSE, but doesn't cause the silent EXPUNGE. This command is + supported by the server if it reports UNSELECT in its CAPABILITY + list. + + 2) SELECT - SELECT causes implicit CLOSE without + EXPUNGE. + + 3) If the client intends to issue LOGOUT after closing the mailbox, + it may just issue LOGOUT, because LOGOUT causes implicit CLOSE + without EXPUNGE as well. + + 4) SELECT - If the client knows a mailbox that + doesn't exist or can't be selected, it MAY SELECT it. + + + +Melnikov Informational [Page 17] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + If the client opened the mailbox with SELECT and just wants to avoid + implicit EXPUNGE without closing the mailbox, it may also use the + following: + + 5) EXAMINE - Reselect the same mailbox in read-only mode. + +4.3. Details of "Normal" Synchronization of a Single Mailbox + + The most common form of synchronization is where the human trusts the + integrity of the client's copy of the state of a particular mailbox + and simply wants to bring the client's cache up to date so that it + accurately reflects the mailbox's current state on the server. + +4.3.1. Discovering New Messages and Changes to Old Messages + + Let represent the highest UID that the client knows + about in this mailbox. Since UIDs are allocated in strictly + ascending order, this is simply the UID of the last message in the + mailbox that the client knows about. Let represent + 's UID plus one. Let represent a list + consisting of all the FETCH data item items that the implementation + considers part of the descriptor; at a minimum this is just the FLAGS + data item, but it usually also includes BODYSTRUCTURE and + RFC822.SIZE. At this step, SHOULD NOT include RFC822. + + With no further information, the client can issue the following two + commands: + + tag1 UID FETCH :* + tag2 UID FETCH 1: FLAGS + + The first command will request some information about "new" messages + (i.e., messages received by the server since the last + synchronization). It will also allow the client to build a message + number to UID map (only for new messages). The second command allows + the client to + + 1) update cached flags for old messages; + + 2) find out which old messages got expunged; and + + 3) build a mapping between message numbers and UIDs (for old + messages). + + The order here is significant. We want the server to start returning + the list of new message descriptors as fast as it can, so that the + client can start issuing more FETCH commands, so we start out by + asking for the descriptors of all the messages we know the client + + + +Melnikov Informational [Page 18] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + cannot possibly have cached yet. The second command fetches the + information we need to determine what changes may have occurred to + messages that the client already has cached. Note that the former + command should only be issued if the UIDNEXT value cached by the + client differs from the one returned by the server. Once the client + has issued these two commands, there's nothing more the client can do + with this mailbox until the responses to the first command start + arriving. A clever synchronization program might use this time to + fetch its local cache state from disk or to start the process of + synchronizing another mailbox. + + The following is an example of the first FETCH: + + C: A011 UID fetch 131:* (FLAGS BODYSTRUCTURE INTERNALDATE + RFC822.SIZE) + + Note 1: The first FETCH may result in the server's sending a huge + volume of data. A smart disconnected client should use message + ranges (see also Section 3.2.1.2 of [RFC2683]), so that the user is + able to execute a different operation between fetching information + for a group of new messages. + + Example 7: + + Knowing the new UIDNEXT returned by the server on SELECT or EXAMINE + (), the client can split the UID range + : into groups, e.g., 100 messages. After + that, the client can issue: + + C: A011 UID fetch : + (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) + ... + C: A012 UID fetch : + (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) + ... + ... + C: A0FF UID fetch : + (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) + + Note that unless a SEARCH command is issued, it is impossible to + determine how many messages will fall into a subrange, as UIDs are + not necessarily contiguous. + + Note 2: The client SHOULD ignore any unsolicited EXPUNGE responses + received during the first FETCH command. EXPUNGE responses contain + message numbers that are useless to a client that doesn't have the + message-number-to-UID translation table. + + + + +Melnikov Informational [Page 19] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + The second FETCH command will result in zero or more untagged fetch + responses. Each response will have a corresponding UID FETCH data + item. All messages that didn't have a matching untagged FETCH + response MUST be removed from the local cache. + + For example, if the had a value 15000 and the local + cache contained 3 messages with the UIDs 12, 777, and 14999, + respectively, then after receiving the following responses from the + server, the client must remove the message with UID 14999 from its + local cache. + + S: * 1 FETCH (UID 12 FLAGS (\Seen)) + S: * 2 FETCH (UID 777 FLAGS (\Answered \Deleted)) + + Note 3: If the client is not interested in flag changes (i.e., the + client only wants to know which old messages are still on the + server), the second FETCH command can be substituted with: + + tag2 UID SEARCH UID 1: + + This command will generate less traffic. However, an implementor + should be aware that in order to build the mapping table from message + numbers to UIDs, the output of the SEARCH command MUST be sorted + first, because there is no requirement for a server to return UIDs in + SEARCH response in any particular order. + +4.3.2. Searching for "Interesting" Messages. + + This step is performed entirely on the client (from the information + received in the step described in 4.3.1), entirely on the server, or + on some combination of both. The decision on what is an + "interesting" message is up to the client software and the human. + One easy criterion that should probably be implemented in any client + is whether the message is "too big" for automatic retrieval, where + "too big" is a parameter defined in the client's configuration. + + Another commonly used criterion is the age of a message. For + example, the client may choose to download only messages received in + the last week (in this case, would be today's date minus 7 + days): + + tag3 UID SEARCH UID SINCE + + Keep in mind that a date search disregards time and time zone. The + client can avoid doing this search if it specified INTERNALDATE in + on the step described in 4.3.1. If the client did, it + can perform the local search on its message cache. + + + + +Melnikov Informational [Page 20] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + At this step, the client also decides what kind of information about + a particular message to fetch from the server. In particular, even + for a message that is considered "too big", the client MAY choose to + fetch some part(s) of it. For example, if the message is a + multipart/mixed containing a text part and a MPEG attachment, there + is no reason for the client not to fetch the text part. The decision + of which part should or should not be fetched can be based on the + information received in the BODYSTRUCTURE FETCH response data item + (i.e., if BODYSTRUCTURE was included in on the step + described in 4.3.1). + +4.3.3. Populating Cache with "Interesting" Messages. + + Once the client has found out which messages are "interesting", it + can start issuing appropriate FETCH commands for "interesting" + messages or parts thereof. + + Note that fetching a message into the disconnected client's local + cache does NOT imply that the human has (or even will) read the + message. Thus, the synchronization program for a disconnected client + should always be careful to use the .PEEK variants of the FETCH data + items that implicitly set the \Seen flag. + + Once the last descriptor has arrived and the last FETCH command has + been issued, the client simply needs to process the incoming fetch + items and use them to update the local message cache. + + In order to avoid deadlock problems, the client must give processing + of received messages priority over issuing new FETCH commands during + this synchronization process. This may necessitate temporary local + queuing of FETCH requests that cannot be issued without causing a + deadlock. In order to achieve the best use of the "expensive" + network connection, the client will almost certainly need to pay + careful attention to any flow-control information that it can obtain + from the underlying transport connection (usually a TCP connection). + + Note: The requirement stated in the previous paragraph might result + in an unpleasant user experience, if followed blindly. For example, + the user might be unwilling to wait for the client to finish + synchronization before starting to process the user's requests. A + smart disconnected client should allow the user to perform requested + operations in between IMAP commands that are part of the + synchronization process. See also Note 1 in Section 4.3.1. + + + + + + + + +Melnikov Informational [Page 21] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Example 8: + + After fetching a message BODYSTRUCTURE, the client discovers a + complex MIME message. Then, it decides to fetch MIME headers of the + nested MIME messages and some body parts. + + C: A011 UID fetch 11 (BODYSTRUCTURE) + S: ... + C: A012 UID fetch 11 (BODY[HEADER] BODY[1.MIME] BODY[1.1.MIME] + BODY[1.2.MIME] BODY[2.MIME] BODY[3.MIME] BODY[4.MIME] + BODY[5.MIME] BODY[6.MIME] BODY[7.MIME] BODY[8.MIME] BODY[9.MIME] + BODY[10.MIME] BODY[11.MIME] BODY[12.MIME] BODY[13.MIME] + BODY[14.MIME] BODY[15.MIME] BODY[16.MIME] BODY[17.MIME] + BODY[18.MIME] BODY[19.MIME] BODY[20.MIME] BODY[21.MIME]) + S: ... + C: A013 UID fetch 11 (BODY[1.1] BODY[1.2]) + S: ... + C: A014 UID fetch 11 (BODY[3] BODY[4] BODY[5] BODY[6] BODY[7] BODY[8] + BODY[9] BODY[10] BODY[11] BODY[13] BODY[14] BODY[15] BODY[16] + BODY[21]) + S: ... + +4.3.4. User-Initiated Synchronization + + After the client has finished the main synchronization process as + described in Sections 4.3.1-4.3.3, the user may optionally request + additional synchronization steps while the client is still online. + This is not any different from the process described in Sections + 4.3.2 and 4.3.3. + + Typical examples are: + + 1) fetch all messages selected in UI. + 2) fetch all messages marked as \Flagged on the server. + +4.4. Special Case: Descriptor-Only Synchronization + + For some mailboxes, fetching the descriptors might be the entire + synchronization step. Practical experience with IMAP has shown that + a certain class of mailboxes (e.g., "archival" mailboxes) are used + primarily for long-term storage of important messages that the human + wants to have instantly available on demand but does not want + cluttering up the disconnected client's cache at any other time. + Messages in this kind of mailbox would be fetched exclusively by + explicit actions queued by the local MUA. Thus, the only + synchronization desirable on this kind of mailbox is fetching enough + descriptor information for the user to be able to identify messages + for subsequent download. + + + +Melnikov Informational [Page 22] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + Special mailboxes that receive messages from a high volume, low + priority mailing list might also be in this category, at least when + the human is in a hurry. + +4.5. Special Case: Fast New-Only Synchronization + + In some cases, the human might be in such a hurry that he or she + doesn't care about changes to old messages, just about new messages. + In this case, the client can skip the UID FETCH command that obtains + the flags and UIDs for old messages (1:). + +4.6. Special Case: Blind FETCH + + In some cases, the human may know (for whatever reason) that he or + she always wants to fetch any new messages in a particular mailbox, + unconditionally. In this case, the client can just fetch the + messages themselves, rather than just the descriptors, by using a + command like: + + tag1 UID FETCH :* (FLAGS BODY.PEEK[]) + + Note that this example ignores the fact that the messages can be + arbitrary long. The disconnected client MUST always check for + message size before downloading, unless explicitly told otherwise. A + well-behaved client should instead use something like the following: + + 1) Issue "tag1 UID FETCH :* (FLAGS RFC822.SIZE)". + + 2) From the message sizes returned in step 1, construct UID set + . + + 3) Issue "tag2 UID FETCH (BODY.PEEK[])". + + or + + 1) Issue "tag1 UID FETCH :* (FLAGS)". + + 2) Construct UID set from the responses of step 1. + + 3) Issue "tag2 SEARCH UID SMALLER ". + Construct UID set from the result of the + SEARCH command. + + 4) Issue "tag3 UID FETCH (BODY.PEEK[])". + + + + + + + +Melnikov Informational [Page 23] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + or + + 1) Issue "tag1 UID FETCH :* (FLAGS + BODY.PEEK[]<0.>)", where should be replaced with + the maximal message size the client is willing to download. + + Note: In response to such a command, the server will only return + partial data if the message is longer than . It will + return the full message data for any message whose size is smaller + than or equal to . In the former case, the client will + not be able to extract the full MIME structure of the message from + the truncated data, so the client should include BODYSTRUCTURE in + the UID FETCH command as well. + +5. Implementation Considerations + + Below are listed some common implementation pitfalls that should be + considered when implementing a disconnected client. + + 1) Implementing fake UIDs on the client. + + A message scheduled to be uploaded has no UID, as UIDs are + selected by the server. The client may implement fake UIDs + internally in order to reference not-yet-uploaded messages in + further operations. (For example, a message could be scheduled to + be uploaded, but subsequently marked as deleted or copied to + another mailbox). Here, the client MUST NOT under any + circumstances send these fake UIDs to the server. Also, client + implementers should be reminded that according to [IMAP4] a UID is + a 32-bit unsigned integer excluding 0. So, both 4294967295 and + 2147483648 are valid UIDs, and 0 and -1 are both invalid. Some + disconnected mail clients have been known to send negative numbers + (e.g., "-1") as message UIDs to servers during synchronization. + + Situation 1: The user starts composing a new message, edits it, + saves it, continues to edit it, and saves it again. + + A disconnected client may record in its replay log (log of + operations to be replayed on the server during synchronization) + the sequence of operations as shown below. For the purpose of + this situation, we assume that all draft messages are stored in + the mailbox called Drafts on an IMAP server. We will also use the + following conventions: is the UID of the intermediate + version of the draft when it was saved for the first time. This + is a fake UID generated on the client. is the UID of + the final version of the draft. This is another fake UID + generated on the client. + + + + +Melnikov Informational [Page 24] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 1) APPEND Drafts (\Seen $MDNSent \Drafts) {} + ...first version of the message follows... + + 2) APPEND Drafts (\Seen $MDNSent \Drafts) {} + ...final version of the message follows... + + 3) STORE +FLAGS (\Deleted) + + Step 1 corresponds to the first attempt to save the draft message, + step 2 corresponds to the second attempt to save the draft + message, and step 3 deletes the first version of the draft message + saved in step 1. + + A naive disconnected client may send the command in step 3 without + replacing the fake client generated with the value + returned by the server in step 1. A server will probably reject + this command, which will make the client believe that the + synchronization sequence has failed. + + 2) Section 5.1 discusses common implementation errors related to + error recovery during playback. + + 3) Don't assume that the disconnected client is the only client used + by the user. + + Situation 2: Some clients may use the \Deleted flag as an + indicator that the message should not appear in the user's view. + Usage of the \Deleted flag for this purpose is not safe, as other + clients (e.g., online clients) might EXPUNGE the mailbox at any + time. + + 4) Beware of data dependencies between synchronization operations. + + It might be very tempting for a client writer to perform some + optimizations on the playback log. Such optimizations might + include removing redundant operations (for example, see + optimization 2 in Section 5.3), or their reordering. + + It is not always safe to reorder or remove redundant operations + during synchronization because some operations may have + dependencies (as Situation 3 demonstrates). So, if in doubt, + don't do this. + + Situation 3: The user copies a message out of a mailbox and then + deletes the mailbox. + + + + + + +Melnikov Informational [Page 25] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + C: A001 SELECT Old-Mail + S: ... + C: A002 UID COPY 111 ToDo + S: A002 OK [COPYUID 1022843345 111 94] Copy completed + ... + C: A015 CLOSE + S: A015 OK Completed + C: A016 DELETE Old-Mail + S: A016 OK Mailbox deletion completed successfully + + If the client performs DELETE (tag A016) first and COPY (tag A002) + second, then the COPY fails. Also, the message that the user so + carefully copied into another mailbox has been lost. + +5.1. Error Recovery during Playback + + Error recovery during synchronization is one of the trickiest parts + to get right. Below, we will discuss certain error conditions and + suggest possible choices for handling them. + + 1) Lost connection to the server. + + The client MUST remember the current position in the playback + (replay) log and replay it starting from the interrupted operation + (the last command issued by the client, but not acknowledged by + the server) the next time it successfully connects to the same + server. If the connection was lost while executing a non- + idempotent IMAP command (see the definition in Section 1), then + when the client is reconnected, it MUST make sure that the + interrupted command was indeed not executed. If it wasn't + executed, the client must restart playback from the interrupted + command, otherwise from the following command. + + Upon reconnect, care must be taken in order to properly reapply + logical operations that are represented by multiple IMAP commands, + e.g., UID EXPUNGE emulation when UID EXPUNGE is not supported by + the server (see Section 4.2.4). + + Once the client detects that the connection to the server was + lost, it MUST stop replaying its log. There are existing + disconnected clients that, to the great annoyance of users, pop up + an error dialog for each and every playback operation that fails. + + 2) Copying/appending messages to a mailbox that doesn't exist. (The + server advertises this condition by sending the TRYCREATE response + code in the tagged NO response to the APPEND or COPY command.) + + + + + +Melnikov Informational [Page 26] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + The user should be advised about the situation and be given one of + the following choices: + + a) Try to recreate a mailbox. + b) Copy/upload messages to another mailbox. + c) Skip copy/upload. + d) Abort replay. + + 3) Copying messages from a mailbox that doesn't exist, or renaming or + getting/changing ACLs [ACL] on a mailbox that doesn't exist: + + a) Skip operation. + b) Abort replay. + + 4) Deleting mailboxes or deleting/expunging messages that no longer + exist. + + This is actually is not an error and should be ignored by the + client. + + 5) Performing operations on messages that no longer exist. + + a) Skip operation. + b) Abort replay. + + In the case of changing flags on an expunged message, the client + should silently ignore the error. + + Note 1: Several synchronization operations map to multiple IMAP + commands (for example, "move" described in 4.2.2). The client must + guarantee atomicity of each such multistep operation. For example, + when performing a "move" between two mailboxes on the same server, if + the server is unable to copy messages, the client MUST NOT attempt to + set the \Deleted flag on the messages being copied, let alone expunge + them. However, the client MAY consider that move operation to have + succeeded even if the server was unable to set the \Deleted flag on + copied messages. + + Note 2: Many synchronization operations have data dependencies. A + failed operation must cause all dependent operations to fail as well. + The client should check this and MUST NOT try to perform all + dependent operations blindly (unless the user corrected the original + problem). For example, a message may be scheduled to be appended to + a mailbox on the server and later on the appended message may be + copied to another mailbox. If the APPEND operation fails, the client + must not attempt to COPY the failed message later on. (See also + Section 5, Situation 3). + + + + +Melnikov Informational [Page 27] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +5.2. Quality of Implementation Issues + + Below, some quality of implementation issues are listed for + disconnected clients. They will help to write a disconnected client + that works correctly, performs synchronization as quickly as possible + (and thus can make the user happier as well as save her some money), + and minimizes the server load: + + 1) Don't lose information. + + No matter how smart your client is in other areas, if it loses + information, users will get very upset. + + 2) Don't do work unless explicitly asked. Be flexible. Ask all + questions BEFORE starting synchronization, if possible. + + 3) Minimize traffic. + + The client MUST NOT issue a command if the client already received + the required information from the server. + + The client MUST make use of UIDPLUS extension if it is supported + by the server. + + See also optimization 1 in Section 5.3. + + 4) Minimize the number of round-trips. + + Round-trips kill performance, especially on links with high + latency. Sections 4.2.2.5 and 5.2 give some advice on how to + minimize the number of round-trips. + + See also optimization 1 in Section 5.3. + +5.3. Optimizations + + Some useful optimizations are described in this section. A + disconnected client that supports the recommendations listed below + will give the user a more pleasant experience. + + 1) The initial OK or PREAUTH responses may contain the CAPABILITY + response code as described in Section 7.1 of [IMAP4]. This + response code gives the same information as returned by the + CAPABILITY command*. A disconnected client that pays attention to + this response code can avoid sending CAPABILITY command and will + save a round-trip. + + + + + +Melnikov Informational [Page 28] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + * Note: Some servers report in the CAPABILITY response code + extensions that are only relevant in unauthenticated state or in + all states. Such servers usually send another CAPABILITY + response code upon successful authentication using LOGIN or + AUTHENTICATE command (that negotiates no security layer; see + Section 6.2.2 of [IMAP4]). The CAPABILITY response code sent + upon successful LOGIN/AUTHENTICATE might be different from the + CAPABILITY response code in the initial OK response, as + extensions only relevant for unauthenticated state will not be + advertised, and some additional extensions available only in + authenticated and/or selected state will be. + + Example 9: + + S: * OK [CAPABILITY IMAP4REV1 LOGIN-REFERRALS STARTTLS + AUTH=DIGEST-MD5 AUTH=SRP] imap.example.com ready + C: 2 authenticate DIGEST-MD5 + S: 2 OK [CAPABILITY IMAP4REV1 IDLE NAMESPACE MAILBOX-REFERRALS SCAN + SORT THREAD=REFERENCES THREAD=ORDEREDSUBJECT MULTIAPPEND] + User authenticated (no layer) + + 2) An advanced disconnected client may choose to optimize its replay + log. For example, there might be some operations that are + redundant (the list is not complete): + + a) an EXPUNGE followed by another EXPUNGE or CLOSE; + b) changing flags (other than the \Deleted flag) on a message that + gets immediately expunged; + c) opening and closing the same mailbox. + + When optimizing, be careful about data dependencies between commands. + For example, if the client is wishing to optimize (see case b, above) + + tag1 UID STORE +FLAGS (\Deleted) + ... + tag2 UID STORE +FLAGS (\Flagged) + ... + tag3 UID COPY "Backup" + ... + tag4 UID EXPUNGE + + it can't remove the second UID STORE command because the message is + being copied before it gets expunged. + + In general, it might be a good idea to keep mailboxes open during + synchronization (see case c above), if possible. This can be more + easily achieved in conjunction with optimization 3 described below. + + + + +Melnikov Informational [Page 29] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 3) Perform some synchronization steps in parallel, if possible. + + Several synchronization steps don't depend on each other and thus + can be performed in parallel. Because the server machine is + usually more powerful than the client machine and can perform some + operations in parallel, this may speed up the total time of + synchronization. + + In order to achieve such parallelization, the client will have to + open more than one connection to the same server. Client writers + should not forget about non-trivial cost associated with + establishing a TCP connection and performing an authentication. + The disconnected client MUST NOT use one connection per mailbox. + In most cases, it is sufficient to have two connections. The + disconnected client SHOULD avoid selecting the same mailbox in + more than one connection; see Section 3.1.1 of [RFC2683] for more + details. + + Any mailbox synchronization MUST start with checking the + UIDVALIDITY as described in Section 4.1 of this document. The + client MAY use STATUS command to check UID Validity of a non- + selected mailbox. This is preferable to opening many connections + to the same server to perform synchronization of multiple + mailboxes simultaneously. As described in Section 5.3.10 of + [IMAP4], this SHOULD NOT be used on the selected mailbox. + +6. IMAP Extensions That May Help + + The following extensions can save traffic and/or the number of + round-trips: + + 1) The use of [UIDPLUS] is discussed in Sections 4.1, 4.2.1, 4.2.2.1 + and 4.2.4. + + 2) The use of the MULTIAPPEND and LITERAL+ extensions for uploading + messages is discussed in Section 4.2.2.5. + + 3) Use the CONDSTORE extension (see Section 6.1) for quick flag + resynchronization. + +6.1. CONDSTORE Extension + + An advanced disconnected mail client should use the [CONDSTORE] + extension when it is supported by the server. The client must cache + the value from HIGHESTMODSEQ OK response code received on mailbox + opening and update it whenever the server sends MODSEQ FETCH data + items. + + + + +Melnikov Informational [Page 30] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + If the client receives NOMODSEQ OK untagged response instead of + HIGHESTMODSEQ, it MUST remove the last known HIGHESTMODSEQ value from + its cache and follow the more general instructions in Section 3. + + When the client opens the mailbox for synchronization, it first + compares UIDVALIDITY as described in step d-1 in Section 3. If the + cached UIDVALIDITY value matches the one returned by the server, the + client MUST compare the cached value of HIGHESTMODSEQ with the one + returned by the server. If the cached HIGHESTMODSEQ value also + matches the one returned by the server, then the client MUST NOT + fetch flags for cached messages, as they hasn't changed. If the + value on the server is higher than the cached one, the client MAY use + "SEARCH MODSEQ " to find all messages with flags + changed since the last time the client was online and had the mailbox + opened. Alternatively, the client MAY use "FETCH 1:* (FLAGS) + (CHANGEDSINCE )". The latter operation combines + searching for changed messages and fetching new information. + + In all cases, the client still needs to fetch information about new + messages (if requested by the user) as well as discover which + messages have been expunged. + + Step d ("Server-to-client synchronization") in Section 4 in the + presence of the CONDSTORE extension is amended as follows: + + d) "Server-to-client synchronization" - For each mailbox that + requires synchronization, do the following: + + 1a) Check the mailbox UIDVALIDITY (see section 4.1 for more + details) with SELECT/EXAMINE/STATUS. + + If the UIDVALIDITY value returned by the server differs, the + client MUST + + * empty the local cache of that mailbox; + * "forget" the cached HIGHESTMODSEQ value for the mailbox; + * remove any pending "actions" that refer to UIDs in that + mailbox (note that this doesn't affect actions performed on + client-generated fake UIDs; see Section 5); and + * skip steps 1b and 2-II; + + 1b) Check the mailbox HIGHESTMODSEQ. If the cached value is the + same as the one returned by the server, skip fetching message + flags on step 2-II, i.e., the client only has to find out + which messages got expunged. + + + + + + +Melnikov Informational [Page 31] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + 2) Fetch the current "descriptors". + + I) Discover new messages. + + II) Discover changes to old messages and flags for new messages + using + "FETCH 1:* (FLAGS) (CHANGEDSINCE )" or + "SEARCH MODSEQ ". + + Discover expunged messages; for example, using + "UID SEARCH 1:". (All messages not returned + in this command are expunged.) + + 3) Fetch the bodies of any "interesting" messages that the client + doesn't already have. + + Example 10: + + The UIDVALIDITY value is the same, but the HIGHESTMODSEQ value + has changed on the server while the client was offline. + + C: A142 SELECT INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 201] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [HIGHESTMODSEQ 20010715194045007] + S: A142 OK [READ-WRITE] SELECT completed + + After that, either: + + C: A143 UID FETCH 1:* (FLAGS) (CHANGEDSINCE 20010715194032001) + S: * 2 FETCH (UID 6 MODSEQ (20010715205008000) FLAGS (\Deleted)) + S: * 5 FETCH (UID 9 MODSEQ (20010715195517000) FLAGS ($NoJunk + $AutoJunk $MDNSent)) + ... + S: A143 OK FETCH completed + + or: + + + + + + + + + +Melnikov Informational [Page 32] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + + C: A143 UID SEARCH MODSEQ 20010715194032001 UID 1:20 + S: * SEARCH 6 9 11 12 18 19 20 23 (MODSEQ 20010917162500) + S: A143 OK Search complete + C: A144 UID SEARCH 1:20 + S: * SEARCH 6 9 ... + S: A144 OK FETCH completed + +7. Security Considerations + + It is believed that this document does not raise any new security + concerns that are not already present in the base [IMAP4] protocol, + and these issues are discussed in [IMAP4]. Additional security + considerations may be found in different extensions mentioned in this + document; in particular, in [UIDPLUS], [LITERAL+], [CONDSTORE], + [MULTIAPPEND], and [UNSELECT]. + + Implementers are also reminded about the importance of thorough + testing. + +8. References + +8.1. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - + VERSION 4rev1", RFC 3501, March 2003. + + [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - + UIDPLUS extension", RFC 4315, December 2005. + + [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC + 2088, January 1997. + + [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for + Conditional STORE Operation or Quick Flag Changes + Resynchronization", RFC 4551, June 2006. + + [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + + [UNSELECT] Melnikov, A., "Internet Message Access Protocol (IMAP) + UNSELECT command", RFC 3691, February 2004. + + [RFC2683] Leiba, B., "IMAP4 Implementation Recommendations", RFC + 2683, September 1999. + + + + +Melnikov Informational [Page 33] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +8.2. Informative References + + [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) + Extension", RFC 4314, December 2005. + + [IMAP-MODEL] Crispin, M., "Distributed Electronic Mail Models in + IMAP4", RFC 1733, December 1994. + +9. Acknowledgements + + This document is based on version 01 of the text written by Rob + Austein in November 1994. + + The editor appreciates comments posted by Mark Crispin to the IMAP + mailing list and the comments/corrections/ideas received from Grant + Baillie, Cyrus Daboo, John G. Myers, Chris Newman, and Timo Sirainen. + + The editor would also like to thank the developers of Netscape + Messenger and Mozilla mail clients for providing examples of + disconnected mail clients that served as a base for many + recommendations in this document. + +Editor's Address + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex + TW12 2BX + United Kingdom + + Phone: +44 77 53759732 + EMail: alexey.melnikov@isode.com + + + + + + + + + + + + + + + + + +Melnikov Informational [Page 34] + +RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Melnikov Informational [Page 35] + diff --git a/docs/rfcs/rfc4551.IMAP_Conditional_STORE_or_Quick_flag_changes_resync.txt b/docs/rfcs/rfc4551.IMAP_Conditional_STORE_or_Quick_flag_changes_resync.txt new file mode 100644 index 0000000..894b510 --- /dev/null +++ b/docs/rfcs/rfc4551.IMAP_Conditional_STORE_or_Quick_flag_changes_resync.txt @@ -0,0 +1,1403 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 4551 Isode Ltd. +Updates: 3501 S. Hole +Category: Standards Track ACI WorldWide/MessagingDirect + June 2006 + + + IMAP Extension for Conditional STORE Operation + or Quick Flag Changes Resynchronization + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2006). + +Abstract + + Often, multiple IMAP (RFC 3501) clients need to coordinate changes to + a common IMAP mailbox. Examples include different clients working on + behalf of the same user, and multiple users accessing shared + mailboxes. These clients need a mechanism to synchronize state + changes for messages within the mailbox. They must be able to + guarantee that only one client can change message state (e.g., + message flags) at any time. An example of such an application is use + of an IMAP mailbox as a message queue with multiple dequeueing + clients. + + The Conditional Store facility provides a protected update mechanism + for message state information that can detect and resolve conflicts + between multiple writing mail clients. + + The Conditional Store facility also allows a client to quickly + resynchronize mailbox flag changes. + + This document defines an extension to IMAP (RFC 3501). + + + + + + + + + +Melnikov & Hole Standards Track [Page 1] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +Table of Contents + + 1. Introduction and Overview ................................. 3 + 2. Conventions Used in This Document ......................... 5 + 3. IMAP Protocol Changes ..................................... 6 + 3.1. New OK untagged responses for SELECT and EXAMINE ......... 6 + 3.1.1. HIGHESTMODSEQ response code ............................ 6 + 3.1.2. NOMODSEQ response code ................................. 7 + 3.2. STORE and UID STORE Commands ............................. 7 + 3.3 FETCH and UID FETCH Commands ..............................13 + 3.3.1. CHANGEDSINCE FETCH modifier ............................13 + 3.3.2. MODSEQ message data item in FETCH Command ..............14 + 3.4. MODSEQ search criterion in SEARCH ........................16 + 3.5. Modified SEARCH untagged response ........................17 + 3.6. HIGHESTMODSEQ status data items ..........................17 + 3.7. CONDSTORE parameter to SELECT and EXAMINE ................18 + 3.8. Additional quality of implementation issues ..............18 + 4. Formal Syntax .............................................19 + 5. Server implementation considerations ......................21 + 6. Security Considerations ...................................22 + 7. IANA Considerations .......................................22 + 8. References ................................................23 + 8.1. Normative References .....................................23 + 8.2. Informative References ...................................23 + 9. Acknowledgements ..........................................23 + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Hole Standards Track [Page 2] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +1. Introduction and Overview + + The Conditional STORE extension is present in any IMAP4 + implementation that returns "CONDSTORE" as one of the supported + capabilities in the CAPABILITY command response. + + An IMAP server that supports this extension MUST associate a positive + unsigned 64-bit value called a modification sequence (mod-sequence) + with every IMAP message. This is an opaque value updated by the + server whenever a metadata item is modified. The server MUST + guarantee that each STORE command performed on the same mailbox + (including simultaneous stores to different metadata items from + different connections) will get a different mod-sequence value. + Also, for any two successful STORE operations performed in the same + session on the same mailbox, the mod-sequence of the second completed + operation MUST be greater than the mod-sequence of the first + completed. Note that the latter rule disallows the use of the system + clock as a mod-sequence, because if system time changes (e.g., an NTP + [NTP] client adjusting the time), the next generated value might be + less than the previous one. + + Mod-sequences allow a client that supports the CONDSTORE extension to + determine if a message metadata has changed since some known moment. + Whenever the state of a flag changes (i.e., the flag is added where + previously it wasn't set, or the flag is removed and before it was + set) the value of the modification sequence for the message MUST be + updated. Adding the flag when it is already present or removing when + it is not present SHOULD NOT change the mod-sequence. + + When a message is appended to a mailbox (via the IMAP APPEND command, + COPY to the mailbox, or using an external mechanism) the server + generates a new modification sequence that is higher than the highest + modification sequence of all messages in the mailbox and assigns it + to the appended message. + + The server MAY store separate (per-message) modification sequence + values for different metadata items. If the server does so, per- + message mod-sequence is the highest mod-sequence of all metadata + items for the specified message. + + The server that supports this extension is not required to be able to + store mod-sequences for every available mailbox. Section 3.1.2 + describes how the server may act if a particular mailbox doesn't + support the persistent storage of mod-sequences. + + + + + + + +Melnikov & Hole Standards Track [Page 3] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + This extension makes the following changes to the IMAP4 protocol: + + a) adds UNCHANGEDSINCE STORE modifier. + + b) adds the MODIFIED response code which should be used with an OK + response to the STORE command. (It can also be used in a NO + response.) + + c) adds a new MODSEQ message data item for use with the FETCH + command. + + d) adds CHANGEDSINCE FETCH modifier. + + e) adds a new MODSEQ search criterion. + + f) extends the syntax of untagged SEARCH responses to include + mod-sequence. + + g) adds new OK untagged responses for the SELECT and EXAMINE + commands. + + h) defines an additional parameter to SELECT/EXAMINE commands. + + i) adds the HIGHESTMODSEQ status data item to the STATUS command. + + A client supporting CONDSTORE extension indicates its willingness to + receive mod-sequence updates in all untagged FETCH responses by + issuing: + + - a SELECT or EXAMINE command with the CONDSTORE parameter, + - a STATUS (HIGHESTMODSEQ) command, + - a FETCH or SEARCH command that includes the MODSEQ message data + item, + - a FETCH command with the CHANGEDSINCE modifier, or + - a STORE command with the UNCHANGEDSINCE modifier. + + The server MUST include mod-sequence data in all subsequent untagged + FETCH responses (until the connection is closed), whether they were + caused by a regular STORE, a STORE with UNCHANGEDSINCE modifier, or + an external agent. + + This document uses the term "CONDSTORE-aware client" to refer to a + client that announces its willingness to receive mod-sequence updates + as described above. The term "CONDSTORE enabling command" will refer + any of the commands listed above. A future extension to this + document may extend the list of CONDSTORE enabling commands. A first + CONDSTORE enabling command executed in the session MUST cause the + + + + +Melnikov & Hole Standards Track [Page 4] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + server to return HIGHESTMODSEQ (Section 3.1.1) unless the server has + sent NOMODSEQ (Section 3.1.2) response code when the currently + selected mailbox was selected. + + The rest of this document describes the protocol changes more + rigorously. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + + In examples, lines beginning with "S:" are sent by the IMAP server, + and lines beginning with "C:" are sent by the client. Line breaks + may appear in example commands solely for editorial clarity; when + present in the actual message, they are represented by "CRLF". + + Formal syntax is defined using ABNF [ABNF]. + + The term "metadata" or "metadata item" is used throughout this + document. It refers to any system or user-defined keyword. Future + documents may extend "metadata" to include other dynamic message + data. + + Some IMAP mailboxes are private, accessible only to the owning user. + Other mailboxes are not, either because the owner has set an Access + Control List [ACL] that permits access by other users, or because it + is a shared mailbox. Let's call a metadata item "shared" for the + mailbox if any changes to the metadata items are persistent and + visible to all other users accessing the mailbox. Otherwise, the + metadata item is called "private". Note that private metadata items + are still visible to all sessions accessing the mailbox as the same + user. Also note that different mailboxes may have different metadata + items as shared. + + See Section 1 for the definition of a "CONDSTORE-aware client" and a + "CONDSTORE enabling command". + + + + + + + + + + + + + +Melnikov & Hole Standards Track [Page 5] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +3. IMAP Protocol Changes + +3.1. New OK Untagged Responses for SELECT and EXAMINE + + This document adds two new response codes, HIGHESTMODSEQ and + NOMODSEQ. One of those response codes MUST be returned in the OK + untagged response for a successful SELECT/EXAMINE command. + + When opening a mailbox, the server must check if the mailbox supports + the persistent storage of mod-sequences. If the mailbox supports the + persistent storage of mod-sequences and the mailbox open operation + succeeds, the server MUST send the OK untagged response including + HIGHESTMODSEQ response code. If the persistent storage for the + mailbox is not supported, the server MUST send the OK untagged + response including NOMODSEQ response code instead. + +3.1.1. HIGHESTMODSEQ Response Code + + This document adds a new response code that is returned in the OK + untagged response for the SELECT and EXAMINE commands. A server + supporting the persistent storage of mod-sequences for the mailbox + MUST send the OK untagged response including HIGHESTMODSEQ response + code with every successful SELECT or EXAMINE command: + + OK [HIGHESTMODSEQ ] + + where is the highest mod-sequence value of + all messages in the mailbox. When the server changes UIDVALIDITY + for a mailbox, it doesn't have to keep the same HIGHESTMODSEQ for + the mailbox. + + A disconnected client can use the value of HIGHESTMODSEQ to check if + it has to refetch metadata from the server. If the UIDVALIDITY value + has changed for the selected mailbox, the client MUST delete the + cached value of HIGHESTMODSEQ. If UIDVALIDITY for the mailbox is the + same, and if the HIGHESTMODSEQ value stored in the client's cache is + less than the value returned by the server, then some metadata items + on the server have changed since the last synchronization, and the + client needs to update its cache. The client MAY use SEARCH MODSEQ + (Section 3.4) to find out exactly which metadata items have changed. + Alternatively, the client MAY issue FETCH with the CHANGEDSINCE + modifier (Section 3.3.1) in order to fetch data for all messages that + have metadata items changed since some known modification sequence. + + Example 1: + + C: A142 SELECT INBOX + S: * 172 EXISTS + + + +Melnikov & Hole Standards Track [Page 6] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [HIGHESTMODSEQ 715194045007] + S: A142 OK [READ-WRITE] SELECT completed + +3.1.2. NOMODSEQ Response Code + + A server that doesn't support the persistent storage of mod-sequences + for the mailbox MUST send the OK untagged response including NOMODSEQ + response code with every successful SELECT or EXAMINE command. A + server that returned NOMODSEQ response code for a mailbox, which + subsequently receives one of the following commands while the mailbox + is selected: + + - a FETCH command with the CHANGEDSINCE modifier, + - a FETCH or SEARCH command that includes the MODSEQ message data + item, or + - a STORE command with the UNCHANGEDSINCE modifier + + MUST reject any such command with the tagged BAD response. + + Example 2: + + C: A142 SELECT INBOX + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [NOMODSEQ] Sorry, this mailbox format doesn't support + modsequences + S: A142 OK [READ-WRITE] SELECT completed + +3.2. STORE and UID STORE Commands + + This document defines the following STORE modifier (see Section 2.5 + of [IMAPABNF]): + + UNCHANGEDSINCE + + For each message specified in the message set, the server performs + the following. If the mod-sequence of any metadata item of the + + + +Melnikov & Hole Standards Track [Page 7] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + message is equal or less than the specified UNCHANGEDSINCE value, + then the requested operation (as described by the message data + item) is performed. If the operation is successful, the server + MUST update the mod-sequence attribute of the message. An + untagged FETCH response MUST be sent, even if the .SILENT suffix + is specified, and the response MUST include the MODSEQ message + data item. This is required to update the client's cache with the + correct mod-sequence values. See Section 3.3.2 for more details. + + However, if the mod-sequence of any metadata item of the message + is greater than the specified UNCHANGEDSINCE value, then the + requested operation MUST NOT be performed. In this case, the + mod-sequence attribute of the message is not updated, and the + message number (or unique identifier in the case of the UID STORE + command) is added to the list of messages that failed the + UNCHANGESINCE test. + + When the server finished performing the operation on all the + messages in the message set, it checks for a non-empty list of + messages that failed the UNCHANGESINCE test. If this list is + non-empty, the server MUST return in the tagged response a + MODIFIED response code. The MODIFIED response code includes the + message set (for STORE) or set of UIDs (for UID STORE) of all + messages that failed the UNCHANGESINCE test. + + Example 3: + + All messages pass the UNCHANGESINCE test. + + C: a103 UID STORE 6,4,8 (UNCHANGEDSINCE 12121230045) + +FLAGS.SILENT (\Deleted) + S: * 1 FETCH (UID 4 MODSEQ (12121231000)) + S: * 2 FETCH (UID 6 MODSEQ (12121230852)) + S: * 4 FETCH (UID 8 MODSEQ (12121130956)) + S: a103 OK Conditional Store completed + + Example 4: + + C: a104 STORE * (UNCHANGEDSINCE 12121230045) +FLAGS.SILENT + (\Deleted $Processed) + S: * 50 FETCH (MODSEQ (12111230047)) + S: a104 OK Store (conditional) completed + + Example 5: + + C: c101 STORE 1 (UNCHANGEDSINCE 12121230045) -FLAGS.SILENT + (\Deleted) + S: * OK [HIGHESTMODSEQ 12111230047] + + + +Melnikov & Hole Standards Track [Page 8] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + S: * 50 FETCH (MODSEQ (12111230048)) + S: c101 OK Store (conditional) completed + + HIGHESTMODSEQ response code was sent by the server presumably + because this was the first CONDSTORE enabling command. + + Example 6: + + In spite of the failure of the conditional STORE operation for + message 7, the server continues to process the conditional STORE + in order to find all messages that fail the test. + + C: d105 STORE 7,5,9 (UNCHANGEDSINCE 320162338) + +FLAGS.SILENT (\Deleted) + S: * 5 FETCH (MODSEQ (320162350)) + S: d105 OK [MODIFIED 7,9] Conditional STORE failed + + Example 7: + + Same as above, but the server follows the SHOULD recommendation in + Section 6.4.6 of [IMAP4]. + + C: d105 STORE 7,5,9 (UNCHANGEDSINCE 320162338) + +FLAGS.SILENT (\Deleted) + S: * 7 FETCH (MODSEQ (320162342) FLAGS (\Seen \Deleted)) + S: * 5 FETCH (MODSEQ (320162350)) + S: * 9 FETCH (MODSEQ (320162349) FLAGS (\Answered)) + S: d105 OK [MODIFIED 7,9] Conditional STORE failed + + Use of UNCHANGEDSINCE with a modification sequence of 0 always + fails if the metadata item exists. A system flag MUST always be + considered existent, whether it was set or not. + + Example 8: + + C: a102 STORE 12 (UNCHANGEDSINCE 0) + +FLAGS.SILENT ($MDNSent) + S: a102 OK [MODIFIED 12] Conditional STORE failed + + The client has tested the presence of the $MDNSent user-defined + keyword. + + Note: A client trying to make an atomic change to the state of a + particular metadata item (or a set of metadata items) should be + prepared to deal with the case when the server returns the MODIFIED + response code if the state of the metadata item being watched hasn't + changed (but the state of some other metadata item has). This is + necessary, because some servers don't store separate mod-sequences + + + +Melnikov & Hole Standards Track [Page 9] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + for different metadata items. However, a server implementation + SHOULD avoid generating spurious MODIFIED responses for +FLAGS/-FLAGS + STORE operations, even when the server stores a single mod-sequence + per message. Section 5 describes how this can be achieved. + + Unless the server has included an unsolicited FETCH to update + client's knowledge about messages that have failed the UNCHANGEDSINCE + test, upon receipt of the MODIFIED response code, the client SHOULD + try to figure out if the required metadata items have indeed changed + by issuing FETCH or NOOP command. It is RECOMMENDED that the server + avoids the need for the client to do that by sending an unsolicited + FETCH response (Examples 9 and 10). + + If the required metadata items haven't changed, the client SHOULD + retry the command with the new mod-sequence. The client SHOULD allow + for a configurable but reasonable number of retries (at least 2). + + Example 9: + + In the example below, the server returns the MODIFIED response + code without sending information describing why the STORE + UNCHANGEDSINCE operation has failed. + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + S: * 100 FETCH (MODSEQ (303181230852)) + S: * 102 FETCH (MODSEQ (303181230852)) + ... + S: * 150 FETCH (MODSEQ (303181230852)) + S: a106 OK [MODIFIED 101] Conditional STORE failed + + The flag $Processed was set on the message 101... + + C: a107 NOOP + S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed)) + S: a107 OK + + Or the flag hasn't changed, but another has (note that this server + behaviour is discouraged. Server implementers should also see + Section 5)... + + C: b107 NOOP + S: * 101 FETCH (MODSEQ (303011130956) FLAGS (\Deleted \Answered)) + S: b107 OK + + ...and the client retries the operation for the message 101 with + the updated UNCHANGEDSINCE value + + + + +Melnikov & Hole Standards Track [Page 10] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + C: b108 STORE 101 (UNCHANGEDSINCE 303011130956) + +FLAGS.SILENT ($Processed) + S: * 101 FETCH (MODSEQ (303181230852)) + S: b108 OK Conditional Store completed + + Example 10: + + Same as above, but the server avoids the need for the client to + poll for changes. + + The flag $Processed was set on the message 101 by another + client... + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + S: * 100 FETCH (MODSEQ (303181230852)) + S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed)) + S: * 102 FETCH (MODSEQ (303181230852)) + ... + S: * 150 FETCH (MODSEQ (303181230852)) + S: a106 OK [MODIFIED 101] Conditional STORE failed + + Or the flag hasn't changed, but another has (note that this server + behaviour is discouraged. Server implementers should also see + Section 5)... + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + S: * 100 FETCH (MODSEQ (303181230852)) + S: * 101 FETCH (MODSEQ (303011130956) FLAGS (\Deleted \Answered)) + S: * 102 FETCH (MODSEQ (303181230852)) + ... + S: * 150 FETCH (MODSEQ (303181230852)) + S: a106 OK [MODIFIED 101] Conditional STORE failed + + ...and the client retries the operation for the message 101 with + the updated UNCHANGEDSINCE value + + C: b108 STORE 101 (UNCHANGEDSINCE 303011130956) + +FLAGS.SILENT ($Processed) + S: * 101 FETCH (MODSEQ (303181230852)) + S: b108 OK Conditional Store completed + + Or the flag hasn't changed, but another has (nice server + behaviour. Server implementers should also see Section 5)... + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + + + +Melnikov & Hole Standards Track [Page 11] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + S: * 100 FETCH (MODSEQ (303181230852)) + S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed \Deleted + \Answered)) + S: * 102 FETCH (MODSEQ (303181230852)) + ... + S: * 150 FETCH (MODSEQ (303181230852)) + S: a106 OK Conditional STORE completed + + Example 11: + + The following example is based on the example from the Section + 4.2.3 of [RFC-2180] and demonstrates that the MODIFIED response + code may be also returned in the tagged NO response. + + Client tries to conditionally STORE flags on a mixture of expunged + and non-expunged messages; one message fails the UNCHANGEDSINCE + test. + + C: B001 STORE 1:7 (UNCHANGEDSINCE 320172338) +FLAGS (\SEEN) + S: * 1 FETCH (MODSEQ (320172342) FLAGS (\SEEN)) + S: * 3 FETCH (MODSEQ (320172342) FLAGS (\SEEN)) + S: B001 NO [MODIFIED 2] Some of the messages no longer exist. + + C: B002 NOOP + S: * 4 EXPUNGE + S: * 4 EXPUNGE + S: * 4 EXPUNGE + S: * 4 EXPUNGE + S: * 2 FETCH (MODSEQ (320172340) FLAGS (\Deleted \Answered)) + S: B002 OK NOOP Completed. + + By receiving FETCH responses for messages 1 and 3, and EXPUNGE + responses that indicate that messages 4 through 7 have been + expunged, the client retries the operation only for the message 2. + The updated UNCHANGEDSINCE value is used. + + C: b003 STORE 2 (UNCHANGEDSINCE 320172340) +FLAGS (\Seen) + S: * 2 FETCH (MODSEQ (320180050)) + S: b003 OK Conditional Store completed + + Note: If a message is specified multiple times in the message set, + and the server doesn't internally eliminate duplicates from the + message set, it MUST NOT fail the conditional STORE operation for the + second (or subsequent) occurrence of the message if the operation + completed successfully for the first occurrence. For example, if the + client specifies: + + + + + +Melnikov & Hole Standards Track [Page 12] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + e105 STORE 7,3:9 (UNCHANGEDSINCE 12121230045) + +FLAGS.SILENT (\Deleted) + + the server must not fail the operation for message 7 as part of + processing "3:9" if it succeeded when message 7 was processed the + first time. + + Once the client specified the UNCHANGEDSINCE modifier in a STORE + command, the server MUST include the MODSEQ fetch response data items + in all subsequent unsolicited FETCH responses. + + This document also changes the behaviour of the server when it has + performed a STORE or UID STORE command and the UNCHANGEDSINCE + modifier is not specified. If the operation is successful for a + message, the server MUST update the mod-sequence attribute of the + message. The server is REQUIRED to include the mod-sequence value + whenever it decides to send the unsolicited FETCH response to all + CONDSTORE-aware clients that have opened the mailbox containing the + message. + + Server implementers should also see Section 3.8 for additional + quality of implementation issues related to the STORE command. + +3.3. FETCH and UID FETCH Commands + +3.3.1. CHANGEDSINCE FETCH Modifier + + This document defines the following FETCH modifier (see Section 2.4 + of [IMAPABNF]): + + CHANGEDSINCE + + CHANGEDSINCE FETCH modifier allows to create a further subset of + the list of messages described by sequence set. The information + described by message data items is only returned for messages that + have mod-sequence bigger than . + + When CHANGEDSINCE FETCH modifier is specified, it implicitly adds + MODSEQ FETCH message data item (Section 3.3.2). + + Example 12: + + C: s100 UID FETCH 1:* (FLAGS) (CHANGEDSINCE 12345) + S: * 1 FETCH (UID 4 MODSEQ (65402) FLAGS (\Seen)) + S: * 2 FETCH (UID 6 MODSEQ (75403) FLAGS (\Deleted)) + S: * 4 FETCH (UID 8 MODSEQ (29738) FLAGS ($NoJunk $AutoJunk + $MDNSent)) + S: s100 OK FETCH completed + + + +Melnikov & Hole Standards Track [Page 13] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +3.3.2. MODSEQ Message Data Item in FETCH Command + + This extension adds a MODSEQ message data item to the FETCH command. + The MODSEQ message data item allows clients to retrieve mod-sequence + values for a range of messages in the currently selected mailbox. + + Once the client specified the MODSEQ message data item in a FETCH + request, the server MUST include the MODSEQ fetch response data items + in all subsequent unsolicited FETCH responses. + + Syntax: MODSEQ + + The MODSEQ message data item causes the server to return MODSEQ + fetch response data items. + + Syntax: MODSEQ ( ) + + MODSEQ response data items contain per-message mod-sequences. + + The MODSEQ response data item is returned if the client issued + FETCH with MODSEQ message data item. It also allows the server to + notify the client about mod-sequence changes caused by conditional + STOREs (Section 3.2) and/or changes caused by external sources. + + Example 13: + + C: a FETCH 1:3 (MODSEQ) + S: * 1 FETCH (MODSEQ (624140003)) + S: * 2 FETCH (MODSEQ (624140007)) + S: * 3 FETCH (MODSEQ (624140005)) + S: a OK Fetch complete + + In this example, the client requests per-message mod-sequences for + a set of messages. + + When a flag for a message is modified in a different session, the + server sends an unsolicited FETCH response containing the mod- + sequence for the message. + + Example 14: + + (Session 1, authenticated as a user "alex"). The user adds a + shared flag \Deleted: + + C: A142 SELECT INBOX + ... + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Answered \Deleted \Seen \*)] Limited + + + +Melnikov & Hole Standards Track [Page 14] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + ... + + C: A160 STORE 7 +FLAGS.SILENT (\Deleted) + S: * 7 FETCH (MODSEQ (2121231000)) + S: A160 OK Store completed + + (Session 2, also authenticated as the user "alex"). Any changes + to flags are always reported to all sessions authenticated as the + same user as in the session 1. + + C: C180 NOOP + S: * 7 FETCH (FLAGS (\Deleted \Answered) MODSEQ (12121231000)) + S: C180 OK Noop completed + + (Session 3, authenticated as a user "andrew"). As \Deleted is a + shared flag, changes in session 1 are also reported in session 3: + + C: D210 NOOP + S: * 7 FETCH (FLAGS (\Deleted \Answered) MODSEQ (12121231000)) + S: D210 OK Noop completed + + The user modifies a private flag \Seen in session 1... + + C: A240 STORE 7 +FLAGS.SILENT (\Seen) + S: * 7 FETCH (MODSEQ (12121231777)) + S: A240 OK Store completed + + ...which is only reported in session 2... + + C: C270 NOOP + S: * 7 FETCH (FLAGS (\Deleted \Answered \Seen) MODSEQ + (12121231777)) + S: C270 OK Noop completed + + ...but not in session 3. + + C: D300 NOOP + S: D300 OK Noop completed + + And finally, the user removes flags \Answered (shared) and \Seen + (private) in session 1. + + C: A330 STORE 7 -FLAGS.SILENT (\Answered \Seen) + S: * 7 FETCH (MODSEQ (12121245160)) + S: A330 OK Store completed + + + + + + +Melnikov & Hole Standards Track [Page 15] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + Both changes are reported in the session 2... + + C: C360 NOOP + S: * 7 FETCH (FLAGS (\Deleted) MODSEQ (12121245160)) + S: C360 OK Noop completed + + ...and only changes to shared flags are reported in session 3. + + C: D390 NOOP + S: * 7 FETCH (FLAGS (\Deleted) MODSEQ (12121245160)) + S: D390 OK Noop completed + + Server implementers should also see Section 3.8 for additional + quality of implementation issues related to the FETCH command. + +3.4. MODSEQ Search Criterion in SEARCH + + The MODSEQ criterion for the SEARCH command allows a client to search + for the metadata items that were modified since a specified moment. + + Syntax: MODSEQ [ ] + + Messages that have modification values that are equal to or + greater than . This allows a client, for + example, to find out which messages contain metadata items that + have changed since the last time it updated its disconnected + cache. The client may also specify (name of metadata + item) and (type of metadata item) before + . can be one of "shared", + "priv" (private), or "all". The latter means that the server + should use the biggest value among "priv" and "shared" mod- + sequences for the metadata item. If the server doesn't store + internally separate mod-sequences for different metadata items, it + MUST ignore and . Otherwise, the + server should use them to narrow down the search. + + For a flag , the corresponding has a form + "/flags/" as defined in [IMAPABNF]. Note that the + leading "\" character that denotes a system flag has to be escaped + as per Section 4.3 of [IMAP4], as the uses syntax for + quoted strings. + + If client specifies a MODSEQ criterion in a SEARCH command and the + server returns a non-empty SEARCH result, the server MUST also append + (to the end of the untagged SEARCH response) the highest mod-sequence + for all messages being returned. See also Section 3.5. + + + + + +Melnikov & Hole Standards Track [Page 16] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + Example 15: + + C: a SEARCH MODSEQ "/flags/\\draft" all 620162338 + S: * SEARCH 2 5 6 7 11 12 18 19 20 23 (MODSEQ 917162500) + S: a OK Search complete + + In the above example, the message numbers of any messages + containing the string "IMAP4" in the "value" attribute of the + "/comment" entry and having a mod-sequence equal to or greater + than 620162338 for the "\Draft" flag are returned in the search + results. + + Example 16: + + C: t SEARCH OR NOT MODSEQ 720162338 LARGER 50000 + S: * SEARCH + S: t OK Search complete, nothing found + +3.5. Modified SEARCH Untagged Response + + Data: zero or more numbers + mod-sequence value (omitted if no match) + + This document extends syntax of the untagged SEARCH response to + include the highest mod-sequence for all messages being returned. + + If a client specifies a MODSEQ criterion in a SEARCH (or UID SEARCH) + command and the server returns a non-empty SEARCH result, the server + MUST also append (to the end of the untagged SEARCH response) the + highest mod-sequence for all messages being returned. See Section + 3.4 for examples. + +3.6. HIGHESTMODSEQ Status Data Items + + This document defines a new status data item: + + HIGHESTMODSEQ + + The highest mod-sequence value of all messages in the mailbox. + This is the same value that is returned by the server in the + HIGHESTMODSEQ response code in an OK untagged response (see + Section 3.1.1). If the server doesn't support the persistent + storage of mod-sequences for the mailbox (see Section 3.1.2), the + server MUST return 0 as the value of HIGHESTMODSEQ status data + item. + + + + + + +Melnikov & Hole Standards Track [Page 17] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + Example 17: + + C: A042 STATUS blurdybloop (UIDNEXT MESSAGES HIGHESTMODSEQ) + S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292 + HIGHESTMODSEQ 7011231777) + S: A042 OK STATUS completed + +3.7. CONDSTORE Parameter to SELECT and EXAMINE + + The CONDSTORE extension defines a single optional select parameter, + "CONDSTORE", which tells the server that it MUST include the MODSEQ + fetch response data items in all subsequent unsolicited FETCH + responses. + + The CONDSTORE parameter to SELECT/EXAMINE helps avoid a race + condition that might arise when one or more metadata items are + modified in another session after the server has sent the + HIGHESTMODSEQ response code and before the client was able to issue a + CONDSTORE enabling command. + + Example 18: + + C: A142 SELECT INBOX (CONDSTORE) + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 4392] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [HIGHESTMODSEQ 715194045007] + S: A142 OK [READ-WRITE] SELECT completed, CONDSTORE is now enabled + +3.8. Additional Quality-of-Implementation Issues + + Server implementations should follow the following rule, which + applies to any successfully completed STORE/UID STORE (with and + without UNCHANGEDSINCE modifier), as well as to a FETCH command that + implicitly sets \Seen flag: + + Adding the flag when it is already present or removing when it is + not present SHOULD NOT change the mod-sequence. + + This will prevent spurious client synchronization requests. + + + + + + + +Melnikov & Hole Standards Track [Page 18] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + However, note that client implementers MUST NOT rely on this server + behavior. A client can't distinguish between the case when a server + has violated the SHOULD mentioned above, and that when one or more + clients set and unset (or unset and set) the flag in another session. + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) [ABNF] notation. Elements not defined here can be found + in the formal syntax of the ABNF [ABNF], IMAP [IMAP4], and IMAP ABNF + extensions [IMAPABNF] specifications. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper- or lowercase characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + capability =/ "CONDSTORE" + + status-att =/ "HIGHESTMODSEQ" + ;; extends non-terminal defined in RFC 3501. + + status-att-val =/ "HIGHESTMODSEQ" SP mod-sequence-valzer + ;; extends non-terminal defined in [IMAPABNF]. + ;; Value 0 denotes that the mailbox doesn't + ;; support persistent mod-sequences + ;; as described in Section 3.1.2 + + store-modifier =/ "UNCHANGEDSINCE" SP mod-sequence-valzer + ;; Only a single "UNCHANGEDSINCE" may be + ;; specified in a STORE operation + + fetch-modifier =/ chgsince-fetch-mod + ;; conforms to the generic "fetch-modifier" + ;; syntax defined in [IMAPABNF]. + + chgsince-fetch-mod = "CHANGEDSINCE" SP mod-sequence-value + ;; CHANGEDSINCE FETCH modifier conforms to + ;; the fetch-modifier syntax + + fetch-att =/ fetch-mod-sequence + ;; modifies original IMAP4 fetch-att + + fetch-mod-sequence = "MODSEQ" + + fetch-mod-resp = "MODSEQ" SP "(" permsg-modsequence ")" + + msg-att-dynamic =/ fetch-mod-resp + + + +Melnikov & Hole Standards Track [Page 19] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + search-key =/ search-modsequence + ;; modifies original IMAP4 search-key + ;; + ;; This change applies to all commands + ;; referencing this non-terminal, in + ;; particular SEARCH. + + search-modsequence = "MODSEQ" [search-modseq-ext] SP + mod-sequence-valzer + + search-modseq-ext = SP entry-name SP entry-type-req + + resp-text-code =/ "HIGHESTMODSEQ" SP mod-sequence-value / + "NOMODSEQ" / + "MODIFIED" SP set + + entry-name = entry-flag-name + + entry-flag-name = DQUOTE "/flags/" attr-flag DQUOTE + ;; each system or user defined flag + ;; is mapped to "/flags/". + ;; + ;; follows the escape rules + ;; used by "quoted" string as described in + ;; Section 4.3 of [IMAP4], e.g., for the flag + ;; \Seen the corresponding is + ;; "/flags/\\seen", and for the flag + ;; $MDNSent, the corresponding + ;; is "/flags/$mdnsent". + + entry-type-resp = "priv" / "shared" + ;; metadata item type + + entry-type-req = entry-type-resp / "all" + ;; perform SEARCH operation on private + ;; metadata item, shared metadata item or both + + permsg-modsequence = mod-sequence-value + ;; per message mod-sequence + + mod-sequence-value = 1*DIGIT + ;; Positive unsigned 64-bit integer + ;; (mod-sequence) + ;; (1 <= n < 18,446,744,073,709,551,615) + + mod-sequence-valzer = "0" / mod-sequence-value + + search-sort-mod-seq = "(" "MODSEQ" SP mod-sequence-value ")" + + + +Melnikov & Hole Standards Track [Page 20] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + select-param =/ condstore-param + ;; conforms to the generic "select-param" + ;; non-terminal syntax defined in [IMAPABNF]. + + condstore-param = "CONDSTORE" + + mailbox-data =/ "SEARCH" [1*(SP nz-number) SP + search-sort-mod-seq] + + attr-flag = "\\Answered" / "\\Flagged" / "\\Deleted" / + "\\Seen" / "\\Draft" / attr-flag-keyword / + attr-flag-extension + ;; Does not include "\\Recent" + + attr-flag-extension = "\\" atom + ;; Future expansion. Client implementations + ;; MUST accept flag-extension flags. Server + ;; implementations MUST NOT generate + ;; flag-extension flags except as defined by + ;; future standard or standards-track + ;; revisions of [IMAP4]. + + attr-flag-keyword = atom + +5. Server Implementation Considerations + + This section describes how a server implementation that doesn't store + separate per-metadata mod-sequences for different metadata items can + avoid sending the MODIFIED response to any of the following + conditional STORE operations: + + +FLAGS + -FLAGS + +FLAGS.SILENT + -FLAGS.SILENT + + Note that the optimization described in this section can't be + performed in case of a conditional STORE FLAGS operation. + + Let's use the following example. The client has issued + + C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) + +FLAGS.SILENT ($Processed) + + When the server receives the command and parses it successfully, it + iterates through the message set and tries to execute the conditional + STORE command for each message. + + + + +Melnikov & Hole Standards Track [Page 21] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + + Each server internally works as a client, i.e., it has to cache the + current state of all IMAP flags as it is known to the client. In + order to report flag changes to the client, the server compares the + cached values with the values in its database for IMAP flags. + + Imagine that another client has changed the state of a flag \Deleted + on the message 101 and that the change updated the mod-sequence for + the message. The server knows that the mod-sequence for the mailbox + has changed; however, it also knows that: + + a) the client is not interested in \Deleted flag, as it hasn't + included it in +FLAGS.SILENT operation; and + + b) the state of the flag $Processed hasn't changed (the server can + determine this by comparing cached flag state with the state of + the flag in the database). + + Therefore, the server doesn't have to report MODIFIED to the client. + Instead, the server may set $Processed flag, update the mod-sequence + for the message 101 once again and send an untagged FETCH response + with new mod-sequence and flags: + + S: * 101 FETCH (MODSEQ (303011130956) + FLAGS ($Processed \Deleted \Answered)) + + See also Section 3.8 for additional quality-of-implementation issues. + +6. Security Considerations + + It is believed that the Conditional STORE extension doesn't raise any + new security concerns that are not already discussed in [IMAP4]. + However, the availability of this extension may make it possible for + IMAP4 to be used in critical applications it could not be used for + previously, making correct IMAP server implementation and operation + even more important. + +7. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + This document defines the CONDSTORE IMAP capability. IANA has added + it to the registry accordingly. + + + + + +Melnikov & Hole Standards Track [Page 22] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +8. References + +8.1. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + +8.2. Informative References + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [ANN] Daboo, C. and R. Gellens, "IMAP ANNOTATE Extension", Work + in Progress, March 2006. + + [NTP] Mills, D., "Network Time Protocol (Version 3) + Specification, Implementation and Analysis", RFC 1305, + March 1992. + + [RFC-2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC + 2180, July 1997. + +9. Acknowledgements + + Some text was borrowed from "IMAP ANNOTATE Extension" [ANN] by + Randall Gellens and Cyrus Daboo and from "ACAP -- Application + Configuration Access Protocol" [ACAP] by Chris Newman and John Myers. + + Many thanks to Randall Gellens for his thorough review of the + document. + + The authors also acknowledge the feedback provided by Cyrus Daboo, + Larry Greenfield, Chris Newman, Harrie Hazewinkel, Arnt Gulbrandsen, + Timo Sirainen, Mark Crispin, Ned Freed, Ken Murchison, and Dave + Cridland. + + + + +Melnikov & Hole Standards Track [Page 23] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex + TW12 2BX, + United Kingdom + + EMail: Alexey.Melnikov@isode.com + + + Steve Hole + ACI WorldWide/MessagingDirect + #1807, 10088 102 Ave + Edmonton, AB + T5J 2Z1 + Canada + + EMail: Steve.Hole@messagingdirect.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Hole Standards Track [Page 24] + +RFC 4551 IMAP Extension for Conditional STORE June 2006 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is provided by the IETF + Administrative Support Activity (IASA). + + + + + + + +Melnikov & Hole Standards Track [Page 25] + diff --git a/docs/rfcs/rfc4731.IMAP4_Extension_to_SEARCH_command.txt b/docs/rfcs/rfc4731.IMAP4_Extension_to_SEARCH_command.txt new file mode 100644 index 0000000..8c4869a --- /dev/null +++ b/docs/rfcs/rfc4731.IMAP4_Extension_to_SEARCH_command.txt @@ -0,0 +1,451 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 4731 Isode Ltd +Category: Standards Track D. Cridland + Inventure Systems Ltd + November 2006 + + + IMAP4 Extension to SEARCH Command for Controlling + What Kind of Information Is Returned + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The IETF Trust (2006). + +Abstract + + This document extends IMAP (RFC 3501) SEARCH and UID SEARCH commands + with several result options, which can control what kind of + information is returned. The following result options are defined: + minimal value, maximal value, all found messages, and number of found + messages. + +Table of Contents + + 1. Introduction ....................................................2 + 2. Conventions Used in This Document ...............................2 + 3. IMAP Protocol Changes ...........................................2 + 3.1. New SEARCH/UID SEARCH Result Options .......................2 + 3.2. Interaction with CONDSTORE extension .......................4 + 4. Formal Syntax ...................................................5 + 5. Security Considerations .........................................6 + 6. IANA Considerations .............................................6 + 7. Normative References ............................................6 + 8. Acknowledgments .................................................6 + + + + + + + + + +Melnikov & Cridland Standards Track [Page 1] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + +1. Introduction + + [IMAPABNF] extended SEARCH and UID SEARCH commands with result + specifiers (also known as result options), which can control what + kind of information is returned. + + A server advertising the ESEARCH capability supports the following + result options: minimal value, maximal value, all found messages, + and number of found messages. These result options allow clients to + get SEARCH results in more convenient forms, while also saving + bandwidth required to transport the results, for example, by finding + the first unseen message or returning the number of unseen or deleted + messages. Also, when a single MIN or a single MAX result option is + specified, servers can optimize execution of SEARCHes. + +2. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + +3. IMAP Protocol Changes + +3.1. New SEARCH/UID SEARCH Result Options + + The SEARCH/UID SEARCH commands are extended to allow for the + following result options: + + MIN + Return the lowest message number/UID that satisfies the SEARCH + criteria. + + If the SEARCH results in no matches, the server MUST NOT + include the MIN result option in the ESEARCH response; however, + it still MUST send the ESEARCH response. + + MAX + Return the highest message number/UID that satisfies the SEARCH + criteria. + + If the SEARCH results in no matches, the server MUST NOT + include the MAX result option in the ESEARCH response; however, + it still MUST send the ESEARCH response. + + + + + +Melnikov & Cridland Standards Track [Page 2] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + + ALL + Return all message numbers/UIDs that satisfy the SEARCH + criteria. Unlike regular (unextended) SEARCH, the messages are + always returned using the sequence-set syntax. A sequence-set + representation may be more compact and can be used as is in a + subsequent command that accepts sequence-set. Note, the client + MUST NOT assume that messages/UIDs will be listed in any + particular order. + + If the SEARCH results in no matches, the server MUST NOT + include the ALL result option in the ESEARCH response; however, + it still MUST send the ESEARCH response. + + COUNT + Return number of the messages that satisfy the SEARCH criteria. + This result option MUST always be included in the ESEARCH + response. + + If one or more result options described above are specified, the + extended SEARCH command MUST return a single ESEARCH response + [IMAPABNF], instead of the SEARCH response. + + An extended UID SEARCH command MUST cause an ESEARCH response with + the UID indicator present. + + Note that future extensions to this document can allow servers to + return multiple ESEARCH responses for a single extended SEARCH + command. These extensions will have to describe how results from + multiple ESEARCH responses are to be amalgamated. + + If the list of result options is empty, that requests the server to + return an ESEARCH response instead of the SEARCH response. This is + equivalent to "(ALL)". + + Example: C: A282 SEARCH RETURN (MIN COUNT) FLAGGED + SINCE 1-Feb-1994 NOT FROM "Smith" + S: * ESEARCH (TAG "A282") MIN 2 COUNT 3 + S: A282 OK SEARCH completed + + Example: C: A283 SEARCH RETURN () FLAGGED + SINCE 1-Feb-1994 NOT FROM "Smith" + S: * ESEARCH (TAG "A283") ALL 2,10:11 + S: A283 OK SEARCH completed + + The following example demonstrates finding the first unseen message + as returned in the UNSEEN response code on a successful SELECT + command: + + + + +Melnikov & Cridland Standards Track [Page 3] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + + Example: C: A284 SEARCH RETURN (MIN) UNSEEN + S: * ESEARCH (TAG "A284") MIN 4 + S: A284 OK SEARCH completed + + The following example demonstrates that if the ESEARCH UID indicator + is present, all data in the ESEARCH response is referring to UIDs; + for example, the MIN result specifier will be followed by a UID. + + Example: C: A285 UID SEARCH RETURN (MIN MAX) 1:5000 + S: * ESEARCH (TAG "A285") UID MIN 7 MAX 3800 + S: A285 OK SEARCH completed + + The following example demonstrates returning the number of deleted + messages: + + Example: C: A286 SEARCH RETURN (COUNT) DELETED + S: * ESEARCH (TAG "A286") COUNT 15 + S: A286 OK SEARCH completed + +3.2. Interaction with CONDSTORE extension + + When the server supports both the ESEARCH and the CONDSTORE + [CONDSTORE] extension, and the client requests one or more result + option described in section 3.1 together with the MODSEQ search + criterion in the same SEARCH/UID SEARCH command, then the server MUST + return the ESEARCH response containing the MODSEQ result option + (described in the following paragraph) instead of the extended SEARCH + response described in section 3.5 of [CONDSTORE]. + + If the SEARCH/UID SEARCH command contained a single MIN or MAX result + option, the MODSEQ result option contains the mod-sequence for the + found message. If the SEARCH/UID SEARCH command contained both MIN + and MAX result options and no ALL/COUNT option, the MODSEQ result + option contains the highest mod-sequence for the two returned + messages. Otherwise the MODSEQ result option contains the highest + mod-sequence for all messages being returned. + + Example: The following example demonstrates how Example 15 from + [CONDSTORE] would look in the presence of one or more result option: + + C: a1 SEARCH RETURN (MIN) MODSEQ "/flags/\\draft" + all 620162338 + S: * ESEARCH (TAG "a1") MIN 2 MODSEQ 917162488 + S: a1 OK Search complete + + C: a2 SEARCH RETURN (MAX) MODSEQ "/flags/\\draft" + all 620162338 + S: * ESEARCH (TAG "a2") MAX 23 MODSEQ 907162321 + + + +Melnikov & Cridland Standards Track [Page 4] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + + S: a2 OK Search complete + + C: a3 SEARCH RETURN (MIN MAX) MODSEQ "/flags/\\draft" + all 620162338 + S: * ESEARCH (TAG "a3") MIN 2 MAX 23 MODSEQ 917162488 + S: a3 OK Search complete + + C: a4 SEARCH RETURN (MIN COUNT) MODSEQ "/flags/\\draft" + all 620162338 + S: * ESEARCH (TAG "a4") MIN 2 COUNT 10 MODSEQ 917162500 + S: a4 OK Search complete + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + Non-terminals referenced but not defined below are as defined by + [IMAP4], [CONDSTORE], or [IMAPABNF]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lowercase characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + capability =/ "ESEARCH" + + search-return-data = "MIN" SP nz-number / + "MAX" SP nz-number / + "ALL" SP sequence-set / + "COUNT" SP number + ;; conforms to the generic + ;; search-return-data syntax defined + ;; in [IMAPABNF] + + search-return-opt = "MIN" / "MAX" / "ALL" / "COUNT" + ;; conforms to generic search-return-opt + ;; syntax defined in [IMAPABNF] + + When the CONDSTORE [CONDSTORE] IMAP extension is also supported, + the ABNF is updated as follows: + + search-return-data =/ "MODSEQ" SP mod-sequence-value + ;; mod-sequence-value is defined + ;; in [CONDSTORE] + + + + + + +Melnikov & Cridland Standards Track [Page 5] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + +5. Security Considerations + + In the general case, the IMAP SEARCH/UID SEARCH commands can be CPU + and/or IO intensive, and are seen by some as a potential attack point + for denial of service attacks, so some sites/implementations even + disable them entirely. This is quite unfortunate, as SEARCH command + is one of the best examples demonstrating IMAP advantage over POP3. + + The ALL and COUNT return options don't change how SEARCH is working + internally; they only change how information about found messages is + returned. MIN and MAX SEARCH result options described in this + document can lighten the load on IMAP servers that choose to optimize + SEARCHes containing only one or both of them. + + It is believed that this extension doesn't raise any additional + security concerns not already discussed in [IMAP4]. + +6. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track RFC + or an IESG-approved experimental RFC. The registry is currently + located at . + + This document defines the ESEARCH IMAP capability, which IANA added + to the registry. + +7. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [ABNF] Crocker, D. (Ed.) and P. Overell , "Augmented BNF for + Syntax Specifications: ABNF", RFC 4234, October 2005. + + [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006.. + + [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for Conditional + STORE", RFC 4551, June 2006. + +8. Acknowledgments + + Thanks to Michael Wener, Arnt Gulbrandsen, Cyrus Daboo, Mark Crispin, + and Pete Maclean for comments and corrections. + + + + +Melnikov & Cridland Standards Track [Page 6] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex, TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + Dave A. Cridland + Inventure Systems Limited + + EMail: dave.cridland@inventuresystems.co.uk + URL: http://invsys.co.uk/dave/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Cridland Standards Track [Page 7] + +RFC 4731 IMAP4 Extension to SEARCH November 2006 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2006). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST, + AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT + THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY + IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR + PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + +Melnikov & Cridland Standards Track [Page 8] + diff --git a/docs/rfcs/rfc4978.IMAP_Compress_extension.txt b/docs/rfcs/rfc4978.IMAP_Compress_extension.txt new file mode 100644 index 0000000..14b56b6 --- /dev/null +++ b/docs/rfcs/rfc4978.IMAP_Compress_extension.txt @@ -0,0 +1,507 @@ + + + + + + +Network Working Group A. Gulbrandsen +Request for Comments: 4978 Oryx Mail Systems GmbH +Category: Standards Track August 2007 + + + The IMAP COMPRESS Extension + +Status of this Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + The COMPRESS extension allows an IMAP connection to be effectively + and efficiently compressed. + + Table of Contents + + 1. Introduction and Overview .......................................2 + 2. Conventions Used in This Document ...............................2 + 3. The COMPRESS Command ............................................3 + 4. Compression Efficiency ..........................................4 + 5. Formal Syntax ...................................................6 + 6. Security Considerations .........................................6 + 7. IANA Considerations .............................................6 + 8. Acknowledgements ................................................7 + 9. References ......................................................7 + 9.1. Normative References .......................................7 + 9.2. Informative References .....................................7 + + + + + + + + + + + + + + + + + + +Gulbrandsen Standards Track [Page 1] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + +1. Introduction and Overview + + A server which supports the COMPRESS extension indicates this with + one or more capability names consisting of "COMPRESS=" followed by a + supported compression algorithm name as described in this document. + + The goal of COMPRESS is to reduce the bandwidth usage of IMAP. + + Compared to PPP compression (see [RFC1962]) and modem-based + compression (see [MNP] and [V42BIS]), COMPRESS offers much better + compression efficiency. COMPRESS can be used together with Transport + Security Layer (TLS) [RFC4346], Simple Authentication and Security + layer (SASL) encryption, Virtual Private Networks (VPNs), etc. + Compared to TLS compression [RFC3749], COMPRESS has the following + (dis)advantages: + + - COMPRESS can be implemented easily both by IMAP servers and + clients. + + - IMAP COMPRESS benefits from an intimate knowledge of the IMAP + protocol's state machine, allowing for dynamic and aggressive + optimization of the underlying compression algorithm's parameters. + + - When the TLS layer implements compression, any protocol using that + layer can transparently benefit from that compression (e.g., SMTP + and IMAP). COMPRESS is specific to IMAP. + + In order to increase interoperation, it is desirable to have as few + different compression algorithms as possible, so this document + specifies only one. The DEFLATE algorithm (defined in [RFC1951]) is + standard, widely available and fairly efficient, so it is the only + algorithm defined by this document. + + In order to increase interoperation, IMAP servers that advertise this + extension SHOULD also advertise the TLS DEFLATE compression mechanism + as defined in [RFC3749]. IMAP clients MAY use either COMPRESS or TLS + compression, however, if the client and server support both, it is + RECOMMENDED that the client choose TLS compression. + + The extension adds one new command (COMPRESS) and no new responses. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + Formal syntax is defined by [RFC4234] as modified by [RFC3501]. + + + +Gulbrandsen Standards Track [Page 2] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + In the examples, "C:" and "S:" indicate lines sent by the client and + server respectively. "[...]" denotes elision. + +3. The COMPRESS Command + + Arguments: Name of compression mechanism: "DEFLATE". + + Responses: None + + Result: OK The server will compress its responses and expects the + client to compress its commands. + NO Compression is already active via another layer. + BAD Command unknown, invalid or unknown argument, or COMPRESS + already active. + + The COMPRESS command instructs the server to use the named + compression mechanism ("DEFLATE" is the only one defined) for all + commands and/or responses after COMPRESS. + + The client MUST NOT send any further commands until it has seen the + result of COMPRESS. If the response was OK, the client MUST compress + starting with the first command after COMPRESS. If the server + response was BAD or NO, the client MUST NOT turn on compression. + + If the server responds NO because it knows that the same mechanism is + active already (e.g., because TLS has negotiated the same mechanism), + it MUST send COMPRESSIONACTIVE as resp-text-code (see [RFC3501], + Section 7.1), and the resp-text SHOULD say which layer compresses. + + If the server issues an OK response, the server MUST compress + starting immediately after the CRLF which ends the tagged OK + response. (Responses issued by the server before the OK response + will, of course, still be uncompressed.) If the server issues a BAD + or NO response, the server MUST NOT turn on compression. + + For DEFLATE (as for many other compression mechanisms), the + compressor can trade speed against quality. When decompressing there + isn't much of a tradeoff. Consequently, the client and server are + both free to pick the best reasonable rate of compression for the + data they send. + + When COMPRESS is combined with TLS (see [RFC4346]) or SASL (see + [RFC4422]) security layers, the sending order of the three extensions + MUST be first COMPRESS, then SASL, and finally TLS. That is, before + data is transmitted it is first compressed. Second, if a SASL + security layer has been negotiated, the compressed data is then + signed and/or encrypted accordingly. Third, if a TLS security layer + has been negotiated, the data from the previous step is signed and/or + + + +Gulbrandsen Standards Track [Page 3] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + encrypted accordingly. When receiving data, the processing order + MUST be reversed. This ensures that before sending, data is + compressed before it is encrypted, independent of the order in which + the client issues COMPRESS, AUTHENTICATE, and STARTTLS. + + The following example illustrates how commands and responses are + compressed during a simple login sequence: + + S: * OK [CAPABILITY IMAP4REV1 STARTTLS COMPRESS=DEFLATE] + C: a starttls + S: a OK TLS active + + From this point on, everything is encrypted. + + C: b login arnt tnra + S: b OK Logged in as arnt + C: c compress deflate + S: d OK DEFLATE active + + From this point on, everything is compressed before being + encrypted. + + The following example demonstrates how a server may refuse to + compress twice: + + S: * OK [CAPABILITY IMAP4REV1 STARTTLS COMPRESS=DEFLATE] + [...] + C: c compress deflate + S: c NO [COMPRESSIONACTIVE] DEFLATE active via TLS + +4. Compression Efficiency + + This section is informative, not normative. + + IMAP poses some unusual problems for a compression layer. + + Upstream is fairly simple. Most IMAP clients send the same few + commands again and again, so any compression algorithm that can + exploit repetition works efficiently. The APPEND command is an + exception; clients that send many APPEND commands may want to + surround large literals with flushes in the same way as is + recommended for servers later in this section. + + Downstream has the unusual property that several kinds of data are + sent, confusing all dictionary-based compression algorithms. + + + + + + +Gulbrandsen Standards Track [Page 4] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + One type is IMAP responses. These are highly compressible; zlib + using its least CPU-intensive setting compresses typical responses to + 25-40% of their original size. + + Another type is email headers. These are equally compressible, and + benefit from using the same dictionary as the IMAP responses. + + A third type is email body text. Text is usually fairly short and + includes much ASCII, so the same compression dictionary will do a + good job here, too. When multiple messages in the same thread are + read at the same time, quoted lines etc. can often be compressed + almost to zero. + + Finally, attachments (non-text email bodies) are transmitted, either + in binary form or encoded with base-64. + + When attachments are retrieved in binary form, DEFLATE may be able to + compress them, but the format of the attachment is usually not IMAP- + like, so the dictionary built while compressing IMAP does not help. + The compressor has to adapt its dictionary from IMAP to the + attachment's format, and then back. A few file formats aren't + compressible at all using deflate, e.g., .gz, .zip, and .jpg files. + + When attachments are retrieved in base-64 form, the same problems + apply, but the base-64 encoding adds another problem. 8-bit + compression algorithms such as deflate work well on 8-bit file + formats, however base-64 turns a file into something resembling 6-bit + bytes, hiding most of the 8-bit file format from the compressor. + + When using the zlib library (see [RFC1951]), the functions + deflateInit2(), deflate(), inflateInit2(), and inflate() suffice to + implement this extension. The windowBits value must be in the range + -8 to -15, or else deflateInit2() uses the wrong format. + deflateParams() can be used to improve compression rate and resource + use. The Z_FULL_FLUSH argument to deflate() can be used to clear the + dictionary (the receiving peer does not need to do anything). + + A client can improve downstream compression by implementing BINARY + (defined in [RFC3516]) and using FETCH BINARY instead of FETCH BODY. + In the author's experience, the improvement ranges from 5% to 40% + depending on the attachment being downloaded. + + A server can improve downstream compression if it hints to the + compressor that the data type is about to change strongly, e.g., by + sending a Z_FULL_FLUSH at the start and end of large non-text + literals (before and after '*CHAR8' in the definition of literal in + RFC 3501, page 86). Small literals are best left alone. A possible + boundary is 5k. + + + +Gulbrandsen Standards Track [Page 5] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + A server can improve the CPU efficiency both of the server and the + client if it adjusts the compression level (e.g., using the + deflateParams() function in zlib) at these points, to avoid trying to + compress incompressible attachments. A very simple strategy is to + change the level to 0 at the start of a literal provided the first + two bytes are either 0x1F 0x8B (as in deflate-compressed files) or + 0xFF 0xD8 (JPEG), and to keep it at 1-5 the rest of the time. More + complex strategies are possible. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC4234]. This syntax augments + the grammar specified in [RFC3501]. [RFC4234] defines SP and + [RFC3501] defines command-auth, capability, and resp-text-code. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + command-auth =/ compress + + compress = "COMPRESS" SP algorithm + + capability =/ "COMPRESS=" algorithm + ;; multiple COMPRESS capabilities allowed + + algorithm = "DEFLATE" + + resp-text-code =/ "COMPRESSIONACTIVE" + + Note that due the syntax of capability names, future algorithm names + must be atoms. + +6. Security Considerations + + As for TLS compression [RFC3749]. + +7. IANA Considerations + + The IANA has added COMPRESS=DEFLATE to the list of IMAP capabilities. + + + + + + + + + +Gulbrandsen Standards Track [Page 6] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + +8. Acknowledgements + + Eric Burger, Dave Cridland, Tony Finch, Ned Freed, Philip Guenther, + Randall Gellens, Tony Hansen, Cullen Jennings, Stephane Maes, Alexey + Melnikov, Lyndon Nerenberg, and Zoltan Ordogh have all helped with + this document. + + The author would also like to thank various people in the rooms at + meetings, whose help is real, but not reflected in the author's + mailbox. + +9. References + +9.1. Normative References + + [RFC1951] Deutsch, P., "DEFLATE Compressed Data Format Specification + version 1.3", RFC 1951, May 1996. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + +9.2. Informative References + + [RFC1962] Rand, D., "The PPP Compression Control Protocol (CCP)", + RFC 1962, June 1996. + + [RFC3516] Nerenberg, L., "IMAP4 Binary Content Extension", RFC 3516, + April 2003. + + [RFC3749] Hollenbeck, S., "Transport Layer Security Protocol + Compression Methods", RFC 3749, May 2004. + + [RFC4346] Dierks, T. and E. Rescorla, "The Transport Layer Security + (TLS) Protocol Version 1.1", RFC 4346, April 2006. + + [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication and + Security Layer (SASL)", RFC 4422, June 2006. + + [V42BIS] ITU, "V.42bis: Data compression procedures for data + circuit-terminating equipment (DCE) using error correction + procedures", http://www.itu.int/rec/T-REC-V.42bis, January + 1990. + + + +Gulbrandsen Standards Track [Page 7] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + + [MNP] Gilbert Held, "The Complete Modem Reference", Second + Edition, Wiley Professional Computing, ISBN 0-471-00852-4, + May 1994. + +Author's Address + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + Fax: +49 89 4502 9758 + EMail: arnt@oryx.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gulbrandsen Standards Track [Page 8] + +RFC 4978 The IMAP COMPRESS Extension August 2007 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2007). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Gulbrandsen Standards Track [Page 9] + diff --git a/docs/rfcs/rfc5032.IMAP_WITHIN_Search_extension.txt b/docs/rfcs/rfc5032.IMAP_WITHIN_Search_extension.txt new file mode 100644 index 0000000..f8e4895 --- /dev/null +++ b/docs/rfcs/rfc5032.IMAP_WITHIN_Search_extension.txt @@ -0,0 +1,283 @@ + + + + + + +Network Working Group E. Burger, Ed. +Request for Comments: 5032 BEA Systems, Inc. +Updates: 3501 September 2007 +Category: Standards Track + + + WITHIN Search Extension to the IMAP Protocol + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + This document describes the WITHIN extension to IMAP SEARCH. IMAP + SEARCH returns messages whose internal date is within or outside a + specified interval. The mechanism described here, OLDER and YOUNGER, + differs from BEFORE and SINCE in that the client specifies an + interval, rather than a date. WITHIN is useful for persistent + searches where either the device does not have the capacity to + perform the search at regular intervals or the network is of limited + bandwidth and thus there is a desire to reduce network traffic from + sending repeated requests and redundant responses. + +1. Introduction + + This extension exposes two new search keys, OLDER and YOUNGER, each + of which takes a non-zero integer argument corresponding to a time + interval in seconds. The server calculates the time of interest by + subtracting the time interval the client presents from the current + date and time of the server. The server then either returns messages + older or younger than the resultant time and date, depending on the + search key used. + +1.1. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + + + + + +Burger Standards Track [Page 1] + +RFC 5032 Search Within September 2007 + + + When describing the general syntax, we omit some definitions, as RFC + 3501 [RFC3501] defines them. + +2. Protocol Operation + + An IMAP4 server that supports the capability described here MUST + return "WITHIN" as one of the server supported capabilities in the + CAPABILITY command. + + For both the OLDER and YOUNGER search keys, the server calculates a + target date and time by subtracting the interval, specified in + seconds, from the current date and time of the server. The server + then compares the target time with the INTERNALDATE of the message, + as specified in IMAP [RFC3501]. For OLDER, messages match if the + INTERNALDATE is less recent than or equal to the target time. For + YOUNGER, messages match if the INTERNALDATE is more recent than or + equal to the target time. + + Both OLDER and YOUNGER searches always result in exact matching, to + the resolution of a second. However, if one is doing a dynamic + evaluation, for example, in a context [CONTEXT], one needs to be + aware that the server might perform the evaluation periodically. + Thus, the server may delay the updates. Clients MUST be aware that + dynamic search results may not reflect the current state of the + mailbox. If the client needs a search result that reflects the + current state of the mailbox, we RECOMMEND that the client issue a + new search. + +3. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation. Elements not defined here can be found in the + formal syntax of ABNF [RFC4234] and IMAP [RFC3501]. + + This document extends RFC 3501 [RFC3501] with two new search keys: + OLDER and YOUNGER . + + search-key =/ ( "OLDER" / "YOUNGER" ) SP nz-number + ; search-key defined in RFC 3501 + +4. Example + + C: a1 SEARCH UNSEEN YOUNGER 259200 + S: a1 * SEARCH 4 8 15 16 23 42 + + Search for all unseen messages within the past 3 days, or 259200 + seconds, according to the server's current time. + + + + +Burger Standards Track [Page 2] + +RFC 5032 Search Within September 2007 + + +5. Security Considerations + + The WITHIN extension does not raise any security considerations that + are not present in the base protocol. Considerations are the same as + for IMAP [RFC3501]. + +6. IANA Considerations + + Per the IMAP RFC [RFC3501], registration of a new IMAP capability in + the IMAP Capability registry requires the publication of a standards- + track RFC or an IESG approved experimental RFC. The registry is + currently located at + . This + standards-track document defines the WITHIN IMAP capability. IANA + has added this extension to the IANA IMAP Capability registry. + +7. References + +7.1. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, BCP 14, March 1997. + + [RFC3501] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [RFC4234] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 4234, October 2005. + +7.2. Informative References + + [CONTEXT] Melnikov, D. and C. King, "Contexts for IMAP4", Work + in Progress, May 2006. + + + + + + + + + + + + + + + + + + +Burger Standards Track [Page 3] + +RFC 5032 Search Within September 2007 + + +Appendix A. Contributors + + Stephane Maes and Ray Cromwell wrote the original version of this + document as part of P-IMAP, as well as the first versions for the + IETF. From an attribution perspective, they are clearly authors. + +Appendix B. Acknowledgements + + The authors want to thank all who have contributed key insight and + who have extensively reviewed and discussed the concepts of LPSEARCH. + They also thank the authors of its early introduction in P-IMAP. + + We also want to give a special thanks to Arnt Gilbrandsen, Ken + Murchison, Zoltan Ordogh, and most especially Dave Cridland for their + review and suggestions. A special thank you goes to Alexey Melnikov + for his choice submission of text. + +Author's Address + + Eric W. Burger (editor) + BEA Systems, Inc. + USA + + EMail: eric.burger@bea.com + URI: http://www.standardstrack.com + + + + + + + + + + + + + + + + + + + + + + + + + + +Burger Standards Track [Page 4] + +RFC 5032 Search Within September 2007 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2007). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Burger Standards Track [Page 5] + diff --git a/docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt b/docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt new file mode 100644 index 0000000..13bbbf7 --- /dev/null +++ b/docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt @@ -0,0 +1,395 @@ + + + + + + +Network Working Group A. Gulbrandsen, Ed. +Request for Comments: 5161 Oryx Mail Systems GmbH +Category: Standards Track A. Melnikov, Ed. + Isode Limited + March 2008 + + + The IMAP ENABLE Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + Most IMAP extensions are used by the client when it wants to and the + server supports it. However, a few extensions require the server to + know whether a client supports that extension. The ENABLE extension + allows an IMAP client to say which extensions it supports. + +1. Overview + + Several IMAP extensions allow the server to return unsolicited + responses specific to these extensions in certain circumstances. + However, servers cannot send those unsolicited responses until they + know that the clients support such extensions and thus won't choke on + the extension response data. + + Up until now, extensions have typically stated that a server cannot + send the unsolicited responses until after the client has used a + command with the extension data (i.e., at that point the server knows + the client is aware of the extension). CONDSTORE ([RFC4551]), + ANNOTATE ([ANNOTATE]), and some extensions under consideration at the + moment use various commands to enable server extensions. For + example, CONDSTORE uses a SELECT or FETCH parameter, and ANNOTATE + uses a side effect of FETCH. + + The ENABLE extension provides an explicit indication from the client + that it supports particular extensions. This is done using a new + ENABLE command. + + An IMAP server that supports ENABLE advertises this by including the + word ENABLE in its capability list. + + + + +Gulbrandsen & Melnikov Standards Track [Page 1] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + + Most IMAP extensions do not require the client to enable the + extension in any way. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + Formal syntax is defined by [RFC5234] and [RFC3501]. + + Example lines prefaced by "C:" are sent by the client and ones + prefaced by "S:" by the server. The five characters [...] means that + something has been elided. + +3. Protocol Changes + +3.1. The ENABLE Command + + Arguments: capability names + + Result: OK: Relevant capabilities enabled + BAD: No arguments, or syntax error in an argument + + The ENABLE command takes a list of capability names, and requests the + server to enable the named extensions. Once enabled using ENABLE, + each extension remains active until the IMAP connection is closed. + For each argument, the server does the following: + + - If the argument is not an extension known to the server, the server + MUST ignore the argument. + + - If the argument is an extension known to the server, and it is not + specifically permitted to be enabled using ENABLE, the server MUST + ignore the argument. (Note that knowing about an extension doesn't + necessarily imply supporting that extension.) + + - If the argument is an extension that is supported by the server and + that needs to be enabled, the server MUST enable the extension for + the duration of the connection. At present, this applies only to + CONDSTORE ([RFC4551]). Note that once an extension is enabled, + there is no way to disable it. + + If the ENABLE command is successful, the server MUST send an untagged + ENABLED response (see Section 3.2). + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 2] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + + Clients SHOULD only include extensions that need to be enabled by the + server. At the time of publication, CONDSTORE is the only such + extension (i.e., ENABLE CONDSTORE is an additional "CONDSTORE + enabling command" as defined in [RFC4551]). Future RFCs may add to + this list. + + The ENABLE command is only valid in the authenticated state (see + [RFC3501]), before any mailbox is selected. Clients MUST NOT issue + ENABLE once they SELECT/EXAMINE a mailbox; however, server + implementations don't have to check that no mailbox is selected or + was previously selected during the duration of a connection. + + The ENABLE command can be issued multiple times in a session. It is + additive; i.e., "ENABLE a b", followed by "ENABLE c" is the same as a + single command "ENABLE a b c". When multiple ENABLE commands are + issued, each corresponding ENABLED response SHOULD only contain + extensions enabled by the corresponding ENABLE command. + + There are no limitations on pipelining ENABLE. For example, it is + possible to send ENABLE and then immediately SELECT, or a LOGIN + immediately followed by ENABLE. + + The server MUST NOT change the CAPABILITY list as a result of + executing ENABLE; i.e., a CAPABILITY command issued right after an + ENABLE command MUST list the same capabilities as a CAPABILITY + command issued before the ENABLE command. This is demonstrated in + the following example: + + C: t1 CAPABILITY + S: * CAPABILITY IMAP4rev1 ID LITERAL+ ENABLE X-GOOD-IDEA + S: t1 OK foo + C: t2 ENABLE CONDSTORE X-GOOD-IDEA + S: * ENABLED X-GOOD-IDEA + S: t2 OK foo + C: t3 CAPABILITY + S: * CAPABILITY IMAP4rev1 ID LITERAL+ ENABLE X-GOOD-IDEA + S: t3 OK foo again + + In the following example, the client enables CONDSTORE: + + C: a1 ENABLE CONDSTORE + S: * ENABLED CONDSTORE + S: a1 OK Conditional Store enabled + + + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 3] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + +3.2. The ENABLED Response + + Contents: capability listing + + The ENABLED response occurs as a result of an ENABLE command. The + capability listing contains a space-separated listing of capability + names that the server supports and that were successfully enabled. + The ENABLED response may contain no capabilities, which means that no + extensions listed by the client were successfully enabled. + +3.3. Note to Designers of Extensions That May Use the ENABLE Command + + Designers of IMAP extensions are discouraged from creating extensions + that require ENABLE unless there is no good alternative design. + Specifically, extensions that cause potentially incompatible behavior + changes to deployed server responses (and thus benefit from ENABLE) + have a higher complexity cost than extensions that do not. + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234] including the core + rules in Appendix B.1. [RFC3501] defines the non-terminals + "capability" and "command-any". + + Except as noted otherwise, all alphabetic characters are + case-insensitive. The use of upper or lower case characters to + define token strings is for editorial clarity only. Implementations + MUST accept these strings in a case-insensitive fashion. + + capability =/ "ENABLE" + + command-any =/ "ENABLE" 1*(SP capability) + + response-data =/ "*" SP enable-data CRLF + + enable-data = "ENABLED" *(SP capability) + +5. Security Considerations + + It is believed that this extension doesn't add any security + considerations that are not already present in the base IMAP protocol + [RFC3501]. + +6. IANA Considerations + + The IANA has added ENABLE to the IMAP4 Capabilities Registry. + + + + +Gulbrandsen & Melnikov Standards Track [Page 4] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + +7. Acknowledgments + + The editors would like to thank Randy Gellens, Chris Newman, Peter + Coates, Dave Cridland, Mark Crispin, Ned Freed, Dan Karp, Cyrus + Daboo, Ken Murchison, and Eric Burger for comments and corrections. + However, this doesn't necessarily mean that they endorse this + extension, agree with all details, or are responsible for errors + introduced by the editors. + +8. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional + STORE Operation or Quick Flag Changes Resynchronization", + RFC 4551, June 2006. + +9. Informative References + + [ANNOTATE] Daboo, C. and R. Gellens, "IMAP ANNOTATE Extension", Work + in Progress, August 2006. + + + + + + + + + + + + + + + + + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 5] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + +Editors' Addresses + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + Fax: +49 89 4502 9758 + EMail: arnt@oryx.com + + + Alexey Melnikov + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 6] + +RFC 5161 The IMAP ENABLE Extension March 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Gulbrandsen & Melnikov Standards Track [Page 7] + diff --git a/docs/rfcs/rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt b/docs/rfcs/rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt new file mode 100644 index 0000000..305c54f --- /dev/null +++ b/docs/rfcs/rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt @@ -0,0 +1,1291 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 5162 D. Cridland +Category: Standards Track Isode Ltd + C. Wilson + Nokia + March 2008 + + + IMAP4 Extensions for Quick Mailbox Resynchronization + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + This document defines an IMAP4 extension, which gives an IMAP client + the ability to quickly resynchronize any previously opened mailbox as + part of the SELECT command, without the need for server-side state or + additional client round-trips. This extension also introduces a new + response that allows for a more compact representation of a list of + expunged messages (and always includes the Unique Identifiers (UIDs) + expunged). + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov, et al. Standards Track [Page 1] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +Table of Contents + + 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 2 + 2. Requirements Notation . . . . . . . . . . . . . . . . . . . . 4 + 3. IMAP Protocol Changes . . . . . . . . . . . . . . . . . . . . 4 + 3.1. QRESYNC Parameter to SELECT/EXAMINE . . . . . . . . . . . 4 + 3.2. VANISHED UID FETCH Modifier . . . . . . . . . . . . . . . 8 + 3.3. EXPUNGE Command . . . . . . . . . . . . . . . . . . . . . 10 + 3.4. CLOSE Command . . . . . . . . . . . . . . . . . . . . . . 11 + 3.5. UID EXPUNGE Command . . . . . . . . . . . . . . . . . . . 11 + 3.6. VANISHED Response . . . . . . . . . . . . . . . . . . . . 12 + 3.7. CLOSED Response Code . . . . . . . . . . . . . . . . . . . 15 + 4. Server Implementation Considerations . . . . . . . . . . . . . 15 + 4.1. Server Implementations That Don't Store Extra State . . . 15 + 4.2. Server Implementations Storing Minimal State . . . . . . . 16 + 4.3. Additional State Required on the Server . . . . . . . . . 16 + 5. Updated Synchronization Sequence . . . . . . . . . . . . . . . 17 + 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 19 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 20 + 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 21 + 9. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 21 + 10. References . . . . . . . . . . . . . . . . . . . . . . . . . . 21 + 10.1. Normative References . . . . . . . . . . . . . . . . . . . 21 + 10.2. Informative References . . . . . . . . . . . . . . . . . . 22 + +1. Introduction and Overview + + The [CONDSTORE] extension gives a disconnected client the ability to + quickly resynchronize IMAP flag changes for previously seen messages. + This can be done using the CHANGEDSINCE FETCH modifier once a mailbox + is opened. In order for the client to discover which messages have + been expunged, the client still has to issue a UID FETCH or a UID + SEARCH command. This document defines an extension to [CONDSTORE] + that allows a reconnecting client to perform full resynchronization, + including discovery of expunged messages, in a single round-trip. + This extension also introduces a new response, VANISHED, that allows + for a more compact representation of a list of expunged messages. + + This extension can be useful for mobile clients that can experience + frequent disconnects caused by environmental factors (battery life, + signal strength, etc.). Such clients need a way to quickly reconnect + to the IMAP server, while minimizing delay experienced by the user as + well as the amount of traffic (and hence the expense) generated by + resynchronization. + + + + + + + +Melnikov, et al. Standards Track [Page 2] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + By extending the SELECT command to perform the additional + resynchronization, this also allows clients to reduce concurrent + connections to the IMAP server held purely for the sake of avoiding + the resynchronization. + + The quick resync IMAP extension is present if an IMAP4 server returns + "QRESYNC" as one of the supported capabilities to the CAPABILITY + command. + + Servers supporting this extension MUST implement and advertise + support for the [ENABLE] IMAP extension. Also, the presence of the + "QRESYNC" capability implies support for the [CONDSTORE] IMAP + extension even if the CONDSTORE capability isn't advertised. A + server compliant with this specification is REQUIREd to support + "ENABLE QRESYNC" and "ENABLE QRESYNC CONDSTORE" (which are "CONDSTORE + enabling commands", as defined in [CONDSTORE], and have identical + results), but there is no requirement for a compliant server to + support "ENABLE CONDSTORE" by itself. The "ENABLE QRESYNC"/"ENABLE + QRESYNC CONDSTORE" command also tells the server that it SHOULD start + sending VANISHED responses (see Section 3.6) instead of EXPUNGE + responses. This change remains in effect until the connection is + closed. + + For compatibility with clients that only support the [CONDSTORE] IMAP + extension, servers SHOULD advertise CONDSTORE in the CAPABILITY + response as well. + + A client making use of this extension MUST issue "ENABLE QRESYNC" + once it is authenticated. A server MUST respond with a tagged BAD + response if the QRESYNC parameter to the SELECT/EXAMINE command or + the VANISHED UID FETCH modifier is specified and the client hasn't + issued "ENABLE QRESYNC" in the current connection. + + This document puts additional requirements on a server implementing + the [CONDSTORE] extension. Each mailbox that supports persistent + storage of mod-sequences, i.e., for which the server has sent a + HIGHESTMODSEQ untagged OK response code on a successful SELECT/ + EXAMINE, MUST increment the per-mailbox mod-sequence when one or more + messages are expunged due to EXPUNGE, UID EXPUNGE or CLOSE; the + server MUST associate the incremented mod-sequence with the UIDs of + the expunged messages. + + A client that supports CONDSTORE but not this extension might + resynchronize a mailbox and discover that its HIGHESTMODSEQ has + increased from the value cached by the client. If the increase is + only due to messages having been expunged since the client last + synchronized, the client is likely to send a FETCH ... CHANGEDSINCE + command that returns no data. Thus, a client that supports CONDSTORE + + + +Melnikov, et al. Standards Track [Page 3] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + but not this extension might incur a penalty of an unneeded round- + trip when resynchronizing some mailboxes (those that have had + messages expunged but no flag changes since the last + synchronization). + + This extra round-trip is only incurred by clients that support + CONDSTORE but not this extension, and only when a mailbox has had + messages expunged but no flag changes to non-expunged messages. + Since CONDSTORE is a relatively new extension, it is thought likely + that clients that support it will also support this extension. + +2. Requirements Notation + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If a single "C:" or "S:" label applies to + multiple lines, then the line breaks between those lines are for + editorial clarity only and are not part of the actual protocol + exchange. The five characters [...] means that something has been + elided. + + Understanding of the IMAP message sequence numbers and UIDs and the + EXPUNGE response [RFC3501] is essential when reading this document. + +3. IMAP Protocol Changes + +3.1. QRESYNC Parameter to SELECT/EXAMINE + + The Quick Resynchronization parameter to SELECT/EXAMINE commands has + four arguments: + + o the last known UIDVALIDITY, + + o the last known modification sequence, + + o the optional set of known UIDs, and + + o an optional parenthesized list of known sequence ranges and their + corresponding UIDs. + + A server MUST respond with a tagged BAD response if the Quick + Resynchronization parameter to SELECT/EXAMINE command is specified + and the client hasn't issued "ENABLE QRESYNC" in the current + connection. + + + + +Melnikov, et al. Standards Track [Page 4] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Before opening the specified mailbox, the server verifies all + arguments for syntactic validity. If any parameter is not + syntactically valid, the server returns the tagged BAD response, and + the mailbox remains unselected. Once the check is done, the server + opens the mailbox as if no SELECT/EXAMINE parameters are specified + (this is subject to processing of other parameters as defined in + other extensions). In particular this means that the server MUST + send all untagged responses as specified in Sections 6.3.1 and 6.3.2 + of [RFC3501]. + + After that, the server checks the UIDVALIDITY value provided by the + client. If the provided UIDVALIDITY doesn't match the UIDVALIDITY + for the mailbox being opened, then the server MUST ignore the + remaining parameters and behave as if no dynamic message data + changed. The client can discover this situation by comparing the + UIDVALIDITY value returned by the server. This behavior allows the + client not to synchronize the mailbox or decide on the best + synchronization strategy. + + Example: Attempting to resynchronize INBOX, but the provided + UIDVALIDITY parameter doesn't match the current UIDVALIDITY + value. + + C: A02 SELECT INBOX (QRESYNC (67890007 20050715194045000 + 41,43:211,214:541)) + S: * 464 EXISTS + S: * 3 RECENT + S: * OK [UIDVALIDITY 3857529045] UIDVALIDITY + S: * OK [UIDNEXT 550] Predicted next UID + S: * OK [HIGHESTMODSEQ 90060128194045007] + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] Permanent flags + S: A02 OK [READ-WRITE] Sorry, UIDVALIDITY mismatch + + Modification Sequence and UID Parameters: + + A server that doesn't support the persistent storage of mod-sequences + for the mailbox MUST send the OK untagged response including the + NOMODSEQ response code with every successful SELECT or EXAMINE + command, as described in [CONDSTORE]. Such a server doesn't need to + remember mod-sequences for expunged messages in the mailbox. It MUST + ignore the remaining parameters and behave as if no dynamic message + data changed. + + If the provided UIDVALIDITY matches that of the selected mailbox, the + server then checks the last known modification sequence. + + + +Melnikov, et al. Standards Track [Page 5] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + The server sends the client any pending flag changes (using FETCH + responses that MUST contain UIDs) and expunges those that have + occurred in this mailbox since the provided modification sequence. + + If the list of known UIDs was also provided, the server should only + report flag changes and expunges for the specified messages. If the + client did not provide the list of UIDs, the server acts as if the + client has specified "1:", where is the mailbox's + UIDNEXT value minus 1. If the mailbox is empty and never had any + messages in it, then lack of the list of UIDs is interpreted as an + empty set of UIDs. + + Thus, the client can process just these pending events and need not + perform a full resynchronization. Without the message sequence + number matching information, the result of this step is semantically + equivalent to the client issuing: + tag1 UID FETCH "known-uids" (FLAGS) (CHANGEDSINCE + "mod-sequence-value" VANISHED) + + Example: + C: A03 SELECT INBOX (QRESYNC (67890007 + 90060115194045000 41,43:211,214:541)) + S: * OK [CLOSED] + S: * 314 EXISTS + S: * 15 RECENT + S: * OK [UIDVALIDITY 67890007] UIDVALIDITY + S: * OK [UIDNEXT 567] Predicted next UID + S: * OK [HIGHESTMODSEQ 90060115205545359] + S: * OK [UNSEEN 7] There are some unseen messages in the mailbox + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] Permanent flags + S: * VANISHED (EARLIER) 41,43:116,118,120:211,214:540 + S: * 49 FETCH (UID 117 FLAGS (\Seen \Answered) MODSEQ + (90060115194045001)) + S: * 50 FETCH (UID 119 FLAGS (\Draft $MDNSent) MODSEQ + (90060115194045308)) + S: ... + S: * 100 FETCH (UID 541 FLAGS (\Seen $Forwarded) MODSEQ + (90060115194045001)) + S: A03 OK [READ-WRITE] mailbox selected + + Message sequence match data: + + A client MAY provide a parenthesized list of a message sequence set + and the corresponding UID sets. Both MUST be provided in ascending + order. The server uses this data to restrict the range for which it + provides expunged message information. + + + +Melnikov, et al. Standards Track [Page 6] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Conceptually, the client provides a small sample of sequence numbers + for which it knows the corresponding UIDs. The server then compares + each sequence number and UID pair the client provides with the + current state of the mailbox. If a pair matches, then the client + knows of any expunges up to, and including, the message, and thus + will not include that range in the VANISHED response, even if the + "mod-sequence-value" provided by the client is too old for the server + to have data of when those messages were expunged. + + Thus, if the Nth message number in the first set in the list is 4, + and the Nth UID in the second set in the list is 8, and the mailbox's + fourth message has UID 8, then no UIDs equal to or less than 8 are + present in the VANISHED response. If the (N+1)th message number is + 12, and the (N+1)th UID is 24, and the (N+1)th message in the mailbox + has UID 25, then the lowest UID included in the VANISHED response + would be 9. + + In the following two examples, the server is unable to remember + expunges at all, and only UIDs with messages divisible by three are + present in the mailbox. In the first example, the client does not + use the fourth parameter; in the second, it provides it. This + example is somewhat extreme, but shows that judicious usage of the + sequence match data can save a substantial amount of bandwidth. + + Example: + C: A04 SELECT INBOX (QRESYNC (67890007 + 90060115194045000 1:29997)) + S: * 10003 EXISTS + S: * 5 RECENT + S: * OK [UIDVALIDITY 67890007] UIDVALIDITY + S: * OK [UIDNEXT 30013] Predicted next UID + S: * OK [HIGHESTMODSEQ 90060115205545359] + S: * OK [UNSEEN 7] There are some unseen messages in the mailbox + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] Permanent flags + S: * VANISHED (EARLIER) 1:2,4:5,7:8,10:11,13:14 [...] + 29998:29999,30001:30002,30004:30005,30007:30008 + S: * 9889 FETCH (UID 29667 FLAGS (\Seen \Answered) MODSEQ + (90060115194045027)) + S: * 9890 FETCH (UID 29670 FLAGS (\Draft $MDNSent) MODSEQ + (90060115194045028)) + S: ... + S: * 9999 FETCH (UID 29997 FLAGS (\Seen $Forwarded) MODSEQ + (90060115194045031)) + S: A04 OK [READ-WRITE] mailbox selected + + + + + +Melnikov, et al. Standards Track [Page 7] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Example: + C: B04 SELECT INBOX (QRESYNC (67890007 + 90060115194045000 1:29997 (5000,7500,9000,9990:9999 15000, + 22500,27000,29970,29973,29976,29979,29982,29985,29988,29991, + 29994,29997))) + S: * 10003 EXISTS + S: * 5 RECENT + S: * OK [UIDVALIDITY 67890007] UIDVALIDITY + S: * OK [UIDNEXT 30013] Predicted next UID + S: * OK [HIGHESTMODSEQ 90060115205545359] + S: * OK [UNSEEN 7] There are some unseen messages in the mailbox + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] Permanent flags + S: * VANISHED (EARLIER) 29998:29999,30001:30002,30004:30005,30007: + 30008 + S: * 9889 FETCH (UID 29667 FLAGS (\Seen \Answered) MODSEQ + (90060115194045027)) + S: * 9890 FETCH (UID 29670 FLAGS (\Draft $MDNSent) MODSEQ + (90060115194045028)) + S: ... + S: * 9999 FETCH (UID 29997 FLAGS (\Seen $Forwarded) MODSEQ + (90060115194045031)) + S: B04 OK [READ-WRITE] mailbox selected + +3.2. VANISHED UID FETCH Modifier + + [IMAPABNF] has extended the syntax of the FETCH and UID FETCH + commands to include an optional FETCH modifier. This document + defines a new UID FETCH modifier: VANISHED. + + Note, that the VANISHED UID FETCH modifier is NOT allowed with a + FETCH command. The server MUST return a tagged BAD response if this + response is specified as a modifier to the FETCH command. + + A server MUST respond with a tagged BAD response if the VANISHED UID + FETCH modifier is specified and the client hasn't issued "ENABLE + QRESYNC" in the current connection. + + The VANISHED UID FETCH modifier MUST only be specified together with + the CHANGEDSINCE UID FETCH modifier. + + The VANISHED UID FETCH modifier instructs the server to report those + messages from the UID set parameter that have been expunged and whose + associated mod-sequence is larger than the specified mod-sequence. + That is, the client requests to be informed of messages from the + specified set that were expunged since the specified mod-sequence. + Note that the mod-sequence(s) associated with these messages were + + + +Melnikov, et al. Standards Track [Page 8] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + updated when the messages were expunged (as described above). The + expunged messages are reported using the VANISHED response as + described in Section 3.6, which MUST contain the EARLIER tag. Any + VANISHED (EARLIER) responses MUST be returned before any FETCH + responses, as otherwise the client might get confused about how + message numbers map to UIDs. + + Note: A server that receives a mod-sequence smaller than , + where is the value of the smallest expunged mod-sequence + it remembers minus one, MUST behave as if it was requested to report + all expunged messages from the provided UID set parameter. + + Example 1: Without the VANISHED UID FETCH modifier, a CONDSTORE-aware + client [CONDSTORE] needs to issue separate commands to learn of flag + changes and expunged messages since the last synchronization: + + C: s100 UID FETCH 300:500 (FLAGS) (CHANGEDSINCE 12345) + S: * 1 FETCH (UID 404 MODSEQ (65402) FLAGS (\Seen)) + S: * 2 FETCH (UID 406 MODSEQ (75403) FLAGS (\Deleted)) + S: * 4 FETCH (UID 408 MODSEQ (29738) FLAGS ($NoJunk + $AutoJunk $MDNSent)) + S: s100 OK FETCH completed + C: s101 UID SEARCH 300:500 + S: * SEARCH 404 406 407 408 410 412 + S: s101 OK search completed + + Where 300 and 500 are the lowest and highest UIDs from client's + cache. The second SEARCH response tells the client that the messages + with UIDs 407, 410, and 412 are still present, but their flags + haven't changed since the specified modification sequence. + + Using the VANISHED UID FETCH modifier, it is sufficient to issue only + a single command: + + C: s100 UID FETCH 300:500 (FLAGS) (CHANGEDSINCE 12345 + VANISHED) + S: * VANISHED (EARLIER) 300:310,405,411 + S: * 1 FETCH (UID 404 MODSEQ (65402) FLAGS (\Seen)) + S: * 2 FETCH (UID 406 MODSEQ (75403) FLAGS (\Deleted)) + S: * 4 FETCH (UID 408 MODSEQ (29738) FLAGS ($NoJunk + $AutoJunk $MDNSent)) + S: s100 OK FETCH completed + + + + + + + + + +Melnikov, et al. Standards Track [Page 9] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +3.3. EXPUNGE Command + + Arguments: none + + Responses: untagged responses: EXPUNGE or VANISHED + + Result: OK - expunge completed + NO - expunge failure: can't expunge (e.g., permission denied) + BAD - command unknown or arguments invalid + + This section updates the definition of the EXPUNGE command described + in Section 6.4.3 of [RFC3501]. + + The EXPUNGE command permanently removes all messages that have the + \Deleted flag set from the currently selected mailbox. Before + returning an OK to the client, those messages that are removed are + reported using a VANISHED response or EXPUNGE responses. + + If the server is capable of storing modification sequences for the + selected mailbox, it MUST increment the per-mailbox mod-sequence if + at least one message was permanently removed due to the execution of + the EXPUNGE command. For each permanently removed message, the + server MUST remember the incremented mod-sequence and corresponding + UID. If at least one message got expunged, the server MUST send the + updated per-mailbox modification sequence using the HIGHESTMODSEQ + response code (defined in [CONDSTORE]) in the tagged OK response. + + Example: C: A202 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 5 EXPUNGE + S: * 8 EXPUNGE + S: A202 OK [HIGHESTMODSEQ 20010715194045319] expunged + + Note: In this example, messages 3, 4, 7, and 11 had the \Deleted flag + set. The first "* 3 EXPUNGE" reports message # 3 as expunged. The + second "* 3 EXPUNGE" reports message # 4 as expunged (the message + number got decremented due to the previous EXPUNGE response). See + the description of the EXPUNGE response in [RFC3501] for further + explanation. + + Note that if the server chooses to always send VANISHED responses + instead of EXPUNGE responses, the previous example might look like + this: + + Example: C: B202 EXPUNGE + S: * VANISHED 405,407,410,425 + S: B202 OK [HIGHESTMODSEQ 20010715194045319] expunged + + + +Melnikov, et al. Standards Track [Page 10] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Here messages with message numbers 3, 4, 7, and 11 have respective + UIDs 405, 407, 410, and 425. + +3.4. CLOSE Command + + Arguments: none + + Responses: no specific responses for this command + + Result: OK - close completed, now in authenticated state + BAD - command unknown or arguments invalid + + This section updates the definition of the CLOSE command described in + Section 6.4.2 of [RFC3501]. + + The CLOSE command permanently removes all messages that have the + \Deleted flag set from the currently selected mailbox, and returns to + the authenticated state from the selected state. No untagged EXPUNGE + (or VANISHED) responses are sent. + + If the server is capable of storing modification sequences for the + selected mailbox, it MUST increment the per-mailbox mod-sequence if + at least one message was permanently removed due to the execution of + the CLOSE command. For each permanently removed message, the server + MUST remember the incremented mod-sequence and corresponding UID. If + at least one message got expunged, the server MUST send the updated + per-mailbox modification sequence using the HIGHESTMODSEQ response + code (defined in [CONDSTORE]) in the tagged OK response. + + Example: C: A202 CLOSE + S: A202 OK [HIGHESTMODSEQ 20010715194045319] done + +3.5. UID EXPUNGE Command + + Arguments: message set + + Responses: untagged responses: EXPUNGE or VANISHED + + Result: OK - expunge completed + NO - expunge failure: can't expunge (e.g., permission denied) + BAD - command unknown or arguments invalid + + This section updates the definition of the UID EXPUNGE command + described in Section 2.1 of [UIDPLUS]. Servers that implement both + [UIDPLUS] and QRESYNC extensions must implement UID EXPUNGE as + described in this section. + + + + + +Melnikov, et al. Standards Track [Page 11] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + The UID EXPUNGE command permanently removes from the currently + selected mailbox all messages that both have the \Deleted flag set + and have a UID that is included in the specified message set. If a + message either does not have the \Deleted flag set or has a UID that + is not included in the specified message set, it is not affected. + + This command is particularly useful for disconnected mode clients. + By using UID EXPUNGE instead of EXPUNGE when resynchronizing with the + server, the client can avoid inadvertently removing any messages that + have been marked as \Deleted by other clients between the time that + the client was last connected and the time the client resynchronizes. + + Before returning an OK to the client, those messages that are removed + are reported using a VANISHED response or EXPUNGE responses. + + If the server is capable of storing modification sequences for the + selected mailbox, it MUST increment the per-mailbox mod-sequence if + at least one message was permanently removed due to the execution of + the UID EXPUNGE command. For each permanently removed message, the + server MUST remember the incremented mod-sequence and corresponding + UID. If at least one message got expunged, the server MUST send the + updated per-mailbox modification sequence using the HIGHESTMODSEQ + response code (defined in [CONDSTORE]) in the tagged OK response. + + Example: C: . UID EXPUNGE 3000:3002 + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: . OK [HIGHESTMODSEQ 20010715194045319] Ok + + Note: In this example, at least messages with message numbers 3, 4, + and 5 (UIDs 3000 to 3002) had the \Deleted flag set. The first "* 3 + EXPUNGE" reports message # 3 as expunged. The second "* 3 EXPUNGE" + reports message # 4 as expunged (the message number got decremented + due to the previous EXPUNGE response). See the description of the + EXPUNGE response in [RFC3501] for further explanation. + +3.6. VANISHED Response + + Contents: an optional EARLIER tag + + list of UIDs + + The VANISHED response reports that the specified UIDs have been + permanently removed from the mailbox. This response is similar to + the EXPUNGE response [RFC3501]; however, it can return information + about multiple messages, and it returns UIDs instead of message + + + + +Melnikov, et al. Standards Track [Page 12] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + numbers. The first benefit saves bandwidth, while the second is more + convenient for clients that only use UIDs to access the IMAP server. + + The VANISHED response has the same restrictions on when it can be + sent as does the EXPUNGE response (see below). + + The VANISHED response has two forms. The first form contains the + EARLIER tag, which signifies that the response was caused by a UID + FETCH (VANISHED) or a SELECT/EXAMINE (QRESYNC) command. This + response is sent if the UID set parameter to the UID FETCH (VANISHED) + command includes UIDs of messages that are no longer in the mailbox. + When the client sees a VANISHED EARLIER response, it MUST NOT + decrement message sequence numbers for each successive message in the + mailbox. + + The second form doesn't contain the EARLIER tag and is described + below. Once a client has issued "ENABLE QRESYNC", the server SHOULD + use the VANISHED response without the EARLIER tag instead of the + EXPUNGE response. The server SHOULD continue using VANISHED in lieu + of EXPUNGE for the duration of the connection. In particular, this + affects the EXPUNGE [RFC3501] and UID EXPUNGE [UIDPLUS] commands, as + well as messages expunged in other connections. Such a VANISHED + response MUST NOT contain the EARLIER tag. + + A VANISHED response sent because of an EXPUNGE or UID EXPUNGE command + or because messages were expunged in other connections (i.e., the + VANISHED response without the EARLIER tag) also decrements the number + of messages in the mailbox; it is not necessary for the server to + send an EXISTS response with the new value. It also decrements + message sequence numbers for each successive message in the mailbox + (see the example at the end of this section). Note that a VANISHED + response caused by EXPUNGE, UID EXPUNGE, or messages expunged in + other connections SHOULD only contain UIDs for messages expunged + since the last VANISHED/EXPUNGE response sent for the currently + opened mailbox or since the mailbox was opened. That is, servers + SHOULD NOT send UIDs for previously expunged messages, unless + explicitly requested to do so by the UID FETCH (VANISHED) command. + + Note that client implementors must take care to properly decrement + the number of messages in the mailbox even if a server violates this + last SHOULD or repeats the same UID multiple times in the returned + UID set. In general, this means that a client using this extension + should either avoid using message numbers entirely, or have a + complete mapping of UIDs to message sequence numbers for the selected + mailbox. + + + + + + +Melnikov, et al. Standards Track [Page 13] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Because clients handle the two different forms of the VANISHED + response differently, servers MUST NOT report UIDs resulting from a + UID FETCH (VANISHED) or a SELECT/EXAMINE (QRESYNC) in the same + VANISHED response as UIDs of messages expunged now (i.e., messages + expunged in other connections). Instead, the server MUST send + separate VANISHED responses: one with the EARLIER tag and one + without. + + A VANISHED response MUST NOT be sent when no command is in progress, + nor while responding to a FETCH, STORE, or SEARCH command. This rule + is necessary to prevent a loss of synchronization of message sequence + numbers between client and server. A command is not "in progress" + until the complete command has been received; in particular, a + command is not "in progress" during the negotiation of command + continuation. + + Note: UID FETCH, UID STORE, and UID SEARCH are different commands + from FETCH, STORE, and SEARCH. A VANISHED response MAY be sent + during a UID command. However, the VANISHED response MUST NOT be + sent during a UID SEARCH command that contains message numbers in the + search criteria. + + The update from the VANISHED response MUST be recorded by the client. + + Example: Let's assume that there is the following mapping between + message numbers and UIDs in the currently selected mailbox (here "X" + marks messages with the \Deleted flag set, and "x" represents UIDs + which are not relevant for the example): + + Message numbers: 1 2 3 4 5 6 7 8 9 10 11 + UIDs: x 504 505 507 508 x 510 x x x 625 + \Deleted messages: X X X X + + In the presence of the extension defined in this document: + + C: A202 EXPUNGE + S: * VANISHED 505,507,510,625 + S: A202 OK EXPUNGE completed + + Without the QRESYNC extension, the same example might look like: + + C: A202 EXPUNGE + S: * 3 EXPUNGE + S: * 3 EXPUNGE + S: * 5 EXPUNGE + S: * 8 EXPUNGE + S: A202 OK EXPUNGE completed + + + + +Melnikov, et al. Standards Track [Page 14] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + (Continuing previous example) If subsequently messages with UIDs 504 + and 508 got marked as \Deleted: + + C: A210 EXPUNGE + S: * VANISHED 504,508 + S: A210 OK EXPUNGE completed + + i.e., the last VANISHED response only contains UIDs of messages + expunged since the previous VANISHED response. + +3.7. CLOSED Response Code + + The CLOSED response code has no parameters. A server implementing + the extension defined in this document MUST return the CLOSED + response code when the currently selected mailbox is closed + implicitly using the SELECT/EXAMINE command on another mailbox. The + CLOSED response code serves as a boundary between responses for the + previously opened mailbox (which was closed) and the newly selected + mailbox: all responses before the CLOSED response code relate to the + mailbox that was closed, and all subsequent responses relate to the + newly opened mailbox. + + There is no need to return the CLOSED response code on completion of + the CLOSE or the UNSELECT [UNSELECT] command (or similar) whose + purpose is to close the currently selected mailbox without opening a + new one. + +4. Server Implementation Considerations + + This section describes a minimalist implementation, a moderate + implementation, and an example of a full implementation. + +4.1. Server Implementations That Don't Store Extra State + + Strictly speaking, a server implementation that doesn't remember mod- + sequences associated with expunged messages can be considered + compliant with this specification. Such implementations return all + expunged messages specified in the UID set of the UID FETCH + (VANISHED) command every time, without paying attention to the + specified CHANGEDSINCE mod-sequence. Such implementations are + discouraged, as they can end up returning VANISHED responses that are + bigger than the result of a UID SEARCH command for the same UID set. + + Clients that use the message sequence match data can reduce the scope + of this VANISHED response substantially in the typical case where + expunges have not happened, or happen only toward the end of the + mailbox. + + + + +Melnikov, et al. Standards Track [Page 15] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +4.2. Server Implementations Storing Minimal State + + A server that stores the HIGHESTMODSEQ value at the time of the last + EXPUNGE can omit the VANISHED response when a client provides a + MODSEQ value that is equal to, or higher than, the current value of + this datum, that is, when there have been no EXPUNGEs. + + A client providing message sequence match data can reduce the scope + as above. In the case where there have been no expunges, the server + can ignore this data. + +4.3. Additional State Required on the Server + + When compared to the [CONDSTORE] extension, this extension requires + servers to store additional state associated with expunged messages. + Note that implementations are not required to store this state in + persistent storage; however, use of persistent storage is advisable. + + One possible way to correctly implement the extension described in + this document is to store a queue of pairs. + can be represented as a sequence of + pairs. + + When messages are expunged, one or more entries are added to the + queue tail. + + When the server receives a request to return messages expunged since + a given mod-sequence, it will search the queue from the tail (i.e., + going from the highest expunged mod-sequence to the lowest) until it + sees the first record with a mod-sequence less than or equal to the + given mod-sequence or it reaches the head of the queue. + + Note that indefinitely storing information about expunged messages + can cause storage and related problems for an implementation. In the + worst case, this could result in almost 64Gb of storage for each IMAP + mailbox. For example, consider an implementation that stores triples for each range of messages + expunged at the same time. Each triple requires 16 octets: 4 octets + for each of the two UIDs, and 8 octets for the mod-sequence. Assume + that there is a mailbox containing a single message with a UID of + 2**32-1 (the maximum possible UID value), where messages had + previously existed with UIDs starting at 1, and have been expunged + one at a time. For this mailbox alone, storage is required for the + triples <1, 1, modseq1>, <2, 2, modseq2>, ..., <2**32-2, 2**32-2, + modseq4294967294>. + + + + + + +Melnikov, et al. Standards Track [Page 16] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + Hence, implementations are encouraged to adopt strategies to protect + against such storage problems, such as limiting the size of the queue + used to store mod-sequences for expunged messages and "expiring" + older records when this limit is reached. When the selected + implementation-specific queue limit is reached, the oldest record(s) + are deleted from the queue (note that such records are located at the + queue head). For all such "expired" records, the server needs to + store a single mod-sequence, which is the highest mod-sequence for + all "expired" expunged messages. + + Note that if the client provides the message sequence match data, + this can heavily reduce the data cost of sending a complete set of + missing UIDs; thus, reducing the problems for clients if a server is + unable to persist much of this queue. If the queue contains data + back to the requested mod-sequence, this data can be ignored. + + Also, note that if the UIDVALIDITY of the mailbox changes or if the + mailbox is deleted, then any state associated with expunged messages + doesn't need to be preserved and SHOULD be deleted. + +5. Updated Synchronization Sequence + + This section updates the description of optimized synchronization in + Section 6.1 of the [IMAP-DISC]. + + An advanced disconnected mail client should use the QRESYNC and + [CONDSTORE] extensions when they are supported by the server. The + client uses the value from the HIGHESTMODSEQ OK response code + received on mailbox opening to determine if it needs to + resynchronize. Once the synchronization is complete, it MUST cache + the received value (unless the mailbox UIDVALIDITY value has changed; + see below). The client MUST update its copy of the HIGHESTMODSEQ + value whenever the server sends a subsequent HIGHESTMODSEQ OK + response code. + + After completing a full synchronization, the client MUST also take + note of any unsolicited MODSEQ FETCH data items received from the + server. Whenever the client receives a tagged response to a command, + it calculates the highest value among all MODSEQ FETCH data items + received since the last tagged response. If this value is bigger + than the client's copy of the HIGHESTMODSEQ value, then the client + MUST use this value as its new HIGHESTMODSEQ value. + + Note: It is not safe to update the client's copy of the HIGHESTMODSEQ + value with a MODSEQ FETCH data item value as soon as it is received + because servers are not required to send MODSEQ FETCH data items in + increasing modseqence order. This can lead to the client missing + some changes in case of connectivity loss. + + + +Melnikov, et al. Standards Track [Page 17] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + When opening the mailbox for synchronization, the client uses the + QRESYNC parameter to the SELECT/EXAMINE command. The QRESYNC + parameter is followed by the UIDVALIDITY and mailbox HIGHESTMODSEQ + values, as known to the client. It can be optionally followed by the + set of UIDs, for example, if the client is only interested in partial + synchronization of the mailbox. The client may also transmit a list + containing its knowledge of message numbers. + + If the SELECT/EXAMINE command is successful, the client compares + UIDVALIDITY as described in step d)1) in Section 3 of the + [IMAP-DISC]. If the cached UIDVALIDITY value matches the one + returned by the server and the server also returns the HIGHESTMODSEQ + response code, then the server reports expunged messages and returns + flag changes for all messages specified by the client in the UID set + parameter (or for all messages in the mailbox, if the client omitted + the UID set parameter). At this point, the client is synchronized, + except for maybe the new messages. + + If upon a successful SELECT/EXAMINE (QRESYNC) command the client + receives a NOMODSEQ OK untagged response (instead of the + HIGHESTMODSEQ response code), it MUST remove the last known + HIGHESTMODSEQ value from its cache and follow the more general + instructions in Section 3 of the [IMAP-DISC]. + + At this point, the client is in sync with the server regarding old + messages. This client can now fetch information about new messages + (if requested by the user). + + Step d) ("Server-to-client synchronization") in Section 4 of the + [IMAP-DISC] in the presence of the QRESYNC & CONDSTORE extensions is + amended as follows: + + d) "Server-to-client synchronization" -- for each mailbox that + requires synchronization, do the following: + + 1a) Check the mailbox UIDVALIDITY (see Section 4.1 of the [IMAP-DISC] + for more details) after issuing SELECT/EXAMINE (QRESYNC) command. + + If the UIDVALIDITY value returned by the server differs, the + client MUST + + * empty the local cache of that mailbox; + + * "forget" the cached HIGHESTMODSEQ value for the mailbox; + + + + + + + +Melnikov, et al. Standards Track [Page 18] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + * remove any pending "actions" which refer to UIDs in that + mailbox. Note, this doesn't affect actions performed on + client generated fake UIDs (see Section 5 of the + [IMAP-DISC]); + + 2) Fetch the current "descriptors"; + + I) Discover new messages. + + 3) Fetch the bodies of any "interesting" messages that the client + doesn't already have. + + Example: The UIDVALIDITY value is the same, but the HIGHESTMODSEQ + value has changed on the server while the client was + offline: + + C: A142 SELECT INBOX (QRESYNC (3857529045 20010715194032001 1:198)) + S: * 172 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 12] Message 12 is first unseen + S: * OK [UIDVALIDITY 3857529045] UIDs valid + S: * OK [UIDNEXT 201] Predicted next UID + S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) + S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited + S: * OK [HIGHESTMODSEQ 20010715194045007] + S: * VANISHED (EARLIER) 1:5,7:8,10:15 + S: * 2 FETCH (UID 6 MODSEQ (20010715205008000) + FLAGS (\Deleted)) + S: * 5 FETCH (UID 9 MODSEQ (20010715195517000) + FLAGS ($NoJunk $AutoJunk $MDNSent)) + ... + S: A142 OK [READ-WRITE] SELECT completed + +6. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. + + Non-terminals referenced but not defined below are as defined by + [RFC3501], [CONDSTORE], or [IMAPABNF]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + + + + + +Melnikov, et al. Standards Track [Page 19] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + + capability =/ "QRESYNC" + + select-param = "QRESYNC" SP "(" uidvalidity SP + mod-sequence-value [SP known-uids] + [SP seq-match-data] ")" + ;; conforms to the generic select-param + ;; syntax defined in [IMAPABNF] + + seq-match-data = "(" known-sequence-set SP known-uid-set ")" + + uidvalidity = nz-number + + known-uids = sequence-set + ;; sequence of UIDs, "*" is not allowed + + known-sequence-set = sequence-set + ;; set of message numbers corresponding to + ;; the UIDs in known-uid-set, in ascending order. + ;; * is not allowed. + + known-uid-set = sequence-set + ;; set of UIDs corresponding to the messages in + ;; known-sequence-set, in ascending order. + ;; * is not allowed. + + message-data =/ expunged-resp + + expunged-resp = "VANISHED" [SP "(EARLIER)"] SP known-uids + + rexpunges-fetch-mod = "VANISHED" + ;; VANISHED UID FETCH modifier conforms + ;; to the fetch-modifier syntax + ;; defined in [IMAPABNF]. It is only + ;; allowed in the UID FETCH command. + + resp-text-code =/ "CLOSED" + +7. Security Considerations + + As always, it is important to thoroughly test clients and servers + implementing this extension, as it changes how the server reports + expunged messages to the client. + + Security considerations relevant to [CONDSTORE] are relevant to this + extension. + + This document doesn't raise any new security concerns not already + raised by [CONDSTORE] or [RFC3501]. + + + +Melnikov, et al. Standards Track [Page 20] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +8. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG approved experimental RFC. The registry is currently located + at: + + http://www.iana.org/assignments/imap4-capabilities + + This document defines the QRESYNC IMAP capability. IANA has added + this capability to the registry. + +9. Acknowledgments + + Thanks to Steve Hole, Cyrus Daboo, and Michael Wener for encouraging + creation of this document. + + Valuable comments, both in agreement and in dissent, were received + from Timo Sirainen, Michael Wener, Randall Gellens, Arnt Gulbrandsen, + Chris Newman, Peter Coates, Mark Crispin, Elwyn Davies, Dan Karp, + Eric Rescorla, and Mike Zraly. + + This document takes substantial text from [RFC3501] by Mark Crispin. + +10. References + +10.1. Normative References + + [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for + Conditional STORE Operation or Quick Flag Changes + Resynchronization", RFC 4551, June 2006. + + [ENABLE] Gulbrandsen, A., Ed. and A. Melnikov, Ed., "The IMAP + ENABLE Extension", RFC 5161, March 2008. + + [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to + IMAP4 ABNF", RFC 4466, April 2006. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - + UIDPLUS extension", RFC 4315, December 2005. + + + +Melnikov, et al. Standards Track [Page 21] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +10.2. Informative References + + [IMAP-DISC] Melnikov, A., Ed., "Synchronization Operations For + Disconnected Imap4 Clients", RFC 4549, June 2006. + + [UNSELECT] Melnikov, A., "Internet Message Access Protocol (IMAP) + UNSELECT command", RFC 3691, February 2004. + +Authors' Addresses + + Alexey Melnikov + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + Dave Cridland + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: dave.cridland@isode.com + + + Corby Wilson + Nokia + 5 Wayside Rd. + Burlington, MA 01803 + USA + + EMail: corby@computer.org + + + + + + + + + + + + + + +Melnikov, et al. Standards Track [Page 22] + +RFC 5162 IMAP Quick Mailbox Resync March 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Melnikov, et al. Standards Track [Page 23] + diff --git a/docs/rfcs/rfc5182.IMAP_extension_last_SEARCH_result.txt b/docs/rfcs/rfc5182.IMAP_extension_last_SEARCH_result.txt new file mode 100644 index 0000000..a7f9147 --- /dev/null +++ b/docs/rfcs/rfc5182.IMAP_extension_last_SEARCH_result.txt @@ -0,0 +1,731 @@ + + + + + + +Network Working Group A. Melnikov +Request for Comments: 5182 Isode Ltd. +Updates: 3501 March 2008 +Category: Standards Track + + + IMAP Extension for Referencing the Last SEARCH Result + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + Many IMAP clients use the result of a SEARCH command as the input to + perform another operation, for example, fetching the found messages, + deleting them, or copying them to another mailbox. + + This can be achieved using standard IMAP operations described in RFC + 3501; however, this would be suboptimal. The server will send the + list of found messages to the client; after that, the client will + have to parse the list, reformat it, and send it back to the server. + The client can't pipeline the SEARCH command with the subsequent + command, and, as a result, the server might not be able to perform + some optimizations. + + This document proposes an IMAP extension that allows a client to tell + a server to use the result of a SEARCH (or Unique Identifier (UID) + SEARCH) command as an input to any subsequent command. + +1. Introduction + + Many IMAP clients use the result of a SEARCH command as the input to + perform another operation, for example, fetching the found messages, + deleting them, or copying them to another mailbox. + + This document proposes an IMAP extension that allows a client to tell + a server to use the result of a SEARCH (or UID SEARCH) command as an + input to any subsequent command. + + The SEARCH result reference extension defines a new SEARCH result + option [IMAPABNF] "SAVE" that tells the server to remember the result + of the SEARCH or UID SEARCH command (as well as any command based on + SEARCH, e.g., SORT and THREAD [SORT]) and store it in an internal + + + +Melnikov Standards Track [Page 1] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + variable that we will reference as the "search result variable". The + client can use the "$" marker to reference the content of this + internal variable. The "$" marker can be used instead of message + sequence or UID sequence in order to indicate that the server should + substitute it with the list of messages from the search result + variable. Thus, the client can use the result of the latest + remembered SEARCH command as a parameter to another command. The + search result marker has several advantages: + + * it avoids wasted bandwidth and associated delay; + + * it allows the client to pipeline a SEARCH [IMAP4] command with a + subsequent FETCH/STORE/COPY/SEARCH [IMAP4] or UID EXPUNGE + [UIDPLUS] command; + + * the client doesn't need to spend time reformatting the result of + a SEARCH command into a message set used in the subsequent + command; + + * it allows the server to perform optimizations. For example, if + the server can execute several pipelined commands in parallel + (or out of order), presence of the search result marker can + allow the server to decide which commands may or may not be + executed out of order. + + In absence of any other SEARCH result option, the SAVE result option + also suppresses any SEARCH response that would have been otherwise + returned by the SEARCH command. + +1.1. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [KEYWORDS]. + + Explanatory comments in examples start with // and are not part of + the protocol. + + + + + + + + + + + +Melnikov Standards Track [Page 2] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +2. Overview + +2.1. Normative Description of the SEARCHRES Extension + + The SEARCH result reference extension described in this document is + present in any IMAP4 server implementation that returns "SEARCHRES" + as one of the supported capabilities in the CAPABILITY command + response. Any such server MUST also implement the [ESEARCH] + extension. + + Upon successful completion of a SELECT or an EXAMINE command (after + the tagged OK response), the current search result variable is reset + to the empty sequence. + + A successful SEARCH command with the SAVE result option sets the + value of the search result variable to the list of messages found in + the SEARCH command. For example, if no messages were found, the + search result variable will contain the empty list. + + Any of the following SEARCH commands MUST NOT change the search + result variable: + + o a SEARCH command that caused the server to return the BAD tagged + response, + + o a SEARCH command with no SAVE result option that caused the + server to return NO tagged response, + + o a successful SEARCH command with no SAVE result option. + + A SEARCH command with the SAVE result option that caused the server + to return the NO tagged response sets the value of the search result + variable to the empty sequence. + + When a message listed in the search result variable is EXPUNGEd, it + is automatically removed from the list. Implementors are reminded + that if the server stores the list as a list of message numbers, it + MUST automatically adjust them when notifying the client about + expunged messages, as described in Section 7.4.1 of [IMAP4]. + + If the server decides to send a new UIDVALIDITY value while the + mailbox is opened, this causes resetting of the search variable to + the empty list. + + + + + + + + +Melnikov Standards Track [Page 3] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + Note that even if the "$" marker contains the empty list of messages, + it must be treated by all commands accepting message sets as + parameters as a valid, but non-matching list of messages. For + example, the "FETCH $" command would return a tagged OK response and + no FETCH responses. See also the Example 5 below. + + Note that even if the "$" marker contains the empty list of messages, + it must be treated as a valid but non-matching list of messages, by + all commands that accept message sets as parameters. + + Implementation note: server implementors should note that "$" can + reference IMAP message sequences or UID sequences, depending on the + context where it is used. For example, the "$" marker can be set as + a result of a SEARCH (SAVE) command and used as a parameter to a UID + FETCH command (which accepts a UID sequence, not a message sequence), + or the "$" marker can be set as a result of a UID SEARCH (SAVE) + command and used as a parameter to a FETCH command (which accepts a + message sequence, not a UID sequence). + +2.2. Examples + + 1) The following example demonstrates how the client can use the + result of a SEARCH command to FETCH headers of interesting + messages: + + Example 1: + C: A282 SEARCH RETURN (SAVE) FLAGGED SINCE 1-Feb-1994 + NOT FROM "Smith" + S: A282 OK SEARCH completed, result saved + C: A283 FETCH $ (UID INTERNALDATE FLAGS RFC822.HEADER) + S: * 2 FETCH (UID 14 ... + S: * 84 FETCH (UID 100 ... + S: * 882 FETCH (UID 1115 ... + S: A283 OK completed + + The client can also pipeline the two commands: + + Example 2: + C: A282 SEARCH RETURN (SAVE) FLAGGED SINCE 1-Feb-1994 + NOT FROM "Smith" + C: A283 FETCH $ (UID INTERNALDATE FLAGS RFC822.HEADER) + S: A282 OK SEARCH completed + S: * 2 FETCH (UID 14 ... + S: * 84 FETCH (UID 100 ... + S: * 882 FETCH (UID 1115 ... + S: A283 OK completed + + + + + +Melnikov Standards Track [Page 4] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + 2) The following example demonstrates that the result of one SEARCH + command can be used as input to another SEARCH command: + + Example 3: + C: A300 SEARCH RETURN (SAVE) SINCE 1-Jan-2004 + NOT FROM "Smith" + S: A300 OK SEARCH completed + C: A301 UID SEARCH UID $ SMALLER 4096 + S: * SEARCH 17 900 901 + S: A301 OK completed + + Note that the second command in Example 3 can be replaced with: + C: A301 UID SEARCH $ SMALLER 4096 + and the result of the command would be the same. + + 3) The following example shows that the "$" + marker can be combined with other message numbers using the OR + SEARCH criterion. + + Example 4: + C: P282 SEARCH RETURN (SAVE) SINCE 1-Feb-1994 + NOT FROM "Smith" + S: P282 OK SEARCH completed + C: P283 SEARCH CHARSET UTF-8 (OR $ 1,3000:3021) TEXT {8} + C: YYYYYYYY + S: * SEARCH 882 1102 3003 3005 3006 + S: P283 OK completed + + Note: Since this document format is restricted to 7-bit ASCII text, + it is not possible to show actual UTF-8 data. The "YYYYYYYY" is a + placeholder for what would be 8 octets of 8-bit data in an actual + transaction. + + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 5] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + 4) The following example demonstrates that a failed SEARCH sets the + search result variable to the empty list. + + Example 5: + C: B282 SEARCH RETURN (SAVE) SINCE 1-Feb-1994 + NOT FROM "Smith" + S: B282 OK SEARCH completed + C: B283 SEARCH CHARSET KOI8-R (OR $ 1,3000:3021) TEXT {4} + C: XXXX + S: B283 NO [BADCHARSET UTF-8] KOI8-R is not supported + //After this command the saved result variable contains + //no messages. A client that wants to reissue the B283 + //SEARCH command with another CHARSET would have to reissue + //the B282 command as well. One possible workaround for + //this is to include the desired CHARSET parameter + //in the earliest SEARCH RETURN (SAVE) command in a + //sequence of related SEARCH commands. + //A better approach might be to always use CHARSET UTF-8 + //instead. + + Note: Since this document format is restricted to 7-bit ASCII text, + it is not possible to show actual KOI8-R data. The "XXXX" is a + placeholder for what would be 4 octets of 8-bit data in an actual + transaction. + + 5) The following example demonstrates that it is not an error to use + the "$" marker when it contains no messages. + + Example 6: + C: E282 SEARCH RETURN (SAVE) SINCE 28-Oct-2006 + NOT FROM "Eric" + C: E283 COPY $ "Other Messages" + //The "$" contains no messages + S: E282 OK SEARCH completed + S: E283 OK COPY completed, nothing copied + +2.3. Multiple Commands in Progress + + Use of a SEARCH RETURN (SAVE) command followed by a command using the + "$" marker creates direct dependency between the two commands. As + directed by Section 5.5 of [IMAP4], a server MUST execute the two + commands in the order they were received. (A server capable of + out-of-order execution can in some cases execute the two commands in + parallel, for example, if a SEARCH RETURN (SAVE) is followed by + "SEARCH $", the search criteria from the first command can be + directly substituted into the second command.) + + + + + +Melnikov Standards Track [Page 6] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + A client supporting this extension MAY pipeline a SEARCH RETURN + (SAVE) command with one or more command using the "$" marker, as long + as this doesn't create an ambiguity, as described in Section 5.5 of + [IMAP4]. + + Example 7: + C: F282 SEARCH RETURN (SAVE) KEYWORD $Junk + C: F283 COPY $ "Junk" + C: F284 STORE $ +FLAGS.Silent (\Deleted) + S: F282 OK SEARCH completed + S: F283 OK COPY completed + S: F284 OK STORE completed + + Example 8: + C: G282 SEARCH RETURN (SAVE) KEYWORD $Junk + C: G283 SEARCH RETURN (ALL) SINCE 28-Oct-2006 + FROM "Eric" + //The server can execute the two SEARCH commands + //in any order, as they don't have any dependency. + //Note that the second command is making use of + //the [ESEARCH] extension. + S: * ESEARCH (TAG "G283") ALL 3:15,27,29:103 + S: G283 OK SEARCH completed + S: G282 OK SEARCH completed + + The following example demonstrates that the result of the second + SEARCH always overrides the result of the first. + + Example 9: + C: H282 SEARCH RETURN (SAVE) KEYWORD $Junk + C: H283 SEARCH RETURN (SAVE) SINCE 28-Oct-2006 + FROM "Eric" + S: H282 OK SEARCH completed + S: H283 OK SEARCH completed + +2.4. Interaction with ESEARCH Extension + + Servers that implement the extension defined in this document MUST + implement [ESEARCH] and conform to additional requirements listed in + this section. + + The SAVE result option doesn't change whether the server would return + items corresponding to MIN, MAX, ALL, or COUNT [ESEARCH] result + options. + + + + + + + +Melnikov Standards Track [Page 7] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + When the SAVE result option is combined with the MIN or MAX [ESEARCH] + result option, and none of the other ESEARCH result options are + present, the corresponding MIN/MAX is returned (if the search result + is not empty), but the "$" marker would contain a single message as + returned in the MIN/MAX return item. + + If the SAVE result option is combined with both MIN and MAX result + options, and none of the other ESEARCH result options are present, + the "$" marker would contain one or two messages as returned in the + MIN/MAX return items. + + If the SAVE result option is combined with the ALL and/or COUNT + result option(s), the "$" marker would always contain all messages + found by the SEARCH or UID SEARCH command. (Note that the last rule + might affect ESEARCH implementations that optimize how the COUNT + result is constructed.) + + The following table summarizes the additional requirement on ESEARCH + server implementations described in this section. + + +----------------+-------------------+ + | Combination of | "$" marker value | + | Result option | | + +----------------+-------------------+ + | SAVE MIN | MIN | + +----------------+-------------------+ + | SAVE MAX | MAX | + +----------------+-------------------+ + | SAVE MIN MAX | MIN & MAX | + +----------------+-------------------+ + | SAVE * [m] | all found messages| + +----------------+-------------------+ + + where '*' means "ALL" and/or "COUNT" + '[m]' means optional "MIN" and/or "MAX" + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 8] + +RFC 5182 Last SEARCH Result Reference March 2008 + + + The following example demonstrates behavioral difference for + different combinations of ESEARCH result options. Explanatory + comments start with // and are not part of the protocol: + + Example 10: + C: C282 SEARCH RETURN (ALL) SINCE 12-Feb-2006 + NOT FROM "Smith" + S: * ESEARCH (TAG "C283") ALL 2,10:15,21 + //$ value hasn't changed + S: C282 OK SEARCH completed + + C: C283 SEARCH RETURN (ALL SAVE) SINCE 12-Feb-2006 + NOT FROM "Smith" + S: * ESEARCH (TAG "C283") ALL 2,10:15,21 + //$ value is 2,10:15,21 + S: C283 OK SEARCH completed + + C: C284 SEARCH RETURN (SAVE MIN) SINCE 12-Feb-2006 + NOT FROM "Smith" + S: * ESEARCH (TAG "C284") MIN 2 + //$ value is 2 + S: C284 OK SEARCH completed + + C: C285 SEARCH RETURN (MAX SAVE MIN) SINCE + 12-Feb-2006 NOT FROM "Smith" + S: * ESEARCH (TAG "C285") MIN 2 MAX 21 + //$ value is 2,21 + S: C285 OK SEARCH completed + + C: C286 SEARCH RETURN (MAX SAVE MIN COUNT) + SINCE 12-Feb-2006 NOT FROM "Smith" + S: * ESEARCH (TAG "C286") MIN 2 MAX 21 COUNT 8 + //$ value is 2,10:15,21 + S: C286 OK SEARCH completed + + C: C286 SEARCH RETURN (ALL SAVE MIN) SINCE + 12-Feb-2006 NOT FROM "Smith" + S: * ESEARCH (TAG "C286") MIN 2 ALL 2,10:15,21 + //$ value is 2,10:15,21 + S: C286 OK SEARCH completed + + + + + + + + + + + +Melnikov Standards Track [Page 9] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +2.5. Refusing to Save Search Results + + In some cases, the server MAY refuse to save a SEARCH (SAVE) result, + for example, if an internal limit on the number of saved results is + reached. + + In this case, the server MUST return a tagged NO response containing + the NOTSAVED response code and set the search result variable to the + empty sequence, as described in Section 2.1. + +3. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [ABNF]. Non-terminals + referenced but not defined below are as defined in [IMAP4] or + [IMAPABNF]. + + Except as noted otherwise, all alphabetic characters are + case-insensitive. The use of upper- or lower-case characters to + define token strings is for editorial clarity only. Implementations + MUST accept these strings in a case-insensitive fashion. + + capability =/ "SEARCHRES" + ;; capability is defined in [IMAP4] + + sequence-set =/ seq-last-command + ;; extends sequence-set to allow for + ;; "result of the last command" indicator. + + seq-last-command = "$" + + search-return-opt = "SAVE" + ;; conforms to generic search-return-opt + ;; syntax defined in [IMAPABNF] + + resp-text-code =/ "NOTSAVED" + ;; from [IMAP4] + + + + + + + + + + + + + + +Melnikov Standards Track [Page 10] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +4. Security Considerations + + This extension requires the server to keep additional state, that may + be used to simplify Denial of Service attacks. In order to minimize + damage from such attacks, server implementations MAY limit the number + of saved searches they allow across all connections at any given time + and return the tagged NO response containing the NOTSAVED response + code (see Section 2.5) to a SEARCH RETURN (SAVE) command when this + limit is exceeded. + + Apart from that, it is believed that this extension doesn't raise any + additional security concerns not already discussed in [IMAP4]. + +5. IANA Considerations + + This document defines the "SEARCHRES" IMAP capability. IANA has + added it to the IMAP4 Capabilities Registry, which is currently + located at: + + http://www.iana.org/assignments/imap4-capabilities + +6. Acknowledgments + + The author would like to thank Mark Crispin, Cyrus Daboo, and Curtis + King for remembering that this document had to be written, as well as + for comments and corrections received. + + The author would also like to thank Dave Cridland, Mark Crispin, + Chris Newman, Dan Karp, and Spencer Dawkins for comments and + corrections received. + + Valuable comments, both in agreement and in dissent, were received + from Arnt Gulbrandsen. + + + + + + + + + + + + + + + + + + +Melnikov Standards Track [Page 11] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +7. References + +7.1. Normative References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [ESEARCH] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH + Command for Controlling What Kind of Information Is + Returned", RFC 4731, November 2006. + +7.2. Informative References + + [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - + UIDPLUS extension", RFC 4315, December 2005. + + [SORT] Crispin, M. and K. Murchison, "INTERNET MESSAGE ACCESS + PROTOCOL - SORT AND THREAD EXTENSIONS", Work in Progress, + Septemeber 2007. + +Author's Address + + Alexey Melnikov + Isode Ltd. + 5 Castle Business Village, + 36 Station Road, + Hampton, Middlesex, + TW12 2BX, United Kingdom + + EMail: Alexey.Melnikov@isode.com + + + + + + + + + + + +Melnikov Standards Track [Page 12] + +RFC 5182 Last SEARCH Result Reference March 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Melnikov Standards Track [Page 13] + diff --git a/docs/rfcs/rfc5182.Sieve_and_extensions.txt b/docs/rfcs/rfc5182.Sieve_and_extensions.txt new file mode 100644 index 0000000..e5a02c6 --- /dev/null +++ b/docs/rfcs/rfc5182.Sieve_and_extensions.txt @@ -0,0 +1,2355 @@ + + + + + + +Network Working Group P. Guenther, Ed. +Request for Comments: 5228 Sendmail, Inc. +Obsoletes: 3028 T. Showalter, Ed. +Category: Standards Track January 2008 + + + Sieve: An Email Filtering Language + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + This document describes a language for filtering email messages at + time of final delivery. It is designed to be implementable on either + a mail client or mail server. It is meant to be extensible, simple, + and independent of access protocol, mail architecture, and operating + system. It is suitable for running on a mail server where users may + not be allowed to execute arbitrary programs, such as on black box + Internet Message Access Protocol (IMAP) servers, as the base language + has no variables, loops, or ability to shell out to external + programs. + + + + + + + + + + + + + + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 1] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +Table of Contents + + 1. Introduction ....................................................4 + 1.1. Conventions Used in This Document ..........................4 + 1.2. Example Mail Messages ......................................5 + 2. Design ..........................................................6 + 2.1. Form of the Language .......................................6 + 2.2. Whitespace .................................................7 + 2.3. Comments ...................................................7 + 2.4. Literal Data ...............................................7 + 2.4.1. Numbers .............................................7 + 2.4.2. Strings .............................................8 + 2.4.2.1. String Lists ...............................9 + 2.4.2.2. Headers ....................................9 + 2.4.2.3. Addresses .................................10 + 2.4.2.4. Encoding Characters Using + "encoded-character" .......................10 + 2.5. Tests .....................................................11 + 2.5.1. Test Lists .........................................12 + 2.6. Arguments .................................................12 + 2.6.1. Positional Arguments ...............................12 + 2.6.2. Tagged Arguments ...................................12 + 2.6.3. Optional Arguments .................................13 + 2.6.4. Types of Arguments .................................13 + 2.7. String Comparison .........................................13 + 2.7.1. Match Type .........................................14 + 2.7.2. Comparisons across Character Sets ..................15 + 2.7.3. Comparators ........................................15 + 2.7.4. Comparisons against Addresses ......................16 + 2.8. Blocks ....................................................17 + 2.9. Commands ..................................................17 + 2.10. Evaluation ...............................................18 + 2.10.1. Action Interaction ................................18 + 2.10.2. Implicit Keep .....................................18 + 2.10.3. Message Uniqueness in a Mailbox ...................19 + 2.10.4. Limits on Numbers of Actions ......................19 + 2.10.5. Extensions and Optional Features ..................19 + 2.10.6. Errors ............................................20 + 2.10.7. Limits on Execution ...............................20 + 3. Control Commands ...............................................21 + 3.1. Control if ................................................21 + 3.2. Control require ...........................................22 + 3.3. Control stop ..............................................22 + 4. Action Commands ................................................23 + 4.1. Action fileinto ...........................................23 + 4.2. Action redirect ...........................................23 + 4.3. Action keep ...............................................24 + 4.4. Action discard ............................................25 + + + +Guenther & Showalter Standards Track [Page 2] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + 5. Test Commands ..................................................26 + 5.1. Test address ..............................................26 + 5.2. Test allof ................................................27 + 5.3. Test anyof ................................................27 + 5.4. Test envelope .............................................27 + 5.5. Test exists ...............................................28 + 5.6. Test false ................................................28 + 5.7. Test header ...............................................29 + 5.8. Test not ..................................................29 + 5.9. Test size .................................................29 + 5.10. Test true ................................................30 + 6. Extensibility ..................................................30 + 6.1. Capability String .........................................31 + 6.2. IANA Considerations .......................................31 + 6.2.1. Template for Capability Registrations ..............32 + 6.2.2. Handling of Existing Capability Registrations ......32 + 6.2.3. Initial Capability Registrations ...................32 + 6.3. Capability Transport ......................................33 + 7. Transmission ...................................................33 + 8. Parsing ........................................................34 + 8.1. Lexical Tokens ............................................34 + 8.2. Grammar ...................................................36 + 8.3. Statement Elements ........................................36 + 9. Extended Example ...............................................37 + 10. Security Considerations .......................................38 + 11. Acknowledgments ...............................................39 + 12. Normative References ..........................................39 + 13. Informative References ........................................40 + 14. Changes from RFC 3028 .........................................41 + + + + + + + + + + + + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 3] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +1. Introduction + + This memo documents a language that can be used to create filters for + electronic mail. It is not tied to any particular operating system + or mail architecture. It requires the use of [IMAIL]-compliant + messages, but should otherwise generalize to many systems. + + The language is powerful enough to be useful but limited in order to + allow for a safe server-side filtering system. The intention is to + make it impossible for users to do anything more complex (and + dangerous) than write simple mail filters, along with facilitating + the use of graphical user interfaces (GUIs) for filter creation and + manipulation. The base language was not designed to be Turing- + complete: it does not have a loop control structure or functions. + + Scripts written in Sieve are executed during final delivery, when the + message is moved to the user-accessible mailbox. In systems where + the Mail Transfer Agent (MTA) does final delivery, such as + traditional Unix mail, it is reasonable to filter when the MTA + deposits mail into the user's mailbox. + + There are a number of reasons to use a filtering system. Mail + traffic for most users has been increasing due to increased usage of + email, the emergence of unsolicited email as a form of advertising, + and increased usage of mailing lists. + + Experience at Carnegie Mellon has shown that if a filtering system is + made available to users, many will make use of it in order to file + messages from specific users or mailing lists. However, many others + did not make use of the Andrew system's FLAMES filtering language + [FLAMES] due to difficulty in setting it up. + + Because of the expectation that users will make use of filtering if + it is offered and easy to use, this language has been made simple + enough to allow many users to make use of it, but rich enough that it + can be used productively. However, it is expected that GUI-based + editors will be the preferred way of editing filters for a large + number of users. + +1.1. Conventions Used in This Document + + In the sections of this document that discuss the requirements of + various keywords and operators, the following conventions have been + adopted. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [KEYWORDS]. + + + +Guenther & Showalter Standards Track [Page 4] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Each section on a command (test, action, or control) has a line + labeled "Usage:". This line describes the usage of the command, + including its name and its arguments. Required arguments are listed + inside angle brackets ("<" and ">"). Optional arguments are listed + inside square brackets ("[" and "]"). Each argument is followed by + its type, so "" represents an argument called "key" that + is a string. Literal strings are represented with double-quoted + strings. Alternatives are separated with slashes, and parentheses + are used for grouping, similar to [ABNF]. + + In the "Usage:" line, there are three special pieces of syntax that + are frequently repeated, MATCH-TYPE, COMPARATOR, and ADDRESS-PART. + These are discussed in sections 2.7.1, 2.7.3, and 2.7.4, + respectively. + + The formal grammar for these commands is defined in section 8 and is + the authoritative reference on how to construct commands, but the + formal grammar does not specify the order, semantics, number or types + of arguments to commands, or the legal command names. The intent is + to allow for extension without changing the grammar. + +1.2. Example Mail Messages + + The following mail messages will be used throughout this document in + examples. + + Message A + ----------------------------------------------------------- + Date: Tue, 1 Apr 1997 09:06:31 -0800 (PST) + From: coyote@desert.example.org + To: roadrunner@acme.example.com + Subject: I have a present for you + + Look, I'm sorry about the whole anvil thing, and I really + didn't mean to try and drop it on you from the top of the + cliff. I want to try to make it up to you. I've got some + great birdseed over here at my place--top of the line + stuff--and if you come by, I'll have it all wrapped up + for you. I'm really sorry for all the problems I've caused + for you over the years, but I know we can work this out. + -- + Wile E. Coyote "Super Genius" coyote@desert.example.org + ----------------------------------------------------------- + + + + + + + + +Guenther & Showalter Standards Track [Page 5] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Message B + ----------------------------------------------------------- + From: youcouldberich!@reply-by-postal-mail.invalid + Sender: b1ff@de.res.example.com + To: rube@landru.example.com + Date: Mon, 31 Mar 1997 18:26:10 -0800 + Subject: $$$ YOU, TOO, CAN BE A MILLIONAIRE! $$$ + + YOU MAY HAVE ALREADY WON TEN MILLION DOLLARS, BUT I DOUBT + IT! SO JUST POST THIS TO SIX HUNDRED NEWSGROUPS! IT WILL + GUARANTEE THAT YOU GET AT LEAST FIVE RESPONSES WITH MONEY! + MONEY! MONEY! COLD HARD CASH! YOU WILL RECEIVE OVER + $20,000 IN LESS THAN TWO MONTHS! AND IT'S LEGAL!!!!!!!!! + !!!!!!!!!!!!!!!!!!111111111!!!!!!!11111111111!!1 JUST + SEND $5 IN SMALL, UNMARKED BILLS TO THE ADDRESSES BELOW! + ----------------------------------------------------------- + +2. Design + +2.1. Form of the Language + + The language consists of a set of commands. Each command consists of + a set of tokens delimited by whitespace. The command identifier is + the first token and it is followed by zero or more argument tokens. + Arguments may be literal data, tags, blocks of commands, or test + commands. + + With the exceptions of strings and comments, the language is limited + to US-ASCII characters. Strings and comments may contain octets + outside the US-ASCII range. Specifically, they will normally be in + UTF-8, as specified in [UTF-8]. NUL (US-ASCII 0) is never permitted + in scripts, while CR and LF can only appear as the CRLF line ending. + + Note: While this specification permits arbitrary octets to appear + in Sieve scripts inside strings and comments, this has made it + difficult to robustly handle Sieve scripts in programs that are + sensitive to the encodings used. The "encoded-character" + capability (section 2.4.2.4) provides an alternative means of + representing such octets in strings using just US-ASCII + characters. As such, the use of non-UTF-8 text in scripts should + be considered a deprecated feature that may be abandoned. + + Tokens other than strings are considered case-insensitive. + + + + + + + + +Guenther & Showalter Standards Track [Page 6] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.2. Whitespace + + Whitespace is used to separate tokens. Whitespace is made up of + tabs, newlines (CRLF, never just CR or LF), and the space character. + The amount of whitespace used is not significant. + +2.3. Comments + + Two types of comments are offered. Comments are semantically + equivalent to whitespace and can be used anyplace that whitespace is + (with one exception in multi-line strings, as described in the + grammar). + + Hash comments begin with a "#" character that is not contained within + a string and continue until the next CRLF. + + Example: if size :over 100k { # this is a comment + discard; + } + + Bracketed comments begin with the token "/*" and end with "*/" + outside of a string. Bracketed comments may span multiple lines. + Bracketed comments do not nest. + + Example: if size :over 100K { /* this is a comment + this is still a comment */ discard /* this is a comment + */ ; + } + +2.4. Literal Data + + Literal data means data that is not executed, merely evaluated "as + is", to be used as arguments to commands. Literal data is limited to + numbers, strings, and string lists. + +2.4.1. Numbers + + Numbers are given as ordinary decimal numbers. As a shorthand for + expressing larger values, such as message sizes, a suffix of "K", + "M", or "G" MAY be appended to indicate a multiple of a power of two. + To be comparable with the power-of-two-based versions of SI units + that computers frequently use, "K" specifies kibi-, or 1,024 (2^10) + times the value of the number; "M" specifies mebi-, or 1,048,576 + (2^20) times the value of the number; and "G" specifies gibi-, or + 1,073,741,824 (2^30) times the value of the number [BINARY-SI]. + + + + + + +Guenther & Showalter Standards Track [Page 7] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Implementations MUST support integer values in the inclusive range + zero to 2,147,483,647 (2^31 - 1), but MAY support larger values. + + Only non-negative integers are permitted by this specification. + +2.4.2. Strings + + Scripts involve large numbers of string values as they are used for + pattern matching, addresses, textual bodies, etc. Typically, short + quoted strings suffice for most uses, but a more convenient form is + provided for longer strings such as bodies of messages. + + A quoted string starts and ends with a single double quote (the <"> + character, US-ASCII 34). A backslash ("\", US-ASCII 92) inside of a + quoted string is followed by either another backslash or a double + quote. These two-character sequences represent a single backslash or + double quote within the value, respectively. + + Scripts SHOULD NOT escape other characters with a backslash. + + An undefined escape sequence (such as "\a" in a context where "a" has + no special meaning) is interpreted as if there were no backslash (in + this case, "\a" is just "a"), though that may be changed by + extensions. + + Non-printing characters such as tabs, CRLF, and control characters + are permitted in quoted strings. Quoted strings MAY span multiple + lines. An unencoded NUL (US-ASCII 0) is not allowed in strings; see + section 2.4.2.4 for how it can be encoded. + + As message header data is converted to [UTF-8] for comparison (see + section 2.7.2), most string values will use the UTF-8 encoding. + However, implementations MUST accept all strings that match the + grammar in section 8. The ability to use non-UTF-8 encoded strings + matches existing practice and has proven to be useful both in tests + for invalid data and in arguments containing raw MIME parts for + extension actions that generate outgoing messages. + + For entering larger amounts of text, such as an email message, a + multi-line form is allowed. It starts with the keyword "text:", + followed by a CRLF, and ends with the sequence of a CRLF, a single + period, and another CRLF. The CRLF before the final period is + considered part of the value. In order to allow the message to + contain lines with a single dot, lines are dot-stuffed. That is, + when composing a message body, an extra '.' is added before each line + that begins with a '.'. When the server interprets the script, these + extra dots are removed. Note that a line that begins with a dot + followed by a non-dot character is not interpreted as dot-stuffed; + + + +Guenther & Showalter Standards Track [Page 8] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + that is, ".foo" is interpreted as ".foo". However, because this is + potentially ambiguous, scripts SHOULD be properly dot-stuffed so such + lines do not appear. + + Note that a hashed comment or whitespace may occur in between the + "text:" and the CRLF, but not within the string itself. Bracketed + comments are not allowed here. + +2.4.2.1. String Lists + + When matching patterns, it is frequently convenient to match against + groups of strings instead of single strings. For this reason, a list + of strings is allowed in many tests, implying that if the test is + true using any one of the strings, then the test is true. + + For instance, the test 'header :contains ["To", "Cc"] + ["me@example.com", "me00@landru.example.com"]' is true if either a To + header or Cc header of the input message contains either of the email + addresses "me@example.com" or "me00@landru.example.com". + + Conversely, in any case where a list of strings is appropriate, a + single string is allowed without being a member of a list: it is + equivalent to a list with a single member. This means that the test + 'exists "To"' is equivalent to the test 'exists ["To"]'. + +2.4.2.2. Headers + + Headers are a subset of strings. In the Internet Message + Specification [IMAIL], each header line is allowed to have whitespace + nearly anywhere in the line, including after the field name and + before the subsequent colon. Extra spaces between the header name + and the ":" in a header field are ignored. + + A header name never contains a colon. The "From" header refers to a + line beginning "From:" (or "From :", etc.). No header will match + the string "From:" due to the trailing colon. + + Similarly, no header will match a syntactically invalid header name. + An implementation MUST NOT cause an error for syntactically invalid + header names in tests. + + Header lines are unfolded as described in [IMAIL] section 2.2.3. + Interpretation of header data SHOULD be done according to [MIME3] + section 6.2 (see section 2.7.2 below for details). + + + + + + + +Guenther & Showalter Standards Track [Page 9] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.4.2.3. Addresses + + A number of commands call for email addresses, which are also a + subset of strings. When these addresses are used in outbound + contexts, addresses must be compliant with [IMAIL], but are further + constrained within this document. Using the symbols defined in + [IMAIL], section 3, the syntax of an address is: + + sieve-address = addr-spec ; simple address + / phrase "<" addr-spec ">" ; name & addr-spec + + That is, routes and group syntax are not permitted. If multiple + addresses are required, use a string list. Named groups are not + permitted. + + It is an error for a script to execute an action with a value for use + as an outbound address that doesn't match the "sieve-address" syntax. + +2.4.2.4. Encoding Characters Using "encoded-character" + + When the "encoded-character" extension is in effect, certain + character sequences in strings are replaced by their decoded value. + This happens after escape sequences are interpreted and dot- + unstuffing has been done. Implementations SHOULD support "encoded- + character". + + Arbitrary octets can be embedded in strings by using the syntax + encoded-arb-octets. The sequence is replaced by the octets with the + hexadecimal values given by each hex-pair. + + blank = WSP / CRLF + encoded-arb-octets = "${hex:" hex-pair-seq "}" + hex-pair-seq = *blank hex-pair *(1*blank hex-pair) *blank + hex-pair = 1*2HEXDIG + + Where WSP and HEXDIG non-terminals are defined in Appendix B.1 of + [ABNF]. + + It may be inconvenient or undesirable to enter Unicode characters + verbatim, and for these cases the syntax encoded-unicode-char can be + used. The sequence is replaced by the UTF-8 encoding of the + specified Unicode characters, which are identified by the hexadecimal + value of unicode-hex. + + encoded-unicode-char = "${unicode:" unicode-hex-seq "}" + unicode-hex-seq = *blank unicode-hex + *(1*blank unicode-hex) *blank + unicode-hex = 1*HEXDIG + + + +Guenther & Showalter Standards Track [Page 10] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + It is an error for a script to use a hexadecimal value that isn't in + either the range 0 to D7FF or the range E000 to 10FFFF. (The range + D800 to DFFF is excluded as those character numbers are only used as + part of the UTF-16 encoding form and are not applicable to the UTF-8 + encoding that the syntax here represents.) + + Note: Implementations MUST NOT raise an error for an out-of-range + Unicode value unless the sequence containing it is well-formed + according to the grammar. + + The capability string for use with the require command is "encoded- + character". + + In the following script, message B is discarded, since the specified + test string is equivalent to "$$$". + + Example: require "encoded-character"; + if header :contains "Subject" "$${hex:24 24}" { + discard; + } + The following examples demonstrate valid and invalid encodings and + how they are handled: + + "$${hex:40}" -> "$@" + "${hex: 40 }" -> "@" + "${HEX: 40}" -> "@" + "${hex:40" -> "${hex:40" + "${hex:400}" -> "${hex:400}" + "${hex:4${hex:30}}" -> "${hex:40}" + "${unicode:40}" -> "@" + "${ unicode:40}" -> "${ unicode:40}" + "${UNICODE:40}" -> "@" + "${UnICoDE:0000040}" -> "@" + "${Unicode:40}" -> "@" + "${Unicode:Cool}" -> "${Unicode:Cool}" + "${unicode:200000}" -> error + "${Unicode:DF01} -> error + +2.5. Tests + + Tests are given as arguments to commands in order to control their + actions. In this document, tests are given to if/elsif to decide + which block of code is run. + + + + + + + + +Guenther & Showalter Standards Track [Page 11] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.5.1. Test Lists + + Some tests ("allof" and "anyof", which implement logical "and" and + logical "or", respectively) may require more than a single test as an + argument. The test-list syntax element provides a way of grouping + tests as a comma-separated list in parentheses. + + Example: if anyof (not exists ["From", "Date"], + header :contains "from" "fool@example.com") { + discard; + } + +2.6. Arguments + + In order to specify what to do, most commands take arguments. There + are three types of arguments: positional, tagged, and optional. + + It is an error for a script, on a single command, to use conflicting + arguments or to use a tagged or optional argument more than once. + +2.6.1. Positional Arguments + + Positional arguments are given to a command that discerns their + meaning based on their order. When a command takes positional + arguments, all positional arguments must be supplied and must be in + the order prescribed. + +2.6.2. Tagged Arguments + + This document provides for tagged arguments in the style of + CommonLISP. These are also similar to flags given to commands in + most command-line systems. + + A tagged argument is an argument for a command that begins with ":" + followed by a tag naming the argument, such as ":contains". This + argument means that zero or more of the next tokens have some + particular meaning depending on the argument. These next tokens may + be literal data, but they are never blocks. + + Tagged arguments are similar to positional arguments, except that + instead of the meaning being derived from the command, it is derived + from the tag. + + Tagged arguments must appear before positional arguments, but they + may appear in any order with other tagged arguments. For simplicity + of the specification, this is not expressed in the syntax definitions + + + + + +Guenther & Showalter Standards Track [Page 12] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + with commands, but they still may be reordered arbitrarily provided + they appear before positional arguments. Tagged arguments may be + mixed with optional arguments. + + Tagged arguments SHOULD NOT take tagged arguments as arguments. + +2.6.3. Optional Arguments + + Optional arguments are exactly like tagged arguments except that they + may be left out, in which case a default value is implied. Because + optional arguments tend to result in shorter scripts, they have been + used far more than tagged arguments. + + One particularly noteworthy case is the ":comparator" argument, which + allows the user to specify which comparator [COLLATION] will be used + to compare two strings, since different languages may impose + different orderings on UTF-8 [UTF-8] strings. + +2.6.4. Types of Arguments + + Abstractly, arguments may be literal data, tests, or blocks of + commands. In this way, an "if" control structure is merely a command + that happens to take a test and a block as arguments and may execute + the block of code. + + However, this abstraction is ambiguous from a parsing standpoint. + + The grammar in section 8.2 presents a parsable version of this: + Arguments are string lists (string-lists), numbers, and tags, which + may be followed by a test or a test list (test-list), which may be + followed by a block of commands. No more than one test or test list, + or more than one block of commands, may be used, and commands that + end with a block of commands do not end with semicolons. + +2.7. String Comparison + + When matching one string against another, there are a number of ways + of performing the match operation. These are accomplished with three + types of matches: an exact match, a substring match, and a wildcard + glob-style match. These are described below. + + In order to provide for matches between character sets and case + insensitivity, Sieve uses the comparators defined in the Internet + Application Protocol Collation Registry [COLLATION]. + + + + + + + +Guenther & Showalter Standards Track [Page 13] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + However, when a string represents the name of a header, the + comparator is never user-specified. Header comparisons are always + done with the "i;ascii-casemap" operator, i.e., case-insensitive + comparisons, because this is the way things are defined in the + message specification [IMAIL]. + +2.7.1. Match Type + + Commands that perform string comparisons may have an optional match + type argument. The three match types in this specification are + ":contains", ":is", and ":matches". + + The ":contains" match type describes a substring match. If the value + argument contains the key argument as a substring, the match is true. + For instance, the string "frobnitzm" contains "frob" and "nit", but + not "fbm". The empty key ("") is contained in all values. + + The ":is" match type describes an absolute match; if the contents of + the first string are absolutely the same as the contents of the + second string, they match. Only the string "frobnitzm" is the string + "frobnitzm". The empty key ("") only ":is" matches with the empty + value. + + The ":matches" match type specifies a wildcard match using the + characters "*" and "?"; the entire value must be matched. "*" + matches zero or more characters in the value and "?" matches a single + character in the value, where the comparator that is used (see + section 2.7.3) defines what a character is. For example, the + comparators "i;octet" and "i;ascii-casemap" define a character to be + a single octet, so "?" will always match exactly one octet when one + of those comparators is in use. In contrast, a Unicode-based + comparator would define a character to be any UTF-8 octet sequence + encoding one Unicode character and thus "?" may match more than one + octet. "?" and "*" may be escaped as "\\?" and "\\*" in strings to + match against themselves. The first backslash escapes the second + backslash; together, they escape the "*". This is awkward, but it is + commonplace in several programming languages that use globs and + regular expressions. + + In order to specify what type of match is supposed to happen, + commands that support matching take optional arguments ":matches", + ":is", and ":contains". Commands default to using ":is" matching if + no match type argument is supplied. Note that these modifiers + interact with comparators; in particular, only comparators that + support the "substring match" operation are suitable for matching + with ":contains" or ":matches". It is an error to use a comparator + with ":contains" or ":matches" that is not compatible with it. + + + + +Guenther & Showalter Standards Track [Page 14] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + It is an error to give more than one of these arguments to a given + command. + + For convenience, the "MATCH-TYPE" syntax element is defined here as + follows: + + Syntax: ":is" / ":contains" / ":matches" + +2.7.2. Comparisons across Character Sets + + Messages may involve a number of character sets. In order for + comparisons to work across character sets, implementations SHOULD + implement the following behavior: + + Comparisons are performed on octets. Implementations convert text + from header fields in all charsets [MIME3] to Unicode, encoded as + UTF-8, as input to the comparator (see section 2.7.3). + Implementations MUST be capable of converting US-ASCII, ISO-8859- + 1, the US-ASCII subset of ISO-8859-* character sets, and UTF-8. + Text that the implementation cannot convert to Unicode for any + reason MAY be treated as plain US-ASCII (including any [MIME3] + syntax) or processed according to local conventions. An encoded + NUL octet (character zero) SHOULD NOT cause early termination of + the header content being compared against. + + If implementations fail to support the above behavior, they MUST + conform to the following: + + No two strings can be considered equal if one contains octets + greater than 127. + +2.7.3. Comparators + + In order to allow for language-independent, case-independent matches, + the match type may be coupled with a comparator name. The Internet + Application Protocol Collation Registry [COLLATION] provides the + framework for describing and naming comparators. + + All implementations MUST support the "i;octet" comparator (simply + compares octets) and the "i;ascii-casemap" comparator (which treats + uppercase and lowercase characters in the US-ASCII subset of UTF-8 as + the same). If left unspecified, the default is "i;ascii-casemap". + + Some comparators may not be usable with substring matches; that is, + they may only work with ":is". It is an error to try to use a + comparator with ":matches" or ":contains" that is not compatible with + it. + + + + +Guenther & Showalter Standards Track [Page 15] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Sieve treats a comparator result of "undefined" the same as a result + of "no-match". That is, this base specification does not provide any + means to directly detect invalid comparator input. + + A comparator is specified by the ":comparator" option with commands + that support matching. This option is followed by a string providing + the name of the comparator to be used. For convenience, the syntax + of a comparator is abbreviated to "COMPARATOR", and (repeated in + several tests) is as follows: + + Syntax: ":comparator" + + So in this example, + + Example: if header :contains :comparator "i;octet" "Subject" + "MAKE MONEY FAST" { + discard; + } + + would discard any message with subjects like "You can MAKE MONEY + FAST", but not "You can Make Money Fast", since the comparator used + is case-sensitive. + + Comparators other than "i;octet" and "i;ascii-casemap" must be + declared with require, as they are extensions. If a comparator + declared with require is not known, it is an error, and execution + fails. If the comparator is not declared with require, it is also an + error, even if the comparator is supported. (See section 2.10.5.) + + Both ":matches" and ":contains" match types are compatible with the + "i;octet" and "i;ascii-casemap" comparators and may be used with + them. + + It is an error to give more than one of these arguments to a given + command. + +2.7.4. Comparisons against Addresses + + Addresses are one of the most frequent things represented as strings. + These are structured, and being able to compare against the local- + part or the domain of an address is useful, so some tests that act + exclusively on addresses take an additional optional argument that + specifies what the test acts on. + + These optional arguments are ":localpart", ":domain", and ":all", + which act on the local-part (left side), the domain-part (right + side), and the whole address. + + + + +Guenther & Showalter Standards Track [Page 16] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + If an address is not syntactically valid, then it will not be matched + by tests specifying ":localpart" or ":domain". + + The kind of comparison done, such as whether or not the test done is + case-insensitive, is specified as a comparator argument to the test. + + If an optional address-part is omitted, the default is ":all". + + It is an error to give more than one of these arguments to a given + command. + + For convenience, the "ADDRESS-PART" syntax element is defined here as + follows: + + Syntax: ":localpart" / ":domain" / ":all" + +2.8. Blocks + + Blocks are sets of commands enclosed within curly braces and supplied + as the final argument to a command. Such a command is a control + structure: when executed it has control over the number of times the + commands in the block are executed. + + With the commands supplied in this memo, there are no loops. The + control structures supplied--if, elsif, and else--run a block either + once or not at all. + +2.9. Commands + + Sieve scripts are sequences of commands. Commands can take any of + the tokens above as arguments, and arguments may be either tagged or + positional arguments. Not all commands take all arguments. + + There are three kinds of commands: test commands, action commands, + and control commands. + + The simplest is an action command. An action command is an + identifier followed by zero or more arguments, terminated by a + semicolon. Action commands do not take tests or blocks as arguments. + The actions referenced in this document are: + + - keep, to save the message in the default location + - fileinto, to save the message in a specific mailbox + - redirect, to forward the message to another address + - discard, to silently throw away the message + + + + + + +Guenther & Showalter Standards Track [Page 17] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + A control command is a command that affects the parsing or the flow + of execution of the Sieve script in some way. A control structure is + a control command that ends with a block instead of a semicolon. + + A test command is used as part of a control command. It is used to + specify whether or not the block of code given to the control command + is executed. + +2.10. Evaluation + +2.10.1. Action Interaction + + Some actions cannot be used with other actions because the result + would be absurd. These restrictions are noted throughout this memo. + + Extension actions MUST state how they interact with actions defined + in this specification. + +2.10.2. Implicit Keep + + Previous experience with filtering systems suggests that cases tend + to be missed in scripts. To prevent errors, Sieve has an "implicit + keep". + + An implicit keep is a keep action (see section 4.3) performed in + absence of any action that cancels the implicit keep. + + An implicit keep is performed if a message is not written to a + mailbox, redirected to a new address, or explicitly thrown out. That + is, if a fileinto, a keep, a redirect, or a discard is performed, an + implicit keep is not. + + Some actions may be defined to not cancel the implicit keep. These + actions may not directly affect the delivery of a message, and are + used for their side effects. None of the actions specified in this + document meet that criteria, but extension actions may. + + For instance, with any of the short messages offered above, the + following script produces no actions. + + Example: if size :over 500K { discard; } + + As a result, the implicit keep is taken. + + + + + + + + +Guenther & Showalter Standards Track [Page 18] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.10.3. Message Uniqueness in a Mailbox + + Implementations SHOULD NOT deliver a message to the same mailbox more + than once, even if a script explicitly asks for a message to be + written to a mailbox twice. + + The test for equality of two messages is implementation-defined. + + If a script asks for a message to be written to a mailbox twice, it + MUST NOT be treated as an error. + +2.10.4. Limits on Numbers of Actions + + Site policy MAY limit the number of actions taken and MAY impose + restrictions on which actions can be used together. In the event + that a script hits a policy limit on the number of actions taken for + a particular message, an error occurs. + + Implementations MUST allow at least one keep or one fileinto. If + fileinto is not implemented, implementations MUST allow at least one + keep. + +2.10.5. Extensions and Optional Features + + Because of the differing capabilities of many mail systems, several + features of this specification are optional. Before any of these + extensions can be executed, they must be declared with the "require" + action. + + If an extension is not enabled with "require", implementations MUST + treat it as if they did not support it at all. This protects scripts + from having their behavior altered by extensions that the script + author might not have even been aware of. + + Implementations MUST NOT execute any Sieve script test or command + subsequent to "require" if one of the required extensions is + unavailable. + + Note: The reason for this restriction is that prior experiences + with languages such as LISP and Tcl suggest that this is a + workable way of noting that a given script uses an extension. + + Extensions that define actions MUST state how they interact with + actions discussed in the base specification. + + + + + + + +Guenther & Showalter Standards Track [Page 19] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +2.10.6. Errors + + In any programming language, there are compile-time and run-time + errors. + + Compile-time errors are ones in syntax that are detectable if a + syntax check is done. + + Run-time errors are not detectable until the script is run. This + includes transient failures like disk full conditions, but also + includes issues like invalid combinations of actions. + + When an error occurs in a Sieve script, all processing stops. + + Implementations MAY choose to do a full parse, then evaluate the + script, then do all actions. Implementations might even go so far as + to ensure that execution is atomic (either all actions are executed + or none are executed). + + Other implementations may choose to parse and run at the same time. + Such implementations are simpler, but have issues with partial + failure (some actions happen, others don't). + + Implementations MUST perform syntactic, semantic, and run-time checks + on code that is actually executed. Implementations MAY perform those + checks or any part of them on code that is not reached during + execution. + + When an error happens, implementations MUST notify the user that an + error occurred and which actions (if any) were taken, and do an + implicit keep. + +2.10.7. Limits on Execution + + Implementations may limit certain constructs. However, this + specification places a lower bound on some of these limits. + + Implementations MUST support fifteen levels of nested blocks. + + Implementations MUST support fifteen levels of nested test lists. + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 20] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +3. Control Commands + + Control structures are needed to allow for multiple and conditional + actions. + +3.1. Control if + + There are three pieces to if: "if", "elsif", and "else". Each is + actually a separate command in terms of the grammar. However, an + elsif or else MUST only follow an if or elsif. An error occurs if + these conditions are not met. + + Usage: if + + Usage: elsif + + Usage: else + + The semantics are similar to those of any of the many other + programming languages these control structures appear in. When the + interpreter sees an "if", it evaluates the test associated with it. + If the test is true, it executes the block associated with it. + + If the test of the "if" is false, it evaluates the test of the first + "elsif" (if any). If the test of "elsif" is true, it runs the + elsif's block. An elsif may be followed by an elsif, in which case, + the interpreter repeats this process until it runs out of elsifs. + + When the interpreter runs out of elsifs, there may be an "else" case. + If there is, and none of the if or elsif tests were true, the + interpreter runs the else's block. + + This provides a way of performing exactly one of the blocks in the + chain. + + In the following example, both messages A and B are dropped. + + Example: require "fileinto"; + if header :contains "from" "coyote" { + discard; + } elsif header :contains ["subject"] ["$$$"] { + discard; + } else { + fileinto "INBOX"; + } + + + + + + +Guenther & Showalter Standards Track [Page 21] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + When the script below is run over message A, it redirects the message + to acm@example.com; message B, to postmaster@example.com; any other + message is redirected to field@example.com. + + Example: if header :contains ["From"] ["coyote"] { + redirect "acm@example.com"; + } elsif header :contains "Subject" "$$$" { + redirect "postmaster@example.com"; + } else { + redirect "field@example.com"; + } + + Note that this definition prohibits the "... else if ..." sequence + used by C. This is intentional, because this construct produces a + shift-reduce conflict. + +3.2. Control require + + Usage: require + + The require action notes that a script makes use of a certain + extension. Such a declaration is required to use the extension, as + discussed in section 2.10.5. Multiple capabilities can be declared + with a single require. + + The require command, if present, MUST be used before anything other + than a require can be used. An error occurs if a require appears + after a command other than require. + + Example: require ["fileinto", "reject"]; + + Example: require "fileinto"; + require "vacation"; + +3.3. Control stop + + Usage: stop + + The "stop" action ends all processing. If the implicit keep has not + been cancelled, then it is taken. + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 22] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +4. Action Commands + + This document supplies four actions that may be taken on a message: + keep, fileinto, redirect, and discard. + + Implementations MUST support the "keep", "discard", and "redirect" + actions. + + Implementations SHOULD support "fileinto". + + Implementations MAY limit the number of certain actions taken (see + section 2.10.4). + +4.1. Action fileinto + + Usage: fileinto + + The "fileinto" action delivers the message into the specified + mailbox. Implementations SHOULD support fileinto, but in some + environments this may be impossible. Implementations MAY place + restrictions on mailbox names; use of an invalid mailbox name MAY be + treated as an error or result in delivery to an implementation- + defined mailbox. If the specified mailbox doesn't exist, the + implementation MAY treat it as an error, create the mailbox, or + deliver the message to an implementation-defined mailbox. If the + implementation uses a different encoding scheme than UTF-8 for + mailbox names, it SHOULD reencode the mailbox name from UTF-8 to its + encoding scheme. For example, the Internet Message Access Protocol + [IMAP] uses modified UTF-7, such that a mailbox argument of "odds & + ends" would appear in IMAP as "odds &- ends". + + The capability string for use with the require command is "fileinto". + + In the following script, message A is filed into mailbox + "INBOX.harassment". + + Example: require "fileinto"; + if header :contains ["from"] "coyote" { + fileinto "INBOX.harassment"; + } + +4.2. Action redirect + + Usage: redirect + + The "redirect" action is used to send the message to another user at + a supplied address, as a mail forwarding feature does. The + "redirect" action makes no changes to the message body or existing + + + +Guenther & Showalter Standards Track [Page 23] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + headers, but it may add new headers. In particular, existing + Received headers MUST be preserved and the count of Received headers + in the outgoing message MUST be larger than the same count on the + message as received by the implementation. (An implementation that + adds a Received header before processing the message does not need to + add another when redirecting.) + + The message is sent back out with the address from the redirect + command as an envelope recipient. Implementations MAY combine + separate redirects for a given message into a single submission with + multiple envelope recipients. (This is not a Mail User Agent (MUA)- + style forward, which creates a new message with a different sender + and message ID, wrapping the old message in a new one.) + + The envelope sender address on the outgoing message is chosen by the + sieve implementation. It MAY be copied from the message being + processed. However, if the message being processed has an empty + envelope sender address the outgoing message MUST also have an empty + envelope sender address. This last requirement is imposed to prevent + loops in the case where a message is redirected to an invalid address + when then returns a delivery status notification that also ends up + being redirected to the same invalid address. + + A simple script can be used for redirecting all mail: + + Example: redirect "bart@example.com"; + + Implementations MUST take measures to implement loop control, + possibly including adding headers to the message or counting Received + headers as specified in section 6.2 of [SMTP]. If an implementation + detects a loop, it causes an error. + + Implementations MUST provide means of limiting the number of + redirects a Sieve script can perform. See section 10 for more + details. + + Implementations MAY ignore a redirect action silently due to policy + reasons. For example, an implementation MAY choose not to redirect + to an address that is known to be undeliverable. Any ignored + redirect MUST NOT cancel the implicit keep. + +4.3. Action keep + + Usage: keep + + The "keep" action is whatever action is taken in lieu of all other + actions, if no filtering happens at all; generally, this simply means + to file the message into the user's main mailbox. This command + + + +Guenther & Showalter Standards Track [Page 24] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + provides a way to execute this action without needing to know the + name of the user's main mailbox, providing a way to call it without + needing to understand the user's setup or the underlying mail system. + + For instance, in an implementation where the IMAP server is running + scripts on behalf of the user at time of delivery, a keep command is + equivalent to a fileinto "INBOX". + + Example: if size :under 1M { keep; } else { discard; } + + Note that the above script is identical to the one below. + + Example: if not size :under 1M { discard; } + +4.4. Action discard + + Usage: discard + + Discard is used to silently throw away the message. It does so by + simply canceling the implicit keep. If discard is used with other + actions, the other actions still happen. Discard is compatible with + all other actions. (For instance, fileinto+discard is equivalent to + fileinto.) + + Discard MUST be silent; that is, it MUST NOT return a non-delivery + notification of any kind ([DSN], [MDN], or otherwise). + + In the following script, any mail from "idiot@example.com" is thrown + out. + + Example: if header :contains ["from"] ["idiot@example.com"] { + discard; + } + + While an important part of this language, "discard" has the potential + to create serious problems for users: Students who leave themselves + logged in to an unattended machine in a public computer lab may find + their script changed to just "discard". In order to protect users in + this situation (along with similar situations), implementations MAY + keep messages destroyed by a script for an indefinite period, and MAY + disallow scripts that throw out all mail. + + + + + + + + + + +Guenther & Showalter Standards Track [Page 25] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +5. Test Commands + + Tests are used in conditionals to decide which part(s) of the + conditional to execute. + + Implementations MUST support these tests: "address", "allof", + "anyof", "exists", "false", "header", "not", "size", and "true". + + Implementations SHOULD support the "envelope" test. + +5.1. Test address + + Usage: address [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] + + + The "address" test matches Internet addresses in structured headers + that contain addresses. It returns true if any header contains any + key in the specified part of the address, as modified by the + comparator and the match keyword. Whether there are other addresses + present in the header doesn't affect this test; this test does not + provide any way to determine whether an address is the only address + in a header. + + Like envelope and header, this test returns true if any combination + of the header-list and key-list arguments match and returns false + otherwise. + + Internet email addresses [IMAIL] have the somewhat awkward + characteristic that the local-part to the left of the at-sign is + considered case sensitive, and the domain-part to the right of the + at-sign is case insensitive. The "address" command does not deal + with this itself, but provides the ADDRESS-PART argument for allowing + users to deal with it. + + The address primitive never acts on the phrase part of an email + address or on comments within that address. It also never acts on + group names, although it does act on the addresses within the group + construct. + + Implementations MUST restrict the address test to headers that + contain addresses, but MUST include at least From, To, Cc, Bcc, + Sender, Resent-From, and Resent-To, and it SHOULD include any other + header that utilizes an "address-list" structured header body. + + Example: if address :is :all "from" "tim@example.com" { + discard; + } + + + + +Guenther & Showalter Standards Track [Page 26] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +5.2. Test allof + + Usage: allof + + The "allof" test performs a logical AND on the tests supplied to it. + + Example: allof (false, false) => false + allof (false, true) => false + allof (true, true) => true + + The allof test takes as its argument a test-list. + +5.3. Test anyof + + Usage: anyof + + The "anyof" test performs a logical OR on the tests supplied to it. + + Example: anyof (false, false) => false + anyof (false, true) => true + anyof (true, true) => true + +5.4. Test envelope + + Usage: envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] + + + The "envelope" test is true if the specified part of the [SMTP] (or + equivalent) envelope matches the specified key. This specification + defines the interpretation of the (case insensitive) "from" and "to" + envelope-parts. Additional envelope-parts may be defined by other + extensions; implementations SHOULD consider unknown envelope parts an + error. + + If one of the envelope-part strings is (case insensitive) "from", + then matching occurs against the FROM address used in the SMTP MAIL + command. The null reverse-path is matched against as the empty + string, regardless of the ADDRESS-PART argument specified. + + If one of the envelope-part strings is (case insensitive) "to", then + matching occurs against the TO address used in the SMTP RCPT command + that resulted in this message getting delivered to this user. Note + that only the most recent TO is available, and only the one relevant + to this user. + + The envelope-part is a string list and may contain more than one + parameter, in which case all of the strings specified in the key-list + are matched against all parts given in the envelope-part list. + + + +Guenther & Showalter Standards Track [Page 27] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Like address and header, this test returns true if any combination of + the envelope-part list and key-list arguments match and returns false + otherwise. + + All tests against envelopes MUST drop source routes. + + If the SMTP transaction involved several RCPT commands, only the data + from the RCPT command that caused delivery to this user is available + in the "to" part of the envelope. + + If a protocol other than SMTP is used for message transport, + implementations are expected to adapt this command appropriately. + + The envelope command is optional. Implementations SHOULD support it, + but the necessary information may not be available in all cases. The + capability string for use with the require command is "envelope". + + Example: require "envelope"; + if envelope :all :is "from" "tim@example.com" { + discard; + } + +5.5. Test exists + + Usage: exists + + The "exists" test is true if the headers listed in the header-names + argument exist within the message. All of the headers must exist or + the test is false. + + The following example throws out mail that doesn't have a From header + and a Date header. + + Example: if not exists ["From","Date"] { + discard; + } + +5.6. Test false + + Usage: false + + The "false" test always evaluates to false. + + + + + + + + + +Guenther & Showalter Standards Track [Page 28] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +5.7. Test header + + Usage: header [COMPARATOR] [MATCH-TYPE] + + + The "header" test evaluates to true if the value of any of the named + headers, ignoring leading and trailing whitespace, matches any key. + The type of match is specified by the optional match argument, which + defaults to ":is" if not specified, as specified in section 2.6. + + Like address and envelope, this test returns true if any combination + of the header-names list and key-list arguments match and returns + false otherwise. + + If a header listed in the header-names argument exists, it contains + the empty key (""). However, if the named header is not present, it + does not match any key, including the empty key. So if a message + contained the header + + X-Caffeine: C8H10N4O2 + + these tests on that header evaluate as follows: + + header :is ["X-Caffeine"] [""] => false + header :contains ["X-Caffeine"] [""] => true + + Testing whether a given header is either absent or doesn't contain + any non-whitespace characters can be done using a negated "header" + test: + + not header :matches "Cc" "?*" + +5.8. Test not + + Usage: not + + The "not" test takes some other test as an argument, and yields the + opposite result. "not false" evaluates to "true" and "not true" + evaluates to "false". + +5.9. Test size + + Usage: size <":over" / ":under"> + + The "size" test deals with the size of a message. It takes either a + tagged argument of ":over" or ":under", followed by a number + representing the size of the message. + + + + +Guenther & Showalter Standards Track [Page 29] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + If the argument is ":over", and the size of the message is greater + than the number provided, the test is true; otherwise, it is false. + + If the argument is ":under", and the size of the message is less than + the number provided, the test is true; otherwise, it is false. + + Exactly one of ":over" or ":under" must be specified, and anything + else is an error. + + The size of a message is defined to be the number of octets in the + [IMAIL] representation of the message. + + Note that for a message that is exactly 4,000 octets, the message is + neither ":over" nor ":under" 4000 octets. + +5.10. Test true + + Usage: true + + The "true" test always evaluates to true. + +6. Extensibility + + New control commands, actions, and tests can be added to the + language. Sites must make these features known to their users; this + document does not define a way to discover the list of extensions + supported by the server. + + Any extensions to this language MUST define a capability string that + uniquely identifies that extension. Capability string are case- + sensitive; for example, "foo" and "FOO" are different capabilities. + If a new version of an extension changes the functionality of a + previously defined extension, it MUST use a different name. + Extensions may register a set of related capabilities by registering + just a unique prefix for them. The "comparator-" prefix is an + example of this. The prefix MUST end with a "-" and MUST NOT overlap + any existing registrations. + + In a situation where there is a script submission protocol and an + extension advertisement mechanism aware of the details of this + language, scripts submitted can be checked against the mail server to + prevent use of an extension that the server does not support. + + Extensions MUST state how they interact with constraints defined in + section 2.10, e.g., whether they cancel the implicit keep, and which + actions they are compatible and incompatible with. Extensions MUST + NOT change the behavior of the "require" control command or alter the + interpretation of the argument to the "require" control. + + + +Guenther & Showalter Standards Track [Page 30] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Extensions that can submit new email messages or otherwise generate + new protocol requests MUST consider loop suppression, at least to + document any security considerations. + +6.1. Capability String + + Capability strings are typically short strings describing what + capabilities are supported by the server. + + Capability strings beginning with "vnd." represent vendor-defined + extensions. Such extensions are not defined by Internet standards or + RFCs, but are still registered with IANA in order to prevent + conflicts. Extensions starting with "vnd." SHOULD be followed by the + name of the vendor and product, such as "vnd.acme.rocket-sled". + + The following capability strings are defined by this document: + + encoded-character The string "encoded-character" indicates that the + implementation supports the interpretation of + "${hex:...}" and "${unicode:...}" in strings. + + envelope The string "envelope" indicates that the implementation + supports the "envelope" command. + + fileinto The string "fileinto" indicates that the implementation + supports the "fileinto" command. + + comparator- The string "comparator-elbonia" is provided if the + implementation supports the "elbonia" comparator. + Therefore, all implementations have at least the + "comparator-i;octet" and "comparator-i;ascii-casemap" + capabilities. However, these comparators may be used + without being declared with require. + +6.2. IANA Considerations + + In order to provide a standard set of extensions, a registry is + maintained by IANA. This registry contains both vendor-controlled + capability names (beginning with "vnd.") and IETF-controlled + capability names. Vendor-controlled capability names may be + registered on a first-come, first-served basis, by applying to IANA + with the form in the following section. Registration of capability + prefixes that do not begin with "vnd." REQUIRES a standards track or + IESG-approved experimental RFC. + + Extensions designed for interoperable use SHOULD use IETF-controlled + capability names. + + + + +Guenther & Showalter Standards Track [Page 31] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +6.2.1. Template for Capability Registrations + + The following template is to be used for registering new Sieve + extensions with IANA. + + To: iana@iana.org + Subject: Registration of new Sieve extension + + Capability name: [the string for use in the 'require' statement] + Description: [a brief description of what the extension adds + or changes] + RFC number: [for extensions published as RFCs] + Contact address: [email and/or physical address to contact for + additional information] + +6.2.2. Handling of Existing Capability Registrations + + In order to bring the existing capability registrations in line with + the new template, IANA has modified each as follows: + + 1. The "capability name" and "capability arguments" fields have been + eliminated + 2. The "capability keyword" field have been renamed to "Capability + name" + 3. An empty "Description" field has been added + 4. The "Standards Track/IESG-approved experimental RFC number" field + has been renamed to "RFC number" + 5. The "Person and email address to contact for further information" + field should be renamed to "Contact address" + +6.2.3. Initial Capability Registrations + + This RFC updates the following entries in the IANA registry for Sieve + extensions. + + Capability name: encoded-character + Description: changes the interpretation of strings to allow + arbitrary octets and Unicode characters to be + represented using US-ASCII + RFC number: RFC 5228 (Sieve base spec) + Contact address: The Sieve discussion list + + Capability name: fileinto + Description: adds the 'fileinto' action for delivering to a + mailbox other than the default + RFC number: RFC 5228 (Sieve base spec) + Contact address: The Sieve discussion list + + + + +Guenther & Showalter Standards Track [Page 32] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Capability name: envelope + Description: adds the 'envelope' test for testing the message + transport sender and recipient address + RFC number: RFC 5228 (Sieve base spec) + Contact address: The Sieve discussion list + + Capability name: comparator-* (anything starting with "comparator-") + Description: adds the indicated comparator for use with the + :comparator argument + RFC number: RFC 5228 (Sieve base spec) and [COLLATION] + Contact address: The Sieve discussion list + +6.3. Capability Transport + + A method of advertising which capabilities an implementation supports + is difficult due to the wide range of possible implementations. Such + a mechanism, however, should have the property that the + implementation can advertise the complete set of extensions that it + supports. + +7. Transmission + + The [MIME] type for a Sieve script is "application/sieve". + + The registration of this type for RFC 2048 requirements is updated as + follows: + + Subject: Registration of MIME media type application/sieve + + MIME media type name: application + MIME subtype name: sieve + Required parameters: none + Optional parameters: none + Encoding considerations: Most Sieve scripts will be textual, + written in UTF-8. When non-7bit characters are used, + quoted-printable is appropriate for transport systems + that require 7bit encoding. + Security considerations: Discussed in section 10 of this RFC. + Interoperability considerations: Discussed in section 2.10.5 + of this RFC. + Published specification: this RFC. + Applications that use this media type: sieve-enabled mail + servers and clients + Additional information: + Magic number(s): + File extension(s): .siv .sieve + Macintosh File Type Code(s): + + + + +Guenther & Showalter Standards Track [Page 33] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + Person & email address to contact for further information: + See the discussion list at ietf-mta-filters@imc.org. + Intended usage: + COMMON + Author/Change controller: + The SIEVE WG, delegated by the IESG. + +8. Parsing + + The Sieve grammar is separated into tokens and a separate grammar as + most programming languages are. Additional rules are supplied here + for common arguments to various language facilities. + +8.1. Lexical Tokens + + Sieve scripts are encoded in UTF-8. The following assumes a valid + UTF-8 encoding; special characters in Sieve scripts are all US-ASCII. + + The following are tokens in Sieve: + + - identifiers + - tags + - numbers + - quoted strings + - multi-line strings + - other separators + + Identifiers, tags, and numbers are case-insensitive, while quoted + strings and multi-line strings are case-sensitive. + + Blanks, horizontal tabs, CRLFs, and comments ("whitespace") are + ignored except as they separate tokens. Some whitespace is required + to separate otherwise adjacent tokens and in specific places in the + multi-line strings. CR and LF can only appear in CRLF pairs. + + The other separators are single individual characters and are + mentioned explicitly in the grammar. + + The lexical structure of sieve is defined in the following grammar + (as described in [ABNF]): + + bracket-comment = "/*" *not-star 1*STAR + *(not-star-slash *not-star 1*STAR) "/" + ; No */ allowed inside a comment. + ; (No * is allowed unless it is the last + ; character, or unless it is followed by a + ; character that isn't a slash.) + + + + +Guenther & Showalter Standards Track [Page 34] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + comment = bracket-comment / hash-comment + + hash-comment = "#" *octet-not-crlf CRLF + + identifier = (ALPHA / "_") *(ALPHA / DIGIT / "_") + + multi-line = "text:" *(SP / HTAB) (hash-comment / CRLF) + *(multiline-literal / multiline-dotstart) + "." CRLF + + multiline-literal = [ octet-not-period *octet-not-crlf ] CRLF + + multiline-dotstart = "." 1*octet-not-crlf CRLF + ; A line containing only "." ends the + ; multi-line. Remove a leading '.' if + ; followed by another '.'. + + not-star = CRLF / %x01-09 / %x0B-0C / %x0E-29 / %x2B-FF + ; either a CRLF pair, OR a single octet + ; other than NUL, CR, LF, or star + + not-star-slash = CRLF / %x01-09 / %x0B-0C / %x0E-29 / %x2B-2E / + %x30-FF + ; either a CRLF pair, OR a single octet + ; other than NUL, CR, LF, star, or slash + + number = 1*DIGIT [ QUANTIFIER ] + + octet-not-crlf = %x01-09 / %x0B-0C / %x0E-FF + ; a single octet other than NUL, CR, or LF + + octet-not-period = %x01-09 / %x0B-0C / %x0E-2D / %x2F-FF + ; a single octet other than NUL, + ; CR, LF, or period + + octet-not-qspecial = %x01-09 / %x0B-0C / %x0E-21 / %x23-5B / %x5D-FF + ; a single octet other than NUL, + ; CR, LF, double-quote, or backslash + + QUANTIFIER = "K" / "M" / "G" + + quoted-other = "\" octet-not-qspecial + ; represents just the octet-no-qspecial + ; character. SHOULD NOT be used + + quoted-safe = CRLF / octet-not-qspecial + ; either a CRLF pair, OR a single octet other + ; than NUL, CR, LF, double-quote, or backslash + + + +Guenther & Showalter Standards Track [Page 35] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + quoted-special = "\" (DQUOTE / "\") + ; represents just a double-quote or backslash + + quoted-string = DQUOTE quoted-text DQUOTE + + quoted-text = *(quoted-safe / quoted-special / quoted-other) + + STAR = "*" + + tag = ":" identifier + + white-space = 1*(SP / CRLF / HTAB) / comment + +8.2. Grammar + + The following is the grammar of Sieve after it has been lexically + interpreted. No whitespace or comments appear below. The start + symbol is "start". + + argument = string-list / number / tag + + arguments = *argument [ test / test-list ] + + block = "{" commands "}" + + command = identifier arguments (";" / block) + + commands = *command + + start = commands + + string = quoted-string / multi-line + + string-list = "[" string *("," string) "]" / string + ; if there is only a single string, the brackets + ; are optional + + test = identifier arguments + + test-list = "(" test *("," test) ")" + +8.3. Statement Elements + + These elements are collected from the "Syntax" sections elsewhere in + this document, and are provided here in [ABNF] syntax so that they + can be modified by extensions. + + ADDRESS-PART = ":localpart" / ":domain" / ":all" + + + +Guenther & Showalter Standards Track [Page 36] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + COMPARATOR = ":comparator" string + + MATCH-TYPE = ":is" / ":contains" / ":matches" + +9. Extended Example + + The following is an extended example of a Sieve script. Note that it + does not make use of the implicit keep. + + # + # Example Sieve Filter + # Declare any optional features or extension used by the script + # + require ["fileinto"]; + + # + # Handle messages from known mailing lists + # Move messages from IETF filter discussion list to filter mailbox + # + if header :is "Sender" "owner-ietf-mta-filters@imc.org" + { + fileinto "filter"; # move to "filter" mailbox + } + # + # Keep all messages to or from people in my company + # + elsif address :DOMAIN :is ["From", "To"] "example.com" + { + keep; # keep in "In" mailbox + } + + # + # Try and catch unsolicited email. If a message is not to me, + # or it contains a subject known to be spam, file it away. + # + elsif anyof (NOT address :all :contains + ["To", "Cc", "Bcc"] "me@example.com", + header :matches "subject" + ["*make*money*fast*", "*university*dipl*mas*"]) + { + fileinto "spam"; # move to "spam" mailbox + } + else + { + # Move all other (non-company) mail to "personal" + # mailbox. + fileinto "personal"; + } + + + +Guenther & Showalter Standards Track [Page 37] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +10. Security Considerations + + Users must get their mail. It is imperative that whatever + implementations use to store the user-defined filtering scripts + protect them from unauthorized modification, to preserve the + integrity of the mail system. An attacker who can modify a script + can cause mail to be discarded, rejected, or forwarded to an + unauthorized recipient. In addition, it's possible that Sieve + scripts might expose private information, such as mailbox names, or + email addresses of favored (or disfavored) correspondents. Because + of that, scripts SHOULD also be protected from unauthorized + retrieval. + + Several commands, such as "discard", "redirect", and "fileinto", + allow for actions to be taken that are potentially very dangerous. + + Use of the "redirect" command to generate notifications may easily + overwhelm the target address, especially if it was not designed to + handle large messages. + + Allowing a single script to redirect to multiple destinations can be + used as a means of amplifying the number of messages in an attack. + Moreover, if loop detection is not properly implemented, it may be + possible to set up exponentially growing message loops. Accordingly, + Sieve implementations: + + (1) MUST implement facilities to detect and break message loops. See + section 6.2 of [SMTP] for additional information on basic loop + detection strategies. + + (2) MUST provide the means for administrators to limit the ability of + users to abuse redirect. In particular, it MUST be possible to + limit the number of redirects a script can perform. + Additionally, if no use cases exist for using redirect to + multiple destinations, this limit SHOULD be set to 1. Additional + limits, such as the ability to restrict redirect to local users, + MAY also be implemented. + + (3) MUST provide facilities to log use of redirect in order to + facilitate tracking down abuse. + + (4) MAY use script analysis to determine whether or not a given + script can be executed safely. While the Sieve language is + sufficiently complex that full analysis of all possible scripts + is computationally infeasible, the majority of real-world scripts + are amenable to analysis. For example, an implementation might + + + + + +Guenther & Showalter Standards Track [Page 38] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + allow scripts that it has determined are safe to run unhindered, + block scripts that are potentially problematic, and subject + unclassifiable scripts to additional auditing and logging. + + Allowing redirects at all may not be appropriate in situations where + email accounts are freely available and/or not trackable to a human + who can be held accountable for creating message bombs or other + abuse. + + As with any filter on a message stream, if the Sieve implementation + and the mail agents 'behind' Sieve in the message stream differ in + their interpretation of the messages, it may be possible for an + attacker to subvert the filter. Of particular note are differences + in the interpretation of malformed messages (e.g., missing or extra + syntax characters) or those that exhibit corner cases (e.g., NUL + octets encoded via [MIME3]). + +11. Acknowledgments + + This document has been revised in part based on comments and + discussions that took place on and off the SIEVE mailing list. + Thanks to Sharon Chisholm, Cyrus Daboo, Ned Freed, Arnt Gulbrandsen, + Michael Haardt, Kjetil Torgrim Homme, Barry Leiba, Mark E. Mallett, + Alexey Melnikov, Eric Rescorla, Rob Siemborski, and Nigel Swinson for + reviews and suggestions. + +12. Normative References + + [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", RFC 4234, October 2005. + + [COLLATION] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet + Application Protocol Collation Registry", RFC 4790, March + 2007. + + [IMAIL] Resnick, P., Ed., "Internet Message Format", RFC 2822, + April 2001. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [MIME] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [MIME3] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII + Text", RFC 2047, November 1996. + + + +Guenther & Showalter Standards Track [Page 39] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + + [SMTP] Klensin, J., Ed., "Simple Mail Transfer Protocol", RFC + 2821, April 2001. + + [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + +13. Informative References + + [BINARY-SI] "Standard IEC 60027-2: Letter symbols to be used in + electrical technology - Part 2: Telecommunications and + electronics", January 1999. + + [DSN] Moore, K. and G. Vaudreuil, "An Extensible Message Format + for Delivery Status Notifications", RFC 3464, January + 2003. + + [FLAMES] Borenstein, N, and C. Thyberg, "Power, Ease of Use, and + Cooperative Work in a Practical Multimedia Message + System", Int. J. of Man-Machine Studies, April, 1991. + Reprinted in Computer-Supported Cooperative Work and + Groupware, Saul Greenberg, editor, Harcourt Brace + Jovanovich, 1991. Reprinted in Readings in Groupware and + Computer-Supported Cooperative Work, Ronald Baecker, + editor, Morgan Kaufmann, 1993. + + [IMAP] Crispin, M., "Internet Message Access Protocol - version + 4rev1", RFC 3501, March 2003. + + [MDN] Hansen, T., Ed., and G. Vaudreuil, Ed., "Message + Disposition Notification", RFC 3798, May 2004. + + [RFC3028] Showalter, T., "Sieve: A Mail Filtering Language", RFC + 3028, January 2001. + + + + + + + + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 40] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +14. Changes from RFC 3028 + + This following list is a summary of the changes that have been made + in the Sieve language base specification from [RFC3028]. + + 1. Removed ban on tests having side-effects + 2. Removed reject extension (will be specified in a separate RFC) + 3. Clarified description of comparators to match [COLLATION], the + new base specification for them + 4. Require stripping of leading and trailing whitespace in "header" + test + 5. Clarified or tightened handling of many minor items, including: + - invalid [MIME3] encoding + - invalid addresses in headers + - invalid header field names in tests + - 'undefined' comparator result + - unknown envelope parts + - null return-path in "envelope" test + 6. Capability strings are case-sensitive + 7. Clarified that fileinto should reencode non-ASCII mailbox + names to match the mailstore's conventions + 8. Errors in the ABNF were corrected + 9. The references were updated and split into normative and + informative + 10. Added encoded-character capability and deprecated (but did not + remove) use of arbitrary binary octets in Sieve scripts. + 11. Updated IANA registration template, and added IANA + considerations to permit capability prefix registrations. + 12. Added .sieve as a valid extension for Sieve scripts. + +Editors' Addresses + + Philip Guenther + Sendmail, Inc. + 6425 Christie St. Ste 400 + Emeryville, CA 94608 + EMail: guenther@sendmail.com + + Tim Showalter + EMail: tjs@psaux.com + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 41] + +RFC 5228 Sieve: An Email Filtering Language January 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Guenther & Showalter Standards Track [Page 42] + diff --git a/docs/rfcs/rfc5255.IMAP_i18n.txt b/docs/rfcs/rfc5255.IMAP_i18n.txt new file mode 100644 index 0000000..df76402 --- /dev/null +++ b/docs/rfcs/rfc5255.IMAP_i18n.txt @@ -0,0 +1,1123 @@ + + + + + + +Network Working Group C. Newman +Request for Comments: 5255 Sun Microsystems +Category: Standards Track A. Gulbrandsen + Oryx Mail Systems GmhH + A. Melnikov + Isode Limited + June 2008 + + + Internet Message Access Protocol Internationalization + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + Internet Message Access Protocol (IMAP) version 4rev1 has basic + support for non-ASCII characters in mailbox names and search + substrings. It also supports non-ASCII message headers and content + encoded as specified by Multipurpose Internet Mail Extensions (MIME). + This specification defines a collection of IMAP extensions that + improve international support including language negotiation for + international error text, translations for namespace prefixes, and + comparator negotiation for search, sort, and thread. + + + + + + + + + + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 1] + +RFC 5255 IMAP Internationalization June 2008 + + +Table of Contents + + 1. Introduction ....................................................3 + 2. Conventions Used in This Document ...............................3 + 3. LANGUAGE Extension ..............................................3 + 3.1. LANGUAGE Extension Requirements ............................4 + 3.2. LANGUAGE Command ...........................................4 + 3.3. LANGUAGE Response ..........................................6 + 3.4. TRANSLATION Extension to the NAMESPACE Response ............7 + 3.5. Formal Syntax ..............................................8 + 4. I18NLEVEL=1 and I18NLEVEL=2 Extensions ..........................9 + 4.1. Introduction and Overview ..................................9 + 4.2. Requirements Common to Both I18NLEVEL=1 and I18NLEVEL=2 ....9 + 4.3. I18NLEVEL=1 Extension Requirements ........................10 + 4.4. I18NLEVEL=2 Extension Requirements ........................10 + 4.5. Compatibility Notes .......................................11 + 4.6. Comparators and Character Encodings .......................11 + 4.7. COMPARATOR Command ........................................13 + 4.8. COMPARATOR Response .......................................14 + 4.9. BADCOMPARATOR Response Code ...............................14 + 4.10. Formal Syntax ............................................14 + 5. Other IMAP Internationalization Issues .........................15 + 5.1. Unicode Userids and Passwords .............................15 + 5.2. UTF-8 Mailbox Names .......................................15 + 5.3. UTF-8 Domains, Addresses, and Mail Headers ................15 + 6. IANA Considerations ............................................16 + 7. Security Considerations ........................................16 + 8. Acknowledgements ...............................................16 + 9. Relevant Sources of Documents for Internationalized IMAP + Implementations ................................................17 + 10. Normative References ..........................................17 + 11. Informative References ........................................18 + + + + + + + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 2] + +RFC 5255 IMAP Internationalization June 2008 + + +1. Introduction + + This specification defines two IMAP4rev1 [RFC3501] extensions to + enhance international support. These extensions can be advertised + and implemented separately. + + The LANGUAGE extension allows the client to request a suitable + language for protocol error messages and in combination with the + NAMESPACE extension [RFC2342] enables namespace translations. + + The I18NLEVEL=2 extension allows the client to request a suitable + collation that will modify the behavior of the base specification's + SEARCH command as well as the SORT and THREAD extensions [SORT]. + This leverages the collation registry [RFC4790]. The I18NLEVEL=1 + extension updates SEARCH/SORT/THREAD to use i;unicode-casemap + comparator, as defined in [UCM]. I18NLEVEL=1 is a simpler version of + I18NLEVEL=2 with no ability to select a different collation. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + The formal syntax uses the Augmented Backus-Naur Form (ABNF) + [RFC5234] notation including the core rules defined in Appendix A. + + The UTF-8-related productions are defined in [RFC3629]. + + In examples, "C:" and "S:" indicate lines sent by the client and + server respectively. If a single "C:" or "S:" label applies to + multiple lines, then the line breaks between those lines are for + editorial clarity only and are not part of the actual protocol + exchange. + +3. LANGUAGE Extension + + IMAP allows server responses to include human-readable text that in + many cases needs to be presented to the user. But that text is + limited to US-ASCII by the IMAP specification [RFC3501] in order to + preserve backwards compatibility with deployed IMAP implementations. + This section specifies a way for an IMAP client to negotiate which + language the server should use when sending human-readable text. + + + + + + + + +Newman, et al. Standards Track [Page 3] + +RFC 5255 IMAP Internationalization June 2008 + + + The LANGUAGE extension only provides a mechanism for altering fixed + server strings such as response text and NAMESPACE folder names. + Assigning localized language aliases to shared mailboxes would be + done with a separate mechanism such as the proposed METADATA + extension (see [METADATA]). + +3.1. LANGUAGE Extension Requirements + + IMAP servers that support this extension MUST list the keyword + LANGUAGE in their CAPABILITY response as well as in the greeting + CAPABILITY data. + + A server that advertises this extension MUST use the language + "i-default" as described in [RFC2277] as its default language until + another supported language is negotiated by the client. A server + MUST include "i-default" as one of its supported languages. IMAP + servers SHOULD NOT advertise the LANGUAGE extension if they discover + that they only support "i-default". + + Clients and servers that support this extension MUST also support the + NAMESPACE extension [RFC2342]. + + The LANGUAGE command is valid in all states. Clients SHOULD issue + LANGUAGE before authentication, since some servers send valuable user + information as part of authentication (e.g., "password is correct, + but expired"). If a security layer (such as SASL or TLS) is + subsequently negotiated by the client, it MUST re-issue the LANGUAGE + command in order to make sure that no previous active attack (if any) + on LANGUAGE negotiation has effect on subsequent error messages. + (See Section 7 for a more detailed explanation of the attack.) + +3.2. LANGUAGE Command + + Arguments: Optional language range arguments. + + Response: A possible LANGUAGE response (see Section 3.3). + A possible NAMESPACE response (see Section 3.4). + + Result: OK - Command completed + NO - Could not complete command + BAD - Arguments invalid + + The LANGUAGE command requests that human-readable text emitted by the + server be localized to a language matching one of the language range + argument as described by Section 2 of [RFC4647]. + + + + + + +Newman, et al. Standards Track [Page 4] + +RFC 5255 IMAP Internationalization June 2008 + + + If the command succeeds, the server will return human-readable + responses in the first supported language specified. These responses + will be in UTF-8 [RFC3629]. The server MUST send a LANGUAGE response + specifying the language used, and the change takes effect immediately + after the LANGUAGE response. + + If the command fails, the server continues to return human-readable + responses in the language it was previously using. + + The special "default" language range argument indicates a request to + use a language designated as preferred by the server administrator. + The preferred language MAY vary based on the currently active user. + + If a language range does not match a known language tag exactly but + does match a language by the rules of [RFC4647], the server MUST send + an untagged LANGUAGE response indicating the language selected. + + If there aren't any arguments, the server SHOULD send an untagged + LANGUAGE response listing the languages it supports. If the server + is unable to enumerate the list of languages it supports it MAY + return a tagged NO response to the enumeration request. If, after + receiving a LANGUAGE request, the server discovers that it doesn't + support any language other than i-default, it MUST return a tagged NO + response to the enumeration request. + + < The server defaults to using English i-default responses until + the user explicitly changes the language. > + + C: A001 LOGIN KAREN PASSWORD + S: A001 OK LOGIN completed + + < Client requested MUL language, which no server supports. > + + C: A002 LANGUAGE MUL + S: A002 NO Unsupported language MUL + + < A LANGUAGE command with no arguments is a request to enumerate + the list of languages the server supports. > + + C: A003 LANGUAGE + S: * LANGUAGE (EN DE IT i-default) + S: A003 OK Supported languages have been enumerated + + C: B001 LANGUAGE + S: B001 NO Server is unable to enumerate supported languages + + + + + + +Newman, et al. Standards Track [Page 5] + +RFC 5255 IMAP Internationalization June 2008 + + + < Once the client changes the language, all responses will be in + that language starting after the LANGUAGE response. Note that + this includes the NAMESPACE response. Because RFCs are in US- + ASCII, this document uses an ASCII transcription rather than + UTF-8 text, e.g., "ue" in the word "ausgefuehrt" > + + C: C001 LANGUAGE DE + S: * LANGUAGE (DE) + S: * NAMESPACE (("" "/")) (("Other Users/" "/" "TRANSLATION" + ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" + "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) + S: C001 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt + + < If a server does not support the requested primary language, + responses will continue to be returned in the current language + the server is using. > + + C: D001 LANGUAGE FR + S: D001 NO Diese Sprache ist nicht unterstuetzt + C: D002 LANGUAGE DE-IT + S: * LANGUAGE (DE-IT) + S: * NAMESPACE (("" "/"))(("Other Users/" "/" "TRANSLATION" + ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" + "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) + S: D002 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt + C: D003 LANGUAGE "default" + S: * LANGUAGE (DE) + S: D003 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt + + < Server does not speak French, but does speak English. User + speaks Canadian French and Canadian English. > + + C: E001 LANGUAGE FR-CA EN-CA + S: * LANGUAGE (EN) + S: E001 OK Now speaking English + +3.3. LANGUAGE Response + + Contents: A list of one or more language tags. + + The LANGUAGE response occurs as a result of a LANGUAGE command. A + LANGUAGE response with a list containing a single language tag + indicates that the server is now using that language. A LANGUAGE + response with a list containing multiple language tags indicates the + server is communicating a list of available languages to the client, + and no change in the active language has been made. + + + + + +Newman, et al. Standards Track [Page 6] + +RFC 5255 IMAP Internationalization June 2008 + + +3.4. TRANSLATION Extension to the NAMESPACE Response + + If localized representations of the namespace prefixes are available + in the selected language, the server SHOULD include these in the + TRANSLATION extension to the NAMESPACE response. + + The TRANSLATION extension to the NAMESPACE response returns a single + string, containing the modified UTF-7 [RFC3501] encoded translation + of the namespace prefix. It is the responsibility of the client to + convert between the namespace prefix and the translation of the + namespace prefix when presenting mailbox names to the user. + + In this example, a server supports the IMAP4 NAMESPACE command. It + uses no prefix to the user's Personal Namespace, a prefix of "Other + Users" to its Other Users' Namespace, and a prefix of "Public + Folders" to its only Shared Namespace. Since a client will often + display these prefixes to the user, the server includes a translation + of them that can be presented to the user. + + C: A001 LANGUAGE DE-IT + S: * NAMESPACE (("" "/")) (("Other Users/" "/" "TRANSLATION" + ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" + "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) + S: A001 OK LANGUAGE-Befehl ausgefuehrt + + + + + + + + + + + + + + + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 7] + +RFC 5255 IMAP Internationalization June 2008 + + +3.5. Formal Syntax + + The following syntax specification inherits ABNF [RFC5234] rules from + IMAP4rev1 [RFC3501], IMAP4 Namespace [RFC2342], Tags for the + Identifying Languages [RFC4646], UTF-8 [RFC3629], and Collected + Extensions to IMAP4 ABNF [RFC4466]. + + command-any =/ language-cmd + ; LANGUAGE command is valid in all states + + language-cmd = "LANGUAGE" *(SP lang-range-quoted) + + response-payload =/ language-data + + language-data = "LANGUAGE" SP "(" lang-tag-quoted *(SP + lang-tag-quoted) ")" + + namespace-trans = SP DQUOTE "TRANSLATION" DQUOTE SP "(" string ")" + ; the string is encoded in Modified UTF-7. + ; this is a subset of the syntax permitted by + ; the Namespace-Response-Extension rule in [RFC4466] + + lang-range-quoted = astring + ; Once any literal wrapper or quoting is removed, this + ; follows the language-range rule in [RFC4647] + + lang-tag-quoted = astring + ; Once any literal wrapper or quoting is removed, this follows + ; the Language-Tag rule in [RFC4646] + + resp-text = ["[" resp-text-code "]" SP ] UTF8-TEXT-CHAR + *(UTF8-TEXT-CHAR / "[") + ; After the server is changed to a language other than + ; i-default, this resp-text rule replaces the resp-text + ; rule from [RFC3501]. + + UTF8-TEXT-CHAR = %x20-5A / %x5C-7E / UTF8-2 / UTF8-3 / UTF8-4 + ; UTF-8 excluding 7-bit control characters and "[" + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 8] + +RFC 5255 IMAP Internationalization June 2008 + + +4. I18NLEVEL=1 and I18NLEVEL=2 Extensions + +4.1. Introduction and Overview + + IMAP4rev1 [RFC3501] includes the SEARCH command that can be used to + locate messages matching criteria including human-readable text. The + SORT extension [SORT] to IMAP allows the client to ask the server to + determine the order of messages based on criteria including human- + readable text. These mechanisms require the ability to support non- + English search and sort functions. + + Section 4 defines two IMAP extensions for internationalizing IMAP + SEARCH, SORT, and THREAD [SORT] using the comparator framework + [RFC4790]. + + The I18NLEVEL=1 extension updates SEARCH/SORT/THREAD to use + i;unicode-casemap comparator, as defined in [UCM]. See Sections 4.2 + and 4.3 for more details. + + The I18NLEVEL=2 extension is a superset of the I18NLEVEL=1 extension. + It adds to I18NLEVEL=1 extension the ability to determine the active + comparator (see definition below) and to negotiate use of comparators + using the COMPARATOR command. It also adds the COMPARATOR response + that indicates the active comparator and possibly other available + comparators. See Sections 4.2 and 4.4 for more details. + +4.2. Requirements Common to Both I18NLEVEL=1 and I18NLEVEL=2 + + The term "default comparator" refers to the comparator that is used + by SEARCH and SORT absent any negotiation using the COMPARATOR + command (see Section 4.7). The term "active comparator" refers to + the comparator which will be used within a session, e.g., by SEARCH + and SORT. The COMPARATOR command is used to change the active + comparator. + + The active comparator applies to the following SEARCH keys: "BCC", + "BODY", "CC", "FROM", "SUBJECT", "TEXT", "TO", and "HEADER". If the + server also advertises the "SORT" extension, then the active + comparator applies to the following SORT keys: "CC", "FROM", + "SUBJECT", and "TO". If the server advertises THREAD=ORDEREDSUBJECT, + then the active comparator applies to the ORDEREDSUBJECT threading + algorithm. If the server advertises THREAD=REFERENCES, then the + active comparator applies to the subject field comparisons done by + REFERENCES threading algorithm. Future extensions may choose to + apply the active comparator to their SEARCH keys. + + + + + + +Newman, et al. Standards Track [Page 9] + +RFC 5255 IMAP Internationalization June 2008 + + + For SORT and THREAD, the pre-processing necessary to extract the base + subject text from a Subject header occurs prior to the application of + a comparator. + + A server that advertises I18NLEVEL=1 or I18NLEVEL=2 extension MUST + implement the i;unicode-casemap comparator, as defined in [UCM]. + + A server that advertises I18NLEVEL=1 or I18NLEVEL=2 extension MUST + support UTF-8 as a SEARCH charset. + +4.3. I18NLEVEL=1 Extension Requirements + + An IMAP server that satisfies all requirements specified in Sections + 4.2 and 4.6 (and that doesn't support/advertise any other + I18NLEVEL= extension, where n > 1) MUST list the keyword + I18NLEVEL=1 in its CAPABILITY data once IMAP enters the authenticated + state, and MAY list that keyword in other states. + +4.4. I18NLEVEL=2 Extension Requirements + + An IMAP server that satisfies all requirements specified in Sections + 4.2, 4.4, and 4.6-4.10 (and that doesn't support/advertise any other + I18NLEVEL= extension, where n > 2) MUST list the keyword + I18NLEVEL=2 in its CAPABILITY data once IMAP enters the authenticated + state, and MAY list that keyword in other states. + + A server that advertises this extension MUST implement the + i;unicode-casemap comparator, as defined in [UCM]. It MAY implement + other comparators from the IANA registry established by [RFC4790]. + See also Section 4.5 of this document. + + A server that advertises this extension SHOULD use i;unicode-casemap + as the default comparator. (Note that i;unicode-casemap is the + default comparator for I18NLEVEL=1, but not necessarily the default + for I18NLEVEL=2.) The selection of the default comparator MAY be + adjustable by the server administrator, and MAY be sensitive to the + current user. Once the IMAP connection enters authenticated state, + the default comparator MUST remain static for the remainder of that + connection. + + Note that since SEARCH uses the substring operation, IMAP servers can + only implement collations that offer the substring operation (see + [RFC4790], Section 4.2.2). Since SORT uses the ordering operation + (which in turn uses the equality operation), IMAP servers that + advertise the SORT extension can only implement collations that offer + all three operations (see [RFC4790], Sections 4.2.2-4.2.4). + + + + + +Newman, et al. Standards Track [Page 10] + +RFC 5255 IMAP Internationalization June 2008 + + + If the active collation does not provide the operations needed by an + IMAP command, the server MUST respond with a tagged BAD. + +4.5. Compatibility Notes + + Several server implementations deployed prior to the publication of + this specification comply with I18NLEVEL=1 (see Section 4.3), but do + not advertise that. Other legacy servers use the i;ascii-casemap + comparator (see [RFC4790]). + + There is no good way for a client to know which comparator a legacy + server uses. If the client has to assume the worst, it may end up + doing expensive local operations to obtain i;unicode-casemap + comparisons even though the server implements it. + + Legacy server implementations which comply with I18NLEVEL=1 should be + updated to advertise I18NLEVEL=1. All server implementations should + eventually be updated to comply with the I18NLEVEL=2 extension. + +4.6. Comparators and Character Encodings + + RFC 3501, Section 6.4.4, says: + + In all search keys that use strings, a message matches the key + if the string is a substring of the field. The matching is + case-insensitive. + + When performing the SEARCH operation, the active comparator is + applied instead of the case-insensitive matching specified above. + + An IMAP server which performs collation operations (e.g., as part of + commands such as SEARCH, SORT, and THREAD) does so according to the + following procedure: + + (a) MIME encoding (for example, see [RFC2047] for headers and + [RFC2045] for body parts) MUST be removed in the texts being + collated. + + If MIME encoding removal fails for a message (e.g., a body part + of the message has an unsupported Content-Transfer-Encoding, uses + characters not allowed by the Content-Transfer-Encoding, etc.), + the collation of this message is undefined by this specification, + and is handled in an implementation-dependent manner. + + (b) The decoded text from (a) MUST be converted to the charset + expected by the active comparator. + + + + + +Newman, et al. Standards Track [Page 11] + +RFC 5255 IMAP Internationalization June 2008 + + + (c) For the substring operation: + + If step (b) failed (e.g., the text is in an unknown charset, + contains a sequence that is not valid according in that charset, + etc.), the original decoded text from (a) (i.e., before the + charset conversion attempt) is collated using the i;octet + comparator (see [RFC4790]). + + If step (b) was successful, the converted text from (b) is + collated according to the active comparator. + + For the ordering operation: + + All strings that were successfully converted by step (b) are + separated from all strings that failed step (b). Strings in each + group are collated independently. All strings successfully + converted by step (b) are then validated by the active + comparator. Strings that pass validation are collated using the + active comparator. All strings that either fail step (b) or fail + the active collation's validity operation are collated (after + applying step (a)) using the i;octet comparator (see [RFC4790]). + The resulting sorted list is produced by appending all collated + "failed" strings after all strings collated using the active + comparator. + + Example: The following example demonstrates ordering of 4 + different strings using the i;unicode-casemap [UCM] comparator. + Strings are represented using hexadecimal notation used by ABNF + [RFC5234]. + + (1) %xD0 %xC0 %xD0 %xBD %xD0 %xB4 %xD1 %x80 %xD0 %xB5 + %xD0 %xB9 (labeled with charset=UTF-8) + (2) %xD1 %x81 %xD0 %x95 %xD0 %xA0 %xD0 %x93 %xD0 %x95 + %xD0 %x99 (labeled with charset=UTF-8) + (3) %xD0 %x92 %xD0 %xB0 %xD1 %x81 %xD0 %xB8 %xD0 %xBB + %xD0 %xB8 %xFF %xB9 (labeled with charset=UTF-8) + (4) %xE1 %xCC %xC5 %xCB %xD3 %xC5 %xCA (labeled with + charset=KOI8-R) + + Step (b) will convert string (4) to the following sequence of + octets (in UTF-8): + + %xD0 %x90 %xD0 %xBB %xD0 %xB5 %xD0 %xBA %xD1 %x81 %xD0 + %xB5 %xD0 %xB9 + + and will reject strings (1) and (3), as they contain octets not + allowed in charset=UTF-8. + + + + +Newman, et al. Standards Track [Page 12] + +RFC 5255 IMAP Internationalization June 2008 + + + After that, using the i;unicode-casemap collation, string (4) + will collate before string (2). Using the i;octet collation on + the original strings, string (3) will collate before string (1). + So the final ordering is as follows: (4) (2) (3) (1). + + If the substring operation (e.g., IMAP SEARCH) of the active + comparator returns the "undefined" result (see Section 4.2.3 of + [RFC4790]) for either the text specified in the SEARCH command or the + message text, then the operation is repeated on the result of step + (a) using the i;octet comparator. + + The ordering operation (e.g., IMAP SORT and THREAD) SHOULD collate + the following together: strings encoded using unknown or invalid + character encodings, strings in unrecognized charsets, and invalid + input (as defined by the active collation). + +4.7. COMPARATOR Command + + Arguments: Optional comparator order arguments. + + Response: A possible COMPARATOR response (see Section 4.8). + + Result: OK - Command completed + NO - No matching comparator found + BAD - Arguments invalid + + The COMPARATOR command is valid in authenticated and selected states. + + The COMPARATOR command is used to determine or change the active + comparator. When issued with no arguments, it results in a + COMPARATOR response indicating the currently active comparator. + + When issued with one or more comparator arguments, it changes the + active comparator as directed. (If more than one installed + comparator is matched by an argument, the first argument wins.) The + COMPARATOR response lists all matching comparators if more than one + matches the specified patterns. + + The argument "default" refers to the server's default comparator. + Otherwise, each argument is a collation specification as defined in + the Internet Application Protocol Comparator Registry [RFC4790]. + + < The client requests activating a Czech comparator if possible, + or else a generic international comparator which it considers + suitable for Czech. The server picks the first supported + comparator. > + + + + + +Newman, et al. Standards Track [Page 13] + +RFC 5255 IMAP Internationalization June 2008 + + + C: A001 COMPARATOR "cz;*" i;basic + S: * COMPARATOR i;basic + S: A001 OK Will use i;basic for collation + +4.8. COMPARATOR Response + + Contents: The active comparator. An optional list of available + matching comparators + + The COMPARATOR response occurs as a result of a COMPARATOR command. + The first argument in the comparator response is the name of the + active comparator. The second argument is a list of comparators + which matched any of the arguments to the COMPARATOR command and is + present only if more than one match is found. + +4.9. BADCOMPARATOR Response Code + + This response code SHOULD be returned as a result of server failing + an IMAP command (returning NO), when the server knows that none of + the specified comparators match the requested comparator(s). + +4.10. Formal Syntax + + The following syntax specification inherits ABNF [RFC5234] rules from + IMAP4rev1 [RFC3501] and the Internet Application Protocol Comparator + Registry [RFC4790]. + + command-auth =/ comparator-cmd + + resp-text-code =/ "BADCOMPARATOR" + + comparator-cmd = "COMPARATOR" *(SP comp-order-quoted) + + response-payload =/ comparator-data + + comparator-data = "COMPARATOR" SP comp-sel-quoted [SP "(" + comp-id-quoted *(SP comp-id-quoted) ")"] + + comp-id-quoted = astring + ; Once any literal wrapper or quoting is removed, this + ; follows the collation-id rule from [RFC4790] + + comp-order-quoted = astring + ; Once any literal wrapper or quoting is removed, this + ; follows the collation-order rule from [RFC4790] + + + + + + +Newman, et al. Standards Track [Page 14] + +RFC 5255 IMAP Internationalization June 2008 + + + comp-sel-quoted = astring + ; Once any literal wrapper or quoting is removed, this + ; follows the collation-selected rule from [RFC4790] + +5. Other IMAP Internationalization Issues + + The following sections provide an overview of various other IMAP + internationalization issues. These issues are not resolved by this + specification, but could be resolved by other standards work, such as + that being done by the EAI working group (see [IMAP-EAI]). + +5.1. Unicode Userids and Passwords + + IMAP4rev1 currently restricts the userid and password fields of the + LOGIN command to US-ASCII. The "userid" and "password" fields of the + IMAP LOGIN command are restricted to US-ASCII only until a future + standards track RFC states otherwise. Servers are encouraged to + validate both fields to make sure they conform to the formal syntax + of UTF-8 and to reject the LOGIN command if that syntax is violated. + Servers MAY reject the LOGIN command if either the "userid" or + "password" field contains an octet with the highest bit set. + + When AUTHENTICATE is used, some servers may support userids and + passwords in Unicode [RFC3490] since SASL (see [RFC4422]) allows + that. However, such userids cannot be used as part of email + addresses. + +5.2. UTF-8 Mailbox Names + + The modified UTF-7 mailbox naming convention described in Section + 5.1.3 of RFC 3501 is best viewed as an transition from the status quo + in 1996 when modified UTF-7 was first specified. At that time, there + was widespread unofficial use of local character sets such as ISO- + 8859-1 and Shift-JIS for non-ASCII mailbox names, with resultant + non-interoperability. + + The requirements in Section 5.1 of RFC 3501 are very important if + we're ever going to be able to deploy UTF-8 mailbox names. Servers + are encouraged to enforce them. + +5.3. UTF-8 Domains, Addresses, and Mail Headers + + There is now an IETF standard for "Internationalizing Domain Names in + Applications (IDNA)" [RFC3490]. While IMAP clients are free to + support this standard, an argument can be made that it would be + helpful to simple clients if the IMAP server could perform this + conversion (the same argument would apply to MIME header encoding + + + + +Newman, et al. Standards Track [Page 15] + +RFC 5255 IMAP Internationalization June 2008 + + + [RFC2047]). However, it would be unwise to move forward with such + work until the work in progress to define the format of international + email addresses is complete. + +6. IANA Considerations + + IANA added LANGUAGE, I18NLEVEL=1, and I18NLEVEL=2 to the IMAP4 + Capabilities Registry. + +7. Security Considerations + + The LANGUAGE extension makes a new command available in "Not + Authenticated" state in IMAP. Some IMAP implementations run with + root privilege when the server is in "Not Authenticated" state and do + not revoke that privilege until after authentication is complete. + Such implementations are particularly vulnerable to buffer overflow + security errors at this stage and need to implement parsing of this + command with extra care. + + A LANGUAGE command issued prior to activation of a security layer is + subject to an active attack that suppresses or modifies the + negotiation, and thus makes STARTTLS or authentication error messages + more difficult to interpret. This is not a new attack as the error + messages themselves are subject to active attack. Clients MUST re- + issue the LANGUAGE command once a security layer is active, in order + to prevent this attack from impacting subsequent protocol operations. + + LANGUAGE, I18NLEVEL=1, and I18NLEVEL=2 extensions use the UTF-8 + charset; thus, the security considerations for UTF-8 [RFC3629] are + relevant. However, neither uses UTF-8 for identifiers, so the most + serious concerns do not apply. + +8. Acknowledgements + + The LANGUAGE extension is based on a previous document by Mike + Gahrns, a substantial portion of the text in that section was written + by him. Many people have participated in discussions about an IMAP + Language extension in the various fora of the IETF and Internet + working groups, so any list of contributors is bound to be + incomplete. However, the authors would like to thank Andrew McCown + for early work on the original proposal, John Myers for suggestions + regarding the namespace issue, along with Jutta Degener, Mark + Crispin, Mark Pustilnik, Larry Osterman, Cyrus Daboo, Martin Duerst, + Timo Sirainen, Ben Campbell, and Magnus Nystrom for their many + suggestions that have been incorporated into this document. + + Initial discussion of the I18NLEVEL=2 extension involved input from + Mark Crispin and other participants of the IMAP Extensions WG. + + + +Newman, et al. Standards Track [Page 16] + +RFC 5255 IMAP Internationalization June 2008 + + +9. Relevant Sources of Documents for Internationalized IMAP + Implementations + + This is a non-normative list of sources to consider when implementing + i18n-aware IMAP software. + + o The LANGUAGE and I18NLEVEL=2 extensions to IMAP (this + specification). + + o The 8-bit rules for mailbox naming in Section 5.1 of RFC 3501. + + o The Mailbox International Naming Convention in Section 5.1.3 of + RFC 3501. + + o MIME [RFC2045] for message bodies. + + o MIME header encoding [RFC2047] for message headers. + + o The IETF EAI working group. + + o MIME Parameter Value and Encoded Word Extensions [RFC2231] for + filenames. Quality IMAP server implementations will + automatically combine multipart parameters when generating the + BODYSTRUCTURE. There is also some deployed non-standard use of + MIME header encoding inside double quotes for filenames. + + o IDNA [RFC3490] and punycode [RFC3492] for domain names + (currently only relevant to IMAP clients). + + o The UTF-8 charset [RFC3629]. + + o The IETF policy on Character Sets and Languages [RFC2277]. + +10. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2277] Alvestrand, H., "IETF Policy on Character Sets and + Languages", BCP 18, RFC 2277, January 1998. + + [RFC2342] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, May + 1998. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + + + + +Newman, et al. Standards Track [Page 17] + +RFC 5255 IMAP Internationalization June 2008 + + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [RFC4422] Melnikov, A., Ed., and K. Zeilenga, Ed., "Simple + Authentication and Security Layer (SASL)", RFC 4422, June + 2006. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC4646] Phillips, A. and M. Davis, "Tags for Identifying + Languages", BCP 47, RFC 4646, September 2006. + + [RFC4647] Phillips, A. and M. Davis, "Matching of Language Tags", + BCP 47, RFC 4647, September 2006. + + [RFC4790] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet + Application Protocol Collation Registry", RFC 4790, March + 2007. + + [SORT] Crispin, M. and K. Murchison, "Internet Message Access + Protocol - SORT and THREAD Extensions", RFC 5256, June + 2008. + + [UCM] Crispin, M., "i;unicode-casemap - Simple Unicode Collation + Algorithm", RFC 5051, October 2007. + + [RFC2045] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII Text", + RFC 2047, November 1996. + +11. Informative References + + [RFC2231] Freed, N. and K. Moore, "MIME Parameter Value and Encoded + Word Extensions: Character Sets, Languages, and + Continuations", RFC 2231, November 1997. + + [RFC3490] Faltstrom, P., Hoffman, P., and A. Costello, + "Internationalizing Domain Names in Applications (IDNA)", + RFC 3490, March 2003. + + + +Newman, et al. Standards Track [Page 18] + +RFC 5255 IMAP Internationalization June 2008 + + + [RFC3492] Costello, A., "Punycode: A Bootstring encoding of Unicode + for Internationalized Domain Names in Applications + (IDNA)", RFC 3492, March 2003. + + [METADATA] Daboo, C., "IMAP METADATA Extension", Work in Progress, + April 2008. + + [IMAP-EAI] Resnick, P., and C. Newman, "IMAP Support for UTF-8", Work + in Progress, November 2007. + +Authors' Addresses + + Chris Newman + Sun Microsystems + 3401 Centrelake Dr., Suite 410 + Ontario, CA 91761 + US + + EMail: chris.newman@sun.com + + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + EMail: arnt@oryx.com + Fax: +49 89 4502 9758 + + + Alexey Melnikov + Isode Limited + 5 Castle Business Village, 36 Station Road, + Hampton, Middlesex, TW12 2BX, UK + + EMail: Alexey.Melnikov@isode.com + + + + + + + + + + + + + + +Newman, et al. Standards Track [Page 19] + +RFC 5255 IMAP Internationalization June 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Newman, et al. Standards Track [Page 20] + diff --git a/docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt b/docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt new file mode 100644 index 0000000..1d088e5 --- /dev/null +++ b/docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt @@ -0,0 +1,1739 @@ + + + + + + +Network Working Group C. Daboo +Request for Comments: 5257 Apple Inc. +Category: Experimental R. Gellens + QUALCOMM Incorporated + June 2008 + + + Internet Message Access Protocol - ANNOTATE Extension + +Status of This Memo + + This memo defines an Experimental Protocol for the Internet + community. It does not specify an Internet standard of any kind. + Discussion and suggestions for improvement are requested. + Distribution of this memo is unlimited. + +Abstract + + The ANNOTATE extension to the Internet Message Access Protocol + permits clients and servers to maintain "meta data" for messages, or + individual message parts, stored in a mailbox on the server. For + example, this can be used to attach comments and other useful + information to a message. It is also possible to attach annotations + to specific parts of a message, so that, for example, they could be + marked as seen, or important, or a comment added. + + Note that this document was the product of a WG that had good + consensus on how to approach the problem. Nevertheless, the WG felt + it did not have enough information on implementation and deployment + hurdles to meet all of the requirements of a Proposed Standard. The + IETF solicits implementations and implementation reports in order to + make further progress. + + Implementers should be aware that this specification may change in an + incompatible manner when going to Proposed Standard status. However, + any incompatible changes will result in a new capability name being + used to prevent problems with any deployments of the experimental + extension. + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 1] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +Table of Contents + + 1. Introduction and Overview .......................................3 + 2. Conventions Used in This Document ...............................4 + 3. Data Model ......................................................4 + 3.1. Overview ...................................................4 + 3.2. Namespace of Entries and Attributes ........................4 + 3.2.1. Entry Names .........................................5 + 3.2.2. Attribute Names .....................................7 + 3.3. Private Versus Shared ......................................7 + 3.4. Access Control .............................................8 + 3.5. Access to Standard IMAP Flags and Keywords ................11 + 4. IMAP Protocol Changes ..........................................11 + 4.1. General Considerations ....................................11 + 4.2. ANNOTATE Parameter with the SELECT/EXAMINE Commands .......12 + 4.3. ANNOTATION Message Data Item in FETCH Command .............12 + 4.4. ANNOTATION Message Data Item in FETCH Response ............14 + 4.5. ANNOTATION Message Data Item in STORE .....................16 + 4.6. ANNOTATION Interaction with COPY ..........................18 + 4.7. ANNOTATION Message Data Item in APPEND ....................18 + 4.8. ANNOTATION Criterion in SEARCH ............................19 + 4.9. ANNOTATION Key in SORT ....................................20 + 4.10. New ACL Rights ...........................................21 + 5. Formal Syntax ..................................................21 + 6. IANA Considerations ............................................23 + 6.1. Entry and Attribute Registration Template .................23 + 6.2. Entry Registrations .......................................24 + 6.2.1. /comment ...........................................24 + 6.2.2. /flags .............................................24 + 6.2.3. /altsubject ........................................25 + 6.2.4. //comment ............................25 + 6.2.5. //flags/seen .........................26 + 6.2.6. //flags/answered .....................26 + 6.2.7. //flags/flagged ......................27 + 6.2.8. //flags/forwarded ....................27 + 6.3. Attribute Registrations ...................................28 + 6.3.1. value ..............................................28 + 6.3.2. size ...............................................28 + 6.4. Capability Registration ...................................28 + 7. Internationalization Considerations ............................29 + 8. Security Considerations ........................................29 + 9. References .....................................................29 + 9.1. Normative References ......................................29 + 9.2. Informative References ....................................30 + 10. Acknowledgments ...............................................30 + + + + + + +Daboo & Gellens Experimental [Page 2] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +1. Introduction and Overview + + The ANNOTATE extension is present in any IMAP [RFC3501] + implementation that returns "ANNOTATE-EXPERIMENT-1" as one of the + supported capabilities in the CAPABILITY response. + + This extension makes the following changes to the IMAP protocol: + + a. adds a new ANNOTATION message data item for use in FETCH. + + b. adds a new ANNOTATION message data item for use in STORE. + + c. adds a new ANNOTATION search criterion for use in SEARCH. + + d. adds a new ANNOTATION sort key for use in the SORT extension. + + e. adds a new ANNOTATION data item for use in APPEND. + + f. adds a new requirement on the COPY command. + + g. adds a new ANNOTATE parameter for use with the SELECT/EXAMINE + commands. + + h. adds two new response codes to indicate store failures of + annotations. + + i. adds a new untagged response code for the SELECT or EXAMINE + commands to indicate the maximum sized annotation that can be + stored. + + j. adds a new Access Control List (ACL) "bit" for use with the ACL + extensions [RFC2086] and [RFC4314]. + + The data model used for the storage of annotations is based on the + Application Configuration Access Protocol [RFC2244]. Note that there + is no inheritance in annotations. + + If a server supports annotations, then it MUST store all annotation + data permanently, i.e., there is no concept of "session only" + annotations that would correspond to the behavior of "session" flags + as defined in the IMAP base specification. + + In order to provide optimum support for a disconnected client (one + that needs to synchronize annotations for use when offline), servers + SHOULD also support the Conditional STORE [RFC4551] extension. + + The rest of this document describes the data model and protocol + changes more rigorously. + + + +Daboo & Gellens Experimental [Page 3] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +2. Conventions Used in This Document + + The examples in this document use "C:" and "S:" to indicate lines + sent by the client and server, respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + +3. Data Model + +3.1. Overview + + The data model for annotations in ANNOTATE uses a uniquely named + entry that contains a set of standard attributes. Thus, a single + coherent unit of "meta data" for a message is stored as a single + entry, made up of several attributes. + + For example, a comment annotation added to a message has an entry + name of "/comment". This entry is composed of several attributes + such as "value", "size", etc., that contain the properties and data + of the entry. + + The protocol changes to IMAP, described below, allow a client to + access or change the values of any attribute in any entry in a + message annotation, assuming it has sufficient access rights to do so + (see Section 3.4 for specifics). + +3.2. Namespace of Entries and Attributes + + A message may contain zero or more annotations, each of which is a + single uniquely named entry. Each entry has a hierarchical name, + with each component of the name separated by a slash ("/"). An entry + name MUST NOT contain two consecutive "/" characters and MUST NOT end + with a "/" character. + + Each entry is made up of a set of attributes. Each attribute has a + hierarchical name, with each component of the name separated by a + period ("."). An attribute name MUST NOT contain two consecutive "." + characters and MUST NOT end with a "." character. + + The value of an attribute is "NIL" (has no value), or is a string of + zero or more octets. + + Entry and attribute names MUST NOT contain asterisk ("*") or percent + ("%") characters, and MUST NOT contain non-ASCII characters or the + NULL octet. Invalid entry or attribute names result in a BAD + response in any IMAP commands where they are used. + + + +Daboo & Gellens Experimental [Page 4] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Attribute names MUST NOT contain any hierarchical components with the + names "priv" or "shared", as those have special meaning (see Section + 3.3). + + Entry and attribute names are case-sensitive. + + Use of control or punctuation characters in entry and attribute names + is strongly discouraged. + + This specification defines an initial set of entry and attribute + names available for use in message annotations. In addition, an + extension mechanism is described to allow additional names to be + added as needed. + +3.2.1. Entry Names + + Entry names MUST be specified in a standards track or IESG approved + experimental RFC, or fall under the vendor namespace. See Section + 6.1 for the registration template. + + / + Defines the top-level of entries associated with an entire + message. This entry itself does not contain any attributes. All + entries that start with a numeric character ("0" - "9") refer to + an annotation on a specific body part. All other entries are for + annotations on the entire message. + + /comment + Defines a comment or note associated with an entire message. + + /flags + This entry hierarchy is reserved for future use. + + /altsubject + Contains text supplied by the message recipient to be used by the + client, instead of the original message Subject. + + /vendor/ + Defines the top-level of entries associated with an entire message + as created by a particular product of some vendor. These sub- + entries can be used by vendors to provide client-specific + annotations. The vendor-token MUST be registered with IANA, using + the [RFC2244] vendor subtree registry. + + / + Defines the top-level of entries associated with a specific body + part of a message. This entry itself does not contain any + attributes. The section-part is a numeric part specifier. Its + + + +Daboo & Gellens Experimental [Page 5] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + syntax is the same as the section-part ABNF element defined in + [RFC3501]. The server MUST return a BAD response if the client + uses an incorrect part specifier (either incorrect syntax or a + specifier referring to a non-existent part). The server MUST + return a BAD response if the client uses an empty part specifier + (which is used in IMAP to represent the entire message). + + //comment + Defines a comment or note associated with a specific body part of + a message. + + //flags + Defines the top-level of entries associated with the flag state + for a specific body part of a message. All sub-entries are + maintained entirely by the client. There is no implicit change to + any flag by the server. + + //flags/seen + This is similar to the IMAP \Seen flag, except it applies + to only the body part referenced by the entry. + + //flags/answered + This is similar to the IMAP \Answered flag, except it + applies to only the body part referenced by the entry. + + //flags/flagged + This is similar to the IMAP \Flagged flag, except it + applies to only the body part referenced by the entry. + + //flags/forwarded + This is similar to the IMAP $Forwarded keyword, except it + applies to only the body part referenced by the entry. + + Defines flags for a specific body part of a message. The "value" + attribute of each of the entries described above must be either + "1", "0", or "NIL". "1" corresponds to the flag being set. + + //vendor/ + Defines the top-level of entries associated with a specific body + part of a message as created by a particular product of some + vendor. This entry can be used by vendors to provide client + specific annotations. The vendor-token MUST be registered with + IANA. + + + + + + + + +Daboo & Gellens Experimental [Page 6] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +3.2.2. Attribute Names + + Attribute names MUST be specified in a standards track or IESG + approved experimental RFC. See Section 6.1 for the registration + template. + + All attribute names implicitly have a ".priv" and a ".shared" suffix + that maps to private and shared versions of the entry. Searching or + fetching without using either suffix will include both. The client + MUST specify either a ".priv" or ".shared" suffix when storing an + annotation or sorting on annotations. + + value + A string or binary data representing the value of the annotation. + To delete an annotation, the client can store "NIL" into the + value. If the client requests the value attribute for a non- + existent entry, then the server MUST return "NIL" for the value. + The content represented by the string is determined by the + content-type used to register the entry (see Section 6.1 for entry + registration templates). Where applicable, the registered + content-type MUST include a charset parameter. Text values SHOULD + use the utf-8 [RFC3629] character set. Note that binary data + (data which may contain the NULL octet) is allowed (e.g., for + storing images), and this extension uses the "literal8" syntax + element [RFC4466] to allow such data to be written to or read from + the server. + + size + The size of the value, in octets. Set automatically by the + server, read-only to clients. If the client requests the size + attribute for a non-existent entry, then the server MUST return + "0" (zero) for the size. + +3.3. Private Versus Shared + + Some IMAP mailboxes are private, accessible only to the owning user. + Other mailboxes are not, either because the owner has set an ACL + [RFC4314] that permits access by other users, or because it is a + shared mailbox. + + This raises the issue of shared versus private annotations. + + If all annotations are private, it is then impossible to have + annotations in a shared or otherwise non-private mailbox be visible + to other users. This eliminates what could be a useful aspect of + annotations in a shared environment. An example of such use is a + shared IMAP folder containing bug reports. Engineers may want to use + + + + +Daboo & Gellens Experimental [Page 7] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + annotations to add information to existing messages, indicate + assignments, status, etc. This use requires shared annotations. + + If all annotations are shared, it is impossible to use annotations + for private notes on messages in shared mailboxes. Also, modifying + an ACL to permit access to a mailbox by other users may + unintentionally expose private information. + + There are also situations in which both shared and private + annotations are useful. For example, an administrator may want to + set shared annotations on messages in a shared folder, which + individual users may wish to supplement with additional notes. + + If shared and private annotations are to coexist, we need a clear way + to differentiate them. Also, it should be as easy as possible for a + client to access both and not overlook either. There is also a + danger in allowing a client to store an annotation without knowing if + it is shared or private. + + This document proposes two standard suffixes for all attributes: + ".shared" and ".priv". A SEARCH or FETCH command that specifies + neither, uses both. STORE, APPEND, and SORT commands MUST explicitly + use ".priv" or ".shared" suffixes. + + If the ANNOTATE extension is present, support for shared annotations + in servers is REQUIRED, while support for private annotations in + servers is OPTIONAL. This recognizes the fact that support for + private annotations may introduce a significant increase in + complexity to a server in terms of tracking ownership of the + annotations, how quota is determined for users based on their own + annotations, etc. Clients that support the ANNOTATE extension MUST + handle both shared and private annotations. + +3.4. Access Control + + A user needs to have appropriate rights in order to read or write + ".priv" or ".shared" annotation values. How those rights are + calculated depends on whether or not the ACL [RFC2086] extension or + its update [RFC4314] is present. If a client attempts to store or + fetch an annotation to which they do not have the appropriate rights, + the server MUST respond with a NO response. + + When the ACL extension is not present, access to annotation values is + governed by the nature of the selected state, in particular whether + the mailbox was SELECTED or EXAMINED in READ-ONLY or READ-WRITE mode. + + + + + + +Daboo & Gellens Experimental [Page 8] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + When the ACL extension is present, the server MUST recognize the new + ACL "n" right, in addition to the ones defined by the ACL extension + itself. + + For ".priv" annotation values, the "r" right controls both read and + write access. When it is on, access to ".priv" annotations is + allowed; when it is off, access to ".priv" annotations is disallowed. + + For ".shared" annotation values, the "r" right controls read access. + When it is on, ".shared" annotations can be read; when it is off, + ".shared" annotations cannot be read. + + For ".shared" annotation values, the "n" right controls write access. + When it is on, ".shared" annotations can be changed or created + through either a STORE or APPEND command; when it is off, ".shared" + annotations cannot be changed or created. The "n" right constitutes + a "shared flag right" as defined in Section 6.2 of [RFC4314]. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 9] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + A summary of all the access control restrictions is tabulated below + + +---------------+---------------+-----------------------------------+ + | Server Type | Action on | Type of mailbox | + | | annotation | | + +===============+===============+===================================+ + | | | | + | | read .priv | Any mailbox that can be SELECTED | + | | values | or EXAMINED. | + | | | | + | +---------------+-----------------------------------+ + | | | | + | | write .priv | Any SELECTED [READ-WRITE] mailbox.| + | | values | SELECTED [READ-ONLY] mailboxes MAY| + | Server | | also permit writes. | + | without | | | + | ACL Extension +---------------+-----------------------------------+ + | | | | + | | read .shared | Any mailbox that can be SELECTED | + | | values | or EXAMINED. | + | | | | + | +---------------+-----------------------------------+ + | | | | + | | write .shared | Any mailbox that can be SELECTED | + | | values | or EXAMINED and is [READ-WRITE]. | + | | | | + +---------------+---------------+-----------------------------------+ + | | | | + | | read .priv | Any mailbox with the "r" | + | | values | ACL right. | + | | | | + | +---------------+-----------------------------------+ + | | | | + | | write .priv | Any mailbox with the "r" | + | Server | values | ACL right. | + | with | | | + | ACL Extension +---------------+-----------------------------------+ + | | | | + | | read .shared | Any mailbox with the "r" | + | | values | ACL right. | + | | | | + | +---------------+-----------------------------------+ + | | | | + | | write .shared | Any mailbox with the "n" | + | | values | ACL right. | + | | | | + +---------------+---------------+-----------------------------------+ + + + + +Daboo & Gellens Experimental [Page 10] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +3.5. Access to Standard IMAP Flags and Keywords + + Due to the ambiguity of how private and shared values would map to + the base IMAP flag and keyword values, the ANNOTATE extension does + not expose IMAP flags or keywords as entries. However, the /flags + annotation entry is reserved for future use and MUST NOT be used by + clients or servers supporting this extension. + + Clients that need to implement shared and private "flags" can create + their own annotation entries for those, completely bypassing the base + IMAP flag/keyword behavior. + +4. IMAP Protocol Changes + +4.1. General Considerations + + Servers may be able to offer only a limited level of support for + annotations in mailboxes, and it is useful for clients to be able to + know what level of support is available. Servers MUST return an + ANNOTATIONS response code during the SELECT or EXAMINE command for a + mailbox to indicate the level of support. Possible data items used + with the ANNOTATIONS response code are: + + "NONE" - this indicates that the mailbox being selected does not + support annotations at all. Clients MUST NOT attempt to use + annotation extensions in commands for this mailbox. + + "READ-ONLY" - this indicates that the annotations supported by the + mailbox cannot be changed by the client. Clients MUST NOT attempt + to store annotations on any messages in a mailbox with this + response code. + + "NOPRIVATE" - this indicates that the server does not support + private annotations on the mailbox. Only shared annotations are + supported. Clients SHOULD only attempt to read or store + annotations attributes with the ".shared" suffix. If a client + uses an attribute with the ".priv" suffix in a FETCH command, then + servers should return the attribute value in the FETCH response as + "NIL". If a client uses an attribute with the ".priv" suffix in a + STORE command (or an APPEND command targeted at the mailbox), then + the server MUST return a NO response. + + numeric values - if servers support writable annotations, then the + server MUST indicate the maximum size in octets for an annotation + value by providing the maximum size value in the response code. + Clients MUST NOT store annotation values of a size greater than + the amount indicated by the server. Servers MUST accept a minimum + + + + +Daboo & Gellens Experimental [Page 11] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + annotation data size of at least 1024 octets if annotations can be + written. + + In addition, the server MAY limit the total number of annotations for + a single message. However, the server MUST provide a minimum + annotation count per message of at least 10. + +4.2. ANNOTATE Parameter with the SELECT/EXAMINE Commands + + The ANNOTATE extension defines a single optional SELECT parameter + [RFC4466] "ANNOTATE", which is used to turn on unsolicited responses + for annotations as described in Section 4.4. This optional parameter + results in a per-mailbox state change, i.e., it must be used in each + SELECT/EXAMINE command in order to be effective, irrespective of + whether it was used in a previous SELECT/EXAMINE during the same + session. + + Example: + + C: a SELECT INBOX (ANNOTATE) + S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) + S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft + \Deleted \Seen \*)] + S: * 10268 EXISTS + S: * 1 RECENT + S: * OK [UNSEEN 10268] + S: * OK [UIDVALIDITY 890061587] + S: * OK [UIDNEXT 34643] + S: * OK [ANNOTATIONS 20480 NOPRIVATE] + S: a OK [READ-WRITE] Completed + + In the above example, a SELECT command with the ANNOTATE parameter + is issued. The response from the server includes the required + ANNOTATIONS response that indicates that the server supports + annotations up to a maximum size of 20480 octets, and does not + support private annotations (only shared). + +4.3. ANNOTATION Message Data Item in FETCH Command + + This extension adds an ANNOTATION message data item to the FETCH + command. This allows clients to retrieve annotations for a range of + messages in the currently selected mailbox. + + ANNOTATION + + The ANNOTATION message data item, when used by the client in the + FETCH command, takes an entry specifier and an attribute + specifier. + + + +Daboo & Gellens Experimental [Page 12] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a FETCH 1 (ANNOTATION (/comment value)) + S: * 1 FETCH (ANNOTATION (/comment + (value.priv "My comment" + value.shared "Group note"))) + S: a OK Fetch complete + + In the above example, the content of the "value" attribute for the + "/comment" entry is requested by the client and returned by the + server. Since neither ".shared" nor ".priv" was specified, both + are returned. + + "*" and "%" wild card characters can be used in entry specifiers to + match one or more characters at that position, with the exception + that "%" does not match the "/" hierarchy delimiter. Thus, an entry + specifier of "/%" matches entries such as "/comment" and + "/altsubject", but not "/1/comment". + + Example: + + C: a UID FETCH 1123 (UID ANNOTATION + (/* (value.priv size.priv))) + S: * 12 FETCH (UID 1123 ANNOTATION + (/comment (value.priv "My comment" + size.priv "10") + /altsubject (value.priv "Rhinoceroses!" + size.priv "13") + /vendor/foobar/label.priv + (value.priv "label43" + size.priv "7") + /vendor/foobar/personality + (value.priv "Tallulah Bankhead" + size.priv "17"))) + S: a OK Fetch complete + + In the above example, the contents of the private "value" and + "size" attributes for any entries in the "/" hierarchy are + requested by the client and returned by the server. + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 13] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a FETCH 1 (ANNOTATION (/% value.shared)) + S: * 1 FETCH (ANNOTATION + (/comment (value.shared "Patch Mangler") + /altsubject (value.shared "Patches? We don't + need no steenkin patches!"))) + S: a OK Fetch complete + + In the above example, the contents of the shared "value" + attributes for entries at the top level only of the "/" hierarchy + are requested by the client and returned by the server. + + Entry and attribute specifiers can be lists of atomic specifiers, so + that multiple items of each type may be returned in a single FETCH + command. + + Example: + + C: a FETCH 1 (ANNOTATION + ((/comment /altsubject) value.priv)) + S: * 1 FETCH (ANNOTATION + (/comment (value.priv "What a chowder-head") + /altsubject (value.priv "How to crush beer cans"))) + S: a OK Fetch complete + + In the above example, the contents of the private "value" + attributes for the two entries "/comment" and "/altsubject" are + requested by the client and returned by the server. + +4.4. ANNOTATION Message Data Item in FETCH Response + + The ANNOTATION message data item in the FETCH response displays + information about annotations in a message. + + ANNOTATION parenthesized list + + The response consists of a list of entries, each of which have a + list of attribute-value pairs. + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 14] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a FETCH 1 (ANNOTATION (/comment value)) + S: * 1 FETCH (ANNOTATION (/comment + (value.priv "My comment" + value.shared NIL))) + S: a OK Fetch complete + + In the above example, a single entry with a single attribute-value + pair is returned by the server. Since the client did not specify + a ".shared" or ".priv" suffix, both are returned. Only the + private attribute has a value (the shared value is "NIL"). + + Example: + + C: a FETCH 1 (ANNOTATION + ((/comment /altsubject) value)) + S: * 1 FETCH (ANNOTATION + (/comment (value.priv "My comment" + value.shared NIL) + /altsubject (value.priv "My subject" + value.shared NIL))) + S: a OK Fetch complete + + In the above example, two entries, each with a single attribute- + value pair, are returned by the server. Since the client did not + specify a ".shared" or ".priv" suffix, both are returned. Only + the private attributes have values; the shared attributes are + "NIL". + + Example: + + C: a FETCH 1 (ANNOTATION + (/comment (value size))) + S: * 1 FETCH (ANNOTATION + (/comment + (value.priv "My comment" + value.shared NIL + size.priv "10" + size.shared "0"))) + S: a OK Fetch complete + + In the above example, a single entry with two attribute-value + pairs is returned by the server. Since the client did not specify + a ".shared" or ".priv" suffix, both are returned. Only the + private attributes have values; the shared attributes are "NIL". + + + + + +Daboo & Gellens Experimental [Page 15] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Servers SHOULD send ANNOTATION message data items in unsolicited + FETCH responses if an annotation entry is changed by a third-party, + and the ANNOTATE select parameter was used. This allows servers to + keep clients updated with changes to annotations by other clients. + + Unsolicited ANNOTATION responses MUST NOT include ANNOTATION data + values -- only the entry name of the ANNOTATION that has changed. + This restriction avoids sending ANNOTATION data values (which may be + large) to a client unless the client explicitly asks for the value. + + Example: + + C: a STORE 1 +FLAGS (\Seen) + S: * 1 FETCH (FLAGS (\Seen)) + ANNOTATION (/comment)) + S: a OK STORE complete + + In the above example, an unsolicited ANNOTATION response is + returned during a STORE command. The unsolicited response + contains only the entry name of the annotation that changed, and + not its value. + +4.5. ANNOTATION Message Data Item in STORE + + ANNOTATION + + Sets the specified list of entries by adding or replacing the + specified attributes with the values provided. Clients can use + "NIL" for values of attributes it wants to remove from entries. + + The ANNOTATION message data item used with the STORE command has an + implicit ".SILENT" behavior. This means the server does not generate + an untagged FETCH in response to the STORE command and assumes that + the client updates its own cache if the command succeeds. Though + note, that if the Conditional STORE extension [RFC4551] is present, + then an untagged FETCH response with a MODSEQ data item will be + returned by the server as required by [RFC4551]. + + If the server is unable to store an annotation because the size of + its value is too large, the server MUST return a tagged NO response + with a "[ANNOTATE TOOBIG]" response code. + + If the server is unable to store a new annotation because the maximum + number of allowed annotations has already been reached, the server + MUST return a tagged NO response with a "[ANNOTATE TOOMANY]" response + code. + + + + + +Daboo & Gellens Experimental [Page 16] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a STORE 1 ANNOTATION (/comment + (value.priv "My new comment")) + S: a OK Store complete + + In the above example, the entry "/comment" is created (if not + already present). Its private attribute "value" is created if not + already present, or replaced if it exists. "value.priv" is set to + "My new comment". + + Example: + + C: a STORE 1 ANNOTATION (/comment + (value.shared NIL)) + S: a OK Store complete + + In the above example, the shared "value" attribute of the entry + "/comment" is removed by storing "NIL" into the attribute. + + Multiple entries can be set in a single STORE command by listing + entry-attribute-value pairs in the list. + + Example: + + C: a STORE 1 ANNOTATION (/comment + (value.priv "Get tix Tuesday") + /altsubject + (value.priv "Wots On")) + S: a OK Store complete + + In the above example, the entries "/comment" and "/altsubject" are + created (if not already present) and the private attribute "value" + is created or replaced for each entry. + + Multiple attributes can be set in a single STORE command by listing + multiple attribute-value pairs in the entry list. + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 17] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a STORE 1 ANNOTATION (/comment + (value.priv "My new comment" + value.shared "foo's bar")) + S: a OK Store complete + + In the above example, the entry "/comment" is created (if not + already present) and the private and shared "value" attributes are + created if not already present, or replaced if they exist. + +4.6. ANNOTATION Interaction with COPY + + The COPY command can be used to move messages from one mailbox to + another on the same server. Servers that support the ANNOTATION + extension MUST, for each message being copied, copy all ".priv" + annotation data for the current user only, and all ".shared" + annotation data along with the message to the new mailbox. The only + exceptions to this are if the destination mailbox permissions are + such that either the ".priv" or ".shared" annotations are not + allowed, or if the destination mailbox is of a type that does not + support annotations or does not support storing of annotations (a + mailbox that returns a "NONE" or "READ-ONLY" response code in its + ANNOTATIONS response), or if the destination mailbox cannot support + the size of an annotation because it exceeds the ANNOTATIONS value. + Servers MUST NOT copy ".priv" annotation data for users other than + the current user. + +4.7. ANNOTATION Message Data Item in APPEND + + ANNOTATION + + Sets the specified list of entries and attributes in the resulting + message. + + The APPEND command can include annotations for the message being + appended via the addition of a new append data item [RFC4466]. The + new data item can also be used with the multi-append [RFC3502] + extension that allows multiple messages to be appended via a single + APPEND command. + + + + + + + + + + + +Daboo & Gellens Experimental [Page 18] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a APPEND drafts ANNOTATION (/comment + (value.priv "Don't send until I say so")) {310} + S: + Ready for literal data + C: MIME-Version: 1.0 + ... + C: + S: a OK APPEND completed + + In the above example, a comment with a private value is added to a + new message appended to the mailbox. The ellipsis represents the + bulk of the message. + +4.8. ANNOTATION Criterion in SEARCH + + ANNOTATION + + The ANNOTATION criterion for the SEARCH command allows a client to + search for a specified string in the value of an annotation entry of + a message. + + Messages that have annotations with entries matching , + attributes matching , and the specified string + in their values are returned in the SEARCH results. The "*" + character can be used in the entry name field to match any content in + those items. The "%" character can be used in the entry name field + to match a single level of hierarchy only. + + Only the "value", "value.priv", and "value.shared" attributes can be + searched. Clients MUST NOT specify an attribute other than either + "value", "value.priv", or "value.shared". Servers MUST return a BAD + response if the client tries to search any other attribute. + + Example: + + C: a SEARCH ANNOTATION /comment value "IMAP4" + S: * SEARCH 2 3 5 7 11 13 17 19 23 + S: a OK Search complete + + In the above example, the message numbers of any messages + containing the string "IMAP4" in the shared or private "value" + attribute of the "/comment" entry are returned in the search + results. + + + + + + + +Daboo & Gellens Experimental [Page 19] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Example: + + C: a SEARCH ANNOTATION * value.priv "IMAP4" + S: * SEARCH 1 2 3 5 8 13 21 34 + S: a OK Search complete + + In the above example, the message numbers of any messages + containing the string "IMAP4" in the private "value" attribute of + any entry are returned in the search results. + +4.9. ANNOTATION Key in SORT + + ANNOTATION + + The ANNOTATION criterion for the SORT command [RFC5256] instructs the + server to return the sequence numbers or Unique Identifiers (UIDs) of + messages in a mailbox, sorted using the values of the specified + annotations. The ANNOTATION criterion is available if the server + returns both ANNOTATE-EXPERIMENT-1 and SORT as supported capabilities + in the CAPABILITY command response. + + Messages are sorted using the values of the + attributes in the entries. + + Clients MUST provide either the ".priv" or ".shared" suffix to the + attribute name to ensure that the server knows which specific value + to sort on. + + Only the "value.priv" and "value.shared" attributes can be used for + sorting. Clients MUST NOT specify an attribute other than either + "value.priv" or "value.shared". Servers MUST return a BAD response + if the client tries to sort on any other attribute. + + When either "value.priv" or "value.shared" is being sorted, the + server MUST use the character set value specified in the SORT command + to determine the appropriate sort order. + + Example: + + C: a SORT (ANNOTATION /altsubject value.shared) UTF-8 ALL + S: * SORT 2 3 4 5 1 11 10 6 7 9 8 + S: a OK Sort complete + + In the above example, the message numbers of all messages are + returned, sorted according to the shared "value" attribute of the + "/altsubject" entry. + + + + + +Daboo & Gellens Experimental [Page 20] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + Note that the ANNOTATION sort key must include a fully specified + entry -- wild cards are not allowed. + +4.10. New ACL Rights + + As discussed in Section 3.4, this extension adds a new "n" right to + the list of rights provided by the ACL extensions [RFC2086] and + [RFC4314]. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. + + Non-terminals referenced but not defined below are as defined by + [RFC3501] with the new definitions in [RFC4466] superseding those in + [RFC3501]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + ann-size = "NONE" / + (("READ-ONLY" / nz-number) + [SP "NOPRIVATE"]) + ; response codes indicating the level of + ; support for annotations in a mailbox + + append-ext =/ att-annotate + ; modifies [RFC3501] extension behaviour + + att-annotate = "ANNOTATION" SP + "(" entry-att *(SP entry-att) ")" + + att-search = "value" / "value.priv" / "value.shared" + ; the only attributes that can be searched + + att-sort = "value.priv" / "value.shared" + ; the only attributes that can be sorted + + att-value = attrib SP value + + attrib = astring + ; dot-separated attribute name + ; MUST NOT contain "*" or "%" + + + + + +Daboo & Gellens Experimental [Page 21] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + attribs = attrib / "(" attrib *(SP attrib) ")" + ; one or more attribute specifiers + + capability =/ "ANNOTATE-EXPERIMENT-1" + ; defines the capability for this extension + + entries = entry-match / + "(" entry-match *(SP entry-match) ")" + + entry = astring + ; slash-separated path to entry + ; MUST NOT contain "*" or "%" + + entry-att = entry SP "(" att-value *(SP att-value) ")" + + entry-match = list-mailbox + ; slash-separated path to entry + ; MAY contain "*" or "%" for use as wild cards + + fetch-att =/ "ANNOTATION" SP "(" entries SP attribs ")" + ; modifies original IMAP fetch-att + + msg-att-dynamic =/ "ANNOTATION" SP + ( "(" entry-att *(SP entry-att) ")" / + "(" entry *(SP entry) ")" ) + ; extends FETCH response with annotation data + + resp-text-code =/ "ANNOTATE" SP "TOOBIG" / + "ANNOTATE" SP "TOOMANY" / + "ANNOTATIONS" SP ann-size + ; new response codes + + search-key =/ "ANNOTATION" SP entry-match SP att-search + SP value + ; modifies original IMAP search-key + + select-param =/ "ANNOTATE" + ; defines the select parameter used with + ; ANNOTATE extension + + sort-key =/ "ANNOTATION" SP entry SP att-sort + ; modifies original sort-key + + store-att-flags =/ att-annotate + ; modifies original IMAP STORE command + + value = nstring / literal8 + + + + +Daboo & Gellens Experimental [Page 22] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6. IANA Considerations + + Entry names MUST be specified in a standards track or IESG approved + experimental RFC, or fall under the vendor namespace. Vendor names + MUST be registered. + + Attribute names MUST be specified in a standards track or IESG + approved experimental RFC. + + Each entry registration MUST include a content-type that is used to + indicate the nature of the annotation value. Where applicable, a + charset parameter MUST be included with the content-type. + +6.1. Entry and Attribute Registration Template + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [] Entry [] Attribute + + Name: ______________________________ + + Description: _______________________ + + ____________________________________ + + ____________________________________ + + Content-Type:_______________________ + + Contact person: ____________________ + + email: ____________________ + + + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 23] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.2. Entry Registrations + + The following templates specify the IANA registrations of annotation + entries specified in this document. + +6.2.1. /comment + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: /comment + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.2.2. /flags + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: /flags + + Description: Reserved entry hierarchy. + + Content-Type: - + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + + + + + + + + + +Daboo & Gellens Experimental [Page 24] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.2.3. /altsubject + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: /altsubject + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.2.4. //comment + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //comment + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 25] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.2.5. //flags/seen + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //flags/seen + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.2.6. //flags/answered + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //flags/answered + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 26] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.2.7. //flags/flagged + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //flags/flagged + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.2.8. //flags/forwarded + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [X] Entry [] Attribute + + Name: //flags/forwarded + + Description: Defined in IMAP ANNOTATE extension document. + + Content-Type: text/plain; charset=utf-8 + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 27] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +6.3. Attribute Registrations + + The following templates specify the IANA registrations of annotation + attributes specified in this document. + +6.3.1. value + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [] Entry [X] Attribute + + Name: value + + Description: Defined in IMAP ANNOTATE extension document. + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.3.2. size + + To: iana@iana.org + Subject: IMAP Annotate Registration + + Please register the following IMAP Annotate item: + + [] Entry [X] Attribute + + Name: size + + Description: Defined in IMAP ANNOTATE extension document. + + Contact person: Cyrus Daboo + + email: cyrus@daboo.name + +6.4. Capability Registration + + This document registers "ANNOTATE-EXPERIMENT-1" as an IMAPEXT + capability. + + + + + + + + +Daboo & Gellens Experimental [Page 28] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +7. Internationalization Considerations + + Annotations may contain values that include text strings, and both + searching and sorting are possible with annotations. Servers MUST + follow standard IMAP text normalization, character set conversion, + and collation rules when such operations are carried out, as would be + done for other textual fields being searched or sorted on. + +8. Security Considerations + + Annotations whose values are intended to remain private MUST be + stored in ".priv" values instead of ".shared" values, which may be + accessible to other users. + + Excluding the above issues, the ANNOTATE extension does not raise any + security considerations that are not present in the base IMAP + protocol; these issues are discussed in [RFC3501]. + +9. References + +9.1. Normative References + + [RFC2086] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2244] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC3502] Crispin, M., "Internet Message Access Protocol (IMAP) - + MULTIAPPEND Extension", RFC 3502, March 2003. + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + + +Daboo & Gellens Experimental [Page 29] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + + [RFC5256] Crispin, M. and K. Murchison, "Internet Message Access + Protocol - SORT and THREAD Extensions", RFC 5256, June + 2008. + +9.2. Informative References + + [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional + STORE Operation or Quick Flag Changes Resynchronization", + RFC 4551, June 2006. + +10. Acknowledgments + + Many thanks to Chris Newman for his detailed comments on the first + draft of this document, and to the participants at the ACAP working + dinner in Pittsburgh. The participants of the IMAPext working group + made significant contributions to this work. + +Authors' Addresses + + Cyrus Daboo + Apple Inc. + 1 Infinite Loop + Cupertino, CA 95014 + USA + + EMail: cyrus@daboo.name + URI: http://www.apple.com/ + + + Randall Gellens + QUALCOMM Incorporated + 5775 Morehouse Dr. + San Diego, CA 92121-2779 + USA + + EMail: randy@qualcomm.com + + + + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 30] + +RFC 5257 IMAP ANNOTATE Extension June 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Daboo & Gellens Experimental [Page 31] + diff --git a/docs/rfcs/rfc5258.IMAP4_LIST_command_extension.txt b/docs/rfcs/rfc5258.IMAP4_LIST_command_extension.txt new file mode 100644 index 0000000..a80ec15 --- /dev/null +++ b/docs/rfcs/rfc5258.IMAP4_LIST_command_extension.txt @@ -0,0 +1,1739 @@ + + + + + + +Network Working Group B. Leiba +Request for Comments: 5258 IBM T.J. Watson Research Center +Obsoletes: 3348 A. Melnikov +Updates: 2193 Isode Limited +Category: Standards Track June 2008 + + + Internet Message Access Protocol version 4 - LIST Command Extensions + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + IMAP4 has two commands for listing mailboxes: LIST and LSUB. As we + have added extensions, such as Mailbox Referrals, that have required + specialized lists we have had to expand the number of list commands, + since each extension must add its function to both LIST and LSUB, and + these commands are not, as they are defined, extensible. If we've + needed the extensions to work together, we've had to add a set of + commands to mix the different options, the set increasing in size + with each new extension. This document describes an extension to the + base LIST command that will allow these additions to be done with + mutually compatible options to the LIST command, avoiding the + exponential increase in specialized list commands. + + + + + + + + + + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 1] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +Table of Contents + + 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . 4 + 3. Extended LIST Command . . . . . . . . . . . . . . . . . . . . 4 + 3.1. Initial List of Selection Options . . . . . . . . . . . . 7 + 3.2. Initial List of Return Options . . . . . . . . . . . . . . 8 + 3.3. General Principles for Returning LIST Responses . . . . . 9 + 3.4. Additional Requirements on LIST-EXTENDED Clients . . . . . 9 + 3.5. CHILDINFO Extended Data Item . . . . . . . . . . . . . . . 10 + 4. The CHILDREN Return Option . . . . . . . . . . . . . . . . . . 11 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 12 + 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 19 + 7. Internationalization Considerations . . . . . . . . . . . . . 22 + 8. Security Considerations . . . . . . . . . . . . . . . . . . . 23 + 9. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 23 + 9.1. Guidelines for IANA . . . . . . . . . . . . . . . . . . . 23 + 9.2. Registration Procedure and Change Control . . . . . . . . 23 + 9.3. Registration Template for LIST-EXTENDED Options . . . . . 25 + 9.4. Initial LIST-EXTENDED Option Registrations . . . . . . . . 25 + 9.5. Registration Template for LIST-EXTENDED Extended Data + Item . . . . . . . . . . . . . . . . . . . . . . . . . . . 28 + 9.6. Initial LIST-EXTENDED Extended Data Item Registrations . . 28 + 10. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . . 29 + 11. References . . . . . . . . . . . . . . . . . . . . . . . . . . 29 + 11.1. Normative References . . . . . . . . . . . . . . . . . . . 29 + 11.2. Informative References . . . . . . . . . . . . . . . . . . 30 + + + + + + + + + + + + + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 2] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +1. Introduction and Overview + + The LIST command is extended by amending the syntax to allow options + and multiple patterns to be specified. The list of options replaces + the several commands that are currently used to mix and match the + information requested. The new syntax is backward compatible, with + no ambiguity: the new syntax is being used if one of the following + conditions is true: + + 1. if the first word after the command name begins with a + parenthesis ("LIST selection options") + + 2. if the second word after the command name begins with a + parenthesis ("multiple mailbox patterns") + + 3. if the LIST command has more than 2 parameters ("LIST return + options") + + Otherwise the original syntax is used. + + By adding options to the LIST command, we are announcing the intent + to phase out and eventually to deprecate the RLIST and RLSUB commands + described in [MBRef]. We are also defining the mechanism to request + extended mailbox information, such as is described in the Child + Mailbox Extension [CMbox]. The base LSUB command is not deprecated + by this extension; rather, this extension adds a way to obtain + subscription information with more options, with those server + implementations that support it. Clients that simply need a list of + subscribed mailboxes, as provided by the LSUB command, SHOULD + continue to use that command. + + This document defines an IMAP4 extension that is identified by the + capability string "LIST-EXTENDED". The LIST-EXTENDED extension makes + the following changes to the IMAP4 protocol, which are described in + more detail in Section 3 and Section 4: + + a. defines new syntax for LIST command options. + + b. extends LIST to allow for multiple mailbox patterns. + + c. adds LIST command selection options: SUBSCRIBED, REMOTE, and + RECURSIVEMATCH. + + d. adds LIST command return options: SUBSCRIBED and CHILDREN. + + e. adds new mailbox attributes: "\NonExistent", "\Subscribed", + "\Remote", "\HasChildren", and "\HasNoChildren". + + + + +Leiba & Melnikov Standards Track [Page 3] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + f. adds CHILDINFO extended data item. + +2. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + are used in this document as specified in RFC 2119 [Kwds]. + + The term "canonical LIST pattern" refers to the canonical pattern + constructed internally by the server from the reference and mailbox + name arguments (Section 6.3.8 of [IMAP4]). The [IMAP4] LIST command + returns only mailboxes that match the canonical LIST pattern. + + Other terms are introduced where they are referenced for the first + time. + +3. Extended LIST Command + + This extension updates the syntax of the LIST command to allow for + multiple mailbox patterns to be specified, if they are enclosed in + parentheses. A mailbox name matches a list of mailbox patterns if it + matches at least one mailbox pattern. If a mailbox name matches + multiple mailbox patterns from the list, the server SHOULD return + only a single LIST response. + + Note that the non-extended LIST command is required to treat an empty + ("" string) mailbox name argument as a special request to return the + hierarchy delimiter and the root name of the name given in the + reference parameter (as per [IMAP4]). However, ANY extended LIST + command (extended in any of 3 ways specified in Section 1, or any + combination thereof) MUST NOT treat the empty mailbox name as such a + special request, and any regular processing described in this + document applies. In particular, if an extended LIST command has + multiple mailbox names and one (or more) of them is the empty string, + the empty string MUST be ignored for the purpose of matching. + + Some servers might restrict which patterns are allowed in a LIST + command. If a server doesn't accept a particular pattern, it MUST + silently ignore it. + + The LIST command syntax is also extended in two additional ways: by + adding a parenthesized list of command options between the command + name and the reference name (LIST selection options) and an optional + + + + + + +Leiba & Melnikov Standards Track [Page 4] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + list of options at the end that control what kind of information + should be returned (LIST return options). See the formal syntax in + Section 6 for specific details. + + A LIST selection option tells the server which mailbox names should + be selected by the LIST operation. The server should return + information about all mailbox names that match any of the "canonical + LIST pattern" (as described above) and satisfy additional selection + criteria (if any) specified by the LIST selection options. Let's + call any such mailbox name a "matched mailbox name". When multiple + selection options are specified, the server MUST return information + about mailbox names that satisfy every selection option, unless a + description of a particular specified option prescribes special + rules. An example of an option prescribing special rules is the + RECURSIVEMATCH selection option described later in this section. We + will use the term "selection criteria" when referring collectively to + all selection options specified in a LIST command. + + A LIST return option controls which information is returned for each + matched mailbox name. Note that return options MUST NOT cause the + server to report information about additional mailbox names. If the + client has not specified any return option, only information about + attributes should be returned by the server. (Of course, the server + is allowed to include any other information at will.) + + Both selection and return command options will be defined in this + document and in approved extension documents; each option will be + enabled by a capability string (one capability may enable multiple + options), and a client MUST NOT send an option for which the server + has not advertised support. A server MUST respond to options it does + not recognize with a BAD response. The client SHOULD NOT specify any + option more than once; however, if the client does this, the server + MUST act as if it received the option only once. The order in which + options are specified by the client is not significant. + + In general, each selection option except RECURSIVEMATCH will have a + corresponding return option. The REMOTE selection option is an + anomaly in this regard, and does not have a corresponding return + option. That is because it expands, rather than restricts, the set + of mailboxes that are returned. Future extensions to this + specification should keep parallelism in mind and define a pair of + corresponding options. + + This extension is identified by the capability string + "LIST-EXTENDED", and support for it is a prerequisite for any future + extensions that require specialized forms of the LIST command. Such + extensions MUST refer to this document and MUST add their function + through command options as described herein. Note that extensions + + + +Leiba & Melnikov Standards Track [Page 5] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + that don't require support for an extended LIST command, but use + extended LIST responses (see below), don't need to advertise the + "LIST-EXTENDED" capability string. + + This extension also defines extensions to the LIST response, allowing + a series of extended fields at the end, a parenthesized list of + tagged data (also referred to as "extended data item"). The first + element of an extended field is a tag, which identifies the type of + data. Tags MUST be registered with IANA, as described in Section 9.5 + of this document. An example of such an extended set might be + + tablecloth (("edge" "lacy") ("color" "red"))) (X-Sample "text")) + + or + + tablecloth ("edge" "lacy")) (X-Sample "text" "more text")) + + See the formal syntax, in Section 6, for the full syntactic details. + The server MUST NOT return any extended data item unless the client + has expressed its ability to support extended LIST responses, for + example, by using an extended LIST command. The server MAY return + data in the extended fields that was not directly solicited by the + client in the corresponding LIST command. For example, the client + can enable extra extended fields by using another IMAP extension that + make use of the extended LIST responses. The client MUST ignore all + extended fields it doesn't recognize. + + The LIST-EXTENDED capability also defines several new mailbox + attributes. + + The "\NonExistent" attribute indicates that a mailbox name does not + refer to an existing mailbox. Note that this attribute is not + meaningful by itself, as mailbox names that match the canonical LIST + pattern but don't exist must not be returned unless one of the two + conditions listed below is also satisfied: + + a. The mailbox name also satisfies the selection criteria (for + example, it is subscribed and the "SUBSCRIBED" selection option + has been specified). + + b. "RECURSIVEMATCH" has been specified, and the mailbox name has at + least one descendant mailbox name that does not match the LIST + pattern and does match the selection criteria. + + In practice, this means that the "\NonExistent" attribute is usually + returned with one or more of "\Subscribed", "\Remote", + "\HasChildren", or the CHILDINFO extended data item (see their + description below). + + + +Leiba & Melnikov Standards Track [Page 6] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The "\NonExistent" attribute implies "\NoSelect". The "\NonExistent" + attribute MUST be supported and MUST be accurately computed. + +3.1. Initial List of Selection Options + + The selection options defined in this specification are as follows: + + SUBSCRIBED - causes the LIST command to list subscribed names, + rather than the existing mailboxes. This will often be a subset + of the actual mailboxes. It's also possible for this list to + contain the names of mailboxes that don't exist. In any case, the + list MUST include exactly those mailbox names that match the + canonical list pattern and are subscribed to. This option is + intended to supplement the LSUB command. Of particular note are + the mailbox attributes as returned by this option, compared with + what is returned by LSUB. With the latter, the attributes + returned may not reflect the actual attribute status on the + mailbox name, and the \NoSelect attribute has a second special + meaning (it indicates that this mailbox is not, itself, + subscribed, but that it has descendant mailboxes that are). With + the SUBSCRIBED selection option described here, the attributes are + accurate and complete, and have no special meanings. "LSUB" and + "LIST (SUBSCRIBED)" are, thus, not the same thing, and some + servers must do significant extra work to respond to "LIST + (SUBSCRIBED)". Because of this, clients SHOULD continue to use + "LSUB" unless they specifically want the additional information + offered by "LIST (SUBSCRIBED)". + + This option defines a new mailbox attribute, "\Subscribed", that + indicates that a mailbox name is subscribed to. The "\Subscribed" + attribute MUST be supported and MUST be accurately computed when + the SUBSCRIBED selection option is specified. + + Note that the SUBSCRIBED selection option implies the SUBSCRIBED + return option (see below). + + REMOTE - causes the LIST command to show remote mailboxes as well as + local ones, as described in [MBRef]. This option is intended to + replace the RLIST command and, in conjunction with the SUBSCRIBED + selection option, the RLSUB command. + + This option defines a new mailbox attribute, "\Remote", that + indicates that a mailbox is a remote mailbox. The "\Remote" + attribute MUST be accurately computed when the REMOTE option is + specified. + + The REMOTE selection option has no interaction with other options. + Its effect is to tell the server to apply the other options, if + + + +Leiba & Melnikov Standards Track [Page 7] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + any, to remote mailboxes, in addition to local ones. In + particular, it has no interaction with RECURSIVEMATCH (see below). + A request for (REMOTE RECURSIVEMATCH) is invalid, because a + request for (RECURSIVEMATCH) is. A request for (REMOTE + RECURSIVEMATCH SUBSCRIBED) is asking for all subscribed mailboxes, + both local and remote. + + RECURSIVEMATCH - this option forces the server to return information + about parent mailboxes that don't match other selection options, + but have some submailboxes that do. Information about children is + returned in the CHILDINFO extended data item, as described in + Section 3.5. + + Note 1: In order for a parent mailbox to be returned, it still has + to match the canonical LIST pattern. + + Note 2: When returning the CHILDINFO extended data item, it + doesn't matter whether or not the submailbox matches the canonical + LIST pattern. See also example 9 in Section 5. + + The RECURSIVEMATCH option MUST NOT occur as the only selection + option (or only with REMOTE), as it only makes sense when other + selection options are also used. The server MUST return BAD + tagged response in such case. + + Note that even if the RECURSIVEMATCH option is specified, the + client MUST still be able to handle a case when a CHILDINFO + extended data item is returned and there are no submailboxes that + meet the selection criteria of the subsequent LIST command, as + they can be deleted/renamed after the LIST response was sent, but + before the client had a chance to access them. + +3.2. Initial List of Return Options + + The return options defined in this specification are as follows: + + SUBSCRIBED - causes the LIST command to return subscription state + for all matching mailbox names. The "\Subscribed" attribute MUST + be supported and MUST be accurately computed when the SUBSCRIBED + return option is specified. Further, all mailbox flags MUST be + accurately computed (this differs from the behavior of the LSUB + command). + + CHILDREN - requests mailbox child information as originally proposed + in [CMbox]. See Section 4, below, for details. This option MUST + be supported by all servers. + + + + + +Leiba & Melnikov Standards Track [Page 8] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +3.3. General Principles for Returning LIST Responses + + This section outlines several principles that can be used by server + implementations of this document to decide whether a LIST response + should be returned, as well as how many responses and what kind of + information they may contain. + + 1. At most one LIST response should be returned for each mailbox + name that matches the canonical LIST pattern. Server + implementors must not assume that clients will be able to + assemble mailbox attributes and other information returned in + multiple LIST responses. + + 2. There are only two reasons for including a matching mailbox name + in the responses to the LIST command (note that the server is + allowed to return unsolicited responses at any time, and such + responses are not governed by this rule): + + A. The mailbox name also satisfies the selection criteria. + + B. The mailbox name doesn't satisfy the selection criteria, but + it has at least one descendant mailbox name that satisfies + the selection criteria and that doesn't match the canonical + LIST pattern. + + For more information on this case, see the CHILDINFO extended + data item described in Section 3.5. Note that the CHILDINFO + extended data item can only be returned when the + RECURSIVEMATCH selection option is specified. + + 3. Attributes returned in the same LIST response must be treated + additively. For example, the following response + + S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" + + means that the "Fruit/Peach" mailbox doesn't exist, but it is + subscribed. + +3.4. Additional Requirements on LIST-EXTENDED Clients + + All clients that support this extension MUST treat an attribute with + a stronger meaning as implying any attribute that can be inferred + from it. For example, the client must treat the presence of the + \NoInferiors attribute as if the \HasNoChildren attribute was also + sent by the server. + + + + + + +Leiba & Melnikov Standards Track [Page 9] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The following table summarizes inference rules described in + Section 3. + + +--------------------+-------------------+ + | returned attribute | implied attribute | + +--------------------+-------------------+ + | \NoInferiors | \HasNoChildren | + | \NonExistent | \NoSelect | + +--------------------+-------------------+ + +3.5. CHILDINFO Extended Data Item + + The CHILDINFO extended data item MUST NOT be returned unless the + client has specified the RECURSIVEMATCH selection option. + + The CHILDINFO extended data item in a LIST response describes the + selection criteria that has caused it to be returned and indicates + that the mailbox has at least one descendant mailbox that matches the + selection criteria. + + The LSUB command indicates this condition by using the "\NoSelect" + attribute, but the LIST (SUBSCRIBED) command MUST NOT do that, since + "\NoSelect" retains its original meaning here. Further, the + CHILDINFO extended data item is more general, in that it can be used + with any extended set of selection criteria. + + Note: Some servers allow for mailboxes to exist without requiring + their parent to exist. For example, a mailbox "Customers/ABC" can + exist while the mailbox "Customers" does not. As CHILDINFO extended + data item is not allowed if the RECURSIVEMATCH selection option is + not specified, such servers SHOULD use the "\NonExistent + \HasChildren" attribute pair to signal to the client that there is a + descendant mailbox that matches the selection criteria. See example + 11 in Section 5. + + The returned selection criteria allow the client to distinguish a + solicited response from an unsolicited one, as well as to distinguish + among solicited responses caused by multiple pipelined LIST commands + that specify different criteria. + + Servers SHOULD ONLY return a non-matching mailbox name along with + CHILDINFO if at least one matching child is not also being returned. + That is, servers SHOULD suppress redundant CHILDINFO responses. + + Examples 8 and 10 in Section 5 demonstrate the difference between + present CHILDINFO extended data item and the "\HasChildren" + attribute. + + + + +Leiba & Melnikov Standards Track [Page 10] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The following table summarizes interaction between the "\NonExistent" + attribute and CHILDINFO (the first column indicates whether the + parent mailbox exists): + + +--------+--------------+--------------------+----------------------+ + | exists | meets the | has a child that | returned | + | | selection | meets the | LIST-EXTENDED | + | | criteria | selection criteria | attributes and | + | | | | CHILDINFO | + +--------+--------------+--------------------+----------------------+ + | no | no | no | no LIST response | + | | | | returned | + | yes | no | no | no LIST response | + | | | | returned | + | no | yes | no | (\NonExistent | + | | | | ) | + | yes | yes | no | () | + | no | no | yes | (\NonExistent) + | + | | | | CHILDINFO | + | yes | no | yes | () + CHILDINFO | + | no | yes | yes | (\NonExistent | + | | | | ) + CHILDINFO | + | yes | yes | yes | () + CHILDINFO | + +--------+--------------+--------------------+----------------------+ + + where is one or more attributes that correspond to the + selection criteria; for example, for the SUBSCRIBED option the + is \Subscribed. + +4. The CHILDREN Return Option + + The CHILDREN return option implements the Child Mailbox Extension, + originally proposed by Mike Gahrns and Raymond Cheng, of Microsoft + Corporation. Most of the information in this section is taken + directly from their original specification [CMbox]. The CHILDREN + return option is simply an indication that the client wants this + information; a server MAY provide it even if the option is not + specified. + + Many IMAP4 [IMAP4] clients present to the user a hierarchical view of + the mailboxes that a user has access to. Rather than initially + presenting to the user the entire mailbox hierarchy, it is often + preferable to show to the user a collapsed outline list of the + mailbox hierarchy (particularly if there is a large number of + mailboxes). The user can then expand the collapsed outline hierarchy + as needed. It is common to include within the collapsed hierarchy a + visual clue (such as a ''+'') to indicate that there are child + mailboxes under a particular mailbox. When the visual clue is + + + +Leiba & Melnikov Standards Track [Page 11] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + clicked, the hierarchy list is expanded to show the child mailboxes. + The CHILDREN return option provides a mechanism for a client to + efficiently determine whether a particular mailbox has children, + without issuing a LIST "" * or a LIST "" % for each mailbox name. + The CHILDREN return option defines two new attributes that MUST be + returned within a LIST response: \HasChildren and \HasNoChildren. + Although these attributes MAY be returned in response to any LIST + command, the CHILDREN return option is provided to indicate that the + client particularly wants this information. If the CHILDREN return + option is present, the server MUST return these attributes even if + their computation is expensive. + + \HasChildren + + The presence of this attribute indicates that the mailbox has child + mailboxes. A server SHOULD NOT set this attribute if there are + child mailboxes and the user does not have permission to access + any of them. In this case, \HasNoChildren SHOULD be used. In + many cases, however, a server may not be able to efficiently + compute whether a user has access to any child mailbox. Note + that even though the \HasChildren attribute for a mailbox must + be correct at the time of processing of the mailbox, a client + must be prepared to deal with a situation when a mailbox is + marked with the \HasChildren attribute, but no child mailbox + appears in the response to the LIST command. This might happen, + for example, due to children mailboxes being deleted or made + inaccessible to the user (using access control) by another + client before the server is able to list them. + + \HasNoChildren + + The presence of this attribute indicates that the mailbox has NO + child mailboxes that are accessible to the currently + authenticated user. + + It is an error for the server to return both a \HasChildren and a + \HasNoChildren attribute in the same LIST response. + + Note: the \HasNoChildren attribute should not be confused with the + IMAP4 [IMAP4] defined attribute \NoInferiors, which indicates that no + child mailboxes exist now and none can be created in the future. + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 12] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +5. Examples + + 1: The first example shows the complete local hierarchy that will + be used for the other examples. + + C: A01 LIST "" "*" + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST () "/" "Fruit" + S: * LIST () "/" "Fruit/Apple" + S: * LIST () "/" "Fruit/Banana" + S: * LIST () "/" "Tofu" + S: * LIST () "/" "Vegetable" + S: * LIST () "/" "Vegetable/Broccoli" + S: * LIST () "/" "Vegetable/Corn" + S: A01 OK done + + 2: In the next example, we will see the subscribed mailboxes. This + is similar to, but not equivalent with, . Note + that the mailbox called "Fruit/Peach" is subscribed to, but does + not actually exist (perhaps it was deleted while still + subscribed). The "Fruit" mailbox is not subscribed to, but it + has two subscribed children. The "Vegetable" mailbox is + subscribed and has two children; one of them is subscribed as + well. + + C: A02 LIST (SUBSCRIBED) "" "*" + S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" + S: * LIST (\Subscribed) "/" "Fruit/Banana" + S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" + S: * LIST (\Subscribed) "/" "Vegetable" + S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" + S: A02 OK done + + 3: The next example shows the use of the CHILDREN option. The + client, without having to list the second level of hierarchy, + now knows which of the top-level mailboxes have submailboxes + (children) and which do not. Note that it's not necessary for + the server to return the \HasNoChildren attribute for the inbox, + because the \NoInferiors attribute already implies that, and has + a stronger meaning. + + C: A03 LIST () "" "%" RETURN (CHILDREN) + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST (\HasChildren) "/" "Fruit" + S: * LIST (\HasNoChildren) "/" "Tofu" + S: * LIST (\HasChildren) "/" "Vegetable" + S: A03 OK done + + + + +Leiba & Melnikov Standards Track [Page 13] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + 4: In this example, we see more mailboxes that reside on another + server. This is similar to the command . + + C: A04 LIST (REMOTE) "" "%" RETURN (CHILDREN) + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST (\HasChildren) "/" "Fruit" + S: * LIST (\HasNoChildren) "/" "Tofu" + S: * LIST (\HasChildren) "/" "Vegetable" + S: * LIST (\Remote) "/" "Bread" + S: * LIST (\HasChildren \Remote) "/" "Meat" + S: A04 OK done + + 5: The following example also requests the server to include + mailboxes that reside on another server. The server returns + information about all mailboxes that are subscribed. This is + similar to the command . We also see the use of + two selection options. + + C: A05 LIST (REMOTE SUBSCRIBED) "" "*" + S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" + S: * LIST (\Subscribed) "/" "Fruit/Banana" + S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" + S: * LIST (\Subscribed) "/" "Vegetable" + S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" + S: * LIST (\Remote \Subscribed) "/" "Bread" + S: A05 OK done + + 6: The following example requests the server to include mailboxes + that reside on another server. The server is asked to return + subscription information for all returned mailboxes. This is + different from the example above. + + Note that the output of this command is not a superset of the + output in the previous example, as it doesn't include LIST + response for the non-existent "Fruit/Peach". + + C: A06 LIST (REMOTE) "" "*" RETURN (SUBSCRIBED) + S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" + S: * LIST () "/" "Fruit" + S: * LIST () "/" "Fruit/Apple" + S: * LIST (\Subscribed) "/" "Fruit/Banana" + S: * LIST () "/" "Tofu" + S: * LIST (\Subscribed) "/" "Vegetable" + S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" + S: * LIST () "/" "Vegetable/Corn" + S: * LIST (\Remote \Subscribed) "/" "Bread" + S: * LIST (\Remote) "/" "Meat" + S: A06 OK done + + + +Leiba & Melnikov Standards Track [Page 14] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + 7: In the following example, the client has specified multiple + mailbox patterns. Note that this example does not use the + mailbox hierarchy used in the previous examples. + + C: BBB LIST "" ("INBOX" "Drafts" "Sent/%") + S: * LIST () "/" "INBOX" + S: * LIST (\NoInferiors) "/" "Drafts" + S: * LIST () "/" "Sent/March2004" + S: * LIST (\Marked) "/" "Sent/December2003" + S: * LIST () "/" "Sent/August2004" + S: BBB OK done + + 8: The following example demonstrates the difference between the + \HasChildren attribute and the CHILDINFO extended data item. + + Let's assume there is the following hierarchy: + + C: C01 LIST "" "*" + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST () "/" "Foo" + S: * LIST () "/" "Foo/Bar" + S: * LIST () "/" "Foo/Baz" + S: * LIST () "/" "Moo" + S: C01 OK done + + If the client asks RETURN (CHILDREN), it will get this: + + C: CA3 LIST "" "%" RETURN (CHILDREN) + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST (\HasChildren) "/" "Foo" + S: * LIST (\HasNoChildren) "/" "Moo" + S: CA3 OK done + + A) Let's also assume that the mailbox "Foo/Baz" is the only + subscribed mailbox. Then we get this result: + + C: C02 LIST (SUBSCRIBED) "" "*" + S: * LIST (\Subscribed) "/" "Foo/Baz" + S: C02 OK done + + Now, if the client issues , the server will + return no mailboxes (as the mailboxes "Moo", "Foo", and "Inbox" are + NOT subscribed). However, if the client issues this: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" + S: * LIST () "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) + S: C04 OK done + + + + +Leiba & Melnikov Standards Track [Page 15] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + (i.e., the mailbox "Foo" is not subscribed, but it has a child that + is.) + + A1) If the mailbox "Foo" had also been subscribed, the last command + would return this: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" + S: * LIST (\Subscribed) "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) + S: C04 OK done + + or even this: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" + S: * LIST (\Subscribed \HasChildren) "/" "Foo" ("CHILDINFO" + ("SUBSCRIBED")) + S: C04 OK done + + A2) If we assume instead that the mailbox "Foo" is not part of the + original hierarchy and is not subscribed, the last command will give + this result: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" + S: * LIST (\NonExistent) "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) + S: C04 OK done + + B) Now, let's assume that no mailbox is subscribed. In this case, + the command will return no + responses, as there are no subscribed children (even though "Foo" has + children). + + C) And finally, suppose that only the mailboxes "Foo" and "Moo" are + subscribed. In that case, we see this result: + + C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" RETURN (CHILDREN) + S: * LIST (\HasChildren \Subscribed) "/" "Foo" + S: * LIST (\HasNoChildren \Subscribed) "/" "Moo" + S: C04 OK done + + (which means that the mailbox "Foo" has children, but none of them is + subscribed). + + 9: The following example demonstrates that the CHILDINFO extended + data item is returned whether or not children mailboxes match + the canonical LIST pattern. + + + + + + + +Leiba & Melnikov Standards Track [Page 16] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Let's assume there is the following hierarchy: + + C: D01 LIST "" "*" + S: * LIST (\Marked \NoInferiors) "/" "inbox" + S: * LIST () "/" "foo2" + S: * LIST () "/" "foo2/bar1" + S: * LIST () "/" "foo2/bar2" + S: * LIST () "/" "baz2" + S: * LIST () "/" "baz2/bar2" + S: * LIST () "/" "baz2/bar22" + S: * LIST () "/" "baz2/bar222" + S: * LIST () "/" "eps2" + S: * LIST () "/" "eps2/mamba" + S: * LIST () "/" "qux2/bar2" + S: D01 OK done + + And that the following mailboxes are subscribed: + + C: D02 LIST (SUBSCRIBED) "" "*" + S: * LIST (\Subscribed) "/" "foo2/bar1" + S: * LIST (\Subscribed) "/" "foo2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar22" + S: * LIST (\Subscribed) "/" "baz2/bar222" + S: * LIST (\Subscribed) "/" "eps2" + S: * LIST (\Subscribed) "/" "eps2/mamba" + S: * LIST (\Subscribed) "/" "qux2/bar2" + S: D02 OK done + + The client issues the following command first: + + C: D03 LIST (RECURSIVEMATCH SUBSCRIBED) "" "*2" + S: * LIST () "/" "foo2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "foo2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar22" + S: * LIST (\Subscribed) "/" "baz2/bar222" + S: * LIST (\Subscribed) "/" "eps2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "qux2/bar2" + S: D03 OK done + + and the server may also include (but this would violate a SHOULD NOT + in Section 3.5, because CHILDINFO is redundant) + + S: * LIST () "/" "baz2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\NonExistent) "/" "qux2" ("CHILDINFO" ("SUBSCRIBED")) + + + + + +Leiba & Melnikov Standards Track [Page 17] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The CHILDINFO extended data item is returned for mailboxes "foo2", + "baz2", and "eps2", because all of them have subscribed children, + even though for the mailbox "foo2" only one of the two subscribed + children matches the pattern, for the mailbox "baz2" all the + subscribed children match the pattern, and for the mailbox "eps2" + none of the subscribed children matches the pattern. + + Note that if the client issues + + C: D03 LIST (RECURSIVEMATCH SUBSCRIBED) "" "*" + S: * LIST () "/" "foo2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "foo2/bar1" + S: * LIST (\Subscribed) "/" "foo2/bar2" + S: * LIST () "/" "baz2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "baz2/bar2" + S: * LIST (\Subscribed) "/" "baz2/bar22" + S: * LIST (\Subscribed) "/" "baz2/bar222" + S: * LIST (\Subscribed) "/" "eps2" ("CHILDINFO" ("SUBSCRIBED")) + S: * LIST (\Subscribed) "/" "eps2/mamba" + S: * LIST (\Subscribed) "/" "qux2/bar2" + S: D03 OK done + + The LIST responses for mailboxes "foo2", "baz2", and "eps2" still + have the CHILDINFO extended data item, even though this information + is redundant and the client can determine it by itself. + + 10: The following example shows usage of multiple mailbox patterns. + It also demonstrates that the presence of the CHILDINFO extended + data item doesn't necessarily imply \HasChildren. + + C: a1 LIST "" ("foo" "foo/*") + S: * LIST () "/" foo + S: a1 OK done + + C: a2 LIST (SUBSCRIBED) "" "foo/*" + S: * LIST (\Subscribed \NonExistent) "/" foo/bar + S: a2 OK done + + C: a3 LIST (SUBSCRIBED RECURSIVEMATCH) "" foo RETURN (CHILDREN) + S: * LIST (\HasNoChildren) "/" foo ("CHILDINFO" ("SUBSCRIBED")) + S: a3 OK done + + 11: The following example shows how a server that supports missing + mailbox hierarchy elements can signal to a client that didn't + specify the RECURSIVEMATCH selection option that there is a + child mailbox that matches the selection criteria. + + + + + +Leiba & Melnikov Standards Track [Page 18] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + C: a1 LIST (REMOTE) "" * + S: * LIST () "/" music/rock + S: * LIST (\Remote) "/" also/jazz + S: a1 OK done + + C: a2 LIST () "" % + S: * LIST (\NonExistent \HasChildren) "/" music + S: a2 OK done + + C: a3 LIST (REMOTE) "" % + S: * LIST (\NonExistent \HasChildren) "/" music + S: * LIST (\NonExistent \HasChildren) "/" also + S: a3 OK done + + C: a3.1 LIST "" (% music/rock) + S: * LIST () "/" music/rock + S: a3.1 OK done + + Because "music/rock" is the only mailbox under "music", there's no + need for the server to also return "music". However clients must + handle both cases. + +6. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) as described in [ABNF]. Terms not defined here are taken + from [IMAP4]. In particular, note that the version of "mailbox-list" + below, which defines the payload of the LIST response, updates the + version defined in the IMAP specification. It is pointed to by + "mailbox-data", which is defined in [IMAP4]. + + "vendor-token" is defined in [ACAP]. Note that this normative + reference to ACAP will be an issue in moving this spec forward, since + it introduces a dependency on ACAP. The definitions of + "vendor-token" and of the IANA registry must eventually go somewhere + else, in a document that can be moved forward on the standards track + independently of ACAP. + + + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 19] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + childinfo-extended-item = "CHILDINFO" SP "(" + list-select-base-opt-quoted + *(SP list-select-base-opt-quoted) ")" + ; Extended data item (mbox-list-extended-item) + ; returned when the RECURSIVEMATCH + ; selection option is specified. + ; Note 1: the CHILDINFO tag can be returned + ; with and without surrounding quotes, as per + ; mbox-list-extended-item-tag production. + ; Note 2: The selection options are always returned + ; quoted, unlike their specification in + ; the extended LIST command. + + child-mbox-flag = "\HasChildren" / "\HasNoChildren" + ; attributes for CHILDREN return option, at most one + ; possible per LIST response + + eitem-standard-tag = atom + ; a tag for extended list data defined in a Standard + ; Track or Experimental RFC. + + eitem-vendor-tag = vendor-token "-" atom + ; a vendor-specific tag for extended list data + + list = "LIST" [SP list-select-opts] SP mailbox SP mbox-or-pat + [SP list-return-opts] + + list-return-opts = "RETURN" SP + "(" [return-option *(SP return-option)] ")" + ; list return options, e.g., CHILDREN + + list-select-base-opt = "SUBSCRIBED" / option-extension + ; options that can be used by themselves + + list-select-base-opt-quoted = DQUOTE list-select-base-opt DQUOTE + + list-select-independent-opt = "REMOTE" / option-extension + ; options that do not syntactically interact with + ; other options + + list-select-mod-opt = "RECURSIVEMATCH" / option-extension + ; options that require a list-select-base-opt + ; to also be present + + list-select-opt = list-select-base-opt / list-select-independent-opt + / list-select-mod-opt + ; An option registration template is described in + ; Section 9.3 of this document. + + + +Leiba & Melnikov Standards Track [Page 20] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + list-select-opts = "(" [ + (*(list-select-opt SP) list-select-base-opt + *(SP list-select-opt)) + / (list-select-independent-opt + *(SP list-select-independent-opt)) + ] ")" + ; Any number of options may be in any order. + ; If a list-select-mod-opt appears, then a + ; list-select-base-opt must also appear. + ; This allows these: + ; () + ; (REMOTE) + ; (SUBSCRIBED) + ; (SUBSCRIBED REMOTE) + ; (SUBSCRIBED RECURSIVEMATCH) + ; (SUBSCRIBED REMOTE RECURSIVEMATCH) + ; But does NOT allow these: + ; (RECURSIVEMATCH) + ; (REMOTE RECURSIVEMATCH) + + mailbox-list = "(" [mbx-list-flags] ")" SP + (DQUOTE QUOTED-CHAR DQUOTE / nil) SP mailbox + [SP mbox-list-extended] + ; This is the list information pointed to by the ABNF + ; item "mailbox-data", which is defined in [IMAP4] + + mbox-list-extended = "(" [mbox-list-extended-item + *(SP mbox-list-extended-item)] ")" + + mbox-list-extended-item = mbox-list-extended-item-tag SP + tagged-ext-val + + mbox-list-extended-item-tag = astring + ; The content MUST conform to either "eitem-vendor-tag" + ; or "eitem-standard-tag" ABNF productions. + ; A tag registration template is described in this + ; document in Section 9.5. + + mbx-list-oflag =/ child-mbox-flag / "\Subscribed" / "\Remote" + + mbx-list-sflag =/ "\NonExistent" + + mbox-or-pat = list-mailbox / patterns + + option-extension = (option-standard-tag / option-vendor-tag) + [SP option-value] + + + + + +Leiba & Melnikov Standards Track [Page 21] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + option-standard-tag = atom + ; an option defined in a Standards Track or + ; Experimental RFC + + option-val-comp = astring / + option-val-comp *(SP option-val-comp) / + "(" option-val-comp ")" + + option-value = "(" option-val-comp ")" + + option-vendor-tag = vendor-token "-" atom + ; a vendor-specific option, non-standard + + patterns = "(" list-mailbox *(SP list-mailbox) ")" + + return-option = "SUBSCRIBED" / "CHILDREN" / option-extension + + tagged-ext-comp = astring / + tagged-ext-comp *(SP tagged-ext-comp) / + "(" tagged-ext-comp ")" + ; Extensions that follow this general + ; syntax should use nstring instead of + ; astring when appropriate in the context + ; of the extension. + ; Note that a message set or a "number" + ; can always be represented as an "atom". + ; A URL should be represented as + ; a "quoted" string. + + tagged-ext-simple = sequence-set / number + + tagged-ext-val = tagged-ext-simple / + "(" [tagged-ext-comp] ")" + +7. Internationalization Considerations + + The LIST command selection option types defined in this specification + involve simple tests of mailbox properties. However, future + extensions to LIST-EXTENDED may define selection options that do more + sophisticated tests. In the case of a test that requires matching + text, in the presence of the COMPARATOR [I18N] extension, the active + comparator must be used to do comparisons. Such LIST-EXTENDED + extensions MUST indicate in their specification the interaction with + the COMPARATOR [I18N] extension. + + + + + + + +Leiba & Melnikov Standards Track [Page 22] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +8. Security Considerations + + This document describes syntactic changes to the specification of the + IMAP4 commands LIST, LSUB, RLIST, and RLSUB, and the modified LIST + command has the same security considerations as those commands. They + are described in [IMAP4] and [MBRef]. + + The Child Mailbox Extension provides a client a more efficient means + of determining whether a particular mailbox has children. If a + mailbox has children, but the currently authenticated user does not + have access to any of them, the server SHOULD respond with a + \HasNoChildren attribute. In many cases, however, a server may not + be able to efficiently compute whether a user has access to any child + mailbox. If such a server responds with a \HasChildren attribute, + when in fact the currently authenticated user does not have access to + any child mailboxes, potentially more information is conveyed about + the mailbox than intended. In most situations, this will not be a + security concern, because if information regarding whether a mailbox + has children is considered sensitive, a user would not be granted + access to that mailbox in the first place. + + The CHILDINFO extended data item has the same security considerations + as the \HasChildren attribute described above. + +9. IANA Considerations + +9.1. Guidelines for IANA + + IANA has created two new registries for LIST-EXTENDED options and + LIST-EXTENDED response data. The templates and the initial + registrations are detailed below. + +9.2. Registration Procedure and Change Control + + Registration of a LIST-EXTENDED option is done by filling in the + template in Section 9.3 and sending it via electronic mail to + iana@iana.org. Registration of a LIST-EXTENDED extended data item is + done by filling in the template in Section 9.5 and sending it via + electronic mail to iana@iana.org. IANA has the right to reject + obviously bogus registrations, but will perform no review of claims + made in the registration form. + + A LIST-EXTENDED option/extended data item name that starts with "V-" + is reserved for vendor-specific options/extended data items. All + options, whether they are vendor specific or global, should be + registered with IANA. If a LIST-EXTENDED extended data item is + returned as a result of requesting a particular LIST-EXTENDED option, + + + + +Leiba & Melnikov Standards Track [Page 23] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + the name of the option SHOULD be used as the name of the + LIST-EXTENDED extended data item. + + Each vendor-specific option/extended data item MUST start with its + vendor-token ("vendor prefix"). The vendor-token MUST be registered + with IANA, using the [ACAP] vendor subtree registry. + + Standard LIST-EXTENDED option/extended data item names are case + insensitive. If the vendor prefix is omitted from a vendor-specific + LIST-EXTENDED option/extended data item name, the rest is case + insensitive. The vendor prefix itself is not case sensitive, as it + might contain non-ASCII characters. While the registration + procedures do not require it, authors of + LIST-EXTENDED options/extended data items are encouraged to seek + community review and comment whenever that is feasible. Authors may + seek community review by posting a specification of their proposed + mechanism as an + Internet-Draft. LIST-EXTENDED option/extended data items intended + for widespread use should be standardized through the normal IETF + process, when appropriate. + + Comments on registered LIST-EXTENDED options/extended response data + should first be sent to the "owner" of the mechanism and/or to the + IMAPEXT WG mailing list. Submitters of comments may, after a + reasonable attempt to contact the owner, request IANA to attach their + comment to the registration itself. If IANA approves of this, the + comment will be made accessible in conjunction with the registration + LIST-EXTENDED options/extended response data itself. + + Once a LIST-EXTENDED registration has been published by IANA, the + author may request a change to its definition. The change request + follows the same procedure as the registration request. + + The owner of a LIST-EXTENDED registration may pass responsibility for + the registered option/extended data item to another person or agency + by informing IANA; this can be done without discussion or review. + + The IESG may reassign responsibility for a LIST-EXTENDED + option/extended data item. The most common case of this will be to + enable changes to be made to mechanisms where the author of the + registration has died, has moved out of contact, or is otherwise + unable to make changes that are important to the community. + + LIST-EXTENDED registrations may not be deleted; mechanisms that are + no longer believed appropriate for use can be declared OBSOLETE by a + change to their "intended use" field. Such LIST-EXTENDED + options/extended data items will be clearly marked in the lists + published by IANA. + + + +Leiba & Melnikov Standards Track [Page 24] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + The IESG is considered to be the owner of all LIST-EXTENDED + options/extended data items that are on the IETF standards track. + +9.3. Registration Template for LIST-EXTENDED Options + + To: iana@iana.org + Subject: Registration of LIST-EXTENDED option X + + LIST-EXTENDED option name: + + LIST-EXTENDED option type: (One of SELECTION or RETURN) + + Implied return options(s), if the option type is SELECTION: (zero or + more) + + LIST-EXTENDED option description: + + Published specification (optional, recommended): + + Security considerations: + + Intended usage: + (One of COMMON, LIMITED USE, or OBSOLETE) + + Person and email address to contact for further information: + + Owner/Change controller: + + (Any other information that the author deems interesting may be added + below this line.) + +9.4. Initial LIST-EXTENDED Option Registrations + + The LIST-EXTENDED option registry has been populated with the + following entries: + + 1. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option SUBSCRIBED + + LIST-EXTENDED option name: SUBSCRIBED + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): SUBSCRIBED + + LIST-EXTENDED option description: Causes the LIST command to list + subscribed mailboxes, rather than the actual mailboxes. + + + + +Leiba & Melnikov Standards Track [Page 25] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Published specification: RFC 5258, Section 3. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + + 2. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option REMOTE + + LIST-EXTENDED option name: REMOTE + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): (none) + + LIST-EXTENDED option description: Causes the LIST command to + return remote mailboxes as well as local ones, as described in + RFC 2193. + + Published specification: RFC 5258, Section 3. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + + 3. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option SUBSCRIBED + + LIST-EXTENDED option name: SUBSCRIBED + + LIST-EXTENDED option type: RETURN + + LIST-EXTENDED option description: Causes the LIST command to + return subscription state. + + Published specification: RFC 5258, Section 3. + + Security considerations: RFC 5258, Section 8. + + + +Leiba & Melnikov Standards Track [Page 26] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + + 4. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option RECURSIVEMATCH + + LIST-EXTENDED option name: RECURSIVEMATCH + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): (none) + + LIST-EXTENDED option description: Requests that CHILDINFO + extended data item (childinfo-extended-item) is to be returned. + + Published specification: RFC 5258, Section 3. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + + 5. To: iana@iana.org + Subject: Registration of LIST-EXTENDED option CHILDREN + + LIST-EXTENDED option name: CHILDREN + + LIST-EXTENDED option type: RETURN + + LIST-EXTENDED option description: Requests mailbox child + information. + + Published specification: RFC 5258, Section 3 and Section 4. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + + +Leiba & Melnikov Standards Track [Page 27] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Owner/Change controller: iesg@ietf.org + +9.5. Registration Template for LIST-EXTENDED Extended Data Item + + To: iana@iana.org + Subject: Registration of X LIST-EXTENDED extended data item + + LIST-EXTENDED extended data item tag: + + LIST-EXTENDED extended data item description: + + Which LIST-EXTENDED option(s) (and their types) causes this extended + data item to be returned (if any): + + Published specification (optional, recommended): + + Security considerations: + + Intended usage: + (One of COMMON, LIMITED USE, or OBSOLETE) + + Person and email address to contact for further information: + + Owner/Change controller: + + (Any other information that the author deems interesting may be added + below this line.) + +9.6. Initial LIST-EXTENDED Extended Data Item Registrations + + The LIST-EXTENDED extended data item registry has been populated with + the following entries: + + 1. To: iana@iana.org + Subject: Registration of CHILDINFO LIST-EXTENDED extended data + item + + LIST-EXTENDED extended data item tag: CHILDINFO + + LIST-EXTENDED extended data item description: The CHILDINFO + extended data item describes the selection criteria that has + caused it to be returned and indicates that the mailbox has one + or more child mailboxes that match the selection criteria. + + Which LIST-EXTENDED option(s) (and their types) causes this + extended data item to be returned (if any): RECURSIVEMATCH + selection option + + + + +Leiba & Melnikov Standards Track [Page 28] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + + Published specification: RFC 5258, Section 3.5. + + Security considerations: RFC 5258, Section 8. + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + +10. Acknowledgements + + Mike Gahrns and Raymond Cheng of Microsoft Corporation originally + devised the Child Mailbox Extension and proposed it in 1997; the + idea, as well as most of the text in Section 4, is theirs. + + This document is the result of discussions on the IMAP4 and IMAPEXT + mailing lists and is meant to reflect consensus of those groups. In + particular, Mark Crispin, Philip Guenther, Cyrus Daboo, Timo + Sirainen, Ken Murchison, Rob Siemborski, Steve Hole, Arnt + Gulbrandsen, Larry Greenfield, Dave Cridland, and Pete Maclean were + active participants in those discussions or made suggestions to this + document. + +11. References + +11.1. Normative References + + [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [ACAP] Newman, C. and J. Myers, "ACAP -- Application Configuration + Access Protocol", RFC 2244, November 1997. + + [I18N] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet + Message Access Protocol Internationalization", RFC 5255, + June 2008. + + [IMAP4] Crispin, M., "Internet Message Access Protocol - Version + 4rev1", RFC 3501, March 2003. + + [Kwds] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", RFC 2119, March 1997. + + [MBRef] Gahrns, M., "IMAP4 Mailbox Referrals", RFC 2193, + September 1997. + + + + +Leiba & Melnikov Standards Track [Page 29] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +11.2. Informative References + + [CMbox] Gahrns, M. and R. Cheng, "The Internet Message Action + Protocol (IMAP4) Child Mailbox Extension", RFC 3348, + July 2002. + +Authors' Addresses + + Barry Leiba + IBM T.J. Watson Research Center + 19 Skyline Drive + Hawthorne, NY 10532 + US + + Phone: +1 914 784 7941 + EMail: leiba@watson.ibm.com + + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + + + + + + + + + + + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 30] + +RFC 5258 IMAP4 LIST Command Extensions June 2008 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2008). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Leiba & Melnikov Standards Track [Page 31] + diff --git a/docs/rfcs/rfc5423.IM_Store_Events.txt b/docs/rfcs/rfc5423.IM_Store_Events.txt new file mode 100644 index 0000000..0326d94 --- /dev/null +++ b/docs/rfcs/rfc5423.IM_Store_Events.txt @@ -0,0 +1,955 @@ + + + + + + +Network Working Group R. Gellens +Request for Comments: 5423 QUALCOMM Inc. +Category: Standards Track C. Newman + Sun Microsystems + March 2009 + + + Internet Message Store Events + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (c) 2009 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents in effect on the date of + publication of this document (http://trustee.ietf.org/license-info). + Please review these documents carefully, as they describe your rights + and restrictions with respect to this document. + +Abstract + + One of the missing features in the existing Internet mail and + messaging standards is a facility for server-to-server and server-to- + client event notifications related to message store events. As the + scope of Internet mail expands to support more diverse media (such as + voice mail) and devices (such as cell phones) and to provide rich + interactions with other services (such as web portals and legal + compliance systems), the need for an interoperable notification + system increases. This document attempts to enumerate the types of + events that interest real-world consumers of such a system. + + This document describes events and event parameters that are useful + for several cases, including notification to administrative systems + and end users. This is not intended as a replacement for a message + access facility such as IMAP. + + + + + + + +Gellens & Newman Standards Track [Page 1] + +RFC 5423 Internet Message Store Events March 2009 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 1.1. Conventions Used in This Document . . . . . . . . . . . . 3 + 2. Terminology . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 3. Event Model . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 4. Event Types . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 4.1. Message Addition and Deletion . . . . . . . . . . . . . . 5 + 4.2. Message Flags . . . . . . . . . . . . . . . . . . . . . . 7 + 4.3. Access Accounting . . . . . . . . . . . . . . . . . . . . 8 + 4.4. Mailbox Management . . . . . . . . . . . . . . . . . . . . 8 + 5. Event Parameters . . . . . . . . . . . . . . . . . . . . . . . 10 + 6. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 14 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 14 + 8. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 15 + 9. References . . . . . . . . . . . . . . . . . . . . . . . . . . 15 + 9.1. Normative References . . . . . . . . . . . . . . . . . . . 15 + 9.2. Informative References . . . . . . . . . . . . . . . . . . 15 + Appendix A. Future Extensions . . . . . . . . . . . . . . . . . . 17 + +1. Introduction + + A message store is used to organize Internet Messages [RFC5322] into + one or more mailboxes (possibly hierarchical), annotate them in + various ways, and provide access to these messages and associated + metadata. Three different standards-based protocols have been widely + deployed to remotely access a message store. The Post Office + Protocol (POP) [RFC1939] provides simple download-and-delete access + to a single mail drop (which is a subset of the functionality + typically associated with a message store). The Internet Message + Access Protocol (IMAP) [RFC3501] provides an extensible feature-rich + model for online, offline, and disconnected access to a message store + with minimal constraints on any associated "fat-client" user + interface. Finally, mail access applications built on top of the + Hypertext Transfer Protocol (HTTP) [RFC2616] that run in standards- + based web browsers provide a third standards-based access mechanism + for online-only access. + + While simple and/or ad-hoc mechanisms for notifications have sufficed + to some degree in the past (e.g., "Simple New Mail Notification" + [RFC4146], "IMAP4 IDLE Command" [RFC2177]), as the scope and + importance of message stores expand, the demand for a more complete + store notification system increases. Some of the driving forces + behind this demand include: + + o Mobile devices with intermittent network connectivity that have + "new mail" or "message count" indicators. + + + + +Gellens & Newman Standards Track [Page 2] + +RFC 5423 Internet Message Store Events March 2009 + + + o Unified messaging systems that include both Internet and voice + mail require support for a message-waiting indicator on phones. + + o Interaction with systems for event-based or utility-computing + billing. + + o Simplification of the process of passing message store events to + non-Internet notification systems. + + o A calendar system may wish to subscribe to MessageNew + notifications in order to support iMIP [RFC2447]. + + o Some jurisdictions have laws or regulations for information + protection and auditing that require interoperable protocols + between message stores built by messaging experts and compliance + auditing systems built by compliance experts. + + Vendors who have deployed proprietary notification systems for their + Internet message stores have seen significant demand to provide + notifications for more and more events. As a first step towards + building a notification system, this document attempts to enumerate + the core events that real-world customers demand. + + This document includes those events that can be generated by the use + of IMAP4rev1 [RFC3501] and some existing extensions. As new IMAP + extensions are defined, or additional event types or parameters need + to be added, the set specified here can be extended by means of an + IANA registry with update requirements, as specified in Section 6. + +1.1. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + When these words appear in lower-case or with initial capital + letters, they are not RFC 2119 key words. + +2. Terminology + + The following terminology is used in this document: + + mailbox + A container for Internet messages and/or child mailboxes. A + mailbox may or may not permit delivery of new messages via a mail + delivery agent. + + + + + + +Gellens & Newman Standards Track [Page 3] + +RFC 5423 Internet Message Store Events March 2009 + + + mailbox identifier + A mailbox identifier provides sufficient information to identify a + specific mailbox on a specific server instance. An IMAP URL can + be a mailbox identifier. + + message access protocols + Protocols that provide clients (e.g., a mail user agent or web + browser) with access to the message store, including but not + limited to IMAP, POP, and HTTP. + + message context + As defined in [RFC3458]. + + UIDVALIDITY + As defined in IMAP4rev1 [RFC3501]. UIDVALIDITY is critical to the + correct operation of a caching mail client. When it changes, the + client MUST flush its cache. It's particularly important to + include UIDVALIDITY with event notifications related to message + addition or removal in order to keep the message data correctly + synchronized. + +3. Event Model + + The events that are generated by a message store depend to some + degree on the model used to represent a message store. The model the + IETF has for a message store is implicit from IMAP4rev1 and + extensions, so that model is assumed by this document. + + A message store event typically has an associated mailbox name and + usually has an associated user name (or authorization identity if + using the terminology from "Simple Authentication and Security Layer" + (SASL) [RFC4422]). Events referring to a specific message can use an + IMAP URL [RFC5092] to do so. Events referring to a set of messages + can use an IMAP URL to the mailbox plus an IMAP UID (Unique + Identifier) set. + + Each notification has a type and parameters. The type determines the + type of event, while the parameters supply information about the + context of the event that may be used to adjust subscription + preferences or may simply supply data associated with the event. The + types and parameter names in this document are restricted to US-ASCII + printable characters, so these events can be easily mapped to an + arbitrary notification system. However, this document assumes that + arbitrary parameter values (including large and multi-line values) + can be encoded with the notification system. Systems which lack that + feature could only implement a subset of these events. + + + + + +Gellens & Newman Standards Track [Page 4] + +RFC 5423 Internet Message Store Events March 2009 + + + This document does not indicate which event parameters are mandatory + or optional. That is done in documents that specify specific message + formats or bindings to a notification system. + + For scalability reasons, some degree of filtering at event generation + is necessary. At the very least, the ability to turn on and off + groups of related events and to suppress inclusion of large + parameters (such as messageContent) is needed. A sophisticated + publish/subscribe notification system may be able to propagate + cumulative subscription information to the publisher. + + Some of these events might be logically collapsed into a single event + type with a required parameter to distinguish between the cases + (e.g., QuotaExceed and QuotaWithin). However, until such time that + an event subscription model is formulated, it's not practical to make + such decisions. We thus note only the fact that some of these events + may be viewed as a single event type. + +4. Event Types + + This section discusses the different types of events useful in a + message store event notification system. The intention is to + document the events sufficient to cover an overwhelming majority of + known use cases while leaving less common event types for the future. + This section mentions parameters that are important or specific to + the events described here. Event parameters likely to be included in + most or all notifications are discussed in the next section. + +4.1. Message Addition and Deletion + + This section includes events related to message addition and + deletion. + + MessageAppend + A message was appended or concatenated to a mailbox by a message + access client. For the most part, this is identical to the + MessageNew event type except that the SMTP envelope information is + not included as a parameter, but information about which protocol + triggered the event MAY be included. See the MessageNew event for + more information. + + MessageExpire + One or more messages were expired from a mailbox due to server + expiration policy and are no longer accessible by the end user. + + The parameters include a mailbox identifier that MUST include + UIDVALIDITY and a UID set that describes the messages. + + + + +Gellens & Newman Standards Track [Page 5] + +RFC 5423 Internet Message Store Events March 2009 + + + Information about which server expiration policy was applied may + be included in the future. + + MessageExpunge + One or more messages were expunged from a mailbox by an IMAP + CLOSE/EXPUNGE, POP3 DELE+QUIT, HTTP, or equivalent client action + and are no longer accessible by the end user. + + The parameters include a mailbox identifier that MUST include + UIDVALIDITY, a UID set, and MAY also indicate which access + protocol triggered the event. + + MessageNew + A new message was received into a mailbox via a message delivery + agent. + + The parameters include a message identifier that, for IMAP- + accessible message stores, MUST include UIDVALIDITY and a UID. + The parameters MAY also include an SMTP envelope and other + arbitrary message and mailbox metadata. In some cases, the entire + new message itself may be included. The set of parameters SHOULD + be adjustable to the client's preference, with limits set by + server policy. An interesting policy, for example, would be to + include messages up to 2K in size with the notification, but to + include a URLAUTH [RFC4467] reference for larger messages. + + QuotaExceed + An operation failed (typically MessageNew) because the user's + mailbox exceeded one of the quotas (e.g., disk quota, message + quota, quota by message context, etc.). The parameters SHOULD + include at least the relevant user and quota and, optionally, the + mailbox. Quota usage SHOULD be included if possible. Parameters + needed to extend this to support quota by context are not + presently described in this document but could be added in the + future. + + QuotaWithin + An operation occurred (typically MessageExpunge or MessageExpire) + that reduced the user's quota usage under the limit. + + QuotaChange + The user's quota was changed. + + + + + + + + + +Gellens & Newman Standards Track [Page 6] + +RFC 5423 Internet Message Store Events March 2009 + + +4.2. Message Flags + + This section includes events related to changes in message flags. + + MessageRead + One or more messages in the mailbox were marked as read or seen by + a user. Note that POP has no concept of read or seen messages, so + these events are only generated by IMAP or HTTP clients (or + equivalent). + + The parameters include a mailbox identifier and a set of message + UIDs. + + MessageTrash + One or more messages were marked for future deletion by the user + but are still accessible over the protocol (the user's client may + or may not make these messages accessible through its user + interface). + + The parameters include a mailbox identifier and a set of message + UIDs. + + FlagsSet + One or more messages in the mailbox had one or more IMAP flags or + keywords set. + + The parameters include a list of IMAP flag or keyword names that + were set, a mailbox identifier, and the set of UIDs of affected + messages. The flagNames MUST NOT include \Recent. For + compatibility with simpler clients, it SHOULD be configurable + whether setting the \Seen or \Deleted flags results in this event + or the simpler MessageRead/MessageTrash events. By default, the + simpler message forms SHOULD be used for MessageRead and + MessageTrash. + + FlagsClear + One or more messages in the mailbox had one or more IMAP flags or + keywords cleared. + + The parameters include a list of IMAP flag or keyword names that + were cleared, a mailbox identifier, and the set of UIDs of + affected messages. The flagNames parameter MUST NOT include + \Recent. + + + + + + + + +Gellens & Newman Standards Track [Page 7] + +RFC 5423 Internet Message Store Events March 2009 + + +4.3. Access Accounting + + This section lists events related to message store access accounting. + + Login + A user has logged into the system via IMAP, HTTP, POP, or some + other mechanism. + + The parameters include the domain name and port used to access the + server and the user's authorization identity. Additional possible + parameters include the client's IP address and port, the + authentication identity (if different from the authorization + identity), the service name, the authentication mechanism, + information about any negotiated security layers, a timestamp, and + other information. + + Logout + A user has logged out or otherwise been disconnected from the + message store via IMAP, HTTP, POP, or some other mechanism. + + The parameters include the server domain name and the user's + authorization identity. Additional parameters MAY include any of + the information from the "Login" event as well as information + about the type of disconnect (suggested values include graceful, + abort, timeout, and security layer error), the duration of the + connection or session, and other information. + +4.4. Mailbox Management + + This section lists events related to the management of mailboxes. + + MailboxCreate + A mailbox has been created, or an access control changed on an + existing mailbox so that it is now accessible by the user. If the + mailbox creation caused the creation of new mailboxes earlier in + the hierarchy, separate MailboxCreate events are not generated, as + their creation is implied. + + The parameters include the created mailbox identifier, its + UIDVALIDITY for IMAP-accessible message stores, and MAY also + indicate which access protocol triggered the event. Access and + permissions information (such as Access Control List (ACL) + [RFC4314] settings) require a standardized format to be included, + and so are left for future extension. + + + + + + + +Gellens & Newman Standards Track [Page 8] + +RFC 5423 Internet Message Store Events March 2009 + + + MailboxDelete + A mailbox has been deleted, or an access control changed on an + existing mailbox so that it is no longer accessible by the user. + Note that if the mailbox has child mailboxes, only the specified + mailbox has been deleted, not the children. The mailbox becomes + \NOSELECT, and the hierarchy remains unchanged, as per the + description of the DELETE command in IMAP4rev1 [RFC3501]. + + The parameters include the deleted mailbox identifier and MAY also + indicate which access protocol triggered the event. + + MailboxRename + A mailbox has been renamed. Note that, per the description of the + RENAME command in IMAP4rev1 [RFC3501], special semantics regarding + the mailbox hierarchy apply when INBOX is renamed (child mailboxes + are usually included in the rename, but are excluded when INBOX is + renamed). When a mailbox other than INBOX is renamed and its + child mailboxes are also renamed as a result, separate + MailboxRename events are not generated for the child mailboxes, as + their renaming is implied. If the rename caused the creation of + new mailboxes earlier in the hierarchy, separate MailboxCreate + events are not generated for those, as their creation is implied. + When INBOX is renamed, a new INBOX is created. A MailboxCreate + event is not generated for the new INBOX, since it is implied. + + The parameters include the old mailbox identifier, the new mailbox + identifier, and MAY also indicate which access protocol triggered + the event. + + MailboxSubscribe + A mailbox has been added to the server-stored subscription list, + such as the one managed by the IMAP SUBSCRIBE and UNSUBSCRIBE + commands. + + The parameters include the user whose subscription list has been + affected, the mailbox identifier, and MAY also indicate which + access protocol triggered the event. + + MailboxUnSubscribe + A mailbox has been removed from the subscription list. + + The parameters include the user whose subscription list has been + affected, the mailbox identifier, and MAY also indicate which + access protocol triggered the event. + + + + + + + +Gellens & Newman Standards Track [Page 9] + +RFC 5423 Internet Message Store Events March 2009 + + +5. Event Parameters + + This section lists parameters included with these events. + + admin + Included with all events generated by message access protocols. + + The authentication identity associated with this event, as + distinct from the authorization identity (see "user"). This is + not included when it is the same as the value of the user + parameter. + + bodyStructure + May be included with MessageAppend and MessageNew. + + The IMAP BODYSTRUCTURE of the message. + + clientIP + Included with all events generated by message access protocols. + + The IPv4 or IPv6 address of the message store access client that + performed the action that triggered the notification. + + clientPort + Included with all events generated by message access protocols. + + The port number of the message store access client that performed + an action that triggered the notification (the port from which the + connection occurred). + + diskQuota + Included with QuotaExceed, QuotaWithin, and QuotaChange + notifications relating to a user or mailbox disk quota. May be + included with other notifications. + + Disk quota limit in kilobytes (1024 octets). + + diskUsed + Included with QuotaExceed and QuotaWithin notifications relating + to a user or mailbox disk quota. May be included with other + notifications. + + Disk space used in kilobytes (1024 octets). Only disk space that + counts against the quota is included. + + + + + + + +Gellens & Newman Standards Track [Page 10] + +RFC 5423 Internet Message Store Events March 2009 + + + envelope + May be included with the MessageNew notification. + + The message transfer envelope associated with final delivery of + the message for the MessageNew notification. This includes the + MAIL FROM and relevant RCPT TO line(s) used for final delivery + with CRLF delimiters and any ESMTP parameters. + + flagNames + Included with FlagsSet and FlagsClear events. May be included + with MessageAppend and MessageNew to indicate flags that were set + initially by the APPEND command or delivery agent, respectively. + + A list (likely to be space-separated) of IMAP flag or keyword + names that were set or cleared. Flag names begin with a backslash + while keyword names do not. The \Recent flag is explicitly not + permitted in the list. + + mailboxID + Included in events that affect mailboxes. A URI describing the + mailbox. In the case of MailboxRename, this refers to the new + name. + + maxMessages + Included with QuotaExceed and QuotaWithin notifications relating + to a user or mailbox message count quota. May be included with + other notifications. + + Quota limit on the number of messages in the mailbox, for events + referring to a mailbox. + + messageContent + May be included with MessageAppend and MessageNew. + + The entire message itself. Size-based suppression of this SHOULD + be available. + + messageSize + May be included with MessageAppend and MessageNew. + + Size of the RFC 5322 message itself in octets. This value matches + the length of the IMAP literal returned in response to an IMAP + FETCH of BODY[] for the referenced message. + + + + + + + + +Gellens & Newman Standards Track [Page 11] + +RFC 5423 Internet Message Store Events March 2009 + + + messages + Included with QuotaExceed and QuotaWithin notifications relating + to a user or mailbox message count quota. May be included with + other notifications. + + Number of messages in the mailbox. This is typically included + with message addition and deletion events. + + modseq + May be included with any notification referring to one message. + + This is the 64-bit integer MODSEQ as defined in [RFC4551]. No + assumptions about MODSEQ can be made if this is omitted. + + oldMailboxID + A URI describing the old name of a renamed or moved mailbox. + + pid + May be included with any notification. + + The process ID of the process that generated the notification. + + process + May be included with any notification. + + The name of the process that generated the notification. + + serverDomain + Included in Login and optionally in Logout or other events. The + domain name or IP address (v4 or v6) used to access the server or + mailbox. + + serverPort + Included in Login and optionally in Logout or other events. The + port number used to access the server. This is often a well-known + port. + + serverFQDN + May be included with any notification. + + The fully qualified domain name of the server that generated the + event. Note that this may be different from the server name used + to access the mailbox included in the mailbox identifier. + + + + + + + + +Gellens & Newman Standards Track [Page 12] + +RFC 5423 Internet Message Store Events March 2009 + + + service + May be included with any notification. + + The name of the service that triggered the event. Suggested + values include "imap", "pop", "http", and "admincli" (for an + administrative client). + + tags + May be included with any notification. + + A list of UTF-8 tags (likely to be comma-separated). One or more + tags can be set at the time a notification criteria or + notification subscription is created. Subscribers can use tags + for additional client-side filtering or dispatch of events. + + timestamp + May be included with any notification. + + The time at which the event occurred that triggered the + notification (the underlying protocol carrying the notification + may contain a timestamp for when the notification was generated). + This MAY be an approximate time. + + Timestamps are expressed in local time and contain the offset from + UTC (this information is used in several places in Internet mail) + and are normally in [RFC3339] format. + + uidnext + May be included with any notification referring to a mailbox. + + The UID that is projected to be assigned next in the mailbox. + This is typically included with message addition and deletion + events. This is equivalent to the UIDNEXT status item in the IMAP + STATUS command. + + uidset + Included with MessageExpires, MessageExpunges, MessageRead, + MessageTrash, FlagsSet, and FlagsClear. + + This includes the set of IMAP UIDs referenced. + + uri + Included with all notifications. A reference to the IMAP server, + a mailbox, or a message. + + Typically an IMAP URL. This can include the name of the server + used to access the mailbox/message, the mailbox name, the + UIDVALIDITY of the mailbox, and the UID of a specific message. + + + +Gellens & Newman Standards Track [Page 13] + +RFC 5423 Internet Message Store Events March 2009 + + + user + Included with all events generated by message access protocols. + + This is the authorization identifier used when the client + connected to the access protocol that triggered the event. Some + protocols (for example, many SASL mechanisms) distinguish between + authorization and authentication identifiers. For events + associated with a mailbox, this may be different from the owner of + the mailbox specified in the IMAP URL. + +6. IANA Considerations + + The IANA has created a new registry for "Internet Message Store + Events" that contains two sub-registries: event names and event + parameters. For both event names and event parameters, entries that + do not start with "vnd." are added by the IETF and are intended for + interoperable use. Entries that start with "vnd." are intended for + private use by one or more parties and are allocated to avoid + collisions. + + The initial values are contained in this document. + + Using IANA Considerations [RFC5226] terminology, entries that do not + start with "vnd." are allocated by IETF Consensus, while those + starting with "vnd." are allocated First Come First Served. + +7. Security Considerations + + Notifications can produce a large amount of traffic and expose + sensitive information. When notification mechanisms are used to + maintain state between different entities, the ability to corrupt or + manipulate notification messages could enable an attacker to modulate + the state of these entities. For example, if an attacker were able + to modify notifications sent from a message store to an auditing + server, he could modify the "user" and "messageContent" parameters in + MessageNew notifications to create false audit log entries. + + A competent transfer protocol for notifications must consider + authentication, authorization, privacy, and message integrity, as + well as denial-of-service issues. While the IETF has adequate tools + and experience to address these issues for mechanisms that involve + only one TCP connection, notification or publish/subscribe protocols + that are more sophisticated than a single end-to-end TCP connection + will need to pay extra attention to these issues and carefully + balance requirements to successfully deploy a system with security + and privacy considerations. + + + + + +Gellens & Newman Standards Track [Page 14] + +RFC 5423 Internet Message Store Events March 2009 + + +8. Acknowledgments + + Alexey Melnikov, Arnt Gulbrandsen, and Zoltan Ordogh have reviewed + and offered improvements to this document. Richard Barnes did a nice + review during Last Call. + +9. References + +9.1. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC5092] Melnikov, A. and C. Newman, "IMAP URL Scheme", RFC 5092, + November 2007. + + [RFC5226] Narten, T. and H. Alvestrand, "Guidelines for Writing an + IANA Considerations Section in RFCs", BCP 26, RFC 5226, + May 2008. + +9.2. Informative References + + [RFC1939] Myers, J. and M. Rose, "Post Office Protocol - Version 3", + STD 53, RFC 1939, May 1996. + + [RFC2177] Leiba, B., "IMAP4 IDLE command", RFC 2177, June 1997. + + [RFC2447] Dawson, F., Mansour, S., and S. Silverberg, "iCalendar + Message-Based Interoperability Protocol (iMIP)", RFC 2447, + November 1998. + + [RFC2616] Fielding, R., Gettys, J., Mogul, J., Frystyk, H., + Masinter, L., Leach, P., and T. Berners-Lee, "Hypertext + Transfer Protocol -- HTTP/1.1", RFC 2616, June 1999. + + [RFC3339] Klyne, G., Ed. and C. Newman, "Date and Time on the + Internet: Timestamps", RFC 3339, July 2002. + + [RFC3458] Burger, E., Candell, E., Eliot, C., and G. Klyne, "Message + Context for Internet Mail", RFC 3458, January 2003. + + [RFC4146] Gellens, R., "Simple New Mail Notification", RFC 4146, + August 2005. + + + + + +Gellens & Newman Standards Track [Page 15] + +RFC 5423 Internet Message Store Events March 2009 + + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication and + Security Layer (SASL)", RFC 4422, June 2006. + + [RFC4467] Crispin, M., "Internet Message Access Protocol (IMAP) - + URLAUTH Extension", RFC 4467, May 2006. + + [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional + STORE Operation or Quick Flag Changes Resynchronization", + RFC 4551, June 2006. + + [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, + October 2008. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Gellens & Newman Standards Track [Page 16] + +RFC 5423 Internet Message Store Events March 2009 + + +Appendix A. Future Extensions + + This document specifies core functionality based on events that are + believed to be well understood, have known use cases, and are + implemented by at least one deployed real-world Internet message + store. (A few events are exceptions to the last test only: FlagsSet, + FlagsClear, MailboxCreate, MailboxDelete, MailboxRename, + MailboxSubscribe, and MailboxUnSubscribe.) + + Some events have been suggested but are postponed to future + extensions because they do not meet this criteria. These events + include messages that have been moved to archive storage and may + require extra time to access, quota by message context, + authentication failure, user mail account disabled, annotations, and + mailbox ACL or metadata change. The descriptions of several events + note additional parameters that are likely candidates for future + inclusion. See Section 6 for how the list of events and parameters + can be extended. + + In order to narrow the scope of this document to something that can + be completed, only events generated from the message store (by a + message access module, administrative module, or message delivery + agent) are considered. A complete mail system is normally linked + with an identity system that would also publish events of interest to + a message store event subscriber. Events of interest include account + created/deleted/disabled and password changed/expired. + +Authors' Addresses + + Randall Gellens + QUALCOMM Incorporated + 5775 Morehouse Drive + San Diego, CA 92651 + USA + + Phone: + EMail: rg+ietf@qualcomm.com + + + Chris Newman + Sun Microsystems + 800 Royal Oaks + Monrovia, CA 91016-6347 + USA + + Phone: + EMail: chris.newman@sun.com + + + + +Gellens & Newman Standards Track [Page 17] + diff --git a/docs/rfcs/rfc5464.IMAP_METADATA_extension.txt b/docs/rfcs/rfc5464.IMAP_METADATA_extension.txt new file mode 100644 index 0000000..645bfd9 --- /dev/null +++ b/docs/rfcs/rfc5464.IMAP_METADATA_extension.txt @@ -0,0 +1,1177 @@ + + + +Network Working Group C. Daboo +Request for Comments: 5464 Apple, Inc. +Category: Standards Track February 2009 + + + The IMAP METADATA Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Abstract + + The METADATA extension to the Internet Message Access Protocol + permits clients and servers to maintain "annotations" or "metadata" + on IMAP servers. It is possible to have annotations on a per-mailbox + basis or on the server as a whole. For example, this would allow + comments about the purpose of a particular mailbox to be "attached" + to that mailbox, or a "message of the day" containing server status + information to be made available to anyone logging in to the server. + + + + + + + + + + + + + + + + + + + + + + + + + + + +Daboo Standards Track [Page 1] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +Table of Contents + + 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . 3 + 3. Data Model . . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 3.1. Overview . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 3.2. Namespace of Entries . . . . . . . . . . . . . . . . . . . 4 + 3.2.1. Entry Names . . . . . . . . . . . . . . . . . . . . . 5 + 3.3. Private versus Shared and Access Control . . . . . . . . . 6 + 4. IMAP Protocol Changes . . . . . . . . . . . . . . . . . . . . 7 + 4.1. General Considerations . . . . . . . . . . . . . . . . . . 7 + 4.2. GETMETADATA Command . . . . . . . . . . . . . . . . . . . 8 + 4.2.1. MAXSIZE GETMETADATA Command Option . . . . . . . . . . 9 + 4.2.2. DEPTH GETMETADATA Command Option . . . . . . . . . . . 10 + 4.3. SETMETADATA Command . . . . . . . . . . . . . . . . . . . 10 + 4.4. METADATA Response . . . . . . . . . . . . . . . . . . . . 12 + 4.4.1. METADATA Response with Values . . . . . . . . . . . . 13 + 4.4.2. Unsolicited METADATA Response without Values . . . . . 13 + 5. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 14 + 6. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 16 + 6.1. Entry and Attribute Registration Template . . . . . . . . 16 + 6.2. Server Entry Registrations . . . . . . . . . . . . . . . . 16 + 6.2.1. /shared/comment . . . . . . . . . . . . . . . . . . . 17 + 6.2.2. /shared/admin . . . . . . . . . . . . . . . . . . . . 17 + 6.3. Mailbox Entry Registrations . . . . . . . . . . . . . . . 17 + 6.3.1. /shared/comment . . . . . . . . . . . . . . . . . . . 18 + 6.3.2. /private/comment . . . . . . . . . . . . . . . . . . . 18 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 18 + 8. Normative References . . . . . . . . . . . . . . . . . . . . . 19 + Appendix A. Acknowledgments . . . . . . . . . . . . . . . . . . . 19 + + + + + + + + + + + + + + + + + + + + + +Daboo Standards Track [Page 2] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +1. Introduction and Overview + + The goal of the METADATA extension is to provide a means for clients + to set and retrieve "annotations" or "metadata" on an IMAP server. + The annotations can be associated with specific mailboxes or the + server as a whole. The server can choose to support only server + annotations or both server and mailbox annotations. + + A server that supports both server and mailbox annotations indicates + the presence of this extension by returning "METADATA" as one of the + supported capabilities in the CAPABILITY command response. + + A server that supports only server annotations indicates the presence + of this extension by returning "METADATA-SERVER" as one of the + supported capabilities in the CAPABILITY command response. + + A server that supports unsolicited annotation change responses MUST + support the "ENABLE" [RFC5161] extension to allow clients to turn + that feature on. + + The METADATA extension adds two new commands and one new untagged + response to the IMAP base protocol. + + This extension makes the following changes to the IMAP protocol: + + o adds a new SETMETADATA command + + o adds a new GETMETADATA command + + o adds a new METADATA untagged response + + o adds a new METADATA response code + + The rest of this document describes the data model and protocol + changes more rigorously. + +2. Conventions Used in This Document + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + Whitespace and line breaks have been added to the examples in this + document to promote readability. + + + + +Daboo Standards Track [Page 3] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +3. Data Model + +3.1. Overview + + Mailboxes or the server as a whole may have zero or more annotations + associated with them. An annotation contains a uniquely named entry, + which has a value. Annotations can be added to mailboxes when a + mailbox name is provided as the first argument to the SETMETADATA + command, or to the server as a whole when the empty string is + provided as the first argument to the command. + + For example, a general comment being added to a mailbox may have an + entry name of "/comment" and a value of "Really useful mailbox". + + The protocol changes to IMAP described below allow a client to access + or change the values of any annotation entry, assuming it has + sufficient access rights to do so. + +3.2. Namespace of Entries + + Each annotation is an entry that has a hierarchical name, with each + component of the name separated by a slash ("/"). An entry name MUST + NOT contain two consecutive "/" characters and MUST NOT end with a + "/" character. + + The value of an entry is NIL (has no value), or a string or binary + data of zero or more octets. A string MAY contain multiple lines of + text. Clients MUST use the CRLF (0x0D 0x0A) character octet sequence + to represent line ends in a multi-line string value. + + Entry names MUST NOT contain asterisk ("*") or percent ("%") + characters and MUST NOT contain non-ASCII characters or characters + with octet values in the range 0x00 to 0x19. Invalid entry names + result in a BAD response in any IMAP command in which they are used. + + Entry names are case-insensitive. + + Use of control or punctuation characters in entry names is strongly + discouraged. + + This specification defines an initial set of entry names available + for use with mailbox and server annotations. In addition, an + extension mechanism is described to allow additional names to be + added for extensibility. + + The first component in entry names defines the scope of the + annotation. Currently, only the prefixes "/private" or "/shared" are + defined. These prefixes are used to indicate whether an annotation + + + +Daboo Standards Track [Page 4] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + is stored on a per-user basis ("/private") and not visible to other + users, or whether an annotation is shared between authorized users + ("/shared") with a single value that can be read and changed by + authorized users with appropriate access. See Section 3.3 for + details. + + Entry names can have any number of components starting at 2, unless + they fall under the vendor namespaces (i.e., have a /shared/vendor/ + or /private/vendor/ prefix as described + below), in which case they have at least 4 components. + +3.2.1. Entry Names + + Entry names MUST be specified in a Standards Track or IESG-approved + Experimental RFC, or fall under the vendor namespace. See + Section 6.1 for the registration template. + +3.2.1.1. Server Entries + + These entries are set or retrieved when the mailbox name argument to + the new SETMETADATA or GETMETADATA command is the empty string. + + /shared/comment + + Defines a comment or note that is associated with the server and + that is shared with authorized users of the server. + + /shared/admin + + Indicates a method for contacting the server administrator. The + value MUST be a URI (e.g., a mailto: or tel: URL). This entry is + always read-only -- clients cannot change it. It is visible to + authorized users of the system. + + /shared/vendor/ + + Defines the top level of shared entries associated with the + server, as created by a particular product of some vendor. This + entry can be used by vendors to provide server- or client-specific + annotations. The vendor-token MUST be registered with IANA, using + the Application Configuration Access Protocol (ACAP) [RFC2244] + vendor subtree registry. + + /private/vendor/ + + Defines the top level of private entries associated with the + server, as created by a particular product of some vendor. This + entry can be used by vendors to provide server- or client-specific + + + +Daboo Standards Track [Page 5] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + annotations. The vendor-token MUST be registered with IANA, using + the ACAP [RFC2244] vendor subtree registry. + +3.2.1.2. Mailbox Entries + + These entries are set or retrieved when the mailbox name argument to + the new SETMETADATA or GETMETADATA command is not the empty string. + + /shared/comment + + Defines a shared comment or note associated with a mailbox. + + /private/comment + + Defines a private (per-user) comment or note associated with a + mailbox. + + /shared/vendor/ + + Defines the top level of shared entries associated with a specific + mailbox, as created by a particular product of some vendor. This + entry can be used by vendors to provide client-specific + annotations. The vendor-token MUST be registered with IANA, using + the ACAP [RFC2244] vendor subtree registry. + + /private/vendor/ + + Defines the top level of private entries associated with a + specific mailbox, as created by a particular product of some + vendor. This entry can be used by vendors to provide client- + specific annotations. The vendor-token MUST be registered with + IANA, using the ACAP [RFC2244] vendor subtree registry. + +3.3. Private versus Shared and Access Control + + In the absence of the ACL (Access Control List) extension [RFC4314], + users can only set and retrieve private or shared mailbox annotations + on a mailbox that exists and is returned to them via a LIST or LSUB + command, and on which they have either read or write access to the + actual message content of the mailbox (as determined by the READ-ONLY + and READ-WRITE response codes as described in Section 5.2 of + [RFC4314]). + + When the ACL extension [RFC4314] is present, users can only set and + retrieve private or shared mailbox annotations on a mailbox on which + they have the "l" right and any one of the "r", "s", "w", "i", or "p" + rights. + + + + +Daboo Standards Track [Page 6] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + If a client attempts to set or retrieve annotations on mailboxes that + do not satisfy the conditions above, the server MUST respond with a + NO response. + + Users can always retrieve private or shared server annotations if + they exist. Servers MAY restrict the creation of private or shared + server annotations as appropriate. When restricted, the server MUST + return a NO response when the SETMETADATA command is used to try to + create a server annotation. + + If the METADATA extension is present, support for shared annotations + is REQUIRED, whilst support for private annotations is OPTIONAL. + This recognizes the fact that support for private annotations may + introduce significantly more complexity to a server in terms of + tracking ownership of the annotations, how quota is determined for + users based on their own annotations, etc. + +4. IMAP Protocol Changes + +4.1. General Considerations + + The new SETMETADATA command and the METADATA response each have a + mailbox name argument. An empty string is used for the mailbox name + to signify server annotations. A non-empty string is used to signify + mailbox annotations attached to the corresponding mailbox. + + Servers SHOULD ensure that mailbox annotations are automatically + moved when the mailbox they refer to is renamed, i.e., the + annotations follow the mailbox. This applies to a rename of the + INBOX, with the additional behavior that the annotations are copied + from the original INBOX to the renamed mailbox, i.e., mailbox + annotations are preserved on the INBOX when it is renamed. + + Servers SHOULD delete annotations for a mailbox when the mailbox is + deleted, so that a mailbox created with the same name as a previously + existing mailbox does not inherit the old mailbox annotations. + + Servers SHOULD allow annotations on all 'types' of mailboxes, + including ones reporting \Noselect for their LIST response. Servers + can implicitly remove \Noselect mailboxes when all child mailboxes + are removed, and, at that time any annotations associated with the + \Noselect mailbox SHOULD be removed. + + The server is allowed to impose limitations on the size of any one + annotation or the total number of annotations for a single mailbox or + for the server as a whole. However, the server MUST accept an + annotation data size of at least 1024 bytes, and an annotation count + per server or mailbox of at least 10. + + + +Daboo Standards Track [Page 7] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + Some annotations may be "read-only" -- i.e., they are set by the + server and cannot be changed by the client. Also, such annotations + may be "computed" -- i.e., the value changes based on underlying + properties of the mailbox or server. For example, an annotation + reporting the total size of all messages in the mailbox would change + as messages are added or removed. Or, an annotation containing an + IMAP URL for the mailbox would change if the mailbox was renamed. + + Servers MAY support sending unsolicited responses for use when + annotations are changed by some "third-party" (see Section 4.4). In + order to do so, servers MUST support the ENABLE command [RFC5161] and + MUST only send unsolicited responses if the client used the ENABLE + command [RFC5161] extension with the capability string "METADATA" or + "METADATA-SERVER" earlier in the session, depending on which of those + capabilities is supported by the server. + +4.2. GETMETADATA Command + + This extension adds the GETMETADATA command. This allows clients to + retrieve server or mailbox annotations. + + This command is only available in authenticated or selected state + [RFC3501]. + + Arguments: mailbox-name + options + entry-specifier + + Responses: required METADATA response + + Result: OK - command completed + NO - command failure: can't access annotations on + the server + BAD - command unknown or arguments invalid + + When the mailbox name is the empty string, this command retrieves + server annotations. When the mailbox name is not empty, this command + retrieves annotations on the specified mailbox. + + Options MAY be included with this command and are defined below. + + Example: + + C: a GETMETADATA "" /shared/comment + S: * METADATA "" (/shared/comment "Shared comment") + S: a OK GETMETADATA complete + + + + + +Daboo Standards Track [Page 8] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + In the above example, the contents of the value of the "/shared/ + comment" server entry is requested by the client and returned by + the server. + + Example: + + C: a GETMETADATA "INBOX" /private/comment + S: * METADATA "INBOX" (/private/comment "My own comment") + S: a OK GETMETADATA complete + + In the above example, the contents of the value of the "/private/ + comment" mailbox entry for the mailbox "INBOX" is requested by the + client and returned by the server. + + Entry specifiers can be lists of atomic specifiers, so that multiple + annotations may be returned in a single GETMETADATA command. + + Example: + + C: a GETMETADATA "INBOX" (/shared/comment /private/comment) + S: * METADATA "INBOX" (/shared/comment "Shared comment" + /private/comment "My own comment") + S: a OK GETMETADATA complete + + In the above example, the values of the two server entries + "/shared/comment" and "/private/comment" on the mailbox "INBOX" + are requested by the client and returned by the server. + +4.2.1. MAXSIZE GETMETADATA Command Option + + When the MAXSIZE option is specified with the GETMETADATA command, it + restricts which entry values are returned by the server. Only entry + values that are less than or equal in octet size to the specified + MAXSIZE limit are returned. If there are any entries with values + larger than the MAXSIZE limit, the server MUST include the METADATA + LONGENTRIES response code in the tagged OK response for the + GETMETADATA command. The METADATA LONGENTRIES response code returns + the size of the biggest entry value requested by the client that + exceeded the MAXSIZE limit. + + Example: + + C: a GETMETADATA "INBOX" (MAXSIZE 1024) + (/shared/comment /private/comment) + S: * METADATA "INBOX" (/private/comment "My own comment") + S: a OK [METADATA LONGENTRIES 2199] GETMETADATA complete + + + + + +Daboo Standards Track [Page 9] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + In the above example, the values of the two server entries + "/shared/comment" and "/private/comment" on the mailbox "INBOX" + are requested by the client, which wants to restrict the size of + returned values to 1024 octets. In this case, the "/shared/ + comment" entry value is 2199 octets and is not returned. + +4.2.2. DEPTH GETMETADATA Command Option + + When the DEPTH option is specified with the GETMETADATA command, it + extends the list of entry values returned by the server. For each + entry name specified in the GETMETADATA command, the server returns + the value of the specified entry name (if it exists), plus all + entries below the entry name up to the specified DEPTH. Three values + are allowed for DEPTH: + + "0" - no entries below the specified entry are returned + "1" - only entries immediately below the specified entry are returned + "infinity" - all entries below the specified entry are returned + + Thus, "depth 1" for an entry "/a" will match "/a" as well as its + children entries (e.g., "/a/b"), but will not match grandchildren + entries (e.g., "/a/b/c"). + + If the DEPTH option is not specified, this is the same as specifying + "DEPTH 0". + + Example: + + C: a GETMETADATA "INBOX" (DEPTH 1) + (/private/filters/values) + S: * METADATA "INBOX" (/private/filters/values/small + "SMALLER 5000" /private/filters/values/boss + "FROM \"boss@example.com\"") + S: a OK GETMETADATA complete + + In the above example, 2 entries below the /private/filters/values + entry exist on the mailbox "INBOX": "/private/filters/values/ + small" and "/private/filters/values/boss". + +4.3. SETMETADATA Command + + This extension adds the SETMETADATA command. This allows clients to + set annotations. + + This command is only available in authenticated or selected state + [RFC3501]. + + + + + +Daboo Standards Track [Page 10] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + Arguments: mailbox-name + entry + value + list of entry, values + + Responses: no specific responses for this command + + Result: OK - command completed + NO - command failure: can't set annotations, + or annotation too big or too many + BAD - command unknown or arguments invalid + + This command sets the specified list of entries by adding or + replacing the specified values provided, on the specified existing + mailboxes or on the server (if the mailbox argument is the empty + string). Clients can use NIL for the value of entries it wants to + remove. The server SHOULD NOT return a METADATA response containing + the updated annotation data. Clients MUST NOT assume that a METADATA + response will be sent, and MUST assume that if the command succeeds, + then the annotation has been changed. + + If the server is unable to set an annotation because the size of its + value is too large, the server MUST return a tagged NO response with + a "[METADATA MAXSIZE NNN]" response code when NNN is the maximum + octet count that it is willing to accept. + + If the server is unable to set a new annotation because the maximum + number of allowed annotations has already been reached, the server + MUST return a tagged NO response with a "[METADATA TOOMANY]" response + code. + + If the server is unable to set a new annotation because it does not + support private annotations on one of the specified mailboxes, the + server MUST return a tagged NO response with a "[METADATA NOPRIVATE]" + response code. + + When any one annotation fails to be set, resulting in a tagged NO + response from the server, then the server MUST NOT change the values + for other annotations specified in the SETMETADATA command. + + Example: + + C: a SETMETADATA INBOX (/private/comment {33} + S: + ready for data + My new comment across + two lines. + ) + S: a OK SETMETADATA complete + + + +Daboo Standards Track [Page 11] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + In the above example, the entry "/private/comment" for the mailbox + "INBOX" is created (if not already present) and the value set to a + multi-line string. + + Example: + + C: a SETMETADATA INBOX (/private/comment NIL) + S: a OK SETMETADATA complete + + In the above example, the entry "/private/comment" is removed from + the mailbox "INBOX". + + Multiple entries can be set in a single SETMETADATA command by + listing entry-value pairs in the list. + + Example: + + C: a SETMETADATA INBOX (/private/comment "My new comment" + /shared/comment "This one is for you!") + S: a OK SETMETADATA complete + + In the above example, the entries "/private/comment" and "/shared/ + comment" for the mailbox "INBOX" are created (if not already + present) and the values set as specified. + + Example: + + C: a SETMETADATA INBOX (/private/comment "My new comment") + S: a NO [METADATA TOOMANY] SETMETADATA failed + + In the above example, the server is unable to set the requested + (new) annotation as it has reached the limit on the number of + annotations it can support on the specified mailbox. + +4.4. METADATA Response + + The METADATA response displays results of a GETMETADATA command, or + can be returned as an unsolicited response at any time by the server + in response to a change in a server or mailbox annotation. + + When unsolicited responses are activated by the ENABLE [RFC5161] + command for this extension, servers MUST send unsolicited METADATA + responses if server or mailbox annotations are changed by a third- + party, allowing servers to keep clients updated with changes. + + Unsolicited METADATA responses MUST only contain entry names, not the + values. If the client wants to update any cached values, it must + explicitly retrieve those using a GETMETADATA command. + + + +Daboo Standards Track [Page 12] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + The METADATA response can contain multiple entries in a single + response, but the server is free to return multiple responses for + each entry or group of entries, if it desires. + + This response is only available in authenticated or selected state + [RFC3501]. + +4.4.1. METADATA Response with Values + + The response consists of a list of entry-value pairs. + + Example: + + C: a GETMETADATA "" /shared/comment + S: * METADATA "" (/shared/comment "My comment") + S: a OK GETMETADATA complete + + In the above example, a single entry with its value is returned by + the server. + + Example: + + C: a GETMETADATA "INBOX" /private/comment /shared/comment + S: * METADATA "INBOX" (/private/comment "My comment" + /shared/comment "Its sunny outside!") + S: a OK GETMETADATA complete + + In the above example, two entries and their values are returned by + the server. + + Example: + + C: a GETMETADATA "INBOX" /private/comment /shared/comment + S: * METADATA "INBOX" (/private/comment "My comment") + S: * METADATA "INBOX" (/shared/comment "Its sunny outside!") + S: a OK GETMETADATA complete + + In the above example, the server returns two separate responses + for each of the two entries requested. + +4.4.2. Unsolicited METADATA Response without Values + + The response consists of a list of entries, each of which have + changed on the server or mailbox. + + Example: + + + + + +Daboo Standards Track [Page 13] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + C: a NOOP + S: * METADATA "" /shared/comment + S: a OK NOOP complete + + In the above example, the server indicates that the "/shared/ + comment" server entry has been changed. + + Example: + + C: a NOOP + S: * METADATA "INBOX" /shared/comment /private/comment + S: a OK NOOP complete + + In the above example, the server indicates a change to two mailbox + entries. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. + + Non-terminals referenced but not defined below are as defined by + [RFC3501], with the new definitions in [RFC4466] superseding those in + [RFC3501]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. + + capability =/ "METADATA" / "METADATA-SERVER" + ; defines the capabilities for this extension. + + command-auth =/ setmetadata / getmetadata + ; adds to original IMAP command + + entries = entry / + "(" entry *(SP entry) ")" + ; entry specifiers + + entry = astring + ; slash-separated path to entry + ; MUST NOT contain "*" or "%" + + entry-value = entry SP value + + entry-values = "(" entry-value *(SP entry-value) ")" + + + + +Daboo Standards Track [Page 14] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + entry-list = entry *(SP entry) + ; list of entries used in unsolicited + ; METADATA response + + getmetadata = "GETMETADATA" [SP getmetadata-options] + SP mailbox SP entries + ; empty string for mailbox implies + ; server annotation. + + getmetadata-options = "(" getmetadata-option + *(SP getmetadata-option) ")" + + getmetadata-option = tagged-ext-label [SP tagged-ext-val] + ; tagged-ext-label and tagged-ext-val + ; are defined in [RFC4466]. + + maxsize-opt = "MAXSIZE" SP number + ; Used as a getmetadata-option + + metadata-resp = "METADATA" SP mailbox SP + (entry-values / entry-list) + ; empty string for mailbox implies + ; server annotation. + + response-payload =/ metadata-resp + ; adds to original IMAP data responses + + resp-text-code =/ "METADATA" SP "LONGENTRIES" SP number + ; new response codes for GETMETADATA + + resp-text-code =/ "METADATA" SP ("MAXSIZE" SP number / + "TOOMANY" / "NOPRIVATE") + ; new response codes for SETMETADATA + ; failures + + scope-opt = "DEPTH" SP ("0" / "1" / "infinity") + ; Used as a getmetadata-option + + setmetadata = "SETMETADATA" SP mailbox + SP entry-values + ; empty string for mailbox implies + ; server annotation. + + value = nstring / literal8 + + + + + + + +Daboo Standards Track [Page 15] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +6. IANA Considerations + + All entries MUST have either "/shared" or "/private" as a prefix. + Entry names MUST be specified in a Standards Track or IESG-approved + Experimental RFC, or fall under the vendor namespace (i.e., use + /shared/vendor/ or /private/vendor/ as + the prefix). + + Each entry registration MUST include a content-type that is used to + indicate the nature of the annotation value. Where applicable, a + charset parameter MUST be included with the content-type. + +6.1. Entry and Attribute Registration Template + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: [Either "Mailbox" or "Server"] + + Name: [the name of the entry] + + Description: [a description of what the entry is for] + + Content-type: [MIME Content-Type and charset for the entry value] + + RFC Number: [for entries published as RFCs] + + Contact: [email and/or physical address to contact for + additional information] + +6.2. Server Entry Registrations + + The following templates specify the IANA registrations of annotation + entries specified in this document. + + + + + + + + + + + + + + + + + +Daboo Standards Track [Page 16] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +6.2.1. /shared/comment + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: Server + + Name: /shared/comment + + Description: Defines a comment or note that is associated + with the server and that is shared with + authorized users of the server. + + Content-type: text/plain; charset=utf-8 + + RFC Number: RFC 5464 + + Contact: IMAP Extensions mailto:ietf-imapext@imc.org + +6.2.2. /shared/admin + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: Server + + Name: /shared/admin + + Description: Indicates a method for contacting the server + administrator. The value MUST be a URI (e.g., a + mailto: or tel: URL). This entry is always + read-only -- clients cannot change it. It is visible + to authorized users of the system. + + Content-type: text/plain; charset=utf-8 + + RFC Number: RFC 5464 + + Contact: IMAP Extensions mailto:ietf-imapext@imc.org + +6.3. Mailbox Entry Registrations + + The following templates specify the IANA registrations of annotation + entries specified in this document. + + + + + + + +Daboo Standards Track [Page 17] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +6.3.1. /shared/comment + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: Mailbox + + Name: /shared/comment + + Description: Defines a shared comment or note associated with a + mailbox. + + Content-type: text/plain; charset=utf-8 + + RFC Number: RFC 5464 + + Contact: IMAP Extensions mailto:ietf-imapext@imc.org + +6.3.2. /private/comment + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + + Type: Mailbox + + Name: /private/comment + + Description: Defines a private comment or note associated with a + mailbox. + + Content-type: text/plain; charset=utf-8 + + RFC Number: RFC 5464 + + Contact: IMAP Extensions mailto:ietf-imapext@imc.org + +7. Security Considerations + + The security considerations in Section 11 of [RFC3501] apply here + with respect to protecting annotations from snooping. Servers MAY + choose to only support the METADATA and/or METADATA-SERVER extensions + after a privacy layer has been negotiated by the client. + + Annotations can contain arbitrary data of varying size. As such, + servers MUST ensure that size limits are enforced to prevent a user + from using up all available space on a server and preventing use by + others. Clients MUST treat annotation data values as an "untrusted" + source of data as it is possible for it to contain malicious content. + + + +Daboo Standards Track [Page 18] + +RFC 5464 The IMAP METADATA Extension February 2009 + + + Annotations whose values are intended to remain private MUST be + stored only in entries that have the "/private" prefix on the entry + name. + + Excluding the above issues, the METADATA extension does not raise any + security considerations that are not present in the base IMAP + protocol, and these issues are discussed in [RFC3501]. + +8. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2244] Newman, C. and J. Myers, "ACAP -- Application + Configuration Access Protocol", RFC 2244, November 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC5161] Gulbrandsen, A. and A. Melnikov, "The IMAP ENABLE + Extension", RFC 5161, March 2008. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + +Appendix A. Acknowledgments + + The ideas expressed in this document are based on the message + annotation document that was co-authored by Randall Gellens. The + author would like to thank the following individuals for contributing + their ideas and support for writing this specification: Dave + Cridland, Arnt Gulbrandsen, Dan Karp, Alexey Melnikov, Ken Murchison, + Chris Newman, and Michael Wener. + + + + + + + + + + + + +Daboo Standards Track [Page 19] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +Author's Address + + Cyrus Daboo + Apple Inc. + 1 Infinite Loop + Cupertino, CA 95014 + USA + + EMail: cyrus@daboo.name + URI: http://www.apple.com/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Daboo Standards Track [Page 20] + +RFC 5464 The IMAP METADATA Extension February 2009 + + +Full Copyright Statement + + Copyright (C) The IETF Trust (2009). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND + THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF + THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the procedures with respect to rights in RFC documents can be + found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at + ietf-ipr@ietf.org. + + + + + + + + + + + + +Daboo Standards Track [Page 21] + + diff --git a/docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt b/docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt new file mode 100644 index 0000000..3fe5bcf --- /dev/null +++ b/docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt @@ -0,0 +1,1235 @@ + + + + + + +Network Working Group A. Gulbrandsen +Request for Comments: 5465 Oryx Mail Systems GmbH +Updates: 5267 C. King +Category: Standards Track A. Melnikov + Isode Ltd. + February 2009 + + + The IMAP NOTIFY Extension + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (c) 2009 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents (http://trustee.ietf.org/ + license-info) in effect on the date of publication of this document. + Please review these documents carefully, as they describe your rights + and restrictions with respect to this document. + +Abstract + + This document defines an IMAP extension that allows a client to + request specific kinds of unsolicited notifications for specified + mailboxes, such as messages being added to or deleted from such + mailboxes. + + + + + + + + + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 1] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +Table of Contents + + 1. Overview and Rationale ..........................................3 + 2. Conventions Used in This Document ...............................4 + 3. The NOTIFY Extension ............................................4 + 3.1. The NOTIFY Command .........................................4 + 4. Interaction with the IDLE Command ...............................8 + 5. Event Types .....................................................8 + 5.1. FlagChange and AnnotationChange ............................9 + 5.2. MessageNew .................................................9 + 5.3. MessageExpunge ............................................10 + 5.4. MailboxName ...............................................11 + 5.5. SubscriptionChange ........................................12 + 5.6. MailboxMetadataChange .....................................12 + 5.7. ServerMetadataChange ......................................13 + 5.8. Notification Overflow .....................................13 + 5.9. ACL (Access Control List) Changes .........................13 + 6. Mailbox Specification ..........................................14 + 6.1. Mailbox Specifiers Affecting the Currently + Selected Mailbox ..........................................14 + 6.2. Personal ..................................................15 + 6.3. Inboxes ...................................................15 + 6.4. Subscribed ................................................15 + 6.5. Subtree ...................................................15 + 6.6. Mailboxes .................................................16 + 7. Extension to SEARCH and SORT Commands ..........................16 + 8. Formal Syntax ..................................................16 + 9. Security Considerations ........................................19 + 10. IANA Considerations ...........................................19 + 10.1. Initial LIST-EXTENDED Extended Data Item Registrations ...19 + 11. Acknowledgements ..............................................20 + 12. Normative References ..........................................20 + 13. Informative References ........................................21 + + + + + + + + + + + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 2] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +1. Overview and Rationale + + The IDLE command (defined in [RFC2177]) provides a way for the client + to go into a mode where the IMAP server pushes it notifications about + IMAP mailstore events for the selected mailbox. However, the IDLE + extension doesn't restrict or control which server events can be + sent, or what information the server sends in response to each event. + Also, IDLE only applies to the selected mailbox, thus requiring an + additional TCP connection per mailbox. + + This document defines an IMAP extension that allows clients to + express their preferences about unsolicited events generated by the + server. The extension allows clients to only receive events that + they are interested in, while servers know that they don't need to go + to the effort of generating certain types of untagged responses. + + Without the NOTIFY command defined in this document, an IMAP server + will only send information about mailstore changes to the client in + the following cases: + + - as the result of a client command (e.g., FETCH responses to a + FETCH or STORE command), + - as unsolicited responses sent just before the end of a command + (e.g., EXISTS or EXPUNGE) as the result of changes in other + sessions, and + - during an IDLE command. + + The NOTIFY command extends what information may be returned in those + last two cases, and also permits and requires the server to send + information about updates between commands. The NOTIFY command also + allows for the client to extend what information is sent unsolicited + about the selected mailbox and to request some update information to + be sent regarding other mailboxes. + + The interaction between IDLE and NOTIFY commands is described in + Section 4. + + For the new messages delivered to or appended to the selected + mailbox, the NOTIFY command can be used to request that a set of + attributes be sent to the client in an unsolicited FETCH response. + This allows a client to be a passive recipient of events and new mail + and to be able to maintain full synchronisation without having to + issue any subsequent commands except to modify the state of the + mailbox on the server. + + + + + + + +Gulbrandsen, et al. Standards Track [Page 3] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + Some mobile clients, however, may want mail "pushed" only for mail + that matches a SEARCH pattern. To meet that need, [RFC5267] is + augmented by this document to extend the UPDATE return option to + specify a list of fetch-atts to be returned when a new message is + delivered or appended in another session. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + + The acronym MSN stands for Message Sequence Numbers (see Section + 2.3.1.2 of [RFC3501]). + + Example lines prefaced by "C:" are sent by the client and ones + prefaced by "S:", by the server. "[...]" means elision. + +3. The NOTIFY Extension + + IMAP servers that support this extension advertise the NOTIFY + capability. This extension adds the NOTIFY command as defined in + Section 5.1. + + A server implementing this extension is not required to implement + LIST-EXTENDED [RFC5258], even though a NOTIFY-compliant server must + be able to return extended LIST responses, defined in [RFC5258]. + +3.1. The NOTIFY Command + + Arguments: "SET" + Optional STATUS indicator + Mailboxes to be watched + Events about which to notify the client + + Or + Arguments: "NONE" + + Responses: Possibly untagged STATUS responses (for SET) + + Result: OK - The server will notify the client as requested. + NO - Unsupported NOTIFY event, NOTIFY too complex or + expensive, etc. + BAD - Command unknown, invalid, unsupported, or has + unknown arguments. + + + + + + +Gulbrandsen, et al. Standards Track [Page 4] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + The NOTIFY command informs the server that the client listens for + event notifications all the time (even when no command is in + progress), and requests the server to notify it about the specified + set of events. The NOTIFY command has two forms. NOTIFY NONE + specifies that the client is not interested in any kind of event + happening on the server. NOTIFY SET replaces the current list of + interesting events with a new list of events. + + Until the NOTIFY command is used for the first time, the server only + sends notifications while a command is being processed, and notifies + the client about these events on the selected mailbox (see Section 5 + for definitions): MessageNew, MessageExpunge, or FlagChange. It does + not notify the client about any events on other mailboxes. + + The effect of a successful NOTIFY command lasts until the next NOTIFY + command or until the IMAP connection is closed. + + A successful NOTIFY SET command MUST cause the server to immediately + return any accumulated changes to the currently selected mailbox (if + any), such as flag changes and new or expunged messages. Thus, a + successful NOTIFY SET command implies an implicit NOOP command. + + The NOTIFY SET command can request notifications of message-related + changes to the selected mailbox, whatever that may be at the time the + message notifications are being generated. This is done by + specifying either the SELECTED or the SELECTED-DELAYED mailbox + selector (see Section 6.1) in the NOTIFY SET command. If the + SELECTED/SELECTED-DELAYED mailbox selector is not specified in the + NOTIFY SET command, this means that the client doesn't want to + receive any s for the currently selected mailbox. + This is the same as specifying SELECTED NONE. + + The client can also request notifications on other mailboxes by name + or by a limited mailbox pattern match. Message-related notifications + returned for the currently selected mailbox will be those specified + by the SELECTED/SELECTED-DELAYED mailbox specifier, even if the + selected mailbox also appears by name (or matches a pattern) in the + command. Non-message-related notifications are controlled by mailbox + specifiers other than SELECTED/SELECTED-DELAYED. + + If the NOTIFY command enables MessageNew, MessageExpunge, + AnnotationChange, or FlagChange notifications for a mailbox other + than the currently selected mailbox, and the client has specified the + STATUS indicator parameter, then the server MUST send a STATUS + response for that mailbox before NOTIFY's tagged OK. If MessageNew + is enabled, the STATUS response MUST contain MESSAGES, UIDNEXT, and + UIDVALIDITY. If MessageExpunge is enabled, the STATUS response MUST + contain MESSAGES. If either AnnotationChange or FlagChange are + + + +Gulbrandsen, et al. Standards Track [Page 5] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + included and the server also supports the CONDSTORE [RFC4551] and/or + QRESYNC [RFC5162] extensions, the STATUS response MUST contain + UIDVALIDITY and HIGHESTMODSEQ. Absence of the STATUS indicator + parameter allows the client to avoid the additional STATUS responses. + This might be useful if the client already retrieved this information + before issuing the NOTIFY command. + + Clients are advised to limit the number of mailboxes used with + NOTIFY. Particularly, if a client asks for events for all accessible + mailboxes, the server may swamp the client with updates about shared + mailboxes. This may reduce the client's battery life. Also, this + wastes both server and network resources. + + For each mailbox specified, the server verifies that the client has + access using the following test: + + - If the name does not refer to an existing mailbox, the server MUST + ignore it. + + - If the name refers to a mailbox that the client can't LIST, the + server MUST ignore it. For a server that implements [RFC4314], + this means that if the client doesn't have the 'l' (lookup) right + for the name, then the server MUST ignore the mailbox. This + behavior prevents disclosure of potentially confidential + information to clients who don't have rights to know it. + + - If the name refers to a mailbox that the client can LIST (e.g., it + has the 'l' right from [RFC4314]), but the client doesn't have + another right required for processing of the specified event(s), + then the server MUST respond with an untagged extended LIST + response containing the \NoAccess name attribute. + + The server SHOULD return the tagged OK response if the client has + access to at least one of the mailboxes specified in the current list + of interesting events. The server MAY return the tagged NO response + if the client has no access to any of the specified mailboxes and no + access can ever be granted in the future (e.g., the client specified + an event for 'Subtree Bar/Foo', 'Bar/Foo' doesn't exist, and LIST + returns \Noinferiors for the parent 'Bar'). + + If the notification would be prohibitively expensive for the server + (e.g., "notify me of all flag changes in all mailboxes"), the server + MAY refuse the command with a tagged NO [NOTIFICATIONOVERFLOW] + response. + + + + + + + +Gulbrandsen, et al. Standards Track [Page 6] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + If the client requests information for events of an unsupported type, + the server MUST refuse the command with a tagged NO response (not a + BAD). This response SHOULD contain the BADEVENT response code, which + MUST list names of all events supported by the server. + + Here's an example: + + S: * OK [CAPABILITY IMAP4REV1 NOTIFY] + C: a login bob alice + S: a OK Password matched + C: b notify set status (selected MessageNew (uid + body.peek[header.fields (from to subject)]) MessageExpunge) + (subtree Lists MessageNew) + S: * STATUS Lists/Lemonade (UIDVALIDITY 4 UIDNEXT 9999 MESSAGES + 500) + S: [...] + S: * STATUS Lists/Im2000 (UIDVALIDITY 901 UIDNEXT 1 MESSAGES 0) + S: b OK done + C: c select inbox + S: [...] (the usual 7-8 responses to SELECT) + S: c OK INBOX selected + (Time passes. A new message is delivered to mailbox + Lists/Lemonade.) + S: * STATUS Lists/Lemonade (UIDVALIDITY 4 UIDNEXT 10000 + MESSAGES 501) + (Time passes. A new message is delivered to inbox.) + S: * 127 FETCH (UID 127001 BODY[HEADER.FIELDS (From To + Subject)] {75} + S: Subject: Re: good morning + S: From: alice@example.org + S: To: bob@example.org + S: + S: ) + (Time passes. The client decides it wants to know about + one more mailbox. As the client already knows necessary + STATUS information for all mailboxes below the Lists + mailbox, and because "notify set status" would cause + STATUS responses for *all* mailboxes specified in the + NOTIFY command, including the ones for which the client + already knows STATUS information, the client issues an + explicit STATUS request for the mailbox to be added to + the watch list, followed by the NOTIFY SET without the + STATUS parameter.) + C: d STATUS misc (UIDVALIDITY UIDNEXT MESSAGES) + S: * STATUS misc (UIDVALIDITY 1 UIDNEXT 999) + S: d STATUS completed + + + + + +Gulbrandsen, et al. Standards Track [Page 7] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + C: e notify set (selected MessageNew (uid + body.peek[header.fields (from to subject)]) MessageExpunge) + (subtree Lists MessageNew) (mailboxes misc MessageNew) + S: e OK done + +4. Interaction with the IDLE Command + + If IDLE [RFC2177] (as well as this extension) is supported, then + while processing any IDLE command, the server MUST send exactly the + same events as instructed by the client using the NOTIFY command. + + NOTIFY makes IDLE unnecessary for some clients. If a client does not + use MSNs and '*' in commands, it can request MessageExpunge and + MessageNew for the selected mailbox by using the NOTIFY command + instead of entering the IDLE mode. + + A client that uses MSNs and '*' in commands can still use the NOTIFY + command if it specifies the SELECTED-DELAYED mailbox specifier in the + NOTIFY command. + +5. Event Types + + Only some of the events in [RFC5423] can be expressed in IMAP, and + for some of them there are several possible ways to express the + event. + + This section specifies the events of which an IMAP server can notify + an IMAP client, and how. + + The server SHOULD omit notifying the client if the event is caused by + this client. For example, if the client issues CREATE and has + requested a MailboxName event that would cover the newly created + mailbox, the server SHOULD NOT notify the client of the MailboxName + change. + + All event types described in this document require the 'l' and 'r' + rights (see [RFC4314]) on all observed mailboxes. Servers that don't + implement [RFC4314] should map the above rights to their access- + control model. + + If the FlagChange and/or AnnotationChange events are specified, + MessageNew and MessageExpunge MUST also be specified by the client. + Otherwise, the server MUST respond with the tagged BAD response. + + If one of MessageNew or MessageExpunge is specified, then both events + MUST be specified. Otherwise, the server MUST respond with the + tagged BAD response. + + + + +Gulbrandsen, et al. Standards Track [Page 8] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + The client can instruct the server not to send an event by omitting + the necessary event from the list of events specified in NOTIFY SET, + by using the NONE event specifier in the NOTIFY SET, or by using + NOTIFY NONE. In particular, NOTIFY SET ... NONE can be used as a + snapshot facility by clients. + +5.1. FlagChange and AnnotationChange + + If the flag and/or message annotation change happens in the selected + mailbox, the server MUST notify the client by sending an unsolicited + FETCH response, which MUST include UID and FLAGS/ANNOTATION FETCH + data items. It MAY also send new FLAGS and/or OK [PERMANENTFLAGS + ...] responses. + + If a search context is in effect as specified in [RFC5267], an + ESEARCH ADDTO or ESEARCH REMOVEFROM will also be generated, if + appropriate. In this case, the FETCH response MUST precede the + ESEARCH response. + + If the change happens in another mailbox, then the server responds + with a STATUS response. The exact content of the STATUS response + depends on various factors. If CONDSTORE [RFC4551] and/or QRESYNC + [RFC5162] are enabled by the client, then the server sends a STATUS + response that includes at least HIGHESTMODSEQ and UIDVALIDITY status + data items. If the number of messages with the \Seen flag changes, + the server MAY also include the UNSEEN data item in the STATUS + response. If CONDSTORE/QRESYNC is not enabled by the client and the + server chooses not to include the UNSEEN data item, the server does + not notify the client. When this event is requested, the server MUST + notify the client about mailbox UIDVALIDITY changes. This is done by + sending a STATUS response that includes UIDVALIDITY. + + FlagChange covers the MessageRead, MessageTrash, FlagsSet, and + FlagsClear events in [RFC5423]. + + Example in the selected mailbox: + S: * 99 FETCH (UID 9999 FLAGS ($Junk)) + + And in another mailbox, with CONDSTORE in use: + S: * STATUS Lists/Lemonade (HIGHESTMODSEQ 65666665 UIDVALIDITY + 101) + + + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 9] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +5.2. MessageNew + + This covers both MessageNew and MessageAppend in [RFC5423]. + + If the new/appended message is in the selected mailbox, the server + notifies the client by sending an unsolicited EXISTS response, + followed by an unsolicited FETCH response containing the information + requested by the client. A FETCH response SHOULD NOT be generated + for a new message created by the client on this particular + connection, for instance, as the result of an APPEND or COPY command + to the selected mailbox performed by the client itself. The server + MAY also send a RECENT response, if the server marks the message as + \Recent. + + Note that a single EXISTS response can be returned for multiple + MessageAppend/MessageNew events. + + If a search context is in effect as specified in [RFC5267], an + ESEARCH ADDTO will also be generated, if appropriate. In this case, + the EXISTS response MUST precede the ESEARCH response. Both the + NOTIFY command and the SEARCH and SORT commands (see Section 7) can + specify attributes to be returned for new messages. These attributes + SHOULD be combined into a single FETCH response. The server SHOULD + avoid sending duplicate data. The FETCH response(s) MUST follow any + ESEARCH ADDTO responses. + + If the new/appended message is in another mailbox, the server sends + an unsolicited STATUS (UIDNEXT MESSAGES) response for the relevant + mailbox. If the CONDSTORE extension [RFC4551] and/or the QRESYNC + extension [RFC5162] is enabled, the HIGHESTMODSEQ status data item + MUST be included in the STATUS response. + + The client SHOULD NOT use FETCH attributes that implicitly set the + \seen flag, or that presuppose the existence of a given bodypart. + UID, MODSEQ, FLAGS, ENVELOPE, BODY.PEEK[HEADER.FIELDS... and + BODY/BODYSTRUCTURE may be the most useful attributes. + + Note that if a client asks to be notified of MessageNew events with + the SELECTED mailbox specifier, the number of messages can increase + at any time, and therefore the client cannot refer to a specific + message using the MSN/UID '*'. + + Example in the selected mailbox: + S: * 444 EXISTS + S: * 444 FETCH (UID 9999) + + And in another mailbox, without CONDSTORE enabled: + S: * STATUS Lists/Lemonade (UIDNEXT 10002 MESSAGES 503) + + + +Gulbrandsen, et al. Standards Track [Page 10] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +5.3. MessageExpunge + + If the expunged message or messages are in the selected mailbox, the + server notifies the client using EXPUNGE (or VANISHED, if [RFC5162] + is supported by the server and enabled by the client). + + If a search context is in effect, as specified in [RFC5267], an + ESEARCH REMOVEFROM will also be generated, if appropriate. + + If the expunged message or messages are in another mailbox, the + server sends an unsolicited STATUS (UIDNEXT MESSAGES) response for + the relevant mailbox. If the QRESYNC [RFC5162] extension is enabled, + the HIGHESTMODSEQ data item MUST be included in the STATUS response + as well. + + Note that if a client requests MessageExpunge with the SELECTED + mailbox specifier, the meaning of an MSN can change at any time, so + the client cannot use MSNs in commands anymore. For example, such a + client cannot use FETCH, but has to use UID FETCH. The meaning of + '*' can also change when messages are added or expunged. A client + wishing to keep using MSNs can either use the SELECTED-DELAYED + mailbox specifier or can avoid using the MessageExpunge event + entirely. + + The MessageExpunge notification covers both MessageExpunge and + MessageExpire events from [RFC5423]. + + Example in the selected mailbox, without QRESYNC: + S: * 444 EXPUNGE + + The same example in the selected mailbox, with QRESYNC: + S: * VANISHED 5444 + + And in another mailbox, when QRESYNC is not enabled: + S: * STATUS misc (UIDNEXT 999 MESSAGES 554) + +5.4. MailboxName + + These notifications are sent if an affected mailbox name was created + (with CREATE), deleted (with DELETE), or renamed (with RENAME). For + a server that implements [RFC4314], granting or revocation of the 'l' + right to the current user on the affected mailbox MUST be considered + mailbox creation or deletion, respectively. If a mailbox is created + or deleted, the mailbox itself and its direct parent (whether it is + an existing mailbox or not) are considered to be affected. + + + + + + +Gulbrandsen, et al. Standards Track [Page 11] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + The server notifies the client by sending an unsolicited LIST + response for each affected mailbox name. If, after the event, the + mailbox name does not refer to a mailbox accessible to the client, + the \Nonexistent flag MUST be included. + + For each LISTable mailbox renamed, the server sends an extended LIST + response [RFC5258] for the new mailbox name, containing the OLDNAME + extended data item with the old mailbox name. When a mailbox is + renamed, its children are renamed too. No additional MailboxName + events are sent for children in this case. When INBOX is renamed, a + new INBOX is assumed to be created. No MailboxName event is sent for + INBOX in this case. + + If the server automatically subscribes a mailbox when it is created + or renamed, then the unsolicited LIST response for each affected + subscribed mailbox name MUST include the \Subscribed attribute (see + [RFC5258]). The server SHOULD also include \HasChildren or + \HasNoChildren attributes [RFC5258] as appropriate. + + Example of a newly created mailbox (or granting of the 'l' right on + the mailbox): + S: * LIST () "/" "NewMailbox" + + And a deleted mailbox (or revocation of the 'l' right on the + mailbox): + S: * LIST (\NonExistent) "." "INBOX.DeletedMailbox" + + Example of a renamed mailbox: + S: * LIST () "/" "NewMailbox" ("OLDNAME" ("OldMailbox")) + +5.5. SubscriptionChange + + The server notifies the client by sending an unsolicited LIST + response for each affected mailbox name. If and only if the mailbox + is subscribed after the event, the \Subscribed attribute (see + [RFC5258]) is included. Note that in the LIST response, all mailbox + attributes MUST be accurately computed (this differs from the + behavior of the LSUB command). + + Example: + S: * LIST (\Subscribed) "/" "SubscribedMailbox" + +5.6. MailboxMetadataChange + + Support for this event type is OPTIONAL unless the METADATA extension + [RFC5464] is also supported by the server, in which case support for + this event type is REQUIRED. + + + + +Gulbrandsen, et al. Standards Track [Page 12] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + A client willing to receive unsolicited METADATA responses as a + result of using the MailboxMetadataChange event in the NOTIFY command + doesn't have to issue ENABLE METADATA. + + The server sends an unsolicited METADATA response (as per Section + 4.4.2 of [RFC5464]). If possible, only the changed metadata SHOULD + be included, but if the server can't detect a change to a single + metadata item, it MAY include all metadata items set on the mailbox. + If a metadata item is deleted (set to NIL), it MUST always be + included in the METADATA response. + + Example: + S: * METADATA "INBOX" /shared/comment + +5.7. ServerMetadataChange + + Support for this event type is OPTIONAL unless the METADATA or the + METADATA-SERVER extension [RFC5464] is also supported by the server, + in which case support for this event type is REQUIRED. + + A client willing to receive unsolicited METADATA responses as a + result of using the ServerMetadataChange event in the NOTIFY command + doesn't have to issue ENABLE METADATA or ENABLE METADATA-SERVER. + + The server sends an unsolicited METADATA response (as per Section + 4.4.2 of [RFC5464]). Only the names of changed metadata entries + SHOULD be returned in such METADATA responses. If a metadata item is + deleted (set to NIL), it MUST always be included in the METADATA + response. + + Example: + S: * METADATA "" /shared/comment + +5.8. Notification Overflow + + If the server is unable or unwilling to deliver as many notifications + as it is being asked to, it may disable notifications for some or all + clients. It MUST notify these clients by sending an untagged "OK + [NOTIFICATIONOVERFLOW]" response and behave as if a NOTIFY NONE + command had just been received. + + Example: + S: * OK [NOTIFICATIONOVERFLOW] ...A comment can go here... + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 13] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +5.9. ACL (Access Control List) Changes + + Even if NOTIFY succeeds, it is still possible to lose access to the + mailboxes being monitored at a later time. If this happens, the + server MUST stop monitoring these mailboxes. If access is later + granted, the server MUST restart event monitoring. + + The server SHOULD return the LIST response with the \NoAccess name + attribute if and when the mailbox loses the 'l' right. Similarly, + the server SHOULD return the LIST response with no \NoAccess name + attribute if the mailbox was previously reported as having \NoAccess + and the 'l' right is later granted. + +6. Mailbox Specification + + Mailboxes to be monitored can be specified in several different ways. + + Only 'SELECTED' and 'SELECTED-DELAYED' (Section 6.1) match the + currently selected mailbox. All other mailbox specifications affect + other (non-selected) mailboxes. + + Note that multiple s can apply to the same mailbox. The + following example demonstrates this. In this example, MessageNew and + MessageExpunge events are reported for INBOX, due to the first + . A SubscriptionChange event will also be reported for + INBOX, due to the second . + + C: a notify set (mailboxes INBOX (Messagenew messageExpunge)) + (personal (SubscriptionChange)) + + A typical client that supports the NOTIFY extension would ask for + events on the selected mailbox and some named mailboxes. + + In the next example, the client asks for FlagChange events for all + personal mailboxes except the currently selected mailbox. This is + different from the previous example because SELECTED overrides all + other message event definitions for the currently selected mailbox + (see Section 3.1). + + C: a notify set (selected (Messagenew (uid flags) messageExpunge)) + (personal (MessageNew FlagChange MessageExpunge)) + +6.1. Mailbox Specifiers Affecting the Currently Selected Mailbox + + Only one of the mailbox specifiers affecting the currently selected + mailbox can be specified in any NOTIFY command. The two such mailbox + specifiers (SELECTED and SELECTED-DELAYED) are described below. + + + + +Gulbrandsen, et al. Standards Track [Page 14] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + Both refer to the mailbox that was selected using either SELECT or + EXAMINE (see [RFC3501], Sections 6.3.1 and 6.3.2). When the IMAP + connection is not in the selected state, such mailbox specifiers + don't refer to any mailbox. + + The mailbox specifiers only apply to s. It is an + error to specify other types of events with either the SELECTED or + the SELECTED-DELAYED selector. + +6.1.1. Selected + + The SELECTED mailbox specifier requires the server to send immediate + notifications for the currently selected mailbox about all specified + s. + +6.1.2. Selected-Delayed + + The SELECTED-DELAYED mailbox specifier requires the server to delay a + MessageExpunge event until the client issues a command that allows + returning information about expunged messages (see Section 7.4.1 of + [RFC3501] for more details), for example, till a NOOP or an IDLE + command has been issued. When SELECTED-DELAYED is specified, the + server MAY also delay returning other s until the + client issues one of the commands specified above, or it MAY return + them immediately. + +6.2. Personal + + Personal refers to all selectable mailboxes in the user's personal + namespace(s), as defined in [RFC2342]. + +6.3. Inboxes + + Inboxes refers to all selectable mailboxes in the user's personal + namespace(s) to which messages may be delivered by a Message Delivery + Agent (MDA) (see [EMAIL-ARCH], particularly Section 4.3.3). + + If the IMAP server cannot easily compute this set, it MUST treat + "inboxes" as equivalent to "personal". + +6.4. Subscribed + + Subscribed refers to all mailboxes subscribed to by the user. + + If the subscription list changes, the server MUST reevaluate the + list. + + + + + +Gulbrandsen, et al. Standards Track [Page 15] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +6.5. Subtree + + Subtree is followed by a mailbox name or list of mailbox names. A + subtree refers to all selectable mailboxes that are subordinate to + the specified mailbox plus the specified mailbox itself. + +6.6. Mailboxes + + Mailboxes is followed by a mailbox name or a list of mailbox names. + The server MUST NOT do a wildcard expansion. This means there is no + special treatment for the LIST wildcard characters ('*' and '%') if + they are present in mailbox names. + +7. Extension to SEARCH and SORT Commands + + If the server that supports the NOTIFY extension also supports + CONTEXT=SEARCH and/or CONTEXT=SORT as defined in [RFC5267], the + UPDATE return option is extended so that a client can request that + FETCH attributes be returned when a new message is added to the + context result set. + + For example: + + C: a00 SEARCH RETURN (COUNT UPDATE (UID BODY[HEADER.FIELDS (TO + FROM SUBJECT)])) FROM "boss" + S: * ESEARCH (TAG "a00") (COUNT 17) + S: a00 OK + [...a new message is delivered...] + S: * EXISTS 93 + S: * 93 FETCH (UID 127001 BODY[HEADER.FIELDS (FROM TO SUBJECT)] + {76} + S: Subject: Re: good morning + S: From: myboss@example.org + S: To: bob@example.org + S: + S: ) + S: * ESEARCH (TAG "a00") ADDTO (0 93) + + Note that the EXISTS response MUST precede any FETCH responses, and + together they MUST precede the ESEARCH response. + + No untagged FETCH response SHOULD be returned if a message becomes a + member of UPDATE SEARCH due to flag or annotation changes. + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 16] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +8. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. [RFC3501] defines + the non-terminals "capability", "command-auth", "mailbox", "mailbox- + data", "resp-text-code", and "search-key". The "modifier-update" + non-terminal is defined in [RFC5267]. "mbx-list-oflag" is defined in + [RFC3501] and updated by [RFC5258]. + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lower case characters to define + token strings is for editorial clarity only. Implementations MUST + accept these strings in a case-insensitive fashion. For example, the + non-terminal value "SELECTED" must be + treated in the same way as "Selected" or "selected". + + capability =/ "NOTIFY" + + command-auth =/ notify + + notify = "NOTIFY" SP + (notify-set / notify-none) + + notify-set = "SET" [status-indicator] SP event-groups + ; Replace registered notification events + ; with the specified list of events + + notify-none = "NONE" + ; Cancel all registered notification + ; events. The client is not interested + ; in receiving any events. + + status-indicator = SP "STATUS" + + one-or-more-mailbox = mailbox / many-mailboxes + + many-mailboxes = "(" mailbox *(SP mailbox) ")" + + event-groups = event-group *(SP event-group) + + event-group = "(" filter-mailboxes SP events ")" + ;; Only s are allowed in + ;; when is used. + + filter-mailboxes = filter-mailboxes-selected / + filter-mailboxes-other + + + + + +Gulbrandsen, et al. Standards Track [Page 17] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + filter-mailboxes-other = "inboxes" / "personal" / "subscribed" / + ( "subtree" SP one-or-more-mailbox ) / + ( "mailboxes" SP one-or-more-mailbox ) + + filter-mailboxes-selected = "selected" / "selected-delayed" + ;; Apply to the currently selected mailbox only. + ;; Only one of them can be specified in a NOTIFY + ;; command. + + events = ( "(" event *(SP event) ")" ) / "NONE" + ;; As in [MSGEVENT]. + ;; "NONE" means that the client does not wish + ;; to receive any events for the specified + ;; mailboxes. + + event = message-event / + mailbox-event / user-event / event-ext + + message-event = ( "MessageNew" [SP + "(" fetch-att *(SP fetch-att) ")" ] ) + / "MessageExpunge" + / "FlagChange" + / "AnnotationChange" + ;; "MessageNew" includes "MessageAppend" from + ;; [MSGEVENT]. "FlagChange" is any of + ;; "MessageRead", "MessageTrash", "FlagsSet", + ;; "FlagsClear" [MSGEVENT]. "MessageExpunge" + ;; includes "MessageExpire" [MSGEVENT]. + ;; MessageNew and MessageExpunge MUST always + ;; be specified together. If FlagChange is + ;; specified, then MessageNew and MessageExpunge + ;; MUST be specified as well. + ;; The fett-att list may only be present for the + ;; SELECTED/SELECTED-DELAYED mailbox filter + ;; (). + + mailbox-event = "MailboxName" / + "SubscriptionChange" / "MailboxMetadataChange" + ; "SubscriptionChange" includes + ; MailboxSubscribe and MailboxUnSubscribe. + ; "MailboxName" includes MailboxCreate, + ; "MailboxDelete" and "MailboxRename". + + user-event = "ServerMetadataChange" + + event-ext = atom + ;; For future extensions + + + + +Gulbrandsen, et al. Standards Track [Page 18] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + oldname-extended-item = "OLDNAME" SP "(" mailbox ")" + ;; Extended data item (mbox-list-extended-item) + ;; returned in a LIST response when a mailbox is + ;; renamed. + ;; Note 1: the OLDNAME tag can be returned + ;; with or without surrounding quotes, as per + ;; mbox-list-extended-item-tag production. + + resp-text-code =/ "NOTIFICATIONOVERFLOW" / + unsupported-events-code + + message-event-name = "MessageNew" / + "MessageExpunge" / "FlagChange" / + "AnnotationChange" + + event-name = message-event-name / mailbox-event / + user-event + + unsupported-events-code = "BADEVENT" + SP "(" event-name *(SP event-name) ")" + + modifier-update = "UPDATE" + [ "(" fetch-att *(SP fetch-att) ")" ] + + mbx-list-oflag =/ "\NoAccess" + +9. Security Considerations + + It is very easy for a client to deny itself service using NOTIFY. + Asking for all events on all mailboxes may work on a small server, + but with a big server, can swamp the client's network connection or + processing capability. In the worst case, the server's processing + could also degrade the service it offers to other clients. + + Server authors should be aware that if a client issues requests and + does not listen to the resulting responses, the TCP window can easily + fill up, and a careless server might block. This problem also exists + in plain IMAP; however, this extension magnifies the problem. + + This extension makes it possible to retrieve messages immediately + when they are added to the mailbox. This makes it wholly impractical + to delete sensitive messages using programs like imapfilter. Using + SIEVE [RFC5228] or similar is much better. + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 19] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +10. IANA Considerations + + The IANA has added NOTIFY to the list of IMAP extensions. + +10.1. Initial LIST-EXTENDED Extended Data Item Registrations + + The following entry has been added to the LIST-EXTENDED response + registry [RFC5258]: + + To: iana@iana.org + Subject: Registration of OLDNAME LIST-EXTENDED extended data item + + LIST-EXTENDED extended data item tag: OLDNAME + + LIST-EXTENDED extended data item description: The OLDNAME extended + data item describes the old mailbox name for the mailbox + identified by the LIST response. + + Which LIST-EXTENDED option(s) (and their types) causes this extended + data item to be returned (if any): none + + Published specification : RFC 5465, Section 5.4. + + Security considerations: none + + Intended usage: COMMON + + Person and email address to contact for further information: Alexey + Melnikov + + Owner/Change controller: iesg@ietf.org + +11. Acknowledgments + + The authors gratefully acknowledge the help of Peter Coates, Dave + Cridland, Mark Crispin, Cyrus Daboo, Abhijit Menon-Sen, Timo + Sirainen, and Eric Burger. In particular, Peter Coates contributed + lots of text and useful suggestions to this document. + + Various examples are copied from other RFCs. + + This document builds on one published and two unpublished drafts by + the same authors. + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 20] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + +12. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2177] Leiba, B., "IMAP4 IDLE command", RFC 2177, June 1997. + + [RFC2342] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, + May 1998. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) + Extension", RFC 4314, December 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to + IMAP4 ABNF", RFC 4466, April 2006. + + [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for + Conditional STORE Operation or Quick Flag Changes + Resynchronization", RFC 4551, June 2006. + + [RFC5162] Melnikov, A., Cridland, D., and C. Wilson, "IMAP4 + Extensions for Quick Mailbox Resynchronization", RFC + 5162, March 2008. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access + Protocol version 4 - LIST Command Extensions", RFC 5258, + June 2008. + + [RFC5267] Cridland, D. and C. King, "Contexts for IMAP4", RFC + 5267, July 2008. + + [RFC5423] Newman, C. and R. Gellens, "Internet Message Store + Events", RFC 5423, Month 2009. + + [RFC5464] Daboo, C., "The IMAP METADATA Extension", RFC 5464, + February 2009. + +13. Informative References + + [RFC5228] Guenther, P., Ed., and T. Showalter, Ed., "Sieve: An + Email Filtering Language", RFC 5228, January 2008. + + + +Gulbrandsen, et al. Standards Track [Page 21] + +RFC 5465 IMAP NOTIFY Extension February 2009 + + + [EMAIL-ARCH] Crocker, D., "Internet Mail Architecture", Work in + Progress, October 2008. + +Authors' Addresses + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + EMail: arnt@oryx.com + + + Curtis King + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Curtis.King@isode.com + + + Alexey Melnikov + Isode Ltd + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + + + + + + + + + + + + + + + + + + + +Gulbrandsen, et al. Standards Track [Page 22] + diff --git a/docs/rfcs/rfc5530.IMAP_Response_codes.txt b/docs/rfcs/rfc5530.IMAP_Response_codes.txt new file mode 100644 index 0000000..946fbb5 --- /dev/null +++ b/docs/rfcs/rfc5530.IMAP_Response_codes.txt @@ -0,0 +1,507 @@ + + + + + + +Network Working Group A. Gulbrandsen +Request for Comments: 5530 Oryx Mail Systems GmbH +Category: Standards Track May 2009 + + + IMAP Response Codes + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (c) 2009 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents in effect on the date of + publication of this document (http://trustee.ietf.org/license-info). + Please review these documents carefully, as they describe your rights + and restrictions with respect to this document. + +Abstract + + IMAP responses consist of a response type (OK, NO, BAD), an optional + machine-readable response code, and a human-readable text. + + This document collects and documents a variety of machine-readable + response codes, for better interoperation and error reporting. + + + + + + + + + + + + + + + + + + +Gulbrandsen Standards Track [Page 1] + +RFC 5530 IMAP Response Codes May 2009 + + +1. Introduction + + Section 7.1 of [RFC3501] defines a number of response codes that can + help tell an IMAP client why a command failed. However, experience + has shown that more codes are useful. For example, it is useful for + a client to know that an authentication attempt failed because of a + server problem as opposed to a password problem. + + Currently, many IMAP servers use English-language, human-readable + text to describe these errors, and a few IMAP clients attempt to + translate this text into the user's language. + + This document names a variety of errors as response codes. It is + based on errors that have been checked and reported on in some IMAP + server implementations, and on the needs of some IMAP clients. + + This document doesn't require any servers to test for these errors or + any clients to test for these names. It only names errors for better + reporting and handling. + +2. Conventions Used in This Document + + Formal syntax is defined by [RFC5234] as modified by [RFC3501]. + + Example lines prefaced by "C:" are sent by the client and ones + prefaced by "S:" by the server. "[...]" means elision. + +3. Response Codes + + This section defines all the new response codes. Each definition is + followed by one or more examples. + + UNAVAILABLE + Temporary failure because a subsystem is down. For example, an + IMAP server that uses a Lightweight Directory Access Protocol + (LDAP) or Radius server for authentication might use this + response code when the LDAP/Radius server is down. + + C: a LOGIN "fred" "foo" + S: a NO [UNAVAILABLE] User's backend down for maintenance + + AUTHENTICATIONFAILED + Authentication failed for some reason on which the server is + unwilling to elaborate. Typically, this includes "unknown + user" and "bad password". + + + + + + +Gulbrandsen Standards Track [Page 2] + +RFC 5530 IMAP Response Codes May 2009 + + + This is the same as not sending any response code, except that + when a client sees AUTHENTICATIONFAILED, it knows that the + problem wasn't, e.g., UNAVAILABLE, so there's no point in + trying the same login/password again later. + + C: b LOGIN "fred" "foo" + S: b NO [AUTHENTICATIONFAILED] Authentication failed + + AUTHORIZATIONFAILED + Authentication succeeded in using the authentication identity, + but the server cannot or will not allow the authentication + identity to act as the requested authorization identity. This + is only applicable when the authentication and authorization + identities are different. + + C: c1 AUTHENTICATE PLAIN + [...] + S: c1 NO [AUTHORIZATIONFAILED] No such authorization-ID + + C: c2 AUTHENTICATE PLAIN + [...] + S: c2 NO [AUTHORIZATIONFAILED] Authenticator is not an admin + + + EXPIRED + Either authentication succeeded or the server no longer had the + necessary data; either way, access is no longer permitted using + that passphrase. The client or user should get a new + passphrase. + + C: d login "fred" "foo" + S: d NO [EXPIRED] That password isn't valid any more + + PRIVACYREQUIRED + The operation is not permitted due to a lack of privacy. If + Transport Layer Security (TLS) is not in use, the client could + try STARTTLS (see Section 6.2.1 of [RFC3501]) and then repeat + the operation. + + C: d login "fred" "foo" + S: d NO [PRIVACYREQUIRED] Connection offers no privacy + + C: d select inbox + S: d NO [PRIVACYREQUIRED] Connection offers no privacy + + + + + + + +Gulbrandsen Standards Track [Page 3] + +RFC 5530 IMAP Response Codes May 2009 + + + CONTACTADMIN + The user should contact the system administrator or support + desk. + + C: e login "fred" "foo" + S: e OK [CONTACTADMIN] + + NOPERM + The access control system (e.g., Access Control List (ACL), see + [RFC4314]) does not permit this user to carry out an operation, + such as selecting or creating a mailbox. + + C: f select "/archive/projects/experiment-iv" + S: f NO [NOPERM] Access denied + + INUSE + An operation has not been carried out because it involves + sawing off a branch someone else is sitting on. Someone else + may be holding an exclusive lock needed for this operation, or + the operation may involve deleting a resource someone else is + using, typically a mailbox. + + The operation may succeed if the client tries again later. + + C: g delete "/archive/projects/experiment-iv" + S: g NO [INUSE] Mailbox in use + + EXPUNGEISSUED + Someone else has issued an EXPUNGE for the same mailbox. The + client may want to issue NOOP soon. [RFC2180] discusses this + subject in depth. + + C: h search from fred@example.com + S: * SEARCH 1 2 3 5 8 13 21 42 + S: h OK [EXPUNGEISSUED] Search completed + + CORRUPTION + The server discovered that some relevant data (e.g., the + mailbox) are corrupt. This response code does not include any + information about what's corrupt, but the server can write that + to its logfiles. + + C: i select "/archive/projects/experiment-iv" + S: i NO [CORRUPTION] Cannot open mailbox + + + + + + + +Gulbrandsen Standards Track [Page 4] + +RFC 5530 IMAP Response Codes May 2009 + + + SERVERBUG + The server encountered a bug in itself or violated one of its + own invariants. + + C: j select "/archive/projects/experiment-iv" + S: j NO [SERVERBUG] This should not happen + + CLIENTBUG + The server has detected a client bug. This can accompany all + of OK, NO, and BAD, depending on what the client bug is. + + C: k1 select "/archive/projects/experiment-iv" + [...] + S: k1 OK [READ-ONLY] Done + C: k2 status "/archive/projects/experiment-iv" (messages) + [...] + S: k2 OK [CLIENTBUG] Done + + CANNOT + The operation violates some invariant of the server and can + never succeed. + + C: l create "///////" + S: l NO [CANNOT] Adjacent slashes are not supported + + LIMIT + The operation ran up against an implementation limit of some + kind, such as the number of flags on a single message or the + number of flags used in a mailbox. + + C: m STORE 42 FLAGS f1 f2 f3 f4 f5 ... f250 + S: m NO [LIMIT] At most 32 flags in one mailbox supported + + OVERQUOTA + The user would be over quota after the operation. (The user + may or may not be over quota already.) + + Note that if the server sends OVERQUOTA but doesn't support the + IMAP QUOTA extension defined by [RFC2087], then there is a + quota, but the client cannot find out what the quota is. + + C: n1 uid copy 1:* oldmail + S: n1 NO [OVERQUOTA] Sorry + + C: n2 uid copy 1:* oldmail + S: n2 OK [OVERQUOTA] You are now over your soft quota + + + + + +Gulbrandsen Standards Track [Page 5] + +RFC 5530 IMAP Response Codes May 2009 + + + ALREADYEXISTS + The operation attempts to create something that already exists, + such as when the CREATE or RENAME directories attempt to create + a mailbox and there is already one of that name. + + C: o RENAME this that + S: o NO [ALREADYEXISTS] Mailbox "that" already exists + + NONEXISTENT + The operation attempts to delete something that does not exist. + Similar to ALREADYEXISTS. + + C: p RENAME this that + S: p NO [NONEXISTENT] No such mailbox + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. [RFC3501] defines + the non-terminal "resp-text-code". + + Except as noted otherwise, all alphabetic characters are case- + insensitive. The use of upper or lowercase characters to define + token strings is for editorial clarity only. + + resp-text-code =/ "UNAVAILABLE" / "AUTHENTICATIONFAILED" / + "AUTHORIZATIONFAILED" / "EXPIRED" / + "PRIVACYREQUIRED" / "CONTACTADMIN" / "NOPERM" / + "INUSE" / "EXPUNGEISSUED" / "CORRUPTION" / + "SERVERBUG" / "CLIENTBUG" / "CANNOT" / + "LIMIT" / "OVERQUOTA" / "ALREADYEXISTS" / + "NONEXISTENT" + +5. Security Considerations + + Revealing information about a passphrase to unauthenticated IMAP + clients causes bad karma. + + Response codes are easier to parse than human-readable text. This + can amplify the consequences of an information leak. For example, + selecting a mailbox can fail because the mailbox doesn't exist, + because the user doesn't have the "l" right (right to know the + mailbox exists) or "r" right (right to read the mailbox). If the + server sent different responses in the first two cases in the past, + only malevolent clients would discover it. With response codes it's + possible, perhaps probable, that benevolent clients will forward the + + + + + +Gulbrandsen Standards Track [Page 6] + +RFC 5530 IMAP Response Codes May 2009 + + + leaked information to the user. Server authors are encouraged to be + particularly careful with the NOPERM and authentication-related + responses. + +6. IANA Considerations + + The IANA has created the IMAP Response Codes registry. The registry + has been populated with the following codes: + + NEWNAME RFC 2060 (obsolete) + REFERRAL RFC 2221 + ALERT RFC 3501 + BADCHARSET RFC 3501 + PARSE RFC 3501 + PERMANENTFLAGS RFC 3501 + READ-ONLY RFC 3501 + READ-WRITE RFC 3501 + TRYCREATE RFC 3501 + UIDNEXT RFC 3501 + UIDVALIDITY RFC 3501 + UNSEEN RFC 3501 + UNKNOWN-CTE RFC 3516 + UIDNOTSTICKY RFC 4315 + APPENDUID RFC 4315 + COPYUID RFC 4315 + URLMECH RFC 4467 + TOOBIG RFC 4469 + BADURL RFC 4469 + HIGHESTMODSEQ RFC 4551 + NOMODSEQ RFC 4551 + MODIFIED RFC 4551 + COMPRESSIONACTIVE RFC 4978 + CLOSED RFC 5162 + NOTSAVED RFC 5182 + BADCOMPARATOR RFC 5255 + ANNOTATE RFC 5257 + ANNOTATIONS RFC 5257 + TEMPFAIL RFC 5259 + MAXCONVERTMESSAGES RFC 5259 + MAXCONVERTPARTS RFC 5259 + NOUPDATE RFC 5267 + METADATA RFC 5464 + NOTIFICATIONOVERFLOW RFC 5465 + BADEVENT RFC 5465 + UNDEFINED-FILTER RFC 5466 + UNAVAILABLE RFC 5530 + AUTHENTICATIONFAILED RFC 5530 + AUTHORIZATIONFAILED RFC 5530 + + + +Gulbrandsen Standards Track [Page 7] + +RFC 5530 IMAP Response Codes May 2009 + + + EXPIRED RFC 5530 + PRIVACYREQUIRED RFC 5530 + CONTACTADMIN RFC 5530 + NOPERM RFC 5530 + INUSE RFC 5530 + EXPUNGEISSUED RFC 5530 + CORRUPTION RFC 5530 + SERVERBUG RFC 5530 + CLIENTBUG RFC 5530 + CANNOT RFC 5530 + LIMIT RFC 5530 + OVERQUOTA RFC 5530 + ALREADYEXISTS RFC 5530 + NONEXISTENT RFC 5530 + + The new registry can be extended by sending a registration request to + IANA. IANA will forward this request to a Designated Expert, + appointed by the responsible IESG Area Director, CCing it to the IMAP + Extensions mailing list at (or a successor + designated by the Area Director). After either allowing 30 days for + community input on the IMAP Extensions mailing list or a successful + IETF Last Call, the expert will determine the appropriateness of the + registration request and either approve or disapprove the request by + sending a notice of the decision to the requestor, CCing the IMAP + Extensions mailing list and IANA. A denial notice must be justified + by an explanation, and, in cases where it is possible, concrete + suggestions on how the request can be modified so as to become + acceptable should be provided. + + For each response code, the registry contains a list of relevant RFCs + that describe (or extend) the response code and an optional response + code status description, such as "obsolete" or "reserved to prevent + collision with deployed software". (Note that in the latter case, + the RFC number can be missing.) Presence of the response code status + description means that the corresponding response code is NOT + RECOMMENDED for widespread use. + + The intention is that any future allocation will be accompanied by a + published RFC (including direct submissions to the RFC Editor). But + in order to allow for the allocation of values prior to the RFC being + approved for publication, the Designated Expert can approve + allocations once it seems clear that an RFC will be published, for + example, before requesting IETF LC for the document. + + The Designated Expert can also approve registrations for response + codes used in deployed software when no RFC exists. Such + registrations must be marked as "reserved to prevent collision with + deployed software". + + + +Gulbrandsen Standards Track [Page 8] + +RFC 5530 IMAP Response Codes May 2009 + + + Response code registrations may not be deleted; response codes that + are no longer believed appropriate for use (for example, if there is + a problem with the syntax of said response code or if the + specification describing it was moved to Historic) should be marked + "obsolete" in the registry, clearly marking the lists published by + IANA. + +7. Acknowledgements + + Peter Coates, Mark Crispin, Philip Guenther, Alexey Melnikov, Ken + Murchison, Chris Newman, Timo Sirainen, Philip Van Hoof, Dale + Wiggins, and Sarah Wilkin helped with this document. + +8. References + +8.1. Normative References + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + +9. Informative References + + [RFC2087] Myers, J., "IMAP4 QUOTA extension", RFC 2087, January + 1997. + + [RFC2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC + 2180, July 1997. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + +Author's Address + + Arnt Gulbrandsen + Oryx Mail Systems GmbH + Schweppermannstr. 8 + D-81671 Muenchen + Germany + + Fax: +49 89 4502 9758 + EMail: arnt@oryx.com + + + + + + +Gulbrandsen Standards Track [Page 9] + diff --git a/docs/rfcs/rfc5738.IMAP_UTF8.txt b/docs/rfcs/rfc5738.IMAP_UTF8.txt new file mode 100644 index 0000000..2b5daaa --- /dev/null +++ b/docs/rfcs/rfc5738.IMAP_UTF8.txt @@ -0,0 +1,843 @@ + + + + + + +Network Working Group P. Resnick +Request for Comments: 5738 Qualcomm Incorporated +Updates: 3501 C. Newman +Category: Experimental Sun Microsystems + March 2010 + + + IMAP Support for UTF-8 + +Abstract + + This specification extends the Internet Message Access Protocol + version 4rev1 (IMAP4rev1) to support UTF-8 encoded international + characters in user names, mail addresses, and message headers. + +Status of This Memo + + This document is not an Internet Standards Track specification; it is + published for examination, experimental implementation, and + evaluation. + + This document defines an Experimental Protocol for the Internet + community. This document is a product of the Internet Engineering + Task Force (IETF). It represents the consensus of the IETF + community. It has received public review and has been approved for + publication by the Internet Engineering Steering Group (IESG). Not + all documents approved by the IESG are a candidate for any level of + Internet Standard; see Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc5738. + +Copyright Notice + + Copyright (c) 2010 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + +Resnick & Newman Experimental [Page 1] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + This document may contain material from IETF Documents or IETF + Contributions published or made publicly available before November + 10, 2008. The person(s) controlling the copyright in some of this + material may not have granted the IETF Trust the right to allow + modifications of such material outside the IETF Standards Process. + Without obtaining an adequate license from the person(s) controlling + the copyright in such materials, this document may not be modified + outside the IETF Standards Process, and derivative works of it may + not be created outside the IETF Standards Process, except to format + it for publication as an RFC or to translate it into languages other + than English. + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . 3 + 3. UTF8=ACCEPT IMAP Capability . . . . . . . . . . . . . . . . . 3 + 3.1. IMAP UTF-8 Quoted Strings . . . . . . . . . . . . . . . . 3 + 3.2. UTF8 Parameter to SELECT and EXAMINE . . . . . . . . . . . 5 + 3.3. UTF-8 LIST and LSUB Responses . . . . . . . . . . . . . . 5 + 3.4. UTF-8 Interaction with IMAP4 LIST Command Extensions . . . 6 + 3.4.1. UTF8 and UTF8ONLY LIST Selection Options . . . . . . . 6 + 3.4.2. UTF8 LIST Return Option . . . . . . . . . . . . . . . 6 + 4. UTF8=APPEND Capability . . . . . . . . . . . . . . . . . . . . 7 + 5. UTF8=USER Capability . . . . . . . . . . . . . . . . . . . . . 7 + 6. UTF8=ALL Capability . . . . . . . . . . . . . . . . . . . . . 7 + 7. UTF8=ONLY Capability . . . . . . . . . . . . . . . . . . . . . 8 + 8. Up-Conversion Server Requirements . . . . . . . . . . . . . . 8 + 9. Issues with UTF-8 Header Mailstore . . . . . . . . . . . . . . 9 + 10. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 9 + 11. Security Considerations . . . . . . . . . . . . . . . . . . . 11 + 12. References . . . . . . . . . . . . . . . . . . . . . . . . . . 11 + 12.1. Normative References . . . . . . . . . . . . . . . . . . . 11 + 12.2. Informative References . . . . . . . . . . . . . . . . . . 13 + Appendix A. Design Rationale . . . . . . . . . . . . . . . . . . 14 + Appendix B. Examples Demonstrating Relationships between + UTF8= Capabilities . . . . . . . . . . . . . . . . . 15 + Appendix C. Acknowledgments . . . . . . . . . . . . . . . . . . . 15 + + + + + + + + + + + + + +Resnick & Newman Experimental [Page 2] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +1. Introduction + + This specification extends IMAP4rev1 [RFC3501] to permit UTF-8 + [RFC3629] in headers as described in "Internationalized Email + Headers" [RFC5335]. It also adds a mechanism to support mailbox + names, login names, and passwords using the UTF-8 charset. This + specification creates five new IMAP capabilities to allow servers to + advertise these new extensions, along with two new IMAP LIST + selection options and a new IMAP LIST return option. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" + in this document are to be interpreted as defined in "Key words for + use in RFCs to Indicate Requirement Levels" [RFC2119]. + + The formal syntax uses the Augmented Backus-Naur Form (ABNF) + [RFC5234] notation including the core rules defined in Appendix B of + [RFC5234]. In addition, rules from IMAP4rev1 [RFC3501], UTF-8 + [RFC3629], "Collected Extensions to IMAP4 ABNF" [RFC4466], and IMAP4 + LIST Command Extensions [RFC5258] are also referenced. + + In examples, "C:" and "S:" indicate lines sent by the client and + server, respectively. If a single "C:" or "S:" label applies to + multiple lines, then the line breaks between those lines are for + editorial clarity only and are not part of the actual protocol + exchange. + +3. UTF8=ACCEPT IMAP Capability + + The "UTF8=ACCEPT" capability indicates that the server supports UTF-8 + quoted strings, the "UTF8" parameter to SELECT and EXAMINE, and UTF-8 + responses from the LIST and LSUB commands. + + A client MUST use the "ENABLE UTF8=ACCEPT" command (defined in + [RFC5161]) to indicate to the server that the client accepts UTF-8 + quoted-strings. The "ENABLE UTF8=ACCEPT" command MUST only be used + in the authenticated state. (Note that the "UTF8=ONLY" capability + described in Section 7 and the "UTF8=ALL" capability described in + Section 6 imply the "UTF8=ACCEPT" capability. See additional + information in these sections.) + +3.1. IMAP UTF-8 Quoted Strings + + The IMAP4rev1 [RFC3501] base specification forbids the use of 8-bit + characters in atoms or quoted strings. Thus, a UTF-8 string can only + be sent as a literal. This can be inconvenient from a coding + standpoint, and unless the server offers IMAP4 non-synchronizing + + + +Resnick & Newman Experimental [Page 3] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + literals [RFC2088], this requires an extra round trip for each UTF-8 + string sent by the client. When the IMAP server advertises the + "UTF8=ACCEPT" capability, it informs the client that it supports + native UTF-8 quoted-strings with the following syntax: + + string =/ utf8-quoted + + utf8-quoted = "*" DQUOTE *UQUOTED-CHAR DQUOTE + + UQUOTED-CHAR = QUOTED-CHAR / UTF8-2 / UTF8-3 / UTF8-4 + ; UTF8-2, UTF8-3, and UTF8-4 are as defined in RFC 3629 + + When this quoting mechanism is used by the client (specifically an + octet sequence beginning with *" and ending with "), then the server + MUST reject octet sequences with the high bit set that fail to comply + with the formal syntax in [RFC3629] with a BAD response. + + The IMAP server MUST NOT send utf8-quoted syntax to the client unless + the client has indicated support for that syntax by using the "ENABLE + UTF8=ACCEPT" command. + + If the server advertises the "UTF8=ACCEPT" capability, the client MAY + use utf8-quoted syntax with any IMAP argument that permits a string + (including astring and nstring). However, if characters outside the + US-ASCII repertoire are used in an inappropriate place, the results + would be the same as if other syntactically valid but semantically + invalid characters were used. For example, if the client includes + UTF-8 characters in the user or password arguments (and the server + has not advertised "UTF8=USER"), the LOGIN command will fail as it + would with any other invalid user name or password. Specific cases + where UTF-8 characters are permitted or not permitted are described + in the following paragraphs. + + All IMAP servers that advertise the "UTF8=ACCEPT" capability SHOULD + accept UTF-8 in mailbox names, and those that also support the + "Mailbox International Naming Convention" described in RFC 3501, + Section 5.1.3 MUST accept utf8-quoted mailbox names and convert them + to the appropriate internal format. Mailbox names MUST comply with + the Net-Unicode Definition (Section 2 of [RFC5198]) with the specific + exception that they MUST NOT contain control characters (0000-001F, + 0080-009F), delete (007F), line separator (2028), or paragraph + separator (2029). + + An IMAP client MUST NOT issue a SEARCH command that uses a mixture of + utf8-quoted syntax and a SEARCH CHARSET other than UTF-8. If an IMAP + server receives such a SEARCH command, it SHOULD reject the command + with a BAD response (due to the conflicting charset labels). + + + + +Resnick & Newman Experimental [Page 4] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +3.2. UTF8 Parameter to SELECT and EXAMINE + + The "UTF8=ACCEPT" capability also indicates that the server supports + the "UTF8" parameter to SELECT and EXAMINE. When a mailbox is + selected with the "UTF8" parameter, it alters the behavior of all + IMAP commands related to message sizes, message headers, and MIME + body headers so they refer to the message with UTF-8 headers. If the + mailstore is not UTF-8 header native and the SELECT or EXAMINE + command with UTF-8 header modifier succeeds, then the server MUST + return results as if the mailstore were UTF-8 header native with + upconversion requirements as described in Section 8. The server MAY + reject the SELECT or EXAMINE command with the [NOT-UTF-8] response + code, unless the "UTF8=ALL" or "UTF8=ONLY" capability is advertised. + + Servers MAY include mailboxes that can only be selected or examined + if the "UTF8" parameter is provided. However, such mailboxes MUST + NOT be included in the output of an unextended LIST, LSUB, or + equivalent command. If a client attempts to SELECT or EXAMINE such + mailboxes without the "UTF8" parameter, the server MUST reject the + command with a [UTF-8-ONLY] response code. As a result, such + mailboxes will not be accessible by IMAP clients written prior to + this specification and are discouraged unless the server advertises + "UTF8=ONLY" or the server implements IMAP4 LIST Command Extensions + [RFC5258]. + + utf8-select-param = "UTF8" + ;; Conforms to from RFC 4466 + + C: a SELECT newmailbox (UTF8) + S: ... + S: a OK SELECT completed + C: b FETCH 1 (SIZE ENVELOPE BODY) + S: ... < UTF-8 header native results > + S: b OK FETCH completed + + C: c EXAMINE legacymailbox (UTF8) + S: c NO [NOT-UTF-8] Mailbox does not support UTF-8 access + + C: d SELECT funky-new-mailbox + S: d NO [UTF-8-ONLY] Mailbox requires UTF-8 client + +3.3. UTF-8 LIST and LSUB Responses + + After an IMAP client successfully issues an "ENABLE UTF8=ACCEPT" + command, the server MUST NOT return in LIST results any mailbox names + to the client following the IMAP4 Mailbox International Naming + Convention. Instead, the server MUST return any mailbox names with + characters outside the US-ASCII repertoire using utf8-quoted syntax. + + + +Resnick & Newman Experimental [Page 5] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + (The IMAP4 Mailbox International Naming Convention has proved + problematic in the past, so the desire is to make this syntax + obsolete as quickly as possible.) + +3.4. UTF-8 Interaction with IMAP4 LIST Command Extensions + + When an IMAP server advertises both the "UTF8=ACCEPT" capability and + the "LIST-EXTENDED" [RFC5258] capability, the server MUST support the + LIST extensions described in this section. + +3.4.1. UTF8 and UTF8ONLY LIST Selection Options + + The "UTF8" LIST selection option tells the server to include + mailboxes that only support UTF-8 headers in the output of the list + command. The "UTF8ONLY" LIST selection option tells the server to + include all mailboxes that support UTF-8 headers and to exclude + mailboxes that don't support UTF-8 headers. Note that "UTF8ONLY" + implies "UTF8", so it is not necessary for the client to request + both. Use of either selection option will also result in UTF-8 + mailbox names in the result as described in Section 3.3 and implies + the "UTF8" List return option described in Section 3.4.2. + +3.4.2. UTF8 LIST Return Option + + If the client supplies the "UTF8" LIST return option, then the server + MUST include either the "\NoUTF8" or the "\UTF8Only" mailbox + attribute as appropriate. The "\NoUTF8" mailbox attribute indicates + that an attempt to SELECT or EXAMINE that mailbox with the "UTF8" + parameter will fail with a [NOT-UTF-8] response code. The + "\UTF8Only" mailbox attribute indicates that an attempt to SELECT or + EXAMINE that mailbox without the "UTF8" parameter will fail with a + [UTF-8-ONLY] response code. Note that computing this information may + be expensive on some server implementations, so this return option + should not be used unless necessary. + + The ABNF [RFC5234] for these LIST extensions follows: + + list-select-independent-opt =/ "UTF8" + + list-select-base-opt =/ "UTF8ONLY" + + mbx-list-oflag =/ "\NoUTF8" / "\UTF8Only" + + return-option =/ "UTF8" + + resp-text-code =/ "NOT-UTF-8" / "UTF-8-ONLY" + + + + + +Resnick & Newman Experimental [Page 6] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +4. UTF8=APPEND Capability + + If the "UTF8=APPEND" capability is advertised, then the server + accepts UTF-8 headers in the APPEND command message argument. A + client that sends a message with UTF-8 headers to the server MUST + send them using the "UTF8" APPEND data extension. If the server also + advertises the CATENATE capability (as specified in [RFC4469]), the + client can use the same data extension to include such a message in a + CATENATE message part. The ABNF for the APPEND data extension and + CATENATE extension follows: + + utf8-literal = "UTF8" SP "(" literal8 ")" + + append-data =/ utf8-literal + + cat-part =/ utf8-literal + + A server that advertises "UTF8=APPEND" has to comply with the + requirements of the IMAP base specification and [RFC5322] for message + fetching. Mechanisms for 7-bit downgrading to help comply with the + standards are discussed in Downgrading mechanism for + Internationalized eMail Address (IMA) [RFC5504]. + + IMAP servers that do not advertise the "UTF8=APPEND" or "UTF8=ONLY" + capability SHOULD reject an APPEND command that includes any 8-bit in + the message headers with a "NO" response. + + Note that the "UTF8=ONLY" capability described in Section 7 implies + the "UTF8=APPEND" capability. See additional information in that + section. + +5. UTF8=USER Capability + + If the "UTF8=USER" capability is advertised, that indicates the + server accepts UTF-8 user names and passwords and applies SASLprep + [RFC4013] to both arguments of the LOGIN command. The server MUST + reject UTF-8 that fails to comply with the formal syntax in RFC 3629 + [RFC3629] or if it encounters Unicode characters listed in Section + 2.3 of SASLprep RFC 4013 [RFC4013]. + +6. UTF8=ALL Capability + + The "UTF8=ALL" capability indicates all server mailboxes support + UTF-8 headers. Specifically, SELECT and EXAMINE with the "UTF8" + parameter will never fail with a [NOT-UTF-8] response code. + + + + + + +Resnick & Newman Experimental [Page 7] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + Note that the "UTF8=ONLY" capability described in Section 7 implies + the "UTF8=ALL" capability. See additional information in that + section. + + Note that the "UTF8=ALL" capability implies the "UTF8=ACCEPT" + capability. + +7. UTF8=ONLY Capability + + The "UTF8=ONLY" capability permits an IMAP server to advertise that + it does not support the international mailbox name convention + (modified UTF-7), and does not permit selection or examination of any + mailbox unless the "UTF8" parameter is provided. As this is an + incompatible change to IMAP, a clear warning is necessary. IMAP + clients that find implementation of the "UTF8=ONLY" capability + problematic are encouraged to at least detect the "UTF8=ONLY" + capability and provide an informative error message to the end-user. + + When an IMAP mailbox internally uses UTF-8 header native storage, the + down-conversion step is necessary to permit selection or examination + of the mailbox in a backwards compatible fashion will become more + difficult to support. Although it is hoped that deployed IMAP + servers will not advertise "UTF8=ONLY" for some years, this + capability is intended to minimize the disruption when legacy support + finally goes away. + + The "UTF8=ONLY" capability implies the "UTF8=ACCEPT" capability, the + "UTF8=ALL" capability, and the "UTF8=APPEND" capability. A server + that advertises "UTF8=ONLY" need not advertise the three implicit + capabilities. + +8. Up-Conversion Server Requirements + + When an IMAP4 server uses a traditional mailbox format that includes + 7-bit headers and it chooses to permit access to that mailbox with + the "UTF8" parameter, it MUST support minimal up-conversion as + described in this section. + + The server MUST support up-conversion of the following address + header-fields in the message header: From, Sender, To, CC, Bcc, + Resent-From, Resent-Sender, Resent-To, Resent-CC, Resent-Bcc, and + Reply-To. This up-conversion MUST include address local-parts in + fields downgraded according to [RFC5504], address domains encoded + according to Internationalizing Domain Names in Applications (IDNA) + [RFC3490], and MIME header encoding [RFC2047] of display-names and + any [RFC5322] comments. + + + + + +Resnick & Newman Experimental [Page 8] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + The following charsets MUST be supported for up-conversion of MIME + header encoding [RFC2047]: UTF-8, US-ASCII, ISO-8859-1, ISO-8859-2, + ISO-8859-3, ISO-8859-4, ISO-8859-5, ISO-8859-6, ISO-8859-7, + ISO-8859-8, ISO-8859-9, ISO-8859-10, ISO-8859-14, and ISO-8859-15. + If the server supports other charsets in IMAP SEARCH or IMAP CONVERT + [RFC5259], it SHOULD also support those charsets in this conversion. + + Up-conversion of MIME header encoding of the following headers MUST + also be implemented: Subject, Date ([RFC5322] comments only), + Comments, Keywords, and Content-Description. + + Server implementations also SHOULD up-convert all MIME body headers + [RFC2045], SHOULD up-convert or remove the deprecated (and misused) + "name" parameter [RFC1341] on Content-Type, and MUST up-convert the + Content-Disposition [RFC2183] "filename" parameter, except when any + of these are contained within a multipart/signed MIME body part (see + below). These parameters can be encoded using the standard MIME + parameter encoding [RFC2231] mechanism, or via non-standard use of + MIME header encoding [RFC2047] in quoted strings. + + The IMAP server MUST NOT perform up-conversion of headers and content + of multipart/signed, as well as Original-Recipient and Return-Path. + +9. Issues with UTF-8 Header Mailstore + + When an IMAP server uses a mailbox format that supports UTF-8 headers + and it permits selection or examination of that mailbox without the + "UTF8" parameter, it is the responsibility of the server to comply + with the IMAP4rev1 base specification [RFC3501] and [RFC5322] with + respect to all header information transmitted over the wire. + Mechanisms for 7-bit downgrading to help comply with the standards + are discussed in "Downgrading Mechanism for Email Address + Internationalization" [RFC5504]. + + An IMAP server with a mailbox that supports UTF-8 headers MUST comply + with the protocol requirements implicit from Section 8. However, the + code necessary for such compliance need not be part of the IMAP + server itself in this case. For example, the minimal required up- + conversion could be performed when a message is inserted into the + IMAP-accessible mailbox. + +10. IANA Considerations + + This adds five new capabilities ("UTF8=ACCEPT", "UTF8=USER", + "UTF8=APPEND", "UTF8=ALL", and "UTF8=ONLY") to the IMAP4rev1 + Capabilities registry [RFC3501]. + + + + + +Resnick & Newman Experimental [Page 9] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + This adds two new IMAP4 list selection options and one new IMAP4 list + return option. + + 1. LIST-EXTENDED option name: UTF8 + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): UTF8 + + LIST-EXTENDED option description: Causes the LIST response to + include mailboxes that mandate the UTF8 SELECT/EXAMINE parameter. + + Published specification: RFC 5738, Section 3.4.1 + + Security considerations: RFC 5738, Section 11 + + Intended usage: COMMON + + Person and email address to contact for further information: see + the Authors' Addresses at the end of this specification + + Owner/Change controller: iesg@ietf.org + + 2. LIST-EXTENDED option name: UTF8ONLY + + LIST-EXTENDED option type: SELECTION + + Implied return options(s): UTF8 + + LIST-EXTENDED option description: Causes the LIST response to + include mailboxes that mandate the UTF8 SELECT/EXAMINE parameter + and exclude mailboxes that do not support the UTF8 SELECT/EXAMINE + parameter. + + Published specification: RFC 5738, Section 3.4.1 + + Security considerations: RFC 5738, Section 11 + + Intended usage: COMMON + + Person and email address to contact for further information: see + the Authors' Addresses at the end of this specification + + Owner/Change controller: iesg@ietf.org + + + + + + + +Resnick & Newman Experimental [Page 10] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + 3. LIST-EXTENDED option name: UTF8 + + LIST-EXTENDED option type: RETURN + + Implied return options(s): none + + LIST-EXTENDED option description: Causes the LIST response to + include \NoUTF8 and \UTF8Only mailbox attributes. + + Published specification: RFC 5738, Section 3.4.1 + + Security considerations: RFC 5738, Section 11 + + Intended usage: COMMON + + Person and email address to contact for further information: see + the Authors' Addresses at the end of this specification + + Owner/Change controller: iesg@ietf.org + +11. Security Considerations + + The security considerations of UTF-8 [RFC3629] and SASLprep [RFC4013] + apply to this specification, particularly with respect to use of + UTF-8 in user names and passwords. Otherwise, this is not believed + to alter the security considerations of IMAP4rev1. + +12. References + +12.1. Normative References + + [RFC1341] Borenstein, N. and N. Freed, "MIME (Multipurpose Internet + Mail Extensions): Mechanisms for Specifying and Describing + the Format of Internet Message Bodies", RFC 1341, + June 1992. + + [RFC2045] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part One: Format of Internet Message + Bodies", RFC 2045, November 1996. + + [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII Text", + RFC 2047, November 1996. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + + + + +Resnick & Newman Experimental [Page 11] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + [RFC2183] Troost, R., Dorner, S., and K. Moore, "Communicating + Presentation Information in Internet Messages: The + Content-Disposition Header Field", RFC 2183, August 1997. + + [RFC2231] Freed, N. and K. Moore, "MIME Parameter Value and Encoded + Word Extensions: + Character Sets, Languages, and Continuations", RFC 2231, + November 1997. + + [RFC3490] Faltstrom, P., Hoffman, P., and A. Costello, + "Internationalizing Domain Names in Applications (IDNA)", + RFC 3490, March 2003. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC4013] Zeilenga, K., "SASLprep: Stringprep Profile for User Names + and Passwords", RFC 4013, February 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC4469] Resnick, P., "Internet Message Access Protocol (IMAP) + CATENATE Extension", RFC 4469, April 2006. + + [RFC5161] Gulbrandsen, A. and A. Melnikov, "The IMAP ENABLE + Extension", RFC 5161, March 2008. + + [RFC5198] Klensin, J. and M. Padlipsky, "Unicode Format for Network + Interchange", RFC 5198, March 2008. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access + Protocol version 4 - LIST Command Extensions", RFC 5258, + June 2008. + + [RFC5259] Melnikov, A. and P. Coates, "Internet Message Access + Protocol - CONVERT Extension", RFC 5259, July 2008. + + [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, + October 2008. + + + + + +Resnick & Newman Experimental [Page 12] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + + [RFC5335] Abel, Y., "Internationalized Email Headers", RFC 5335, + September 2008. + + [RFC5504] Fujiwara, K. and Y. Yoneya, "Downgrading Mechanism for + Email Address Internationalization", RFC 5504, March 2009. + +12.2. Informative References + + [RFC2049] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part Five: Conformance Criteria and + Examples", RFC 2049, November 1996. + + [RFC2088] Myers, J., "IMAP4 non-synchronizing literals", RFC 2088, + January 1997. + + [RFC2277] Alvestrand, H., "IETF Policy on Character Sets and + Languages", BCP 18, RFC 2277, January 1998. + + [RFC5721] Gellens, R. and C. Newman, "POP3 Support for UTF-8", + RFC 5721, February 2010. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Resnick & Newman Experimental [Page 13] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +Appendix A. Design Rationale + + This non-normative section discusses the reasons behind some of the + design choices in the above specification. + + The basic approach of advertising the ability to access a mailbox in + UTF-8 mode is intended to permit graceful upgrade, including servers + that support multiple mailbox formats. In particular, it would be + undesirable to force conversion of an entire server mailstore to + UTF-8 headers, so being able to phase-in support for new mailboxes + and gradually migrate old mailboxes is permitted by this design. + + "UTF8=USER" is optional because many identity systems are US-ASCII + only, so it's helpful to inform the client up front that UTF-8 won't + work. + + "UTF8=APPEND" is optional because it effectively requires IMAP server + support for down-conversion, which is a much more complex operation + than up-conversion. + + The "UTF8=ONLY" mechanism simplifies diagnosis of interoperability + problems when legacy support goes away. In the situation where + backwards compatibility is broken anyway, just-send-UTF-8 IMAP has + the advantage that it might work with some legacy clients. However, + the difficulty of diagnosing interoperability problems caused by a + just-send-UTF-8 IMAP mechanism is the reason the "UTF8=ONLY" + capability mechanism was chosen. + + The up-conversion requirements are designed to balance the desire to + deprecate and eventually eliminate complicated encodings (like MIME + header encodings) without creating a significant deployment burden + for servers. As IMAP4 servers already require a MIME parser, this + includes additional server up-conversion requirements not present in + POP3 Support for UTF-8 [RFC5721]. + + The set of mandatory charsets comes from two sources: MIME + requirements [RFC2049] and IETF Policy on Character Sets [RFC2277]. + Including a requirement to up-convert widely deployed encoded + ideographic charsets to UTF-8 would be reasonable for most scenarios, + but may require unacceptable table sizes for some embedded devices. + The open-ended recommendation to support widely deployed charsets + avoids the political ramifications of attempting to list such + charsets. The authors believe market forces, existing open-source + software, and public conversion tables are sufficient to deploy the + appropriate charsets. + + + + + + +Resnick & Newman Experimental [Page 14] + +RFC 5738 IMAP Support for UTF-8 March 2010 + + +Appendix B. Examples Demonstrating Relationships between UTF8= + Capabilities + + + UTF8=ACCEPT UTF8=USER UTF8=APPEND + UTF8=ACCEPT UTF8=ALL + UTF8=ALL ; Note, same as above + UTF8=ACCEPT UTF8=USER UTF8=APPEND UTF8=ALL UTF8=ONLY + UTF8=USER UTF8=ONLY ; Note, same as above + +Appendix C. Acknowledgments + + The authors wish to thank the participants of the EAI working group + for their contributions to this document with particular thanks to + Harald Alvestrand, David Black, Randall Gellens, Arnt Gulbrandsen, + Kari Hurtta, John Klensin, Xiaodong Lee, Charles Lindsey, Alexey + Melnikov, Subramanian Moonesamy, Shawn Steele, Daniel Taharlev, and + Joseph Yee for their specific contributions to the discussion. + +Authors' Addresses + + Pete Resnick + Qualcomm Incorporated + 5775 Morehouse Drive + San Diego, CA 92121-1714 + US + + Phone: +1 858 651 4478 + EMail: presnick@qualcomm.com + URI: http://www.qualcomm.com/~presnick/ + + Chris Newman + Sun Microsystems + 800 Royal Oaks + Monrovia, CA 91016 + US + + EMail: chris.newman@sun.com + + + + + + + + + + + + + +Resnick & Newman Experimental [Page 15] + diff --git a/docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt b/docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt new file mode 100644 index 0000000..fe0de70 --- /dev/null +++ b/docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt @@ -0,0 +1,619 @@ + + + + + + +Internet Engineering Task Force (IETF) A. Melnikov +Request for Comments: 5788 D. Cridland +Category: Standards Track Isode Limited +ISSN: 2070-1721 March 2010 + + + IMAP4 Keyword Registry + +Abstract + + The aim of this document is to establish a new IANA registry for IMAP + keywords and to define a procedure for keyword registration, in order + to improve interoperability between different IMAP clients. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc5788. + +Copyright Notice + + Copyright (c) 2010 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + + + + + +Melnikov & Cridland Standards Track [Page 1] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + +Table of Contents + + 1. Introduction ....................................................2 + 2. Conventions Used in This Document ...............................2 + 3. IANA Considerations .............................................3 + 3.1. Review Guidelines for the Designated Expert Reviewer .......4 + 3.2. Comments on IMAP Keywords' Registrations ...................5 + 3.3. Change Control .............................................5 + 3.4. Initial Registrations ......................................6 + 3.4.1. $MDNSent IMAP Keyword Registration ..................6 + 3.4.2. $Forwarded IMAP Keyword Registration ................7 + 3.4.3. $SubmitPending IMAP Keyword Registration ............8 + 3.4.4. $Submitted IMAP Keyword Registration ................9 + 4. Security Considerations ........................................10 + 5. Acknowledgements ...............................................10 + 6. References .....................................................10 + 6.1. Normative References ......................................10 + 6.2. Informative References ....................................11 + +1. Introduction + + IMAP keywords [RFC3501] are boolean named flags that can be used by + clients to annotate messages in an IMAP mailbox. Although IMAP + keywords are an optional feature of IMAP, the majority of IMAP + servers can store arbitrary keywords. Many mainstream IMAP clients + use a limited set of specific keywords, and some can manage (create, + edit, display) arbitrary IMAP keywords. + + Over the years, some IMAP keywords have become de-facto standards, + with some specific semantics associated with them. In some cases, + different client implementors decided to define and use keywords with + different names, but the same semantics. Some server implementors + decided to map such keywords automatically in order to improve cross- + client interoperability. + + In other cases, the same keywords have been used with different + semantics, thus causing interoperability problems. + + This document attempts to prevent further incompatible uses of IMAP + keywords by establishing an "IMAP Keywords" registry and allocating a + special prefix for standardized keywords. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [Kwds]. + + + + +Melnikov & Cridland Standards Track [Page 2] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + +3. IANA Considerations + + IANA has established a new registry for IMAP keywords. + + Registration of an IMAP keyword is requested by filling in the + following template and following the instructions on the IANA pages + for the registry to submit it to IANA: + + Subject: Registration of IMAP keyword X + + IMAP keyword name: + + Purpose (description): + + Private or Shared on a server: (One of PRIVATE, SHARED, or BOTH. + PRIVATE means that each different + user has a distinct copy of the + keyword. SHARED means that all + different users see the same value of + the keyword. BOTH means that an IMAP + server can have the keyword as either + private or shared.) + + Is it an advisory keyword or may it cause an automatic action: + + When/by whom the keyword is set/cleared: + + Related keywords: (for example, "mutually exclusive with keywords Y + and Z") + + Related IMAP capabilities: + + Security considerations: + + Published specification (recommended): + + Person & email address to contact for further information: + + Intended usage: (One of COMMON, LIMITED USE, or DEPRECATED (i.e., + not recommended for use)) + + Owner/Change controller: (MUST be "IESG" for any "common use" + keyword registration specified in an IETF + Review document. See definition of "common + use" below in this section. When the + Owner/Change controller is not a + Standardization Organization, the + registration request MUST make it clear if + + + +Melnikov & Cridland Standards Track [Page 3] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + the registration is controlled by a + company, or the individual performing the + registration.) + + Note: (Any other information that the author deems interesting + may be added here, for example, if the keyword(s) is + supported by existing clients.) + + Registration of an IMAP keyword requires Expert Review [RFC5226]. + Registration of any IMAP keyword is initiated by posting a + registration request to the Message Organization WG mailing list + (or its replacement as chosen by the responsible + Application Area Director) and CCing IANA (). After + allowing for at least two weeks for community input on the designated + mailing list, the expert will determine the appropriateness of the + registration request and either approve or disapprove the request + with notice to the requestor, the mailing list, and IANA. Any + refusal must come with a clear explanation. + + The IESG appoints one or more Expert Reviewers for the IMAP keyword + registry established by this document. + + The Expert Reviewer should strive for timely reviews. The Expert + Reviewer should take no longer than six weeks to make and announce + the decision, or notify the mailing list that more time is required. + + Decisions (or lack of) made by an Expert Reviewer can be first + appealed to Application Area Directors and, if the appellant is not + satisfied with the response, to the full IESG. + + There are two types of IMAP keywords in the "IMAP Keywords" registry: + intended for "common use" and vendor-/organization-specific use (also + known as "limited use"). An IMAP keyword is said to be for "common + use" if it is reasonably expected to be implemented in at least two + independent client implementations. The two types of IMAP keywords + have different levels of requirements for registration (see below). + +3.1. Review Guidelines for the Designated Expert Reviewer + + Expert Reviewers should focus on the following requirements. + + Registration of a vendor-/organization-specific ("limited use") IMAP + keyword is easier. The Expert Reviewer only needs to check that the + requested name doesn't conflict with an already registered name, and + that the name is not too generic, misleading, etc. The Expert + Reviewer MAY request the IMAP keyword name change before approving + + + + + +Melnikov & Cridland Standards Track [Page 4] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + the registration. The Expert Reviewer SHOULD refuse a registration + if there is an already registered IMAP keyword that serves the same + purpose, but has a different name. + + When registering an IMAP keyword for "common use", the Expert + Reviewer performs the checks described for vendor-/ + organization-specific IMAP keywords, plus additional checks as + detailed below. + + Keywords intended for "common use" SHOULD start with the "$" prefix. + (Note that this is a SHOULD because some of the commonly used IMAP + keywords in widespread use don't follow this convention.) + + IMAP keywords intended for "common use" SHOULD be standardized in + IETF Review [RFC5226] documents. (Note that IETF Review documents + still require Expert Review.) + + Values in the "IMAP Keywords" IANA registry intended for "common use" + must be clearly documented and likely to ensure interoperability. + They must be useful, not harmful to the Internet. In cases when an + IMAP keyword being registered is already deployed, Expert Reviewers + should favor registering it over requiring perfect documentation + and/or requesting a change to the name of the IMAP keyword. + + The Expert Reviewer MAY automatically "upgrade" registration requests + for a "limited use" IMAP keyword to "common use" level. The Expert + Reviewer MAY also request that a registration targeted for "common + use" be registered as "limited use" instead. + +3.2. Comments on IMAP Keywords' Registrations + + Comments on registered IMAP keywords should be sent to both the + "owner" of the mechanism and to the mailing list designated to IMAP + keyword review (see Section 3). This improves the chances of getting + a timely response. + + Submitters of comments may, after a reasonable attempt to contact the + owner and after soliciting comments on the IMAP mailing list, request + the designated Expert Reviewer to attach their comment to the IMAP + keyword registration itself. The procedure is similar to requesting + an Expert Review for the affected keyword. + +3.3. Change Control + + Once an IMAP keyword registration has been published by IANA, the + owner may request a change to its definition. The change request + (including a change to the "intended usage" field) follows the same + procedure as the initial registration request, with the exception of + + + +Melnikov & Cridland Standards Track [Page 5] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + changes to the "Person & email address to contact for further + information" and "Owner/Change controller" fields. The latter can be + changed by the owner by informing IANA; this can be done without + discussion or review. + + The IESG may reassign responsibility for an IMAP keyword. The most + common case of this will be to enable clarifications to be made to + keywords where the owner of the registration has died, moved out of + contact, or is otherwise unable to make changes that are important to + the community. + + IMAP keyword registrations MUST NOT be deleted; keywords that are no + longer believed appropriate for use can be declared DEPRECATED by a + change to their "intended usage" field. + + The IESG is considered the owner of all "common use" IMAP keywords + that are published in an IETF Review document. + +3.4. Initial Registrations + + IANA has registered the IMAP keywords specified in following + subsections in the registry established by this document. + +3.4.1. $MDNSent IMAP Keyword Registration + + Subject: Registration of IMAP keyword $MDNSent + + + IMAP keyword name: $MDNSent + + Purpose (description): Specifies that a Message Disposition + Notification (MDN) must not be sent for any + message annotated with the $MDNSent IMAP + keyword. + + Private or Shared on a server: SHARED + + Is it an advisory keyword or may it cause an automatic action: + This keyword can cause automatic action by + the client. See [RFC3503] for more details. + + When/by whom the keyword is set/cleared: + This keyword is set by an IMAP client when it + decides to act on an MDN request, or when + uploading a sent or draft message. It can + also be set by a delivery agent. Once set, + the flag SHOULD NOT be cleared. + + + + +Melnikov & Cridland Standards Track [Page 6] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + Related keywords: None + + Related IMAP capabilities: None + + Security considerations: See Section 6 of [RFC3503] + + Published specification (recommended): [RFC3503] + + Person & email address to contact for further information: + Alexey Melnikov + + Intended usage: COMMON + + Owner/Change controller: IESG + + Note: + +3.4.2. $Forwarded IMAP Keyword Registration + + Subject: Registration of the IMAP keyword $Forwarded + + IMAP keyword name: $Forwarded + + Purpose (description): $Forwarded is used by several IMAP clients to + specify that the message was resent to + another email address, embedded within or + attached to a new message. This keyword is + set by the mail client when it successfully + forwards the message to another email + address. Typical usage of this keyword is to + show a different (or additional) icon for a + message that has been forwarded. + + Private or Shared on a server: BOTH + + Is it an advisory keyword or may it cause an automatic action: + advisory + + When/by whom the keyword is set/cleared: + This keyword can be set by either a delivery + agent or a mail client. Once set, the flag + SHOULD NOT be cleared. Notes: There is no + way to tell if a message with $Forwarded + keyword set was forwarded more than once. + + Related keywords: None + + Related IMAP capabilities: None + + + +Melnikov & Cridland Standards Track [Page 7] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + Security considerations: A server implementing this keyword as a + shared keyword may disclose that a + confidential message was forwarded. + + Published specification (recommended): [RFC5550] + + Person & email address to contact for further information: + Alexey Melnikov + + Intended usage: COMMON + + Owner/Change controller: IESG + + Note: + +3.4.3. $SubmitPending IMAP Keyword Registration + + Subject: Registration of IMAP keyword $SubmitPending + + IMAP keyword name: $SubmitPending + + Purpose (description): The $SubmitPending IMAP keyword designates + the message as awaiting to be submitted. + This keyword allows storing messages waiting + to be submitted in the same mailbox where + messages that were already submitted and/or + are being edited are stored. + + A message that has both $Submitted and + $SubmitPending IMAP keywords set is a message + being actively submitted. + + Private or Shared on a server: SHARED + + Is it an advisory keyword or may it cause an automatic action: + This keyword can cause automatic action by + the client. See Section 5.10 of [RFC5550] + for more details. + + When/by whom the keyword is set/cleared: + This keyword is set by a mail client when it + decides that the message needs to be sent + out. + + Related keywords: $Submitted + + Related IMAP capabilities: None + + + + +Melnikov & Cridland Standards Track [Page 8] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + Security considerations: A server implementing this keyword as a + shared keyword may disclose that a + confidential message is scheduled to be + sent out or is being actively sent out. + + Published specification (recommended): [RFC5550] + + Person & email address to contact for further information: + Alexey Melnikov + + Intended usage: COMMON + + Owner/Change controller: IESG + + Note: + +3.4.4. $Submitted IMAP Keyword Registration + + Subject: Registration of IMAP keyword $Submitted + + IMAP keyword name: $Submitted + + Purpose (description): The $Submitted IMAP keyword designates the + message as being sent out. + + A message that has both $Submitted and + $SubmitPending IMAP keywords set is a message + being actively submitted. + + Private or Shared on a server: SHARED + + Is it an advisory keyword or may it cause an automatic action: + This keyword can cause automatic action by + the client. See Section 5.10 of [RFC5550] + for more details. + + When/by whom the keyword is set/cleared: + This keyword is set by a mail client when it + decides to start sending it. + + Related keywords: $SubmitPending + + Related IMAP capabilities: None + + Security considerations: A server implementing this keyword as a + shared keyword may disclose that a + confidential message was sent or is being + actively sent out. + + + +Melnikov & Cridland Standards Track [Page 9] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + Published specification (recommended): [RFC5550] + + Person & email address to contact for further information: + Alexey Melnikov + + Intended usage: COMMON + + Owner/Change controller: IESG + + Note: + +4. Security Considerations + + IMAP keywords are one of the base IMAP features [RFC3501]. This + document doesn't change their behavior, so it does not add new + security issues. + + A particular IMAP keyword might have specific security + considerations, which are documented in the IMAP keyword + registration template standardized by this document. + +5. Acknowledgements + + The creation of this document was prompted by one of many discussions + on the IMAP mailing list. + + John Neystadt co-authored the first version of this document. + + Special thanks to Chris Newman, David Harris, Lyndon Nerenberg, Mark + Crispin, Samuel Weiler, Alfred Hoenes, Lars Eggert, and Cullen + Jennings for reviewing different versions of this document. However, + all errors or omissions must be attributed to the authors of this + document. + + The authors would also like to thank the developers of Mozilla mail + clients for providing food for thought. + +6. References + +6.1. Normative References + + [Kwds] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + + + + +Melnikov & Cridland Standards Track [Page 10] + +RFC 5788 IMAP4 Keyword Registry March 2010 + + + [RFC5226] Narten, T. and H. Alvestrand, "Guidelines for Writing an + IANA Considerations Section in RFCs", BCP 26, RFC 5226, + May 2008. + +6.2. Informative References + + [RFC3503] Melnikov, A., "Message Disposition Notification (MDN) + profile for Internet Message Access Protocol (IMAP)", + RFC 3503, March 2003. + + [RFC5550] Cridland, D., Melnikov, A., and S. Maes, "The Internet + Email to Support Diverse Service Environments (Lemonade) + Profile", RFC 5550, August 2009. + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + Dave Cridland + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: dave.cridland@isode.com + + + + + + + + + + + + + + + + +Melnikov & Cridland Standards Track [Page 11] + diff --git a/docs/rfcs/rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt b/docs/rfcs/rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt new file mode 100644 index 0000000..e45b855 --- /dev/null +++ b/docs/rfcs/rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt @@ -0,0 +1,339 @@ + + + + + + +Internet Engineering Task Force (IETF) A. Melnikov +Request for Comments: 5819 Isode Limited +Category: Standards Track T. Sirainen +ISSN: 2070-1721 Unaffiliated + March 2010 + + + IMAP4 Extension for Returning STATUS Information in Extended LIST + +Abstract + + Many IMAP clients display information about total number of + messages / total number of unseen messages in IMAP mailboxes. In + order to do that, they are forced to issue a LIST or LSUB command and + to list all available mailboxes, followed by a STATUS command for + each mailbox found. This document provides an extension to LIST + command that allows the client to request STATUS information for + mailboxes together with other information typically returned by the + LIST command. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc5819. + +Copyright Notice + + Copyright (c) 2010 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + +Melnikov & Sirainen Standards Track [Page 1] + +RFC 5819 TITLE* March 2010 + + +Table of Contents + + 1. Introduction ....................................................2 + 1.1. Conventions Used in This Document ..........................2 + 2. STATUS Return Option to LIST Command ............................2 + 3. Examples ........................................................3 + 4. Formal Syntax ...................................................4 + 5. Security Considerations .........................................4 + 6. IANA Considerations .............................................4 + 7. Acknowledgements ................................................5 + 8. Normative References ............................................5 + +1. Introduction + + Many IMAP clients display information about the total number of + messages / total number of unseen messages in IMAP mailboxes. In + order to do that, they are forced to issue a LIST or LSUB command and + to list all available mailboxes, followed by a STATUS command for + each mailbox found. This document provides an extension to LIST + command that allows the client to request STATUS information for + mailboxes together with other information typically returned by the + LIST command. + +1.1. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [Kwds]. + +2. STATUS Return Option to LIST Command + + [RFC3501] explicitly disallows mailbox patterns in the STATUS + command. The main reason was to discourage frequent use of the + STATUS command by clients, as it might be quite expensive for an IMAP + server to perform. However, this prohibition had resulted in an + opposite effect: a new generation of IMAP clients appeared, that + issues a STATUS command for each mailbox returned by the LIST + command. This behavior is suboptimal to say at least. It wastes + extra bandwidth and, in the case of a client that doesn't support + IMAP pipelining, also degrades performance by using too many round + trips. This document tries to remedy the situation by specifying a + single command that can be used by the client to request all the + necessary information. In order to achieve this goal, this document + is extending the LIST command with a new return option, STATUS. This + option takes STATUS data items as parameters. For each selectable + + + +Melnikov & Sirainen Standards Track [Page 2] + +RFC 5819 TITLE* March 2010 + + + mailbox matching the list pattern and selection options, the server + MUST return an untagged LIST response followed by an untagged STATUS + response containing the information requested in the STATUS return + option. + + If an attempted STATUS for a listed mailbox fails because the mailbox + can't be selected (e.g., if the "l" ACL right [ACL] is granted to the + mailbox and the "r" right is not granted, or due to a race condition + between LIST and STATUS changing the mailbox to \NoSelect), the + STATUS response MUST NOT be returned and the LIST response MUST + include the \NoSelect attribute. This means the server may have to + buffer the LIST reply until it has successfully looked up the + necessary STATUS information. + + If the server runs into unexpected problems while trying to look up + the STATUS information, it MAY drop the corresponding STATUS reply. + In such a situation, the LIST command would still return a tagged OK + reply. + +3. Examples + + C: A01 LIST "" % RETURN (STATUS (MESSAGES UNSEEN)) + S: * LIST () "." "INBOX" + S: * STATUS "INBOX" (MESSAGES 17 UNSEEN 16) + S: * LIST () "." "foo" + S: * STATUS "foo" (MESSAGES 30 UNSEEN 29) + S: * LIST (\NoSelect) "." "bar" + S: A01 OK List completed. + + The "bar" mailbox isn't selectable, so it has no STATUS reply. + + C: A02 LIST (SUBSCRIBED RECURSIVEMATCH)"" % RETURN (STATUS + (MESSAGES)) + S: * LIST (\Subscribed) "." "INBOX" + S: * STATUS "INBOX" (MESSAGES 17) + S: * LIST () "." "foo" (CHILDINFO ("SUBSCRIBED")) + S: A02 OK List completed. + + The LIST reply for "foo" is returned because it has matching + children, but no STATUS reply is returned because "foo" itself + doesn't match the selection criteria. + + + + + + + + + + +Melnikov & Sirainen Standards Track [Page 3] + +RFC 5819 TITLE* March 2010 + + +4. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. Terms not defined here are taken + from [RFC3501] and [LISTEXT]. + + return-option =/ status-option + + status-option = "STATUS" SP "(" status-att *(SP status-att) ")" + ;; This ABNF production complies with + ;; syntax. + +5. Security Considerations + + This extension makes it a bit easier for clients to overload the + server by requesting STATUS information for a large number of + mailboxes. However, as already noted in the introduction, existing + clients already try to do that by generating a large number of STATUS + commands for each mailbox in which they are interested. While + performing STATUS information retrieval for big lists of mailboxes, a + server implementation needs to make sure that it can still serve + other IMAP connections and yield execution to other connections, when + necessary. + +6. IANA Considerations + + IMAP4 capabilities are registered by publishing a Standards Track or + IESG-approved Experimental RFC. The "IMAP 4 Capabilities" registry + is available from the IANA webiste: + + http://www.iana.org + + This document defines the LIST-STATUS IMAP capability. IANA has + added it to the registry. + + IANA has also added the following new LIST-EXTENDED option to the + IANA registry established by [LISTEXT]: + + To: iana@iana.org + Subject: Registration of LIST-EXTENDED option STATUS + + LIST-EXTENDED option name: STATUS + + LIST-EXTENDED option type: RETURN + + LIST-EXTENDED option description: Causes the LIST command to return + STATUS responses in addition to LIST responses. + + + + +Melnikov & Sirainen Standards Track [Page 4] + +RFC 5819 TITLE* March 2010 + + + Published specification: RFC 5819 + + Security considerations: RFC 5819 + + Intended usage: COMMON + + Person and email address to contact for further information: + Alexey Melnikov + + Owner/Change controller: iesg@ietf.org + +7. Acknowledgements + + Thanks to Philip Van Hoof who pointed out that STATUS and LIST + commands should be combined in order to optimize traffic and number + of round trips. + +8. Normative References + + [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [Kwds] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [LISTEXT] Leiba, B. and A. Melnikov, "Internet Message Access + Protocol version 4 - LIST Command Extensions", RFC 5258, + June 2008. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + + + + + + + + + + + + + + + +Melnikov & Sirainen Standards Track [Page 5] + +RFC 5819 TITLE* March 2010 + + +Authors' Addresses + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + Timo Sirainen + + EMail: tss@iki.fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Melnikov & Sirainen Standards Track [Page 6] + diff --git a/docs/rfcs/rfc5957.IMAP4_SORT_extension.txt b/docs/rfcs/rfc5957.IMAP4_SORT_extension.txt new file mode 100644 index 0000000..32adb49 --- /dev/null +++ b/docs/rfcs/rfc5957.IMAP4_SORT_extension.txt @@ -0,0 +1,283 @@ + + + + + + +Internet Engineering Task Force (IETF) D. Karp +Request for Comments: 5957 Zimbra +Updates: 5256 July 2010 +Category: Standards Track +ISSN: 2070-1721 + + + Display-Based Address Sorting for the IMAP4 SORT Extension + +Abstract + + This document describes an IMAP protocol extension enabling server- + side message sorting on the commonly displayed portion of the From + and To header fields. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc5957. + +Copyright Notice + + Copyright (c) 2010 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + + + + +Karp IMAP4 Display-Based Address Sorting [Page 1] + +RFC 5957 July 2010 + + +Table of Contents + + 1. Introduction ....................................................2 + 2. Conventions Used in This Document ...............................2 + 3. DISPLAY Sort Value for an Address ...............................2 + 4. The DISPLAYFROM and DISPLAYTO Sort Criteria .....................3 + 5. Formal Syntax ...................................................3 + 6. Security Considerations .........................................3 + 7. Internationalization Considerations .............................4 + 8. IANA Considerations .............................................4 + 9. Normative References ............................................4 + +1. Introduction + + The [SORT] extension to the [IMAP] protocol provides a means for the + server-based sorting of messages. It defines a set of sort criteria + and the mechanism for determining the sort value of a message for + each such ordering. + + The [SORT] FROM and TO orderings sort messages lexically on the + [IMAP] addr-mailbox of the first address in the message's From and To + headers, respectively. This document provides two alternative + orderings, DISPLAYFROM and DISPLAYTO, which sort messages based on + the first From or To address's [IMAP] addr-name (generally the same + as its [RFC5322] display-name), when present. + + A server that supports the full [SORT] extension as well as both the + DISPLAYFROM and DISPLAYTO sort criteria indicates this by returning + "SORT=DISPLAY" in its CAPABILITY response. + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. + +3. DISPLAY Sort Value for an Address + + For the purposes of the sort criteria defined in this document, the + sort value for an [IMAP] address structure is defined as follows: + + o If the address structure's [IMAP] addr-name is non-NIL, apply the + procedure from [RFC5255], Section 4.6. (That is, decode any + [RFC2047] encoded-words and convert the resulting character string + into a charset valid for the currently active [RFC4790] collation, + with a default of UTF-8.) If the resulting octet string is not + the empty string, use it as the sort value for the address. + + + + +Karp IMAP4 Display-Based Address Sorting [Page 2] + +RFC 5957 July 2010 + + + o Otherwise, if the address structure's [IMAP] addr-mailbox and + [IMAP] addr-host are both non-NIL, the sort value for the address + is addr-mailbox@addr-host. + + o Otherwise, if the address structure's [IMAP] addr-mailbox is non- + NIL, the sort value for the address is its addr-mailbox. + + o If none of the above conditions are met, the sort value for the + address is the empty string. + +4. The DISPLAYFROM and DISPLAYTO Sort Criteria + + This document introduces two new [SORT] sort criteria, DISPLAYFROM + and DISPLAYTO. A message's sort value under these orderings MUST be + derived as follows: + + A "derived-addr" value is created from the [IMAP] envelope structure + resulting from a FETCH ENVELOPE on the message. For DISPLAYFROM, the + derived-addr value is the [IMAP] env-from value. For DISPLAYTO, the + derived-addr value is the [IMAP] env-to value. + + o If the derived-addr value is NIL, the message's sort value is the + empty string. + + o Otherwise, the message's sort value is the DISPLAY sort value of + the first [IMAP] address in the derived-addr value. + +5. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) notation as specified in [RFC5234]. [IMAP] defines the + non-terminal "capability", and [SORT] defines "sort-key". + + capability =/ "SORT=DISPLAY" + + sort-key =/ "DISPLAYFROM" / "DISPLAYTO" + +6. Security Considerations + + This document defines an additional IMAP4 capability. As such, it + does not change the underlying security considerations of [IMAP]. + The author believes that no new security issues are introduced with + this additional IMAP4 capability. + + + + + + + + +Karp IMAP4 Display-Based Address Sorting [Page 3] + +RFC 5957 July 2010 + + +7. Internationalization Considerations + + DISPLAYFROM and DISPLAYTO are string-based sort criteria. As stated + in [SORT], the active [RFC4790] collation as per [RFC5255] MUST be + used when sorting such strings. + + The DISPLAYFROM and DISPLAYTO orderings sort on the full decoded + [IMAP] addr-name, when present. They do not attempt to parse this + string in a locale- or language-dependent manner in order to + determine and sort on some semantically meaningful substring such as + the surname. + +8. IANA Considerations + + [IMAP] capabilities are registered by publishing a Standards Track or + IESG-approved Experimental RFC. This document constitutes + registration of the SORT=DISPLAY capability in the [IMAP] + capabilities registry. + +9. Normative References + + [IMAP] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) + Part Three: Message Header Extensions for Non-ASCII Text", + RFC 2047, November 1996. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC4790] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet + Application Protocol Collation Registry", RFC 4790, March + 2007. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, January + 2008. + + [RFC5255] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet + Message Access Protocol Internationalization", RFC 5255, + June 2008. + + [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, + October 2008. + + + + + + +Karp IMAP4 Display-Based Address Sorting [Page 4] + +RFC 5957 July 2010 + + + [SORT] Crispin, M. and K. Murchison, "Internet Message Access + Protocol - SORT and THREAD Extensions", RFC 5256, June + 2008. + +Author's Address + + Dan Karp + Zimbra + 3401 Hillview Avenue + Palo Alto, CA 94304 + USA + + EMail: dkarp@zimbra.com + URI: http://www.zimbra.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Karp IMAP4 Display-Based Address Sorting [Page 5] + diff --git a/docs/rfcs/rfc6154.IMAP_LIST_Special-use_Mailboxes.txt b/docs/rfcs/rfc6154.IMAP_LIST_Special-use_Mailboxes.txt new file mode 100644 index 0000000..14d3e76 --- /dev/null +++ b/docs/rfcs/rfc6154.IMAP_LIST_Special-use_Mailboxes.txt @@ -0,0 +1,675 @@ + + + + + + +Internet Engineering Task Force (IETF) B. Leiba +Request for Comments: 6154 Huawei Technologies +Category: Standards Track J. Nicolson +ISSN: 2070-1721 Google + March 2011 + + + IMAP LIST Extension for Special-Use Mailboxes + +Abstract + + Some IMAP message stores include special-use mailboxes, such as those + used to hold draft messages or sent messages. Many mail clients + allow users to specify where draft or sent messages should be put, + but configuring them requires that the user know which mailboxes the + server has set aside for these purposes. This extension adds new + optional mailbox attributes that a server may include in IMAP LIST + command responses to identify special-use mailboxes to the client, + easing configuration. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc6154. + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + +Leiba & Nicolson Standards Track [Page 1] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 1.1. Conventions Used in This Document . . . . . . . . . . . . 3 + 2. New Mailbox Attributes Identifying Special-Use Mailboxes . . . 3 + 3. Extension to IMAP CREATE Command to Set Special-Use + Attributes . . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 4. IMAP METADATA Entry for Special-Use Attributes . . . . . . . . 6 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 7 + 5.1. Example of an IMAP LIST Command . . . . . . . . . . . . . 7 + 5.2. Example of an Extended IMAP LIST Command . . . . . . . . . 7 + 5.3. Example of an IMAP CREATE Command . . . . . . . . . . . . 8 + 5.4. Example of Using IMAP METADATA to Manipulate + Special-Use Attributes . . . . . . . . . . . . . . . . . . 8 + 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 9 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 9 + 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 10 + 8.1. Registration of USEATTR IMAP Response Code . . . . . . . . 10 + 8.2. Registration of CREATE-SPECIAL-USE IMAP Capability . . . . 10 + 8.3. Registration of SPECIAL-USE IMAP Capability . . . . . . . 10 + 8.4. Registration of SPECIAL-USE Selection Option . . . . . . . 10 + 8.5. Registration of SPECIAL-USE Return Option . . . . . . . . 11 + 8.6. Registration of SPECIAL-USE Metadata . . . . . . . . . . . 11 + 9. References . . . . . . . . . . . . . . . . . . . . . . . . . . 12 + 9.1. Normative References . . . . . . . . . . . . . . . . . . . 12 + 9.2. Informative References . . . . . . . . . . . . . . . . . . 12 + + + + + + + + + + + + + + + + + + + + + + + + + +Leiba & Nicolson Standards Track [Page 2] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +1. Introduction + + Some IMAP message stores include special-use mailboxes, such as those + used to hold draft messages or sent messages. Many mail clients + allow users to specify where draft or sent messages should be put, + but configuring them requires that the user know which mailboxes the + server has set aside for these purposes. This extension adds new + optional mailbox attributes that a server may include in IMAP LIST + command responses to identify special-use mailboxes to the client, + easing configuration. + + In addition, this extension adds an optional parameter on the IMAP + CREATE command, allowing a client to assign a special use to a + mailbox when it is created. Servers may choose to support this part + of the extension, but are not required to. + +1.1. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + +2. New Mailbox Attributes Identifying Special-Use Mailboxes + + An IMAP server that supports this extension MAY include any or all of + the following attributes in responses to the non-extended IMAP LIST + command. The new attributes are included along with existing + attributes, such as "\Marked" and "\Noselect". A given mailbox may + have none, one, or more than one of these attributes. In some cases, + a special use is advice to a client about what to put in that + mailbox. In other cases, it's advice to a client about what to + expect to find there. There is no capability string related to the + support of special-use attributes on the non-extended LIST command. + + For the extended list command [RFC5258], this extension adds a new + capability string, a new selection option, and a new return option, + all called "SPECIAL-USE". Supporting implementations MUST include + the "SPECIAL-USE" capability string in response to an IMAP CAPABILITY + command. If the client specifies the "SPECIAL-USE" selection option, + the LIST command MUST return only those mailboxes that have a + special-use attribute set. If the client specifies the "SPECIAL-USE" + return option, the LIST command MUST return the new special-use + attributes on those mailboxes that have them set. The "SPECIAL-USE" + + + + + +Leiba & Nicolson Standards Track [Page 3] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + return option is implied by the "SPECIAL-USE" selection option. The + extended LIST command MAY return SPECIAL-USE attributes even if the + client does not specify the return option. + + The new attributes defined here are as follows: + + \All + This mailbox presents all messages in the user's message store. + Implementations MAY omit some messages, such as, perhaps, those + in \Trash and \Junk. When this special use is supported, it is + almost certain to represent a virtual mailbox. + + \Archive + This mailbox is used to archive messages. The meaning of an + "archival" mailbox is server-dependent; typically, it will be + used to get messages out of the inbox, or otherwise keep them + out of the user's way, while still making them accessible. + + \Drafts + This mailbox is used to hold draft messages -- typically, + messages that are being composed but have not yet been sent. In + some server implementations, this might be a virtual mailbox, + containing messages from other mailboxes that are marked with + the "\Draft" message flag. Alternatively, this might just be + advice that a client put drafts here. + + \Flagged + This mailbox presents all messages marked in some way as + "important". When this special use is supported, it is likely + to represent a virtual mailbox collecting messages (from other + mailboxes) that are marked with the "\Flagged" message flag. + + \Junk + This mailbox is where messages deemed to be junk mail are held. + Some server implementations might put messages here + automatically. Alternatively, this might just be advice to a + client-side spam filter. + + \Sent + This mailbox is used to hold copies of messages that have been + sent. Some server implementations might put messages here + automatically. Alternatively, this might just be advice that a + client save sent messages here. + + \Trash + This mailbox is used to hold messages that have been deleted or + marked for deletion. In some server implementations, this might + be a virtual mailbox, containing messages from other mailboxes + + + +Leiba & Nicolson Standards Track [Page 4] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + that are marked with the "\Deleted" message flag. + Alternatively, this might just be advice that a client that + chooses not to use the IMAP "\Deleted" model should use this as + its trash location. In server implementations that strictly + expect the IMAP "\Deleted" model, this special use is likely not + to be supported. + + All of the above attributes are OPTIONAL, and any given server or + message store may support any combination of the attributes, or none + at all. In most cases, there will likely be at most one mailbox with + a given attribute for a given user, but in some server or message + store implementations it might be possible for multiple mailboxes to + have the same special-use attribute. + + Special-use attributes are likely to be user-specific. User Adam + might share his \Sent mailbox with user Barb, but that mailbox is + unlikely to also serve as Barb's \Sent mailbox. It's certainly + possible for Adam and Barb to each set the \Sent use on the same + mailbox, but that would be done by specific action (see the sections + below). + +3. Extension to IMAP CREATE Command to Set Special-Use Attributes + + As an OPTIONAL feature, a server MAY allow clients to designate a + mailbox, at creation, as having one or more special uses. This + extension defines the "USE" parameter to the IMAP CREATE command for + that purpose (using the syntax defined in RFC 4466 section 2.2 + [RFC4466]). The new OPTIONAL "USE" parameter is followed by a + parenthesized list of zero or more special-use attributes, as defined + above. + + In some server implementations, some special uses may imply automatic + action by the server. For example, creation of a "\Junk" mailbox + might cause the server to start placing messages that have been + evaluated as spam into the mailbox. + + In some server implementations, some special uses may result in a + mailbox with unusual characteristics or side effects. For example, + creation of an "\All" mailbox might cause the server to create a + virtual mailbox, rather than a standard one, and that mailbox might + behave in unexpected ways (COPY into it might fail, for example). + + Servers MAY allow the creation of a special-use mailbox even if one + so designated already exists. This might have the effect of moving + the special use from the old mailbox to the new one, or might create + multiple mailboxes with the same special use. Alternatively, servers + MAY refuse the creation, considering the designation to be a + conflict. + + + +Leiba & Nicolson Standards Track [Page 5] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + If the server cannot create a mailbox with the designated special use + defined, for whatever reason, it MUST NOT create the mailbox, and + MUST respond to the CREATE command with a tagged NO response. If the + reason for the failure is related to the special-use attribute (the + specified special use is not supported or cannot be assigned to the + specified mailbox), the server SHOULD include the new "USEATTR" + response code in the tagged response (see Section 5.3 for an + example). + + An IMAP server that supports this OPTIONAL feature will advertise the + "CREATE-SPECIAL-USE" capability string. Clients MUST NOT use the + "USE" parameter unless the server advertises the capability. Note + that this capability string is different from the "SPECIAL-USE" + string defined above, and a server that supports both functions MUST + advertise both capability strings. + +4. IMAP METADATA Entry for Special-Use Attributes + + If a server supports this extension and the METADATA extension + [RFC5464], it SHOULD tie the special-use attributes for a mailbox to + its metadata entry "/private/specialuse". The value of /private/ + specialuse is either NIL (if there are no special-use attributes for + that mailbox) or a space-separated list of special-use attributes, + presented the same way they would be presented in the LIST command + response. + + Such a server MAY allow the setting of special-use attributes through + the METADATA mechanisms, thereby allowing clients to change the + special uses of existing mailboxes. These changes might have side + effects, as the server automatically adjusts the special uses + accordingly, just as it might do with CREATE USE, above. See + Section 5.4 for an example. + + A server that supports this MUST check the validity of changes to the + special-use attributes that are done through the metadata in the same + way that it checks validity for the CREATE command and for any + internal mechanisms for setting special uses on mailboxes. It MUST + NOT just blindly accept setting of these metadata by clients, which + might result in the setting of special uses that the implementation + does not support, multiple mailboxes with the same special use, or + other situations that the implementation considers invalid. + + + + + + + + + + +Leiba & Nicolson Standards Track [Page 6] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +5. Examples + +5.1. Example of an IMAP LIST Command + + This example shows an IMAP LIST response from a server that supports + this extension. Note that not all of the attributes are used. This + server also supports the Child Mailbox extension [RFC3348]. + + C: t1 LIST "" "%" + S: * LIST (\Marked \HasNoChildren) "/" Inbox + S: * LIST (\HasNoChildren) "/" ToDo + S: * LIST (\HasChildren) "/" Projects + S: * LIST (\Sent \HasNoChildren) "/" SentMail + S: * LIST (\Marked \Drafts \HasNoChildren) "/" MyDrafts + S: * LIST (\Trash \HasNoChildren) "/" Trash + S: t1 OK done + +5.2. Example of an Extended IMAP LIST Command + + This example shows an IMAP LIST response from a server that supports + this extension. The client uses the extended IMAP LIST command. + + C: t1 CAPABILITY + S: * CAPABILITY IMAP4rev1 SPECIAL-USE + S: t1 OK done + + C: t2 LIST "" "%" RETURN (SPECIAL-USE) + S: * LIST (\Marked) "/" Inbox + S: * LIST () "/" ToDo + S: * LIST () "/" Projects + S: * LIST (\Sent) "/" SentMail + S: * LIST (\Marked \Drafts) "/" MyDrafts + S: * LIST (\Trash) "/" Trash + S: t2 OK done + + Here, the client also includes the "SPECIAL-USE" selection option for + the same list. The "SPECIAL-USE" return option could also have been + specified, but it is unnecessary, as it is implied by the selection + option. Note that in this case, mailboxes that do not have a + special-use attribute are not listed. Also note that we've used the + wildcard "*", rather than "%", to make sure we see all special-use + mailboxes, even ones that might not be at the namespace's root. + + C: t3 LIST (SPECIAL-USE) "" "*" + S: * LIST (\Sent) "/" SentMail + S: * LIST (\Marked \Drafts) "/" MyDrafts + S: * LIST (\Trash) "/" Trash + S: t3 OK done + + + +Leiba & Nicolson Standards Track [Page 7] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +5.3. Example of an IMAP CREATE Command + + This example shows an IMAP CREATE command that might be used to + create a mailbox designated to hold draft and sent messages. It also + attempts to create a mailbox that will contain all the user's + messages, but the server does not support that special use for this + user's message store. + + C: t1 CAPABILITY + S: * CAPABILITY IMAP4rev1 CREATE-SPECIAL-USE + S: t1 OK done + + C: t2 CREATE MySpecial (USE (\Drafts \Sent)) + S: t2 OK MySpecial created + + C: t3 CREATE Everything (USE (\All)) + S: t3 NO [USEATTR] \All not supported + +5.4. Example of Using IMAP METADATA to Manipulate Special-Use + Attributes + + This example shows how IMAP METADATA can be used to manipulate + special-use attributes, if the operation is supported on the server. + + ==> Starting point: + C: t1 LIST "" "%" RETURN (SPECIAL-USE) + S: * LIST (\Sent) "/" SentMail + S: * LIST (\Drafts) "/" MyDrafts + S: * LIST () "/" SavedDrafts + S: * LIST (\Trash) "/" Trash + S: t1 OK done + + ==> Demonstrate the connection: + C: t2 GETMETADATA "MyDrafts" /private/specialuse + S: * METADATA "MyDrafts" (/private/specialuse "\\Drafts") + S: t2 OK done + + ==> Set new use for SavedDrafts; MyDrafts changes automatically: + C: t3 SETMETADATA "SavedDrafts" (/private/specialuse "\\Drafts") + S: * METADATA "MyDrafts" (/private/specialuse NIL) + S: t3 OK SETMETADATA complete + + ==> Remove special use for SentMail: + C: t4 SETMETADATA "SentMail" (/private/specialuse NIL) + S: t4 OK SETMETADATA complete + + + + + + +Leiba & Nicolson Standards Track [Page 8] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + ==> Check the results: + C: t5 LIST "" "%" RETURN (SPECIAL-USE) + S: * LIST () "/" SentMail + S: * LIST () "/" MyDrafts + S: * LIST (\Drafts) "/" SavedDrafts + S: * LIST (\Trash) "/" Trash + S: t5 OK done + +6. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [RFC5234]. + + create-param =/ "USE" SP "(" [use-attr *(SP use-attr)] ")" + ; Extends "create-param" from RFC 4466 [RFC4466] + + mbx-list-oflag =/ use-attr + ; Extends "mbx-list-oflag" from IMAP base [RFC3501] + + list-select-independent-opt =/ "SPECIAL-USE" + ; Extends "list-select-independent-opt" from + ; LIST-extended [RFC5258] + + return-option =/ "SPECIAL-USE" + ; Extends "return-option" from + ; LIST-extended [RFC5258] + + resp-text-code =/ "USEATTR" + ; Extends "resp-text-code" from + ; IMAP [RFC3501] + + use-attr = "\All" / "\Archive" / "\Drafts" / "\Flagged" / + "\Junk" / "\Sent" / "\Trash" / use-attr-ext + + use-attr-ext = "\" atom + ; Reserved for future extensions. Clients + ; MUST ignore list attributes they do not understand + ; Server implementations MUST NOT generate + ; extension attributes except as defined by + ; future Standards-Track revisions of or + ; extensions to this specification. + +7. Security Considerations + + LIST response: + Conveying special-use information to a client exposes a small bit of + extra information that could be of value to an attacker. Knowing, + for example, that a particular mailbox (\All) contains pointers to + + + +Leiba & Nicolson Standards Track [Page 9] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + every message the user has might be of particular value. If the IMAP + channel is not protected from passive eavesdropping, this could be an + issue. + + CREATE command "USE" parameter and metadata extension: In some server + implementations, some special uses may imply automatic action by the + server. For example, creation of a "\Junk" mailbox might cause the + server to start placing messages that have been evaluated as spam + into the mailbox. Implementors SHOULD consider the consequences of + allowing a user (or client program) to designate the target of such + automatic action. + + Example: If a user is allowed to give the "\Junk" attribute to a + shared mailbox, legitimate mail that's misclassified as junk (false + positives) will be put into that shared mailbox, exposing the user's + private mail to others. The server might warn a user of that + possibility, or might refuse to allow the specification to be made on + a shared mailbox. (Note that this problem exists independent of this + specification, if the server allows a user to share a mailbox that's + already in use for such a function.) + +8. IANA Considerations + +8.1. Registration of USEATTR IMAP Response Code + + This document defines a new IMAP response code, "USEATTR", which IANA + added to the IMAP Response Codes registry. + +8.2. Registration of CREATE-SPECIAL-USE IMAP Capability + + This document defines a new IMAP capability, "CREATE-SPECIAL-USE", + which IANA added to the IMAP 4 Capabilities registry. + +8.3. Registration of SPECIAL-USE IMAP Capability + + This document defines a new IMAP capability, "SPECIAL-USE", which + IANA added to the IMAP 4 Capabilities registry. + +8.4. Registration of SPECIAL-USE Selection Option + + This document defines a new IMAP4 List Extended selection option, + "SPECIAL-USE", which IANA added to the IMAP4 List Extended registry, + as follows: + + To: iana@iana.org + Subject: Registration of LIST-EXTENDED selection option SPECIAL-USE + LIST-EXTENDED option name: SPECIAL-USE + LIST-EXTENDED option type: SELECTION + + + +Leiba & Nicolson Standards Track [Page 10] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + + Implied return option(s): SPECIAL-USE + LIST-EXTENDED option description: Limit the list to special-use + mailboxes only + Published specification: RFC 6154 + Security considerations: none + Intended usage: COMMON + Person and email address to contact for further information: Authors' + Addresses at the end of RFC 6154 + Owner/Change controller: iesg@ietf.org + +8.5. Registration of SPECIAL-USE Return Option + + This document defines a new IMAP4 List Extended return option, + "SPECIAL-USE", which IANA added to the IMAP4 List Extended registry, + as follows: + + To: iana@iana.org + Subject: Registration of LIST-EXTENDED return option SPECIAL-USE + LIST-EXTENDED option name: SPECIAL-USE + LIST-EXTENDED option type: RETURN + LIST-EXTENDED option description: Request special-use mailbox + information + Published specification: RFC 6154 + Security considerations: none + Intended usage: COMMON + Person and email address to contact for further information: Authors' + Addresses at the end of RFC 6154 + Owner/Change controller: iesg@ietf.org + +8.6. Registration of SPECIAL-USE Metadata + + This document defines a new IMAP METADATA entry. IANA added the + following to the IMAP METADATA Mailbox Entry registry: + + To: iana@iana.org + Subject: IMAP METADATA Entry Registration + Type: Mailbox + Name: /private/specialuse + Description: Defines any special-use features of a mailbox. See the + reference specification for details of its use. + Content-type: text/plain; charset=us-ascii + RFC Number: RFC 6154 + Contact: MORG mailing list mailto:morg@ietf.org + + + + + + + + +Leiba & Nicolson Standards Track [Page 11] + +RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 + + +9. References + +9.1. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access + Protocol version 4 - LIST Command Extensions", RFC 5258, + June 2008. + + [RFC5464] Daboo, C., "The IMAP METADATA Extension", RFC 5464, + February 2009. + +9.2. Informative References + + [RFC3348] Gahrns, M. and R. Cheng, "The Internet Message Action + Protocol (IMAP4) Child Mailbox Extension", RFC 3348, + July 2002. + +Authors' Addresses + + Barry Leiba + Huawei Technologies + + Phone: +1 646 827 0648 + EMail: barryleiba@computer.org + URI: http://internetmessagingtechnology.org/ + + + Jamie Nicolson + Google + + EMail: nicolson@google.com + + + + + + + + +Leiba & Nicolson Standards Track [Page 12] + diff --git a/docs/rfcs/rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt b/docs/rfcs/rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt new file mode 100644 index 0000000..e4ed001 --- /dev/null +++ b/docs/rfcs/rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt @@ -0,0 +1,395 @@ + + + + + + +Internet Engineering Task Force (IETF) T. Sirainen +Request for Comments: 6203 March 2011 +Category: Standards Track +ISSN: 2070-1721 + + + IMAP4 Extension for Fuzzy Search + +Abstract + + This document describes an IMAP protocol extension enabling a server + to perform searches with inexact matching and assigning relevancy + scores for matched messages. + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc6203. + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + + + + + +Sirainen Standards Track [Page 1] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + +1. Introduction + + When humans perform searches in IMAP clients, they typically want to + see the most relevant search results first. IMAP servers are able to + do this in the most efficient way when they're free to internally + decide how searches should match messages. This document describes a + new SEARCH=FUZZY extension that provides such functionality. + +2. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [KEYWORDS]. + +3. The FUZZY Search Key + + The FUZZY search key takes another search key as its argument. The + server is allowed to perform all matching in an implementation- + defined manner for this search key, including ignoring the active + comparator as defined by [RFC5255]. Typically, this would be used to + search for strings. For example: + + C: A1 SEARCH FUZZY (SUBJECT "IMAP break") + S: * SEARCH 1 5 10 + S: A1 OK Search completed. + + Besides matching messages with a subject of "IMAP break", the above + search may also match messages with subjects "broken IMAP", "IMAP is + broken", or anything else the server decides that might be a good + match. + + This example does a fuzzy SUBJECT search, but a non-fuzzy FROM + search: + + C: A2 SEARCH FUZZY SUBJECT work FROM user@example.com + S: * SEARCH 1 4 + S: A2 OK Search completed. + + How the server handles multiple separate FUZZY search keys is + implementation-defined. + + Fuzzy search algorithms might change, or the results of the + algorithms might be different from search to search, so that fuzzy + searches with the same parameters might give different results for + 1) the same user at different times, 2) different users (searches + + + +Sirainen Standards Track [Page 2] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + executed simultaneously), or 3) different users (searches executed at + different times). For example, a fuzzy search might adapt to a + user's search habits in an attempt to give more relevant results (in + a "learning" manner). Such differences can also occur because of + operational decisions, such as load balancing. Clients asking for + "fuzzy" really are requesting search results in a not-necessarily- + deterministic way and need to give the user appropriate warning about + that. + +4. Relevancy Scores for Search Results + + Servers SHOULD assign a search relevancy score for each matched + message when the FUZZY search key is given. Relevancy scores are + given in the range 1-100, where 100 is the highest relevancy. The + relevancy scores SHOULD use the full 1-100 range, so that clients can + show them to users in a meaningful way, e.g., as a percentage value. + + As the name already indicates, relevancy scores specify how relevant + to the search the matched message is. It's not necessarily the same + as how precisely the message matched. For example, a message whose + subject fuzzily matches the search string might get a higher + relevancy score than a message whose body had the exact string in the + middle of a sentence. When multiple search keys are matched fuzzily, + how the relevancy score is calculated is server-dependent. + + If the server also advertises the ESEARCH capability as defined by + [ESEARCH], the relevancy scores can be retrieved using the new + RELEVANCY return option for SEARCH: + + C: B1 SEARCH RETURN (RELEVANCY ALL) FUZZY TEXT "Helo" + S: * ESEARCH (TAG "B1") ALL 1,5,10 RELEVANCY (4 99 42) + S: B1 OK Search completed. + + In the example above, the server would treat "hello", "help", and + other similar strings as fuzzily matching the misspelled "Helo". + + The RELEVANCY return option MUST NOT be used unless a FUZZY search + key is also given. Note that SEARCH results aren't sorted by + relevancy; SORT is needed for that. + +5. Fuzzy Matching with Non-String Search Keys + + Fuzzy matching is not limited to just string matching. All search + keys SHOULD be matched fuzzily, although exactly what that means for + different search keys is left for server implementations to decide -- + including deciding that fuzzy matching is meaningless for a + particular key, and falling back to exact matching. Some suggestions + are given below. + + + +Sirainen Standards Track [Page 3] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + Dates: + A typical example could be when a user wants to find a message + "from Dave about a week ago". A client could perform this search + using SEARCH FUZZY (FROM "Dave" SINCE 21-Jan-2009 BEFORE + 24-Jan-2009). The server could return messages outside the + specified date range, but the further away the message is, the + lower the relevancy score. + + Sizes: + These should be handled similarly to dates. If a user wants to + search for "about 1 MB attachments", the client could do this by + sending SEARCH FUZZY (LARGER 900000 SMALLER 1100000). Again, the + further away the message size is from the specified range, the + lower the relevancy score. + + Flags: + If other search criteria match, the server could return messages + that don't have the specified flags set, but with lower relevancy + scores. SEARCH SUBJECT "xyz" FUZZY ANSWERED, for example, might + be useful if the user thinks the message he is looking for has the + ANSWERED flag set, but he isn't sure. + + Unique Identifiers (UIDs), sequences, modification sequences: These + are examples of keys for which exact matching probably makes sense. + Alternatively, a server might choose, for instance, to expand a UID + range by 5% on each side. + +6. Extensions to SORT and SEARCH + + If the server also advertises the SORT capability as defined by + [SORT], the results can be sorted by the new RELEVANCY sort criteria: + + C: C1 SORT (RELEVANCY) UTF-8 FUZZY SUBJECT "Helo" + S: * SORT 5 10 1 + S: C1 OK Sort completed. + + The message with the highest score is returned first. As with the + RELEVANCY return option, RELEVANCY sort criteria MUST NOT be used + unless a FUZZY search key is also given. + + If the server also advertises the ESORT capability as defined by + [CONTEXT], the relevancy scores can be retrieved using the new + RELEVANCY return option for SORT: + + C: C2 SORT RETURN (RELEVANCY ALL) (RELEVANCY) UTF-8 FUZZY TEXT + "Helo" + S: * ESEARCH (TAG "C2") ALL 5,10,1 RELEVANCY (99 42 4) + S: C2 OK Sort completed. + + + +Sirainen Standards Track [Page 4] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + Furthermore, if the server advertises the CONTEXT=SORT (or + CONTEXT=SEARCH) capability, then the client can limit the number of + returned messages to a SORT (or a SEARCH) by using the PARTIAL return + option. For example, this returns the 10 most relevant messages: + + C: C3 SORT RETURN (PARTIAL 1:10) (RELEVANCY) UTF-8 FUZZY TEXT + "World" + S: * ESEARCH (TAG "C3") PARTIAL (1:10 42,9,34,13,15,4,2,7,23,82) + S: C3 OK Sort completed. + +7. Formal Syntax + + The following syntax specification uses the augmented Backus-Naur + Form (BNF) as described in [ABNF]. It includes definitions from + [RFC3501], [IMAP-ABNF], and [SORT]. + + capability =/ "SEARCH=FUZZY" + + score = 1*3DIGIT + ;; (1 <= n <= 100) + + score-list = "(" [score *(SP score)] ")" + + search-key =/ "FUZZY" SP search-key + + search-return-data =/ "RELEVANCY" SP score-list + ;; Conforms to , from [IMAP-ABNF] + + search-return-opt =/ "RELEVANCY" + ;; Conforms to , from [IMAP-ABNF] + + sort-key =/ "RELEVANCY" + +8. Security Considerations + + Implementation of this extension might enable denial-of-service + attacks against server resources. Servers MAY limit the resources + that a single search (or a single user) may use. Additionally, + implementors should be aware of the following: Fuzzy search engines + are often complex with non-obvious disk space, memory, and/or CPU + usage patterns. Server implementors should at least test the fuzzy- + search behavior with large messages that contain very long words + and/or unique random strings. Also, very long search keys might + cause excessive memory or CPU usage. + + Invalid input may also be problematic. For example, if the search + engine takes a UTF-8 stream as input, it might fail more or less + badly when illegal UTF-8 sequences are fed to it from a message whose + + + +Sirainen Standards Track [Page 5] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + character set was claimed to be UTF-8. This could be avoided by + validating all the input and, for example, replacing illegal UTF-8 + sequences with the Unicode replacement character (U+FFFD). + + Search relevancy rankings might be susceptible to "poisoning" by + smart attackers using certain keywords or hidden markup (e.g., HTML) + in their messages to boost the rankings. This can't be fully + prevented by servers, so clients should prepare for it by at least + allowing users to see all the search results, rather than hiding + results below a certain score. + +9. IANA Considerations + + IMAP4 capabilities are registered by publishing a standards track or + IESG-approved experimental RFC. The "Internet Message Access + Protocol (IMAP) 4 Capabilities Registry" is available from + http://www.iana.org/. + + This document defines the SEARCH=FUZZY IMAP capability. IANA has + added it to the registry. + +10. Acknowledgements + + Alexey Melnikov, Zoltan Ordogh, Barry Leiba, Cyrus Daboo, and Dave + Cridland have helped with this document. + +11. Normative References + + [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, + January 2008. + + [CONTEXT] Cridland, D. and C. King, "Contexts for IMAP4", + RFC 5267, July 2008. + + [ESEARCH] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH + Command for Controlling What Kind of Information Is + Returned", RFC 4731, November 2006. + + [IMAP-ABNF] Melnikov, A. and C. Daboo, "Collected Extensions to + IMAP4 ABNF", RFC 4466, April 2006. + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + + + +Sirainen Standards Track [Page 6] + +RFC 6203 IMAP4 FUZZY Search March 2011 + + + [RFC5255] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet + Message Access Protocol Internationalization", RFC 5255, + June 2008. + + [SORT] Crispin, M. and K. Murchison, "Internet Message Access + Protocol - SORT and THREAD Extensions", RFC 5256, + June 2008. + +Author's Address + + Timo Sirainen + + EMail: tss@iki.fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Sirainen Standards Track [Page 7] + diff --git a/docs/rfcs/rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt b/docs/rfcs/rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt new file mode 100644 index 0000000..b5a3a61 --- /dev/null +++ b/docs/rfcs/rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt @@ -0,0 +1,563 @@ + + + + + + +Internet Engineering Task Force (IETF) B. Leiba +Request for Comments: 6237 Huawei Technologies +Updates: 4466 A. Melnikov +Category: Experimental Isode Limited +ISSN: 2070-1721 May 2011 + + + IMAP4 Multimailbox SEARCH Extension + +Abstract + + The IMAP4 specification allows the searching of only the selected + mailbox. A user often wants to search multiple mailboxes, and a + client that wishes to support this must issue a series of SELECT and + SEARCH commands, waiting for each to complete before moving on to the + next. This extension allows a client to search multiple mailboxes + with one command, limiting the round trips and waiting for various + searches to complete, and not requiring disruption of the currently + selected mailbox. This extension also uses MAILBOX and TAG fields in + ESEARCH responses, allowing a client to pipeline the searches if it + chooses. This document updates RFC 4466. + +Status of This Memo + + This document is not an Internet Standards Track specification; it is + published for examination, experimental implementation, and + evaluation. + + This document defines an Experimental Protocol for the Internet + community. This document is a product of the Internet Engineering + Task Force (IETF). It represents the consensus of the IETF + community. It has received public review and has been approved for + publication by the Internet Engineering Steering Group (IESG). Not + all documents approved by the IESG are a candidate for any level of + Internet Standard; see Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc6237. + + + + + + + + + + + + +Leiba & Melnikov Experimental [Page 1] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + +Table of Contents + + 1. Introduction ....................................................2 + 1.1. Conventions Used in This Document ..........................3 + 2. New ESEARCH Command .............................................3 + 2.1. The ESEARCH Response .......................................4 + 2.2. Source Options: Specifying Mailboxes to Search .............5 + 3. Examples ........................................................6 + 4. Formal Syntax ...................................................7 + 5. Security Considerations .........................................8 + 6. IANA Considerations .............................................9 + 7. Acknowledgements ................................................9 + 8. Normative References ............................................9 + +1. Introduction + + The IMAP4 specification allows the searching of only the selected + mailbox. A user often wants to search multiple mailboxes, and a + client that wishes to support this must issue a series of SELECT and + SEARCH commands, waiting for each to complete before moving on to the + next. The commands can't be pipelined, because the server might run + them in parallel, and the untagged SEARCH responses could not then be + distinguished from each other. + + This extension allows a client to search multiple mailboxes with one + command, and includes MAILBOX and TAG fields in the ESEARCH response, + yielding the following advantages: + + o A single command limits the number of round trips needed to search + a set of mailboxes. + + o A single command eliminates the need to wait for one search to + complete before starting the next. + + + +Leiba & Melnikov Experimental [Page 2] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + o A single command allows the server to optimize the search, if it + can. + + o A command that is not dependent upon the selected mailbox + eliminates the need to disrupt the selection state or to open + another IMAP connection. + + o The MAILBOX, UIDVALIDITY, and TAG fields in the responses allow a + client to distinguish which responses go with which search (and + which mailbox). A client can safely pipeline these search + commands without danger of confusion. The addition of the MAILBOX + and UIDVALIDITY fields updates the search-correlator item defined + in [RFC4466]. + +1.1. Conventions Used in This Document + + In examples, "C:" indicates lines sent by a client that is connected + to a server. "S:" indicates lines sent by the server to the client. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + +2. New ESEARCH Command + + Arguments: OPTIONAL source options + OPTIONAL result options + OPTIONAL charset specification (see [RFC2978]) + searching criteria (one or more) + + Responses: REQUIRED untagged response: ESEARCH + + Result: OK -- search completed + NO -- error: cannot search that charset or criteria + BAD -- command unknown or arguments invalid + + This section defines a new ESEARCH command, which works similarly to + the UID SEARCH command described in Section 2.6.1 of [RFC4466] + (initially described in Section 6.4.4 of [RFC3501] and extended by + [RFC4731]). + + The ESEARCH command further extends searching by allowing for + optional source and result options. This document does not define + any new result options (see Section 3.1 of [RFC4731]). A server that + supports this extension includes "MULTISEARCH" in its IMAP capability + string. + + + + + +Leiba & Melnikov Experimental [Page 3] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + Because there has been confusion about this, it is worth pointing out + that with ESEARCH, as with *any* SEARCH or UID SEARCH command, it + MUST NOT be considered an error if the search terms include a range + of message numbers that extends (or, in fact, starts) beyond the end + of the mailbox. For example, a client might want to establish a + rolling window through the search results this way: + + C: tag1 UID ESEARCH FROM "frobozz" 1:100 + + ...followed later by this: + + C: tag1 UID ESEARCH FROM "frobozz" 101:200 + + ...and so on. This tells the server to match only the first hundred + messages in the mailbox the first time, the second hundred the second + time, etc. In fact, it might likely allow the server to optimize the + search significantly. In the above example, whether the mailbox + contains 50 or 150 or 250 messages, neither of the search commands + shown will result in an error. It is up to the client to know when + to stop moving its search window. + +2.1. The ESEARCH Response + + In response to an ESEARCH command, the server MUST return ESEARCH + responses [RFC4731] (that is, not SEARCH responses). Because message + numbers are not useful for mailboxes that are not selected, the + responses MUST contain information about UIDs, not message numbers. + This is true even if the source options specify that only the + selected mailbox be searched. + + Presence of a source option in the absence of a result option implies + the "ALL" result option (see Section 3.1 of [RFC4731]). Note that + this is not the same as the result from the SEARCH command described + in the IMAP base protocol [RFC3501]. + + Source options describe which mailboxes must be searched for + messages. An ESEARCH command with source options does not affect + which mailbox, if any, is currently selected, regardless of which + mailboxes are searched. + + For each mailbox satisfying the source options, a single ESEARCH + response MUST be returned if any messages in that mailbox match the + search criteria. An ESEARCH response MUST NOT be returned for + mailboxes that contain no matching messages. This is true even when + result options such as MIN, MAX, and COUNT are specified (see + Section 3.1 of [RFC4731]), and the values returned (lowest UID + matched, highest UID matched, and number of messages matched, + respectively) apply to the mailbox reported in that ESEARCH response. + + + +Leiba & Melnikov Experimental [Page 4] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + Note that it is possible for an ESEARCH command to return *no* + untagged responses (no ESEARCH responses at all), in the case that + there are no matches to the search in any of the mailboxes that + satisfy the source options. Clients can detect this situation by + finding the tagged OK response without having received any matching + untagged ESEARCH responses. + + Each ESEARCH response MUST contain the MAILBOX, TAG, and UIDVALIDITY + correlators. Correlators allow clients to issue several ESEARCH + commands at once (pipelined). If the SEARCHRES [RFC5182] extension + is used in an ESEARCH command, that ESEARCH command MUST be executed + by the server after all previous SEARCH/ESEARCH commands have + completed and before any subsequent SEARCH/ESEARCH commands are + executed. The server MAY perform consecutive ESEARCH commands in + parallel as long as none of them use the SEARCHRES extension. + +2.2. Source Options: Specifying Mailboxes to Search + + The source options, if present, MUST contain a mailbox specifier as + defined in the IMAP NOTIFY extension [RFC5465], Section 6 (using the + "filter-mailboxes" ABNF item), with the following differences: + + 1. The "selected-delayed" specifier is not valid here. + + 2. A "subtree-one" specifier is added. The "subtree" specifier + results in a search of the specified mailbox and all selectable + mailboxes that are subordinate to it, through an indefinitely + deep hierarchy. The "subtree-one" specifier results in a search + of the specified mailbox and all selectable child mailboxes, one + hierarchy level down. + + If "subtree" is specified, the server MUST defend against loops in + the hierarchy (for example, those caused by recursive file-system + links within the message store). The server SHOULD do this by + keeping track of the mailboxes that have been searched, and + terminating the hierarchy traversal when a repeat is found. If it + cannot do that, it MAY do it by limiting the hierarchy depth. + + If the source options are not present, the value "selected" is + assumed -- that is, only the currently selected mailbox is searched. + + The "personal" source option is a particularly convenient way to + search all of the current user's mailboxes. Note that there is no + way to use wildcard characters to search all mailboxes; the + "mailboxes" source option does not do wildcard expansion. + + + + + + +Leiba & Melnikov Experimental [Page 5] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + If the source options include (or default to) "selected", the IMAP + session MUST be in "selected" state. If the source options specify + other mailboxes and NOT "selected", then the IMAP session MUST be in + either "selected" or "authenticated" state. If the session is not in + a correct state, the ESEARCH command MUST return a "BAD" result. + + If the server supports the SEARCHRES [RFC5182] extension, then the + "SAVE" result option is valid *only* if "selected" is specified or + defaulted as the sole mailbox to be searched. If any source option + other than "selected" is specified, the ESEARCH command MUST return a + "BAD" result. + + If the server supports the CONTEXT=SEARCH and/or CONTEXT=SORT + extension [RFC5267], then the following additional rules apply: + + o The CONTEXT return option (Section 4.2 of [RFC5267]) can be used + with an ESEARCH command. + + o If the UPDATE return option is used (Section 4.3 of [RFC5267]), it + MUST apply ONLY to the currently selected mailbox. If UPDATE is + used and there is no mailbox currently selected, the ESEARCH + command MUST return a "BAD" result. + + o The PARTIAL search return option (Section 4.4 of [RFC5267]) can be + used and applies to each mailbox searched by the ESEARCH command. + + If the server supports the Access Control List (ACL) [RFC4314] + extension, then the logged-in user is required to have the "r" right + for each mailbox she wants to search. In addition, any mailboxes + that are not explicitly named (accessed through "personal" or + "subtree", for example) are required to have the "l" right. + Mailboxes matching the source options for which the logged-in user + lacks sufficient rights MUST be ignored by the ESEARCH command + processing. In particular, ESEARCH responses MUST NOT be returned + for those mailboxes. + +3. Examples + + In the following example, note that two ESEARCH commands are + pipelined, and that the server is running them in parallel, + interleaving a response to the second search amid the responses to + the first (watch the tags). + + C: tag1 ESEARCH IN (mailboxes "folder1" subtree "folder2") unseen + C: tag2 ESEARCH IN (mailboxes "folder1" subtree-one "folder2") + subject "chad" + S: * ESEARCH (TAG "tag1" MAILBOX "folder1" UIDVALIDITY 1) UID ALL + 4001,4003,4005,4007,4009 + + + +Leiba & Melnikov Experimental [Page 6] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + S: * ESEARCH (TAG "tag2" MAILBOX "folder1" UIDVALIDITY 1) UID ALL + 3001:3004,3788 + S: * ESEARCH (TAG "tag1" MAILBOX "folder2/banana" UIDVALIDITY 503) + UID ALL 3002,4004 + S: * ESEARCH (TAG "tag1" MAILBOX "folder2/peach" UIDVALIDITY 3) UID + ALL 921691 + S: tag1 OK done + S: * ESEARCH (TAG "tag2" MAILBOX "folder2/salmon" UIDVALIDITY + 1111111) UID ALL 50003,50006,50009,50012 + S: tag2 OK done + +4. Formal Syntax + + The following syntax specification uses the Augmented Backus-Naur + Form (ABNF) as described in [RFC5234]. Terms not defined here are + taken from [RFC3501], [RFC5465], or [RFC4466]. + + command-auth =/ esearch + ; Update definition from IMAP base [RFC3501]. + ; Add new "esearch" command. + + command-select =/ esearch + ; Update definition from IMAP base [RFC3501]. + ; Add new "esearch" command. + + filter-mailboxes-other =/ ("subtree-one" SP one-or-more-mailbox) + ; Update definition from IMAP Notify [RFC5465]. + ; Add new "subtree-one" selector. + + filter-mailboxes-selected = "selected" + ; Update definition from IMAP Notify [RFC5465]. + ; We forbid the use of "selected-delayed". + + one-correlator = ("TAG" SP tag-string) / ("MAILBOX" SP astring) / + ("UIDVALIDITY" SP nz-number) + ; Each correlator MUST appear exactly once. + + scope-option = scope-option-name [SP scope-option-value] + ; No options defined here. Syntax for future extensions. + + scope-option-name = tagged-ext-label + ; No options defined here. Syntax for future extensions. + + scope-option-value = tagged-ext-val + ; No options defined here. Syntax for future extensions. + + + + + + +Leiba & Melnikov Experimental [Page 7] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + scope-options = scope-option *(SP scope-option) + ; A given option may only appear once. + ; No options defined here. Syntax for future extensions. + + esearch = "ESEARCH" [SP esearch-source-opts] + [SP search-return-opts] SP search-program + + search-correlator = SP "(" one-correlator *(SP one-correlator) ")" + ; Updates definition in IMAP4 ABNF [RFC4466]. + + esearch-source-opts = "IN" SP "(" source-mbox [SP + "(" scope-options ")"] ")" + + source-mbox = filter-mailboxes *(SP filter-mailboxes) + ; "filter-mailboxes" is defined in IMAP Notify [RFC5465]. + ; See updated definition of filter-mailboxes-other, above. + ; See updated definition of filter-mailboxes-selected, above. + +5. Security Considerations + + This new IMAP ESEARCH command allows a single command to search many + mailboxes at once. On the one hand, a client could do that by + sending many IMAP SEARCH commands. On the other hand, this makes it + easier for a client to overwork a server, by sending a single command + that results in an expensive search of tens of thousands of + mailboxes. Server implementations need to be aware of that, and + provide mechanisms that prevent a client from adversely affecting + other users. Limitations on the number of mailboxes that may be + searched in one command, and/or on the server resources that will be + devoted to responding to a single client, are reasonable limitations + for an implementation to impose. + + Implementations MUST, of course, apply access controls appropriately, + limiting a user's access to ESEARCH in the same way its access is + limited for any other IMAP commands. This extension has no data- + access risks beyond what may be there in the unextended IMAP + implementation. + + Mailboxes matching the source options for which the logged-in user + lacks sufficient rights MUST be ignored by the ESEARCH command + processing (see the paragraph about this in Section 2.2). In + particular, any attempt to distinguish insufficient access from + non-existent mailboxes may expose information about the mailbox + hierarchy that isn't otherwise available to the client. + + If "subtree" is specified, the server MUST defend against loops in + the hierarchy (see the paragraph about this in Section 2.2). + + + + +Leiba & Melnikov Experimental [Page 8] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + +6. IANA Considerations + + IMAP4 capabilities are registered by publishing a Standards Track or + IESG-approved Experimental RFC. The "IMAP 4 Capabilities" registry + is currently located here: + + http://www.iana.org/ + + This document defines the IMAP capability "MULTISEARCH", and IANA has + added it to the registry. + +7. Acknowledgements + + The authors gratefully acknowledge feedback provided by Timo + Sirainen, Peter Coates, and Arnt Gulbrandsen. + +8. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC2978] Freed, N. and J. Postel, "IANA Charset Registration + Procedures", BCP 19, RFC 2978, October 2000. + + [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION + 4rev1", RFC 3501, March 2003. + + [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", + RFC 4314, December 2005. + + [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 + ABNF", RFC 4466, April 2006. + + [RFC4731] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH + Command for Controlling What Kind of Information Is + Returned", RFC 4731, November 2006. + + [RFC5182] Melnikov, A., "IMAP Extension for Referencing the Last + SEARCH Result", RFC 5182, March 2008. + + [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for + Syntax Specifications: ABNF", STD 68, RFC 5234, + January 2008. + + [RFC5267] Cridland, D. and C. King, "Contexts for IMAP4", RFC 5267, + July 2008. + + + + + +Leiba & Melnikov Experimental [Page 9] + +RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 + + + [RFC5465] Gulbrandsen, A., King, C., and A. Melnikov, "The IMAP + NOTIFY Extension", RFC 5465, February 2009. + +Authors' Addresses + + Barry Leiba + Huawei Technologies + + Phone: +1 646 827 0648 + EMail: barryleiba@computer.org + URI: http://internetmessagingtechnology.org/ + + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Leiba & Melnikov Experimental [Page 10] + diff --git a/docs/rfcs/rfc6331.Moving_Digest-MD5_to_Historic b/docs/rfcs/rfc6331.Moving_Digest-MD5_to_Historic new file mode 100644 index 0000000..3d4172f --- /dev/null +++ b/docs/rfcs/rfc6331.Moving_Digest-MD5_to_Historic @@ -0,0 +1,339 @@ + + + + + + +Internet Engineering Task Force (IETF) A. Melnikov +Request for Comments: 6331 Isode Limited +Obsoletes: 2831 July 2011 +Category: Informational +ISSN: 2070-1721 + + + Moving DIGEST-MD5 to Historic + +Abstract + + This memo describes problems with the DIGEST-MD5 Simple + Authentication and Security Layer (SASL) mechanism as specified in + RFC 2831. It marks DIGEST-MD5 as OBSOLETE in the IANA Registry of + SASL mechanisms and moves RFC 2831 to Historic status. + +Status of This Memo + + This document is not an Internet Standards Track specification; it is + published for informational purposes. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Not all documents + approved by the IESG are a candidate for any level of Internet + Standard; see Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc6331. + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + +Melnikov Informational [Page 1] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + + This document may contain material from IETF Documents or IETF + Contributions published or made publicly available before November + 10, 2008. The person(s) controlling the copyright in some of this + material may not have granted the IETF Trust the right to allow + modifications of such material outside the IETF Standards Process. + Without obtaining an adequate license from the person(s) controlling + the copyright in such materials, this document may not be modified + outside the IETF Standards Process, and derivative works of it may + not be created outside the IETF Standards Process, except to format + it for publication as an RFC or to translate it into languages other + than English. + +Table of Contents + + 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 2 + 2. Security Considerations . . . . . . . . . . . . . . . . . . . 5 + 3. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 5 + 4. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . 5 + 5. References . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 5.1. Normative References . . . . . . . . . . . . . . . . . . . . 5 + 5.2. Informative References . . . . . . . . . . . . . . . . . . . 5 + +1. Introduction and Overview + + [RFC2831] defines how HTTP Digest Authentication [RFC2617] can be + used as a Simple Authentication and Security Layer (SASL) [RFC4422] + mechanism for any protocol that has a SASL profile. It was intended + both as an improvement over CRAM-MD5 [RFC2195] and as a convenient + way to support a single authentication mechanism for web, email, the + Lightweight Directory Access Protocol (LDAP), and other protocols. + While it can be argued that it is an improvement over CRAM-MD5, many + implementors commented that the additional complexity of DIGEST-MD5 + makes it difficult to implement fully and securely. + + Below is an incomplete list of problems with the DIGEST-MD5 mechanism + as specified in [RFC2831]: + + 1. The mechanism has too many options and modes. Some of them are + not well described and are not widely implemented. For example, + DIGEST-MD5 allows the "qop" directive to contain multiple values, + but it also allows for multiple qop directives to be specified. + The handling of multiple options is not specified, which results + in minor interoperability problems. Some implementations + amalgamate multiple qop values into one, while others treat + multiple qops as an error. Another example is the use of an + empty authorization identity. In SASL, an empty authorization + identity means that the client is willing to authorize as the + authentication identity. The document is not clear on whether + + + +Melnikov Informational [Page 2] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + + the authzid must be omitted or if it can be specified with an + empty value to convey this. The requirement for backward + compatibility with HTTP Digest means that the situation is even + worse. For example, DIGEST-MD5 requires all usernames/passwords + that can be entirely represented in the ISO-8859-1 charset to be + down converted from UTF-8 [RFC3629] to ISO-8859-1 [ISO-8859-1]. + Another example is the use of quoted strings. Handling of + characters that need escaping is not properly described, and the + DIGEST-MD5 document has no examples to demonstrate correct + behavior. + + 2. The DIGEST-MD5 document uses ABNF from RFC 822 [RFC0822], which + allows an extra construct and allows for "implied folding + whitespace" to be inserted in many places. The difference from a + more common ABNF defined in [RFC5234] is confusing for some + implementors. As a result, many implementations do not accept + folding whitespace in many places where it is allowed. + + 3. The DIGEST-MD5 document uses the concept of a "realm" to define a + collection of accounts. A DIGEST-MD5 server can support one or + more realms. The DIGEST-MD5 document does not provide any + guidance on how realms should be named and, more importantly, how + they can be entered in User Interfaces (UIs). As a result, many + DIGEST-MD5 clients have confusing UIs, do not allow users to + enter a realm, and/or do not allow users to pick one of the + server-supported realms. + + 4. Use of username in the inner hash is problematic. The inner hash + of DIGEST-MD5 is an MD5 hash of colon-separated username, realm, + and password. Implementations may choose to store inner hashes + instead of clear text passwords. This has some useful + properties, such as protection from compromise of authentication + databases containing the same username and password on other + servers if a server with the username and password is + compromised; however, this is rarely done in practice. First, + the inner hash is not compatible with widely deployed Unix + password databases, and second, changing the username would + invalidate the inner hash. + + 5. Description of DES/3DES [DES] and RC4 security layers are + inadequate to produce independently developed interoperable + implementations. In the DES/3DES case, this is partly a problem + with existing DES APIs. + + 6. DIGEST-MD5 outer hash (the value of the "response" directive) + does not protect the whole authentication exchange, which makes + the mechanism vulnerable to "man-in-the-middle" (MITM) attacks, + such as modification of the list of supported qops or ciphers. + + + +Melnikov Informational [Page 3] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + + 7. The following features are missing from DIGEST-MD5, making it + insecure or unsuitable for use in protocols: + + A. Channel bindings [RFC5056]. + + B. Hash agility (i.e., no easy way to replace the MD5 hash + function with another one). + + C. Support for SASLPrep [RFC4013] or any other type of Unicode + character normalization of usernames and passwords. The + original DIGEST-MD5 document predates SASLPrep and does not + recommend any Unicode character normalization. + + 8. The cryptographic primitives in DIGEST-MD5 are not up to today's + standards, in particular: + + A. The MD5 hash is sufficiently weak to make a brute force + attack on DIGEST-MD5 easy with common hardware [RFC6151]. + + B. The RC4 algorithm is prone to attack when used as the + security layer without discarding the initial key stream + output [RFC6229]. + + C. The DES cipher for the security layer is considered insecure + due to its small key space [RFC3766]. + + Note that most of the problems listed above are already present in + the HTTP Digest authentication mechanism. + + Because DIGEST-MD5 is defined as an extensible mechanism, it is + possible to fix most of the problems listed above. However, this + would increase implementation complexity of an already complex + mechanism even further, so the effort is not worth the cost. In + addition, an implementation of a "fixed" DIGEST-MD5 specification + would likely either not interoperate with any existing implementation + of [RFC2831] or would be vulnerable to various downgrade attacks. + + Note that despite DIGEST-MD5 seeing some deployment on the Internet, + this specification recommends obsoleting DIGEST-MD5 because DIGEST- + MD5, as implemented, is not a reasonable candidate for further + standardization and should be deprecated in favor of one or more new + password-based mechanisms currently being designed. + + The Salted Challenge Response Authentication Mechanism (SCRAM) family + of SASL mechanisms [RFC5802] has been developed to provide similar + features as DIGEST-MD5 but with a better design. + + + + + +Melnikov Informational [Page 4] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + +2. Security Considerations + + Security issues are discussed throughout this document. + +3. IANA Considerations + + IANA has changed the "Intended usage" of the DIGEST-MD5 mechanism + registration in the SASL mechanism registry to OBSOLETE. The SASL + mechanism registry is specified in [RFC4422] and is currently + available at: + + http://www.iana.org/assignments/sasl-mechanisms + +4. Acknowledgements + + The author gratefully acknowledges the feedback provided by Chris + Newman, Simon Josefsson, Kurt Zeilenga, Sean Turner, and Abhijit + Menon-Sen. Various text was copied from other RFCs, in particular, + from [RFC2831]. + +5. References + +5.1. Normative References + + [RFC2617] Franks, J., Hallam-Baker, P., Hostetler, J., Lawrence, + S., Leach, P., Luotonen, A., and L. Stewart, "HTTP + Authentication: Basic and Digest Access + Authentication", RFC 2617, June 1999. + + [RFC2831] Leach, P. and C. Newman, "Using Digest Authentication + as a SASL Mechanism", RFC 2831, May 2000. + +5.2. Informative References + + [DES] National Institute of Standards and Technology, "Data + Encryption Standard (DES)", FIPS PUB 46-3, + October 1999. + + [ISO-8859-1] International Organization for Standardization, + "Information technology - 8-bit single-byte coded + graphic character sets - Part 1: Latin alphabet No. 1", + ISO/IEC 8859-1, 1998. + + [RFC0822] Crocker, D., "Standard for the format of ARPA Internet + text messages", STD 11, RFC 822, August 1982. + + + + + + +Melnikov Informational [Page 5] + +RFC 6331 Moving DIGEST-MD5 to Historic July 2011 + + + [RFC2195] Klensin, J., Catoe, R., and P. Krumviede, "IMAP/POP + AUTHorize Extension for Simple Challenge/Response", + RFC 2195, September 1997. + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC3766] Orman, H. and P. Hoffman, "Determining Strengths For + Public Keys Used For Exchanging Symmetric Keys", + BCP 86, RFC 3766, April 2004. + + [RFC4013] Zeilenga, K., "SASLprep: Stringprep Profile for User + Names and Passwords", RFC 4013, February 2005. + + [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication + and Security Layer (SASL)", RFC 4422, June 2006. + + [RFC5056] Williams, N., "On the Use of Channel Bindings to Secure + Channels", RFC 5056, November 2007. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [RFC5802] Newman, C., Menon-Sen, A., Melnikov, A., and N. + Williams, "Salted Challenge Response Authentication + Mechanism (SCRAM) SASL and GSS-API Mechanisms", + RFC 5802, July 2010. + + [RFC6151] Turner, S. and L. Chen, "Updated Security + Considerations for the MD5 Message-Digest and the HMAC- + MD5 Algorithms", RFC 6151, March 2011. + + [RFC6229] Strombergson, J. and S. Josefsson, "Test Vectors for + the Stream Cipher RC4", RFC 6229, May 2011. + +Author's Address + + Alexey Melnikov + Isode Limited + 5 Castle Business Village + 36 Station Road + Hampton, Middlesex TW12 2BX + UK + + EMail: Alexey.Melnikov@isode.com + URI: http://www.melnikov.ca/ + + + + + +Melnikov Informational [Page 6] + diff --git a/docs/website-doc.sh b/docs/website-doc.sh new file mode 100755 index 0000000..0565c71 --- /dev/null +++ b/docs/website-doc.sh @@ -0,0 +1,119 @@ +#!/bin/sh +# +# vim: expandtab ts=2 : + +ARGS=$* + +SPHINXBUILD=sphinx-build +TMPDIR='/tmp/offlineimap-sphinx-doctrees' +WEBSITE='./website' +DOCBASE="${WEBSITE}/_doc" +DESTBASE="${DOCBASE}/versions" +VERSIONS_YML="${WEBSITE}/_data/versions.yml" +ANNOUNCES_YML="${WEBSITE}/_data/announces.yml" +CONTRIB_YML="${WEBSITE}/_data/contribs.yml" +CONTRIB="${DOCBASE}/contrib" +HEADER="# DO NOT EDIT MANUALLY: it is generated by a script (website-doc.sh)." + + +function fix_pwd () { + cd "$(git rev-parse --show-toplevel)" || \ + exit 2 "cannot determine the root of the repository" + test -d "$DESTBASE" || exit 1 +} + +fix_pwd +version="v$(./offlineimap.py --version)" + + + +# +# Add the doc for the contrib files. +# +function contrib () { + echo $HEADER > "$CONTRIB_YML" + # systemd + cp -afv "./contrib/systemd/README.md" "${CONTRIB}/systemd.md" + echo "- {filename: 'systemd', linkname: 'Integrate with systemd'}" >> "$CONTRIB_YML" +} + + + +# +# Build the sphinx documentation. +# +function api () { + # Build the doc with sphinx. + dest="${DESTBASE}/${version}" + echo "Cleaning target directory: $dest" + rm -rf "$dest" + $SPHINXBUILD -b html -d "$TMPDIR" ./docs/doc-src "$dest" + + # Build the JSON definitions for Jekyll. + # This let know the website about the available APIs documentations. + echo "Building Jekyll data: $VERSIONS_YML" + # Erase previous content. + echo "$HEADER" > "$VERSIONS_YML" + for version in $(ls "$DESTBASE" -1 | sort -nr) + do + echo "- $version" + done >> "$VERSIONS_YML" +} + + + +# +# Make Changelog public and save links to them as JSON. +# +function releases () { + # Copy the Changelogs. + for foo in ./Changelog.md ./Changelog.maint.md + do + cp -afv "$foo" "$DOCBASE" + done + + # Build the announces JSON list. Format is JSON: + # - {version: '', link: ''} + # - ... + echo "$HEADER" > "$ANNOUNCES_YML" + grep -E '^### OfflineIMAP' ./Changelog.md | while read title + do + link="$(echo $title | sed -r -e 's,^### (OfflineIMAP.*)\),\1,' \ + | tr '[:upper:]' '[:lower:]' \ + | sed -r -e 's,[\.("],,g' \ + | sed -r -e 's, ,-,g' + )" + v="$(echo $title \ + | sed -r -e 's,^### [a-Z]+ (v[^ ]+).*,\1,' + )" + echo "- {version: '${v}', link: '$link'}" + done | tee -a "$ANNOUNCES_YML" +} + + +exit_code=0 +test "n$ARGS" = 'n' && ARG='usage' # no option passed +for arg in $ARGS +do + # PWD was fixed at the very beginning. + case "n$arg" in + "nreleases") + releases + ;; + "napi") + api + ;; + "ncontrib") + contrib + ;; + "nusage") + echo "Usage: website-doc.sh " + ;; + *) + echo "unkown option $arg" + exit_code=$(( $exit_code + 1 )) + ;; + esac +done + +exit $exit_code diff --git a/offlineimap.conf b/offlineimap.conf index d3be8be..69ede92 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -1,116 +1,171 @@ -# Sample configuration file -# Copyright (C) 2002-2005 John Goerzen -# -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Offlineimap sample configuration file +# This file documents *all* possible options and can be quite scary. # Looking for a quick start? Take a look at offlineimap.conf.minimal. +# More details can be found at http://offlineimap.org . +################################################## +# Overview +################################################## + +# The default configuration file is "~/.offlineimaprc". +# +# OfflineIMAP ships with a file named "offlineimap.conf" that you should copy to +# that location and then edit. +# +# OfflineIMAP also ships a file named "offlineimap.conf.minimal" that you can +# also try. It's useful if you want to get started with the most basic feature +# set, and you can read about other features later with "offlineimap.conf". +# +# If you want to be XDG-compatible, you can put your configuration file into +# "$XDG_CONFIG_HOME/offlineimap/config". ################################################## # General definitions ################################################## +# NOTE 1: Settings generally support python interpolation. This means +# values can contain python format strings which refer to other values +# in the same section, or values in a special DEFAULT section. This +# allows you for example to use common settings for multiple accounts: +# +# [Repository Gmail1] +# trashfolder: %(gmailtrashfolder)s +# +# [Repository Gmail2] +# trashfolder: %(gmailtrashfolder)s +# +# [DEFAULT] +# gmailtrashfolder = [Gmail]/Papierkorb +# +# would set the trashfolder setting for your German Gmail accounts. + +# NOTE 2: Above feature implies that any '%' needs to be encoded as '%%' + +# NOTE 3: Any variable that is subject to the environment variables +# ($NAME) and tilde (~username/~) expansions will receive tilde +# expansion first and only after the environment variable will be +# expanded in the resulting string. This behaviour is intentional +# as it coincides with typical shell expansion strategy. + +# NOTE 4: multiple same-named sections. +# The library used to parse the configuration file has known issue when multiple +# sections have the same name. In such case, only the last section is considered. +# It is strongly discouraged to have multiple sections with the same name. +# See https://github.com/OfflineIMAP/offlineimap/issues/143 for more details. + [general] -# This specifies where offlineimap is to store its metadata. +# This specifies where OfflineIMAP is to store its metadata. # This directory will be created if it does not already exist. - -metadata = ~/.offlineimap - -# This variable specifies which accounts are defined. Separate them -# with commas. Account names should be alphanumeric only. -# You will need to specify one section per account below. You may -# not use "general" for an account name. # +# Tilde and environment variable expansions will be performed. +# +#metadata = ~/.offlineimap + +# This option stands in the [general] section. +# +# This variable specifies which accounts are defined. Separate them with commas. +# Account names should be alphanumeric only. You will need to specify one +# section per account below. You may not use "general" for an account name. +# +# Always use ASCII characters only. +# accounts = Test -# Offlineimap can synchronize more the one account at a time. If you -# want to enable this feature, set the below value to something -# greater than 1. To force it to synchronize only one account at a -# time, set it to 1. + +# This option stands in the [general] section. # -# Note: if you are using autorefresh and have more than one account, -# you must set this number to be >= to the number of accounts you have; -# since any given sync run never "finishes" due to a timer, you will never -# sync your additional accounts if this is 1. +# Offlineimap can synchronize more than one account at a time. If you want to +# enable this feature, set the below value to something greater than 1. To +# force it to synchronize only one account at a time, set it to 1. +# +# NOTE: if you are using autorefresh and have more than one account, you must +# set this number to be >= to the number of accounts you have; since any given +# sync run never "finishes" due to a timer, you will never sync your additional +# accounts if this is 1. +# +#maxsyncaccounts = 1 -maxsyncaccounts = 1 -# You can specify one or more user interface modules for OfflineIMAP -# to use. OfflineIMAP will try the first in the list, and if it -# fails, the second, and so forth. +# This option stands in the [general] section. +# +# You can specify one or more user interface. OfflineIMAP will try the first in +# the list, and if it fails, the second, and so forth. # # The pre-defined options are: -# Curses.Blinkenlights -- A text-based (terminal) interface similar to -# Tk.Blinkenlights -# TTY.TTYUI -- a text-based (terminal) interface -# Noninteractive.Basic -- Noninteractive interface suitable for cronning -# Noninteractive.Quiet -- Noninteractive interface, generates no output -# except for errors. -# Machine.MachineUI -- Interactive interface suitable for machine -# parsing. +# Blinkenlights -- A fancy (terminal) interface +# TTYUI -- a text-based (terminal) interface +# Basic -- Noninteractive interface suitable for cron'ing +# Quiet -- Noninteractive interface, generates no output +# except for errors. +# MachineUI -- Interactive interface suitable for machine +# parsing. +# +# See also offlineimapui(7) # # You can override this with a command-line option -u. +# +#ui = basic -ui = Curses.Blinkenlights, TTY.TTYUI, - Noninteractive.Basic, Noninteractive.Quiet -# If you try to synchronize messages to a read-only folder, -# OfflineIMAP will generate a warning. If you want to suppress these -# warnings, set ignore-readonly to yes. Read-only IMAP folders allow -# reading but not modification, so if you try to change messages in -# the local copy of such a folder, the IMAP server will prevent -# OfflineIMAP from propagating those changes to the IMAP server. +# This option stands in the [general] section. +# +# If you try to synchronize messages to a folder which the IMAP server +# considers read-only, OfflineIMAP will generate a warning. If you want +# to suppress these warnings, set ignore-readonly to yes. Read-only +# IMAP folders allow reading but not modification, so if you try to +# change messages in the local copy of such a folder, the IMAP server +# will prevent OfflineIMAP from propagating those changes to the IMAP +# server. Note that ignore-readonly is UNRELATED to the "readonly" +# setting which prevents a repository from being modified at all. +# +#ignore-readonly = no -ignore-readonly = no ########## Advanced settings +# This option stands in the [general] section. +# # You can give a Python source filename here and all config file # python snippets will be evaluated in the context of that file. # This allows you to e.g. define helper functions in the Python # source file and call them from this config file. You can find # an example of this in the manual. # -# pythonfile = ~/.offlineimap.py +# Tilde and environment variable expansions will be performed. # +#pythonfile = ~/.offlineimap.py -# By default, OfflineIMAP will not exit due to a network error until -# the operating system returns an error code. Operating systems can sometimes -# take forever to notice this. Here you can activate a timeout on the -# socket. This timeout applies to individual socket reads and writes, -# not to an overall sync operation. You could perfectly well have a 30s -# timeout here and your sync still take minutes. + +# This option is in the [general] section. +# +# By default, OfflineIMAP will not exit due to a network error until the +# operating system returns an error code. Operating systems can sometimes take +# forever to notice this. Here you can activate a timeout on the socket. This +# timeout applies to individual socket reads and writes, not to an overall sync +# operation. You could perfectly well have a 30s timeout here and your sync +# still take minutes. # # Values in the 30-120 second range are reasonable. # # The default is to have no timeout beyond the OS. Times are given in seconds. # -# socktimeout = 60 +#socktimeout = 60 -# By default, OfflineIMAP will use fsync() to force data out to disk at -# opportune times to ensure consistency. This can, however, reduce -# performance. Users where /home is on SSD (Flash) may also wish to reduce -# write cycles. Therefore, you can disable OfflineIMAP's use of fsync(). -# Doing so will come at the expense of greater risk of message duplication -# in the event of a system crash or power loss. Default is fsync = true. -# Set fsync = false ot disable fsync. + +# This option stands in the [general] section. # -# fsync = true +# By default, OfflineIMAP will use fsync() to force data out to disk at +# opportune times to ensure consistency. This can, however, reduce performance. +# Users where /home is on SSD (Flash) may also wish to reduce write cycles. +# Therefore, you can disable OfflineIMAP's use of fsync(). Doing so will come +# at the expense of greater risk of message duplication in the event of a system +# crash or power loss. Default is true. Set it to false to disable fsync. +# +#fsync = true + ################################################## # Mailbox name recorder @@ -118,7 +173,7 @@ ignore-readonly = no [mbnames] -# offlineimap can record your mailbox names in a format you specify. +# OfflineIMAP can record your mailbox names in a format you specify. # You can define the header, each mailbox item, the separator, # and the footer. Here is an example for Mutt. # If enabled is yes, all six setting must be specified, even if they @@ -126,37 +181,63 @@ ignore-readonly = no # # The header, peritem, sep, and footer are all Python expressions passed # through eval, so you can (and must) use Python quoting. - -enabled = no -filename = ~/Mutt/muttrc.mailboxes -header = "mailboxes " -peritem = "+%(accountname)s/%(foldername)s" -sep = " " -footer = "\n" - -# You can also specify a folderfilter. It will apply to the -# *translated* folder name here, and it takes TWO arguments: -# accountname and foldername. In all other ways, it will -# behave identically to the folderfilter for accounts. Please see -# that section for more information and examples. # -# Note that this filter can be used only to further restrict mbnames -# to a subset of folders that pass the account's folderfilter. +# The incremental setting controls whether the file is written after each +# account completes or once all synced accounts are complete. This is usefull if +# an account is sightly slower than the other. It allows keeping the previous +# file rather than having it partially written. +# This works best with "no" if in one-shot mode started by cron or systemd +# timers. Default: no. +# +# The following hash key are available to the expansion for 'peritem': +# - accountname: the name of the corresponding account; +# - foldername: the name of the folder; +# - localfolders: path to the local directory hosting all Maildir +# folders for the account. +# +# Tilde and environment variable expansions will be performed +# for "filename" knob. +# +#enabled = no +#filename = ~/Mutt/muttrc.mailboxes +#header = "mailboxes " +#peritem = "+%(accountname)s/%(foldername)s" +#sep = " " +#footer = "\n" +#incremental = no -[ui.Curses.Blinkenlights] -# Character used to indicate thread status. -statuschar = . +# This option stands in the [mbnames] section. +# +# You can also specify a folderfilter. It will apply to the *translated* folder +# name here, and it takes TWO arguments: accountname and foldername. In all +# other ways, it will behave identically to the folderfilter for accounts. +# Please see the folderfilter option for more information and examples. +# +# This filter can be used only to further restrict mbnames to a subset of +# folders that pass the account's folderfilter. +# +# You can customize the order in which mailbox names are listed in the generated +# file by specifying a sort_keyfunc, which takes a single dict argument +# containing keys 'accountname' and 'foldername'. This function will be called +# once for each mailbox, and should return a suitable sort key that defines this +# mailbox' position in the custom ordering. +# +# This is useful with e.g. Mutt-sidebar, which uses the mailbox order from the +# generated file when listing mailboxes in the sidebar. +# +# Default setting is: +#sort_keyfunc = lambda d: (d['accountname'], d['foldername']) + ################################################## # Accounts ################################################## -# This is an account definition clause. You'll have one of these -# for each account listed in general/accounts above. +# This is an account definition clause. You'll have one of these for each +# account listed in the "accounts" option in [general] section (above). [Account Test] -########## Basic settings # These settings specify the two folders that you will be syncing. # You'll need to have a "Repository ..." section for each one. @@ -164,335 +245,753 @@ statuschar = . localrepository = LocalExample remoterepository = RemoteExample + ########## Advanced settings -# You can have offlineimap continue running indefinitely, automatically -# syncing your mail periodically. If you want that, specify how -# frequently to do that (in minutes) here. You can also specify -# fractional minutes (ie, 3.25). +# This option stands in the [Account Test] section. +# +# You can have OfflineIMAP continue running indefinitely, automatically syncing +# your mail periodically. If you want that, specify how frequently to do that +# (in minutes) here. Fractional minutes (ie, 3.25) is allowed. +# +#autorefresh = 5 -# autorefresh = 5 -# You can tell offlineimap to do a number of quicker synchronizations -# between full updates. A quick synchronization only synchronizes -# if a Maildir folder has changed, or if an IMAP folder has received -# new messages or had messages deleted. It does not update if the -# only changes were to IMAP flags. Specify 0 to never do quick updates, -# -1 to always do quick updates, or a positive integer to do that many -# quick updates between each full synchronization (requires autorefresh). +# This option stands in the [Account Test] section. +# +# OfflineImap can replace a number of full updates by quick synchronizations. +# This option is ignored if maxage or startdate are used. +# +# It only synchronizes a folder if +# +# 1) a Maildir folder has changed +# +# or +# +# 2) if an IMAP folder has received new messages or had messages deleted, ie +# it does not update if only IMAP flags have changed. +# +# Full updates need to fetch ALL flags for all messages, so this makes quite a +# performance difference (especially if syncing between two IMAP servers). +# +# Specify 0 for never, -1 for always (works even in non-autorefresh mode) +# +# A positive integer to do quick updates before doing another full +# synchronization (requires autorefresh). Updates are always performed after +# minutes, be they quick or full. +# +#quick = 10 -# quick = 10 -# You can specify a pre and post sync hook to execute a external command. -# in this case a call to imapfilter to filter mail before the sync process -# starts and a custom shell script after the sync completes. -# The pre sync script has to complete before a sync to the account will -# start. +# This option stands in the [Account Test] section. +# +# You can specify a pre and post sync hook to execute a external command. In +# this case a call to imapfilter to filter mail before the sync process starts +# and a custom shell script after the sync completes. +# +# The pre sync script has to complete before a sync to the account will start. +# +#presynchook = imapfilter -c someotherconfig.lua +#postsynchook = notifysync.sh -# presynchook = imapfilter -# postsynchook = notifysync.sh -# You can also specify parameters to the commands -# presynchook = imapfilter -c someotherconfig.lua +# This option stands in the [Account Test] section. +# +# OfflineImap caches the state of the synchronisation to e.g. be able to +# determine if a mail has been added or deleted on either side. +# +# The default and historical backend is 'plain' which writes out the +# state in plain text files. On Repositories with large numbers of +# mails, the performance might not be optimal, as we write out the +# complete file for each change. Another new backend 'sqlite' is +# available which stores the status in sqlite databases. +# +# If you switch the backend, you may want to delete the old cache +# directory in ~/.offlineimap/Account-/LocalStatus manually +# once you are sure that things work. +# +#status_backend = plain + +# This option stands in the [Account Test] section. +# # If you have a limited amount of bandwidth available you can exclude larger -# messages (e.g. those with large attachments etc). If you do this it -# will appear to offlineimap that these messages do not exist at all. They -# will not be copied, have flags changed etc. For this to work on an IMAP -# server the server must have server side search enabled. This works with gmail -# and most imap servers (e.g. cyrus etc) +# messages (e.g. those with large attachments etc). If you do this it will +# appear to OfflineIMAP that these messages do not exist at all. They will not +# be copied, have flags changed etc. For this to work on an IMAP server the +# server must have server side search enabled. This works with Gmail and most +# imap servers (e.g. cyrus etc) +# # The maximum size should be specified in bytes - e.g. 2000000 for approx 2MB - -# maxsize = 2000000 +# +#maxsize = 2000000 -# When you are starting to sync an already existing account yuo can tell offlineimap -# to sync messages from only the last x days. When you do this messages older than x -# days will be completely ignored. This can be useful for importing existing accounts -# when you do not want to download large amounts of archive email. +# This option stands in the [Account Test] section. +# +# maxage enables you to sync only recent messages. There are two ways to specify +# what "recent" means: if maxage is given as an integer, then only messages from +# the last maxage days will be synced. If maxage is given as a date, then only +# messages later than that date will be synced. +# +# Messages older than the cutoff will not be synced, their flags will not be +# changed, they will not be deleted, etc. For OfflineIMAP it will be like these +# messages do not exist. This will perform an IMAP search in the case of IMAP or +# Gmail and therefore requires that the server support server side searching. +# +# Known edge cases are described in offlineimap(1). +# +# maxage is allowed only when the local folder is of type Maildir. It can't be +# used with startdate. +# +# The maxage option expects an integer (for the number of days) or a date of the +# form yyyy-mm-dd. +# +#maxage = 3 +#maxage = 2015-04-01 -# Messages older than maxage days will not be synced, their flags will -# not be changed, they will not be deleted etc. For offlineimap it will be like these -# messages do not exist. This will perform an IMAP search in the case of IMAP or Gmail -# and therefor requires that the server support server side searching. This will -# calculate the earliest day that would be included in the search and include all -# messages from that day until today. e.g. maxage = 3 to sync only the last 3 days mail -# maxage = 3 +# This option stands in the [Account Test] section. +# +# Maildir file format uses colon (:) separator between uniq name and info. +# Unfortunatelly colon is not allowed character in windows file name. If you +# enable maildir-windows-compatible option, OfflineIMAP will be able to store +# messages on windows drive, but you will probably loose compatibility with +# other programs working with the maildir. +# +#maildir-windows-compatible = no + + +# This option stands in the [Account Test] section. +# +# Specifies if we want to sync GMail labels with the local repository. +# Effective only for GMail IMAP repositories. +# +# Non-ASCII characters in labels are bad handled or won't work at all. +# +#synclabels = no + + +# This option stands in the [Account Test] section. +# +# Name of the header to use for label storage. Format for the header +# value differs for different headers, because there are some de-facto +# "standards" set by popular clients: +# +# - X-Label or Keywords keep values separated with spaces; for these +# you, obviously, should not have label values that contain spaces; +# +# - X-Keywords use comma (',') as the separator. +# +# To be consistent with the usual To-like headers, for the rest of header +# types we use comma as the separator. +# +# Use ASCII characters only. +# +#labelsheader = X-Keywords + + +# This option stands in the [Account Test] section. +# +# Set of labels to be ignored. Comma-separated list. GMail-specific +# labels all start with backslash ('\'). +# +# Use ASCII characters only. +# +#ignorelabels = \Inbox, \Starred, \Sent, \Draft, \Spam, \Trash, \Important + + +# This option stands in the [Account Test] section. +# +# OfflineIMAP can strip off some headers when your messages are propagated +# back to the IMAP server. This option carries the comma-separated list +# of headers to trim off. Header name matching is case-sensitive. +# +# This knob is respected only by IMAP-based accounts. Value of labelsheader +# for GMail-based accounts is automatically added to this list, you don't +# need to specify it explicitely. +# +# Use ASCII characters only. +# +#filterheaders = X-Some-Weird-Header + + +# This option stands in the [Account Test] section. +# +# Use proxy connection for this account. Usefull to bypass the GFW in China. +# To specify a proxy connection, join proxy type, host and port with colons. +# Available proxy types are SOCKS5, SOCKS4, HTTP. +# You also need to install PySocks through pip. +# +# Currently, this feature leaks DNS support. +# +#proxy = SOCKS5:IP:9999 [Repository LocalExample] -# This is one of the two repositories that you'll work with given the -# above example. Each repository requires a "type" declaration. +# Each repository requires a "type" declaration. The types supported for +# local repositories are Maildir, GmailMaildir and IMAP. # -# The types supported are Maildir and IMAP. -# - type = Maildir + +# This option stands in the [Repository LocalExample] section. +# # Specify local repository. Your IMAP folders will be synchronized # to maildirs created under this path. OfflineIMAP will create the # maildirs for you as needed. - +# localfolders = ~/Test -# You can specify the "path separator character" used for your Maildir -# folders. This is inserted in-between the components of the tree. -# It defaults to ".". If you want your Maildir folders to be nested, -# set it to "/". -sep = . - -# Some users on *nix platforms may not want the atime (last access -# time) to be modified by OfflineIMAP. In these cases, they would -# want to set restoreatime to yes. OfflineIMAP will make an effort -# to not touch the atime if you do that. +# This option stands in the [Repository LocalExample] section. # -# In most cases, the default of no should be sufficient. +# You can specify the "folder separator character" used for your Maildir +# folders. It is inserted in-between the components of the tree. If you +# want your folders to be nested directories, set it to "/". 'sep' is +# ignored for IMAP repositories, as it is queried automatically. +# Otherwise, default value is ".". +# +#sep = "." + + +# This option stands in the [Repository LocalExample] section. +# +# startdate syncs mails starting from a given date. It applies the date +# restriction to LocalExample only. The remote repository MUST be empty +# at the first sync where this option is used. +# +# Unlike maxage, this is supported for IMAP-IMAP sync. +# +# startdate can't be used with maxage. +# +# The startdate option expects a date in the format yyyy-mm-dd. +# +#startdate = 2015-04-01 + + +# This option stands in the [Repository LocalExample] section. +# +# Some users may not want the atime (last access time) of folders to be +# modified by OfflineIMAP. If 'restoreatime' is set to yes, OfflineIMAP +# will restore the atime of the "new" and "cur" folders in each maildir +# folder to their original value after each sync. +# +# In nearly all cases, the default should be fine. +# +#restoreatime = no + + +# This option stands in the [Repository LocalExample] section. +# +# Set modification time of messages basing on the message's "Date" header. This +# option makes sense for the Maildir type, only. +# +# This is useful if you are doing some processing/finding on your Maildir (for +# example, finding messages older than 3 months), without parsing each +# file/message content. +# +# If enabled, this forbid the -q (quick mode) CLI option to work correctly. +# This option is still "TESTING" feature. +# +# Default: no. +# +#utime_from_header = no + + +[Repository GmailLocalExample] + +# This type of repository enables syncing of Gmail. All Maildir +# configuration settings are also valid here. +# +# This is a separate Repository type from Maildir because it involves +# some extra overhead which sometimes may be significant. We look for +# modified tags in local messages by looking only to the files +# modified since last run. This is usually rather fast, but the first +# time OfflineIMAP runs with synclabels enabled, it will have to check +# the contents of all individual messages for labels and this may take +# a while. +# +type = GmailMaildir -restoreatime = no [Repository RemoteExample] -# And this is the remote repository. We only support IMAP or Gmail here. - +# The remote repository. We only support IMAP or Gmail here. +# type = IMAP + +# These options stands in the [Repository RemoteExample] section. +# # The following can fetch the account credentials via a python expression that # is parsed from the pythonfile parameter. For example, a function called # "getcredentials" that parses a file "filename" and returns the account # details for "hostname". -# remotehosteval = getcredentials("filename", "hostname", "hostname") -# remoteusereval = getcredentials("filename", "hostname", "user") -# remotepasseval = getcredentials("filename", "hostname", "passwd") +# +#remotehosteval = getcredentials("filename", "hostname", "hostname") +#remoteporteval = getcredentials("filename", "hostname", "port") +#remoteusereval = getcredentials("filename", "hostname", "user") +#remotepasseval = getcredentials("filename", "hostname", "passwd") + +# This option stands in the [Repository RemoteExample] section. +# # Specify the remote hostname. +# remotehost = examplehost + +# This option stands in the [Repository RemoteExample] section. +# # Whether or not to use SSL. -ssl = yes +# +# Note: be care to configure the 'remotehost' line with the domain name defined +# in the certificate. E.g., if you trust your provider and want to use the +# certificate it provides on a shared server. Otherwise, OfflineIMAP will stop +# and say that the domain is not named in the certificate. +# +#ssl = yes -# SSL Client certificate (optional) -# sslclientcert = /path/to/file.crt -# SSL Client key (optional) -# sslclientkey = /path/to/file.key +# This option stands in the [Repository RemoteExample] section. +# +# SSL Client certificate (optional). +# +# Tilde and environment variable expansions will be performed. +# +#sslclientcert = /path/to/file.crt + +# This option stands in the [Repository RemoteExample] section. +# +# SSL Client key (optional). +# +# Tilde and environment variable expansions will be performed. +# +#sslclientkey = /path/to/file.key + + +# This option stands in the [Repository RemoteExample] section. +# # SSL CA Cert(s) to verify the server cert against (optional). # No SSL verification is done without this option. If it is # specified, the CA Cert(s) need to verify the Server cert AND # match the hostname (* wildcard allowed on the left hand side) # The certificate should be in PEM format. -# sslcacertfile = /path/to/cacertfile.crt +# +# Tilde and environment variable expansions will be performed. +# +# Special value OS-DEFAULT makes OfflineIMAP to automatically +# determine system-wide location of standard trusted CA roots file +# for known OS distributions and use the first bundle encountered +# (if any). If no system-wide CA bundle is found, OfflineIMAP +# will refuse to continue; this is done to prevent creation +# of false security expectations ("I had configured CA bundle, +# thou certificate verification shalt be present"). +# +# You can also use fingerprint verification via cert_fingerprint. +# See below for more verbose explanation. +# +#sslcacertfile = /path/to/cacertfile.crt + +# This option stands in the [Repository RemoteExample] section. +# +# If you connect via SSL/TLS (ssl = yes) and you have no CA certificate +# specified, OfflineIMAP will refuse to sync as it connects to a server +# with an unknown "fingerprint". If you are sure you connect to the +# correct server, you can then configure the presented server +# fingerprint here. OfflineIMAP will verify that the server fingerprint +# has not changed on each connect and refuse to connect otherwise. +# +# You can also configure fingerprint validation in addition to +# CA certificate validation above and it will check both: +# OfflineIMAP fill verify certificate first and if things will be fine, +# fingerprint will be validated. +# +# Multiple fingerprints can be specified, separated by commas. +# +# In Windows, Microsoft uses the term "thumbprint" instead of "fingerprint". +# +# Fingerprints must be in hexadecimal form without leading '0x': +# 40 hex digits like bbfe29cf97acb204591edbafe0aa8c8f914287c9. +# +#cert_fingerprint = [, ] + + +# This option stands in the [Repository RemoteExample] section. +# +# SSL version (optional). +# +# It is best to leave this unset, in which case the correct version will be +# automatically detected. In rare cases, it may be necessary to specify a +# particular version from: tls1, ssl2, ssl3, ssl23 (SSLv2 or SSLv3) +# +#ssl_version = ssl23 + + +# This option stands in the [Repository RemoteExample] section. +# # Specify the port. If not specified, use a default port. -# remoteport = 993 +# +#remoteport = 993 + +# This option stands in the [Repository RemoteExample] section. +# # Specify the remote user name. +# remoteuser = username -# There are five ways to give the password for the remote IMAP -# server: + +# This option stands in the [Repository RemoteExample] section. # -# 1. No password at all specified in the config file. If a matching entry is -# found in ~/.netrc (see netrc (5) for information) the password from the -# matching entry will be used. If there is no ~/.netrc file but there is an -# /etc/netrc file, the password will instead be taken from there. Otherwise -# you will be prompted for the password when OfflineIMAP starts. +# Specify the user to be authorized as. Sometimes we want to +# authenticate with our login/password, but tell the server that we +# really want to be treated as some other user; perhaps server will +# allow us to do that (or maybe not). Some IMAP servers migrate +# account names using this functionality: your credentials remain +# intact, but remote identity changes. +# +# Currently this variable is used only for SASL PLAIN authentication +# mechanism, so consider using auth_mechanisms to prioritize PLAIN +# or even make it the only mechanism to be tried. +# +#remote_identity = authzuser + + +# This option stands in the [Repository RemoteExample] section. +# +# Specify which authentication/authorization mechanisms we should try and the +# order in which OfflineIMAP will try them. +# +# NOTE: any given mechanism will be tried ONLY if it is supported by the remote +# IMAP server. +# +# Default value is ranged is from strongest to more weak ones. Due to technical +# limitations, if GSSAPI is set, it will be tried first, no matter where it was +# specified in the list. +# +#auth_mechanisms = GSSAPI, CRAM-MD5, PLAIN, LOGIN + + +########## Passwords + +# There are six ways to specify the password for the IMAP server: +# +# 1. No password at all specified in the config file. +# If a matching entry is found in ~/.netrc (see netrc (5) for +# information) this password will be used. Do note that netrc only +# allows one entry per hostname. If there is no ~/.netrc file but +# there is an /etc/netrc file, the password will instead be taken +# from there. Otherwise you will be prompted for the password when +# OfflineIMAP starts when using a UI that supports this. # # 2. The remote password stored in this file with the remotepass -# option. Example: -# -# remotepass = mypassword +# option. Any '%' needs to be encoded as '%%'. Example: +# remotepass = mypassword # # 3. The remote password stored as a single line in an external # file, which is referenced by the remotefile option. Example: -# -# remotepassfile = ~/Password.IMAP.Account1 +# remotepassfile = ~/Password.IMAP.Account1 # # 4. With a preauth tunnel. With this method, you invoke an external -# program that is guaranteed *NOT* to ask for a password, but rather -# to read from stdin and write to stdout an IMAP procotol stream -# that begins life in the PREAUTH state. When you use a tunnel, -# you do NOT specify a user or password (if you do, they'll be -# ignored.) Instead, you specify a preauthtunnel, as this -# example illustrates for Courier IMAP on Debian: +# program that is guaranteed *NOT* to ask for a password, but rather +# to read from stdin and write to stdout an IMAP procotol stream that +# begins life in the PREAUTH state. When you use a tunnel, you do +# NOT specify a user or password (if you do, they'll be ignored.) +# Instead, you specify a preauthtunnel, as this example illustrates +# for Courier IMAP on Debian: +# preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' # -# preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' +# 5. If you are using Kerberos and have the Python Kerberos package +# installed, you should not specify a remotepass. If the user has a +# valid Kerberos TGT, OfflineIMAP will figure out the rest all by +# itself, and fall back to password authentication if needed. # -# 5. If you are using Kerberos and have the Python Kerberos package installed, -# you should not specify a remotepass. If the user has a valid -# Kerberos TGT, OfflineIMAP will figure out the rest all by itself, and -# fall back to password authentication if needed. +# 6. Using arbitrary python code. With this method, you invoke a +# function from your pythonfile. To use this method assign the name +# of the function to the variable 'remotepasseval'. Example: +# remotepasseval = get_password("imap.example.net") +# You can also query for the username: +# remoteusereval = get_username("imap.example.net") +# This method can be used to design more elaborate setups, e.g. by +# querying the gnome-keyring via its python bindings. + ########## Advanced settings -# Some IMAP servers need a "reference" which often refers to the -# "folder root". This is most commonly needed with UW IMAP, where -# you might need to specify the directory in which your mail is -# stored. Most users will not need this. -# -# reference = Mail +# These options stands in the [Repository RemoteExample] section. +# +# Tunnels. There are two types: +# +# - preauth: they teleport your connection to the remote system +# and you don't need to authenticate yourself there; the sole +# fact that you succeeded to get the tunnel running is enough. +# This tunnel type was explained above in the 'Passwords' section. +# +# - transport: the just provide the transport (probably encrypted) +# to the IMAP server, but you still need to authenticate at the +# IMAP server. +# +# Tunnels are currently working only with IMAP servers and their +# derivatives (GMail currently). Additionally, for GMail accounts +# preauth tunnel settings are ignored: we don't believe that there +# are ways to preauthenticate at Google mail system IMAP servers. +# +# You must choose at most one tunnel type, be wise M'Lord! +# +#preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' +#transporttunnel = openssl s_client -host myimap -port 993 -quiet + + +# This option stands in the [Repository RemoteExample] section. +# +# Some IMAP servers need a "reference" which often refers to the "folder root". +# +# This is most commonly needed with UW IMAP, where you might need to specify the +# directory in which your mail is stored. The 'reference' value will be prefixed +# to all folder paths refering to that repository. E.g. accessing folder 'INBOX' +# with "reference = Mail" will try to access Mail/INBOX. +# +# The nametrans and folderfilter functions will apply to the full path, +# including the reference prefix. Most users will not need this. +# +#reference = Mail + + +# This option stands in the [Repository RemoteExample] section. +# +# In between synchronisations, OfflineIMAP can monitor mailboxes for new +# messages using the IDLE command. If you want to enable this, specify here the +# folders you wish to monitor. IMAP protocol requires a separate connection for +# each folder monitored in this way, so setting this option will force settings +# for: +# +# - maxconnections: to be at least the number of folders you give +# - holdconnectionopen: to be true +# - keepalive: to be 29 minutes unless you specify otherwise +# +# This feature isn't complete and may well have problems. See the "Known Issues" +# entry in the manual for more details. +# +# This option should return a Python list. For example +# +#idlefolders = ['INBOX', 'INBOX.Alerts'] + + +# This option stands in the [Repository RemoteExample] section. +# +# OfflineIMAP can use a compressed connection to the IMAP server. +# This can result in faster downloads for some cases. +# +#usecompression = yes + + +# This option stands in the [Repository RemoteExample] section. +# # OfflineIMAP can use multiple connections to the server in order # to perform multiple synchronization actions simultaneously. # This may place a higher burden on the server. In most cases, # setting this value to 2 or 3 will speed up the sync, but in some # cases, it may slow things down. The safe answer is 1. You should # probably never set it to a value more than 5. +# +#maxconnections = 2 -maxconnections = 1 +# This option stands in the [Repository RemoteExample] section. +# # OfflineIMAP normally closes IMAP server connections between refreshes if # the global option autorefresh is specified. If you wish it to keep the # connection open, set this to true. If not specified, the default is # false. Keeping the connection open means a faster sync start the # next time and may use fewer server resources on connection, but uses # more server memory. This setting has no effect if autorefresh is not set. - -holdconnectionopen = no - -# If you want to have "keepalives" sent while waiting between syncs, -# specify the amount of time IN SECONDS between keepalives here. Note that -# sometimes more than this amount of time might pass, so don't make it -# tight. This setting has no effect if autorefresh and holdconnectionopen -# are not both set. - -# keepalive = 60 - -# Normally, OfflineIMAP will expunge deleted messages from the server. -# You can disable that if you wish. This means that OfflineIMAP will -# mark them deleted on the server, but not actually delete them. -# You must use some other IMAP client to delete them if you use this -# setting; otherwise, the messgaes will just pile up there forever. -# Therefore, this setting is definately NOT recommended. # -# expunge = no +#holdconnectionopen = no + +# This option stands in the [Repository RemoteExample] section. +# +# If you want to have "keepalives" sent while waiting between syncs, specify the +# amount of time IN SECONDS between keepalives here. Note that sometimes more +# than this amount of time might pass, so don't make it tight. This setting has +# no effect if autorefresh and holdconnectionopen are not both set. +# +#keepalive = 60 + + +# This option stands in the [Repository RemoteExample] section. +# +# Normally, OfflineIMAP will expunge deleted messages from the server. You can +# disable that if you wish. This means that OfflineIMAP will mark them deleted +# on the server, but not actually delete them. You must use some other IMAP +# client to delete them if you use this setting; otherwise, the messages will +# just pile up there forever. Therefore, this setting is definitely NOT +# recommended for a long term. +# +#expunge = no + + +# This option stands in the [Repository RemoteExample] section. +# # Specify whether to process all mail folders on the server, or only # those listed as "subscribed". -subscribedonly = no - -# You can specify a folder translator. This must be a eval-able -# Python expression that takes a foldername arg and returns the new -# value. I suggest a lambda. This example below will remove "INBOX." from -# the leading edge of folders (great for Courier IMAP users) # -# WARNING: you MUST construct this such that it NEVER returns -# the same value for two folders, UNLESS the second values are -# filtered out by folderfilter below. Failure to follow this rule -# will result in undefined behavior -# -# nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) +#subscribedonly = no + +# This option stands in the [Repository RemoteExample] section. +# +# You can specify a folder translator. This must be a eval-able. +# +# Python expression that takes a foldername arg and returns the new value. A +# lambda function is suggested. +# +# WARNING: you MUST construct it so that it NEVER returns the same value for two +# folders, UNLESS the second values are filtered out by folderfilter below. +# Failure to follow this rule will result in undefined behavior. +# +# See the user documentation for details and use cases. They are also online at: +# http://docs.offlineimap.org/en/latest/nametrans.html +# +# This example below will remove "INBOX." from the leading edge of folders +# (great for Courier IMAP users). +# +#nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) +# # Using Courier remotely and want to duplicate its mailbox naming # locally? Try this: # -# nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) +#nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) -# You can specify which folders to sync. You can do it several ways. -# I'll provide some examples. The folderfilter operates on the -# *UNTRANSLATED* name, if you specify nametrans. It should return -# true if the folder is to be included; false otherwise. + +# This option stands in the [Repository RemoteExample] section. +# +# Determines if folderfilter will be invoked on each run (dynamic folder +# filtering) or filtering status will be determined at startup (default +# behaviour). +# +#dynamic_folderfilter = False + + +# This option stands in the [Repository RemoteExample] section. +# +# You can specify which folders to sync using the folderfilter setting. You can +# provide any python function (e.g. a lambda function) which will be invoked for +# each foldername. If the filter function returns True, the folder will be +# synced, if it returns False, it. +# +# The folderfilter operates on the *UNTRANSLATED* name (before any nametrans +# translation takes place). # # Example 1: synchronizing only INBOX and Sent. # -# folderfilter = lambda foldername: foldername in ['INBOX', 'Sent'] +#folderfilter = lambda foldername: foldername in ['INBOX', 'Sent'] # # Example 2: synchronizing everything except Trash. # -# folderfilter = lambda foldername: foldername not in ['Trash'] +#folderfilter = lambda foldername: foldername not in ['Trash'] # # Example 3: Using a regular expression to exclude Trash and all folders # containing the characters "Del". # -# folderfilter = lambda foldername: not re.search('(^Trash$|Del)', foldername) +#folderfilter = lambda foldername: not re.search('(^Trash$|Del)', foldername) # -# If folderfilter is not specified, ALL remote folders will be -# synchronized. +# If folderfilter is not specified, ALL remote folders will be synchronized. # -# You can span multiple lines by indenting the others. (Use backslashes -# at the end when required by Python syntax) For instance: +# You can span multiple lines by indenting the others. (Use backslashes at the +# end when required by Python syntax) For instance: # -# folderfilter = lambda foldername: foldername in -# ['INBOX', 'Sent Mail', 'Deleted Items', -# 'Received'] -# -# FYI, you could also include every folder with: -# -# folderfilter = lambda foldername: 1 -# -# And exclude every folder with: -# -# folderfilter = lambda foldername: 0 +#folderfilter = lambda foldername: foldername in [ +# 'INBOX', 'Sent Mail', +# 'Deleted Items', 'Received'] -# You can specify folderincludes to include additional folders. -# It should return a Python list. This might be used to include a -# folder that was excluded by your folderfilter rule, to include a -# folder that your server does not specify with its LIST option, or -# to include a folder that is outside your basic reference. Some examples: -# -# To include debian.user and debian.personal: -# -# folderincludes = ['debian.user', 'debian.personal'] -# -# To include your INBOX (UW IMAPd users will find this useful if they -# specify a reference): -# -# folderincludes = ['INBOX'] -# -# To specify a long list: -# -# folderincludes = ['box1', 'box2', 'box3', 'box4', -# 'box5', 'box6'] -# You can specify foldersort to determine how folders are sorted. -# This affects order of synchronization and mbnames. The expression -# should return -1, 0, or 1, as the default Python cmp() does. The -# two arguments, x and y, are strings representing the names of the folders -# to be sorted. The sorting is applied *AFTER* nametrans, if any. +# This option stands in the [Repository RemoteExample] section. # -# To reverse the sort: +# You can specify folderincludes to include additional folders. It should +# return a Python list. This might be used to include a folder that was +# excluded by your folderfilter rule, to include a folder that your server does +# not specify with its LIST option, or to include a folder that is outside your +# basic reference. # -# foldersort = lambda x, y: -cmp(x, y) +# The 'reference' value will not be prefixed to this folder name, even if you +# have specified one. For example: +# +#folderincludes = ['debian.user', 'debian.personal'] + + +# This option stands in the [Repository RemoteExample] section. +# +# If you do not want to have any folders created on this repository, +# set the createfolders variable to False, the default is True. Using +# this feature you can e.g. disable the propagation of new folders to +# the new repository. +# +#createfolders = True + + +# This option stands in the [Repository RemoteExample] section. +# +# 'foldersort' determines how folders are sorted. +# +# This affects order of synchronization and mbnames. The expression should +# return -1, 0, or 1, as the default Python cmp() does. The two arguments, x +# and y, are strings representing the names of the folders to be sorted. The +# sorting is applied *AFTER* nametrans, if any. The default is to sort IMAP +# folders alphabetically (case-insensitive). Usually, you should never have to +# modify this. To eg. reverse the sort: +# +#foldersort = lambda x, y: -cmp(x, y) + + +# This option stands in the [Repository RemoteExample] section. +# +# Enable 1-way synchronization. When setting 'readonly' to True, this +# repository will not be modified during synchronization. Usefull to +# e.g. backup an IMAP server. The readonly setting can be applied to any +# type of Repository (Maildir, Imap, etc). +# +#readonly = False [Repository GmailExample] -# A repository using Gmail's IMAP interface. Any configuration -# parameter of `IMAP` type repositories can be used here. Only -# `remoteuser` (or `remoteusereval` ) is mandatory. Default values -# for other parameters are OK, and you should not need fiddle with -# those. +# A repository using Gmail's IMAP interface. # -# The Gmail repository will use hard-coded values for `remotehost`, -# `remoteport`, `tunnel` and `ssl`. (See -# http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814) -# Any attempt to set those parameters will be silently ignored. +# Any configuration parameter of "IMAP" type repositories can be used here. +# Only "remoteuser" (or "remoteusereval" ) is mandatory. Default values for +# other parameters are OK, and you should not need fiddle with those. +# +# The Gmail repository will use hard-coded values for "remotehost", +# "remoteport", "tunnel" and "ssl". Any attempt to set those parameters will be +# silently ignored. For details, see +# +# http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814 +# +# To enable GMail labels synchronisation, set the option "synclabels" +# in the corresponding "Account" section. # - type = Gmail + +# This option stands in the [Repository GmailExample] section. +# # Specify the Gmail user name. This is the only mandatory parameter. +# remoteuser = username@gmail.com -# Deleting a message from a Gmail folder via the IMAP interface will -# just remove that folder's label from the message: the message will -# continue to exist in the '[Gmail]/All Mail' folder. If `realdelete` -# is set to `True`, then deleted messages will really be deleted -# during `offlineimap` sync, by moving them to the '[Gmail]/Trash' -# folder. BEWARE: this will delete a messages from *all folders* it -# belongs to! -# -# See http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 -realdelete = no -# The trash folder name may be different from [Gmail]/Trash -# for example on german googlemail, this setting should be +# This option stands in the [Repository GmailExample] section. # -# trashfolder = [Google Mail]/Papierkorb +# The trash folder name may be different from [Gmail]/Trash due to localization. +# You should look for the localized names of the spam folder too: "spamfolder" +# tunable will help you to override the standard name. # -# The same is valid for the spam folder +# For example on German Gmail, this setting should be: # -# spamfolder = [Google Mail]/Spam - +#trashfolder = [Gmail]/Papierkorb diff --git a/offlineimap.conf.minimal b/offlineimap.conf.minimal index 2979d3d..3ca6e09 100644 --- a/offlineimap.conf.minimal +++ b/offlineimap.conf.minimal @@ -1,5 +1,5 @@ # Sample minimal config file. Copy this to ~/.offlineimaprc and edit to -# suit to get started fast. +# get started fast. [general] accounts = Test diff --git a/offlineimap.py b/offlineimap.py index ce1c70a..05da925 100755 --- a/offlineimap.py +++ b/offlineimap.py @@ -17,6 +17,19 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import os +import sys + +if not 'DEVELOPING_OFFLINEIMAP_PYTHON3_SUPPORT' in os.environ: + if sys.version_info[0] > 2: + sys.stderr.write("""IIMAPS! + +Sorry, OfflineIMAP currently doesn't support Python higher than 2.x. +We're doing our best to bring in support for 3.x really soon. You can +also join us at https://github.com/OfflineIMAP/offlineimap/ and help. +""") + sys.exit(1) + from offlineimap import OfflineImap oi = OfflineImap() diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 0c10271..44cfcab 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -1,5 +1,4 @@ -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,60 +14,142 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from ConfigParser import ConfigParser -from offlineimap.localeval import LocalEval import os +import re +from sys import exc_info + +try: + from ConfigParser import SafeConfigParser, Error +except ImportError: #python3 + from configparser import SafeConfigParser, Error +from offlineimap.localeval import LocalEval + +class CustomConfigParser(SafeConfigParser): + def __init__(self): + SafeConfigParser.__init__(self) + self.localeval = None -class CustomConfigParser(ConfigParser): def getdefault(self, section, option, default, *args, **kwargs): - """Same as config.get, but returns the "default" option if there - is no such option specified.""" + """Same as config.get, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): - return apply(self.get, [section, option] + list(args), kwargs) + return self.get(*(section, option) + args, **kwargs) else: return default - + + def getdefaultint(self, section, option, default, *args, **kwargs): + """Same as config.getint, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): - return apply(self.getint, [section, option] + list(args), kwargs) + return self.getint(*(section, option) + args, **kwargs) else: return default + def getdefaultfloat(self, section, option, default, *args, **kwargs): + """Same as config.getfloat, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): - return apply(self.getfloat, [section, option] + list(args), kwargs) + return self.getfloat(*(section, option) + args, **kwargs) else: return default def getdefaultboolean(self, section, option, default, *args, **kwargs): + """Same as config.getboolean, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): - return apply(self.getboolean, [section, option] + list(args), - kwargs) + return self.getboolean(*(section, option) + args, **kwargs) + else: + return default + + def getlist(self, section, option, separator_re): + """Parses option as the list of values separated + by the given regexp.""" + + try: + val = self.get(section, option).strip() + return re.split(separator_re, val) + except re.error as e: + raise Error("Bad split regexp '%s': %s" % \ + (separator_re, e)), None, exc_info()[2] + + def getdefaultlist(self, section, option, default, separator_re): + """Same as getlist, but returns the value of `default` + if there is no such option specified.""" + + if self.has_option(section, option): + return self.getlist(*(section, option, separator_re)) else: return default def getmetadatadir(self): - metadatadir = os.path.expanduser(self.getdefault("general", "metadata", "~/.offlineimap")) + xforms = [os.path.expanduser, os.path.expandvars] + d = self.getdefault("general", "metadata", "~/.offlineimap") + metadatadir = self.apply_xforms(d, xforms) if not os.path.exists(metadatadir): - os.mkdir(metadatadir, 0700) + os.mkdir(metadatadir, 0o700) return metadatadir def getlocaleval(self): + # We already loaded pythonfile, so return this copy. + if self.localeval is not None: + return self.localeval + + xforms = [os.path.expanduser, os.path.expandvars] if self.has_option("general", "pythonfile"): - path = os.path.expanduser(self.get("general", "pythonfile")) + path = self.get("general", "pythonfile") + path = self.apply_xforms(path, xforms) else: path = None - return LocalEval(path) + + self.localeval = LocalEval(path) + return self.localeval def getsectionlist(self, key): - """Returns a list of sections that start with key + " ". That is, - if key is "Account", returns all section names that start with - "Account ", but strips off the "Account ". For instance, for - "Account Test", returns "Test".""" + """Returns a list of sections that start with (str) key + " ". + + That is, if key is "Account", returns all section names that + start with "Account ", but strips off the "Account ". + + For instance, for "Account Test", returns "Test".""" key = key + ' ' return [x[len(key):] for x in self.sections() \ - if x.startswith(key)] + if x.startswith(key)] + + def set_if_not_exists(self, section, option, value): + """Set a value if it does not exist yet. + + This allows to set default if the user has not explicitly + configured anything.""" + + if not self.has_option(section, option): + self.set(section, option, value) + + + def apply_xforms(self, string, transforms): + """Applies set of transformations to a string. + + Arguments: + - string: source string; if None, then no processing will + take place. + - transforms: iterable that returns transformation function + on each turn. + + Returns transformed string.""" + + if string == None: + return None + for f in transforms: + string = f(string) + return string + + def CustomConfigDefault(): """Just a constant that won't occur anywhere else. @@ -76,43 +157,156 @@ def CustomConfigDefault(): This allows us to differentiate if the user has passed in any default value to the getconf* functions in ConfigHelperMixin derived classes.""" + pass + + class ConfigHelperMixin: - """Allow comfortable retrieving of config values pertaining to a section. + """Allow comfortable retrieving of config values pertaining + to a section. - If a class inherits from this cls:`ConfigHelperMixin`, it needs - to provide 2 functions: meth:`getconfig` (returning a - ConfigParser object) and meth:`getsection` (returning a string - which represents the section to look up). All calls to getconf* - will then return the configuration values for the ConfigParser - object in the specific section.""" + If a class inherits from cls:`ConfigHelperMixin`, it needs + to provide 2 functions: + - meth:`getconfig` (returning a CustomConfigParser object) + - and meth:`getsection` (returning a string which represents + the section to look up). + All calls to getconf* will then return the configuration values + for the CustomConfigParser object in the specific section. + """ - def _confighelper_runner(self, option, default, defaultfunc, mainfunc): - """Return config value for getsection()""" + def _confighelper_runner(self, option, default, defaultfunc, mainfunc, *args): + """Returns configuration or default value for option + that contains in section identified by getsection(). + + Arguments: + - option: name of the option to retrieve; + - default: governs which function we will call. + * When CustomConfigDefault is passed, we will call + the mainfunc. + * When any other value is passed, we will call + the defaultfunc and the value of `default` will + be passed as the third argument to this function. + - defaultfunc and mainfunc: processing helpers. + - args: additional trailing arguments that will be passed + to all processing helpers. + """ + + lst = [self.getsection(), option] if default == CustomConfigDefault: - return apply(mainfunc, [self.getsection(), option]) + return mainfunc(*(lst + list(args))) else: - return apply(defaultfunc, [self.getsection(), option, default]) + lst.append(default) + return defaultfunc(*(lst + list(args))) + + def getconfig(self): + """Returns CustomConfigParser object that we will use + for all our actions. + + Must be overriden in all classes that use this mix-in.""" + + raise NotImplementedError("ConfigHelperMixin.getconfig() " + "is to be overriden") - def getconf(self, option, - default = CustomConfigDefault): + + def getsection(self): + """Returns name of configuration section in which our + class keeps its configuration. + + Must be overriden in all classes that use this mix-in.""" + + raise NotImplementedError("ConfigHelperMixin.getsection() " + "is to be overriden") + + + def getconf(self, option, default = CustomConfigDefault): + """Retrieves string from the configuration. + + Arguments: + - option: option name whose value is to be retrieved; + - default: default return value if no such option + exists. + """ + return self._confighelper_runner(option, default, self.getconfig().getdefault, self.getconfig().get) + + def getconf_xform(self, option, xforms, default = CustomConfigDefault): + """Retrieves string from the configuration transforming the result. + + Arguments: + - option: option name whose value is to be retrieved; + - xforms: iterable that returns transform functions + to be applied to the value of the option, + both retrieved and default one; + - default: default value for string if no such option + exists. + """ + + value = self.getconf(option, default) + return self.getconfig().apply_xforms(value, xforms) + + def getconfboolean(self, option, default = CustomConfigDefault): + """Retrieves boolean value from the configuration. + + Arguments: + - option: option name whose value is to be retrieved; + - default: default return value if no such option + exists. + """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultboolean, self.getconfig().getboolean) + def getconfint(self, option, default = CustomConfigDefault): + """ + Retrieves integer value from the configuration. + + Arguments: + - option: option name whose value is to be retrieved; + - default: default return value if no such option + exists. + + """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultint, self.getconfig().getint) - + + def getconffloat(self, option, default = CustomConfigDefault): + """Retrieves floating-point value from the configuration. + + Arguments: + - option: option name whose value is to be retrieved; + - default: default return value if no such option + exists. + """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultfloat, self.getconfig().getfloat) + + + def getconflist(self, option, separator_re, + default = CustomConfigDefault): + """Retrieves strings from the configuration and splits it + into the list of strings. + + Arguments: + - option: option name whose value is to be retrieved; + - separator_re: regular expression for separator + to be used for split operation; + - default: default return value if no such option + exists. + """ + + return self._confighelper_runner(option, default, + self.getconfig().getdefaultlist, + self.getconfig().getlist, separator_re) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index e228282..8b93c96 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,23 +1,21 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.3.2.3" -__copyright__ = "Copyright (C) 2002 - 2010 John Goerzen" +__version__ = "6.5.7" +__revision__ = "" +__bigversion__ = __version__ + __revision__ +__copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" -__author_email__= "john@complete.org" +__author_email__= "offlineimap-project@lists.alioth.debian.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" -__bigcopyright__ = """%(__productname__)s %(__version__)s -%(__copyright__)s <%(__author_email__)s>""" % locals() +__license__ = "Licensed under the GNU GPL v2 or any later version (with an OpenSSL exception)" +__bigcopyright__ = """%(__productname__)s %(__bigversion__)s + %(__license__)s""" % locals() +__homepage__ = "http://offlineimap.org" -banner = __bigcopyright__ + """ +banner = __bigcopyright__ -This software comes with ABSOLUTELY NO WARRANTY; see the file -COPYING for details. This is free software, and you are welcome -to distribute it under the conditions laid out in COPYING.""" - -__homepage__ = "http://github.com/nicolas33/offlineimap" -__license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)." - -# put this last, so we don't run into circular dependencies using +from offlineimap.error import OfflineImapError +# put this last, so we don't run into circular dependencies using # e.g. offlineimap.__version__. from offlineimap.init import OfflineImap diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 63e83fa..515c11d 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -1,5 +1,4 @@ -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,104 +14,77 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import threadutil, mbnames, CustomConfig -import offlineimap.repository.Base, offlineimap.repository.LocalStatus -from offlineimap.ui import getglobalui -from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread from subprocess import Popen, PIPE -from threading import Event, Lock +from threading import Event import os -from Queue import Queue, Empty -import sys +import time +from sys import exc_info import traceback -class SigListener(Queue): - def __init__(self): - self.folderlock = Lock() - self.folders = None - Queue.__init__(self, 20) - def put_nowait(self, sig): - self.folderlock.acquire() - try: - if sig == 1: - if self.folders is None or not self.autorefreshes: - # folders haven't yet been added, or this account is once-only; drop signal - return - elif self.folders: - for foldernr in range(len(self.folders)): - # requeue folder - self.folders[foldernr][1] = True - self.quick = False - return - # else folders have already been cleared, put signal... - finally: - self.folderlock.release() - Queue.put_nowait(self, sig) - def addfolders(self, remotefolders, autorefreshes, quick): - self.folderlock.acquire() - try: - self.folders = [] - self.quick = quick - self.autorefreshes = autorefreshes - for folder in remotefolders: - # new folders are queued - self.folders.append([folder, True]) - finally: - self.folderlock.release() - def clearfolders(self): - self.folderlock.acquire() - try: - for folder, queued in self.folders: - if queued: - # some folders still in queue - return False - self.folders[:] = [] - return True - finally: - self.folderlock.release() - def queuedfolders(self): - self.folderlock.acquire() - try: - dirty = True - while dirty: - dirty = False - for foldernr, (folder, queued) in enumerate(self.folders): - if queued: - # mark folder as no longer queued - self.folders[foldernr][1] = False - dirty = True - quick = self.quick - self.folderlock.release() - yield (folder, quick) - self.folderlock.acquire() - except: - self.folderlock.release() - raise - self.folderlock.release() +from offlineimap import mbnames, CustomConfig, OfflineImapError +from offlineimap import globals +from offlineimap.repository import Repository +from offlineimap.ui import getglobalui +from offlineimap.threadutil import InstanceLimitedThread +try: + import fcntl +except: + pass # ok if this fails, we can do without + +# FIXME: spaghetti code alert! def getaccountlist(customconfig): + # Account names in a list. return customconfig.getsectionlist('Account') +# FIXME: spaghetti code alert! def AccountListGenerator(customconfig): + """Returns a list of instanciated Account class, one per account name.""" + return [Account(customconfig, accountname) for accountname in getaccountlist(customconfig)] +# FIXME: spaghetti code alert! def AccountHashGenerator(customconfig): + """Returns a dict of instanciated Account class with the account name as + key.""" + retval = {} for item in AccountListGenerator(customconfig): retval[item.getname()] = item return retval -mailboxes = [] class Account(CustomConfig.ConfigHelperMixin): + """Represents an account (ie. 2 repositories) to sync. + + Most of the time you will actually want to use the derived + :class:`accounts.SyncableAccount` which contains all functions used + for syncing an account.""" + + # Signal gets set when we should stop looping. + abort_soon_signal = Event() + # Signal gets set on CTRL-C/SIGTERM. + abort_NOW_signal = Event() + def __init__(self, config, name): + """ + :param config: Representing the offlineimap configuration file. + :type config: :class:`offlineimap.CustomConfig.CustomConfigParser` + + :param name: A (str) string denoting the name of the Account + as configured. + """ + self.config = config self.name = name self.metadatadir = config.getmetadatadir() self.localeval = config.getlocaleval() + # current :mod:`offlineimap.ui`, can be used for logging: self.ui = getglobalui() self.refreshperiod = self.getconffloat('autorefresh', 0.0) + # should we run in "dry-run" mode? + self.dryrun = self.config.getboolean('general', 'dry-run') self.quicknum = 0 if self.refreshperiod == 0.0: self.refreshperiod = None @@ -120,22 +92,74 @@ class Account(CustomConfig.ConfigHelperMixin): def getlocaleval(self): return self.localeval + # Interface from CustomConfig.ConfigHelperMixin def getconfig(self): return self.config def getname(self): return self.name + def __str__(self): + return self.name + + def getaccountmeta(self): + return os.path.join(self.metadatadir, 'Account-' + self.name) + + # Interface from CustomConfig.ConfigHelperMixin def getsection(self): return 'Account ' + self.getname() - def sleeper(self, siglistener): - """Sleep handler. Returns same value as UIBase.sleep: - 0 if timeout expired, 1 if there was a request to cancel the timer, - and 2 if there is a request to abort the program. + @classmethod + def set_abort_event(cls, config, signum): + """Set skip sleep/abort event for all accounts. + + If we want to skip a current (or the next) sleep, or if we want + to abort an autorefresh loop, the main thread can use + set_abort_event() to send the corresponding signal. Signum = 1 + implies that we want all accounts to abort or skip the current + or next sleep phase. Signum = 2 will end the autorefresh loop, + ie all accounts will return after they finished a sync. signum=3 + means, abort NOW, e.g. on SIGINT or SIGTERM. + + This is a class method, it will send the signal to all accounts. + """ + + if signum == 1: + # resync signal, set config option for all accounts + for acctsection in getaccountlist(config): + config.set('Account ' + acctsection, "skipsleep", '1') + elif signum == 2: + # don't autorefresh anymore + cls.abort_soon_signal.set() + elif signum == 3: + # abort ASAP + cls.abort_NOW_signal.set() + + def get_abort_event(self): + """Checks if an abort signal had been sent. + + If the 'skipsleep' config option for this account had been set, + with `set_abort_event(config, 1)` it will get cleared in this + function. Ie, we will only skip one sleep and not all. + + :returns: True, if the main thread had called + :meth:`set_abort_event` earlier, otherwise 'False'. + """ + + skipsleep = self.getconfboolean("skipsleep", 0) + if skipsleep: + self.config.set(self.getsection(), "skipsleep", '0') + return skipsleep or Account.abort_soon_signal.is_set() or \ + Account.abort_NOW_signal.is_set() + + def _sleeper(self): + """Sleep if the account is set to autorefresh. + + :returns: 0:timeout expired, 1: canceled the timer, + 2:request to abort the program, + 100: if configured to not sleep at all. + """ - Also, returns 100 if configured to not sleep at all.""" - if not self.refreshperiod: return 100 @@ -148,78 +172,133 @@ class Account(CustomConfig.ConfigHelperMixin): for item in kaobjs: item.startkeepalive() - + refreshperiod = int(self.refreshperiod * 60) -# try: -# sleepresult = siglistener.get_nowait() -# # retrieved signal before sleep started -# if sleepresult == 1: -# # catching signal 1 here means folders were cleared before signal was posted -# pass -# except Empty: -# sleepresult = self.ui.sleep(refreshperiod, siglistener) - sleepresult = self.ui.sleep(refreshperiod, siglistener) - if sleepresult == 1: - self.quicknum = 0 + sleepresult = self.ui.sleep(refreshperiod, self) # Cancel keepalive for item in kaobjs: item.stopkeepalive() - return sleepresult - -class AccountSynchronizationMixin: - def syncrunner(self, siglistener): - self.ui.registerthread(self.name) - self.ui.acct(self.name) - accountmetadata = self.getaccountmeta() - if not os.path.exists(accountmetadata): - os.mkdir(accountmetadata, 0700) - self.remoterepos = offlineimap.repository.Base.LoadRepository(self.getconf('remoterepository'), self, 'remote') + if sleepresult: + if Account.abort_soon_signal.is_set() or \ + Account.abort_NOW_signal.is_set(): + return 2 + self.quicknum = 0 + return 1 + return 0 - # Connect to the local repository. - self.localrepos = offlineimap.repository.Base.LoadRepository(self.getconf('localrepository'), self, 'local') + def serverdiagnostics(self): + """Output diagnostics for all involved repositories.""" - # Connect to the local cache. - self.statusrepos = offlineimap.repository.LocalStatus.LocalStatusRepository(self.getconf('localrepository'), self) + remote_repo = Repository(self, 'remote') + local_repo = Repository(self, 'local') + #status_repo = Repository(self, 'status') + self.ui.serverdiagnostics(remote_repo, 'Remote') + self.ui.serverdiagnostics(local_repo, 'Local') + #self.ui.serverdiagnostics(statusrepos, 'Status') - #might need changes here to ensure that one account sync does not crash others... - if not self.refreshperiod: + +class SyncableAccount(Account): + """A syncable email account connecting 2 repositories. + + Derives from :class:`accounts.Account` but contains the additional + functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`, + used for syncing.""" + + def __init__(self, *args, **kwargs): + Account.__init__(self, *args, **kwargs) + self._lockfd = None + self._lockfilepath = os.path.join( + self.config.getmetadatadir(), "%s.lock"% self) + + def __lock(self): + """Lock the account, throwing an exception if it is locked already.""" + + self._lockfd = open(self._lockfilepath, 'w') + try: + fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB) + except NameError: + #fcntl not available (Windows), disable file locking... :( + pass + except IOError: + self._lockfd.close() + raise OfflineImapError("Could not lock account %s. Is another " + "instance using this account?"% self, + OfflineImapError.ERROR.REPO), None, exc_info()[2] + + def _unlock(self): + """Unlock the account, deleting the lock file""" + + #If we own the lock file, delete it + if self._lockfd and not self._lockfd.closed: + self._lockfd.close() try: - try: - self.sync(siglistener) - except (KeyboardInterrupt, SystemExit): - raise - except: - self.ui.warn("Error occured attempting to sync account " + self.name \ - + ": " + traceback.format_exc()) - finally: - self.ui.acctdone(self.name) + os.unlink(self._lockfilepath) + except OSError: + pass # Failed to delete for some reason. + def syncrunner(self): + self.ui.registerthread(self) + try: + accountmetadata = self.getaccountmeta() + if not os.path.exists(accountmetadata): + os.mkdir(accountmetadata, 0o700) + + self.remoterepos = Repository(self, 'remote') + self.localrepos = Repository(self, 'local') + self.statusrepos = Repository(self, 'status') + except OfflineImapError as e: + self.ui.error(e, exc_info()[2]) + if e.severity >= OfflineImapError.ERROR.CRITICAL: + raise return - - looping = 1 + # Loop account sync if needed (bail out after 3 failures) + looping = 3 while looping: + self.ui.acct(self) try: - try: - self.sync(siglistener) - except (KeyboardInterrupt, SystemExit): - raise - except: - self.ui.warn("Error occured attempting to sync account " + self.name \ - + ": " + traceback.format_exc()) + self.__lock() + self.__sync() + except (KeyboardInterrupt, SystemExit): + raise + except OfflineImapError as e: + # Stop looping and bubble up Exception if needed. + if e.severity >= OfflineImapError.ERROR.REPO: + if looping: + looping -= 1 + if e.severity >= OfflineImapError.ERROR.CRITICAL: + raise + self.ui.error(e, exc_info()[2]) + except Exception as e: + self.ui.error(e, exc_info()[2], msg= + "While attempting to sync account '%s'"% self) + else: + # after success sync, reset the looping counter to 3 + if self.refreshperiod: + looping = 3 finally: - looping = self.sleeper(siglistener) != 2 - self.ui.acctdone(self.name) + self.ui.acctdone(self) + self._unlock() + if looping and self._sleeper() >= 2: + looping = 0 + def get_local_folder(self, remotefolder): + """Return the corresponding local folder for a given remotefolder.""" - def getaccountmeta(self): - return os.path.join(self.metadatadir, 'Account-' + self.name) + return self.localrepos.getfolder( + remotefolder.getvisiblename(). + replace(self.remoterepos.getsep(), self.localrepos.getsep())) - def sync(self, siglistener): - # We don't need an account lock because syncitall() goes through - # each account once, then waits for all to finish. + def __sync(self): + """Synchronize the account once, then return. + + Assumes that `self.remoterepos`, `self.localrepos`, and + `self.statusrepos` has already been populated, so it should only + be called from the :meth:`syncrunner` function.""" + + folderthreads = [] hook = self.getconf('presynchook', '') self.callhook(hook) @@ -241,143 +320,278 @@ class AccountSynchronizationMixin: remoterepos = self.remoterepos localrepos = self.localrepos statusrepos = self.statusrepos - self.ui.syncfolders(remoterepos, localrepos) - remoterepos.syncfoldersto(localrepos, [statusrepos]) - siglistener.addfolders(remoterepos.getfolders(), bool(self.refreshperiod), quick) + # init repos with list of folders, so we have them (and the + # folder delimiter etc) + remoterepos.getfolders() + localrepos.getfolders() - while True: - folderthreads = [] - for remotefolder, quick in siglistener.queuedfolders(): + remoterepos.sync_folder_structure(localrepos, statusrepos) + # replicate the folderstructure between REMOTE to LOCAL + if not localrepos.getconfboolean('readonly', False): + self.ui.syncfolders(remoterepos, localrepos) + + # iterate through all folders on the remote repo and sync + for remotefolder in remoterepos.getfolders(): + # check for CTRL-C or SIGTERM + if Account.abort_NOW_signal.is_set(): break + + if not remotefolder.sync_this: + self.ui.debug('', "Not syncing filtered folder '%s'" + "[%s]"% (remotefolder, remoterepos)) + continue # Ignore filtered folder + localfolder = self.get_local_folder(remotefolder) + if not localfolder.sync_this: + self.ui.debug('', "Not syncing filtered folder '%s'" + "[%s]"% (localfolder, localfolder.repository)) + continue # Ignore filtered folder + if not globals.options.singlethreading: thread = InstanceLimitedThread(\ instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, - name = "Folder sync %s[%s]" % \ - (self.name, remotefolder.getvisiblename()), - args = (self.name, remoterepos, remotefolder, localrepos, - statusrepos, quick)) - thread.setDaemon(1) + name = "Folder %s [acc: %s]"% (remotefolder.getexplainedname(), self), + args = (self, remotefolder, quick)) thread.start() folderthreads.append(thread) - threadutil.threadsreset(folderthreads) - if siglistener.clearfolders(): - break - mbnames.write() + else: + syncfolder(self, remotefolder, quick) + # wait for all threads to finish + for thr in folderthreads: + thr.join() + # Write out mailbox names if required and not in dry-run mode + if not self.dryrun: + mbnames.write(False) localrepos.forgetfolders() remoterepos.forgetfolders() + except: + #error while syncing. Drop all connections that we have, they + #might be bogus by now (e.g. after suspend) + localrepos.dropconnections() + remoterepos.dropconnections() + raise + else: + # sync went fine. Hold or drop depending on config localrepos.holdordropconnections() remoterepos.holdordropconnections() - finally: - pass hook = self.getconf('postsynchook', '') self.callhook(hook) def callhook(self, cmd): + # check for CTRL-C or SIGTERM and run postsynchook + if Account.abort_NOW_signal.is_set(): + return if not cmd: return try: self.ui.callhook("Calling hook: " + cmd) + if self.dryrun: # don't if we are in dry-run mode + return p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) r = p.communicate() - self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n" % r) - self.ui.callhook("Hook return code: %d" % p.returncode) - except: - self.ui.warn("Exception occured while calling hook") - -class SyncableAccount(Account, AccountSynchronizationMixin): - pass + self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n"% r) + self.ui.callhook("Hook return code: %d"% p.returncode) + except (KeyboardInterrupt, SystemExit): + raise + except Exception as e: + self.ui.error(e, exc_info()[2], msg="Calling hook") -def syncfolder(accountname, remoterepos, remotefolder, localrepos, - statusrepos, quick): - global mailboxes - ui = getglobalui() - ui.registerthread(accountname) - try: - # Load local folder. - localfolder = localrepos.\ - getfolder(remotefolder.getvisiblename().\ - replace(remoterepos.getsep(), localrepos.getsep())) - # Write the mailboxes - mbnames.add(accountname, localfolder.getvisiblename()) +def syncfolder(account, remotefolder, quick): + """Synchronizes given remote folder for the specified account. - # Load status folder. - statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ - replace(remoterepos.getsep(), - statusrepos.getsep())) - if localfolder.getuidvalidity() == None: - # This is a new folder, so delete the status cache to be sure - # we don't have a conflict. - statusfolder.deletemessagelist() + Filtered folders on the remote side will not invoke this function.""" - statusfolder.cachemessagelist() - - if quick: - if not localfolder.quickchanged(statusfolder) \ - and not remotefolder.quickchanged(statusfolder): - ui.skippingfolder(remotefolder) - localrepos.restore_atime() - return - - # Load local folder - ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) - ui.loadmessagelist(localrepos, localfolder) - localfolder.cachemessagelist() - ui.messagelistloaded(localrepos, localfolder, len(localfolder.getmessagelist().keys())) - - # If either the local or the status folder has messages and there is a UID - # validity problem, warn and abort. If there are no messages, UW IMAPd - # loses UIDVALIDITY. But we don't really need it if both local folders are - # empty. So, in that case, just save it off. - if len(localfolder.getmessagelist()) or len(statusfolder.getmessagelist()): - if not localfolder.isuidvalidityok(): + def check_uid_validity(localfolder, remotefolder, statusfolder): + # If either the local or the status folder has messages and + # there is a UID validity problem, warn and abort. If there are + # no messages, UW IMAPd loses UIDVALIDITY. But we don't really + # need it if both local folders are empty. So, in that case, + # just save it off. + if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount() > 0: + if not localfolder.check_uidvalidity(): ui.validityproblem(localfolder) - localrepos.restore_atime() + localfolder.repository.restore_atime() return - if not remotefolder.isuidvalidityok(): + if not remotefolder.check_uidvalidity(): ui.validityproblem(remotefolder) localrepos.restore_atime() return else: - localfolder.saveuidvalidity() - remotefolder.saveuidvalidity() + # Both folders empty, just save new UIDVALIDITY + localfolder.save_uidvalidity() + remotefolder.save_uidvalidity() - # Load remote folder. - ui.loadmessagelist(remoterepos, remotefolder) - remotefolder.cachemessagelist() - ui.messagelistloaded(remoterepos, remotefolder, - len(remotefolder.getmessagelist().keys())) + def save_min_uid(folder, min_uid): + uidfile = folder.get_min_uid_file() + fd = open(uidfile, 'wt') + fd.write(str(min_uid) + "\n") + fd.close() + def cachemessagelists_upto_date(localfolder, remotefolder, date): + """ Returns messages with uid > min(uids of messages newer than date).""" + localfolder.cachemessagelist(min_date=date) + check_uid_validity(localfolder, remotefolder, statusfolder) + # local messagelist had date restriction applied already. Restrict + # sync to messages with UIDs >= min_uid from this list. # + # local messagelist might contain new messages (with uid's < 0). + positive_uids = filter( + lambda uid: uid > 0, localfolder.getmessageuidlist()) + if len(positive_uids) > 0: + remotefolder.cachemessagelist(min_uid=min(positive_uids)) + else: + # No messages with UID > 0 in range in localfolder. + # date restriction was applied with respect to local dates but + # remote folder timezone might be different from local, so be + # safe and make sure the range isn't bigger than in local. + remotefolder.cachemessagelist( + min_date=time.gmtime(time.mktime(date) + 24*60*60)) - if not statusfolder.isnewfolder(): - # Delete local copies of remote messages. This way, - # if a message's flag is modified locally but it has been - # deleted remotely, we'll delete it locally. Otherwise, we - # try to modify a deleted message's flags! This step - # need only be taken if a statusfolder is present; otherwise, - # there is no action taken *to* the remote repository. + def cachemessagelists_startdate(new, partial, date): + """ Retrieve messagelists when startdate has been set for + the folder 'partial'. - remotefolder.syncmessagesto_delete(localfolder, [localfolder, - statusfolder]) - ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) - localfolder.syncmessagesto(statusfolder, [remotefolder, statusfolder]) + Idea: suppose you want to clone the messages after date in one + account (partial) to a new one (new). If new is empty, then copy + messages in partial newer than date to new, and keep track of the + min uid. On subsequent syncs, sync all the messages in new against + those after that min uid in partial. This is a partial replacement + for maxage in the IMAP-IMAP sync case, where maxage doesn't work: + the UIDs of the messages in localfolder might not be in the same + order as those of corresponding messages in remotefolder, so if L in + local corresponds to R in remote, the ranges [L, ...] and [R, ...] + might not correspond. But, if we're cloning a folder into a new one, + [min_uid, ...] does correspond to [1, ...]. + + This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead. + """ + + new.cachemessagelist() + min_uid = partial.retrieve_min_uid() + if min_uid == None: # min_uid file didn't exist + if len(new.getmessageuidlist()) > 0: + raise OfflineImapError("To use startdate on Repository %s, " + "Repository %s must be empty"% + (partial.repository.name, new.repository.name), + OfflineImapError.ERROR.MESSAGE) + else: + partial.cachemessagelist(min_date=date) + # messagelist.keys() instead of getuidmessagelist() because in + # the UID mapped case we want the actual local UIDs, not their + # remote counterparts + positive_uids = filter( + lambda uid: uid > 0, partial.messagelist.keys()) + if len(positive_uids) > 0: + min_uid = min(positive_uids) + else: + min_uid = 1 + save_min_uid(partial, min_uid) + else: + partial.cachemessagelist(min_uid=min_uid) + + + remoterepos = account.remoterepos + localrepos = account.localrepos + statusrepos = account.statusrepos + + ui = getglobalui() + ui.registerthread(account) + try: + # Load local folder. + localfolder = account.get_local_folder(remotefolder) + + # Write the mailboxes + mbnames.add(account.name, localfolder.getname(), + localrepos.getlocalroot()) + + # Load status folder. + statusfolder = statusrepos.getfolder(remotefolder.getvisiblename(). + replace(remoterepos.getsep(), statusrepos.getsep())) + + if localfolder.get_uidvalidity() == None: + # This is a new folder, so delete the status cache to be + # sure we don't have a conflict. + # TODO: This does not work. We always return a value, need + # to rework this... + statusfolder.deletemessagelist() + + statusfolder.cachemessagelist() + + + # Load local folder. + ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) + + # Retrieve messagelists, taking into account age-restriction + # options + maxage = localfolder.getmaxage() + localstart = localfolder.getstartdate() + remotestart = remotefolder.getstartdate() + if (maxage != None) + (localstart != None) + (remotestart != None) > 1: + raise OfflineImapError("You can set at most one of the " + "following: maxage, startdate (for the local folder), " + "startdate (for the remote folder)", + OfflineImapError.ERROR.REPO), None, exc_info()[2] + if (maxage != None or localstart or remotestart) and quick: + # IMAP quickchanged isn't compatible with options that + # involve restricting the messagelist, since the "quick" + # check can only retrieve a full list of UIDs in the folder. + ui.warn("Quick syncs (-q) not supported in conjunction " + "with maxage or startdate; ignoring -q.") + if maxage != None: + cachemessagelists_upto_date(localfolder, remotefolder, maxage) + elif localstart != None: + cachemessagelists_startdate(remotefolder, localfolder, + localstart) + check_uid_validity(localfolder, remotefolder, statusfolder) + elif remotestart != None: + cachemessagelists_startdate(localfolder, remotefolder, + remotestart) + check_uid_validity(localfolder, remotefolder, statusfolder) + else: + localfolder.cachemessagelist() + if quick: + if (not localfolder.quickchanged(statusfolder) and + not remotefolder.quickchanged(statusfolder)): + ui.skippingfolder(remotefolder) + localrepos.restore_atime() + return + check_uid_validity(localfolder, remotefolder, statusfolder) + remotefolder.cachemessagelist() # Synchronize remote changes. - ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) - remotefolder.syncmessagesto(localfolder, [localfolder, statusfolder]) + if not localrepos.getconfboolean('readonly', False): + ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) + remotefolder.syncmessagesto(localfolder, statusfolder) + else: + ui.debug('imap', "Not syncing to read-only repository '%s'" \ + % localrepos.getname()) + + # Synchronize local changes + if not remoterepos.getconfboolean('readonly', False): + ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) + localfolder.syncmessagesto(remotefolder, statusfolder) + else: + ui.debug('', "Not syncing to read-only repository '%s'" \ + % remoterepos.getname()) - # Make sure the status folder is up-to-date. - ui.syncingmessages(localrepos, localfolder, statusrepos, statusfolder) - localfolder.syncmessagesto(statusfolder) statusfolder.save() localrepos.restore_atime() except (KeyboardInterrupt, SystemExit): raise - except: - ui.warn("ERROR in syncfolder for %s folder %s: %s" % \ - (accountname,remotefolder.getvisiblename(), - traceback.format_exc())) + except OfflineImapError as e: + # bubble up severe Errors, skip folder otherwise + if e.severity > OfflineImapError.ERROR.FOLDER: + raise + else: + ui.error(e, exc_info()[2], msg = "Aborting sync, folder '%s' " + "[acc: '%s']" % (localfolder, account)) + except Exception as e: + ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s"% + (account, remotefolder.getvisiblename(), traceback.format_exc())) + finally: + for folder in ["statusfolder", "localfolder", "remotefolder"]: + if folder in locals(): + locals()[folder].dropmessagelistcache() diff --git a/offlineimap/emailutil.py b/offlineimap/emailutil.py new file mode 100644 index 0000000..0c732e2 --- /dev/null +++ b/offlineimap/emailutil.py @@ -0,0 +1,36 @@ +# Some useful functions to extract data out of emails +# Copyright (C) 2002-2015 John Goerzen & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +import email +from email.Parser import Parser as MailParser + +def get_message_date(content, header='Date'): + """Parses mail and returns resulting timestamp. + + :param header: the header to extract date from; + :returns: timestamp or `None` in the case of failure. + """ + + message = MailParser().parsestr(content, True) + dateheader = message.get(header) + # parsedate_tz returns a 10-tuple that can be passed to mktime_tz + # Will be None if missing or not in a valid format. Note that + # indexes 6, 7, and 8 of the result tuple are not usable. + datetuple = email.utils.parsedate_tz(dateheader) + if datetuple is None: + return None + return email.utils.mktime_tz(datetuple) diff --git a/offlineimap/error.py b/offlineimap/error.py new file mode 100644 index 0000000..1be64ac --- /dev/null +++ b/offlineimap/error.py @@ -0,0 +1,39 @@ +class OfflineImapError(Exception): + """An Error during offlineimap synchronization""" + + class ERROR: + """Severity level of an Exception + + * **MESSAGE**: Abort the current message, but continue with folder + * **FOLDER_RETRY**: Error syncing folder, but do retry + * **FOLDER**: Abort folder sync, but continue with next folder + * **REPO**: Abort repository sync, continue with next account + * **CRITICAL**: Immediately exit offlineimap + """ + + MESSAGE, FOLDER_RETRY, FOLDER, REPO, CRITICAL = 0, 10, 15, 20, 30 + + def __init__(self, reason, severity, errcode=None): + """ + :param reason: Human readable string suitable for logging + + :param severity: denoting which operations should be + aborted. E.g. a ERROR.MESSAGE can occur on a faulty + message, but a ERROR.REPO occurs when the server is + offline. + + :param errcode: optional number denoting a predefined error + situation (which let's us exit with a predefined exit + value). So far, no errcodes have been defined yet. + + :type severity: OfflineImapError.ERROR value""" + + self.errcode = errcode + self.severity = severity + + # 'reason' is stored in the Exception().args tuple. + super(OfflineImapError, self).__init__(reason) + + @property + def reason(self): + return self.args[0] diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 8e6a6b3..7957b83 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -1,6 +1,5 @@ # Base folder support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,57 +15,152 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import * -from offlineimap import threadutil -from offlineimap.ui import getglobalui import os.path import re -import sys -import traceback +import time +from sys import exc_info + +from offlineimap import threadutil +from offlineimap import globals +from offlineimap.ui import getglobalui +from offlineimap.error import OfflineImapError +import offlineimap.accounts + + +class BaseFolder(object): + def __init__(self, name, repository): + """ + :param name: Path & name of folder minus root or reference + :param repository: Repository() in which the folder is. + """ -class BaseFolder: - def __init__(self): - self.uidlock = Lock() self.ui = getglobalui() - + # Save original name for folderfilter operations + self.ffilter_name = name + # Top level dir name is always '' + self.root = None + self.name = name if not name == self.getsep() else '' + self.repository = repository + self.visiblename = repository.nametrans(name) + # In case the visiblename becomes '.' or '/' (top-level) we use + # '' as that is the name that e.g. the Maildir scanning will + # return for the top-level dir. + if self.visiblename == self.getsep(): + self.visiblename = '' + + self.config = repository.getconfig() + utime_from_header_global = self.config.getdefaultboolean( + "general", "utime_from_header", False) + repo = "Repository " + repository.name + self._utime_from_header = self.config.getdefaultboolean(repo, + "utime_from_header", utime_from_header_global) + + # Determine if we're running static or dynamic folder filtering + # and check filtering status + self._dynamic_folderfilter = self.config.getdefaultboolean( + repo, "dynamic_folderfilter", False) + self._sync_this = repository.should_sync_folder(self.ffilter_name) + if self._dynamic_folderfilter: + self.ui.debug('', "Running dynamic folder filtering on '%s'[%s]"% + (self.ffilter_name, repository)) + elif not self._sync_this: + self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter"% + (self.ffilter_name, repository)) + + # Passes for syncmessagesto + self.syncmessagesto_passes = [('copying messages' , self.__syncmessagesto_copy), + ('deleting messages' , self.__syncmessagesto_delete), + ('syncing flags' , self.__syncmessagesto_flags)] + def getname(self): """Returns name""" return self.name + def __str__(self): + # FIMXE: remove calls of this. We have getname(). + return self.name + + @property + def accountname(self): + """Account name as string""" + + return self.repository.accountname + + @property + def sync_this(self): + """Should this folder be synced or is it e.g. filtered out?""" + + if not self._dynamic_folderfilter: + return self._sync_this + else: + return self.repository.should_sync_folder(self.ffilter_name) + + @property + def utime_from_header(self): + return self._utime_from_header + def suggeststhreads(self): """Returns true if this folder suggests using threads for actions; false otherwise. Probably only IMAP will return true.""" return 0 def waitforthread(self): - """For threading folders, waits until there is a resource available - before firing off a thread. For all others, returns immediately.""" - pass + """Implements method that waits for thread to be usable. + Should be implemented only for folders that suggest threads.""" + raise NotImplementedError + + # XXX: we may need someting like supports_quickstatus() to check + # XXX: if user specifies 'quick' flag for folder that doesn't + # XXX: support quick status queries, so one believes that quick + # XXX: status checks will be done, but it won't really be so. + def quickchanged(self, statusfolder): + """ Runs quick check for folder changes and returns changed + status: True -- changed, False -- not changed. + + :param statusfolder: keeps track of the last known folder state. + """ + + return True def getcopyinstancelimit(self): """For threading folders, returns the instancelimitname for InstanceLimitedThreads.""" - raise NotImplementedException + + raise NotImplementedError def storesmessages(self): """Should be true for any backend that actually saves message bodies. (Almost all of them). False for the LocalStatus backend. Saves us from having to slurp up messages just for localstatus purposes.""" + return 1 def getvisiblename(self): - return self.name + """The nametrans-transposed name of the folder's name.""" + + return self.visiblename + + def getexplainedname(self): + """Name that shows both real and nametrans-mangled values.""" + + if self.name == self.visiblename: + return self.name + else: + return "%s [remote name %s]"% (self.visiblename, self.name) def getrepository(self): """Returns the repository object that this folder is within.""" + return self.repository def getroot(self): """Returns the root of the folder, in a folder-specific fashion.""" + return self.root def getsep(self): """Returns the separator for this folder type.""" + return self.sep def getfullname(self): @@ -74,26 +168,47 @@ class BaseFolder: return self.getroot() + self.getsep() + self.getname() else: return self.getname() - - def getfolderbasename(self): - foldername = self.getname() - foldername = foldername.replace(self.repository.getsep(), '.') - foldername = re.sub('/\.$', '/dot', foldername) - foldername = re.sub('^\.$', 'dot', foldername) - return foldername - def isuidvalidityok(self): - if self.getsaveduidvalidity() != None: - return self.getsaveduidvalidity() == self.getuidvalidity() + def getfolderbasename(self): + """Return base file name of file to store Status/UID info in.""" + + if not self.name: + basename = '.' + else: # Avoid directory hierarchies and file names such as '/'. + basename = self.name.replace('/', '.') + # Replace with literal 'dot' if final path name is '.' as '.' is + # an invalid file name. + basename = re.sub('(^|\/)\.$','\\1dot', basename) + return basename + + def check_uidvalidity(self): + """Tests if the cached UIDVALIDITY match the real current one + + If required it saves the UIDVALIDITY value. In this case the + function is not threadsafe. So don't attempt to call it from + concurrent threads. + + :returns: Boolean indicating the match. Returns True in case it + implicitely saved the UIDVALIDITY.""" + + if self.get_saveduidvalidity() != None: + return self.get_saveduidvalidity() == self.get_uidvalidity() else: - self.saveuidvalidity() - return 1 + self.save_uidvalidity() + return True def _getuidfilename(self): + """provides UIDVALIDITY cache filename for class internal purposes.""" + return os.path.join(self.repository.getuiddir(), self.getfolderbasename()) - - def getsaveduidvalidity(self): + + def get_saveduidvalidity(self): + """Return the previously cached UIDVALIDITY value + + :returns: UIDVALIDITY as (long) number or None, if None had been + saved yet.""" + if hasattr(self, '_base_saved_uidvalidity'): return self._base_saved_uidvalidity uidfilename = self._getuidfilename() @@ -105,325 +220,788 @@ class BaseFolder: file.close() return self._base_saved_uidvalidity - def saveuidvalidity(self): - newval = self.getuidvalidity() - uidfilename = self._getuidfilename() - self.uidlock.acquire() - try: - file = open(uidfilename + ".tmp", "wt") - file.write("%d\n" % newval) - file.close() - os.rename(uidfilename + ".tmp", uidfilename) - self._base_saved_uidvalidity = newval - finally: - self.uidlock.release() + def save_uidvalidity(self): + """Save the UIDVALIDITY value of the folder to the cache - def getuidvalidity(self): - raise NotImplementedException + This function is not threadsafe, so don't attempt to call it + from concurrent threads.""" + + newval = self.get_uidvalidity() + uidfilename = self._getuidfilename() + + with open(uidfilename + ".tmp", "wt") as file: + file.write("%d\n"% newval) + os.rename(uidfilename + ".tmp", uidfilename) + self._base_saved_uidvalidity = newval + + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value + + This function needs to be implemented by each Backend + :returns: UIDVALIDITY as a (long) number""" + + raise NotImplementedError def cachemessagelist(self): """Reads the message list from disk or network and stores it in memory for later use. This list will not be re-read from disk or memory unless this function is called again.""" - raise NotImplementedException + + raise NotImplementedError + + def ismessagelistempty(self): + """Empty everythings we know about messages.""" + + if len(self.messagelist.keys()) < 1: + return True + return False + + def dropmessagelistcache(self): + """Empty everythings we know about messages.""" + + self.messagelist = {} def getmessagelist(self): """Gets the current message list. + You must call cachemessagelist() before calling this function!""" - raise NotImplementedException + + raise NotImplementedError + + def msglist_item_initializer(self, uid): + """Returns value for empty messagelist element with given UID. + + This function must initialize all fields of messagelist item + and must be called every time when one creates new messagelist + entry to ensure that all fields that must be present are present.""" + + raise NotImplementedError + + def uidexists(self, uid): + """Returns True if uid exists""" + + return uid in self.getmessagelist() + + def getmessageuidlist(self): + """Gets a list of UIDs. + + You may have to call cachemessagelist() before calling this function!""" + + return self.getmessagelist().keys() + + def getmessagecount(self): + """Gets the number of messages.""" + + return len(self.getmessagelist()) def getmessage(self, uid): """Returns the content of the specified message.""" - raise NotImplementedException + + raise NotImplementedError + + def getmaxage(self): + """ maxage is allowed to be either an integer or a date of the + form YYYY-mm-dd. This returns a time_struct. """ + + maxagestr = self.config.getdefault("Account %s"% + self.accountname, "maxage", None) + if maxagestr == None: + return None + # is it a number? + try: + maxage = int(maxagestr) + if maxage < 1: + raise OfflineImapError("invalid maxage value %d"% maxage, + OfflineImapError.ERROR.MESSAGE) + return time.gmtime(time.time() - 60*60*24*maxage) + except ValueError: + pass # maybe it was a date + # is it a date string? + try: + date = time.strptime(maxagestr, "%Y-%m-%d") + if date[0] < 1900: + raise OfflineImapError("maxage led to year %d. " + "Abort syncing."% date[0], + OfflineImapError.ERROR.MESSAGE) + return date + except ValueError: + raise OfflineImapError("invalid maxage value %s"% maxagestr, + OfflineImapError.ERROR.MESSAGE) + + def getmaxsize(self): + return self.config.getdefaultint("Account %s"% + self.accountname, "maxsize", None) + + def getstartdate(self): + """ Retrieve the value of the configuration option startdate """ + datestr = self.config.getdefault("Repository " + self.repository.name, + 'startdate', None) + try: + if not datestr: + return None + date = time.strptime(datestr, "%Y-%m-%d") + if date[0] < 1900: + raise OfflineImapError("startdate led to year %d. " + "Abort syncing."% date[0], + OfflineImapError.ERROR.MESSAGE) + return date + except ValueError: + raise OfflineImapError("invalid startdate value %s", + OfflineImapError.ERROR.MESSAGE) + + def get_min_uid_file(self): + startuiddir = os.path.join(self.config.getmetadatadir(), + 'Repository-' + self.repository.name, 'StartUID') + if not os.path.exists(startuiddir): + os.mkdir(startuiddir, 0o700) + return os.path.join(startuiddir, self.getfolderbasename()) + + def retrieve_min_uid(self): + uidfile = self.get_min_uid_file() + if not os.path.exists(uidfile): + return None + try: + fd = open(uidfile, 'rt') + min_uid = long(fd.readline().strip()) + fd.close() + return min_uid + except: + raise IOError("Can't read %s"% uidfile) + def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. - If the uid is < 0, the backend should assign a new uid and return it. - If the backend cannot assign a new uid, it returns the uid passed in - WITHOUT saving the message. + If the uid is < 0: The backend should assign a new uid and + return it. In case it cannot assign a new uid, it returns + the negative uid passed in WITHOUT saving the message. - If the backend CAN assign a new uid, but cannot find out what this UID - is (as is the case with many IMAP servers), it returns 0 but DOES save - the message. - - IMAP backend should be the only one that can assign a new uid. + If the backend CAN assign a new uid, but cannot find out what + this UID is (as is the case with some IMAP servers), it + returns 0 but DOES save the message. + + IMAP backend should be the only one that can assign a new + uid. If the uid is > 0, the backend should set the uid to this, if it can. - If it cannot set the uid to that, it will save it anyway. - It will return the uid assigned in any case. - """ - raise NotImplementedException + If it cannot set the uid to that, it will save it anyway. + It will return the uid assigned in any case. + + Note that savemessage() does not check against dryrun settings, + so you need to ensure that savemessage is never called in a + dryrun mode.""" + + raise NotImplementedError def getmessagetime(self, uid): """Return the received time for the specified message.""" - raise NotImplementedException + + raise NotImplementedError + + def getmessagemtime(self, uid): + """Returns the message modification time of the specified message.""" + + raise NotImplementedError def getmessageflags(self, uid): """Returns the flags for the specified message.""" - raise NotImplementedException + + raise NotImplementedError def savemessageflags(self, uid, flags): - """Sets the specified message's flags to the given set.""" - raise NotImplementedException + """Sets the specified message's flags to the given set. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + + raise NotImplementedError def addmessageflags(self, uid, flags): """Adds the specified flags to the message's flag set. If a given - flag is already present, it will not be duplicated.""" - newflags = self.getmessageflags(uid) - for flag in flags: - if not flag in newflags: - newflags.append(flag) - newflags.sort() + flag is already present, it will not be duplicated. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + + :param flags: A set() of flags""" + + newflags = self.getmessageflags(uid) | flags self.savemessageflags(uid, newflags) def addmessagesflags(self, uidlist, flags): + """Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + for uid in uidlist: - self.addmessageflags(uid, flags) + if self.uidexists(uid): + self.addmessageflags(uid, flags) def deletemessageflags(self, uid, flags): """Removes each flag given from the message's flag set. If a given - flag is already removed, no action will be taken for that flag.""" - newflags = self.getmessageflags(uid) - for flag in flags: - if flag in newflags: - newflags.remove(flag) - newflags.sort() + flag is already removed, no action will be taken for that flag. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + + newflags = self.getmessageflags(uid) - flags self.savemessageflags(uid, newflags) def deletemessagesflags(self, uidlist, flags): + """ + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + for uid in uidlist: self.deletemessageflags(uid, flags) + def getmessagelabels(self, uid): + """Returns the labels for the specified message.""" + + raise NotImplementedError + + def savemessagelabels(self, uid, labels, ignorelabels=set(), mtime=0): + """Sets the specified message's labels to the given set. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + + raise NotImplementedError + + def addmessagelabels(self, uid, labels): + """Adds the specified labels to the message's labels set. If a given + label is already present, it will not be duplicated. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + + :param labels: A set() of labels""" + + newlabels = self.getmessagelabels(uid) | labels + self.savemessagelabels(uid, newlabels) + + def addmessageslabels(self, uidlist, labels): + """Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + + for uid in uidlist: + self.addmessagelabels(uid, labels) + + def deletemessagelabels(self, uid, labels): + """Removes each label given from the message's label set. If a given + label is already removed, no action will be taken for that label. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + + newlabels = self.getmessagelabels(uid) - labels + self.savemessagelabels(uid, newlabels) + + def deletemessageslabels(self, uidlist, labels): + """ + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + + for uid in uidlist: + self.deletemessagelabels(uid, labels) + + def addmessageheader(self, content, linebreak, headername, headervalue): + """Adds new header to the provided message. + + WARNING: This function is a bit tricky, and modifying it in the wrong way, + may easily lead to data-loss. + + Arguments: + - content: message content, headers and body as a single string + - linebreak: string that carries line ending + - headername: name of the header to add + - headervalue: value of the header to add + + .. note:: + + The following documentation will not get displayed correctly after being + processed by Sphinx. View the source of this method to read it. + + This has to deal with strange corner cases where the header is + missing or empty. Here are illustrations for all the cases, + showing where the header gets inserted and what the end result + is. In each illustration, '+' means the added contents. Note + that these examples assume LF for linebreak, not CRLF, so '\n' + denotes a linebreak and '\n\n' corresponds to the transition + between header and body. However if the linebreak parameter + is set to '\r\n' then you would have to substitute '\r\n' for + '\n' in the below examples. + + * Case 1: No '\n\n', leading '\n' + + +X-Flying-Pig-Header: i am here\n + \n + This is the body\n + next line\n + + * Case 2: '\n\n' at position 0 + + +X-Flying-Pig-Header: i am here + \n + \n + This is the body\n + next line\n + + * Case 3: No '\n\n', no leading '\n' + + +X-Flying-Pig-Header: i am here\n + +\n + This is the body\n + next line\n + + * Case 4: '\n\n' at non-zero position + + Subject: Something wrong with OI\n + From: some@person.at + +\nX-Flying-Pig-Header: i am here + \n + \n + This is the body\n + next line\n + """ + + self.ui.debug('', 'addmessageheader: called to add %s: %s'% + (headername, headervalue)) + + insertionpoint = content.find(linebreak * 2) + if insertionpoint == -1: + self.ui.debug('', 'addmessageheader: headers were missing') + else: + self.ui.debug('', 'addmessageheader: headers end at position %d' % insertionpoint) + mark = '==>EOH<==' + contextstart = max(0, insertionpoint - 100) + contextend = min(len(content), insertionpoint + 100) + self.ui.debug('', 'addmessageheader: header/body transition context (marked by %s): %s' % + (mark, repr(content[contextstart:insertionpoint]) + \ + mark + repr(content[insertionpoint:contextend]))) + + # Hoping for case #4 + prefix = linebreak + suffix = '' + # Case #2 + if insertionpoint == 0: + prefix = '' + suffix = '' + # Either case #1 or #3 + elif insertionpoint == -1: + prefix = '' + suffix = linebreak + insertionpoint = 0 + # Case #3: when body starts immediately, without preceding '\n' + # (this shouldn't happen with proper mail messages, but + # we seen many broken ones), we should add '\n' to make + # new (and the only header, in this case) to be properly + # separated from the message body. + if content[0:len(linebreak)] != linebreak: + suffix = suffix + linebreak + + self.ui.debug('', 'addmessageheader: insertionpoint = %d'% insertionpoint) + headers = content[0:insertionpoint] + self.ui.debug('', 'addmessageheader: headers = %s'% repr(headers)) + new_header = prefix + ("%s: %s" % (headername, headervalue)) + suffix + self.ui.debug('', 'addmessageheader: new_header = ' + repr(new_header)) + return headers + new_header + content[insertionpoint:] + + + def __find_eoh(self, content): + """ Searches for the point where mail headers end. + Either double '\n', or end of string. + + Arguments: + - content: contents of the message to search in + Returns: position of the first non-header byte. + """ + + eoh_cr = content.find('\n\n') + if eoh_cr == -1: + eoh_cr = len(content) + + return eoh_cr + + + def getmessageheader(self, content, name): + """Searches for the first occurence of the given header and returns + its value. Header name is case-insensitive. + + Arguments: + - contents: message itself + - name: name of the header to be searched + + Returns: header value or None if no such header was found + """ + + self.ui.debug('', 'getmessageheader: called to get %s'% name) + eoh = self.__find_eoh(content) + self.ui.debug('', 'getmessageheader: eoh = %d'% eoh) + headers = content[0:eoh] + self.ui.debug('', 'getmessageheader: headers = %s'% repr(headers)) + + m = re.search('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE) + if m: + return m.group(1).strip() + else: + return None + + + def getmessageheaderlist(self, content, name): + """Searches for the given header and returns a list of values for + that header. + + Arguments: + - contents: message itself + - name: name of the header to be searched + + Returns: list of header values or emptylist if no such header was found + """ + + self.ui.debug('', 'getmessageheaderlist: called to get %s' % name) + eoh = self.__find_eoh(content) + self.ui.debug('', 'getmessageheaderlist: eoh = %d' % eoh) + headers = content[0:eoh] + self.ui.debug('', 'getmessageheaderlist: headers = %s' % repr(headers)) + + return re.findall('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE) + + + def deletemessageheaders(self, content, header_list): + """Deletes headers in the given list from the message content. + + Arguments: + - content: message itself + - header_list: list of headers to be deleted or just the header name + + We expect our message to have '\n' as line endings. + """ + + if type(header_list) != type([]): + header_list = [header_list] + self.ui.debug('', 'deletemessageheaders: called to delete %s'% (header_list)) + + if not len(header_list): return content + + eoh = self.__find_eoh(content) + self.ui.debug('', 'deletemessageheaders: end of headers = %d'% eoh) + headers = content[0:eoh] + rest = content[eoh:] + self.ui.debug('', 'deletemessageheaders: headers = %s'% repr(headers)) + new_headers = [] + for h in headers.split('\n'): + keep_it = True + for trim_h in header_list: + if len(h) > len(trim_h) and h[0:len(trim_h)+1] == (trim_h + ":"): + keep_it = False + break + if keep_it: new_headers.append(h) + + return ('\n'.join(new_headers) + rest) + + + + + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + + If the backend supports it (IMAP does not). + + :param new_uid: (optional) If given, the old UID will be changed + to a new UID. This allows backends efficient renaming of + messages if the UID has changed.""" + + raise NotImplementedError + def deletemessage(self, uid): - raise NotImplementedException + """Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + + raise NotImplementedError def deletemessages(self, uidlist): + """Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + for uid in uidlist: self.deletemessage(uid) - def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1): - if register: - self.ui.registerthread(self.getaccountname()) - self.ui.copyingmessage(uid, self, applyto) - successobject = None - successuid = None - message = self.getmessage(uid) - flags = self.getmessageflags(uid) - rtime = self.getmessagetime(uid) - for tryappend in applyto: - successuid = tryappend.savemessage(uid, message, flags, rtime) - if successuid >= 0: - successobject = tryappend - break - # Did we succeed? - if successobject != None: - if successuid: # Only if IMAP actually assigned a UID - # Copy the message to the other remote servers. - for appendserver in \ - [x for x in applyto if x != successobject]: - appendserver.savemessage(successuid, message, flags, rtime) - # Copy to its new name on the local server and delete - # the one without a UID. - self.savemessage(successuid, message, flags, rtime) - self.deletemessage(uid) # It'll be re-downloaded. - else: - # Did not find any server to take this message. Ignore. - pass - + def copymessageto(self, uid, dstfolder, statusfolder, register = 1): + """Copies a message from self to dst if needed, updating the status - def syncmessagesto_neguid(self, dest, applyto): - """Pass 1 of folder synchronization. + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. - Look for messages in self with a negative uid. These are messages in - Maildirs that were not added by us. Try to add them to the dests, - and once that succeeds, get the UID, add it to the others for real, - add it to local for real, and delete the fake one.""" + :param uid: uid of the message to be copied. + :param dstfolder: A BaseFolder-derived instance + :param statusfolder: A LocalStatusFolder instance + :param register: whether we should register a new thread." + :returns: Nothing on success, or raises an Exception.""" - uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0] - threads = [] - - usethread = None - if applyto != None: - usethread = applyto[0] - - for uid in uidlist: - if usethread and usethread.suggeststhreads(): - usethread.waitforthread() - thread = threadutil.InstanceLimitedThread(\ - usethread.getcopyinstancelimit(), - target = self.syncmessagesto_neguid_msg, - name = "New msg sync from %s" % self.getvisiblename(), - args = (uid, dest, applyto)) - thread.setDaemon(1) - thread.start() - threads.append(thread) - else: - self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0) - for thread in threads: - thread.join() - - def copymessageto(self, uid, applyto, register = 1): # Sometimes, it could be the case that if a sync takes awhile, # a message might be deleted from the maildir before it can be # synced to the status cache. This is only a problem with # self.getmessage(). So, don't call self.getmessage unless # really needed. + if register: # output that we start a new thread + self.ui.registerthread(self.repository.account) + try: - if register: - self.ui.registerthread(self.getaccountname()) - self.ui.copyingmessage(uid, self, applyto) - message = '' - # If any of the destinations actually stores the message body, - # load it up. - - for object in applyto: - if object.storesmessages(): - message = self.getmessage(uid) - break + message = None flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) - for object in applyto: - newuid = object.savemessage(uid, message, flags, rtime) - if newuid > 0 and newuid != uid: - # Change the local uid. - self.savemessage(newuid, message, flags, rtime) - self.deletemessage(uid) - uid = newuid - except (KeyboardInterrupt): + + # If any of the destinations actually stores the message body, + # load it up. + if dstfolder.storesmessages(): + message = self.getmessage(uid) + # Succeeded? -> IMAP actually assigned a UID. If newid + # remained negative, no server was willing to assign us an + # UID. If newid is 0, saving succeeded, but we could not + # retrieve the new UID. Ignore message in this case. + new_uid = dstfolder.savemessage(uid, message, flags, rtime) + if new_uid > 0: + if new_uid != uid: + # Got new UID, change the local uid to match the new one. + self.change_message_uid(uid, new_uid) + statusfolder.deletemessage(uid) + # Got new UID, change the local uid. + # Save uploaded status in the statusfolder + statusfolder.savemessage(new_uid, message, flags, rtime) + elif new_uid == 0: + # Message was stored to dstfolder, but we can't find it's UID + # This means we can't link current message to the one created + # in IMAP. So we just delete local message and on next run + # we'll sync it back + # XXX This could cause infinite loop on syncing between two + # IMAP servers ... + self.deletemessage(uid) + else: + raise OfflineImapError("Trying to save msg (uid %d) on folder " + "%s returned invalid uid %d"% (uid, dstfolder.getvisiblename(), + new_uid), OfflineImapError.ERROR.MESSAGE) + except (KeyboardInterrupt): # bubble up CTRL-C raise - except: - self.ui.warn("ERROR attempting to copy message " + str(uid) \ - + " for account " + self.getaccountname() + ":" + traceback.format_exc()) - + except OfflineImapError as e: + if e.severity > OfflineImapError.ERROR.MESSAGE: + raise # bubble severe errors up + self.ui.error(e, exc_info()[2]) + except Exception as e: + self.ui.error(e, exc_info()[2], + msg = "Copying message %s [acc: %s]"% (uid, self.accountname)) + raise #raise on unknown errors, so we can fix those - def syncmessagesto_copy(self, dest, applyto): - """Pass 2 of folder synchronization. + def __syncmessagesto_copy(self, dstfolder, statusfolder): + """Pass1: Copy locally existing messages not on the other side. + + This will copy messages to dstfolder that exist locally but are + not in the statusfolder yet. The strategy is: + + 1) Look for messages present in self but not in statusfolder. + 2) invoke copymessageto() on those which: + - If dstfolder doesn't have it yet, add them to dstfolder. + - Update statusfolder + + This function checks and protects us from action in dryrun mode.""" - Look for messages present in self but not in dest. If any, add - them to dest.""" threads = [] - - dest_messagelist = dest.getmessagelist() - for uid in self.getmessagelist().keys(): - if uid < 0: # Ignore messages that pass 1 missed. + + copylist = filter(lambda uid: not statusfolder.uidexists(uid), + self.getmessageuidlist()) + num_to_copy = len(copylist) + if num_to_copy and self.repository.account.dryrun: + self.ui.info("[DRYRUN] Copy {0} messages from {1}[{2}] to {3}".format( + num_to_copy, self, self.repository, dstfolder.repository)) + return + for num, uid in enumerate(copylist): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + if uid > 0 and dstfolder.uidexists(uid): + # dst has message with that UID already, only update status + flags = self.getmessageflags(uid) + rtime = self.getmessagetime(uid) + statusfolder.savemessage(uid, None, flags, rtime) continue - if not uid in dest_messagelist: - if self.suggeststhreads(): - self.waitforthread() - thread = threadutil.InstanceLimitedThread(\ - self.getcopyinstancelimit(), - target = self.copymessageto, - name = "Copy message %d from %s" % (uid, - self.getvisiblename()), - args = (uid, applyto)) - thread.setDaemon(1) - thread.start() - threads.append(thread) - else: - self.copymessageto(uid, applyto, register = 0) + + self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) + # exceptions are caught in copymessageto() + if self.suggeststhreads() and not globals.options.singlethreading: + self.waitforthread() + thread = threadutil.InstanceLimitedThread(\ + self.getcopyinstancelimit(), + target = self.copymessageto, + name = "Copy message from %s:%s" % (self.repository, self), + args = (uid, dstfolder, statusfolder)) + thread.start() + threads.append(thread) + else: + self.copymessageto(uid, dstfolder, statusfolder, + register = 0) for thread in threads: thread.join() - def syncmessagesto_delete(self, dest, applyto): - """Pass 3 of folder synchronization. + def __syncmessagesto_delete(self, dstfolder, statusfolder): + """Pass 2: Remove locally deleted messages on dst. - Look for message present in dest but not in self. - If any, delete them.""" - deletelist = [] - self_messagelist = self.getmessagelist() - for uid in dest.getmessagelist().keys(): - if uid < 0: - continue - if not uid in self_messagelist: - deletelist.append(uid) + Get all UIDS in statusfolder but not self. These are messages + that were deleted in 'self'. Delete those from dstfolder and + statusfolder. + + This function checks and protects us from action in dryrun mode. + """ + + deletelist = filter(lambda uid: uid >= 0 and not + self.uidexists(uid), statusfolder.getmessageuidlist()) if len(deletelist): - self.ui.deletingmessages(deletelist, applyto) - for object in applyto: - object.deletemessages(deletelist) + # Delete in statusfolder first to play safe. In case of abort, we + # won't lose message, we will just unneccessarily retransmit some. + # Delete messages from statusfolder that were either deleted by the + # user, or not being tracked (e.g. because of maxage). + statusfolder.deletemessages(deletelist) + # Filter out untracked messages + deletelist = filter(lambda uid: dstfolder.uidexists(uid), deletelist) + if len(deletelist): + self.ui.deletingmessages(deletelist, [dstfolder]) + if self.repository.account.dryrun: + return #don't delete messages in dry-run mode + dstfolder.deletemessages(deletelist) - def syncmessagesto_flags(self, dest, applyto): - """Pass 4 of folder synchronization. + def __syncmessagesto_flags(self, dstfolder, statusfolder): + """Pass 3: Flag synchronization. - Look for any flag matching issues -- set dest message to have the - same flags that we have.""" + Compare flag mismatches in self with those in statusfolder. If + msg has a valid UID and exists on dstfolder (has not e.g. been + deleted there), sync the flag change to both dstfolder and + statusfolder. - # As an optimization over previous versions, we store up which flags - # are being used for an add or a delete. For each flag, we store - # a list of uids to which it should be added. Then, we can call - # addmessagesflags() to apply them in bulk, rather than one - # call per message as before. This should result in some significant - # performance improvements. + This function checks and protects us from action in ryrun mode. + """ + # For each flag, we store a list of uids to which it should be + # added. Then, we can call addmessagesflags() to apply them in + # bulk, rather than one call per message. addflaglist = {} delflaglist = {} - - for uid in self.getmessagelist().keys(): - if uid < 0: # Ignore messages missed by pass 1 + for uid in self.getmessageuidlist(): + # Ignore messages with negative UIDs missed by pass 1 and + # don't do anything if the message has been deleted remotely + if uid < 0 or not dstfolder.uidexists(uid): continue - selfflags = self.getmessageflags(uid) - destflags = dest.getmessageflags(uid) - addflags = [x for x in selfflags if x not in destflags] + selfflags = self.getmessageflags(uid) + + if statusfolder.uidexists(uid): + statusflags = statusfolder.getmessageflags(uid) + else: + statusflags = set() + + addflags = selfflags - statusflags + delflags = statusflags - selfflags for flag in addflags: if not flag in addflaglist: addflaglist[flag] = [] addflaglist[flag].append(uid) - delflags = [x for x in destflags if x not in selfflags] for flag in delflags: if not flag in delflaglist: delflaglist[flag] = [] delflaglist[flag].append(uid) - for object in applyto: - for flag in addflaglist.keys(): - self.ui.addingflags(addflaglist[flag], flag, [object]) - object.addmessagesflags(addflaglist[flag], [flag]) - for flag in delflaglist.keys(): - self.ui.deletingflags(delflaglist[flag], flag, [object]) - object.deletemessagesflags(delflaglist[flag], [flag]) - - def syncmessagesto(self, dest, applyto = None): - """Syncs messages in this folder to the destination. - If applyto is specified, it should be a list of folders (don't forget - to include dest!) to which all write actions should be applied. - It defaults to [dest] if not specified. It is important that - the UID generator be listed first in applyto; that is, the other - applyto ones should be the ones that "copy" the main action.""" - if applyto == None: - applyto = [dest] + for flag, uids in addflaglist.items(): + self.ui.addingflags(uids, flag, dstfolder) + if self.repository.account.dryrun: + continue #don't actually add in a dryrun + dstfolder.addmessagesflags(uids, set(flag)) + statusfolder.addmessagesflags(uids, set(flag)) - try: - self.syncmessagesto_neguid(dest, applyto) - except (KeyboardInterrupt): - raise - except: - self.ui.warn("ERROR attempting to handle negative uids " \ - + "for account " + self.getaccountname() + ":" + traceback.format_exc()) + for flag,uids in delflaglist.items(): + self.ui.deletingflags(uids, flag, dstfolder) + if self.repository.account.dryrun: + continue #don't actually remove in a dryrun + dstfolder.deletemessagesflags(uids, set(flag)) + statusfolder.deletemessagesflags(uids, set(flag)) - #all threads launched here are in try / except clauses when they copy anyway... - self.syncmessagesto_copy(dest, applyto) + def syncmessagesto(self, dstfolder, statusfolder): + """Syncs messages in this folder to the destination dstfolder. - try: - self.syncmessagesto_delete(dest, applyto) - except (KeyboardInterrupt): - raise - except: - self.ui.warn("ERROR attempting to delete messages " \ - + "for account " + self.getaccountname() + ":" + traceback.format_exc()) + This is the high level entry for syncing messages in one direction. + Syncsteps are: - # Now, the message lists should be identical wrt the uids present. - # (except for potential negative uids that couldn't be placed - # anywhere) + Pass1: Copy locally existing messages + Copy messages in self, but not statusfolder to dstfolder if not + already in dstfolder. dstfolder might assign a new UID (e.g. if + uploading to IMAP). Update statusfolder. - try: - self.syncmessagesto_flags(dest, applyto) - except (KeyboardInterrupt): - raise - except: - self.ui.warn("ERROR attempting to sync flags " \ - + "for account " + self.getaccountname() + ":" + traceback.format_exc()) - - + Pass2: Remove locally deleted messages + Get all UIDS in statusfolder but not self. These are messages + that were deleted in 'self'. Delete those from dstfolder and + statusfolder. + + After this pass, the message lists should be identical wrt the + uids present (except for potential negative uids that couldn't + be placed anywhere). + + Pass3: Synchronize flag changes + Compare flag mismatches in self with those in statusfolder. If + msg has a valid UID and exists on dstfolder (has not e.g. been + deleted there), sync the flag change to both dstfolder and + statusfolder. + + Pass4: Synchronize label changes (Gmail only) + Compares label mismatches in self with those in statusfolder. + If msg has a valid UID and exists on dstfolder, syncs the labels + to both dstfolder and statusfolder. + + :param dstfolder: Folderinstance to sync the msgs to. + :param statusfolder: LocalStatus instance to sync against. + """ + + for (passdesc, action) in self.syncmessagesto_passes: + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + try: + action(dstfolder, statusfolder) + except (KeyboardInterrupt): + raise + except OfflineImapError as e: + if e.severity > OfflineImapError.ERROR.FOLDER: + raise + self.ui.error(e, exc_info()[2]) + except Exception as e: + self.ui.error(e, exc_info()[2], "Syncing folder %s [acc: %s]" %\ + (self, self.accountname)) + raise # raise unknown Exceptions so we can fix them + + def __eq__(self, other): + """Comparisons work either on string comparing folder names or + on the same instance. + + MailDirFolder('foo') == 'foo' --> True + a = MailDirFolder('foo'); a == b --> True + MailDirFolder('foo') == 'moo' --> False + MailDirFolder('foo') == IMAPFolder('foo') --> False + MailDirFolder('foo') == MaildirFolder('foo') --> False + """ + + if isinstance(other, basestring): + return other == self.name + return id(self) == id(other) + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 59d3253..3d83b91 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -16,107 +16,357 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -"""Folder implementation to support features of the Gmail IMAP server. -""" +import re +from sys import exc_info -from IMAP import IMAPFolder -import imaplib -from offlineimap import imaputil, imaplibutil -from copy import copy +from offlineimap import imaputil, OfflineImapError +from offlineimap import imaplibutil +import offlineimap.accounts +from .IMAP import IMAPFolder +"""Folder implementation to support features of the Gmail IMAP server.""" class GmailFolder(IMAPFolder): """Folder implementation to support features of the Gmail IMAP server. - Specifically, deleted messages are moved to folder `Gmail.TRASH_FOLDER` - (by default: ``[Gmail]/Trash``) prior to expunging them, since - Gmail maps to IMAP ``EXPUNGE`` command to "remove label". + + Removing a message from a folder will only remove the "label" from + the message and keep it in the "All mails" folder. To really delete + a message it needs to be copied to the Trash folder. However, this + is dangerous as our folder moves are implemented as a 1) delete in + one folder and 2) append to the other. If 2 comes before 1, this + will effectively delete the message from all folders. So we cannot + do that until we have a smarter folder move mechanism. For more information on the Gmail IMAP server: http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 + https://developers.google.com/google-apps/gmail/imap_extensions """ - def __init__(self, imapserver, name, visiblename, accountname, repository): - self.realdelete = repository.getrealdelete(name) + def __init__(self, imapserver, name, repository): + super(GmailFolder, self).__init__(imapserver, name, repository) self.trash_folder = repository.gettrashfolder(name) - #: Gmail will really delete messages upon EXPUNGE in these folders + # Gmail will really delete messages upon EXPUNGE in these folders self.real_delete_folders = [ self.trash_folder, repository.getspamfolder() ] - IMAPFolder.__init__(self, imapserver, name, visiblename, \ - accountname, repository) - def deletemessages_noconvert(self, uidlist): - uidlist = [uid for uid in uidlist if uid in self.messagelist] - if not len(uidlist): - return + # The header under which labels are stored + self.labelsheader = self.repository.account.getconf('labelsheader', 'X-Keywords') - if self.realdelete and not (self.getname() in self.real_delete_folders): - # IMAP expunge is just "remove label" in this folder, - # so map the request into a "move into Trash" + # enables / disables label sync + self.synclabels = self.repository.account.getconfboolean('synclabels', False) - imapobj = self.imapserver.acquireconnection() - try: - imapobj.select(self.getfullname()) - result = imapobj.uid('copy', - imaputil.listjoin(uidlist), - self.trash_folder) - assert result[0] == 'OK', \ - "Bad IMAPlib result: %s" % result[0] - finally: - self.imapserver.releaseconnection(imapobj) - for uid in uidlist: - del self.messagelist[uid] - else: - IMAPFolder.deletemessages_noconvert(self, uidlist) - - def processmessagesflags(self, operation, uidlist, flags): - # XXX: the imapobj.myrights(...) calls dies with an error - # report from Gmail server stating that IMAP command - # 'MYRIGHTS' is not implemented. So, this - # `processmessagesflags` is just a copy from `IMAPFolder`, - # with the references to `imapobj.myrights()` deleted This - # shouldn't hurt, however, Gmail users always have full - # control over all their mailboxes (apparently). - if len(uidlist) > 101: - # Hack for those IMAP ervers with a limited line length - self.processmessagesflags(operation, uidlist[:100], flags) - self.processmessagesflags(operation, uidlist[100:], flags) - return - + # if synclabels is enabled, add a 4th pass to sync labels + if self.synclabels: + self.imap_query.insert(0, 'X-GM-LABELS') + self.syncmessagesto_passes.append(('syncing labels', self.syncmessagesto_labels)) + + # Labels to be left alone + ignorelabels = self.repository.account.getconf('ignorelabels', '') + self.ignorelabels = set([l for l in re.split(r'\s*,\s*', ignorelabels) if len(l)]) + + + def getmessage(self, uid): + """Retrieve message with UID from the IMAP server (incl body). Also + gets Gmail labels and embeds them into the message. + + :returns: the message body or throws and OfflineImapError + (probably severity MESSAGE) if e.g. no message with + this UID could be found. + """ imapobj = self.imapserver.acquireconnection() try: - imapobj.select(self.getfullname()) - r = imapobj.uid('store', - imaputil.listjoin(uidlist), - operation + 'FLAGS', - imaputil.flagsmaildir2imap(flags)) - assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) - r = r[1] + data = self._fetch_from_imap(imapobj, str(uid), 2) finally: self.imapserver.releaseconnection(imapobj) - needupdate = copy(uidlist) - for result in r: - if result == None: - # Compensate for servers that don't return anything from - # STORE. + # data looks now e.g. + #[('320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....')] + # we only asked for one message, and that msg is in data[0]. + # msbody is in [0][1]. + body = data[0][1].replace("\r\n", "\n") + + # Embed the labels into the message headers + if self.synclabels: + m = re.search('X-GM-LABELS\s*\(([^\)]*)\)', data[0][0]) + if m: + labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) + else: + labels = set() + labels = labels - self.ignorelabels + labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) + + # First remove old label headers that may be in the message content retrieved + # from gmail Then add a labels header with current gmail labels. + body = self.deletemessageheaders(body, self.labelsheader) + body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) + + if len(body)>200: + dbg_output = "%s...%s"% (str(body)[:150], str(body)[-50:]) + else: + dbg_output = body + + self.ui.debug('imap', "Returned object from fetching %d: '%s'"% + (uid, dbg_output)) + return body + + def getmessagelabels(self, uid): + if 'labels' in self.messagelist[uid]: + return self.messagelist[uid]['labels'] + else: + return set() + + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0} + + + # TODO: merge this code with the parent's cachemessagelist: + # TODO: they have too much common logics. + def cachemessagelist(self, min_date=None, min_uid=None): + if not self.synclabels: + return super(GmailFolder, self).cachemessagelist( + min_date=min_date, min_uid=min_uid) + + self.messagelist = {} + + self.ui.collectingdata(None, self) + imapobj = self.imapserver.acquireconnection() + try: + msgsToFetch = self._msgs_to_fetch( + imapobj, min_date=min_date, min_uid=min_uid) + if not msgsToFetch: + return # No messages to sync + + # Get the flags and UIDs for these. single-quotes prevent + # imaplib2 from quoting the sequence. + # + # NB: msgsToFetch are sequential numbers, not UID's + res_type, response = imapobj.fetch("'%s'"% msgsToFetch, + '(FLAGS X-GM-LABELS UID)') + if res_type != 'OK': + raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " % \ + (self.getrepository(), self) + \ + "Server responded '[%s] %s'" % \ + (res_type, response), OfflineImapError.ERROR.FOLDER), \ + None, exc_info()[2] + finally: + self.imapserver.releaseconnection(imapobj) + + for messagestr in response: + # looks like: '1 (FLAGS (\\Seen Old) X-GM-LABELS (\\Inbox \\Favorites) UID 4807)' or None if no msg + # Discard initial message number. + if messagestr == None: continue - attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1]) - if not ('UID' in attributehash and 'FLAGS' in attributehash): - # Compensate for servers that don't return a UID attribute. - continue - flags = attributehash['FLAGS'] - uid = long(attributehash['UID']) - self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags) + messagestr = messagestr.split(' ', 1)[1] + options = imaputil.flags2hash(messagestr) + if not 'UID' in options: + self.ui.warn('No UID in message with options %s' %\ + str(options), + minor = 1) + else: + uid = long(options['UID']) + self.messagelist[uid] = self.msglist_item_initializer(uid) + flags = imaputil.flagsimap2maildir(options['FLAGS']) + m = re.search('\(([^\)]*)\)', options['X-GM-LABELS']) + if m: + labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) + else: + labels = set() + labels = labels - self.ignorelabels + rtime = imaplibutil.Internaldate2epoch(messagestr) + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime} + + def savemessage(self, uid, content, flags, rtime): + """Save the message on the Server + + This backend always assigns a new uid, so the uid arg is ignored. + + This function will update the self.messagelist dict to contain + the new message after sucessfully saving it, including labels. + + See folder/Base for details. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode. + + :param rtime: A timestamp to be used as the mail date + :returns: the UID of the new message as assigned by the server. If the + message is saved, but it's UID can not be found, it will + return 0. If the message can't be written (folder is + read-only for example) it will return -1.""" + + if not self.synclabels: + return super(GmailFolder, self).savemessage(uid, content, flags, rtime) + + labels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) + + ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) + self.savemessagelabels(ret, labels) + return ret + + def _messagelabels_aux(self, arg, uidlist, labels): + """Common code to savemessagelabels and addmessagelabels""" + labels = labels - self.ignorelabels + uidlist = [uid for uid in uidlist if uid > 0] + if len(uidlist) > 0: + imapobj = self.imapserver.acquireconnection() try: - needupdate.remove(uid) - except ValueError: # Let it slide if it's not in the list - pass - for uid in needupdate: - if operation == '+': - for flag in flags: - if not flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].append(flag) - self.messagelist[uid]['flags'].sort() - elif operation == '-': - for flag in flags: - if flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].remove(flag) + labels_str = '(' + ' '.join([imaputil.quote(lb) for lb in labels]) + ')' + # Coalesce uid's into ranges + uid_str = imaputil.uid_sequence(uidlist) + result = self._store_to_imap(imapobj, uid_str, arg, labels_str) + + except imapobj.readonly: + self.ui.labelstoreadonly(self, uidlist, labels) + return None + + finally: + self.imapserver.releaseconnection(imapobj) + + if result: + retlabels = imaputil.flags2hash(imaputil.imapsplit(result)[1])['X-GM-LABELS'] + retlabels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(retlabels)]) + return retlabels + return None + + def savemessagelabels(self, uid, labels): + """Change a message's labels to `labels`. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a dryrun mode.""" + if uid in self.messagelist and 'labels' in self.messagelist[uid]: + oldlabels = self.messagelist[uid]['labels'] + else: + oldlabels = set() + labels = labels - self.ignorelabels + newlabels = labels | (oldlabels & self.ignorelabels) + if oldlabels != newlabels: + result = self._messagelabels_aux('X-GM-LABELS', [uid], newlabels) + if result: + self.messagelist[uid]['labels'] = newlabels + else: + self.messagelist[uid]['labels'] = oldlabels + + def addmessageslabels(self, uidlist, labels): + """Add `labels` to all messages in uidlist. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a dryrun mode.""" + + labels = labels - self.ignorelabels + result = self._messagelabels_aux('+X-GM-LABELS', uidlist, labels) + if result: + for uid in uidlist: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels + + def deletemessageslabels(self, uidlist, labels): + """Delete `labels` from all messages in uidlist. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a dryrun mode.""" + + labels = labels - self.ignorelabels + result = self._messagelabels_aux('-X-GM-LABELS', uidlist, labels) + if result: + for uid in uidlist: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels + + def copymessageto(self, uid, dstfolder, statusfolder, register = 1): + """Copies a message from self to dst if needed, updating the status + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + + :param uid: uid of the message to be copied. + :param dstfolder: A BaseFolder-derived instance + :param statusfolder: A LocalStatusFolder instance + :param register: whether we should register a new thread." + :returns: Nothing on success, or raises an Exception.""" + + # Check if we are really copying + realcopy = uid > 0 and not dstfolder.uidexists(uid) + + # first copy the message + super(GmailFolder, self).copymessageto(uid, dstfolder, statusfolder, register) + + # sync labels and mtime now when the message is new (the embedded labels are up to date) + # otherwise we may be spending time for nothing, as they will get updated on a later pass. + if realcopy and self.synclabels: + try: + mtime = dstfolder.getmessagemtime(uid) + labels = dstfolder.getmessagelabels(uid) + statusfolder.savemessagelabels(uid, labels, mtime=mtime) + + # dstfolder is not GmailMaildir. + except NotImplementedError: + return + + def syncmessagesto_labels(self, dstfolder, statusfolder): + """Pass 4: Label Synchronization (Gmail only) + + Compare label mismatches in self with those in statusfolder. If + msg has a valid UID and exists on dstfolder (has not e.g. been + deleted there), sync the labels change to both dstfolder and + statusfolder. + + This function checks and protects us from action in dryrun mode. + """ + # This applies the labels message by message, as this makes more sense for a + # Maildir target. If applied with an other Gmail IMAP target it would not be + # the fastest thing in the world though... + uidlist = [] + + # filter the uids (fast) + try: + for uid in self.getmessageuidlist(): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + # Ignore messages with negative UIDs missed by pass 1 and + # don't do anything if the message has been deleted remotely + if uid < 0 or not dstfolder.uidexists(uid): + continue + + selflabels = self.getmessagelabels(uid) - self.ignorelabels + + if statusfolder.uidexists(uid): + statuslabels = statusfolder.getmessagelabels(uid) - self.ignorelabels + else: + statuslabels = set() + + if selflabels != statuslabels: + uidlist.append(uid) + + # now sync labels (slow) + mtimes = {} + labels = {} + for i, uid in enumerate(uidlist): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + selflabels = self.getmessagelabels(uid) - self.ignorelabels + + if statusfolder.uidexists(uid): + statuslabels = statusfolder.getmessagelabels(uid) - self.ignorelabels + else: + statuslabels = set() + + if selflabels != statuslabels: + self.ui.settinglabels(uid, i+1, len(uidlist), sorted(selflabels), dstfolder) + if self.repository.account.dryrun: + continue #don't actually add in a dryrun + dstfolder.savemessagelabels(uid, selflabels, ignorelabels = self.ignorelabels) + mtime = dstfolder.getmessagemtime(uid) + mtimes[uid] = mtime + labels[uid] = selflabels + + # Update statusfolder in a single DB transaction. It is safe, as if something fails, + # statusfolder will be updated on the next run. + statusfolder.savemessageslabelsbulk(labels) + statusfolder.savemessagesmtimebulk(mtimes) + + except NotImplementedError: + self.ui.warn("Can't sync labels. You need to configure a local repository of type GmailMaildir") diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py new file mode 100644 index 0000000..05f7d79 --- /dev/null +++ b/offlineimap/folder/GmailMaildir.py @@ -0,0 +1,328 @@ +# Maildir folder support with labels +# Copyright (C) 2002 - 2011 John Goerzen & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +import os +from sys import exc_info +from .Maildir import MaildirFolder +from offlineimap import OfflineImapError +import offlineimap.accounts +from offlineimap import imaputil + +class GmailMaildirFolder(MaildirFolder): + """Folder implementation to support adding labels to messages in a Maildir. + """ + def __init__(self, root, name, sep, repository): + super(GmailMaildirFolder, self).__init__(root, name, sep, repository) + + # The header under which labels are stored + self.labelsheader = self.repository.account.getconf('labelsheader', 'X-Keywords') + + # enables / disables label sync + self.synclabels = self.repository.account.getconfboolean('synclabels', 0) + + # if synclabels is enabled, add a 4th pass to sync labels + if self.synclabels: + self.syncmessagesto_passes.append(('syncing labels', self.syncmessagesto_labels)) + + def quickchanged(self, statusfolder): + """Returns True if the Maildir has changed. Checks uids, flags and mtimes""" + + self.cachemessagelist() + # Folder has different uids than statusfolder => TRUE + if sorted(self.getmessageuidlist()) != \ + sorted(statusfolder.getmessageuidlist()): + return True + # check for flag changes, it's quick on a Maildir + for (uid, message) in self.getmessagelist().iteritems(): + if message['flags'] != statusfolder.getmessageflags(uid): + return True + # check for newer mtimes. it is also fast + for (uid, message) in self.getmessagelist().iteritems(): + if message['mtime'] > statusfolder.getmessagemtime(uid): + return True + return False #Nope, nothing changed + + + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'flags': set(), 'labels': set(), 'labels_cached': False, + 'filename': '/no-dir/no-such-file/', 'mtime': 0} + + + def cachemessagelist(self, min_date=None, min_uid=None): + if self.ismessagelistempty(): + self.messagelist = self._scanfolder(min_date=min_date, min_uid=min_uid) + + # Get mtimes + if self.synclabels: + for uid, msg in self.messagelist.items(): + filepath = os.path.join(self.getfullname(), msg['filename']) + msg['mtime'] = long(os.stat(filepath).st_mtime) + + + def getmessagelabels(self, uid): + # Labels are not cached in cachemessagelist because it is too slow. + if not self.messagelist[uid]['labels_cached']: + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + + if not os.path.exists(filepath): + return set() + + file = open(filepath, 'rt') + content = file.read() + file.close() + + self.messagelist[uid]['labels'] = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + self.messagelist[uid]['labels'].update( + imaputil.labels_from_header(self.labelsheader, hstr)) + self.messagelist[uid]['labels_cached'] = True + + return self.messagelist[uid]['labels'] + + + def getmessagemtime(self, uid): + if not 'mtime' in self.messagelist[uid]: + return 0 + else: + return self.messagelist[uid]['mtime'] + + def savemessage(self, uid, content, flags, rtime): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode.""" + + if not self.synclabels: + return super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) + + labels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) + + ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) + + # Update the mtime and labels + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + self.messagelist[uid]['mtime'] = long(os.stat(filepath).st_mtime) + self.messagelist[uid]['labels'] = labels + return ret + + def savemessagelabels(self, uid, labels, ignorelabels=set()): + """Change a message's labels to `labels`. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a dryrun mode.""" + + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + + file = open(filepath, 'rt') + content = file.read() + file.close() + + oldlabels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + oldlabels.update(imaputil.labels_from_header(self.labelsheader, hstr)) + + labels = labels - ignorelabels + ignoredlabels = oldlabels & ignorelabels + oldlabels = oldlabels - ignorelabels + + # Nothing to change + if labels == oldlabels: + return + + # Change labels into content + labels_str = imaputil.format_labels_string(self.labelsheader, + sorted(labels | ignoredlabels)) + + # First remove old labels header, and then add the new one + content = self.deletemessageheaders(content, self.labelsheader) + content = self.addmessageheader(content, '\n', self.labelsheader, labels_str) + + mtime = long(os.stat(filepath).st_mtime) + + # write file with new labels to a unique file in tmp + messagename = self.new_message_filename(uid, set()) + tmpname = self.save_to_tmp_file(messagename, content) + tmppath = os.path.join(self.getfullname(), tmpname) + + # move to actual location + try: + os.rename(tmppath, filepath) + except OSError as e: + raise OfflineImapError("Can't rename file '%s' to '%s': %s" % \ + (tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER), \ + None, exc_info()[2] + + # if utime_from_header=true, we don't want to change the mtime. + if self.utime_from_header and mtime: + os.utime(filepath, (mtime, mtime)) + + # save the new mtime and labels + self.messagelist[uid]['mtime'] = long(os.stat(filepath).st_mtime) + self.messagelist[uid]['labels'] = labels + + def copymessageto(self, uid, dstfolder, statusfolder, register = 1): + """Copies a message from self to dst if needed, updating the status + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode. + + :param uid: uid of the message to be copied. + :param dstfolder: A BaseFolder-derived instance + :param statusfolder: A LocalStatusFolder instance + :param register: whether we should register a new thread." + :returns: Nothing on success, or raises an Exception.""" + + # Check if we are really copying + realcopy = uid > 0 and not dstfolder.uidexists(uid) + + # first copy the message + super(GmailMaildirFolder, self).copymessageto(uid, dstfolder, statusfolder, register) + + # sync labels and mtime now when the message is new (the embedded labels are up to date, + # and have already propagated to the remote server. + # for message which already existed on the remote, this is useless, as later the labels may + # get updated. + if realcopy and self.synclabels: + try: + labels = dstfolder.getmessagelabels(uid) + statusfolder.savemessagelabels(uid, labels, mtime=self.getmessagemtime(uid)) + + # dstfolder is not GmailMaildir. + except NotImplementedError: + return + + def syncmessagesto_labels(self, dstfolder, statusfolder): + """Pass 4: Label Synchronization (Gmail only) + + Compare label mismatches in self with those in statusfolder. If + msg has a valid UID and exists on dstfolder (has not e.g. been + deleted there), sync the labels change to both dstfolder and + statusfolder. + + Also skips messages whose mtime remains the same as statusfolder, as the + contents have not changed. + + This function checks and protects us from action in ryrun mode. + """ + # For each label, we store a list of uids to which it should be + # added. Then, we can call addmessageslabels() to apply them in + # bulk, rather than one call per message. + addlabellist = {} + dellabellist = {} + uidlist = [] + + try: + # filter uids (fast) + for uid in self.getmessageuidlist(): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + # Ignore messages with negative UIDs missed by pass 1 and + # don't do anything if the message has been deleted remotely + if uid < 0 or not dstfolder.uidexists(uid): + continue + + selfmtime = self.getmessagemtime(uid) + + if statusfolder.uidexists(uid): + statusmtime = statusfolder.getmessagemtime(uid) + else: + statusmtime = 0 + + if selfmtime > statusmtime: + uidlist.append(uid) + + + self.ui.collectingdata(uidlist, self) + # This can be slow if there is a lot of modified files + for uid in uidlist: + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + selflabels = self.getmessagelabels(uid) + + if statusfolder.uidexists(uid): + statuslabels = statusfolder.getmessagelabels(uid) + else: + statuslabels = set() + + addlabels = selflabels - statuslabels + dellabels = statuslabels - selflabels + + for lb in addlabels: + if not lb in addlabellist: + addlabellist[lb] = [] + addlabellist[lb].append(uid) + + for lb in dellabels: + if not lb in dellabellist: + dellabellist[lb] = [] + dellabellist[lb].append(uid) + + for lb, uids in addlabellist.items(): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + self.ui.addinglabels(uids, lb, dstfolder) + if self.repository.account.dryrun: + continue #don't actually add in a dryrun + dstfolder.addmessageslabels(uids, set([lb])) + statusfolder.addmessageslabels(uids, set([lb])) + + for lb, uids in dellabellist.items(): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + self.ui.deletinglabels(uids, lb, dstfolder) + if self.repository.account.dryrun: + continue #don't actually remove in a dryrun + dstfolder.deletemessageslabels(uids, set([lb])) + statusfolder.deletemessageslabels(uids, set([lb])) + + # Update mtimes on StatusFolder. It is done last to be safe. If something els fails + # and the mtime is not updated, the labels will still be synced next time. + mtimes = {} + for uid in uidlist: + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + if self.repository.account.dryrun: + continue #don't actually update statusfolder + + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + mtimes[uid] = long(os.stat(filepath).st_mtime) + + # finally update statusfolder in a single DB transaction + statusfolder.savemessagesmtimebulk(mtimes) + + except NotImplementedError: + self.ui.warn("Can't sync labels. You need to configure a remote repository of type Gmail.") diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 77d6f36..9ed7fec 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -1,6 +1,5 @@ # IMAP folder support -# Copyright (C) 2002-2007 John Goerzen -# +# Copyright (C) 2002-2012 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,403 +15,766 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import imaplib -import rfc822 -import string import random import binascii import re +import os import time -from StringIO import StringIO -from copy import copy -from Base import BaseFolder -from offlineimap import imaputil, imaplibutil, __version__ +from sys import exc_info + +from .Base import BaseFolder +from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError +from offlineimap import globals +from offlineimap.imaplib2 import MonthNames + + +# Globals +CRLF = '\r\n' + + +# NB: message returned from getmessage() will have '\n' all over the place, +# NB: there will be no CRLFs. Just before the sending stage of savemessage() +# NB: '\n' will be transformed back to CRLF. So, for the most parts of the +# NB: code the stored content will be clean of CRLF and one can rely that +# NB: line endings will be pure '\n'. + class IMAPFolder(BaseFolder): - def __init__(self, imapserver, name, visiblename, accountname, repository): - self.config = imapserver.config - self.expunge = repository.getexpunge() - self.name = imaputil.dequote(name) - self.root = None # imapserver.root + def __init__(self, imapserver, name, repository): + # FIXME: decide if unquoted name is from the responsability of the + # caller or not, but not both. + name = imaputil.dequote(name) self.sep = imapserver.delim + super(IMAPFolder, self).__init__(name, repository) + self.expunge = repository.getexpunge() + self.root = None # imapserver.root self.imapserver = imapserver - self.messagelist = None - self.visiblename = visiblename - self.accountname = accountname - self.repository = repository + self.messagelist = {} self.randomgenerator = random.Random() - BaseFolder.__init__(self) #self.ui is set in BaseFolder + self.imap_query = ['BODY.PEEK[]'] - def selectro(self, imapobj): + fh_conf = self.repository.account.getconf('filterheaders', '') + self.filterheaders = [h for h in re.split(r'\s*,\s*', fh_conf) if h] + + + def __selectro(self, imapobj, force=False): """Select this folder when we do not need write access. + Prefer SELECT to EXAMINE if we can, since some servers (Courier) do not stabilize UID validity until the folder is - selected.""" + selected. + .. todo: Still valid? Needs verification + :param: Enforce new SELECT even if we are on that folder already. + :returns: raises :exc:`OfflineImapError` severity FOLDER on error""" try: - imapobj.select(self.getfullname()) + imapobj.select(self.getfullname(), force = force) except imapobj.readonly: - imapobj.select(self.getfullname(), readonly = 1) - - def getaccountname(self): - return self.accountname + imapobj.select(self.getfullname(), readonly = True, force = force) + # Interface from BaseFolder def suggeststhreads(self): - return 1 + return not globals.options.singlethreading + # Interface from BaseFolder def waitforthread(self): self.imapserver.connectionwait() + def getmaxage(self): + if self.config.getdefault("Account %s"% + self.accountname, "maxage", None): + raise OfflineImapError("maxage is not supported on IMAP-IMAP sync", + OfflineImapError.ERROR.REPO), None, exc_info()[2] + + # Interface from BaseFolder def getcopyinstancelimit(self): return 'MSGCOPY_' + self.repository.getname() - def getvisiblename(self): - return self.visiblename + # Interface from BaseFolder + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value - def getuidvalidity(self): + UIDVALIDITY value will be cached on the first call. + :returns: The UIDVALIDITY as (long) number.""" + + if hasattr(self, '_uidvalidity'): + # use cached value if existing + return self._uidvalidity imapobj = self.imapserver.acquireconnection() try: - # Primes untagged_responses - self.selectro(imapobj) - return long(imapobj.untagged_responses['UIDVALIDITY'][0]) + # SELECT (if not already done) and get current UIDVALIDITY + self.__selectro(imapobj) + typ, uidval = imapobj.response('UIDVALIDITY') + assert uidval != [None] and uidval != None, \ + "response('UIDVALIDITY') returned [None]!" + self._uidvalidity = long(uidval[-1]) + return self._uidvalidity finally: self.imapserver.releaseconnection(imapobj) - + + # Interface from BaseFolder def quickchanged(self, statusfolder): # An IMAP folder has definitely changed if the number of # messages or the UID of the last message have changed. Otherwise # only flag changes could have occurred. - imapobj = self.imapserver.acquireconnection() - try: - # Primes untagged_responses - imapobj.select(self.getfullname(), readonly = 1, force = 1) + retry = True # Should we attempt another round or exit? + while retry: + retry = False + imapobj = self.imapserver.acquireconnection() try: - # 1. Some mail servers do not return an EXISTS response - # if the folder is empty. 2. ZIMBRA servers can return - # multiple EXISTS replies in the form 500, 1000, 1500, - # 1623 so check for potentially multiple replies. - maxmsgid = 0 - for msgid in imapobj.untagged_responses['EXISTS']: - maxmsgid = max(long(msgid), maxmsgid) - except KeyError: - return True - - # Different number of messages than last time? - if maxmsgid != len(statusfolder.getmessagelist()): - return True - - if maxmsgid < 1: - # No messages; return - return False - - # Now, get the UID for the last message. - response = imapobj.fetch('%d' % maxmsgid, '(UID)')[1] - finally: - self.imapserver.releaseconnection(imapobj) - - # Discard the message number. - messagestr = string.split(response[0], maxsplit = 1)[1] - options = imaputil.flags2hash(messagestr) - if not options.has_key('UID'): + # Select folder and get number of messages + restype, imapdata = imapobj.select(self.getfullname(), True, + True) + self.imapserver.releaseconnection(imapobj) + except OfflineImapError as e: + # retry on dropped connections, raise otherwise + self.imapserver.releaseconnection(imapobj, True) + if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: + retry = True + else: raise + except: + # cleanup and raise on all other errors + self.imapserver.releaseconnection(imapobj, True) + raise + # 1. Some mail servers do not return an EXISTS response + # if the folder is empty. 2. ZIMBRA servers can return + # multiple EXISTS replies in the form 500, 1000, 1500, + # 1623 so check for potentially multiple replies. + if imapdata == [None]: return True - uid = long(options['UID']) - saveduids = statusfolder.getmessagelist().keys() - saveduids.sort() - if uid != saveduids[-1]: + maxmsgid = 0 + for msgid in imapdata: + maxmsgid = max(long(msgid), maxmsgid) + # Different number of messages than last time? + if maxmsgid != statusfolder.getmessagecount(): return True - return False - # TODO: Make this so that it can define a date that would be the oldest messages etc. - def cachemessagelist(self): - imapobj = self.imapserver.acquireconnection() + def _msgs_to_fetch(self, imapobj, min_date=None, min_uid=None): + """Determines sequence numbers of messages to be fetched. + + Message sequence numbers (MSNs) are more easily compacted + into ranges which makes transactions slightly faster. + + Arguments: + - imapobj: instance of IMAPlib + - min_date (optional): a time_struct; only fetch messages newer than this + - min_uid (optional): only fetch messages with UID >= min_uid + + This function should be called with at MOST one of min_date OR + min_uid set but not BOTH. + + Returns: range(s) for messages or None if no messages + are to be fetched.""" + + def search(search_conditions): + """Actually request the server with the specified conditions. + + Returns: range(s) for messages or None if no messages + are to be fetched.""" + res_type, res_data = imapobj.search(None, search_conditions) + if res_type != 'OK': + raise OfflineImapError("SEARCH in folder [%s]%s failed. " + "Search string was '%s'. Server responded '[%s] %s'"% ( + self.getrepository(), self, search_cond, res_type, res_data), + OfflineImapError.ERROR.FOLDER) + return res_data[0].split() + + res_type, imapdata = imapobj.select(self.getfullname(), True, True) + if imapdata == [None] or imapdata[0] == '0': + # Empty folder, no need to populate message list. + return None + + conditions = [] + # 1. min_uid condition. + if min_uid != None: + conditions.append("UID %d:*"% min_uid) + # 2. date condition. + elif min_date != None: + # Find out what the oldest message is that we should look at. + conditions.append("SINCE %02d-%s-%d"% ( + min_date[2], MonthNames[min_date[1]], min_date[0])) + # 3. maxsize condition. + maxsize = self.getmaxsize() + if maxsize != None: + conditions.append("SMALLER %d"% maxsize) + + if len(conditions) >= 1: + # Build SEARCH command. + search_cond = "(%s)"% ' '.join(conditions) + search_result = search(search_cond) + return imaputil.uid_sequence(search_result) + + # By default consider all messages in this folder. + return '1:*' + + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'uid': uid, 'flags': set(), 'time': 0} + + + # Interface from BaseFolder + def cachemessagelist(self, min_date=None, min_uid=None): + self.ui.loadmessagelist(self.repository, self) self.messagelist = {} + imapobj = self.imapserver.acquireconnection() try: - # Primes untagged_responses - imapobj.select(self.getfullname(), readonly = 1, force = 1) + msgsToFetch = self._msgs_to_fetch( + imapobj, min_date=min_date, min_uid=min_uid) + if not msgsToFetch: + return # No messages to sync - maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1) - maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1) - - if (maxage != -1) | (maxsize != -1): - try: - search_condition = "("; - - if(maxage != -1): - #find out what the oldest message is that we should look at - oldest_time_struct = time.gmtime(time.time() - (60*60*24*maxage)) - - #format this manually - otherwise locales could cause problems - monthnames_standard = ["Jan", "Feb", "Mar", "Apr", "May", \ - "June", "July", "Aug", "Sep", "Oct", "Nov", "Dec"] - - our_monthname = monthnames_standard[oldest_time_struct[1]-1] - daystr = "%(day)02d" % {'day' : oldest_time_struct[2]} - date_search_str = "SINCE " + daystr + "-" + our_monthname \ - + "-" + str(oldest_time_struct[0]) - - search_condition += date_search_str - - if(maxsize != -1): - if(maxage != 1): #There are two conditions - add a space - search_condition += " " - - search_condition += "SMALLER " + self.config.getdefault("Account " + self.accountname, "maxsize", -1) - - search_condition += ")" - searchresult = imapobj.search(None, search_condition) - - #result would come back seperated by space - to change into a fetch - #statement we need to change space to comma - messagesToFetch = searchresult[1][0].replace(" ", ",") - except KeyError: - return - if len(messagesToFetch) < 1: - # No messages; return - return - else: - try: - # 1. Some mail servers do not return an EXISTS response - # if the folder is empty. 2. ZIMBRA servers can return - # multiple EXISTS replies in the form 500, 1000, 1500, - # 1623 so check for potentially multiple replies. - maxmsgid = 0 - for msgid in imapobj.untagged_responses['EXISTS']: - maxmsgid = max(long(msgid), maxmsgid) - messagesToFetch = '1:%d' % maxmsgid; - except KeyError: - return - if maxmsgid < 1: - #no messages; return - return - # Now, get the flags and UIDs for these. - # We could conceivably get rid of maxmsgid and just say - # '1:*' here. - response = imapobj.fetch(messagesToFetch, '(FLAGS UID)')[1] + # Get the flags and UIDs for these. single-quotes prevent + # imaplib2 from quoting the sequence. + res_type, response = imapobj.fetch("'%s'"% + msgsToFetch, '(FLAGS UID INTERNALDATE)') + if res_type != 'OK': + raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " + "Server responded '[%s] %s'"% (self.getrepository(), self, + res_type, response), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) + for messagestr in response: - # Discard the message number. - messagestr = string.split(messagestr, maxsplit = 1)[1] + # looks like: '1 (FLAGS (\\Seen Old) UID 4807)' or None if no msg + # Discard initial message number. + if messagestr == None: + continue + messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) - if not options.has_key('UID'): - self.ui.warn('No UID in message with options %s' %\ + if not 'UID' in options: + self.ui.warn('No UID in message with options %s'% \ str(options), minor = 1) else: uid = long(options['UID']) + self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) + def dropmessagelistcache(self): + self.messagelist = {} + + # Interface from BaseFolder def getmessagelist(self): return self.messagelist + # Interface from BaseFolder def getmessage(self, uid): + """Retrieve message with UID from the IMAP server (incl body). + + After this function all CRLFs will be transformed to '\n'. + + :returns: the message body or throws and OfflineImapError + (probably severity MESSAGE) if e.g. no message with + this UID could be found. + """ + imapobj = self.imapserver.acquireconnection() try: - imapobj.select(self.getfullname(), readonly = 1) - initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])') - self.ui.debug('imap', 'Returned object from fetching %d: %s' % \ - (uid, str(initialresult))) - return initialresult[1][0][1].replace("\r\n", "\n") - + data = self._fetch_from_imap(imapobj, str(uid), 2) finally: self.imapserver.releaseconnection(imapobj) + # data looks now e.g. [('320 (UID 17061 BODY[] + # {2565}','msgbody....')] we only asked for one message, + # and that msg is in data[0]. msbody is in [0][1] + data = data[0][1].replace(CRLF, "\n") + + if len(data)>200: + dbg_output = "%s...%s"% (str(data)[:150], str(data)[-50:]) + else: + dbg_output = data + + self.ui.debug('imap', "Returned object from fetching %d: '%s'"% + (uid, dbg_output)) + + return data + + # Interface from BaseFolder def getmessagetime(self, uid): return self.messagelist[uid]['time'] - + + # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] - def savemessage_getnewheader(self, content): + def __generate_randomheader(self, content): + """Returns a unique X-OfflineIMAP header + + Generate an 'X-OfflineIMAP' mail header which contains a random + unique value (which is based on the mail content, and a random + number). This header allows us to fetch a mail after APPENDing + it to an IMAP server and thus find out the UID that the server + assigned it. + + :returns: (headername, headervalue) tuple, consisting of strings + headername == 'X-OfflineIMAP' and headervalue will be a + random string + """ + headername = 'X-OfflineIMAP' - headervalue = '%s-' % str(binascii.crc32(content)).replace('-', 'x') - headervalue += binascii.hexlify(self.repository.getname()) + '-' - headervalue += binascii.hexlify(self.getname()) - headervalue += '-%d-' % long(time.time()) - headervalue += str(self.randomgenerator.random()).replace('.', '') - headervalue += '-v' + __version__ + # We need a random component too. If we ever upload the same + # mail twice (e.g. in different folders), we would still need to + # get the UID for the correct one. As we won't have too many + # mails with identical content, the randomness requirements are + # not extremly critial though. + + # compute unsigned crc32 of 'content' as unique hash + # NB: crc32 returns unsigned only starting with python 3.0 + headervalue = str( binascii.crc32(content) & 0xffffffff ) + '-' + headervalue += str(self.randomgenerator.randint(0,9999999999)) return (headername, headervalue) - def savemessage_addheader(self, content, headername, headervalue): - self.ui.debug('imap', - 'savemessage_addheader: called to add %s: %s' % (headername, - headervalue)) - insertionpoint = content.find("\r\n") - self.ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint) - leader = content[0:insertionpoint] - self.ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader)) - if insertionpoint == 0 or insertionpoint == -1: - newline = '' - insertionpoint = 0 - else: - newline = "\r\n" - newline += "%s: %s" % (headername, headervalue) - self.ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline)) - trailer = content[insertionpoint:] - self.ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer)) - return leader + newline + trailer - def savemessage_searchforheader(self, imapobj, headername, headervalue): - if imapobj.untagged_responses.has_key('APPENDUID'): - return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1]) - - self.ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \ - (headername, headervalue)) + def __savemessage_searchforheader(self, imapobj, headername, headervalue): + self.ui.debug('imap', '__savemessage_searchforheader called for %s: %s'% \ + (headername, headervalue)) # Now find the UID it got. headervalue = imapobj._quote(headervalue) try: - matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0] - except imapobj.error, err: + matchinguids = imapobj.uid('search', 'HEADER', + headername, headervalue)[1][0] + except imapobj.error as err: # IMAP server doesn't implement search or had a problem. - self.ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername)) + self.ui.debug('imap', "__savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s"% (err, headername)) return 0 - self.ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids)) + self.ui.debug('imap', '__savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids)) if matchinguids == '': - self.ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername) + self.ui.debug('imap', "__savemessage_searchforheader: UID SEARCH for message with header %s yielded no results"% headername) return 0 matchinguids = matchinguids.split(' ') - self.ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \ + self.ui.debug('imap', '__savemessage_searchforheader: matchinguids now ' + \ repr(matchinguids)) if len(matchinguids) != 1 or matchinguids[0] == None: - raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids)) - matchinguids.sort() + raise ValueError("While attempting to find UID for message with " + "header %s, got wrong-sized matchinguids of %s"%\ + (headername, str(matchinguids))) return long(matchinguids[0]) - def savemessage(self, uid, content, flags, rtime): - imapobj = self.imapserver.acquireconnection() - self.ui.debug('imap', 'savemessage: called') + def __savemessage_fetchheaders(self, imapobj, headername, headervalue): + """ We fetch all new mail headers and search for the right + X-OfflineImap line by hand. The response from the server has form: + ( + 'OK', + [ + ( + '185 (RFC822.HEADER {1789}', + '... mail headers ...' + ), + ' UID 2444)', + ( + '186 (RFC822.HEADER {1789}', + '... 2nd mail headers ...' + ), + ' UID 2445)' + ] + ) + We need to locate the UID just after mail headers containing our + X-OfflineIMAP line. + + Returns UID when found, 0 when not found.""" + + self.ui.debug('imap', '__savemessage_fetchheaders called for %s: %s'% \ + (headername, headervalue)) + + # run "fetch X:* rfc822.header" + # since we stored the mail we are looking for just recently, it would + # not be optimal to fetch all messages. So we'll find highest message + # UID in our local messagelist and search from there (exactly from + # UID+1). That works because UIDs are guaranteed to be unique and + # ascending. + + if self.getmessagelist(): + start = 1 + max(self.getmessagelist().keys()) + else: + # Folder was empty - start from 1 + start = 1 + + # Imaplib quotes all parameters of a string type. That must not happen + # with the range X:*. So we use bytearray to stop imaplib from getting + # in our way + + result = imapobj.uid('FETCH', bytearray('%d:*'% start), 'rfc822.header') + if result[0] != 'OK': + raise OfflineImapError('Error fetching mail headers: %s'% + '. '.join(result[1]), OfflineImapError.ERROR.MESSAGE) + + result = result[1] + + found = 0 + for item in result: + if found == 0 and type(item) == type( () ): + # Walk just tuples + if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)"% (headername, headervalue), + item[1], flags=re.IGNORECASE): + found = 1 + elif found == 1: + if type(item) == type (""): + uid = re.search("UID\s+(\d+)", item, flags=re.IGNORECASE) + if uid: + return int(uid.group(1)) + else: + self.ui.warn("Can't parse FETCH response, can't find UID: %s", result.__repr__()) + else: + self.ui.warn("Can't parse FETCH response, we awaited string: %s", result.__repr__()) + + return 0 + + def __getmessageinternaldate(self, content, rtime=None): + """Parses mail and returns an INTERNALDATE string + + It will use information in the following order, falling back as an + attempt fails: + - rtime parameter + - Date header of email + + We return None, if we couldn't find a valid date. In this case + the IMAP server will use the server local time when appening + (per RFC). + + Note, that imaplib's Time2Internaldate is inherently broken as + it returns localized date strings which are invalid for IMAP + servers. However, that function is called for *every* append() + internally. So we need to either pass in `None` or the correct + string (in which case Time2Internaldate() will do nothing) to + append(). The output of this function is designed to work as + input to the imapobj.append() function. + + TODO: We should probably be returning a bytearray rather than a + string here, because the IMAP server will expect plain + ASCII. However, imaplib.Time2INternaldate currently returns a + string so we go with the same for now. + + :param rtime: epoch timestamp to be used rather than analyzing + the email. + :returns: string in the form of "DD-Mmm-YYYY HH:MM:SS +HHMM" + (including double quotes) or `None` in case of failure + (which is fine as value for append).""" + + if rtime is None: + rtime = emailutil.get_message_date(content) + if rtime == None: + return None + datetuple = time.localtime(rtime) + try: - try: - imapobj.select(self.getfullname()) # Needed for search - except imapobj.readonly: - self.ui.msgtoreadonly(self, uid, content, flags) - # Return indicating message taken, but no UID assigned. - # Fudge it. - return 0 - - # This backend always assigns a new uid, so the uid arg is ignored. - # In order to get the new uid, we need to save off the message ID. + # Check for invalid dates + if datetuple[0] < 1981: + raise ValueError - message = rfc822.Message(StringIO(content)) - datetuple_msg = rfc822.parsedate(message.getheader('Date')) - # Will be None if missing or not in a valid format. + # Check for invalid dates + datetuple_check = time.localtime(time.mktime(datetuple)) + if datetuple[:2] != datetuple_check[:2]: + raise ValueError - # If time isn't known - if rtime == None and datetuple_msg == None: - datetuple = time.localtime() - elif rtime == None: - datetuple = datetuple_msg + except (ValueError, OverflowError): + # Argh, sometimes it's a valid format but year is 0102 + # or something. Argh. It seems that Time2Internaldate + # will rause a ValueError if the year is 0102 but not 1902, + # but some IMAP servers nonetheless choke on 1902. + self.ui.debug('imap', "Message with invalid date %s. " + "Server will use local time."% datetuple) + return None + + # Produce a string representation of datetuple that works as + # INTERNALDATE. + num2mon = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', + 7:'Jul', 8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'} + + # tm_isdst coming from email.parsedate is not usable, we still use it + # here, mhh. + if datetuple.tm_isdst == 1: + zone = -time.altzone + else: + zone = -time.timezone + offset_h, offset_m = divmod(zone//60, 60) + + internaldate = '"%02d-%s-%04d %02d:%02d:%02d %+03d%02d"'% \ + (datetuple.tm_mday, num2mon[datetuple.tm_mon], datetuple.tm_year, \ + datetuple.tm_hour, datetuple.tm_min, datetuple.tm_sec, offset_h, offset_m) + + return internaldate + + # Interface from BaseFolder + def savemessage(self, uid, content, flags, rtime): + """Save the message on the Server + + This backend always assigns a new uid, so the uid arg is ignored. + + This function will update the self.messagelist dict to contain + the new message after sucessfully saving it. + + See folder/Base for details. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode. + + :param rtime: A timestamp to be used as the mail date + :returns: the UID of the new message as assigned by the server. If the + message is saved, but it's UID can not be found, it will + return 0. If the message can't be written (folder is + read-only for example) it will return -1.""" + + self.ui.savemessage('imap', uid, flags, self) + + # already have it, just save modified flags + if uid > 0 and self.uidexists(uid): + self.savemessageflags(uid, flags) + return uid + + content = self.deletemessageheaders(content, self.filterheaders) + + # Use proper CRLF all over the message + content = re.sub("(?200: + dbg_output = "%s...%s"% (content[:150], content[-50:]) + else: + dbg_output = content + self.ui.debug('imap', "savemessage: date: %s, content: '%s'"% + (date, dbg_output)) + + try: + # Select folder for append and make the box READ-WRITE + imapobj.select(self.getfullname()) + except imapobj.readonly: + # readonly exception. Return original uid to notify that + # we did not save the message. (see savemessage in Base.py) + self.ui.msgtoreadonly(self, uid, content, flags) + return uid + + #Do the APPEND + try: + (typ, dat) = imapobj.append(self.getfullname(), + imaputil.flagsmaildir2imap(flags), date, content) + # This should only catch 'NO' responses since append() + # will raise an exception for 'BAD' responses: + if typ != 'OK': + # For example, Groupwise IMAP server can return something like: + # + # NO APPEND The 1500 MB storage limit has been exceeded. + # + # In this case, we should immediately abort the repository sync + # and continue with the next account. + msg = \ + "Saving msg (%s) in folder '%s', repository '%s' failed (abort). " \ + "Server responded: %s %s\n"% \ + (msg_id, self, self.getrepository(), typ, dat) + raise OfflineImapError(msg, OfflineImapError.ERROR.REPO) + retry_left = 0 # Mark as success + except imapobj.abort as e: + # connection has been reset, release connection and retry. + retry_left -= 1 + self.imapserver.releaseconnection(imapobj, True) + imapobj = self.imapserver.acquireconnection() + if not retry_left: + raise OfflineImapError("Saving msg (%s) in folder '%s', " + "repository '%s' failed (abort). Server responded: %s\n" + "Message content was: %s"% + (msg_id, self, self.getrepository(), str(e), dbg_output), + OfflineImapError.ERROR.MESSAGE), \ + None, exc_info()[2] + # XXX: is this still needed? + self.ui.error(e, exc_info()[2]) + except imapobj.error as e: # APPEND failed + # If the server responds with 'BAD', append() + # raise()s directly. So we catch that too. + # drop conn, it might be bad. + self.imapserver.releaseconnection(imapobj, True) + imapobj = None + raise OfflineImapError("Saving msg (%s) folder '%s', repo '%s'" + "failed (error). Server responded: %s\nMessage content was: " + "%s" % (msg_id, self, self.getrepository(), str(e), dbg_output), + OfflineImapError.ERROR.MESSAGE), None, exc_info()[2] + # Checkpoint. Let it write out stuff, etc. Eg searches for + # just uploaded messages won't work if we don't do this. + (typ,dat) = imapobj.check() + assert(typ == 'OK') + + # get the new UID, do we use UIDPLUS? + if use_uidplus: + # get new UID from the APPENDUID response, it could look + # like OK [APPENDUID 38505 3955] APPEND completed with + # 38505 bein folder UIDvalidity and 3955 the new UID. + # note: we would want to use .response() here but that + # often seems to return [None], even though we have + # data. TODO + resp = imapobj._get_untagged_response('APPENDUID') + if resp == [None] or resp is None: + self.ui.warn("Server supports UIDPLUS but got no APPENDUID " + "appending a message.") + return 0 + uid = long(resp[-1].split(' ')[1]) + if uid == 0: + self.ui.warn("savemessage: Server supports UIDPLUS, but" + " we got no usable uid back. APPENDUID reponse was " + "'%s'"% str(resp)) else: - datetuple = time.localtime(rtime) - - try: - if datetuple[0] < 1981: - raise ValueError - - # Check for invalid date - datetuple_check = time.localtime(time.mktime(datetuple)) - if datetuple[:2] != datetuple_check[:2]: - raise ValueError - - # This could raise a value error if it's not a valid format. - date = imaplib.Time2Internaldate(datetuple) - except (ValueError, OverflowError): - # Argh, sometimes it's a valid format but year is 0102 - # or something. Argh. It seems that Time2Internaldate - # will rause a ValueError if the year is 0102 but not 1902, - # but some IMAP servers nonetheless choke on 1902. - date = imaplib.Time2Internaldate(time.localtime()) - - self.ui.debug('imap', 'savemessage: using date ' + str(date)) - content = re.sub("(? 101: - # Hack for those IMAP ervers with a limited line length - self.processmessagesflags(operation, uidlist[:100], flags) - self.processmessagesflags(operation, uidlist[100:], flags) - return - + def __processmessagesflags_real(self, operation, uidlist, flags): imapobj = self.imapserver.acquireconnection() try: try: @@ -421,9 +783,8 @@ class IMAPFolder(BaseFolder): self.ui.flagstoreadonly(self, uidlist, flags) return r = imapobj.uid('store', - imaputil.listjoin(uidlist), - operation + 'FLAGS', - imaputil.flagsmaildir2imap(flags)) + imaputil.uid_sequence(uidlist), operation + 'FLAGS', + imaputil.flagsmaildir2imap(flags)) assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) r = r[1] finally: @@ -431,7 +792,7 @@ class IMAPFolder(BaseFolder): # Some IMAP servers do not always return a result. Therefore, # only update the ones that it talks about, and manually fix # the others. - needupdate = copy(uidlist) + needupdate = list(uidlist) for result in r: if result == None: # Compensate for servers that don't return anything from @@ -441,37 +802,51 @@ class IMAPFolder(BaseFolder): if not ('UID' in attributehash and 'FLAGS' in attributehash): # Compensate for servers that don't return a UID attribute. continue - lflags = attributehash['FLAGS'] + flagstr = attributehash['FLAGS'] uid = long(attributehash['UID']) - self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(lflags) + self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flagstr) try: needupdate.remove(uid) except ValueError: # Let it slide if it's not in the list pass for uid in needupdate: if operation == '+': - for flag in flags: - if not flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].append(flag) - self.messagelist[uid]['flags'].sort() + self.messagelist[uid]['flags'] |= flags elif operation == '-': - for flag in flags: - if flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].remove(flag) + self.messagelist[uid]['flags'] -= flags + + def __processmessagesflags(self, operation, uidlist, flags): + # Hack for those IMAP servers with a limited line length + batch_size = 100 + for i in xrange(0, len(uidlist), batch_size): + self.__processmessagesflags_real(operation, + uidlist[i:i + batch_size], flags) + return + + + # Interface from BaseFolder + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + + If the backend supports it. IMAP does not and will throw errors.""" + + raise OfflineImapError('IMAP backend cannot change a messages UID from ' + '%d to %d'% (uid, new_uid), OfflineImapError.ERROR.MESSAGE) + + # Interface from BaseFolder def deletemessage(self, uid): - self.deletemessages_noconvert([uid]) + self.__deletemessages_noconvert([uid]) + # Interface from BaseFolder def deletemessages(self, uidlist): - self.deletemessages_noconvert(uidlist) + self.__deletemessages_noconvert(uidlist) - def deletemessages_noconvert(self, uidlist): - # Weed out ones not in self.messagelist - uidlist = [uid for uid in uidlist if uid in self.messagelist] + def __deletemessages_noconvert(self, uidlist): if not len(uidlist): - return + return - self.addmessagesflags_noconvert(uidlist, ['T']) + self.__addmessagesflags_noconvert(uidlist, set('T')) imapobj = self.imapserver.acquireconnection() try: try: @@ -485,5 +860,3 @@ class IMAPFolder(BaseFolder): self.imapserver.releaseconnection(imapobj) for uid in uidlist: del self.messagelist[uid] - - diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 3a0a16d..78f2134 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -1,6 +1,5 @@ # Local status cache virtual folder -# Copyright (C) 2002 - 2008 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,45 +15,38 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from Base import BaseFolder -import os, threading +from sys import exc_info +import os +import threading + +from .Base import BaseFolder -magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" class LocalStatusFolder(BaseFolder): - def __init__(self, root, name, repository, accountname, config): - self.name = name - self.root = root - self.sep = '.' - self.config = config - self.dofsync = config.getdefaultboolean("general", "fsync", True) - self.filename = os.path.join(root, name) - self.filename = repository.getfolderfilename(name) - self.messagelist = None - self.repository = repository + """LocalStatus backend implemented as a plain text file.""" + + cur_version = 2 + magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT %d" + + def __init__(self, name, repository): + self.sep = '.' #needs to be set before super.__init__() + super(LocalStatusFolder, self).__init__(name, repository) + self.root = repository.root + self.filename = os.path.join(self.getroot(), self.getfolderbasename()) + self.messagelist = {} self.savelock = threading.Lock() - self.doautosave = 1 - self.accountname = accountname - BaseFolder.__init__(self) - - def getaccountname(self): - return self.accountname + # Should we perform fsyncs as often as possible? + self.doautosave = self.config.getdefaultboolean( + "general", "fsync", False) + # Interface from BaseFolder def storesmessages(self): return 0 def isnewfolder(self): return not os.path.exists(self.filename) - def getname(self): - return self.name - - def getroot(self): - return self.root - - def getsep(self): - return self.sep - + # Interface from BaseFolder def getfullname(self): return self.filename @@ -62,89 +54,214 @@ class LocalStatusFolder(BaseFolder): if not self.isnewfolder(): os.unlink(self.filename) - def cachemessagelist(self): - if self.isnewfolder(): - self.messagelist = {} - return - file = open(self.filename, "rt") - self.messagelist = {} - line = file.readline().strip() - if not line and not line.read(): - # The status file is empty - should not have happened, - # but somehow did. - file.close() - return - assert(line == magicline) - for line in file.xreadlines(): + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0} + + def readstatus_v1(self, fp): + """Read status folder in format version 1. + + Arguments: + - fp: I/O object that points to the opened database file. + """ + + for line in fp.xreadlines(): line = line.strip() try: uid, flags = line.split(':') uid = long(uid) - except ValueError, e: - errstr = "Corrupt line '%s' in cache file '%s'" % (line, self.filename) + flags = set(flags) + except ValueError as e: + errstr = "Corrupt line '%s' in cache file '%s'" % \ + (line, self.filename) + self.ui.warn(errstr) + raise ValueError(errstr), None, exc_info()[2] + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags + + def readstatus(self, fp): + """Read status file in the current format. + + Arguments: + - fp: I/O object that points to the opened database file. + """ + + for line in fp.xreadlines(): + line = line.strip() + try: + uid, flags, mtime, labels = line.split('|') + uid = long(uid) + flags = set(flags) + mtime = long(mtime) + labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) + except ValueError as e: + errstr = "Corrupt line '%s' in cache file '%s'"% \ + (line, self.filename) + self.ui.warn(errstr) + raise ValueError(errstr), None, exc_info()[2] + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags + self.messagelist[uid]['mtime'] = mtime + self.messagelist[uid]['labels'] = labels + + + # Interface from BaseFolder + def cachemessagelist(self): + if self.isnewfolder(): + self.messagelist = {} + return + + # Loop as many times as version, and update format. + for i in range(1, self.cur_version + 1): + self.messagelist = {} + cachefd = open(self.filename, "rt") + line = cachefd.readline().strip() + + # Format is up to date. break. + if line == (self.magicline % self.cur_version): + break + + # Convert from format v1. + elif line == (self.magicline % 1): + self.ui._msg('Upgrading LocalStatus cache from version 1' + 'to version 2 for %s:%s'% (self.repository, self)) + self.readstatus_v1(cachefd) + cachefd.close() + self.save() + + # NOTE: Add other format transitions here in the future. + # elif line == (self.magicline % 2): + # self.ui._msg(u'Upgrading LocalStatus cache from version 2' + # 'to version 3 for %s:%s'% (self.repository, self)) + # self.readstatus_v2(cache) + # cache.close() + # cache.save() + + # Something is wrong. + else: + errstr = "Unrecognized cache magicline in '%s'" % self.filename self.ui.warn(errstr) raise ValueError(errstr) - flags = [x for x in flags] - self.messagelist[uid] = {'uid': uid, 'flags': flags} - file.close() - def autosave(self): - if self.doautosave: - self.save() + if not line: + # The status file is empty - should not have happened, + # but somehow did. + errstr = "Cache file '%s' is empty."% self.filename + self.ui.warn(errstr) + cachefd.close() + return + + assert(line == (self.magicline % self.cur_version)) + self.readstatus(cachefd) + cachefd.close() def save(self): - self.savelock.acquire() - try: - file = open(self.filename + ".tmp", "wt") - file.write(magicline + "\n") + """Save changed data to disk. For this backend it is the same as saveall.""" + + self.saveall() + + def saveall(self): + """Saves the entire messagelist to disk.""" + + with self.savelock: + cachefd = open(self.filename + ".tmp", "wt") + cachefd.write((self.magicline % self.cur_version) + "\n") for msg in self.messagelist.values(): - flags = msg['flags'] - flags.sort() - flags = ''.join(flags) - file.write("%s:%s\n" % (msg['uid'], flags)) - file.flush() - if self.dofsync: - os.fsync(file.fileno()) - file.close() + flags = ''.join(sorted(msg['flags'])) + labels = ', '.join(sorted(msg['labels'])) + cachefd.write("%s|%s|%d|%s\n" % (msg['uid'], flags, msg['mtime'], labels)) + cachefd.flush() + if self.doautosave: + os.fsync(cachefd.fileno()) + cachefd.close() os.rename(self.filename + ".tmp", self.filename) - if self.dofsync: + if self.doautosave: fd = os.open(os.path.dirname(self.filename), os.O_RDONLY) os.fsync(fd) os.close(fd) - finally: - self.savelock.release() - + # Interface from BaseFolder def getmessagelist(self): return self.messagelist - def savemessage(self, uid, content, flags, rtime): + # Interface from BaseFolder + def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode.""" + if uid < 0: # We cannot assign a uid. return uid - if uid in self.messagelist: # already have it + if self.uidexists(uid): # already have it self.savemessageflags(uid, flags) return uid - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} - self.autosave() + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags + self.messagelist[uid]['time'] = rtime + self.messagelist[uid]['mtime'] = mtime + self.messagelist[uid]['labels'] = labels + self.save() return uid + # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] + # Interface from BaseFolder def getmessagetime(self, uid): return self.messagelist[uid]['time'] + # Interface from BaseFolder def savemessageflags(self, uid, flags): self.messagelist[uid]['flags'] = flags - self.autosave() + self.save() + def savemessagelabels(self, uid, labels, mtime=None): + self.messagelist[uid]['labels'] = labels + if mtime: self.messagelist[uid]['mtime'] = mtime + self.save() + + def savemessageslabelsbulk(self, labels): + """Saves labels from a dictionary in a single database operation.""" + + for uid, lb in labels.items(): + self.messagelist[uid]['labels'] = lb + self.save() + + def addmessageslabels(self, uids, labels): + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels + self.save() + + def deletemessageslabels(self, uids, labels): + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels + self.save() + + def getmessagelabels(self, uid): + return self.messagelist[uid]['labels'] + + def savemessagesmtimebulk(self, mtimes): + """Saves mtimes from the mtimes dictionary in a single database operation.""" + + for uid, mt in mtimes.items(): + self.messagelist[uid]['mtime'] = mt + self.save() + + def getmessagemtime(self, uid): + return self.messagelist[uid]['mtime'] + + # Interface from BaseFolder def deletemessage(self, uid): self.deletemessages([uid]) + # Interface from BaseFolder def deletemessages(self, uidlist): # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if uid in self.messagelist] @@ -153,4 +270,4 @@ class LocalStatusFolder(BaseFolder): for uid in uidlist: del(self.messagelist[uid]) - self.autosave() + self.save() diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py new file mode 100644 index 0000000..64adcd1 --- /dev/null +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -0,0 +1,408 @@ +# Local status cache virtual folder: SQLite backend +# Copyright (C) 2009-2011 Stewart Smith and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os +from sys import exc_info +from threading import Lock +try: + import sqlite3 as sqlite +except: + pass #fail only if needed later on, not on import + +from .Base import BaseFolder + + +class LocalStatusSQLiteFolder(BaseFolder): + """LocalStatus backend implemented with an SQLite database + + As python-sqlite currently does not allow to access the same sqlite + objects from various threads, we need to open get and close a db + connection and cursor for all operations. This is a big disadvantage + and we might want to investigate if we cannot hold an object open + for a thread somehow.""" + # Though. According to sqlite docs, you need to commit() before + # the connection is closed or your changes will be lost! + # get db connection which autocommits + # connection = sqlite.connect(self.filename, isolation_level=None) + # cursor = connection.cursor() + # return connection, cursor + + # Current version of our db format. + cur_version = 2 + + def __init__(self, name, repository): + self.sep = '.' # Needs to be set before super.__init__() + super(LocalStatusSQLiteFolder, self).__init__(name, repository) + self.root = repository.root + self.filename = os.path.join(self.getroot(), self.getfolderbasename()) + self.messagelist = {} + + self._newfolder = False # Flag if the folder is new. + + dirname = os.path.dirname(self.filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + if not os.path.isdir(dirname): + raise UserWarning("SQLite database path '%s' is not a directory."% + dirname) + + # dblock protects against concurrent writes in same connection. + self._dblock = Lock() + + # Try to establish connection, no need for threadsafety in __init__. + try: + self.connection = sqlite.connect(self.filename, check_same_thread=False) + except NameError: + # sqlite import had failed. + raise UserWarning("SQLite backend chosen, but cannot connect " + "with available bindings to '%s'. Is the sqlite3 package " + "installed?."% self.filename), None, exc_info()[2] + except sqlite.OperationalError as e: + # Operation had failed. + raise UserWarning("cannot open database file '%s': %s.\nYou might " + "want to check the rights to that file and if it cleanly opens " + "with the 'sqlite<3>' command."% + (self.filename, e)), None, exc_info()[2] + + # Make sure sqlite is in multithreading SERIALIZE mode. + assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' + + # Test if db version is current enough and if db is readable. + try: + cursor = self.connection.execute( + "SELECT value from metadata WHERE key='db_version'") + except sqlite.DatabaseError: + #db file missing or corrupt, recreate it. + self.__create_db() + else: + # fetch db version and upgrade if needed + version = int(cursor.fetchone()[0]) + if version < LocalStatusSQLiteFolder.cur_version: + self.__upgrade_db(version) + + + def storesmessages(self): + return False + + def getfullname(self): + return self.filename + + # Interface from LocalStatusFolder + def isnewfolder(self): + return self._newfolder + + + # Interface from LocalStatusFolder + def deletemessagelist(self): + """Delete all messages in the db.""" + + self.__sql_write('DELETE FROM status') + + + def __sql_write(self, sql, vars=None, executemany=False): + """Execute some SQL, retrying if the db was locked. + + :param sql: the SQL string passed to execute() + :param vars: the variable values to `sql`. E.g. (1,2) or {uid:1, + flags:'T'}. See sqlite docs for possibilities. + :param executemany: bool indicating whether we want to + perform conn.executemany() or conn.execute(). + :returns: the Cursor() or raises an Exception""" + + success = False + while not success: + self._dblock.acquire() + try: + if vars is None: + if executemany: + cursor = self.connection.executemany(sql) + else: + cursor = self.connection.execute(sql) + else: + if executemany: + cursor = self.connection.executemany(sql, vars) + else: + cursor = self.connection.execute(sql, vars) + success = True + self.connection.commit() + except sqlite.OperationalError as e: + if e.args[0] == 'cannot commit - no transaction is active': + pass + elif e.args[0] == 'database is locked': + self.ui.debug('', "Locked sqlite database, retrying.") + success = False + else: + raise + finally: + self._dblock.release() + return cursor + + def __upgrade_db(self, from_ver): + """Upgrade the sqlite format from version 'from_ver' to current""" + + if hasattr(self, 'connection'): + self.connection.close() #close old connections first + self.connection = sqlite.connect(self.filename, + check_same_thread = False) + + # Upgrade from database version 1 to version 2 + # This change adds labels and mtime columns, to be used by Gmail IMAP and Maildir folders. + if from_ver <= 1: + self.ui._msg('Upgrading LocalStatus cache from version 1 to version 2 for %s:%s'% + (self.repository, self)) + self.connection.executescript("""ALTER TABLE status ADD mtime INTEGER DEFAULT 0; + ALTER TABLE status ADD labels VARCHAR(256) DEFAULT ''; + UPDATE metadata SET value='2' WHERE key='db_version'; + """) + self.connection.commit() + + # Future version upgrades come here... + # if from_ver <= 2: ... #upgrade from 2 to 3 + # if from_ver <= 3: ... #upgrade from 3 to 4 + + + def __create_db(self): + """Create a new db file. + + self.connection must point to the opened and valid SQlite + database connection.""" + self.ui._msg('Creating new Local Status db for %s:%s' \ + % (self.repository, self)) + self.connection.executescript(""" + CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128)); + INSERT INTO metadata VALUES('db_version', '2'); + CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50), mtime INTEGER, labels VARCHAR(256)); + """) + self.connection.commit() + self._newfolder = True + + + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0} + + + # Interface from BaseFolder + def cachemessagelist(self): + self.messagelist = {} + cursor = self.connection.execute('SELECT id,flags,mtime,labels from status') + for row in cursor: + uid = row[0] + self.messagelist[uid] = self.msglist_item_initializer(uid) + flags = set(row[1]) + try: + labels = set([lb.strip() for lb in + row[3].split(',') if len(lb.strip()) > 0]) + except AttributeError: + # FIXME: This except clause was introduced because row[3] from + # database can be found of unexpected type NoneType. See + # https://github.com/OfflineIMAP/offlineimap/issues/103 + # + # We are fixing the type here but this would require more + # researches to find the true root cause. row[3] is expected to + # be a (empty) string, not None. + # + # Also, since database might return None, we have to fix the + # database, too. + labels = set() + self.messagelist[uid]['flags'] = flags + self.messagelist[uid]['labels'] = labels + self.messagelist[uid]['mtime'] = row[2] + + def dropmessagelistcache(self): + self.messagelist = {} + + # Interface from LocalStatusFolder + def save(self): + pass + # Noop. every transaction commits to database! + + def saveall(self): + """Saves the entire messagelist to the database.""" + + data = [] + for uid, msg in self.messagelist.items(): + mtime = msg['mtime'] + flags = ''.join(sorted(msg['flags'])) + labels = ', '.join(sorted(msg['labels'])) + data.append((uid, flags, mtime, labels)) + + self.__sql_write('INSERT OR REPLACE INTO status ' + '(id,flags,mtime,labels) VALUES (?,?,?,?)', + data, executemany=True) + + + # Following some pure SQLite functions, where we chose to use + # BaseFolder() methods instead. Doing those on the in-memory list is + # quicker anyway. If our db becomes so big that we don't want to + # maintain the in-memory list anymore, these might come in handy + # in the future though. + # + #def uidexists(self,uid): + # conn, cursor = self.get_cursor() + # with conn: + # cursor.execute('SELECT id FROM status WHERE id=:id',{'id': uid}) + # return cursor.fetchone() + # This would be the pure SQLite solution, use BaseFolder() method, + # to avoid threading with sqlite... + #def getmessageuidlist(self): + # conn, cursor = self.get_cursor() + # with conn: + # cursor.execute('SELECT id from status') + # r = [] + # for row in cursor: + # r.append(row[0]) + # return r + #def getmessagecount(self): + # conn, cursor = self.get_cursor() + # with conn: + # cursor.execute('SELECT count(id) from status'); + # return cursor.fetchone()[0] + #def getmessageflags(self, uid): + # conn, cursor = self.get_cursor() + # with conn: + # cursor.execute('SELECT flags FROM status WHERE id=:id', + # {'id': uid}) + # for row in cursor: + # flags = [x for x in row[0]] + # return flags + # assert False,"getmessageflags() called on non-existing message" + + + # Interface from BaseFolder + def getmessagelist(self): + return self.messagelist + + + # Interface from BaseFolder + def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode.""" + + if uid < 0: + # We cannot assign a uid. + return uid + + if self.uidexists(uid): # already have it + self.savemessageflags(uid, flags) + return uid + + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels} + flags = ''.join(sorted(flags)) + labels = ', '.join(sorted(labels)) + self.__sql_write('INSERT INTO status (id,flags,mtime,labels) VALUES (?,?,?,?)', + (uid,flags,mtime,labels)) + return uid + + + # Interface from BaseFolder + def savemessageflags(self, uid, flags): + assert self.uidexists(uid) + self.messagelist[uid]['flags'] = flags + flags = ''.join(sorted(flags)) + self.__sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) + + + def getmessageflags(self, uid): + return self.messagelist[uid]['flags'] + + + def savemessagelabels(self, uid, labels, mtime=None): + self.messagelist[uid]['labels'] = labels + if mtime: self.messagelist[uid]['mtime'] = mtime + + labels = ', '.join(sorted(labels)) + if mtime: + self.__sql_write('UPDATE status SET labels=?, mtime=? WHERE id=?',(labels,mtime,uid)) + else: + self.__sql_write('UPDATE status SET labels=? WHERE id=?',(labels,uid)) + + + def savemessageslabelsbulk(self, labels): + """ + Saves labels from a dictionary in a single database operation. + + """ + data = [(', '.join(sorted(l)), uid) for uid, l in labels.items()] + self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) + for uid, l in labels.items(): + self.messagelist[uid]['labels'] = l + + + def addmessageslabels(self, uids, labels): + data = [] + for uid in uids: + newlabels = self.messagelist[uid]['labels'] | labels + data.append((', '.join(sorted(newlabels)), uid)) + self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels + + + def deletemessageslabels(self, uids, labels): + data = [] + for uid in uids: + newlabels = self.messagelist[uid]['labels'] - labels + data.append((', '.join(sorted(newlabels)), uid)) + self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels + + + def getmessagelabels(self, uid): + return self.messagelist[uid]['labels'] + + + def savemessagesmtimebulk(self, mtimes): + """Saves mtimes from the mtimes dictionary in a single database operation.""" + + data = [(mt, uid) for uid, mt in mtimes.items()] + self.__sql_write('UPDATE status SET mtime=? WHERE id=?', data, executemany=True) + for uid, mt in mtimes.items(): + self.messagelist[uid]['mtime'] = mt + + + def getmessagemtime(self, uid): + return self.messagelist[uid]['mtime'] + + + # Interface from BaseFolder + def deletemessage(self, uid): + if not uid in self.messagelist: + return + self.__sql_write('DELETE FROM status WHERE id=?', (uid, )) + del(self.messagelist[uid]) + + # Interface from BaseFolder + def deletemessages(self, uidlist): + """Delete list of UIDs from status cache + + This function uses sqlites executemany() function which is + much faster than iterating through deletemessage() when we have + many messages to delete.""" + + # Weed out ones not in self.messagelist + uidlist = [uid for uid in uidlist if uid in self.messagelist] + if not len(uidlist): + return + # arg2 needs to be an iterable of 1-tuples [(1,),(2,),...] + self.__sql_write('DELETE FROM status WHERE id=?', zip(uidlist, ), True) + for uid in uidlist: + del(self.messagelist[uid]) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index a01d071..3f5c071 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -1,6 +1,5 @@ # Maildir folder support -# Copyright (C) 2002 - 2007 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,25 +15,34 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import os.path, os, re, time, socket -from Base import BaseFolder -from offlineimap import imaputil +import socket +import time +import re +import os +from sys import exc_info +from .Base import BaseFolder from threading import Lock - try: from hashlib import md5 except ImportError: from md5 import md5 +try: # python 2.6 has set() built in + set +except NameError: + from sets import Set as set -uidmatchre = re.compile(',U=(\d+)') -flagmatchre = re.compile(':.*2,([A-Z]+)') -timestampmatchre = re.compile('(\d+)'); +from offlineimap import OfflineImapError, emailutil + +# Find the UID in a message filename +re_uidmatch = re.compile(',U=(\d+)') +# Find a numeric timestamp in a string (filename prefix) +re_timestampmatch = re.compile('(\d+)'); timeseq = 0 -lasttime = long(0) +lasttime = 0 timelock = Lock() -def gettimeseq(): +def _gettimeseq(): global lasttime, timeseq, timelock timelock.acquire() try: @@ -50,258 +58,400 @@ def gettimeseq(): timelock.release() class MaildirFolder(BaseFolder): - def __init__(self, root, name, sep, repository, accountname, config): - self.name = name - self.config = config - self.dofsync = config.getdefaultboolean("general", "fsync", True) + def __init__(self, root, name, sep, repository): + self.sep = sep # needs to be set before super().__init__ + super(MaildirFolder, self).__init__(name, repository) + self.dofsync = self.config.getdefaultboolean("general", "fsync", True) self.root = root - self.sep = sep - self.messagelist = None - self.repository = repository - self.accountname = accountname - BaseFolder.__init__(self) + self.messagelist = {} + # check if we should use a different infosep to support Win file systems + self.wincompatible = self.config.getdefaultboolean( + "Account "+self.accountname, "maildir-windows-compatible", False) + self.infosep = '!' if self.wincompatible else ':' + """infosep is the separator between maildir name and flag appendix""" + self.re_flagmatch = re.compile('%s2,(\w*)'% self.infosep) #self.ui is set in BaseFolder.init() + # Everything up to the first comma or colon (or ! if Windows): + self.re_prefixmatch = re.compile('([^'+ self.infosep + ',]*)') + # folder's md, so we can match with recorded file md5 for validity. + self._foldermd5 = md5(self.getvisiblename()).hexdigest() + # Cache the full folder path, as we use getfullname() very often. + self._fullname = os.path.join(self.getroot(), self.getname()) - def getaccountname(self): - return self.accountname - + # Interface from BaseFolder def getfullname(self): - return os.path.join(self.getroot(), self.getname()) + """Return the absolute file path to the Maildir folder (sans cur|new)""" + return self._fullname - def getuidvalidity(self): - """Maildirs have no notion of uidvalidity, so we just return a magic + # Interface from BaseFolder + def get_uidvalidity(self): + """Retrieve the current connections UIDVALIDITY value + + Maildirs have no notion of uidvalidity, so we just return a magic token.""" return 42 - #Checks to see if the given message is within the maximum age according - #to the maildir name which should begin with a timestamp - def _iswithinmaxage(self, messagename, maxage): - #In order to have the same behaviour as SINCE in an IMAP search - #we must convert this to the oldest time and then strip off hrs/mins - #from that day - oldest_time_utc = time.time() - (60*60*24*maxage) - oldest_time_struct = time.gmtime(oldest_time_utc) - oldest_time_today_seconds = ((oldest_time_struct[3] * 3600) \ - + (oldest_time_struct[4] * 60) \ - + oldest_time_struct[5]) - oldest_time_utc -= oldest_time_today_seconds + def _iswithintime(self, messagename, date): + """Check to see if the given message is newer than date (a + time_struct) according to the maildir name which should begin + with a timestamp.""" - timestampmatch = timestampmatchre.search(messagename) + timestampmatch = re_timestampmatch.search(messagename) + if not timestampmatch: + return True timestampstr = timestampmatch.group() timestamplong = long(timestampstr) - if(timestamplong < oldest_time_utc): + if(timestamplong < time.mktime(date)): return False else: return True + def _parse_filename(self, filename): + """Returns a messages file name components + + Receives the file name (without path) of a msg. Usual format is + '<%d_%d.%d.%s>,U=<%d>,FMD5=<%s>:2,' (pointy brackets + denoting the various components). + + If FMD5 does not correspond with the current folder MD5, we will + return None for the UID & FMD5 (as it is not valid in this + folder). If UID or FMD5 can not be detected, we return `None` + for the respective element. If flags are empty or cannot be + detected, we return an empty flags list. + + :returns: (prefix, UID, FMD5, flags). UID is a numeric "long" + type. flags is a set() of Maildir flags. + """ + + prefix, uid, fmd5, flags = None, None, None, set() + prefixmatch = self.re_prefixmatch.match(filename) + if prefixmatch: + prefix = prefixmatch.group(1) + folderstr = ',FMD5=%s'% self._foldermd5 + foldermatch = folderstr in filename + # If there was no folder MD5 specified, or if it mismatches, + # assume it is a foreign (new) message and ret: uid, fmd5 = None, None + if foldermatch: + uidmatch = re_uidmatch.search(filename) + if uidmatch: + uid = long(uidmatch.group(1)) + flagmatch = self.re_flagmatch.search(filename) + if flagmatch: + # Filter out all lowercase (custom maildir) flags. We don't + # handle them yet. + flags = set((c for c in flagmatch.group(1) if not c.islower())) + return prefix, uid, fmd5, flags + + def _scanfolder(self, min_date=None, min_uid=None): + """Cache the message list from a Maildir. + + If min_date is set, this finds the min UID of all messages newer than + min_date and uses it as the real cutoff for considering messages. + This handles the edge cases where the date is much earlier than messages + with similar UID's (e.g. the UID was reassigned much later). + + Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F + (flagged). + :returns: dict that can be used as self.messagelist. + """ + + maxsize = self.getmaxsize() - def _scanfolder(self): - """Cache the message list. Maildir flags are: - R (replied) - S (seen) - T (trashed) - D (draft) - F (flagged) - and must occur in ASCII order.""" retval = {} files = [] - nouidcounter = -1 # Messages without UIDs get - # negative UID numbers. - foldermd5 = md5(self.getvisiblename()).hexdigest() - folderstr = ',FMD5=' + foldermd5 + nouidcounter = -1 # Messages without UIDs get negative UIDs. for dirannex in ['new', 'cur']: fulldirname = os.path.join(self.getfullname(), dirannex) - files.extend(os.path.join(fulldirname, filename) for + files.extend((dirannex, filename) for filename in os.listdir(fulldirname)) - for file in files: - messagename = os.path.basename(file) - #check if there is a parameter for maxage / maxsize - then see if this - #message should be considered or not - maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1) - maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1) + date_excludees = {} + for dirannex, filename in files: + # We store just dirannex and filename, ie 'cur/123...' + filepath = os.path.join(dirannex, filename) + # Check maxsize if this message should be considered. + if maxsize and (os.path.getsize(os.path.join( + self.getfullname(), filepath)) > maxsize): + continue - if(maxage != -1): - isnewenough = self._iswithinmaxage(messagename, maxage) - if(isnewenough != True): - #this message is older than we should consider.... - continue - - #Check and see if the message is too big if the maxsize for this account is set - if(maxsize != -1): - filesize = os.path.getsize(file) - if(filesize > maxsize): - continue - - - foldermatch = messagename.find(folderstr) != -1 - if not foldermatch: - # If there is no folder MD5 specified, or if it mismatches, - # assume it is a foreign (new) message and generate a - # negative uid for it + (prefix, uid, fmd5, flags) = self._parse_filename(filename) + if uid is None: # Assign negative uid to upload it. uid = nouidcounter nouidcounter -= 1 else: # It comes from our folder. - uidmatch = uidmatchre.search(messagename) + uidmatch = re_uidmatch.search(filename) uid = None if not uidmatch: uid = nouidcounter nouidcounter -= 1 else: uid = long(uidmatch.group(1)) - flagmatch = flagmatchre.search(messagename) - flags = [] - if flagmatch: - flags = [x for x in flagmatch.group(1)] - flags.sort() - retval[uid] = {'uid': uid, - 'flags': flags, - 'filename': file} + if min_uid != None and uid > 0 and uid < min_uid: + continue + if min_date != None and not self._iswithintime(filename, min_date): + # Keep track of messages outside of the time limit, because they + # still might have UID > min(UIDs of within-min_date). We hit + # this case for maxage if any message had a known/valid datetime + # and was re-uploaded because the UID in the filename got lost + # (e.g. local copy/move). On next sync, it was assigned a new + # UID from the server and will be included in the SEARCH + # condition. So, we must re-include them later in this method + # in order to avoid inconsistent lists of messages. + date_excludees[uid] = self.msglist_item_initializer(uid) + date_excludees[uid]['flags'] = flags + date_excludees[uid]['filename'] = filepath + else: + # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S + retval[uid] = self.msglist_item_initializer(uid) + retval[uid]['flags'] = flags + retval[uid]['filename'] = filepath + if min_date != None: + # Re-include messages with high enough uid's. + positive_uids = filter(lambda uid: uid > 0, retval) + if positive_uids: + min_uid = min(positive_uids) + for uid in date_excludees.keys(): + if uid > min_uid: + # This message was originally excluded because of + # its date. It is re-included now because we want all + # messages with UID > min_uid. + retval[uid] = date_excludees[uid] return retval + # Interface from BaseFolder def quickchanged(self, statusfolder): - self.cachemessagelist() - savedmessages = statusfolder.getmessagelist() - if len(self.messagelist) != len(savedmessages): - return True - for uid in self.messagelist.keys(): - if uid not in savedmessages: - return True - if self.messagelist[uid]['flags'] != savedmessages[uid]['flags']: - return True - return False + """Returns True if the Maildir has changed - def cachemessagelist(self): - if self.messagelist is None: - self.messagelist = self._scanfolder() - + Assumes cachemessagelist() has already been called """ + # Folder has different uids than statusfolder => TRUE. + if sorted(self.getmessageuidlist()) != \ + sorted(statusfolder.getmessageuidlist()): + return True + # Also check for flag changes, it's quick on a Maildir. + for (uid, message) in self.getmessagelist().iteritems(): + if message['flags'] != statusfolder.getmessageflags(uid): + return True + return False # Nope, nothing changed. + + + # Interface from BaseFolder + def msglist_item_initializer(self, uid): + return {'flags': set(), 'filename': '/no-dir/no-such-file/'} + + # Interface from BaseFolder + def cachemessagelist(self, min_date=None, min_uid=None): + if self.ismessagelistempty(): + self.ui.loadmessagelist(self.repository, self) + self.messagelist = self._scanfolder(min_date=min_date, + min_uid=min_uid) + self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) + + # Interface from BaseFolder def getmessagelist(self): return self.messagelist + # Interface from BaseFolder def getmessage(self, uid): + """Return the content of the message.""" + filename = self.messagelist[uid]['filename'] - file = open(filename, 'rt') + filepath = os.path.join(self.getfullname(), filename) + file = open(filepath, 'rt') retval = file.read() file.close() + #TODO: WHY are we replacing \r\n with \n here? And why do we + # read it as text? return retval.replace("\r\n", "\n") - def getmessagetime( self, uid ): + # Interface from BaseFolder + def getmessagetime(self, uid): filename = self.messagelist[uid]['filename'] - st = os.stat(filename) - return st.st_mtime + filepath = os.path.join(self.getfullname(), filename) + return os.path.getmtime(filepath) + def new_message_filename(self, uid, flags=set()): + """Creates a new unique Maildir filename + + :param uid: The UID`None`, or a set of maildir flags + :param flags: A set of maildir flags + :returns: String containing unique message filename""" + + timeval, timeseq = _gettimeseq() + return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \ + (timeval, timeseq, os.getpid(), socket.gethostname(), + uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) + + + def save_to_tmp_file(self, filename, content): + """Saves given content to the named temporary file in the + 'tmp' subdirectory of $CWD. + + Arguments: + - filename: name of the temporary file; + - content: data to be saved. + + Returns: relative path to the temporary file + that was created.""" + + tmpname = os.path.join('tmp', filename) + # open file and write it out + tries = 7 + while tries: + tries = tries - 1 + try: + fd = os.open(os.path.join(self.getfullname(), tmpname), + os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666) + break + except OSError as e: + if e.errno == e.EEXIST: + if tries: + time.sleep(0.23) + continue + severity = OfflineImapError.ERROR.MESSAGE + raise OfflineImapError("Unique filename %s already exists."% + filename, severity), None, exc_info()[2] + else: + raise + + fd = os.fdopen(fd, 'wt') + fd.write(content) + # Make sure the data hits the disk. + fd.flush() + if self.dofsync: + os.fsync(fd) + fd.close() + + return tmpname + + + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode.""" # This function only ever saves to tmp/, # but it calls savemessageflags() to actually save to cur/ or new/. - self.ui.debug('maildir', 'savemessage: called to write with flags %s and content %s' % \ - (repr(flags), repr(content))) + self.ui.savemessage('maildir', uid, flags, self) if uid < 0: # We cannot assign a new uid. return uid + if uid in self.messagelist: - # We already have it. + # We already have it, just update flags. self.savemessageflags(uid, flags) return uid # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - messagename = None - attempts = 0 - while 1: - if attempts > 15: - raise IOError, "Couldn't write to file %s" % messagename - timeval, timeseq = gettimeseq() - messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \ - (timeval, - timeseq, - os.getpid(), - socket.gethostname(), - uid, - md5(self.getvisiblename()).hexdigest()) - if os.path.exists(os.path.join(tmpdir, messagename)): - time.sleep(2) - attempts += 1 - else: - break - tmpmessagename = messagename.split(',')[0] - self.ui.debug('maildir', 'savemessage: using temporary name %s' % tmpmessagename) - file = open(os.path.join(tmpdir, tmpmessagename), "wt") - file.write(content) + messagename = self.new_message_filename(uid, flags) + tmpname = self.save_to_tmp_file(messagename, content) - # Make sure the data hits the disk - file.flush() - if self.dofsync: - os.fsync(file.fileno()) + if self.utime_from_header: + date = emailutil.get_message_date(content, 'Date') + if date != None: + os.utime(os.path.join(self.getfullname(), tmpname), (date, date)) - file.close() - if rtime != None: - os.utime(os.path.join(tmpdir,tmpmessagename), (rtime,rtime)) - self.ui.debug('maildir', 'savemessage: moving from %s to %s' % \ - (tmpmessagename, messagename)) - if tmpmessagename != messagename: # then rename it - os.rename(os.path.join(tmpdir, tmpmessagename), - os.path.join(tmpdir, messagename)) - - if self.dofsync: - try: - # fsync the directory (safer semantics in Linux) - fd = os.open(tmpdir, os.O_RDONLY) - os.fsync(fd) - os.close(fd) - except: - pass - - self.messagelist[uid] = {'uid': uid, 'flags': [], - 'filename': os.path.join(tmpdir, messagename)} + self.messagelist[uid] = self.msglist_item_initializer(uid) + self.messagelist[uid]['flags'] = flags + self.messagelist[uid]['filename'] = tmpname + # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) self.ui.debug('maildir', 'savemessage: returning uid %d' % uid) return uid - + + # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] + # Interface from BaseFolder def savemessageflags(self, uid, flags): + """Sets the specified message's flags to the given set. + + This function moves the message to the cur or new subdir, + depending on the 'S'een flag. + + Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + + assert uid in self.messagelist + oldfilename = self.messagelist[uid]['filename'] - newpath, newname = os.path.split(oldfilename) - tmpdir = os.path.join(self.getfullname(), 'tmp') - if 'S' in flags: - # If a message has been seen, it goes into the cur - # directory. CR debian#152482, [complete.org #4] - newpath = os.path.join(self.getfullname(), 'cur') - else: - newpath = os.path.join(self.getfullname(), 'new') - infostr = ':' - infomatch = re.search('(:.*)$', newname) - if infomatch: # If the info string is present.. - infostr = infomatch.group(1) - newname = newname.split(':')[0] # Strip off the info string. - infostr = re.sub('2,[A-Z]*', '', infostr) - flags.sort() - infostr += '2,' + ''.join(flags) - newname += infostr - - newfilename = os.path.join(newpath, newname) + dir_prefix, filename = os.path.split(oldfilename) + # If a message has been seen, it goes into 'cur' + dir_prefix = 'cur' if 'S' in flags else 'new' + + if flags != self.messagelist[uid]['flags']: + # Flags have actually changed, construct new filename Strip + # off existing infostring (possibly discarding small letter + # flags that dovecot uses TODO) + infomatch = self.re_flagmatch.search(filename) + if infomatch: + filename = filename[:-len(infomatch.group())] #strip off + infostr = '%s2,%s'% (self.infosep, ''.join(sorted(flags))) + filename += infostr + + newfilename = os.path.join(dir_prefix, filename) if (newfilename != oldfilename): - os.rename(oldfilename, newfilename) + try: + os.rename(os.path.join(self.getfullname(), oldfilename), + os.path.join(self.getfullname(), newfilename)) + except OSError as e: + raise OfflineImapError("Can't rename file '%s' to '%s': %s" % ( + oldfilename, newfilename, e[1]), + OfflineImapError.ERROR.FOLDER), \ + None, exc_info()[2] + self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename - # By now, the message had better not be in tmp/ land! - final_dir, final_name = os.path.split(self.messagelist[uid]['filename']) - assert final_dir != tmpdir + # Interface from BaseFolder + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + + This will not update the statusfolder UID, you need to do that yourself. + :param new_uid: (optional) If given, the old UID will be changed + to a new UID. The Maildir backend can implement this as + an efficient rename. + """ - def deletemessage(self, uid): if not uid in self.messagelist: - return + raise OfflineImapError("Cannot change unknown Maildir UID %s"% uid) + if uid == new_uid: return + + oldfilename = self.messagelist[uid]['filename'] + dir_prefix, filename = os.path.split(oldfilename) + flags = self.getmessageflags(uid) + newfilename = os.path.join(dir_prefix, + self.new_message_filename(new_uid, flags)) + os.rename(os.path.join(self.getfullname(), oldfilename), + os.path.join(self.getfullname(), newfilename)) + self.messagelist[new_uid] = self.messagelist[uid] + self.messagelist[new_uid]['filename'] = newfilename + del self.messagelist[uid] + + # Interface from BaseFolder + def deletemessage(self, uid): + """Unlinks a message file from the Maildir. + + :param uid: UID of a mail message + :type uid: String + :return: Nothing, or an Exception if UID but no corresponding file + found. + """ filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) try: - os.unlink(filename) + os.unlink(filepath) except OSError: # Can't find the file -- maybe already deleted? newmsglist = self._scanfolder() if uid in newmsglist: # Nope, try new filename. - os.unlink(newmsglist[uid]['filename']) + filename = newmsglist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + os.unlink(filepath) # Yep -- return. del(self.messagelist[uid]) - diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index c77fd14..1e54f2b 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -1,6 +1,5 @@ # Base folder support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,22 +15,36 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import * -from offlineimap import threadutil -from offlineimap.threadutil import InstanceLimitedThread -from IMAP import IMAPFolder -import os.path, re +from sys import exc_info +from threading import Lock +from offlineimap import OfflineImapError +from .IMAP import IMAPFolder +import os.path -class MappingFolderMixIn: - def _initmapping(self): +class MappedIMAPFolder(IMAPFolder): + """IMAP class to map between Folder() instances where both side assign a uid + + This Folder is used on the local side, while the remote side should + be an IMAPFolder. + + Instance variables (self.): + r2l: dict mapping message uids: self.r2l[remoteuid]=localuid + l2r: dict mapping message uids: self.r2l[localuid]=remoteuid + #TODO: what is the difference, how are they used? + diskr2l: dict mapping message uids: self.r2l[remoteuid]=localuid + diskl2r: dict mapping message uids: self.r2l[localuid]=remoteuid""" + + def __init__(self, *args, **kwargs): + IMAPFolder.__init__(self, *args, **kwargs) self.maplock = Lock() (self.diskr2l, self.diskl2r) = self._loadmaps() - self._mb = self.__class__.__bases__[1] + self._mb = IMAPFolder(*args, **kwargs) + """Representing the local IMAP Folder using local UIDs""" def _getmapfilename(self): return os.path.join(self.repository.getmapdir(), self.getfolderbasename()) - + def _loadmaps(self): self.maplock.acquire() try: @@ -48,8 +61,8 @@ class MappingFolderMixIn: try: line = line.strip() except ValueError: - raise Exception("Corrupt line '%s' in UID mapping file '%s'" \ - %(line, mapfilename)) + raise Exception("Corrupt line '%s' in UID mapping file '%s'"% + (line, mapfilename)), None, exc_info()[2] (str1, str2) = line.split(':') loc = long(str1) rem = long(str2) @@ -65,18 +78,26 @@ class MappingFolderMixIn: try: file = open(mapfilename + ".tmp", 'wt') for (key, value) in self.diskl2r.iteritems(): - file.write("%d:%d\n" % (key, value)) + file.write("%d:%d\n"% (key, value)) file.close() os.rename(mapfilename + '.tmp', mapfilename) finally: if dolock: self.maplock.release() def _uidlist(self, mapping, items): - return [mapping[x] for x in items] + try: + return [mapping[x] for x in items] + except KeyError as e: + raise OfflineImapError("Could not find UID for msg '{0}' (f:'{1}'." + " This is usually a bad thing and should be reported on the ma" + "iling list.".format(e.args[0], self), + OfflineImapError.ERROR.MESSAGE), None, exc_info()[2] - def cachemessagelist(self): - self._mb.cachemessagelist(self) - reallist = self._mb.getmessagelist(self) + # Interface from BaseFolder + def cachemessagelist(self, min_date=None, min_uid=None): + self._mb.cachemessagelist(min_date=min_date, min_uid=min_uid) + reallist = self._mb.getmessagelist() + self.messagelist = self._mb.messagelist self.maplock.acquire() try: @@ -84,7 +105,7 @@ class MappingFolderMixIn: # summary that have been deleted from the folder. for luid in self.diskl2r.keys(): - if not reallist.has_key(luid): + if not luid in reallist: ruid = self.diskl2r[luid] del self.diskr2l[ruid] del self.diskl2r[luid] @@ -97,7 +118,7 @@ class MappingFolderMixIn: self.l2r = self.diskl2r.copy() for luid in reallist.keys(): - if not self.l2r.has_key(luid): + if not luid in self.l2r: ruid = nextneg nextneg -= 1 self.l2r[luid] = ruid @@ -105,12 +126,40 @@ class MappingFolderMixIn: finally: self.maplock.release() + def dropmessagelistcache(self): + self._mb.dropmessagelistcache() + + # Interface from BaseFolder + def uidexists(self, ruid): + """Checks if the (remote) UID exists in this Folder""" + # This implementation overrides the one in BaseFolder, as it is + # much more efficient for the mapped case. + return ruid in self.r2l + + # Interface from BaseFolder + def getmessageuidlist(self): + """Gets a list of (remote) UIDs. + You may have to call cachemessagelist() before calling this function!""" + # This implementation overrides the one in BaseFolder, as it is + # much more efficient for the mapped case. + return self.r2l.keys() + + # Interface from BaseFolder + def getmessagecount(self): + """Gets the number of messages in this folder. + You may have to call cachemessagelist() before calling this function!""" + # This implementation overrides the one in BaseFolder, as it is + # much more efficient for the mapped case. + return len(self.r2l) + + # Interface from BaseFolder def getmessagelist(self): - """Gets the current message list. - You must call cachemessagelist() before calling this function!""" + """Gets the current message list. This function's implementation + is quite expensive for the mapped UID case. You must call + cachemessagelist() before calling this function!""" retval = {} - localhash = self._mb.getmessagelist(self) + localhash = self._mb.getmessagelist() self.maplock.acquire() try: for key, value in localhash.items(): @@ -129,36 +178,46 @@ class MappingFolderMixIn: finally: self.maplock.release() + # Interface from BaseFolder def getmessage(self, uid): """Returns the content of the specified message.""" - return self._mb.getmessage(self, self.r2l[uid]) + return self._mb.getmessage(self.r2l[uid]) + # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. - If the uid is < 0, the backend should assign a new uid and return it. - If the backend cannot assign a new uid, it returns the uid passed in - WITHOUT saving the message. - - If the backend CAN assign a new uid, but cannot find out what this UID - is (as is the case with many IMAP servers), it returns 0 but DOES save - the message. - - IMAP backend should be the only one that can assign a new uid. + The UIDMaps class will not return a newly assigned uid, as it + internally maps different uids between IMAP servers. So a + successful savemessage() invocation will return the same uid it + has been invoked with. As it maps between 2 IMAP servers which + means the source message must already have an uid, it requires a + positive uid to be passed in. Passing in a message with a + negative uid will do nothing and return the negative uid. If the uid is > 0, the backend should set the uid to this, if it can. If it cannot set the uid to that, it will save it anyway. It will return the uid assigned in any case. + + See folder/Base for details. Note that savemessage() does not + check against dryrun settings, so you need to ensure that + savemessage is never called in a dryrun mode. """ + self.ui.savemessage('imap', uid, flags, self) + # Mapped UID instances require the source to already have a + # positive UID, so simply return here. if uid < 0: - # We cannot assign a new uid. return uid + + # If msg uid already exists, just modify the flags. if uid in self.r2l: self.savemessageflags(uid, flags) return uid - newluid = self._mb.savemessage(self, -1, content, flags, rtime) + + newluid = self._mb.savemessage(-1, content, flags, rtime) if newluid < 1: - raise ValueError, "Backend could not find uid for message" + raise ValueError("Backend could not find uid for message, " + "returned %s"% newluid) self.maplock.acquire() try: self.diskl2r[newluid] = uid @@ -168,23 +227,59 @@ class MappingFolderMixIn: self._savemaps(dolock = 0) finally: self.maplock.release() + return uid + # Interface from BaseFolder def getmessageflags(self, uid): - return self._mb.getmessageflags(self, self.r2l[uid]) + return self._mb.getmessageflags(self.r2l[uid]) + # Interface from BaseFolder def getmessagetime(self, uid): return None + # Interface from BaseFolder def savemessageflags(self, uid, flags): - self._mb.savemessageflags(self, self.r2l[uid], flags) + """Note that this function does not check against dryrun settings, + so you need to ensure that it is never called in a + dryrun mode.""" + self._mb.savemessageflags(self.r2l[uid], flags) + + # Interface from BaseFolder def addmessageflags(self, uid, flags): - self._mb.addmessageflags(self, self.r2l[uid], flags) + self._mb.addmessageflags(self.r2l[uid], flags) + # Interface from BaseFolder def addmessagesflags(self, uidlist, flags): - self._mb.addmessagesflags(self, self._uidlist(self.r2l, uidlist), + self._mb.addmessagesflags(self._uidlist(self.r2l, uidlist), flags) + # Interface from BaseFolder + def change_message_uid(self, ruid, new_ruid): + """Change the message from existing ruid to new_ruid + + :param new_uid: The old remote UID will be changed to a new + UID. The UIDMaps case handles this efficiently by simply + changing the mappings file.""" + if ruid not in self.r2l: + raise OfflineImapError("Cannot change unknown Maildir UID %s"% + ruid, OfflineImapError.ERROR.MESSAGE) + if ruid == new_ruid: return # sanity check shortcut + self.maplock.acquire() + try: + luid = self.r2l[ruid] + self.l2r[luid] = new_ruid + del self.r2l[ruid] + self.r2l[new_ruid] = luid + # TODO: diskl2r|r2l are a pain to sync and should be done away with + # diskl2r only contains positive UIDs, so wrap in ifs. + if luid > 0: self.diskl2r[luid] = new_ruid + if ruid > 0: del self.diskr2l[ruid] + if new_ruid > 0: self.diskr2l[new_ruid] = luid + self._savemaps(dolock = 0) + finally: + self.maplock.release() + def _mapped_delete(self, uidlist): self.maplock.acquire() try: @@ -202,28 +297,21 @@ class MappingFolderMixIn: finally: self.maplock.release() + # Interface from BaseFolder def deletemessageflags(self, uid, flags): - self._mb.deletemessageflags(self, self.r2l[uid], flags) + self._mb.deletemessageflags(self.r2l[uid], flags) + # Interface from BaseFolder def deletemessagesflags(self, uidlist, flags): - self._mb.deletemessagesflags(self, self._uidlist(self.r2l, uidlist), - flags) + self._mb.deletemessagesflags(self._uidlist(self.r2l, uidlist), + flags) + # Interface from BaseFolder def deletemessage(self, uid): - self._mb.deletemessage(self, self.r2l[uid]) + self._mb.deletemessage(self.r2l[uid]) self._mapped_delete([uid]) + # Interface from BaseFolder def deletemessages(self, uidlist): - self._mb.deletemessages(self, self._uidlist(self.r2l, uidlist)) + self._mb.deletemessages(self._uidlist(self.r2l, uidlist)) self._mapped_delete(uidlist) - - #def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1): - # does not need changes because it calls functions that make the changes - # same goes for all other sync messages types. - - -# Define a class for local part of IMAP. -class MappedIMAPFolder(MappingFolderMixIn, IMAPFolder): - def __init__(self, *args, **kwargs): - apply(IMAPFolder.__init__, (self,) + args, kwargs) - self._initmapping() diff --git a/offlineimap/folder/__init__.py b/offlineimap/folder/__init__.py index 425148b..2b54a71 100644 --- a/offlineimap/folder/__init__.py +++ b/offlineimap/folder/__init__.py @@ -1,2 +1,2 @@ -import Base, Gmail, IMAP, Maildir, LocalStatus +from . import Base, Gmail, IMAP, Maildir, LocalStatus diff --git a/offlineimap/globals.py b/offlineimap/globals.py new file mode 100644 index 0000000..b4253f9 --- /dev/null +++ b/offlineimap/globals.py @@ -0,0 +1,12 @@ +# Copyright 2013 Eygene A. Ryabinkin. +# +# Module that holds various global objects. + +from offlineimap.utils import const + +# Holds command-line options for OfflineIMAP. +options = const.ConstProxy() + +def set_options (source): + """ Sets the source for options variable """ + options.set_source (source) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py new file mode 100644 index 0000000..eeb4792 --- /dev/null +++ b/offlineimap/imaplib2.py @@ -0,0 +1,2581 @@ +#!/usr/bin/env python + +"""Threaded IMAP4 client. + +Based on RFC 3501 and original imaplib module. + +Public classes: IMAP4 + IMAP4_SSL + IMAP4_stream + +Public functions: Internaldate2Time + ParseFlags + Time2Internaldate +""" + + +__all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", + "Internaldate2Time", "ParseFlags", "Time2Internaldate") + +__version__ = "2.43" +__release__ = "2" +__revision__ = "43" +__credits__ = """ +Authentication code contributed by Donn Cave June 1998. +String method conversion by ESR, February 2001. +GET/SETACL contributed by Anthony Baxter April 2001. +IMAP4_SSL contributed by Tino Lange March 2002. +GET/SETQUOTA contributed by Andreas Zeidler June 2002. +PROXYAUTH contributed by Rick Holbert November 2002. +IDLE via threads suggested by Philippe Normand January 2005. +GET/SETANNOTATION contributed by Tomas Lindroos June 2005. +COMPRESS/DEFLATE contributed by Bron Gondwana May 2009. +STARTTLS from Jython's imaplib by Alan Kennedy. +ID contributed by Dave Baggett November 2009. +Improved untagged responses handling suggested by Dave Baggett November 2009. +Improved thread naming, and 0 read detection contributed by Grant Edwards June 2010. +Improved timeout handling contributed by Ivan Vovnenko October 2010. +Timeout handling further improved by Ethan Glasser-Camp December 2010. +Time2Internaldate() patch to match RFC2060 specification of English month names from bugs.python.org/issue11024 March 2011. +starttls() bug fixed with the help of Sebastian Spaeth April 2011. +Threads now set the "daemon" flag (suggested by offlineimap-project) April 2011. +Single quoting introduced with the help of Vladimir Marek August 2011. +Support for specifying SSL version by Ryan Kavanagh July 2013. +Fix for gmail "read 0" error provided by Jim Greenleaf August 2013. +Fix for offlineimap "indexerror: string index out of range" bug provided by Eygene Ryabinkin August 2013. +Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014. +Conversion to Python3 provided by F. Malina February 2015. +Fix for READ-ONLY error from multiple EXAMINE/SELECT calls by Pierre-Louis Bonicoli March 2015. +Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli March 2015.""" +__author__ = "Piers Lauder " +__URL__ = "http://imaplib2.sourceforge.net" +__license__ = "Python License" + +import binascii, errno, os, random, re, select, socket, sys, time, threading, zlib + +try: + import queue # py3 + string_types = str +except ImportError: + import Queue as queue # py2 + string_types = basestring + + +select_module = select + +# Globals + +CRLF = '\r\n' +Debug = None # Backward compatibility +IMAP4_PORT = 143 +IMAP4_SSL_PORT = 993 + +IDLE_TIMEOUT_RESPONSE = '* IDLE TIMEOUT\r\n' +IDLE_TIMEOUT = 60*29 # Don't stay in IDLE state longer +READ_POLL_TIMEOUT = 30 # Without this timeout interrupted network connections can hang reader +READ_SIZE = 32768 # Consume all available in socket + +DFLT_DEBUG_BUF_LVL = 3 # Level above which the logging output goes directly to stderr + +AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first + +# Commands + +CMD_VAL_STATES = 0 +CMD_VAL_ASYNC = 1 +NONAUTH, AUTH, SELECTED, LOGOUT = 'NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT' + +Commands = { + # name valid states asynchronous + 'APPEND': ((AUTH, SELECTED), False), + 'AUTHENTICATE': ((NONAUTH,), False), + 'CAPABILITY': ((NONAUTH, AUTH, SELECTED), True), + 'CHECK': ((SELECTED,), True), + 'CLOSE': ((SELECTED,), False), + 'COMPRESS': ((AUTH,), False), + 'COPY': ((SELECTED,), True), + 'CREATE': ((AUTH, SELECTED), True), + 'DELETE': ((AUTH, SELECTED), True), + 'DELETEACL': ((AUTH, SELECTED), True), + 'EXAMINE': ((AUTH, SELECTED), False), + 'EXPUNGE': ((SELECTED,), True), + 'FETCH': ((SELECTED,), True), + 'GETACL': ((AUTH, SELECTED), True), + 'GETANNOTATION':((AUTH, SELECTED), True), + 'GETQUOTA': ((AUTH, SELECTED), True), + 'GETQUOTAROOT': ((AUTH, SELECTED), True), + 'ID': ((NONAUTH, AUTH, LOGOUT, SELECTED), True), + 'IDLE': ((SELECTED,), False), + 'LIST': ((AUTH, SELECTED), True), + 'LOGIN': ((NONAUTH,), False), + 'LOGOUT': ((NONAUTH, AUTH, LOGOUT, SELECTED), False), + 'LSUB': ((AUTH, SELECTED), True), + 'MYRIGHTS': ((AUTH, SELECTED), True), + 'NAMESPACE': ((AUTH, SELECTED), True), + 'NOOP': ((NONAUTH, AUTH, SELECTED), True), + 'PARTIAL': ((SELECTED,), True), + 'PROXYAUTH': ((AUTH,), False), + 'RENAME': ((AUTH, SELECTED), True), + 'SEARCH': ((SELECTED,), True), + 'SELECT': ((AUTH, SELECTED), False), + 'SETACL': ((AUTH, SELECTED), False), + 'SETANNOTATION':((AUTH, SELECTED), True), + 'SETQUOTA': ((AUTH, SELECTED), False), + 'SORT': ((SELECTED,), True), + 'STARTTLS': ((NONAUTH,), False), + 'STATUS': ((AUTH, SELECTED), True), + 'STORE': ((SELECTED,), True), + 'SUBSCRIBE': ((AUTH, SELECTED), False), + 'THREAD': ((SELECTED,), True), + 'UID': ((SELECTED,), True), + 'UNSUBSCRIBE': ((AUTH, SELECTED), False), + } + +UID_direct = ('SEARCH', 'SORT', 'THREAD') + + +def Int2AP(num): + + """string = Int2AP(num) + Return 'num' converted to a string using characters from the set 'A'..'P' + """ + + val, a2p = [], 'ABCDEFGHIJKLMNOP' + num = int(abs(num)) + while num: + num, mod = divmod(num, 16) + val.insert(0, a2p[mod]) + return ''.join(val) + + + +class Request(object): + + """Private class to represent a request awaiting response.""" + + def __init__(self, parent, name=None, callback=None, cb_arg=None, cb_self=False): + self.parent = parent + self.name = name + self.callback = callback # Function called to process result + if not cb_self: + self.callback_arg = cb_arg # Optional arg passed to "callback" + else: + self.callback_arg = (self, cb_arg) # Self reference required in callback arg + + self.tag = '%s%s' % (parent.tagpre, parent.tagnum) + parent.tagnum += 1 + + self.ready = threading.Event() + self.response = None + self.aborted = None + self.data = None + + + def abort(self, typ, val): + self.aborted = (typ, val) + self.deliver(None) + + + def get_response(self, exc_fmt=None): + self.callback = None + if __debug__: self.parent._log(3, '%s:%s.ready.wait' % (self.name, self.tag)) + self.ready.wait() + + if self.aborted is not None: + typ, val = self.aborted + if exc_fmt is None: + exc_fmt = '%s - %%s' % typ + raise typ(exc_fmt % str(val)) + + return self.response + + + def deliver(self, response): + if self.callback is not None: + self.callback((response, self.callback_arg, self.aborted)) + return + + self.response = response + self.ready.set() + if __debug__: self.parent._log(3, '%s:%s.ready.set' % (self.name, self.tag)) + + + + +class IMAP4(object): + + """Threaded IMAP4 client class. + + Instantiate with: + IMAP4(host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 port); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response (default: no timeout), + debug_buf_lvl - debug level at which buffering is turned off. + + All IMAP4rev1 commands are supported by methods of the same name. + + Each command returns a tuple: (type, [data, ...]) where 'type' + is usually 'OK' or 'NO', and 'data' is either the text from the + tagged response, or untagged results from command. Each 'data' is + either a string, or a tuple. If a tuple, then the first part is the + header of the response, and the second part contains the data (ie: + 'literal' value). + + Errors raise the exception class .error(""). + IMAP4 server errors raise .abort(""), which is + a sub-class of 'error'. Mailbox status changes from READ-WRITE to + READ-ONLY raise the exception class .readonly(""), + which is a sub-class of 'abort'. + + "error" exceptions imply a program error. + "abort" exceptions imply the connection should be reset, and + the command re-tried. + "readonly" exceptions imply the command should be re-tried. + + All commands take two optional named arguments: + 'callback' and 'cb_arg' + If 'callback' is provided then the command is asynchronous, so after + the command is queued for transmission, the call returns immediately + with the tuple (None, None). + The result will be posted by invoking "callback" with one arg, a tuple: + callback((result, cb_arg, None)) + or, if there was a problem: + callback((None, cb_arg, (exception class, reason))) + + Otherwise the command is synchronous (waits for result). But note + that state-changing commands will both block until previous commands + have completed, and block subsequent commands until they have finished. + + All (non-callback) arguments to commands are converted to strings, + except for AUTHENTICATE, and the last argument to APPEND which is + passed as an IMAP4 literal. If necessary (the string contains any + non-printing characters or white-space and isn't enclosed with + either parentheses or double or single quotes) each string is + quoted. However, the 'password' argument to the LOGIN command is + always quoted. If you want to avoid having an argument string + quoted (eg: the 'flags' argument to STORE) then enclose the string + in parentheses (eg: "(\Deleted)"). If you are using "sequence sets" + containing the wildcard character '*', then enclose the argument + in single quotes: the quotes will be removed and the resulting + string passed unquoted. Note also that you can pass in an argument + with a type that doesn't evaluate to 'string_types' (eg: 'bytearray') + and it will be converted to a string without quoting. + + There is one instance variable, 'state', that is useful for tracking + whether the client needs to login to the server. If it has the + value "AUTH" after instantiating the class, then the connection + is pre-authenticated (otherwise it will be "NONAUTH"). Selecting a + mailbox changes the state to be "SELECTED", closing a mailbox changes + back to "AUTH", and once the client has logged out, the state changes + to "LOGOUT" and no further commands may be issued. + + Note: to use this module, you must read the RFCs pertaining to the + IMAP4 protocol, as the semantics of the arguments to each IMAP4 + command are left to the invoker, not to mention the results. Also, + most IMAP servers implement a sub-set of the commands available here. + + Note also that you must call logout() to shut down threads before + discarding an instance. + """ + + class error(Exception): pass # Logical errors - debug required + class abort(error): pass # Service errors - close and retry + class readonly(abort): pass # Mailbox status changed to READ-ONLY + + + continuation_cre = re.compile(r'\+( (?P.*))?') + literal_cre = re.compile(r'.*{(?P\d+)}$') + mapCRLF_cre = re.compile(r'\r\n|\r|\n') + # Need to quote "atom-specials" :- + # "(" / ")" / "{" / SP / 0x00 - 0x1f / 0x7f / "%" / "*" / DQUOTE / "\" / "]" + # so match not the inverse set + mustquote_cre = re.compile(r"[^!#$&'+,./0-9:;<=>?@A-Z\[^_`a-z|}~-]") + response_code_cre = re.compile(r'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') + # sequence_set_cre = re.compile(r"^[0-9]+(:([0-9]+|\*))?(,[0-9]+(:([0-9]+|\*))?)*$") + untagged_response_cre = re.compile(r'\* (?P[A-Z-]+)( (?P.*))?') + untagged_status_cre = re.compile(r'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?') + + + def __init__(self, host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): + + self.state = NONAUTH # IMAP4 protocol state + self.literal = None # A literal argument to a command + self.tagged_commands = {} # Tagged commands awaiting response + self.untagged_responses = [] # [[typ: [data, ...]], ...] + self.mailbox = None # Current mailbox selected + self.is_readonly = False # READ-ONLY desired state + self.idle_rqb = None # Server IDLE Request - see _IdleCont + self.idle_timeout = None # Must prod server occasionally + + self._expecting_data = False # Expecting message data + self._expecting_data_len = 0 # How many characters we expect + self._accumulated_data = [] # Message data accumulated so far + self._literal_expected = None # Message data descriptor + + self.compressor = None # COMPRESS/DEFLATE if not None + self.decompressor = None + + # Create unique tag for this session, + # and compile tagged response matcher. + + self.tagnum = 0 + self.tagpre = Int2AP(random.randint(4096, 65535)) + self.tagre = re.compile(r'(?P' + + self.tagpre + + r'\d+) (?P[A-Z]+) (?P.*)') + + if __debug__: self._init_debug(debug, debug_file, debug_buf_lvl) + + self.resp_timeout = timeout # Timeout waiting for command response + + if timeout is not None and timeout < READ_POLL_TIMEOUT: + self.read_poll_timeout = timeout + else: + self.read_poll_timeout = READ_POLL_TIMEOUT + self.read_size = READ_SIZE + + # Open socket to server. + + self.open(host, port) + + if __debug__: + if debug: + self._mesg('connected to %s on port %s' % (self.host, self.port)) + + # Threading + + if identifier is not None: + self.identifier = identifier + else: + self.identifier = self.host + if self.identifier: + self.identifier += ' ' + + self.Terminate = self.TerminateReader = False + + self.state_change_free = threading.Event() + self.state_change_pending = threading.Lock() + self.commands_lock = threading.Lock() + self.idle_lock = threading.Lock() + + self.ouq = queue.Queue(10) + self.inq = queue.Queue() + + self.wrth = threading.Thread(target=self._writer) + self.wrth.setDaemon(True) + self.wrth.start() + self.rdth = threading.Thread(target=self._reader) + self.rdth.setDaemon(True) + self.rdth.start() + self.inth = threading.Thread(target=self._handler) + self.inth.setDaemon(True) + self.inth.start() + + # Get server welcome message, + # request and store CAPABILITY response. + + try: + self.welcome = self._request_push(tag='continuation').get_response('IMAP4 protocol error: %s')[1] + + if self._get_untagged_response('PREAUTH'): + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + elif self._get_untagged_response('OK'): + if __debug__: self._log(1, 'state => NONAUTH') + else: + raise self.error('unrecognised server welcome message: %s' % repr(self.welcome)) + + typ, dat = self.capability() + if dat == [None]: + raise self.error('no CAPABILITY response from server') + self.capabilities = tuple(dat[-1].upper().split()) + if __debug__: self._log(1, 'CAPABILITY: %r' % (self.capabilities,)) + + for version in AllowedVersions: + if not version in self.capabilities: + continue + self.PROTOCOL_VERSION = version + break + else: + raise self.error('server not IMAP4 compliant') + except: + self._close_threads() + raise + + + def __getattr__(self, attr): + # Allow UPPERCASE variants of IMAP4 command methods. + if attr in Commands: + return getattr(self, attr.lower()) + raise AttributeError("Unknown IMAP4 command: '%s'" % attr) + + + + # Overridable methods + + + def open(self, host=None, port=None): + """open(host=None, port=None) + Setup connection to remote server on "host:port" + (default: localhost:standard IMAP4 port). + This connection will be used by the routines: + read, send, shutdown, socket.""" + + self.host = self._choose_nonull_or_dflt('', host) + self.port = self._choose_nonull_or_dflt(IMAP4_PORT, port) + self.sock = self.open_socket() + self.read_fd = self.sock.fileno() + + + def open_socket(self): + """open_socket() + Open socket choosing first address family available.""" + + msg = (-1, 'could not open socket') + for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + s = socket.socket(af, socktype, proto) + except socket.error as msg: + continue + try: + for i in (0, 1): + try: + s.connect(sa) + break + except socket.error as msg: + if len(msg.args) < 2 or msg.args[0] != errno.EINTR: + raise + else: + raise socket.error(msg) + except socket.error as msg: + s.close() + continue + break + else: + raise socket.error(msg) + + return s + + + def ssl_wrap_socket(self): + + # Allow sending of keep-alive messages - seems to prevent some servers + # from closing SSL, leading to deadlocks. + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + try: + import ssl + if self.ca_certs is not None: + cert_reqs = ssl.CERT_REQUIRED + else: + cert_reqs = ssl.CERT_NONE + + if self.ssl_version == "tls1": + ssl_version = ssl.PROTOCOL_TLSv1 + elif self.ssl_version == "ssl2": + ssl_version = ssl.PROTOCOL_SSLv2 + elif self.ssl_version == "ssl3": + ssl_version = ssl.PROTOCOL_SSLv3 + elif self.ssl_version == "ssl23" or self.ssl_version is None: + ssl_version = ssl.PROTOCOL_SSLv23 + else: + raise socket.sslerror("Invalid SSL version requested: %s", self.ssl_version) + + self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version) + ssl_exc = ssl.SSLError + self.read_fd = self.sock.fileno() + except ImportError: + # No ssl module, and socket.ssl has no fileno(), and does not allow certificate verification + raise socket.sslerror("imaplib2 SSL mode does not work without ssl module") + + if self.cert_verify_cb is not None: + cert_err = self.cert_verify_cb(self.sock.getpeercert(), self.host) + if cert_err: + raise ssl_exc(cert_err) + + + + def start_compressing(self): + """start_compressing() + Enable deflate compression on the socket (RFC 4978).""" + + # rfc 1951 - pure DEFLATE, so use -15 for both windows + self.decompressor = zlib.decompressobj(-15) + self.compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) + + + def read(self, size): + """data = read(size) + Read at most 'size' bytes from remote.""" + + if self.decompressor is None: + return self.sock.recv(size) + + if self.decompressor.unconsumed_tail: + data = self.decompressor.unconsumed_tail + else: + data = self.sock.recv(READ_SIZE) + + return self.decompressor.decompress(data, size) + + + def send(self, data): + """send(data) + Send 'data' to remote.""" + + if self.compressor is not None: + data = self.compressor.compress(data) + data += self.compressor.flush(zlib.Z_SYNC_FLUSH) + + if bytes != str: + self.sock.sendall(bytes(data, 'utf8')) + else: + self.sock.sendall(data) + + + def shutdown(self): + """shutdown() + Close I/O established in "open".""" + + self.sock.close() + + + def socket(self): + """socket = socket() + Return socket instance used to connect to IMAP4 server.""" + + return self.sock + + + + # Utility methods + + + def enable_compression(self): + """enable_compression() + Ask the server to start compressing the connection. + Should be called from user of this class after instantiation, as in: + if 'COMPRESS=DEFLATE' in imapobj.capabilities: + imapobj.enable_compression()""" + + try: + typ, dat = self._simple_command('COMPRESS', 'DEFLATE') + if typ == 'OK': + self.start_compressing() + if __debug__: self._log(1, 'Enabled COMPRESS=DEFLATE') + finally: + self._release_state_change() + + + def pop_untagged_responses(self): + """ for typ,data in pop_untagged_responses(): pass + Generator for any remaining untagged responses. + Returns and removes untagged responses in order of reception. + Use at your own risk!""" + + while self.untagged_responses: + self.commands_lock.acquire() + try: + yield self.untagged_responses.pop(0) + finally: + self.commands_lock.release() + + + def recent(self, **kw): + """(typ, [data]) = recent() + Return 'RECENT' responses if any exist, + else prompt server for an update using the 'NOOP' command. + 'data' is None if no new messages, + else list of RECENT responses, most recent last.""" + + name = 'RECENT' + typ, dat = self._untagged_response(None, [None], name) + if dat != [None]: + return self._deliver_dat(typ, dat, kw) + kw['untagged_response'] = name + return self.noop(**kw) # Prod server for response + + + def response(self, code, **kw): + """(code, [data]) = response(code) + Return data for response 'code' if received, or None. + Old value for response 'code' is cleared.""" + + typ, dat = self._untagged_response(code, [None], code.upper()) + return self._deliver_dat(typ, dat, kw) + + + + + # IMAP4 commands + + + def append(self, mailbox, flags, date_time, message, **kw): + """(typ, [data]) = append(mailbox, flags, date_time, message) + Append message to named mailbox. + All args except `message' can be None.""" + + name = 'APPEND' + if not mailbox: + mailbox = 'INBOX' + if flags: + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags + else: + flags = None + if date_time: + date_time = Time2Internaldate(date_time) + else: + date_time = None + self.literal = self.mapCRLF_cre.sub(CRLF, message) + try: + return self._simple_command(name, mailbox, flags, date_time, **kw) + finally: + self._release_state_change() + + + def authenticate(self, mechanism, authobject, **kw): + """(typ, [data]) = authenticate(mechanism, authobject) + Authenticate command - requires response processing. + + 'mechanism' specifies which authentication mechanism is to + be used - it must appear in .capabilities in the + form AUTH=. + + 'authobject' must be a callable object: + + data = authobject(response) + + It will be called to process server continuation responses. + It should return data that will be encoded and sent to server. + It should return None if the client abort response '*' should + be sent instead.""" + + self.literal = _Authenticator(authobject).process + try: + typ, dat = self._simple_command('AUTHENTICATE', mechanism.upper()) + if typ != 'OK': + self._deliver_exc(self.error, dat[-1], kw) + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + finally: + self._release_state_change() + return self._deliver_dat(typ, dat, kw) + + + def capability(self, **kw): + """(typ, [data]) = capability() + Fetch capabilities list from server.""" + + name = 'CAPABILITY' + kw['untagged_response'] = name + return self._simple_command(name, **kw) + + + def check(self, **kw): + """(typ, [data]) = check() + Checkpoint mailbox on server.""" + + return self._simple_command('CHECK', **kw) + + + def close(self, **kw): + """(typ, [data]) = close() + Close currently selected mailbox. + + Deleted messages are removed from writable mailbox. + This is the recommended command before 'LOGOUT'.""" + + if self.state != 'SELECTED': + raise self.error('No mailbox selected.') + try: + typ, dat = self._simple_command('CLOSE') + finally: + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + self._release_state_change() + return self._deliver_dat(typ, dat, kw) + + + def copy(self, message_set, new_mailbox, **kw): + """(typ, [data]) = copy(message_set, new_mailbox) + Copy 'message_set' messages onto end of 'new_mailbox'.""" + + return self._simple_command('COPY', message_set, new_mailbox, **kw) + + + def create(self, mailbox, **kw): + """(typ, [data]) = create(mailbox) + Create new mailbox.""" + + return self._simple_command('CREATE', mailbox, **kw) + + + def delete(self, mailbox, **kw): + """(typ, [data]) = delete(mailbox) + Delete old mailbox.""" + + return self._simple_command('DELETE', mailbox, **kw) + + + def deleteacl(self, mailbox, who, **kw): + """(typ, [data]) = deleteacl(mailbox, who) + Delete the ACLs (remove any rights) set for who on mailbox.""" + + return self._simple_command('DELETEACL', mailbox, who, **kw) + + + def examine(self, mailbox='INBOX', **kw): + """(typ, [data]) = examine(mailbox='INBOX') + Select a mailbox for READ-ONLY access. (Flushes all untagged responses.) + 'data' is count of messages in mailbox ('EXISTS' response). + Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so + other responses should be obtained via "response('FLAGS')" etc.""" + + return self.select(mailbox=mailbox, readonly=True, **kw) + + + def expunge(self, **kw): + """(typ, [data]) = expunge() + Permanently remove deleted items from selected mailbox. + Generates 'EXPUNGE' response for each deleted message. + 'data' is list of 'EXPUNGE'd message numbers in order received.""" + + name = 'EXPUNGE' + kw['untagged_response'] = name + return self._simple_command(name, **kw) + + + def fetch(self, message_set, message_parts, **kw): + """(typ, [data, ...]) = fetch(message_set, message_parts) + Fetch (parts of) messages. + 'message_parts' should be a string of selected parts + enclosed in parentheses, eg: "(UID BODY[TEXT])". + 'data' are tuples of message part envelope and data, + followed by a string containing the trailer.""" + + name = 'FETCH' + kw['untagged_response'] = name + return self._simple_command(name, message_set, message_parts, **kw) + + + def getacl(self, mailbox, **kw): + """(typ, [data]) = getacl(mailbox) + Get the ACLs for a mailbox.""" + + kw['untagged_response'] = 'ACL' + return self._simple_command('GETACL', mailbox, **kw) + + + def getannotation(self, mailbox, entry, attribute, **kw): + """(typ, [data]) = getannotation(mailbox, entry, attribute) + Retrieve ANNOTATIONs.""" + + kw['untagged_response'] = 'ANNOTATION' + return self._simple_command('GETANNOTATION', mailbox, entry, attribute, **kw) + + + def getquota(self, root, **kw): + """(typ, [data]) = getquota(root) + Get the quota root's resource usage and limits. + (Part of the IMAP4 QUOTA extension defined in rfc2087.)""" + + kw['untagged_response'] = 'QUOTA' + return self._simple_command('GETQUOTA', root, **kw) + + + def getquotaroot(self, mailbox, **kw): + # Hmmm, this is non-std! Left for backwards-compatibility, sigh. + # NB: usage should have been defined as: + # (typ, [QUOTAROOT responses...]) = getquotaroot(mailbox) + # (typ, [QUOTA responses...]) = response('QUOTA') + """(typ, [[QUOTAROOT responses...], [QUOTA responses...]]) = getquotaroot(mailbox) + Get the list of quota roots for the named mailbox.""" + + typ, dat = self._simple_command('GETQUOTAROOT', mailbox) + typ, quota = self._untagged_response(typ, dat, 'QUOTA') + typ, quotaroot = self._untagged_response(typ, dat, 'QUOTAROOT') + return self._deliver_dat(typ, [quotaroot, quota], kw) + + + def id(self, *kv_pairs, **kw): + """(typ, [data]) = .id(kv_pairs) + 'kv_pairs' is a possibly empty list of keys and values. + 'data' is a list of ID key value pairs or NIL. + NB: a single argument is assumed to be correctly formatted and is passed through unchanged + (for backward compatibility with earlier version). + Exchange information for problem analysis and determination. + The ID extension is defined in RFC 2971. """ + + name = 'ID' + kw['untagged_response'] = name + + if not kv_pairs: + data = 'NIL' + elif len(kv_pairs) == 1: + data = kv_pairs[0] # Assume invoker passing correctly formatted string (back-compat) + else: + data = '(%s)' % ' '.join([(arg and self._quote(arg) or 'NIL') for arg in kv_pairs]) + + return self._simple_command(name, data, **kw) + + + def idle(self, timeout=None, **kw): + """"(typ, [data]) = idle(timeout=None) + Put server into IDLE mode until server notifies some change, + or 'timeout' (secs) occurs (default: 29 minutes), + or another IMAP4 command is scheduled.""" + + name = 'IDLE' + self.literal = _IdleCont(self, timeout).process + try: + return self._simple_command(name, **kw) + finally: + self._release_state_change() + + + def list(self, directory='""', pattern='*', **kw): + """(typ, [data]) = list(directory='""', pattern='*') + List mailbox names in directory matching pattern. + 'data' is list of LIST responses. + + NB: for 'pattern': + % matches all except separator ( so LIST "" "%" returns names at root) + * matches all (so LIST "" "*" returns whole directory tree from root)""" + + name = 'LIST' + kw['untagged_response'] = name + return self._simple_command(name, directory, pattern, **kw) + + + def login(self, user, password, **kw): + """(typ, [data]) = login(user, password) + Identify client using plaintext password. + NB: 'password' will be quoted.""" + + try: + typ, dat = self._simple_command('LOGIN', user, self._quote(password)) + if typ != 'OK': + self._deliver_exc(self.error, dat[-1], kw) + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + finally: + self._release_state_change() + return self._deliver_dat(typ, dat, kw) + + + def login_cram_md5(self, user, password, **kw): + """(typ, [data]) = login_cram_md5(user, password) + Force use of CRAM-MD5 authentication.""" + + self.user, self.password = user, password + return self.authenticate('CRAM-MD5', self._CRAM_MD5_AUTH, **kw) + + + def _CRAM_MD5_AUTH(self, challenge): + """Authobject to use with CRAM-MD5 authentication.""" + import hmac + return self.user + " " + hmac.HMAC(self.password, challenge).hexdigest() + + + def logout(self, **kw): + """(typ, [data]) = logout() + Shutdown connection to server. + Returns server 'BYE' response. + NB: You must call this to shut down threads before discarding an instance.""" + + self.state = LOGOUT + if __debug__: self._log(1, 'state => LOGOUT') + + try: + try: + typ, dat = self._simple_command('LOGOUT') + except: + typ, dat = 'NO', ['%s: %s' % sys.exc_info()[:2]] + if __debug__: self._log(1, dat) + + self._close_threads() + finally: + self._release_state_change() + + if __debug__: self._log(1, 'connection closed') + + bye = self._get_untagged_response('BYE', leave=True) + if bye: + typ, dat = 'BYE', bye + return self._deliver_dat(typ, dat, kw) + + + def lsub(self, directory='""', pattern='*', **kw): + """(typ, [data, ...]) = lsub(directory='""', pattern='*') + List 'subscribed' mailbox names in directory matching pattern. + 'data' are tuples of message part envelope and data.""" + + name = 'LSUB' + kw['untagged_response'] = name + return self._simple_command(name, directory, pattern, **kw) + + + def myrights(self, mailbox, **kw): + """(typ, [data]) = myrights(mailbox) + Show my ACLs for a mailbox (i.e. the rights that I have on mailbox).""" + + name = 'MYRIGHTS' + kw['untagged_response'] = name + return self._simple_command(name, mailbox, **kw) + + + def namespace(self, **kw): + """(typ, [data, ...]) = namespace() + Returns IMAP namespaces ala rfc2342.""" + + name = 'NAMESPACE' + kw['untagged_response'] = name + return self._simple_command(name, **kw) + + + def noop(self, **kw): + """(typ, [data]) = noop() + Send NOOP command.""" + + if __debug__: self._dump_ur(3) + return self._simple_command('NOOP', **kw) + + + def partial(self, message_num, message_part, start, length, **kw): + """(typ, [data, ...]) = partial(message_num, message_part, start, length) + Fetch truncated part of a message. + 'data' is tuple of message part envelope and data. + NB: obsolete.""" + + name = 'PARTIAL' + kw['untagged_response'] = 'FETCH' + return self._simple_command(name, message_num, message_part, start, length, **kw) + + + def proxyauth(self, user, **kw): + """(typ, [data]) = proxyauth(user) + Assume authentication as 'user'. + (Allows an authorised administrator to proxy into any user's mailbox.)""" + + try: + return self._simple_command('PROXYAUTH', user, **kw) + finally: + self._release_state_change() + + + def rename(self, oldmailbox, newmailbox, **kw): + """(typ, [data]) = rename(oldmailbox, newmailbox) + Rename old mailbox name to new.""" + + return self._simple_command('RENAME', oldmailbox, newmailbox, **kw) + + + def search(self, charset, *criteria, **kw): + """(typ, [data]) = search(charset, criterion, ...) + Search mailbox for matching messages. + 'data' is space separated list of matching message numbers.""" + + name = 'SEARCH' + kw['untagged_response'] = name + if charset: + return self._simple_command(name, 'CHARSET', charset, *criteria, **kw) + return self._simple_command(name, *criteria, **kw) + + + def select(self, mailbox='INBOX', readonly=False, **kw): + """(typ, [data]) = select(mailbox='INBOX', readonly=False) + Select a mailbox. (Flushes all untagged responses.) + 'data' is count of messages in mailbox ('EXISTS' response). + Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so + other responses should be obtained via "response('FLAGS')" etc.""" + + self.mailbox = mailbox + + self.is_readonly = bool(readonly) + if readonly: + name = 'EXAMINE' + else: + name = 'SELECT' + try: + rqb = self._command(name, mailbox) + typ, dat = rqb.get_response('command: %s => %%s' % rqb.name) + if typ != 'OK': + if self.state == SELECTED: + self.state = AUTH + if __debug__: self._log(1, 'state => AUTH') + if typ == 'BAD': + self._deliver_exc(self.error, '%s command error: %s %s. Data: %.100s' % (name, typ, dat, mailbox), kw) + return self._deliver_dat(typ, dat, kw) + self.state = SELECTED + if __debug__: self._log(1, 'state => SELECTED') + finally: + self._release_state_change() + + if self._get_untagged_response('READ-ONLY', leave=True) and not readonly: + if __debug__: self._dump_ur(1) + self._deliver_exc(self.readonly, '%s is not writable' % mailbox, kw) + typ, dat = self._untagged_response(typ, [None], 'EXISTS') + return self._deliver_dat(typ, dat, kw) + + + def setacl(self, mailbox, who, what, **kw): + """(typ, [data]) = setacl(mailbox, who, what) + Set a mailbox acl.""" + + try: + return self._simple_command('SETACL', mailbox, who, what, **kw) + finally: + self._release_state_change() + + + def setannotation(self, *args, **kw): + """(typ, [data]) = setannotation(mailbox[, entry, attribute]+) + Set ANNOTATIONs.""" + + kw['untagged_response'] = 'ANNOTATION' + return self._simple_command('SETANNOTATION', *args, **kw) + + + def setquota(self, root, limits, **kw): + """(typ, [data]) = setquota(root, limits) + Set the quota root's resource limits.""" + + kw['untagged_response'] = 'QUOTA' + try: + return self._simple_command('SETQUOTA', root, limits, **kw) + finally: + self._release_state_change() + + + def sort(self, sort_criteria, charset, *search_criteria, **kw): + """(typ, [data]) = sort(sort_criteria, charset, search_criteria, ...) + IMAP4rev1 extension SORT command.""" + + name = 'SORT' + if (sort_criteria[0],sort_criteria[-1]) != ('(',')'): + sort_criteria = '(%s)' % sort_criteria + kw['untagged_response'] = name + return self._simple_command(name, sort_criteria, charset, *search_criteria, **kw) + + + def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", **kw): + """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23") + Start TLS negotiation as per RFC 2595.""" + + name = 'STARTTLS' + + if name not in self.capabilities: + raise self.abort('TLS not supported by server') + + if hasattr(self, '_tls_established') and self._tls_established: + raise self.abort('TLS session already established') + + # Must now shutdown reader thread after next response, and restart after changing read_fd + + self.read_size = 1 # Don't consume TLS handshake + self.TerminateReader = True + + try: + typ, dat = self._simple_command(name) + finally: + self._release_state_change() + self.rdth.join() + self.TerminateReader = False + self.read_size = READ_SIZE + + if typ != 'OK': + # Restart reader thread and error + self.rdth = threading.Thread(target=self._reader) + self.rdth.setDaemon(True) + self.rdth.start() + raise self.error("Couldn't establish TLS session: %s" % dat) + + self.keyfile = keyfile + self.certfile = certfile + self.ca_certs = ca_certs + self.cert_verify_cb = cert_verify_cb + self.ssl_version = ssl_version + + try: + self.ssl_wrap_socket() + finally: + # Restart reader thread + self.rdth = threading.Thread(target=self._reader) + self.rdth.setDaemon(True) + self.rdth.start() + + typ, dat = self.capability() + if dat == [None]: + raise self.error('no CAPABILITY response from server') + self.capabilities = tuple(dat[-1].upper().split()) + + self._tls_established = True + + typ, dat = self._untagged_response(typ, dat, name) + return self._deliver_dat(typ, dat, kw) + + + def status(self, mailbox, names, **kw): + """(typ, [data]) = status(mailbox, names) + Request named status conditions for mailbox.""" + + name = 'STATUS' + kw['untagged_response'] = name + return self._simple_command(name, mailbox, names, **kw) + + + def store(self, message_set, command, flags, **kw): + """(typ, [data]) = store(message_set, command, flags) + Alters flag dispositions for messages in mailbox.""" + + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags # Avoid quoting the flags + kw['untagged_response'] = 'FETCH' + return self._simple_command('STORE', message_set, command, flags, **kw) + + + def subscribe(self, mailbox, **kw): + """(typ, [data]) = subscribe(mailbox) + Subscribe to new mailbox.""" + + try: + return self._simple_command('SUBSCRIBE', mailbox, **kw) + finally: + self._release_state_change() + + + def thread(self, threading_algorithm, charset, *search_criteria, **kw): + """(type, [data]) = thread(threading_alogrithm, charset, search_criteria, ...) + IMAPrev1 extension THREAD command.""" + + name = 'THREAD' + kw['untagged_response'] = name + return self._simple_command(name, threading_algorithm, charset, *search_criteria, **kw) + + + def uid(self, command, *args, **kw): + """(typ, [data]) = uid(command, arg, ...) + Execute "command arg ..." with messages identified by UID, + rather than message number. + Assumes 'command' is legal in current state. + Returns response appropriate to 'command'.""" + + command = command.upper() + if command in UID_direct: + resp = command + else: + resp = 'FETCH' + kw['untagged_response'] = resp + return self._simple_command('UID', command, *args, **kw) + + + def unsubscribe(self, mailbox, **kw): + """(typ, [data]) = unsubscribe(mailbox) + Unsubscribe from old mailbox.""" + + try: + return self._simple_command('UNSUBSCRIBE', mailbox, **kw) + finally: + self._release_state_change() + + + def xatom(self, name, *args, **kw): + """(typ, [data]) = xatom(name, arg, ...) + Allow simple extension commands notified by server in CAPABILITY response. + Assumes extension command 'name' is legal in current state. + Returns response appropriate to extension command 'name'.""" + + name = name.upper() + if not name in Commands: + Commands[name] = ((self.state,), False) + try: + return self._simple_command(name, *args, **kw) + finally: + self._release_state_change() + + + + # Internal methods + + + def _append_untagged(self, typ, dat): + + # Append new 'dat' to end of last untagged response if same 'typ', + # else append new response. + + if dat is None: dat = '' + + self.commands_lock.acquire() + + if self.untagged_responses: + urn, urd = self.untagged_responses[-1] + if urn != typ: + urd = None + else: + urd = None + + if urd is None: + urd = [] + self.untagged_responses.append([typ, urd]) + + urd.append(dat) + + self.commands_lock.release() + + if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%s"]' % (typ, len(urd)-1, dat)) + + + def _check_bye(self): + + bye = self._get_untagged_response('BYE', leave=True) + if bye: + raise self.abort(bye[-1]) + + + def _checkquote(self, arg): + + # Must quote command args if "atom-specials" present, + # and not already quoted. NB: single quotes are removed. + + if not isinstance(arg, string_types): + return arg + if len(arg) >= 2 and (arg[0],arg[-1]) in (('(',')'),('"','"')): + return arg + if len(arg) >= 2 and (arg[0],arg[-1]) in (("'","'"),): + return arg[1:-1] + if arg and self.mustquote_cre.search(arg) is None: + return arg + return self._quote(arg) + + + def _choose_nonull_or_dflt(self, dflt, *args): + if isinstance(dflt, string_types): + dflttyp = string_types # Allow any string type + else: + dflttyp = type(dflt) + for arg in args: + if arg is not None: + if isinstance(arg, dflttyp): + return arg + if __debug__: self._log(0, 'bad arg is %s, expecting %s' % (type(arg), dflttyp)) + return dflt + + + def _command(self, name, *args, **kw): + + if Commands[name][CMD_VAL_ASYNC]: + cmdtyp = 'async' + else: + cmdtyp = 'sync' + + if __debug__: self._log(1, '[%s] %s %s' % (cmdtyp, name, args)) + + if __debug__: self._log(3, 'state_change_pending.acquire') + self.state_change_pending.acquire() + + self._end_idle() + + if cmdtyp == 'async': + self.state_change_pending.release() + if __debug__: self._log(3, 'state_change_pending.release') + else: + # Need to wait for all async commands to complete + self._check_bye() + self.commands_lock.acquire() + if self.tagged_commands: + self.state_change_free.clear() + need_event = True + else: + need_event = False + self.commands_lock.release() + if need_event: + if __debug__: self._log(3, 'sync command %s waiting for empty commands Q' % name) + self.state_change_free.wait() + if __debug__: self._log(3, 'sync command %s proceeding' % name) + + if self.state not in Commands[name][CMD_VAL_STATES]: + self.literal = None + raise self.error('command %s illegal in state %s' + % (name, self.state)) + + self._check_bye() + + if name in ('EXAMINE', 'SELECT'): + self.commands_lock.acquire() + self.untagged_responses = [] # Flush all untagged responses + self.commands_lock.release() + else: + for typ in ('OK', 'NO', 'BAD'): + while self._get_untagged_response(typ): + continue + + if self._get_untagged_response('READ-ONLY', leave=True) and not self.is_readonly: + self.literal = None + raise self.readonly('mailbox status changed to READ-ONLY') + + if self.Terminate: + raise self.abort('connection closed') + + rqb = self._request_push(name=name, **kw) + + data = '%s %s' % (rqb.tag, name) + for arg in args: + if arg is None: continue + data = '%s %s' % (data, self._checkquote(arg)) + + literal = self.literal + if literal is not None: + self.literal = None + if isinstance(literal, string_types): + literator = None + data = '%s {%s}' % (data, len(literal)) + else: + literator = literal + + if __debug__: self._log(4, 'data=%s' % data) + + rqb.data = '%s%s' % (data, CRLF) + + if literal is None: + self.ouq.put(rqb) + return rqb + + # Must setup continuation expectancy *before* ouq.put + crqb = self._request_push(tag='continuation') + + self.ouq.put(rqb) + + while True: + # Wait for continuation response + + ok, data = crqb.get_response('command: %s => %%s' % name) + if __debug__: self._log(4, 'continuation => %s, %s' % (ok, data)) + + # NO/BAD response? + + if not ok: + break + + # Send literal + + if literator is not None: + literal = literator(data, rqb) + + if literal is None: + break + + if literator is not None: + # Need new request for next continuation response + crqb = self._request_push(tag='continuation') + + if __debug__: self._log(4, 'write literal size %s' % len(literal)) + crqb.data = '%s%s' % (literal, CRLF) + self.ouq.put(crqb) + + if literator is None: + break + + return rqb + + + def _command_complete(self, rqb, kw): + + # Called for non-callback commands + + self._check_bye() + typ, dat = rqb.get_response('command: %s => %%s' % rqb.name) + if typ == 'BAD': + if __debug__: self._print_log() + raise self.error('%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) + if 'untagged_response' in kw: + return self._untagged_response(typ, dat, kw['untagged_response']) + return typ, dat + + + def _command_completer(self, cb_arg_list): + + # Called for callback commands + (response, cb_arg, error) = cb_arg_list + rqb, kw = cb_arg + rqb.callback = kw['callback'] + rqb.callback_arg = kw.get('cb_arg') + if error is not None: + if __debug__: self._print_log() + typ, val = error + rqb.abort(typ, val) + return + bye = self._get_untagged_response('BYE', leave=True) + if bye: + rqb.abort(self.abort, bye[-1]) + return + typ, dat = response + if typ == 'BAD': + if __debug__: self._print_log() + rqb.abort(self.error, '%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) + return + if 'untagged_response' in kw: + response = self._untagged_response(typ, dat, kw['untagged_response']) + rqb.deliver(response) + + + def _deliver_dat(self, typ, dat, kw): + + if 'callback' in kw: + kw['callback'](((typ, dat), kw.get('cb_arg'), None)) + return typ, dat + + + def _deliver_exc(self, exc, dat, kw): + + if 'callback' in kw: + kw['callback']((None, kw.get('cb_arg'), (exc, dat))) + raise exc(dat) + + + def _end_idle(self): + + self.idle_lock.acquire() + irqb = self.idle_rqb + if irqb is None: + self.idle_lock.release() + return + self.idle_rqb = None + self.idle_timeout = None + self.idle_lock.release() + irqb.data = 'DONE%s' % CRLF + self.ouq.put(irqb) + if __debug__: self._log(2, 'server IDLE finished') + + + def _get_untagged_response(self, name, leave=False): + + self.commands_lock.acquire() + + for i, (typ, dat) in enumerate(self.untagged_responses): + if typ == name: + if not leave: + del self.untagged_responses[i] + self.commands_lock.release() + if __debug__: self._log(5, '_get_untagged_response(%s) => %s' % (name, dat)) + return dat + + self.commands_lock.release() + return None + + + def _match(self, cre, s): + + # Run compiled regular expression 'cre' match method on 's'. + # Save result, return success. + + self.mo = cre.match(s) + return self.mo is not None + + + def _put_response(self, resp): + + if self._expecting_data: + rlen = len(resp) + dlen = min(self._expecting_data_len, rlen) + if __debug__: self._log(5, '_put_response expecting data len %s, got %s' % (self._expecting_data_len, rlen)) + self._expecting_data_len -= dlen + self._expecting_data = (self._expecting_data_len != 0) + if rlen <= dlen: + self._accumulated_data.append(resp) + return + self._accumulated_data.append(resp[:dlen]) + resp = resp[dlen:] + + if self._accumulated_data: + typ, dat = self._literal_expected + self._append_untagged(typ, (dat, ''.join(self._accumulated_data))) + self._accumulated_data = [] + + # Protocol mandates all lines terminated by CRLF + resp = resp[:-2] + if __debug__: self._log(5, '_put_response(%s)' % resp) + + if 'continuation' in self.tagged_commands: + continuation_expected = True + else: + continuation_expected = False + + if self._literal_expected is not None: + dat = resp + if self._match(self.literal_cre, dat): + self._literal_expected[1] = dat + self._expecting_data = True + self._expecting_data_len = int(self.mo.group('size')) + if __debug__: self._log(4, 'expecting literal size %s' % self._expecting_data_len) + return + typ = self._literal_expected[0] + self._literal_expected = None + if dat: + self._append_untagged(typ, dat) # Tail + if __debug__: self._log(4, 'literal completed') + else: + # Command completion response? + if self._match(self.tagre, resp): + tag = self.mo.group('tag') + typ = self.mo.group('type') + dat = self.mo.group('data') + if typ in ('OK', 'NO', 'BAD') and self._match(self.response_code_cre, dat): + self._append_untagged(self.mo.group('type'), self.mo.group('data')) + if not tag in self.tagged_commands: + if __debug__: self._log(1, 'unexpected tagged response: %s' % resp) + else: + self._request_pop(tag, (typ, [dat])) + else: + dat2 = None + + # '*' (untagged) responses? + + if not self._match(self.untagged_response_cre, resp): + if self._match(self.untagged_status_cre, resp): + dat2 = self.mo.group('data2') + + if self.mo is None: + # Only other possibility is '+' (continuation) response... + + if self._match(self.continuation_cre, resp): + if not continuation_expected: + if __debug__: self._log(1, "unexpected continuation response: '%s'" % resp) + return + self._request_pop('continuation', (True, self.mo.group('data'))) + return + + if __debug__: self._log(1, "unexpected response: '%s'" % resp) + return + + typ = self.mo.group('type') + dat = self.mo.group('data') + if dat is None: dat = '' # Null untagged response + if dat2: dat = dat + ' ' + dat2 + + # Is there a literal to come? + + if self._match(self.literal_cre, dat): + self._expecting_data = True + self._expecting_data_len = int(self.mo.group('size')) + if __debug__: self._log(4, 'read literal size %s' % self._expecting_data_len) + self._literal_expected = [typ, dat] + return + + self._append_untagged(typ, dat) + if typ in ('OK', 'NO', 'BAD') and self._match(self.response_code_cre, dat): + self._append_untagged(self.mo.group('type'), self.mo.group('data')) + + if typ != 'OK': # NO, BYE, IDLE + self._end_idle() + + # Command waiting for aborted continuation response? + + if continuation_expected: + self._request_pop('continuation', (False, resp)) + + # Bad news? + + if typ in ('NO', 'BAD', 'BYE'): + if typ == 'BYE': + self.Terminate = True + if __debug__: self._log(1, '%s response: %s' % (typ, dat)) + + + def _quote(self, arg): + + return '"%s"' % arg.replace('\\', '\\\\').replace('"', '\\"') + + + def _release_state_change(self): + + if self.state_change_pending.locked(): + self.state_change_pending.release() + if __debug__: self._log(3, 'state_change_pending.release') + + + def _request_pop(self, name, data): + + self.commands_lock.acquire() + rqb = self.tagged_commands.pop(name) + if not self.tagged_commands: + if __debug__: self._log(3, 'state_change_free.set') + self.state_change_free.set() + self.commands_lock.release() + if __debug__: self._log(4, '_request_pop(%s, %s) = %s' % (name, data, rqb.tag)) + rqb.deliver(data) + + + def _request_push(self, tag=None, name=None, **kw): + + self.commands_lock.acquire() + rqb = Request(self, name=name, **kw) + if tag is None: + tag = rqb.tag + self.tagged_commands[tag] = rqb + self.commands_lock.release() + if __debug__: self._log(4, '_request_push(%s, %s, %s) = %s' % (tag, name, repr(kw), rqb.tag)) + return rqb + + + def _simple_command(self, name, *args, **kw): + + if 'callback' in kw: + # Note: old calling sequence for back-compat with python <2.6 + self._command(name, callback=self._command_completer, cb_arg=kw, cb_self=True, *args) + return (None, None) + return self._command_complete(self._command(name, *args), kw) + + + def _untagged_response(self, typ, dat, name): + + if typ == 'NO': + return typ, dat + data = self._get_untagged_response(name) + if not data: + return typ, [None] + while True: + dat = self._get_untagged_response(name) + if not dat: + break + data += dat + if __debug__: self._log(4, '_untagged_response(%s, ?, %s) => %s' % (typ, name, data)) + return typ, data + + + + # Threads + + + def _close_threads(self): + + if __debug__: self._log(1, '_close_threads') + + self.ouq.put(None) + self.wrth.join() + + if __debug__: self._log(1, 'call shutdown') + + self.shutdown() + + self.rdth.join() + self.inth.join() + + + def _handler(self): + + resp_timeout = self.resp_timeout + + threading.currentThread().setName(self.identifier + 'handler') + + time.sleep(0.1) # Don't start handling before main thread ready + + if __debug__: self._log(1, 'starting') + + typ, val = self.abort, 'connection terminated' + + while not self.Terminate: + + self.idle_lock.acquire() + if self.idle_timeout is not None: + timeout = self.idle_timeout - time.time() + if timeout <= 0: + timeout = 1 + if __debug__: + if self.idle_rqb is not None: + self._log(5, 'server IDLING, timeout=%.2f' % timeout) + else: + timeout = resp_timeout + self.idle_lock.release() + + try: + line = self.inq.get(True, timeout) + except queue.Empty: + if self.idle_rqb is None: + if resp_timeout is not None and self.tagged_commands: + if __debug__: self._log(1, 'response timeout') + typ, val = self.abort, 'no response after %s secs' % resp_timeout + break + continue + if self.idle_timeout > time.time(): + continue + if __debug__: self._log(2, 'server IDLE timedout') + line = IDLE_TIMEOUT_RESPONSE + + if line is None: + if __debug__: self._log(1, 'inq None - terminating') + break + + if not isinstance(line, string_types): + typ, val = line + break + + try: + self._put_response(line) + except: + typ, val = self.error, 'program error: %s - %s' % sys.exc_info()[:2] + break + + self.Terminate = True + + if __debug__: self._log(1, 'terminating: %s' % repr(val)) + + while not self.ouq.empty(): + try: + qel = self.ouq.get_nowait() + if qel is not None: + qel.abort(typ, val) + except queue.Empty: + break + self.ouq.put(None) + + self.commands_lock.acquire() + for name in list(self.tagged_commands.keys()): + rqb = self.tagged_commands.pop(name) + rqb.abort(typ, val) + self.state_change_free.set() + self.commands_lock.release() + if __debug__: self._log(3, 'state_change_free.set') + + if __debug__: self._log(1, 'finished') + + + if hasattr(select_module, "poll"): + + def _reader(self): + + threading.currentThread().setName(self.identifier + 'reader') + + if __debug__: self._log(1, 'starting using poll') + + def poll_error(state): + PollErrors = { + select.POLLERR: 'Error', + select.POLLHUP: 'Hang up', + select.POLLNVAL: 'Invalid request: descriptor not open', + } + return ' '.join([PollErrors[s] for s in PollErrors.keys() if (s & state)]) + + line_part = '' + + poll = select.poll() + + poll.register(self.read_fd, select.POLLIN) + + rxzero = 0 + terminate = False + read_poll_timeout = self.read_poll_timeout * 1000 # poll() timeout is in millisecs + + while not (terminate or self.Terminate): + if self.state == LOGOUT: + timeout = 1 + else: + timeout = read_poll_timeout + try: + r = poll.poll(timeout) + if __debug__: self._log(5, 'poll => %s' % repr(r)) + if not r: + continue # Timeout + + fd,state = r[0] + + if state & select.POLLIN: + data = self.read(self.read_size) # Drain ssl buffer if present + start = 0 + dlen = len(data) + if __debug__: self._log(5, 'rcvd %s' % dlen) + if dlen == 0: + rxzero += 1 + if rxzero > 5: + raise IOError("Too many read 0") + time.sleep(0.1) + continue # Try again + rxzero = 0 + + while True: + if bytes != str: + stop = data.find(b'\n', start) + if stop < 0: + line_part += data[start:].decode() + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop].decode() + else: + stop = data.find('\n', start) + if stop < 0: + line_part += data[start:] + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop] + if __debug__: self._log(4, '< %s' % line) + self.inq.put(line) + if self.TerminateReader: + terminate = True + + if state & ~(select.POLLIN): + raise IOError(poll_error(state)) + except: + reason = 'socket error: %s - %s' % sys.exc_info()[:2] + if __debug__: + if not self.Terminate: + self._print_log() + if self.debug: self.debug += 4 # Output all + self._log(1, reason) + self.inq.put((self.abort, reason)) + break + + poll.unregister(self.read_fd) + + if __debug__: self._log(1, 'finished') + + else: + + # No "poll" - use select() + + def _reader(self): + + threading.currentThread().setName(self.identifier + 'reader') + + if __debug__: self._log(1, 'starting using select') + + line_part = '' + + rxzero = 0 + terminate = False + + while not (terminate or self.Terminate): + if self.state == LOGOUT: + timeout = 1 + else: + timeout = self.read_poll_timeout + try: + r,w,e = select.select([self.read_fd], [], [], timeout) + if __debug__: self._log(5, 'select => %s, %s, %s' % (r,w,e)) + if not r: # Timeout + continue + + data = self.read(self.read_size) # Drain ssl buffer if present + start = 0 + dlen = len(data) + if __debug__: self._log(5, 'rcvd %s' % dlen) + if dlen == 0: + rxzero += 1 + if rxzero > 5: + raise IOError("Too many read 0") + time.sleep(0.1) + continue # Try again + rxzero = 0 + + while True: + if bytes != str: + stop = data.find(b'\n', start) + if stop < 0: + line_part += data[start:].decode() + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop].decode() + else: + stop = data.find('\n', start) + if stop < 0: + line_part += data[start:] + break + stop += 1 + line_part, start, line = \ + '', stop, line_part + data[start:stop] + if __debug__: self._log(4, '< %s' % line) + self.inq.put(line) + if self.TerminateReader: + terminate = True + except: + reason = 'socket error: %s - %s' % sys.exc_info()[:2] + if __debug__: + if not self.Terminate: + self._print_log() + if self.debug: self.debug += 4 # Output all + self._log(1, reason) + self.inq.put((self.abort, reason)) + break + + if __debug__: self._log(1, 'finished') + + + def _writer(self): + + threading.currentThread().setName(self.identifier + 'writer') + + if __debug__: self._log(1, 'starting') + + reason = 'Terminated' + + while not self.Terminate: + rqb = self.ouq.get() + if rqb is None: + break # Outq flushed + + try: + self.send(rqb.data) + if __debug__: self._log(4, '> %s' % rqb.data) + except: + reason = 'socket error: %s - %s' % sys.exc_info()[:2] + if __debug__: + if not self.Terminate: + self._print_log() + if self.debug: self.debug += 4 # Output all + self._log(1, reason) + rqb.abort(self.abort, reason) + break + + self.inq.put((self.abort, reason)) + + if __debug__: self._log(1, 'finished') + + + + # Debugging + + + if __debug__: + + def _init_debug(self, debug=None, debug_file=None, debug_buf_lvl=None): + self.debug_lock = threading.Lock() + + self.debug = self._choose_nonull_or_dflt(0, debug, Debug) + self.debug_file = self._choose_nonull_or_dflt(sys.stderr, debug_file) + self.debug_buf_lvl = self._choose_nonull_or_dflt(DFLT_DEBUG_BUF_LVL, debug_buf_lvl) + + self._cmd_log_len = 20 + self._cmd_log_idx = 0 + self._cmd_log = {} # Last `_cmd_log_len' interactions + if self.debug: + self._mesg('imaplib2 version %s' % __version__) + self._mesg('imaplib2 debug level %s, buffer level %s' % (self.debug, self.debug_buf_lvl)) + + + def _dump_ur(self, lvl): + if lvl > self.debug: + return + + l = self.untagged_responses + if not l: + return + + t = '\n\t\t' + l = ['%s: "%s"' % (x[0], x[1][0] and '" "'.join(x[1]) or '') for x in l] + self.debug_lock.acquire() + self._mesg('untagged responses dump:%s%s' % (t, t.join(l))) + self.debug_lock.release() + + + def _log(self, lvl, line): + if lvl > self.debug: + return + + if line[-2:] == CRLF: + line = line[:-2] + '\\r\\n' + + tn = threading.currentThread().getName() + + if lvl <= 1 or self.debug > self.debug_buf_lvl: + self.debug_lock.acquire() + self._mesg(line, tn) + self.debug_lock.release() + if lvl != 1: + return + + # Keep log of last `_cmd_log_len' interactions for debugging. + self.debug_lock.acquire() + self._cmd_log[self._cmd_log_idx] = (line, tn, time.time()) + self._cmd_log_idx += 1 + if self._cmd_log_idx >= self._cmd_log_len: + self._cmd_log_idx = 0 + self.debug_lock.release() + + + def _mesg(self, s, tn=None, secs=None): + if secs is None: + secs = time.time() + if tn is None: + tn = threading.currentThread().getName() + tm = time.strftime('%M:%S', time.localtime(secs)) + try: + self.debug_file.write(' %s.%02d %s %s\n' % (tm, (secs*100)%100, tn, s)) + self.debug_file.flush() + finally: + pass + + + def _print_log(self): + self.debug_lock.acquire() + i, n = self._cmd_log_idx, self._cmd_log_len + if n: self._mesg('last %d log messages:' % n) + while n: + try: + self._mesg(*self._cmd_log[i]) + except: + pass + i += 1 + if i >= self._cmd_log_len: + i = 0 + n -= 1 + self.debug_lock.release() + + + +class IMAP4_SSL(IMAP4): + + """IMAP4 client class over SSL connection + + Instantiate with: + IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 SSL port); + keyfile - PEM formatted file that contains your private key (default: None); + certfile - PEM formatted certificate chain file (default: None); + ca_certs - PEM formatted certificate chain file used to validate server certificates (default: None); + cert_verify_cb - function to verify authenticity of server certificates (default: None); + ssl_version - SSL version to use (default: "ssl23", choose from: "tls1","ssl2","ssl3","ssl23"); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response. + debug_buf_lvl - debug level at which buffering is turned off. + + For more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): + self.keyfile = keyfile + self.certfile = certfile + self.ca_certs = ca_certs + self.cert_verify_cb = cert_verify_cb + self.ssl_version = ssl_version + IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout, debug_buf_lvl) + + + def open(self, host=None, port=None): + """open(host=None, port=None) + Setup secure connection to remote server on "host:port" + (default: localhost:standard IMAP4 SSL port). + This connection will be used by the routines: + read, send, shutdown, socket, ssl.""" + + self.host = self._choose_nonull_or_dflt('', host) + self.port = self._choose_nonull_or_dflt(IMAP4_SSL_PORT, port) + self.sock = self.open_socket() + self.ssl_wrap_socket() + + + def read(self, size): + """data = read(size) + Read at most 'size' bytes from remote.""" + + if self.decompressor is None: + return self.sock.read(size) + + if self.decompressor.unconsumed_tail: + data = self.decompressor.unconsumed_tail + else: + data = self.sock.read(READ_SIZE) + + return self.decompressor.decompress(data, size) + + + def send(self, data): + """send(data) + Send 'data' to remote.""" + + if self.compressor is not None: + data = self.compressor.compress(data) + data += self.compressor.flush(zlib.Z_SYNC_FLUSH) + + if bytes != str: + if hasattr(self.sock, "sendall"): + self.sock.sendall(bytes(data, 'utf8')) + else: + dlen = len(data) + while dlen > 0: + sent = self.sock.write(bytes(data, 'utf8')) + if sent == dlen: + break # avoid copy + data = data[sent:] + dlen = dlen - sent + else: + if hasattr(self.sock, "sendall"): + self.sock.sendall(data) + else: + dlen = len(data) + while dlen > 0: + sent = self.sock.write(data) + if sent == dlen: + break # avoid copy + data = data[sent:] + dlen = dlen - sent + + + def ssl(self): + """ssl = ssl() + Return ssl instance used to communicate with the IMAP4 server.""" + + return self.sock + + + +class IMAP4_stream(IMAP4): + + """IMAP4 client class over a stream + + Instantiate with: + IMAP4_stream(command, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None) + + command - string that can be passed to subprocess.Popen(); + debug - debug level (default: 0 - no debug); + debug_file - debug stream (default: sys.stderr); + identifier - thread identifier prefix (default: host); + timeout - timeout in seconds when expecting a command response. + debug_buf_lvl - debug level at which buffering is turned off. + + For more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, command, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): + self.command = command + self.host = command + self.port = None + self.sock = None + self.writefile, self.readfile = None, None + self.read_fd = None + IMAP4.__init__(self, None, None, debug, debug_file, identifier, timeout, debug_buf_lvl) + + + def open(self, host=None, port=None): + """open(host=None, port=None) + Setup a stream connection via 'self.command'. + This connection will be used by the routines: + read, send, shutdown, socket.""" + + from subprocess import Popen, PIPE + + if __debug__: self._log(0, 'opening stream from command "%s"' % self.command) + self._P = Popen(self.command, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True) + self.writefile, self.readfile = self._P.stdin, self._P.stdout + self.read_fd = self.readfile.fileno() + + + def read(self, size): + """Read 'size' bytes from remote.""" + + if self.decompressor is None: + return os.read(self.read_fd, size) + + if self.decompressor.unconsumed_tail: + data = self.decompressor.unconsumed_tail + else: + data = os.read(self.read_fd, READ_SIZE) + + return self.decompressor.decompress(data, size) + + + def send(self, data): + """Send data to remote.""" + + if self.compressor is not None: + data = self.compressor.compress(data) + data += self.compressor.flush(zlib.Z_SYNC_FLUSH) + + if bytes != str: + self.writefile.write(bytes(data, 'utf8')) + else: + self.writefile.write(data) + self.writefile.flush() + + + def shutdown(self): + """Close I/O established in "open".""" + + self.readfile.close() + self.writefile.close() + + +class _Authenticator(object): + + """Private class to provide en/de-coding + for base64 authentication conversation.""" + + def __init__(self, mechinst): + self.mech = mechinst # Callable object to provide/process data + + def process(self, data, rqb): + ret = self.mech(self.decode(data)) + if ret is None: + return '*' # Abort conversation + return self.encode(ret) + + def encode(self, inp): + # + # Invoke binascii.b2a_base64 iteratively with + # short even length buffers, strip the trailing + # line feed from the result and append. "Even" + # means a number that factors to both 6 and 8, + # so when it gets to the end of the 8-bit input + # there's no partial 6-bit output. + # + oup = '' + while inp: + if len(inp) > 48: + t = inp[:48] + inp = inp[48:] + else: + t = inp + inp = '' + e = binascii.b2a_base64(t) + if e: + oup = oup + e[:-1] + return oup + + def decode(self, inp): + if not inp: + return '' + return binascii.a2b_base64(inp) + + + + +class _IdleCont(object): + + """When process is called, server is in IDLE state + and will send asynchronous changes.""" + + def __init__(self, parent, timeout): + self.parent = parent + self.timeout = parent._choose_nonull_or_dflt(IDLE_TIMEOUT, timeout) + self.parent.idle_timeout = self.timeout + time.time() + + def process(self, data, rqb): + self.parent.idle_lock.acquire() + self.parent.idle_rqb = rqb + self.parent.idle_timeout = self.timeout + time.time() + self.parent.idle_lock.release() + if __debug__: self.parent._log(2, 'server IDLE started, timeout in %.2f secs' % self.timeout) + return None + + + +MonthNames = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + +Mon2num = dict(list(zip((x.encode() for x in MonthNames[1:]), list(range(1, 13))))) + +InternalDate = re.compile(r'.*INTERNALDATE "' + r'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' + r' (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])' + r' (?P[-+])(?P[0-9][0-9])(?P[0-9][0-9])' + r'"') + + +def Internaldate2Time(resp): + + """time_tuple = Internaldate2Time(resp) + Convert IMAP4 INTERNALDATE to UT.""" + + mo = InternalDate.match(resp) + if not mo: + return None + + mon = Mon2num[mo.group('mon')] + zonen = mo.group('zonen') + + day = int(mo.group('day')) + year = int(mo.group('year')) + hour = int(mo.group('hour')) + min = int(mo.group('min')) + sec = int(mo.group('sec')) + zoneh = int(mo.group('zoneh')) + zonem = int(mo.group('zonem')) + + # INTERNALDATE timezone must be subtracted to get UT + + zone = (zoneh*60 + zonem)*60 + if zonen == '-': + zone = -zone + + tt = (year, mon, day, hour, min, sec, -1, -1, -1) + + utc = time.mktime(tt) + + # Following is necessary because the time module has no 'mkgmtime'. + # 'mktime' assumes arg in local timezone, so adds timezone/altzone. + + lt = time.localtime(utc) + if time.daylight and lt[-1]: + zone = zone + time.altzone + else: + zone = zone + time.timezone + + return time.localtime(utc - zone) + +Internaldate2tuple = Internaldate2Time # (Backward compatible) + + + +def Time2Internaldate(date_time): + + """'"DD-Mmm-YYYY HH:MM:SS +HHMM"' = Time2Internaldate(date_time) + Convert 'date_time' to IMAP4 INTERNALDATE representation.""" + + if isinstance(date_time, (int, float)): + tt = time.localtime(date_time) + elif isinstance(date_time, (tuple, time.struct_time)): + tt = date_time + elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'): + return date_time # Assume in correct format + else: + raise ValueError("date_time not of a known type") + + if time.daylight and tt[-1]: + zone = -time.altzone + else: + zone = -time.timezone + return ('"%2d-%s-%04d %02d:%02d:%02d %+03d%02d"' % + ((tt[2], MonthNames[tt[1]], tt[0]) + tt[3:6] + + divmod(zone//60, 60))) + + + +FLAGS_cre = re.compile(r'.*FLAGS \((?P[^\)]*)\)') + +def ParseFlags(resp): + + """('flag', ...) = ParseFlags(line) + Convert IMAP4 flags response to python tuple.""" + + mo = FLAGS_cre.match(resp) + if not mo: + return () + + return tuple(mo.group('flags').split()) + + + +if __name__ == '__main__': + + # To test: invoke either as 'python imaplib2.py [IMAP4_server_hostname]', + # or as 'python imaplib2.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' + # or as 'python imaplib2.py -l "keyfile[:certfile]" [IMAP4_SSL_server_hostname]' + # Option "-i" tests that IDLE is interruptible + + import getopt, getpass + + try: + optlist, args = getopt.getopt(sys.argv[1:], 'd:il:s:p:') + except getopt.error as val: + optlist, args = (), () + + debug, debug_buf_lvl, port, stream_command, keyfile, certfile, idle_intr = (None,)*7 + for opt,val in optlist: + if opt == '-d': + debug = int(val) + debug_buf_lvl = debug - 1 + elif opt == '-i': + idle_intr = 1 + elif opt == '-l': + try: + keyfile,certfile = val.split(':') + except ValueError: + keyfile,certfile = val,val + elif opt == '-p': + port = int(val) + elif opt == '-s': + stream_command = val + if not args: args = (stream_command,) + + if not args: args = ('',) + if not port: port = (keyfile is not None) and IMAP4_SSL_PORT or IMAP4_PORT + + host = args[0] + + USER = getpass.getuser() + + data = open(os.path.exists("test.data") and "test.data" or __file__).read(1000) + test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)s%(data)s' \ + % {'user':USER, 'lf':'\n', 'data':data} + + test_seq1 = [ + ('list', ('""', '""')), + ('list', ('""', '%')), + ('create', ('imaplib2_test0',)), + ('rename', ('imaplib2_test0', 'imaplib2_test1')), + ('CREATE', ('imaplib2_test2',)), + ('append', ('imaplib2_test2', None, None, test_mesg)), + ('list', ('', 'imaplib2_test%')), + ('select', ('imaplib2_test2',)), + ('search', (None, 'SUBJECT', 'IMAP4 test')), + ('fetch', ("'1:*'", '(FLAGS INTERNALDATE RFC822)')), + ('store', ('1', 'FLAGS', '(\Deleted)')), + ('namespace', ()), + ('expunge', ()), + ('recent', ()), + ('close', ()), + ] + + test_seq2 = ( + ('select', ()), + ('response', ('UIDVALIDITY',)), + ('response', ('EXISTS',)), + ('append', (None, None, None, test_mesg)), + ('examine', ()), + ('select', ()), + ('fetch', ("'1:*'", '(FLAGS UID)')), + ('examine', ()), + ('select', ()), + ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')), + ('uid', ('SEARCH', 'ALL')), + ('uid', ('THREAD', 'references', 'UTF-8', '(SEEN)')), + ('recent', ()), + ) + + + AsyncError = None + + def responder(cb_arg_list): + (response, cb_arg, error) = cb_arg_list + global AsyncError + cmd, args = cb_arg + if error is not None: + AsyncError = error + M._log(0, '[cb] ERROR %s %.100s => %s' % (cmd, args, error)) + return + typ, dat = response + M._log(0, '[cb] %s %.100s => %s %.100s' % (cmd, args, typ, dat)) + if typ == 'NO': + AsyncError = (Exception, dat[0]) + + def run(cmd, args, cb=True): + if AsyncError: + M._log(1, 'AsyncError %s' % repr(AsyncError)) + M.logout() + typ, val = AsyncError + raise typ(val) + if not M.debug: M._log(0, '%s %.100s' % (cmd, args)) + try: + if cb: + typ, dat = getattr(M, cmd)(callback=responder, cb_arg=(cmd, args), *args) + M._log(1, '%s %.100s => %s %.100s' % (cmd, args, typ, dat)) + else: + typ, dat = getattr(M, cmd)(*args) + M._log(1, '%s %.100s => %s %.100s' % (cmd, args, typ, dat)) + except: + M._log(1, '%s - %s' % sys.exc_info()[:2]) + M.logout() + raise + if typ == 'NO': + M._log(1, 'NO') + M.logout() + raise Exception(dat[0]) + return dat + + try: + threading.currentThread().setName('main') + + if keyfile is not None: + if not keyfile: keyfile = None + if not certfile: certfile = None + M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) + elif stream_command: + M = IMAP4_stream(stream_command, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) + else: + M = IMAP4(host=host, port=port, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) + if M.state != 'AUTH': # Login needed + PASSWD = getpass.getpass("IMAP password for %s on %s: " % (USER, host or "localhost")) + test_seq1.insert(0, ('login', (USER, PASSWD))) + M._log(0, 'PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) + if 'COMPRESS=DEFLATE' in M.capabilities: + M.enable_compression() + + for cmd,args in test_seq1: + run(cmd, args) + + for ml in run('list', ('', 'imaplib2_test%'), cb=False): + mo = re.match(r'.*"([^"]+)"$', ml) + if mo: path = mo.group(1) + else: path = ml.split()[-1] + run('delete', (path,)) + + if 'ID' in M.capabilities: + run('id', ()) + run('id', ("(name imaplib2)",)) + run('id', ("version", __version__, "os", os.uname()[0])) + + for cmd,args in test_seq2: + if (cmd,args) != ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')): + run(cmd, args) + continue + + dat = run(cmd, args, cb=False) + uid = dat[-1].split() + if not uid: continue + run('uid', ('FETCH', uid[-1], + '(FLAGS INTERNALDATE RFC822.SIZE RFC822.HEADER RFC822.TEXT)')) + run('uid', ('STORE', uid[-1], 'FLAGS', '(\Deleted)')) + run('expunge', ()) + + if 'IDLE' in M.capabilities: + run('idle', (2,), cb=False) + run('idle', (99,)) # Asynchronous, to test interruption of 'idle' by 'noop' + time.sleep(1) + run('noop', (), cb=False) + + run('append', (None, None, None, test_mesg), cb=False) + num = run('search', (None, 'ALL'), cb=False)[0].split()[0] + dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) + M._mesg('fetch %s => %s' % (num, repr(dat))) + run('idle', (2,)) + run('store', (num, '-FLAGS', '(\Seen)'), cb=False), + dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) + M._mesg('fetch %s => %s' % (num, repr(dat))) + run('uid', ('STORE', num, 'FLAGS', '(\Deleted)')) + run('expunge', ()) + if idle_intr: + M._mesg('HIT CTRL-C to interrupt IDLE') + try: + run('idle', (99,), cb=False) # Synchronous, to test interruption of 'idle' by INTR + except KeyboardInterrupt: + M._mesg('Thanks!') + M._mesg('') + raise + elif idle_intr: + M._mesg('chosen server does not report IDLE capability') + + run('logout', (), cb=False) + + if debug: + M._mesg('') + M._print_log() + M._mesg('') + M._mesg('unused untagged responses in order, most recent last:') + for typ,dat in M.pop_untagged_responses(): M._mesg('\t%s %s' % (typ, dat)) + + print('All tests OK.') + + except: + if not idle_intr or not 'IDLE' in M.capabilities: + print('Tests failed.') + + if not debug: + print(''' +If you would like to see debugging output, +try: %s -d5 +''' % sys.argv[0]) + + raise diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index cf82996..952404a 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -1,6 +1,5 @@ # imaplib utilities -# Copyright (C) 2002-2007 John Goerzen -# 2010 Sebastian Spaeth +# Copyright (C) 2002-2015 John Goerzen & contributors # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -15,20 +14,94 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import re, socket, time, subprocess +import os +import fcntl +import time +import subprocess +from sys import exc_info +import threading +from hashlib import sha1 +import socket +import errno + from offlineimap.ui import getglobalui -from imaplib import * +from offlineimap import OfflineImapError +from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, InternalDate, Mon2num -# Import the symbols we need that aren't exported by default -from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num -try: - import ssl -except ImportError: - #fails on python <2.6 - pass +class UsefulIMAPMixIn(object): + def __getselectedfolder(self): + if self.state == 'SELECTED': + return self.mailbox + return None -class IMAP4_Tunnel(IMAP4): + def select(self, mailbox='INBOX', readonly=False, force=False): + """Selects a mailbox on the IMAP server + + :returns: 'OK' on success, nothing if the folder was already + selected or raises an :exc:`OfflineImapError`.""" + + if self.__getselectedfolder() == mailbox and \ + self.is_readonly == readonly and \ + not force: + # No change; return. + return + try: + result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) + except self.readonly as e: + # pass self.readonly to our callers + raise + except self.abort as e: + # self.abort is raised when we are supposed to retry + errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ + "ver said: %s" % (self.host, mailbox, e.args[0]) + severity = OfflineImapError.ERROR.FOLDER_RETRY + raise OfflineImapError(errstr, severity), None, exc_info()[2] + if result[0] != 'OK': + #in case of error, bail out with OfflineImapError + errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\ + (mailbox, result) + severity = OfflineImapError.ERROR.FOLDER + raise OfflineImapError(errstr, severity) + return result + + # Overrides private function from IMAP4 (@imaplib2) + def _mesg(self, s, tn=None, secs=None): + new_mesg(self, s, tn, secs) + + # Overrides private function from IMAP4 (@imaplib2) + def open_socket(self): + """open_socket() + Open socket choosing first address family available.""" + msg = (-1, 'could not open socket') + for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + # use socket of our own, possiblly socksified socket. + s = self.socket(af, socktype, proto) + except socket.error, msg: + continue + try: + for i in (0, 1): + try: + s.connect(sa) + break + except socket.error, msg: + if len(msg.args) < 2 or msg.args[0] != errno.EINTR: + raise + else: + raise socket.error(msg) + except socket.error, msg: + s.close() + continue + break + else: + raise socket.error(msg) + + return s + + +class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): """IMAP4 client class over a tunnel Instantiate with: IMAP4_Tunnel(tunnelcmd) @@ -36,25 +109,51 @@ class IMAP4_Tunnel(IMAP4): tunnelcmd -- shell command to generate the tunnel. The result will be in PREAUTH stage.""" - def __init__(self, tunnelcmd): - IMAP4.__init__(self, tunnelcmd) + def __init__(self, tunnelcmd, **kwargs): + if "use_socket" in kwargs: + self.socket = kwargs['use_socket'] + del kwargs['use_socket'] + IMAP4.__init__(self, tunnelcmd, **kwargs) def open(self, host, port): """The tunnelcmd comes in on host!""" + + self.host = host self.process = subprocess.Popen(host, shell=True, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) (self.outfd, self.infd) = (self.process.stdin, self.process.stdout) + # imaplib2 polls on this fd + self.read_fd = self.infd.fileno() + + self.set_nonblocking(self.read_fd) + + def set_nonblocking(self, fd): + """Mark fd as nonblocking""" + + # get the file's current flag settings + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + # clear non-blocking mode from flags + fl = fl & ~os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, fl) def read(self, size): - retval = '' - while len(retval) < size: - retval += self.infd.read(size - len(retval)) - return retval + """data = read(size) + Read at most 'size' bytes from remote.""" - def readline(self): - return self.infd.readline() + if self.decompressor is None: + return os.read(self.read_fd, size) + + if self.decompressor.unconsumed_tail: + data = self.decompressor.unconsumed_tail + else: + data = os.read(self.read_fd, 8192) + + return self.decompressor.decompress(data, size) def send(self, data): + if self.compressor is not None: + data = self.compressor.compress(data) + data += self.compressor.flush(zlib.Z_SYNC_FLUSH) self.outfd.write(data) def shutdown(self): @@ -63,189 +162,65 @@ class IMAP4_Tunnel(IMAP4): self.process.wait() -def new_mesg(self, s, secs=None): +def new_mesg(self, s, tn=None, secs=None): if secs is None: secs = time.time() + if tn is None: + tn = threading.currentThread().getName() tm = time.strftime('%M:%S', time.localtime(secs)) - getglobalui().debug('imap', ' %s.%02d %s' % (tm, (secs*100)%100, s)) + getglobalui().debug('imap', ' %s.%02d %s %s' % (tm, (secs*100)%100, tn, s)) -class WrappedIMAP4_SSL(IMAP4_SSL): - """Provides an improved version of the standard IMAP4_SSL - It provides a better readline() implementation as impaplib's - readline() is extremly inefficient. It can also connect to IPv6 - addresses.""" +class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): + """Improved version of imaplib.IMAP4_SSL overriding select().""" + def __init__(self, *args, **kwargs): - self._readbuf = '' - self._cacertfile = kwargs.get('cacertfile', None) - if kwargs.has_key('cacertfile'): - del kwargs['cacertfile'] - IMAP4_SSL.__init__(self, *args, **kwargs) + if "use_socket" in kwargs: + self.socket = kwargs['use_socket'] + del kwargs['use_socket'] + self._fingerprint = kwargs.get('fingerprint', None) + if type(self._fingerprint) != type([]): + self._fingerprint = [self._fingerprint] + if 'fingerprint' in kwargs: + del kwargs['fingerprint'] + super(WrappedIMAP4_SSL, self).__init__(*args, **kwargs) - def open(self, host = '', port = IMAP4_SSL_PORT): - """Do whatever IMAP4_SSL would do in open, but call sslwrap - with cert verification""" - #IMAP4_SSL.open(self, host, port) uses the below 2 lines: - self.host = host - self.port = port - - #rather than just self.sock = socket.create_connection((host, port)) - #we use the below part to be able to connect to ipv6 addresses too - #This connects to the first ip found ipv4/ipv6 - #Added by Adriaan Peeters based on a socket - #example from the python documentation: - #http://www.python.org/doc/lib/socket-example.html - res = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM) - # Try all the addresses in turn until we connect() - last_error = 0 - for remote in res: - af, socktype, proto, canonname, sa = remote - self.sock = socket.socket(af, socktype, proto) - last_error = self.sock.connect_ex(sa) - if last_error == 0: - break - else: - self.sock.close() - if last_error != 0: - # FIXME - raise socket.error(last_error) - - #connected to socket, now wrap it in SSL - try: - if self._cacertfile: - requirecert = ssl.CERT_REQUIRED - else: - requirecert = ssl.CERT_NONE - - self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, - self.certfile, - ca_certs = self._cacertfile, - cert_reqs = requirecert) - except NameError: - #Python 2.4/2.5 don't have the ssl module, we need to - #socket.ssl() here but that doesn't allow cert - #verification!!! - if self._cacertfile: - #user configured a CA certificate, but python 2.4/5 doesn't - #allow us to easily check it. So bail out here. - raise Exception("SSL CA Certificates cannot be checked with python <=2.6. Abort") - self.sslobj = socket.ssl(self.sock, self.keyfile, - self.certfile) - - else: - #ssl.wrap_socket worked and cert is verified (if configured), - #now check that hostnames also match if we have a CA cert. - if self._cacertfile: - error = self._verifycert(self.sslobj.getpeercert(), host) - if error: - raise ssl.SSLError("SSL Certificate host name mismatch: %s" % error) - - #TODO: Done for now. We should implement a mutt-like behavior - #that offers the users to accept a certificate (presenting a - #fingerprint of it) (get via self.sslobj.getpeercert()), and - #save that, and compare on future connects, rather than having - #to trust what the CA certs say. - - def _verifycert(self, cert, hostname): - '''Verify that cert (in socket.getpeercert() format) matches hostname. - CRLs and subjectAltName are not handled. - - Returns error message if any problems are found and None on success. - ''' - if not cert: - return ('no certificate received') - dnsname = hostname.lower() - for s in cert.get('subject', []): - key, value = s[0] - if key == 'commonName': - certname = value.lower() - if (certname == dnsname or - '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): - return None - return ('certificate is for %s') % certname - return ('no commonName found in certificate') - - def _read_upto (self, n): - """Read up to n bytes, emptying existing _readbuffer first""" - bytesfrombuf = min(n, len(self._readbuf)) - if bytesfrombuf: - # Return the stuff in readbuf, even if less than n. - # It might contain the rest of the line, and if we try to - # read more, might block waiting for data that is not - # coming to arrive. - retval = self._readbuf[:bytesfrombuf] - self._readbuf = self._readbuf[bytesfrombuf:] - return retval - return self.sslobj.read(min(n, 16384)) - - def read(self, n): - """Read exactly n bytes - - As done in IMAP4_SSL.read() API. If read returns less than n - bytes, things break left and right.""" - chunks = [] - read = 0 - while read < n: - data = self._read_upto (n-read) - read += len(data) - chunks.append(data) - - return ''.join(chunks) - - def readline(self): - """Get the next line. This implementation is more efficient - than IMAP4_SSL.readline() which reads one char at a time and - reassembles the string by appending those chars. Uggh.""" - retval = '' - while 1: - linebuf = self._read_upto(1024) - nlindex = linebuf.find("\n") - if nlindex != -1: - retval += linebuf[:nlindex + 1] - self._readbuf = linebuf[nlindex + 1:] + self._readbuf - return retval - else: - retval += linebuf + def open(self, host=None, port=None): + if not self.ca_certs and not self._fingerprint: + raise OfflineImapError("No CA certificates " + "and no server fingerprints configured. " + "You must configure at least something, otherwise " + "having SSL helps nothing.", OfflineImapError.ERROR.REPO) + super(WrappedIMAP4_SSL, self).open(host, port) + if self._fingerprint: + # compare fingerprints + fingerprint = sha1(self.sock.getpeercert(True)).hexdigest() + if fingerprint not in self._fingerprint: + raise OfflineImapError("Server SSL fingerprint '%s' " + "for hostname '%s' " + "does not match configured fingerprint(s) %s. " + "Please verify and set 'cert_fingerprint' accordingly " + "if not set yet."% + (fingerprint, host, self._fingerprint), + OfflineImapError.ERROR.REPO) -class WrappedIMAP4(IMAP4): - """Improved version of imaplib.IMAP4 that can also connect to IPv6""" +class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): + """Improved version of imaplib.IMAP4 overriding select().""" - def open(self, host = '', port = IMAP4_PORT): - """Setup connection to remote server on "host:port" - (default: localhost:standard IMAP4 port). - """ - #self.host and self.port are needed by the parent IMAP4 class - self.host = host - self.port = port - res = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM) + def __init__(self, *args, **kwargs): + if "use_socket" in kwargs: + self.socket = kwargs['use_socket'] + del kwargs['use_socket'] + IMAP4.__init__(self, *args, **kwargs) - # Try each address returned by getaddrinfo in turn until we - # manage to connect to one. - # Try all the addresses in turn until we connect() - last_error = 0 - for remote in res: - af, socktype, proto, canonname, sa = remote - self.sock = socket.socket(af, socktype, proto) - last_error = self.sock.connect_ex(sa) - if last_error == 0: - break - else: - self.sock.close() - if last_error != 0: - # FIXME - raise socket.error(last_error) - self.file = self.sock.makefile('rb') - -mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]") def Internaldate2epoch(resp): """Convert IMAP4 INTERNALDATE to UT. - Returns seconds since the epoch. - """ + Returns seconds since the epoch.""" + + from calendar import timegm mo = InternalDate.match(resp) if not mo: @@ -270,4 +245,4 @@ def Internaldate2epoch(resp): tt = (year, mon, day, hour, min, sec, -1, -1, -1) - return time.mktime(tt) + return timegm(tt) - zone diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 2a9f247..f0b2248 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -1,6 +1,5 @@ # IMAP server support -# Copyright (C) 2002 - 2007 John Goerzen -# +# Copyright (C) 2002 - 2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,15 +15,20 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import imaplib -from offlineimap import imaplibutil, imaputil, threadutil -from offlineimap.ui import getglobalui -from threading import * -import thread, hmac, os, time +from threading import Lock, BoundedSemaphore, Thread, Event, currentThread +import hmac +import socket import base64 +import time +import errno +from sys import exc_info +from socket import gaierror +from ssl import SSLError, cert_time_to_seconds + +from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError +import offlineimap.accounts +from offlineimap.ui import getglobalui -from StringIO import StringIO -from platform import system try: # do we have a recent pykerberos? @@ -35,178 +39,345 @@ try: except ImportError: pass -class UsefulIMAPMixIn: - def getstate(self): - return self.state - def getselectedfolder(self): - if self.getstate() == 'SELECTED': - return self.selectedfolder - return None - - def select(self, mailbox='INBOX', readonly=None, force = 0): - if (not force) and self.getselectedfolder() == mailbox \ - and self.is_readonly == readonly: - # No change; return. - return - result = self.__class__.__bases__[1].select(self, mailbox, readonly) - if result[0] != 'OK': - raise ValueError, "Error from select: %s" % str(result) - if self.getstate() == 'SELECTED': - self.selectedfolder = mailbox - else: - self.selectedfolder = None - - def _mesg(self, s, secs=None): - imaplibutil.new_mesg(self, s, secs) - -class UsefulIMAP4(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4): - - # This is a hack around Darwin's implementation of realloc() (which - # Python uses inside the socket code). On Darwin, we split the - # message into 100k chunks, which should be small enough - smaller - # might start seriously hurting performance ... - def read(self, size): - if (system() == 'Darwin') and (size>0) : - read = 0 - io = StringIO() - while read < size: - data = imaplib.IMAP4.read (self, min(size-read,8192)) - read += len(data) - io.write(data) - return io.getvalue() - else: - return imaplib.IMAP4.read (self, size) - -class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): - # This is the same hack as above, to be used in the case of an SSL - # connexion. - def read(self, size): - if (system() == 'Darwin') and (size>0) : - read = 0 - io = StringIO() - while read < size: - data = imaplibutil.WrappedIMAP4_SSL.read (self, min(size-read,8192)) - read += len(data) - io.write(data) - return io.getvalue() - else: - return imaplibutil.WrappedIMAP4_SSL.read (self,size) - -class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass - class IMAPServer: + """Initializes all variables from an IMAPRepository() instance + + Various functions, such as acquireconnection() return an IMAP4 + object on which we can operate. + + Public instance variables are: self.: + delim The server's folder delimiter. Only valid after acquireconnection() + """ + GSS_STATE_STEP = 0 GSS_STATE_WRAP = 1 - def __init__(self, config, reposname, - username = None, password = None, hostname = None, - port = None, ssl = 1, maxconnections = 1, tunnel = None, - reference = '""', sslclientcert = None, sslclientkey = None, - sslcacertfile= None): + def __init__(self, repos): self.ui = getglobalui() - self.reposname = reposname - self.config = config - self.username = username - self.password = password + self.repos = repos + self.config = repos.getconfig() + + self.preauth_tunnel = repos.getpreauthtunnel() + self.transport_tunnel = repos.gettransporttunnel() + if self.preauth_tunnel and self.transport_tunnel: + raise OfflineImapError('%s: '% repos + + 'you must enable precisely one ' + 'type of tunnel (preauth or transport), ' + 'not both', OfflineImapError.ERROR.REPO) + self.tunnel = \ + self.preauth_tunnel if self.preauth_tunnel \ + else self.transport_tunnel + + self.username = \ + None if self.preauth_tunnel else repos.getuser() + self.user_identity = repos.get_remote_identity() + self.authmechs = repos.get_auth_mechanisms() + self.password = None self.passworderror = None self.goodpassword = None - self.hostname = hostname - self.tunnel = tunnel - self.port = port - self.usessl = ssl - self.sslclientcert = sslclientcert - self.sslclientkey = sslclientkey - self.sslcacertfile = sslcacertfile + + self.usessl = repos.getssl() + self.hostname = \ + None if self.preauth_tunnel else repos.gethost() + self.port = repos.getport() + if self.port == None: + self.port = 993 if self.usessl else 143 + self.sslclientcert = repos.getsslclientcert() + self.sslclientkey = repos.getsslclientkey() + self.sslcacertfile = repos.getsslcacertfile() + if self.sslcacertfile is None: + self.__verifycert = None # disable cert verification + self.fingerprint = repos.get_ssl_fingerprint() + self.sslversion = repos.getsslversion() + self.delim = None self.root = None - if port == None: - if ssl: - self.port = 993 - else: - self.port = 143 - self.maxconnections = maxconnections + self.maxconnections = repos.getmaxconnections() self.availableconnections = [] self.assignedconnections = [] self.lastowner = {} self.semaphore = BoundedSemaphore(self.maxconnections) self.connectionlock = Lock() - self.reference = reference + self.reference = repos.getreference() + self.idlefolders = repos.getidlefolders() self.gss_step = self.GSS_STATE_STEP self.gss_vc = None self.gssapi = False - def getpassword(self): - if self.goodpassword != None: + # In order to support proxy connection, we have to override the + # default socket instance with our own socksified socket instance. + # We add this option to bypass the GFW in China. + _account_section = 'Account ' + self.repos.account.name + if not self.config.has_option(_account_section, 'proxy'): + self.proxied_socket = socket.socket + else: + proxy = self.config.get(_account_section, 'proxy') + # Powered by PySocks. + try: + import socks + proxy_type, host, port = proxy.split(":") + port = int(port) + socks.setdefaultproxy(getattr(socks, proxy_type), host, port) + self.proxied_socket = socks.socksocket + except ImportError: + self.ui.warn("PySocks not installed, ignoring proxy option.") + self.proxied_socket = socket.socket + except (AttributeError, ValueError) as e: + self.ui.warn("Bad proxy option %s for account %s: %s " + "Ignoring proxy option."% + (proxy, self.repos.account.name, e)) + self.proxied_socket = socket.socket + + + def __getpassword(self): + """Returns the server password or None""" + if self.goodpassword != None: # use cached good one first return self.goodpassword if self.password != None and self.passworderror == None: - return self.password + return self.password # non-failed preconfigured one - self.password = self.ui.getpass(self.reposname, - self.config, - self.passworderror) + # get 1) configured password first 2) fall back to asking via UI + self.password = self.repos.getpassword() or \ + self.ui.getpass(self.repos.getname(), self.config, + self.passworderror) self.passworderror = None - return self.password - def getdelim(self): - """Returns this server's folder delimiter. Can only be called - after one or more calls to acquireconnection.""" - return self.delim - + # XXX: is this function used anywhere? def getroot(self): - """Returns this server's folder root. Can only be called after one + """Returns this server's folder root. Can only be called after one or more calls to acquireconnection.""" + return self.root - def releaseconnection(self, connection): - """Releases a connection, returning it to the pool.""" + def releaseconnection(self, connection, drop_conn=False): + """Releases a connection, returning it to the pool. + + :param drop_conn: If True, the connection will be released and + not be reused. This can be used to indicate broken connections.""" + + if connection is None: return #noop on bad connection self.connectionlock.acquire() self.assignedconnections.remove(connection) - self.availableconnections.append(connection) + # Don't reuse broken connections + if connection.Terminate or drop_conn: + connection.logout() + else: + self.availableconnections.append(connection) self.connectionlock.release() self.semaphore.release() - def md5handler(self, response): + def __md5handler(self, response): challenge = response.strip() - self.ui.debug('imap', 'md5handler: got challenge %s' % challenge) + self.ui.debug('imap', '__md5handler: got challenge %s'% challenge) - passwd = self.getpassword() + passwd = self.__getpassword() retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest() - self.ui.debug('imap', 'md5handler: returning %s' % retval) + self.ui.debug('imap', '__md5handler: returning %s'% retval) return retval - def plainauth(self, imapobj): - self.ui.debug('imap', 'Attempting plain authentication') - imapobj.login(self.username, self.getpassword()) + def __loginauth(self, imapobj): + """ Basic authentication via LOGIN command.""" - def gssauth(self, response): + self.ui.debug('imap', 'Attempting IMAP LOGIN authentication') + imapobj.login(self.username, self.__getpassword()) + + + def __plainhandler(self, response): + """Implements SASL PLAIN authentication, RFC 4616, + http://tools.ietf.org/html/rfc4616""" + + authc = self.username + passwd = self.__getpassword() + authz = '' + if self.user_identity != None: + authz = self.user_identity + NULL = u'\x00' + retval = NULL.join((authz, authc, passwd)).encode('utf-8') + self.ui.debug('imap', '__plainhandler: returning %s' % retval) + return retval + + + # XXX: describe function + def __gssauth(self, response): data = base64.b64encode(response) try: if self.gss_step == self.GSS_STATE_STEP: if not self.gss_vc: - rc, self.gss_vc = kerberos.authGSSClientInit('imap@' + - self.hostname) + rc, self.gss_vc = kerberos.authGSSClientInit( + 'imap@' + self.hostname) response = kerberos.authGSSClientResponse(self.gss_vc) rc = kerberos.authGSSClientStep(self.gss_vc, data) if rc != kerberos.AUTH_GSS_CONTINUE: - self.gss_step = self.GSS_STATE_WRAP + self.gss_step = self.GSS_STATE_WRAP elif self.gss_step == self.GSS_STATE_WRAP: rc = kerberos.authGSSClientUnwrap(self.gss_vc, data) response = kerberos.authGSSClientResponse(self.gss_vc) - rc = kerberos.authGSSClientWrap(self.gss_vc, response, - self.username) + rc = kerberos.authGSSClientWrap( + self.gss_vc, response, self.username) response = kerberos.authGSSClientResponse(self.gss_vc) - except kerberos.GSSError, err: + except kerberos.GSSError as err: # Kerberos errored out on us, respond with None to cancel the # authentication - self.ui.debug('imap', '%s: %s' % (err[0][0], err[1][0])) + self.ui.debug('imap', '%s: %s'% (err[0][0], err[1][0])) return None if not response: response = '' return base64.b64decode(response) + + def __start_tls(self, imapobj): + if 'STARTTLS' in imapobj.capabilities and not self.usessl: + self.ui.debug('imap', 'Using STARTTLS connection') + try: + imapobj.starttls() + except imapobj.error as e: + raise OfflineImapError("Failed to start " + "TLS connection: %s"% str(e), + OfflineImapError.ERROR.REPO, None, exc_info()[2]) + + + ## All __authn_* procedures are helpers that do authentication. + ## They are class methods that take one parameter, IMAP object. + ## + ## Each function should return True if authentication was + ## successful and False if authentication wasn't even tried + ## for some reason (but not when IMAP has no such authentication + ## capability, calling code checks that). + ## + ## Functions can also raise exceptions; two types are special + ## and will be handled by the calling code: + ## + ## - imapobj.error means that there was some error that + ## comes from imaplib2; + ## + ## - OfflineImapError means that function detected some + ## problem by itself. + + def __authn_gssapi(self, imapobj): + if not have_gss: + return False + + self.connectionlock.acquire() + try: + imapobj.authenticate('GSSAPI', self.__gssauth) + return True + except imapobj.error as e: + self.gssapi = False + raise + else: + self.gssapi = True + kerberos.authGSSClientClean(self.gss_vc) + self.gss_vc = None + self.gss_step = self.GSS_STATE_STEP + finally: + self.connectionlock.release() + + def __authn_cram_md5(self, imapobj): + imapobj.authenticate('CRAM-MD5', self.__md5handler) + return True + + def __authn_plain(self, imapobj): + imapobj.authenticate('PLAIN', self.__plainhandler) + return True + + def __authn_login(self, imapobj): + # Use LOGIN command, unless LOGINDISABLED is advertized + # (per RFC 2595) + if 'LOGINDISABLED' in imapobj.capabilities: + raise OfflineImapError("IMAP LOGIN is " + "disabled by server. Need to use SSL?", + OfflineImapError.ERROR.REPO) + else: + self.__loginauth(imapobj) + return True + + + def __authn_helper(self, imapobj): + """Authentication machinery for self.acquireconnection(). + + Raises OfflineImapError() of type ERROR.REPO when + there are either fatal problems or no authentications + succeeded. + + If any authentication method succeeds, routine should exit: + warnings for failed methods are to be produced in the + respective except blocks.""" + + # Authentication routines, hash keyed by method name + # with value that is a tuple with + # - authentication function, + # - tryTLS flag, + # - check IMAP capability flag. + auth_methods = { + "GSSAPI": (self.__authn_gssapi, False, True), + "CRAM-MD5": (self.__authn_cram_md5, True, True), + "PLAIN": (self.__authn_plain, True, True), + "LOGIN": (self.__authn_login, True, False), + } + # Stack stores pairs of (method name, exception) + exc_stack = [] + tried_to_authn = False + tried_tls = False + mechs = self.authmechs + + # GSSAPI must be tried first: we will probably go TLS after it + # and GSSAPI mustn't be tunneled over TLS. + if "GSSAPI" in mechs: + mechs.remove("GSSAPI") + mechs.insert(0, "GSSAPI") + + for m in mechs: + if m not in auth_methods: + raise Exception("Bad authentication method %s, " + "please, file OfflineIMAP bug" % m) + + func, tryTLS, check_cap = auth_methods[m] + + # TLS must be initiated before checking capabilities: + # they could have been changed after STARTTLS. + if tryTLS and not tried_tls: + tried_tls = True + self.__start_tls(imapobj) + + if check_cap: + cap = "AUTH=" + m + if cap not in imapobj.capabilities: + continue + + tried_to_authn = True + self.ui.debug('imap', u'Attempting ' + '%s authentication'% m) + try: + if func(imapobj): + return + except (imapobj.error, OfflineImapError) as e: + self.ui.warn('%s authentication failed: %s'% (m, e)) + exc_stack.append((m, e)) + + if len(exc_stack): + msg = "\n\t".join(map( + lambda x: ": ".join((x[0], str(x[1]))), + exc_stack + )) + raise OfflineImapError("All authentication types " + "failed:\n\t%s"% msg, OfflineImapError.ERROR.REPO) + + if not tried_to_authn: + methods = ", ".join(map( + lambda x: x[5:], filter(lambda x: x[0:5] == "AUTH=", + imapobj.capabilities) + )) + raise OfflineImapError(u"Repository %s: no supported " + "authentication mechanisms found; configured %s, " + "server advertises %s"% (self.repos, + ", ".join(self.authmechs), methods), + OfflineImapError.ERROR.REPO) + + + # XXX: move above, closer to releaseconnection() def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. @@ -215,17 +386,17 @@ class IMAPServer: self.semaphore.acquire() self.connectionlock.acquire() + curThread = currentThread() imapobj = None if len(self.availableconnections): # One is available. # Try to find one that previously belonged to this thread # as an optimization. Start from the back since that's where # they're popped on. - threadid = thread.get_ident() imapobj = None for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] - if self.lastowner[tryobj] == threadid: + if self.lastowner[tryobj] == curThread.ident: imapobj = tryobj del(self.availableconnections[i]) break @@ -233,68 +404,65 @@ class IMAPServer: imapobj = self.availableconnections[0] del(self.availableconnections[0]) self.assignedconnections.append(imapobj) - self.lastowner[imapobj] = thread.get_ident() + self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj - + self.connectionlock.release() # Release until need to modify data - """ Must be careful here that if we fail we should bail out gracefully - and release locks / threads so that the next attempt can try... - """ + # Must be careful here that if we fail we should bail out gracefully + # and release locks / threads so that the next attempt can try... success = 0 try: while not success: # Generate a new connection. if self.tunnel: self.ui.connecting('tunnel', self.tunnel) - imapobj = UsefulIMAP4_Tunnel(self.tunnel) + imapobj = imaplibutil.IMAP4_Tunnel( + self.tunnel, + timeout=socket.getdefaulttimeout(), + use_socket=self.proxied_socket, + ) success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) - imapobj = UsefulIMAP4_SSL(self.hostname, self.port, - self.sslclientkey, self.sslclientcert, - cacertfile = self.sslcacertfile) + imapobj = imaplibutil.WrappedIMAP4_SSL( + self.hostname, + self.port, + self.sslclientkey, + self.sslclientcert, + self.sslcacertfile, + self.__verifycert, + self.sslversion, + timeout=socket.getdefaulttimeout(), + fingerprint=self.fingerprint, + use_socket=self.proxied_socket, + ) else: self.ui.connecting(self.hostname, self.port) - imapobj = UsefulIMAP4(self.hostname, self.port) + imapobj = imaplibutil.WrappedIMAP4( + self.hostname, self.port, + timeout=socket.getdefaulttimeout(), + use_socket=self.proxied_socket, + ) - imapobj.mustquote = imaplibutil.mustquote - - if not self.tunnel: + if not self.preauth_tunnel: try: - # Try GSSAPI and continue if it fails - if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss: - self.ui.debug('imap', - 'Attempting GSSAPI authentication') - try: - imapobj.authenticate('GSSAPI', self.gssauth) - except imapobj.error, val: - self.gssapi = False - self.ui.debug('imap', - 'GSSAPI Authentication failed') - else: - self.gssapi = True - #if we do self.password = None then the next attempt cannot try... - #self.password = None - - if not self.gssapi: - if 'AUTH=CRAM-MD5' in imapobj.capabilities: - self.ui.debug('imap', - 'Attempting CRAM-MD5 authentication') - try: - imapobj.authenticate('CRAM-MD5', self.md5handler) - except imapobj.error, val: - self.plainauth(imapobj) - else: - self.plainauth(imapobj) - # Would bail by here if there was a failure. - success = 1 + self.__authn_helper(imapobj) self.goodpassword = self.password - except imapobj.error, val: - self.passworderror = str(val) + success = 1 + except OfflineImapError as e: + self.passworderror = str(e) raise - #self.password = None + + # Enable compression + if self.repos.getconfboolean('usecompression', 0): + imapobj.enable_compression() + + # update capabilities after login, e.g. gmail serves different ones + typ, dat = imapobj.capability() + if dat != [None]: + imapobj.capabilities = tuple(dat[-1].upper().split()) if self.delim == None: listres = imapobj.list(self.reference, '""')[1] @@ -302,139 +470,301 @@ class IMAPServer: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] + if listres == [None] or listres == None: + # No Folders were returned. This occurs, e.g. if the + # 'reference' prefix does not exist on the mail + # server. Raise exception. + err = "Server '%s' returned no folders in '%s'"% \ + (self.repos.getname(), self.reference) + self.ui.warn(err) + raise Exception(err) self.delim, self.root = \ - imaputil.imapsplit(listres[0])[1:] + imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) - self.connectionlock.acquire() - self.assignedconnections.append(imapobj) - self.lastowner[imapobj] = thread.get_ident() - self.connectionlock.release() + with self.connectionlock: + self.assignedconnections.append(imapobj) + self.lastowner[imapobj] = curThread.ident return imapobj - except: - """If we are here then we did not succeed in getting a connection - - we should clean up and then re-raise the error...""" + except Exception as e: + """If we are here then we did not succeed in getting a + connection - we should clean up and then re-raise the + error...""" + self.semaphore.release() - #Make sure that this can be retried the next time... - self.passworderror = None - if(self.connectionlock.locked()): - self.connectionlock.release() - raise - + severity = OfflineImapError.ERROR.REPO + if type(e) == gaierror: + #DNS related errors. Abort Repo sync + #TODO: special error msg for e.errno == 2 "Name or service not known"? + reason = "Could not resolve name '%s' for repository "\ + "'%s'. Make sure you have configured the ser"\ + "ver name correctly and that you are online."%\ + (self.hostname, self.repos) + raise OfflineImapError(reason, severity), None, exc_info()[2] + + elif isinstance(e, SSLError) and e.errno == errno.EPERM: + # SSL unknown protocol error + # happens e.g. when connecting via SSL to a non-SSL service + if self.port != 993: + reason = "Could not connect via SSL to host '%s' and non-s"\ + "tandard ssl port %d configured. Make sure you connect"\ + " to the correct port."% (self.hostname, self.port) + else: + reason = "Unknown SSL protocol connecting to host '%s' for "\ + "repository '%s'. OpenSSL responded:\n%s"\ + % (self.hostname, self.repos, e) + raise OfflineImapError(reason, severity), None, exc_info()[2] + + elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: + # "Connection refused", can be a non-existing port, or an unauthorized + # webproxy (open WLAN?) + reason = "Connection to host '%s:%d' for repository '%s' was "\ + "refused. Make sure you have the right host and port "\ + "configured and that you are actually able to access the "\ + "network."% (self.hostname, self.port, self.repos) + raise OfflineImapError(reason, severity), None, exc_info()[2] + # Could not acquire connection to the remote; + # socket.error(last_error) raised + if str(e)[:24] == "can't open socket; error": + raise OfflineImapError("Could not connect to remote server '%s' "\ + "for repository '%s'. Remote does not answer." + % (self.hostname, self.repos), + OfflineImapError.ERROR.REPO), None, exc_info()[2] + else: + # re-raise all other errors + raise + def connectionwait(self): - """Waits until there is a connection available. Note that between - the time that a connection becomes available and the time it is - requested, another thread may have grabbed it. This function is - mainly present as a way to avoid spawning thousands of threads - to copy messages, then have them all wait for 3 available connections. - It's OK if we have maxconnections + 1 or 2 threads, which is what - this will help us do.""" - threadutil.semaphorewait(self.semaphore) + """Waits until there is a connection available. + + Note that between the time that a connection becomes available and the + time it is requested, another thread may have grabbed it. This function + is mainly present as a way to avoid spawning thousands of threads to + copy messages, then have them all wait for 3 available connections. + It's OK if we have maxconnections + 1 or 2 threads, which is what this + will help us do.""" + + self.semaphore.acquire() + self.semaphore.release() def close(self): # Make sure I own all the semaphores. Let the threads finish # their stuff. This is a blocking method. - self.connectionlock.acquire() - threadutil.semaphorereset(self.semaphore, self.maxconnections) - for imapobj in self.assignedconnections + self.availableconnections: - imapobj.logout() - self.assignedconnections = [] - self.availableconnections = [] - self.lastowner = {} - # reset kerberos state - self.gss_step = self.GSS_STATE_STEP - self.gss_vc = None - self.gssapi = False - self.connectionlock.release() + with self.connectionlock: + # first, wait till all connections had been released. + # TODO: won't work IMHO, as releaseconnection() also + # requires the connectionlock, leading to a potential + # deadlock! Audit & check! + threadutil.semaphorereset(self.semaphore, self.maxconnections) + for imapobj in self.assignedconnections + self.availableconnections: + imapobj.logout() + self.assignedconnections = [] + self.availableconnections = [] + self.lastowner = {} + # reset kerberos state + self.gss_step = self.GSS_STATE_STEP + self.gss_vc = None + self.gssapi = False def keepalive(self, timeout, event): - """Sends a NOOP to each connection recorded. It will wait a maximum - of timeout seconds between doing this, and will continue to do so - until the Event object as passed is true. This method is expected - to be invoked in a separate thread, which should be join()'d after - the event is set.""" + """Sends a NOOP to each connection recorded. + + It will wait a maximum of timeout seconds between doing this, and will + continue to do so until the Event object as passed is true. This method + is expected to be invoked in a separate thread, which should be join()'d + after the event is set.""" + self.ui.debug('imap', 'keepalive thread started') - while 1: - self.ui.debug('imap', 'keepalive: top of loop') - time.sleep(timeout) - self.ui.debug('imap', 'keepalive: after wait') - if event.isSet(): - self.ui.debug('imap', 'keepalive: event is set; exiting') - return - self.ui.debug('imap', 'keepalive: acquiring connectionlock') + while not event.isSet(): self.connectionlock.acquire() numconnections = len(self.assignedconnections) + \ len(self.availableconnections) self.connectionlock.release() - self.ui.debug('imap', 'keepalive: connectionlock released') + threads = [] - imapobjs = [] - for i in range(numconnections): - self.ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections)) - imapobj = self.acquireconnection() - self.ui.debug('imap', 'keepalive: connection %d acquired' % i) - imapobjs.append(imapobj) - thr = threadutil.ExitNotifyThread(target = imapobj.noop) - thr.setDaemon(1) - thr.start() - threads.append(thr) - self.ui.debug('imap', 'keepalive: thread started') + self.ui.debug('imap', 'keepalive: processing connection %d of %d'% (i, numconnections)) + if len(self.idlefolders) > i: + # IDLE thread + idler = IdleThread(self, self.idlefolders[i]) + else: + # NOOP thread + idler = IdleThread(self) + idler.start() + threads.append(idler) - self.ui.debug('imap', 'keepalive: joining threads') + self.ui.debug('imap', 'keepalive: waiting for timeout') + event.wait(timeout) + self.ui.debug('imap', 'keepalive: after wait') - for thr in threads: + for idler in threads: # Make sure all the commands have completed. - thr.join() + idler.stop() + idler.join() + self.ui.debug('imap', 'keepalive: all threads joined') + self.ui.debug('imap', 'keepalive: event is set; exiting') + return - self.ui.debug('imap', 'keepalive: releasing connections') + def __verifycert(self, cert, hostname): + """Verify that cert (in socket.getpeercert() format) matches hostname. - for imapobj in imapobjs: - self.releaseconnection(imapobj) + CRLs are not handled. + Returns error message if any problems are found and None on success.""" - self.ui.debug('imap', 'keepalive: bottom of loop') + errstr = "CA Cert verifying failed: " + if not cert: + return ('%s no certificate received'% errstr) + dnsname = hostname.lower() + certnames = [] -class ConfigedIMAPServer(IMAPServer): - """This class is designed for easier initialization given a ConfigParser - object and an account name. The passwordhash is used if - passwords for certain accounts are known. If the password for this - account is listed, it will be obtained from there.""" - def __init__(self, repository, passwordhash = {}): - """Initialize the object. If the account is not a tunnel, - the password is required.""" - self.repos = repository - self.config = self.repos.getconfig() - usetunnel = self.repos.getpreauthtunnel() - if not usetunnel: - host = self.repos.gethost() - user = self.repos.getuser() - port = self.repos.getport() - ssl = self.repos.getssl() - sslclientcert = self.repos.getsslclientcert() - sslclientkey = self.repos.getsslclientkey() - sslcacertfile = self.repos.getsslcacertfile() - reference = self.repos.getreference() - server = None - password = None - - if repository.getname() in passwordhash: - password = passwordhash[repository.getname()] + # cert expired? + notafter = cert.get('notAfter') + if notafter: + if time.time() >= cert_time_to_seconds(notafter): + return '%s certificate expired %s'% (errstr, notafter) - # Connect to the remote server. - if usetunnel: - IMAPServer.__init__(self, self.config, self.repos.getname(), - tunnel = usetunnel, - reference = reference, - maxconnections = self.repos.getmaxconnections()) + # First read commonName + for s in cert.get('subject', []): + key, value = s[0] + if key == 'commonName': + certnames.append(value.lower()) + if len(certnames) == 0: + return ('%s no commonName found in certificate'% errstr) + + # Then read subjectAltName + for key, value in cert.get('subjectAltName', []): + if key == 'DNS': + certnames.append(value.lower()) + + # And finally try to match hostname with one of these names + for certname in certnames: + if (certname == dnsname or + '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): + return None + + return ('%s no matching domain name found in certificate'% errstr) + + +class IdleThread(object): + def __init__(self, parent, folder=None): + """If invoked without 'folder', perform a NOOP and wait for + self.stop() to be called. If invoked with folder, switch to IDLE + mode and synchronize once we have a new message""" + + self.parent = parent + self.folder = folder + self.stop_sig = Event() + self.ui = getglobalui() + if folder is None: + self.thread = Thread(target=self.noop) else: - if not password: - password = self.repos.getpassword() - IMAPServer.__init__(self, self.config, self.repos.getname(), - user, password, host, port, ssl, - self.repos.getmaxconnections(), - reference = reference, - sslclientcert = sslclientcert, - sslclientkey = sslclientkey, - sslcacertfile = sslcacertfile) + self.thread = Thread(target=self.__idle) + self.thread.setDaemon(1) + + def start(self): + self.thread.start() + + def stop(self): + self.stop_sig.set() + + def join(self): + self.thread.join() + + def noop(self): + # TODO: AFAIK this is not optimal, we will send a NOOP on one + # random connection (ie not enough to keep all connections + # open). In case we do the noop multiple times, we can well use + # the same connection every time, as we get a random one. This + # function should IMHO send a noop on ALL available connections + # to the server. + imapobj = self.parent.acquireconnection() + try: + imapobj.noop() + except imapobj.abort: + self.ui.warn('Attempting NOOP on dropped connection %s'% + imapobj.identifier) + self.parent.releaseconnection(imapobj, True) + imapobj = None + finally: + if imapobj: + self.parent.releaseconnection(imapobj) + self.stop_sig.wait() # wait until we are supposed to quit + + def __dosync(self): + remoterepos = self.parent.repos + account = remoterepos.account + localrepos = account.localrepos + remoterepos = account.remoterepos + statusrepos = account.statusrepos + remotefolder = remoterepos.getfolder(self.folder) + + hook = account.getconf('presynchook', '') + account.callhook(hook) + offlineimap.accounts.syncfolder(account, remotefolder, quick=False) + hook = account.getconf('postsynchook', '') + account.callhook(hook) + + ui = getglobalui() + ui.unregisterthread(currentThread()) #syncfolder registered the thread + + def __idle(self): + """Invoke IDLE mode until timeout or self.stop() is invoked.""" + + def callback(args): + """IDLE callback function invoked by imaplib2. + + This is invoked when a) The IMAP server tells us something + while in IDLE mode, b) we get an Exception (e.g. on dropped + connections, or c) the standard imaplib IDLE timeout of 29 + minutes kicks in.""" + result, cb_arg, exc_data = args + if exc_data is None and not self.stop_sig.isSet(): + # No Exception, and we are not supposed to stop: + self.needsync = True + self.stop_sig.set() # Continue to sync. + + while not self.stop_sig.isSet(): + self.needsync = False + + success = False # Successfully selected FOLDER? + while not success: + imapobj = self.parent.acquireconnection() + try: + imapobj.select(self.folder) + except OfflineImapError as e: + if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: + # Connection closed, release connection and retry. + self.ui.error(e, exc_info()[2]) + self.parent.releaseconnection(imapobj, True) + elif e.severity == OfflineImapError.ERROR.FOLDER: + # Just continue the process on such error for now. + self.ui.error(e, exc_info()[2]) + else: + # Stops future attempts to sync this account. + raise + else: + success = True + if "IDLE" in imapobj.capabilities: + imapobj.idle(callback=callback) + else: + self.ui.warn("IMAP IDLE not supported on server '%s'." + "Sleep until next refresh cycle."% imapobj.identifier) + imapobj.noop() + self.stop_sig.wait() # self.stop() or IDLE callback are invoked. + try: + # End IDLE mode with noop, imapobj can point to a dropped conn. + imapobj.noop() + except imapobj.abort: + self.ui.warn('Attempting NOOP on dropped connection %s'% + imapobj.identifier) + self.parent.releaseconnection(imapobj, True) + else: + self.parent.releaseconnection(imapobj) + + if self.needsync: + # Here not via self.stop, but because IDLE responded. Do + # another round and invoke actual syncing. + self.stop_sig.clear() + self.__dosync() diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index fe74854..f1f287b 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -1,6 +1,5 @@ # IMAP utility module -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,47 +15,78 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import re, string, types +import re +import string from offlineimap.ui import getglobalui -quotere = re.compile('^("(?:[^"]|\\\\")*")') -def debug(*args): + +## Globals + +# Message headers that use space as the separator (for label storage) +SPACE_SEPARATED_LABEL_HEADERS = ('X-Label', 'Keywords') + + +def __debug(*args): msg = [] for arg in args: msg.append(str(arg)) getglobalui().debug('imap', " ".join(msg)) -def dequote(string): - """Takes a string which may or may not be quoted and returns it, unquoted. - This function does NOT consider parenthised lists to be quoted. +def dequote(s): + """Takes string which may or may not be quoted and unquotes it. + + It only considers double quotes. This function does NOT consider + parenthised lists to be quoted.""" + + if s and s.startswith('"') and s.endswith('"'): + s = s[1:-1] # Strip off the surrounding quotes. + s = s.replace('\\"', '"') + s = s.replace('\\\\', '\\') + return s + +def quote(s): + """Takes an unquoted string and quotes it. + + It only adds double quotes. This function does NOT consider + parenthised lists to be quoted.""" + + s = s.replace('"', '\\"') + s = s.replace('\\', '\\\\') + return '"%s"'% s + +def flagsplit(s): + """Converts a string of IMAP flags to a list + + :returns: E.g. '(\\Draft \\Deleted)' returns ['\\Draft','\\Deleted']. + (FLAGS (\\Seen Old) UID 4807) returns + ['FLAGS,'(\\Seen Old)','UID', '4807'] """ - debug("dequote() called with input:", string) - if not (string[0] == '"' and string[-1] == '"'): - return string - string = string[1:-1] # Strip off quotes. - string = string.replace('\\"', '"') - string = string.replace('\\\\', '\\') - debug("dequote() returning:", string) - return string + if s[0] != '(' or s[-1] != ')': + raise ValueError("Passed s '%s' is not a flag list"% s) + return imapsplit(s[1:-1]) -def flagsplit(string): - if string[0] != '(' or string[-1] != ')': - raise ValueError, "Passed string '%s' is not a flag list" % string - return imapsplit(string[1:-1]) +def __options2hash(list): + """convert list [1,2,3,4,5,6] to {1:2, 3:4, 5:6}""" -def options2hash(list): - debug("options2hash called with input:", list) + # effectively this does dict(zip(l[::2],l[1::2])), however + # measurements seemed to have indicated that the manual variant is + # faster for mosly small lists. retval = {} counter = 0 while (counter < len(list)): retval[list[counter]] = list[counter + 1] counter += 2 - debug("options2hash returning:", retval) + __debug("__options2hash returning:", retval) return retval -def flags2hash(string): - return options2hash(flagsplit(string)) +def flags2hash(flags): + """Converts IMAP response string from eg IMAP4.fetch() to a hash. + + E.g. '(FLAGS (\\Seen Old) UID 4807)' leads to + {'FLAGS': '(\\Seen Old)', 'UID': '4807'}""" + + return __options2hash(flagsplit(flags)) def imapsplit(imapstring): """Takes a string from an IMAP conversation and returns a list containing @@ -68,9 +98,8 @@ def imapsplit(imapstring): ['(\\HasNoChildren)', '"."', '"INBOX.Sent"']""" - debug("imapsplit() called with input:", imapstring) - if type(imapstring) != types.StringType: - debug("imapsplit() got a non-string input; working around.") + if not isinstance(imapstring, basestring): + __debug("imapsplit() got a non-string input; working around.") # Sometimes, imaplib will throw us a tuple if the input # contains a literal. See Python bug # #619732 at https://sourceforge.net/tracker/index.php?func=detail&aid=619732&group_id=5470&atid=105470 @@ -92,8 +121,7 @@ def imapsplit(imapstring): arg = arg.replace('\\', '\\\\') arg = arg.replace('"', '\\"') arg = '"%s"' % arg - debug("imapsplit() non-string [%d]: Appending %s" %\ - (i, arg)) + __debug("imapsplit() non-string [%d]: Appending %s"% (i, arg)) retval.append(arg) else: # Even -- we have a string that ends with a literal @@ -102,31 +130,33 @@ def imapsplit(imapstring): # Recursion to the rescue. arg = imapstring[i] arg = re.sub('\{\d+\}$', '', arg) - debug("imapsplit() non-string [%d]: Feeding %s to recursion" %\ - (i, arg)) + __debug("imapsplit() non-string [%d]: Feeding %s to recursion"%\ + (i, arg)) retval.extend(imapsplit(arg)) - debug("imapsplit() non-string: returning %s" % str(retval)) + __debug("imapsplit() non-string: returning %s" % str(retval)) return retval - + workstr = imapstring.strip() retval = [] while len(workstr): + # handle parenthized fragments (...()...) if workstr[0] == '(': rparenc = 1 # count of right parenthesis to match rpareni = 1 # position to examine - while rparenc: # Find the end of the group. - if workstr[rpareni] == ')': # end of a group - rparenc -= 1 - elif workstr[rpareni] == '(': # start of a group - rparenc += 1 - rpareni += 1 # Move to next character. + while rparenc: # Find the end of the group. + if workstr[rpareni] == ')': # end of a group + rparenc -= 1 + elif workstr[rpareni] == '(': # start of a group + rparenc += 1 + rpareni += 1 # Move to next character. parenlist = workstr[0:rpareni] workstr = workstr[rpareni:].lstrip() retval.append(parenlist) elif workstr[0] == '"': - quotelist = quotere.search(workstr).group(1) - workstr = workstr[len(quotelist):].lstrip() - retval.append(quotelist) + # quoted fragments '"...\"..."' + (quoted, rest) = __split_quoted(workstr) + retval.append(quoted) + workstr = rest else: splits = string.split(workstr, maxsplit = 1) splitslen = len(splits) @@ -144,9 +174,8 @@ def imapsplit(imapstring): elif splitslen == 0: # There was not even an unquoted word. break - debug("imapsplit() returning:", retval) return retval - + flagmap = [('\\Seen', 'S'), ('\\Answered', 'R'), ('\\Flagged', 'F'), @@ -154,54 +183,148 @@ flagmap = [('\\Seen', 'S'), ('\\Draft', 'D')] def flagsimap2maildir(flagstring): - retval = [] - imapflaglist = [x.lower() for x in flagstring[1:-1].split()] + """Convert string '(\\Draft \\Deleted)' into a flags set(DR).""" + + retval = set() + imapflaglist = flagstring[1:-1].split() for imapflag, maildirflag in flagmap: - if imapflag.lower() in imapflaglist: - retval.append(maildirflag) - retval.sort() + if imapflag in imapflaglist: + retval.add(maildirflag) return retval def flagsmaildir2imap(maildirflaglist): + """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'.""" + retval = [] for imapflag, maildirflag in flagmap: if maildirflag in maildirflaglist: retval.append(imapflag) - retval.sort() - return '(' + ' '.join(retval) + ')' + return '(' + ' '.join(sorted(retval)) + ')' -def listjoin(list): - start = None - end = None - retval = [] +def uid_sequence(uidlist): + """Collapse UID lists into shorter sequence sets - def getlist(start, end): + [1,2,3,4,5,10,12,13] will return "1:5,10,12:13". This function sorts + the list, and only collapses if subsequent entries form a range. + :returns: The collapsed UID list as string.""" + + def getrange(start, end): if start == end: return(str(start)) - else: - return(str(start) + ":" + str(end)) - + return "%s:%s"% (start, end) - for item in list: - if start == None: - # First item. - start = item - end = item - elif item == end + 1: - # An addition to the list. - end = item - else: - # Here on: starting a new list. - retval.append(getlist(start, end)) - start = item - end = item + if not len(uidlist): return '' # Empty list, return + start, end = None, None + retval = [] + # Force items to be longs and sort them + sorted_uids = sorted(map(int, uidlist)) - if start != None: - retval.append(getlist(start, end)) + for item in iter(sorted_uids): + item = int(item) + if start == None: # First item + start, end = item, item + elif item == end + 1: # Next item in a range + end = item + else: # Starting a new range + retval.append(getrange(start, end)) + start, end = item, item + retval.append(getrange(start, end)) # Add final range/item return ",".join(retval) +def __split_quoted(s): + """Looks for the ending quote character in the string that starts + with quote character, splitting out quoted component and the + rest of the string (without possible space between these two + parts. + + First character of the string is taken to be quote character. + + Examples: + - "this is \" a test" (\\None) => ("this is \" a test", (\\None)) + - "\\" => ("\\", ) + """ + + if len(s) == 0: + return ('', '') + + q = quoted = s[0] + rest = s[1:] + while True: + next_q = rest.find(q) + if next_q == -1: + raise ValueError("can't find ending quote '%s' in '%s'"% (q, s)) + # If quote is preceeded by even number of backslashes, + # then it is the ending quote, otherwise the quote + # character is escaped by backslash, so we should + # continue our search. + is_escaped = False + i = next_q - 1 + while i >= 0 and rest[i] == '\\': + i -= 1 + is_escaped = not is_escaped + quoted += rest[0:next_q + 1] + rest = rest[next_q + 1:] + if not is_escaped: + return (quoted, rest.lstrip()) + + +def format_labels_string(header, labels): + """Formats labels for embedding into a message, + with format according to header name. + + Headers from SPACE_SEPARATED_LABEL_HEADERS keep space-separated list + of labels, the rest uses comma (',') as the separator. + + Also see parse_labels_string() and modify it accordingly + if logics here gets changed.""" + + if header in SPACE_SEPARATED_LABEL_HEADERS: + sep = ' ' + else: + sep = ',' + + return sep.join(labels) + + +def parse_labels_string(header, labels_str): + """Parses a string into a set of labels, with a format according to + the name of the header. + + See __format_labels_string() for explanation on header handling + and keep these two functions synced with each other. + + TODO: add test to ensure that + - format_labels_string * parse_labels_string is unity + and + - parse_labels_string * format_labels_string is unity + """ + + if header in SPACE_SEPARATED_LABEL_HEADERS: + sep = ' ' + else: + sep = ',' + + labels = labels_str.strip().split(sep) + + return set([l.strip() for l in labels if l.strip()]) + + +def labels_from_header(header_name, header_value): + """Helper that builds label set from the corresponding header value. + + Arguments: + - header_name: name of the header that keeps labels; + - header_value: value of the said header, can be None + + Returns: set of labels parsed from the header (or empty set). + """ + + if header_value: + labels = parse_labels_string(header_name, header_value) + else: + labels = set() + + return labels - - diff --git a/offlineimap/init.py b/offlineimap/init.py index 9e23fdd..a48c152 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -1,6 +1,5 @@ # OfflineIMAP initialization code -# Copyright (C) 2002-2007 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,148 +18,122 @@ import os import sys import threading -import imaplib +import offlineimap.imaplib2 as imaplib import signal import socket import logging from optparse import OptionParser + import offlineimap from offlineimap import accounts, threadutil, syncmaster +from offlineimap import globals from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser +from offlineimap.utils import stacktrace -try: - import fcntl - hasfcntl = 1 -except: - hasfcntl = 0 - -lockfd = None - class OfflineImap: """The main class that encapsulates the high level use of OfflineImap. - To invoke OfflineImap you would call it with: - oi = OfflineImap() - oi.run() + To invoke OfflineImap you would call it with:: + + oi = OfflineImap() + oi.run() """ - def lock(self, config, ui): - global lockfd, hasfcntl - if not hasfcntl: - return - lockfd = open(config.getmetadatadir() + "/lock", "w") - try: - fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - ui.locked() - ui.terminate(1) - + def run(self): """Parse the commandline and invoke everything""" + # next line also sets self.config and self.ui + options, args = self.__parse_cmd_options() + if options.diagnostics: + self.__serverdiagnostics(options) + else: + self.__sync(options) - parser = OptionParser(version=offlineimap.__version__, - description="%s.\n\n%s" % + def __parse_cmd_options(self): + parser = OptionParser(version=offlineimap.__bigversion__, + description="%s.\n\n%s" % (offlineimap.__copyright__, offlineimap.__license__)) + parser.add_option("--dry-run", + action="store_true", dest="dryrun", + default=False, + help="dry run mode") + + parser.add_option("--info", + action="store_true", dest="diagnostics", + default=False, + help="output information on the configured email repositories") + parser.add_option("-1", action="store_true", dest="singlethreading", default=False, - help="Disable all multithreading operations and use " - "solely a single-thread sync. This effectively sets the " - "maxsyncaccounts and all maxconnections configuration file " - "variables to 1.") + help="(the number one) disable all multithreading operations") parser.add_option("-P", dest="profiledir", metavar="DIR", - help="Sets OfflineIMAP into profile mode. The program " - "will create DIR (it must not already exist). " - "As it runs, Python profiling information about each " - "thread is logged into profiledir. Please note: " - "This option is present for debugging and optimization " - "only, and should NOT be used unless you have a " - "specific reason to do so. It will significantly " - "decrease program performance, may reduce reliability, " - "and can generate huge amounts of data. This option " - "implies the -1 option.") + help="sets OfflineIMAP into profile mode.") - parser.add_option("-a", dest="accounts", metavar="ACCOUNTS", - help="""Overrides the accounts section in the config file. - Lets you specify a particular account or set of - accounts to sync without having to edit the config - file. You might use this to exclude certain accounts, - or to sync some accounts that you normally prefer not to.""") + parser.add_option("-a", dest="accounts", + metavar="account1[,account2[,...]]", + help="list of accounts to sync") parser.add_option("-c", dest="configfile", metavar="FILE", - default="~/.offlineimaprc", - help="Specifies a configuration file to use in lieu of " - "%default.") + default=None, + help="specifies a configuration file to use") - parser.add_option("-d", dest="debugtype", metavar="type1,[type2...]", - help="""Enables debugging for OfflineIMAP. This is useful - if you are to track down a malfunction or figure out what is - going on under the hood. This option requires one or more - debugtypes, separated by commas. These define what exactly - will be debugged, and so far include two options: imap, thread, - maildir or ALL. The imap option will enable IMAP protocol - stream and parsing debugging. Note that the output may contain - passwords, so take care to remove that from the debugging - output before sending it to anyone else. The maildir option - will enable debugging for certain Maildir operations. - The use of any debug option (unless 'thread' is included), - implies the single-thread option -1.""") + parser.add_option("-d", dest="debugtype", + metavar="type1[,type2[,...]]", + help="enables debugging for OfflineIMAP " + " (types: imap, maildir, thread)") parser.add_option("-l", dest="logfile", metavar="FILE", - help="Log to FILE") + help="log to FILE") - parser.add_option("-f", dest="folders", metavar="folder1,[folder2...]", - help= - "Only sync the specified folders. The folder names " - "are the *untranslated* foldernames. This " - "command-line option overrides any 'folderfilter' " - "and 'folderincludes' options in the configuration " - "file.") + parser.add_option("-f", dest="folders", + metavar="folder1[,folder2[,...]]", + help="only sync the specified folders") parser.add_option("-k", dest="configoverride", action="append", metavar="[section:]option=value", - help= - """Override configuration file option. If"section" is - omitted, it defaults to "general". Any underscores - in the section name are replaced with spaces: - for instance, to override option "autorefresh" in - the "[Account Personal]" section in the config file - one would use "-k Account_Personal:autorefresh=30".""") + help="override configuration file option") parser.add_option("-o", action="store_true", dest="runonce", default=False, - help="Run only once, ignoring any autorefresh setting " - "in the configuration file.") + help="run only once (ignore autorefresh)") parser.add_option("-q", action="store_true", dest="quick", default=False, - help="Run only quick synchronizations. Ignore any " - "flag updates on IMAP servers (if a flag on the remote IMAP " - "changes, and we have the message locally, it will be left " - "untouched in a quick run.") + help="run only quick synchronizations (don't update flags)") parser.add_option("-u", dest="interface", - help="Specifies an alternative user interface to " - "use. This overrides the default specified in the " - "configuration file. The UI specified with -u will " - "be forced to be used, even if checks determine that it is " - "not usable. Possible interface choices are: %s " % - ", ".join(UI_LIST.keys())) + help="specifies an alternative user interface" + " (quiet, basic, ttyui, blinkenlights, machineui)") (options, args) = parser.parse_args() + globals.set_options (options) #read in configuration file - configfilename = os.path.expanduser(options.configfile) - + if not options.configfile: + # Try XDG location, then fall back to ~/.offlineimaprc + xdg_var = 'XDG_CONFIG_HOME' + if not xdg_var in os.environ or not os.environ[xdg_var]: + xdg_home = os.path.expanduser('~/.config') + else: + xdg_home = os.environ[xdg_var] + options.configfile = os.path.join(xdg_home, "offlineimap", "config") + if not os.path.exists(options.configfile): + options.configfile = os.path.expanduser('~/.offlineimaprc') + configfilename = options.configfile + else: + configfilename = os.path.expanduser(options.configfile) + config = CustomConfigParser() if not os.path.exists(configfilename): - logging.error(" *** Config file '%s' does not exist; aborting!" % + # TODO, initialize and make use of chosen ui for logging + logging.error(" *** Config file '%s' does not exist; aborting!"% configfilename) sys.exit(1) config.read(configfilename) @@ -168,13 +141,19 @@ class OfflineImap: #profile mode chosen? if options.profiledir: if not options.singlethreading: + # TODO, make use of chosen ui for logging logging.warn("Profile mode: Forcing to singlethreaded.") options.singlethreading = True - profiledir = options.profiledir - os.mkdir(profiledir) - threadutil.setprofiledir(profiledir) + if os.path.exists(options.profiledir): + # TODO, make use of chosen ui for logging + logging.warn("Profile mode: Directory '%s' already exists!"% + options.profiledir) + else: + os.mkdir(options.profiledir) + threadutil.ExitNotifyThread.set_profiledir(options.profiledir) + # TODO, make use of chosen ui for logging logging.warn("Profile mode: Potentially large data will be " - "created in '%s'" % profiledir) + "created in '%s'"% options.profiledir) #override a config value if options.configoverride: @@ -187,44 +166,58 @@ class OfflineImap: section = "general" config.set(section, key, value) - #init the ui, cmd line option overrides config file - ui_type = config.getdefault('general','ui', 'TTY.TTYUI') + #which ui to use? cmd line option overrides config file + ui_type = config.getdefault('general', 'ui', 'ttyui') if options.interface != None: ui_type = options.interface + if '.' in ui_type: + #transform Curses.Blinkenlights -> Blinkenlights + ui_type = ui_type.split('.')[-1] + # TODO, make use of chosen ui for logging + logging.warning('Using old interface name, consider using one ' + 'of %s'% ', '.join(UI_LIST.keys())) + if options.diagnostics: ui_type = 'basic' # enforce basic UI for --info + + # dry-run? Set [general]dry-run=True + if options.dryrun: + dryrun = config.set('general', 'dry-run', 'True') + config.set_if_not_exists('general', 'dry-run', 'False') + try: - ui = UI_LIST[ui_type](config) + # create the ui class + self.ui = UI_LIST[ui_type.lower()](config) except KeyError: - logging.error("UI '%s' does not exist, choose one of: %s" % \ - (ui_type,', '.join(UI_LIST.keys()))) + logging.error("UI '%s' does not exist, choose one of: %s"% \ + (ui_type, ', '.join(UI_LIST.keys()))) sys.exit(1) - setglobalui(ui) + setglobalui(self.ui) #set up additional log files if options.logfile: - ui.setlogfd(open(options.logfile, 'wt')) - + self.ui.setlogfile(options.logfile) + #welcome blurb - ui.init_banner() + self.ui.init_banner() if options.debugtype: + self.ui.logger.setLevel(logging.DEBUG) if options.debugtype.lower() == 'all': options.debugtype = 'imap,maildir,thread' #force single threading? if not ('thread' in options.debugtype.split(',') \ - and options.singlethreading): - ui._msg("Debug mode: Forcing to singlethreaded.") + and not options.singlethreading): + self.ui._msg("Debug mode: Forcing to singlethreaded.") options.singlethreading = True - for type in options.debugtype.split(','): - type = type.strip() - ui.add_debug(type) - if type.lower() == 'imap': + debugtypes = options.debugtype.split(',') + [''] + for dtype in debugtypes: + dtype = dtype.strip() + self.ui.add_debug(dtype) + if dtype.lower() == u'imap': imaplib.Debug = 5 - if type.lower() == 'thread': - threading._VERBOSE = 1 if options.runonce: - # FIXME: maybe need a better + # FIXME: spaghetti code alert! for section in accounts.getaccountlist(config): config.remove_option('Account ' + section, "autorefresh") @@ -232,135 +225,143 @@ class OfflineImap: for section in accounts.getaccountlist(config): config.set('Account ' + section, "quick", '-1') + #custom folder list specified? if options.folders: - foldernames = options.folders.replace(" ", "").split(",") - folderfilter = "lambda f: f in %s" % foldernames + foldernames = options.folders.split(",") + folderfilter = "lambda f: f in %s"% foldernames folderincludes = "[]" for accountname in accounts.getaccountlist(config): account_section = 'Account ' + accountname remote_repo_section = 'Repository ' + \ - config.get(account_section, 'remoterepository') - local_repo_section = 'Repository ' + \ - config.get(account_section, 'localrepository') - for section in [remote_repo_section, local_repo_section]: - config.set(section, "folderfilter", folderfilter) - config.set(section, "folderincludes", folderincludes) + config.get(account_section, 'remoterepository') + config.set(remote_repo_section, "folderfilter", folderfilter) + config.set(remote_repo_section, "folderincludes", + folderincludes) - self.lock(config, ui) + if options.logfile: + sys.stderr = self.ui.logfile - - def sigterm_handler(signum, frame): - # die immediately - ui = getglobalui() - ui.terminate(errormsg="terminating...") + socktimeout = config.getdefaultint("general", "socktimeout", 0) + if socktimeout > 0: + socket.setdefaulttimeout(socktimeout) - signal.signal(signal.SIGTERM,sigterm_handler) - + threadutil.initInstanceLimit('ACCOUNTLIMIT', + config.getdefaultint('general', 'maxsyncaccounts', 1)) + + for reposname in config.getsectionlist('Repository'): + for instancename in ["FOLDER_" + reposname, + "MSGCOPY_" + reposname]: + if options.singlethreading: + threadutil.initInstanceLimit(instancename, 1) + else: + threadutil.initInstanceLimit(instancename, + config.getdefaultint('Repository ' + reposname, + 'maxconnections', 2)) + self.config = config + return (options, args) + + def __sync(self, options): + """Invoke the correct single/multithread syncing + + self.config is supposed to have been correctly initialized + already.""" try: - pidfd = open(config.getmetadatadir() + "/pid", "w") + pidfd = open(self.config.getmetadatadir() + "/pid", "w") pidfd.write(str(os.getpid()) + "\n") pidfd.close() except: pass - + try: - if options.logfile: - sys.stderr = ui.logfile - - socktimeout = config.getdefaultint("general", "socktimeout", 0) - if socktimeout > 0: - socket.setdefaulttimeout(socktimeout) - - activeaccounts = config.get("general", "accounts") + # Honor CLI --account option, only. + # Accounts to sync are put into syncaccounts variable. + activeaccounts = self.config.get("general", "accounts") if options.accounts: activeaccounts = options.accounts activeaccounts = activeaccounts.replace(" ", "") activeaccounts = activeaccounts.split(",") - allaccounts = accounts.AccountHashGenerator(config) - + allaccounts = accounts.AccountHashGenerator(self.config) + syncaccounts = [] for account in activeaccounts: if account not in allaccounts: if len(allaccounts) == 0: - errormsg = 'The account "%s" does not exist because no accounts are defined!'%account + errormsg = "The account '%s' does not exist because no" \ + " accounts are defined!"% account else: - errormsg = 'The account "%s" does not exist. Valid accounts are:'%account - for name in allaccounts.keys(): - errormsg += '\n%s'%name - ui.terminate(1, errortitle = 'Unknown Account "%s"'%account, errormsg = errormsg) + errormsg = "The account '%s' does not exist. Valid ac" \ + "counts are: %s"% \ + (account, ", ".join(allaccounts.keys())) + self.ui.terminate(1, errormsg=errormsg) if account not in syncaccounts: syncaccounts.append(account) - - server = None - remoterepos = None - localrepos = None - - threadutil.initInstanceLimit("ACCOUNTLIMIT", - config.getdefaultint("general", "maxsyncaccounts", 1)) - - for reposname in config.getsectionlist('Repository'): - for instancename in ["FOLDER_" + reposname, - "MSGCOPY_" + reposname]: - if options.singlethreading: - threadutil.initInstanceLimit(instancename, 1) - else: - threadutil.initInstanceLimit(instancename, - config.getdefaultint('Repository ' + reposname, "maxconnections", 1)) - siglisteners = [] - def sig_handler(signum, frame): - if signum == signal.SIGUSR1: - # tell each account to do a full sync asap - signum = (1,) - elif signum == signal.SIGHUP: - # tell each account to die asap - signum = (2,) - elif signum == signal.SIGUSR2: - # tell each account to do a full sync asap, then die - signum = (1, 2) - # one listener per account thread (up to maxsyncaccounts) - for listener in siglisteners: - for sig in signum: - listener.put_nowait(sig) - signal.signal(signal.SIGHUP,sig_handler) - signal.signal(signal.SIGUSR1,sig_handler) - signal.signal(signal.SIGUSR2,sig_handler) - + + def sig_handler(sig, frame): + if sig == signal.SIGUSR1: + # tell each account to stop sleeping + accounts.Account.set_abort_event(self.config, 1) + elif sig == signal.SIGUSR2: + # tell each account to stop looping + getglobalui().warn("Terminating after this sync...") + accounts.Account.set_abort_event(self.config, 2) + elif sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + # tell each account to ABORT ASAP (ctrl-c) + getglobalui().warn("Terminating NOW (this may "\ + "take a few seconds)...") + accounts.Account.set_abort_event(self.config, 3) + elif sig == signal.SIGQUIT: + stacktrace.dump(sys.stderr) + os.abort() + + signal.signal(signal.SIGHUP, sig_handler) + signal.signal(signal.SIGUSR1, sig_handler) + signal.signal(signal.SIGUSR2, sig_handler) + signal.signal(signal.SIGTERM, sig_handler) + signal.signal(signal.SIGINT, sig_handler) + signal.signal(signal.SIGQUIT, sig_handler) + #various initializations that need to be performed: - threadutil.initexitnotify() #TODO: Why? - offlineimap.mbnames.init(config, syncaccounts) + offlineimap.mbnames.init(self.config, syncaccounts) if options.singlethreading: #singlethreaded - self.sync_singlethreaded(syncaccounts, config, siglisteners) + self.__sync_singlethreaded(syncaccounts) else: # multithreaded t = threadutil.ExitNotifyThread(target=syncmaster.syncitall, name='Sync Runner', kwargs = {'accounts': syncaccounts, - 'config': config, - 'siglisteners': siglisteners}) - t.setDaemon(1) + 'config': self.config}) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) - except KeyboardInterrupt: - ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') - return + if not options.dryrun: + offlineimap.mbnames.write(True) + + self.ui.terminate() except (SystemExit): raise - except: - ui.mainException() + except Exception as e: + self.ui.error(e) + self.ui.terminate() - def sync_singlethreaded(self, accs, config, siglisteners): + def __sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread :param accs: A list of accounts that should be synced - :param config: The CustomConfig object - :param siglisteners: The signal listeners list, defined in run() """ for accountname in accs: - account = offlineimap.accounts.SyncableAccount(config, accountname) - siglistener = offlineimap.accounts.SigListener() - siglisteners.append(siglistener) - threading.currentThread().name = "Account sync %s" % accountname - account.syncrunner(siglistener=siglistener) + account = offlineimap.accounts.SyncableAccount(self.config, + accountname) + threading.currentThread().name = "Account sync %s"% accountname + account.syncrunner() + + def __serverdiagnostics(self, options): + activeaccounts = self.config.get("general", "accounts") + if options.accounts: + activeaccounts = options.accounts + activeaccounts = activeaccounts.split(",") + allaccounts = accounts.AccountListGenerator(self.config) + for account in allaccounts: + if account.name not in activeaccounts: continue + account.serverdiagnostics() diff --git a/offlineimap/localeval.py b/offlineimap/localeval.py index 22014e6..a9494fb 100644 --- a/offlineimap/localeval.py +++ b/offlineimap/localeval.py @@ -1,7 +1,6 @@ """Eval python code with global namespace of a python source file.""" -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2014 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,18 +23,22 @@ except: pass class LocalEval: + """Here is a powerfull but very dangerous option, of course.""" + def __init__(self, path=None): - self.namespace={} + self.namespace = {} if path is not None: - file=open(path, 'r') - module=imp.load_module( + # FIXME: limit opening files owned by current user with rights set + # to fixed mode 644. + foo = open(path, 'r') + module = imp.load_module( '', - file, + foo, path, ('', 'r', imp.PY_SOURCE)) for attr in dir(module): - self.namespace[attr]=getattr(module, attr) + self.namespace[attr] = getattr(module, attr) def eval(self, text, namespace=None): names = {} diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 5ff0f29..8829ee5 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -1,6 +1,6 @@ # Mailbox name generator -# Copyright (C) 2002 John Goerzen -# +# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,9 +18,10 @@ import os.path import re # for folderfilter -from threading import * +from threading import Lock boxes = {} +localroots = {} config = None accounts = None mblock = Lock() @@ -30,45 +31,65 @@ def init(conf, accts): config = conf accounts = accts -def add(accountname, foldername): +def add(accountname, foldername, localfolders): if not accountname in boxes: boxes[accountname] = [] + localroots[accountname] = localfolders if not foldername in boxes[accountname]: boxes[accountname].append(foldername) -def write(): +def write(allcomplete): + incremental = config.getdefaultboolean("mbnames", "incremental", False) + + # Skip writing if we don't want incremental writing and we're not done. + if not incremental and not allcomplete: + return + + # Skip writing if we want incremental writing and we're done. + if incremental and allcomplete: + return + # See if we're ready to write it out. for account in accounts: if account not in boxes: return - genmbnames() + __genmbnames() -def genmbnames(): +def __genmbnames(): """Takes a configparser object and a boxlist, which is a list of hashes containing 'accountname' and 'foldername' keys.""" + + xforms = [os.path.expanduser, os.path.expandvars] mblock.acquire() try: localeval = config.getlocaleval() if not config.getdefaultboolean("mbnames", "enabled", 0): return - file = open(os.path.expanduser(config.get("mbnames", "filename")), "wt") + path = config.apply_xforms(config.get("mbnames", "filename"), xforms) + file = open(path, "wt") file.write(localeval.eval(config.get("mbnames", "header"))) folderfilter = lambda accountname, foldername: 1 if config.has_option("mbnames", "folderfilter"): folderfilter = localeval.eval(config.get("mbnames", "folderfilter"), {'re': re}) + mb_sort_keyfunc = lambda d: (d['accountname'], d['foldername']) + if config.has_option("mbnames", "sort_keyfunc"): + mb_sort_keyfunc = localeval.eval(config.get("mbnames", "sort_keyfunc"), + {'re': re}) itemlist = [] for accountname in boxes.keys(): + localroot = localroots[accountname] for foldername in boxes[accountname]: if folderfilter(accountname, foldername): - itemlist.append(config.get("mbnames", "peritem", raw=1) % \ - {'accountname': accountname, - 'foldername': foldername}) + itemlist.append({'accountname': accountname, + 'foldername': foldername, + 'localfolders': localroot}) + itemlist.sort(key = mb_sort_keyfunc) + format_string = config.get("mbnames", "peritem", raw=1) + itemlist = [format_string % d for d in itemlist] file.write(localeval.eval(config.get("mbnames", "sep")).join(itemlist)) file.write(localeval.eval(config.get("mbnames", "footer"))) file.close() finally: mblock.release() - - diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 7e1dd4d..0cf44f8 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -1,6 +1,5 @@ # Base repository support -# Copyright (C) 2002-2007 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,61 +15,59 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import re +import os.path +from sys import exc_info + from offlineimap import CustomConfig from offlineimap.ui import getglobalui -import os.path -import traceback +from offlineimap.error import OfflineImapError -def LoadRepository(name, account, reqtype): - from offlineimap.repository.Gmail import GmailRepository - from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository - from offlineimap.repository.Maildir import MaildirRepository - if reqtype == 'remote': - # For now, we don't support Maildirs on the remote side. - typemap = {'IMAP': IMAPRepository, - 'Gmail': GmailRepository} - elif reqtype == 'local': - typemap = {'IMAP': MappedIMAPRepository, - 'Maildir': MaildirRepository} - else: - raise ValueError, "Request type %s not supported" % reqtype - config = account.getconfig() - repostype = config.get('Repository ' + name, 'type').strip() - try: - repo = typemap[repostype] - except KeyError: - raise Exception, "'%s' repository not supported for %s repositories."%\ - (repostype, reqtype) - return repo(name, account) +class BaseRepository(CustomConfig.ConfigHelperMixin, object): -class BaseRepository(CustomConfig.ConfigHelperMixin): def __init__(self, reposname, account): + self.ui = getglobalui() self.account = account self.config = account.getconfig() self.name = reposname self.localeval = account.getlocaleval() - self.accountname = self.account.getname() + self._accountname = self.account.getname() + self._readonly = self.getconfboolean('readonly', False) self.uiddir = os.path.join(self.config.getmetadatadir(), 'Repository-' + self.name) if not os.path.exists(self.uiddir): - os.mkdir(self.uiddir, 0700) + os.mkdir(self.uiddir, 0o700) self.mapdir = os.path.join(self.uiddir, 'UIDMapping') if not os.path.exists(self.mapdir): - os.mkdir(self.mapdir, 0700) + os.mkdir(self.mapdir, 0o700) + # FIXME: self.uiddir variable name is lying about itself. self.uiddir = os.path.join(self.uiddir, 'FolderValidity') if not os.path.exists(self.uiddir): - os.mkdir(self.uiddir, 0700) + os.mkdir(self.uiddir, 0o700) + + self.nametrans = lambda foldername: foldername + self.folderfilter = lambda foldername: 1 + self.folderincludes = [] + self.foldersort = None + if self.config.has_option(self.getsection(), 'nametrans'): + self.nametrans = self.localeval.eval( + self.getconf('nametrans'), {'re': re}) + if self.config.has_option(self.getsection(), 'folderfilter'): + self.folderfilter = self.localeval.eval( + self.getconf('folderfilter'), {'re': re}) + if self.config.has_option(self.getsection(), 'folderincludes'): + self.folderincludes = self.localeval.eval( + self.getconf('folderincludes'), {'re': re}) + if self.config.has_option(self.getsection(), 'foldersort'): + self.foldersort = self.localeval.eval( + self.getconf('foldersort'), {'re': re}) - # The 'restoreatime' config parameter only applies to local Maildir - # mailboxes. def restore_atime(self): - if self.config.get('Repository ' + self.name, 'type').strip() != \ - 'Maildir': - return + """Sets folders' atime back to their values after a sync - if not self.config.has_option('Repository ' + self.name, 'restoreatime') or not self.config.getboolean('Repository ' + self.name, 'restoreatime'): - return - - return self.restore_folder_atimes() + Controlled by the 'restoreatime' config parameter (default + False), applies only to local Maildir mailboxes and does nothing + on all other repository types.""" + pass def connect(self): """Establish a connection to the remote, if necessary. This exists @@ -93,24 +90,36 @@ class BaseRepository(CustomConfig.ConfigHelperMixin): def getname(self): return self.name + def __str__(self): + return self.name + + @property + def accountname(self): + """Account name as string""" + return self._accountname + def getuiddir(self): return self.uiddir def getmapdir(self): return self.mapdir - def getaccountname(self): - return self.accountname - + # Interface from CustomConfig.ConfigHelperMixin def getsection(self): return 'Repository ' + self.name + # Interface from CustomConfig.ConfigHelperMixin def getconfig(self): return self.config + @property + def readonly(self): + """Is the repository readonly?""" + return self._readonly + def getlocaleval(self): return self.account.getlocaleval() - + def getfolders(self): """Returns a list of ALL folders on this server.""" return [] @@ -123,7 +132,22 @@ class BaseRepository(CustomConfig.ConfigHelperMixin): def getsep(self): raise NotImplementedError + def should_sync_folder(self, fname): + """Should this folder be synced?""" + + return fname in self.folderincludes or self.folderfilter(fname) + + def get_create_folders(self): + """Is folder creation enabled on this repository? + + It is disabled by either setting the whole repository + 'readonly' or by using the 'createfolders' setting.""" + + return (not self._readonly) and \ + self.getconfboolean('createfolders', True) + def makefolder(self, foldername): + """Create a new folder.""" raise NotImplementedError def deletefolder(self, foldername): @@ -131,54 +155,106 @@ class BaseRepository(CustomConfig.ConfigHelperMixin): def getfolder(self, foldername): raise NotImplementedError - - def syncfoldersto(self, dest, copyfolders): + + def sync_folder_structure(self, dst_repo, status_repo): """Syncs the folders in this repository to those in dest. - It does NOT sync the contents of those folders. - For every time dest.makefolder() is called, also call makefolder() - on each folder in copyfolders.""" - src = self - srcfolders = src.getfolders() - destfolders = dest.getfolders() + It does NOT sync the contents of those folders. nametrans rules + in both directions will be honored, but there are NO checks yet + that forward and backward nametrans actually match up! + Configuring nametrans on BOTH repositories therefore could lead + to infinite folder creation cycles.""" + if not self.get_create_folders() and not dst_repo.get_create_folders(): + # quick exit if no folder creation is enabled on either side. + return + + src_repo = self + src_folders = src_repo.getfolders() + dst_folders = dst_repo.getfolders() + # Do we need to refresh the folder list afterwards? + src_haschanged, dst_haschanged = False, False # Create hashes with the names, but convert the source folders # to the dest folder's sep. + src_hash = {} + for folder in src_folders: + src_hash[folder.getvisiblename().replace( + src_repo.getsep(), dst_repo.getsep())] = folder + dst_hash = {} + for folder in dst_folders: + dst_hash[folder.getvisiblename().replace( + dst_repo.getsep(), src_repo.getsep())] = folder - srchash = {} - for folder in srcfolders: - srchash[folder.getvisiblename().replace(src.getsep(), dest.getsep())] = \ - folder - desthash = {} - for folder in destfolders: - desthash[folder.getvisiblename()] = folder - - # - # Find new folders. - # - - for key in srchash.keys(): - if not key in desthash: + # Find new folders on src_repo. + for src_name_t, src_folder in src_hash.iteritems(): + # Don't create on dst_repo, if it is readonly + if not dst_repo.get_create_folders(): + break + if src_folder.sync_this and not src_name_t in dst_folders: try: - dest.makefolder(key) - for copyfolder in copyfolders: - copyfolder.makefolder(key.replace(dest.getsep(), copyfolder.getsep())) - except (KeyboardInterrupt): + dst_repo.makefolder(src_name_t) + dst_haschanged = True # Need to refresh list + except OfflineImapError as e: + self.ui.error(e, exc_info()[2], + "Creating folder %s on repository %s"% + (src_name_t, dst_repo)) raise - except: - getglobalui().warn("ERROR Attempting to create folder " \ - + key + ":" +traceback.format_exc()) + status_repo.makefolder(src_name_t.replace(dst_repo.getsep(), + status_repo.getsep())) + # Find new folders on dst_repo. + for dst_name_t, dst_folder in dst_hash.iteritems(): + if not src_repo.get_create_folders(): + # Don't create missing folder on readonly repo. + break - # + if dst_folder.sync_this and not dst_name_t in src_folders: + # nametrans sanity check! + # Does nametrans back&forth lead to identical names? + # 1) would src repo filter out the new folder name? In this + # case don't create it on it: + if not self.should_sync_folder(dst_name_t): + self.ui.debug('', "Not creating folder '%s' (repository '%s" + "') as it would be filtered out on that repository."% + (dst_name_t, self)) + continue + # get IMAPFolder and see if the reverse nametrans + # works fine TODO: getfolder() works only because we + # succeed in getting inexisting folders which I would + # like to change. Take care! + folder = self.getfolder(dst_name_t) + # apply reverse nametrans to see if we end up with the same name + newdst_name = folder.getvisiblename().replace( + src_repo.getsep(), dst_repo.getsep()) + if dst_folder.name != newdst_name: + raise OfflineImapError("INFINITE FOLDER CREATION DETECTED! " + "Folder '%s' (repository '%s') would be created as fold" + "er '%s' (repository '%s'). The latter becomes '%s' in " + "return, leading to infinite folder creation cycles.\n " + "SOLUTION: 1) Do set your nametrans rules on both repos" + "itories so they lead to identical names if applied bac" + "k and forth. 2) Use folderfilter settings on a reposit" + "ory to prevent some folders from being created on the " + "other side." % (dst_folder.name, dst_repo, dst_name_t, + src_repo, newdst_name), + OfflineImapError.ERROR.REPO) + # end sanity check, actually create the folder + try: + src_repo.makefolder(dst_name_t) + src_haschanged = True # Need to refresh list + except OfflineImapError as e: + self.ui.error(e, exc_info()[2], "Creating folder %s on " + "repository %s" % (dst_name_t, src_repo)) + raise + status_repo.makefolder(dst_name_t.replace( + src_repo.getsep(), status_repo.getsep())) # Find deleted folders. - # - # We don't delete folders right now. + # TODO: We don't delete folders right now. - #for key in desthash.keys(): - # if not key in srchash: - # dest.deletefolder(key) - - ##### Keepalive + #Forget old list of cached folders so we get new ones if needed + if src_haschanged: + self.forgetfolders() + if dst_haschanged: + dst_repo.forgetfolders() def startkeepalive(self): """The default implementation will do nothing.""" @@ -188,4 +264,9 @@ class BaseRepository(CustomConfig.ConfigHelperMixin): """Stop keep alive, but don't bother waiting for the threads to terminate.""" pass - + + def getlocalroot(self): + """ Local root folder for storing messages. + Will not be set for remote repositories.""" + return None + diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 4793db7..2e23e62 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -15,35 +15,39 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from IMAP import IMAPRepository -from offlineimap import folder, imaputil -from offlineimap.imapserver import IMAPServer +from offlineimap.repository.IMAP import IMAPRepository +from offlineimap import folder, OfflineImapError class GmailRepository(IMAPRepository): """Gmail IMAP repository. - Uses hard-coded host name and port, see: - http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814 + Falls back to hard-coded gmail host name and port, if none were specified: + http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814 """ - - #: Gmail IMAP server hostname + # Gmail IMAP server hostname HOSTNAME = "imap.gmail.com" - - #: Gmail IMAP server port + # Gmail IMAP server port PORT = 993 - + def __init__(self, reposname, account): """Initialize a GmailRepository object.""" - account.getconfig().set('Repository ' + reposname, - 'remotehost', GmailRepository.HOSTNAME) - account.getconfig().set('Repository ' + reposname, - 'remoteport', GmailRepository.PORT) + # Enforce SSL usage account.getconfig().set('Repository ' + reposname, 'ssl', 'yes') IMAPRepository.__init__(self, reposname, account) + def gethost(self): - return GmailRepository.HOSTNAME + """Return the server name to connect to. + + Gmail implementation first checks for the usual IMAP settings + and falls back to imap.gmail.com if not specified.""" + try: + return super(GmailRepository, self).gethost() + except OfflineImapError: + # nothing was configured, cache and return hardcoded one + self._host = GmailRepository.HOSTNAME + return self._host def getport(self): return GmailRepository.PORT @@ -56,22 +60,15 @@ class GmailRepository(IMAPRepository): def getfolder(self, foldername): return self.getfoldertype()(self.imapserver, foldername, - self.nametrans(foldername), - self.accountname, self) + self) def getfoldertype(self): return folder.Gmail.GmailFolder - def getrealdelete(self, foldername): - # XXX: `foldername` is currently ignored - the `realdelete` - # setting is repository-wide - return self.getconfboolean('realdelete', 0) - def gettrashfolder(self, foldername): #: Where deleted mail should be moved return self.getconf('trashfolder','[Gmail]/Trash') - + def getspamfolder(self): #: Gmail also deletes messages upon EXPUNGE in the Spam folder return self.getconf('spamfolder','[Gmail]/Spam') - diff --git a/offlineimap/repository/GmailMaildir.py b/offlineimap/repository/GmailMaildir.py new file mode 100644 index 0000000..71f89ac --- /dev/null +++ b/offlineimap/repository/GmailMaildir.py @@ -0,0 +1,30 @@ +# Maildir repository support +# Copyright (C) 2002-2015 John Goerzen & contributors +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +from offlineimap.repository.Maildir import MaildirRepository +from offlineimap.folder.GmailMaildir import GmailMaildirFolder + +class GmailMaildirRepository(MaildirRepository): + def __init__(self, reposname, account): + """Initialize a MaildirRepository object. Takes a path name + to the directory holding all the Maildir directories.""" + + super(GmailMaildirRepository, self).__init__(reposname, account) + + def getfoldertype(self): + return GmailMaildirFolder diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 23e92af..68c8e33 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -1,6 +1,5 @@ # IMAP repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,36 +15,30 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from Base import BaseRepository -from offlineimap import folder, imaputil, imapserver +from threading import Event +import os +from sys import exc_info +import netrc +import errno + +from offlineimap.repository.Base import BaseRepository +from offlineimap import folder, imaputil, imapserver, OfflineImapError from offlineimap.folder.UIDMaps import MappedIMAPFolder from offlineimap.threadutil import ExitNotifyThread -import re, types, os, netrc, errno -from threading import * +from offlineimap.utils.distro import get_os_sslcertfile, get_os_sslcertfile_searchpath + class IMAPRepository(BaseRepository): def __init__(self, reposname, account): """Initialize an IMAPRepository object.""" BaseRepository.__init__(self, reposname, account) - self.imapserver = imapserver.ConfigedIMAPServer(self) + # self.ui is being set by the BaseRepository + self._host = None + self.imapserver = imapserver.IMAPServer(self) self.folders = None - self.nametrans = lambda foldername: foldername - self.folderfilter = lambda foldername: 1 - self.folderincludes = [] - self.foldersort = cmp - localeval = self.localeval - if self.config.has_option(self.getsection(), 'nametrans'): - self.nametrans = localeval.eval(self.getconf('nametrans'), - {'re': re}) - if self.config.has_option(self.getsection(), 'folderfilter'): - self.folderfilter = localeval.eval(self.getconf('folderfilter'), - {'re': re}) - if self.config.has_option(self.getsection(), 'folderincludes'): - self.folderincludes = localeval.eval(self.getconf('folderincludes'), - {'re': re}) - if self.config.has_option(self.getsection(), 'foldersort'): - self.foldersort = localeval.eval(self.getconf('foldersort'), - {'re': re}) + if self.getconf('sep', None): + self.ui.info("The 'sep' setting is being ignored for IMAP " + "repository '%s' (it's autodetected)"% self) def startkeepalive(self): keepalivetime = self.getkeepalive() @@ -74,26 +67,83 @@ class IMAPRepository(BaseRepository): self.imapserver.close() def getholdconnectionopen(self): + if self.getidlefolders(): + return 1 return self.getconfboolean("holdconnectionopen", 0) def getkeepalive(self): - return self.getconfint("keepalive", 0) + num = self.getconfint("keepalive", 0) + if num == 0 and self.getidlefolders(): + return 29*60 + else: + return num def getsep(self): + """Return the folder separator for the IMAP repository + + This requires that self.imapserver has been initialized with an + acquireconnection() or it will still be `None`""" + assert self.imapserver.delim != None, "'%s' " \ + "repository called getsep() before the folder separator was " \ + "queried from the server"% self return self.imapserver.delim def gethost(self): - host = None - localeval = self.localeval + """Return the configured hostname to connect to + :returns: hostname as string or throws Exception""" + if self._host: # use cached value if possible + return self._host + + # 1) check for remotehosteval setting if self.config.has_option(self.getsection(), 'remotehosteval'): host = self.getconf('remotehosteval') + try: + host = self.localeval.eval(host) + except Exception as e: + raise OfflineImapError("remotehosteval option for repository " + "'%s' failed:\n%s"% (self, e), OfflineImapError.ERROR.REPO), \ + None, exc_info()[2] + if host: + self._host = host + return self._host + # 2) check for plain remotehost setting + host = self.getconf('remotehost', None) if host != None: - return localeval.eval(host) + self._host = host + return self._host + + # no success + raise OfflineImapError("No remote host for repository " + "'%s' specified."% self, OfflineImapError.ERROR.REPO) + + def get_remote_identity(self): + """Remote identity is used for certain SASL mechanisms + (currently -- PLAIN) to inform server about the ID + we want to authorize as instead of our login name.""" + + return self.getconf('remote_identity', default=None) + + def get_auth_mechanisms(self): + supported = ["GSSAPI", "CRAM-MD5", "PLAIN", "LOGIN"] + # Mechanisms are ranged from the strongest to the + # weakest ones. + # TODO: we need DIGEST-MD5, it must come before CRAM-MD5 + # TODO: due to the chosen-plaintext resistance. + default = ["GSSAPI", "CRAM-MD5", "PLAIN", "LOGIN"] + + mechs = self.getconflist('auth_mechanisms', r',\s*', + default) + + for m in mechs: + if m not in supported: + raise OfflineImapError("Repository %s: "% self + \ + "unknown authentication mechanism '%s'"% m, + OfflineImapError.ERROR.REPO) + + self.ui.debug('imap', "Using authentication mechanisms %s" % mechs) + return mechs - host = self.getconf('remotehost') - if host != None: - return host def getuser(self): user = None @@ -104,13 +154,14 @@ class IMAPRepository(BaseRepository): if user != None: return localeval.eval(user) - user = self.getconf('remoteuser') + if self.config.has_option(self.getsection(), 'remoteuser'): + user = self.getconf('remoteuser') if user != None: return user try: netrcentry = netrc.netrc().authenticators(self.gethost()) - except IOError, inst: + except IOError as inst: if inst.errno != errno.ENOENT: raise else: @@ -119,8 +170,8 @@ class IMAPRepository(BaseRepository): try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) - except IOError, inst: - if inst.errno != errno.ENOENT: + except IOError as inst: + if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: @@ -128,38 +179,96 @@ class IMAPRepository(BaseRepository): def getport(self): + port = None + + if self.config.has_option(self.getsection(), 'remoteporteval'): + port = self.getconf('remoteporteval') + if port != None: + return self.localeval.eval(port) + return self.getconfint('remoteport', None) def getssl(self): return self.getconfboolean('ssl', 0) def getsslclientcert(self): - return self.getconf('sslclientcert', None) + xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] + return self.getconf_xform('sslclientcert', xforms, None) def getsslclientkey(self): - return self.getconf('sslclientkey', None) + xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] + return self.getconf_xform('sslclientkey', xforms, None) def getsslcacertfile(self): - """Return the absolute path of the CA certfile to use, if any""" - cacertfile = self.getconf('sslcacertfile', None) + """Determines CA bundle. + + Returns path to the CA bundle. It is either explicitely specified + or requested via "OS-DEFAULT" value (and we will search known + locations for the current OS and distribution). + + If search via "OS-DEFAULT" route yields nothing, we will throw an + exception to make our callers distinguish between not specified + value and non-existent default CA bundle. + + It is also an error to specify non-existent file via configuration: + it will error out later, but, perhaps, with less verbose explanation, + so we will also throw an exception. It is consistent with + the above behaviour, so any explicitely-requested configuration + that doesn't result in an existing file will give an exception. + """ + + xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] + cacertfile = self.getconf_xform('sslcacertfile', xforms, None) + if self.getconf('sslcacertfile', None) == "OS-DEFAULT": + cacertfile = get_os_sslcertfile() + if cacertfile == None: + searchpath = get_os_sslcertfile_searchpath() + if searchpath: + reason = "Default CA bundle was requested, "\ + "but no existing locations available. "\ + "Tried %s." % (", ".join(searchpath)) + else: + reason = "Default CA bundle was requested, "\ + "but OfflineIMAP doesn't know any for your "\ + "current operating system." + raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) if cacertfile is None: return None - cacertfile = os.path.expanduser(cacertfile) - cacertfile = os.path.abspath(cacertfile) if not os.path.isfile(cacertfile): - raise SyntaxWarning("CA certfile for repository '%s' could " - "not be found. No such file: '%s'" \ - % (self.name, cacertfile)) + reason = "CA certfile for repository '%s' couldn't be found. "\ + "No such file: '%s'" % (self.name, cacertfile) + raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) return cacertfile + def getsslversion(self): + return self.getconf('ssl_version', None) + + def get_ssl_fingerprint(self): + """Return array of possible certificate fingerprints. + + Configuration item cert_fingerprint can contain multiple + comma-separated fingerprints in hex form.""" + + value = self.getconf('cert_fingerprint', "") + return [f.strip().lower() for f in value.split(',') if f] + def getpreauthtunnel(self): return self.getconf('preauthtunnel', None) + def gettransporttunnel(self): + return self.getconf('transporttunnel', None) + def getreference(self): - return self.getconf('reference', '""') + return self.getconf('reference', '') + + def getidlefolders(self): + localeval = self.localeval + return localeval.eval(self.getconf('idlefolders', '[]')) def getmaxconnections(self): - return self.getconfint('maxconnections', 1) + num1 = len(self.getidlefolders()) + num2 = self.getconfint('maxconnections', 1) + return max(num1, num2) def getexpunge(self): return self.getconfboolean('expunge', 1) @@ -176,8 +285,8 @@ class IMAPRepository(BaseRepository): 5. read password from /etc/netrc On success we return the password. - If all strategies fail we return None. - """ + If all strategies fail we return None.""" + # 1. evaluate Repository 'remotepasseval' passwd = self.getconf('remotepasseval', None) if passwd != None: @@ -196,33 +305,32 @@ class IMAPRepository(BaseRepository): # 4. read password from ~/.netrc try: netrcentry = netrc.netrc().authenticators(self.gethost()) - except IOError, inst: + except IOError as inst: if inst.errno != errno.ENOENT: raise else: if netrcentry: - user = self.getconf('remoteuser') + user = self.getuser() if user == None or user == netrcentry[0]: return netrcentry[2] # 5. read password from /etc/netrc try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) - except IOError, inst: - if inst.errno != errno.ENOENT: + except IOError as inst: + if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: - user = self.getconf('remoteuser') + user = self.getuser() if user == None or user == netrcentry[0]: return netrcentry[2] # no strategy yielded a password! return None - def getfolder(self, foldername): - return self.getfoldertype()(self.imapserver, foldername, - self.nametrans(foldername), - self.accountname, self) + """Return instance of OfflineIMAP representative folder.""" + + return self.getfoldertype()(self.imapserver, foldername, self) def getfoldertype(self): return folder.IMAP.IMAPFolder @@ -235,68 +343,98 @@ class IMAPRepository(BaseRepository): self.folders = None def getfolders(self): + """Return a list of instances of OfflineIMAP representative folder.""" + if self.folders != None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list - if self.config.has_option(self.getsection(), 'subscribedonly'): - if self.getconf('subscribedonly') == "yes": + if self.getconfboolean('subscribedonly', False): listfunction = imapobj.lsub try: listresult = listfunction(directory = self.imapserver.reference)[1] finally: self.imapserver.releaseconnection(imapobj) - for string in listresult: - if string == None or \ - (type(string) == types.StringType and string == ''): + for s in listresult: + if s == None or \ + (isinstance(s, basestring) and s == ''): # Bug in imaplib: empty strings in results from - # literals. + # literals. TODO: still relevant? continue - flags, delim, name = imaputil.imapsplit(string) + flags, delim, name = imaputil.imapsplit(s) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue foldername = imaputil.dequote(name) - if not self.folderfilter(foldername): - continue retval.append(self.getfoldertype()(self.imapserver, foldername, - self.nametrans(foldername), - self.accountname, self)) + self)) + # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: - imapobj.select(foldername, readonly = 1) - except ValueError: + imapobj.select(foldername, readonly = True) + except OfflineImapError as e: + # couldn't select this folderinclude, so ignore folder. + if e.severity > OfflineImapError.ERROR.FOLDER: + raise + self.ui.error(e, exc_info()[2], + 'Invalid folderinclude:') continue - retval.append(self.getfoldertype()(self.imapserver, - foldername, - self.nametrans(foldername), - self.accountname, self)) + retval.append(self.getfoldertype()( + self.imapserver, foldername, self)) finally: self.imapserver.releaseconnection(imapobj) - - retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename())) + + if self.foldersort is None: + # default sorting by case insensitive transposed name + retval.sort(key=lambda x: str.lower(x.getvisiblename())) + else: + # do foldersort in a python3-compatible way + # http://bytes.com/topic/python/answers/844614-python-3-sorting-comparison-function + def cmp2key(mycmp): + """Converts a cmp= function into a key= function + We need to keep cmp functions for backward compatibility""" + class K: + def __init__(self, obj, *args): + self.obj = obj + def __cmp__(self, other): + return mycmp(self.obj.getvisiblename(), other.obj.getvisiblename()) + return K + retval.sort(key=cmp2key(self.foldersort)) + self.folders = retval - return retval + return self.folders def makefolder(self, foldername): - #if self.getreference() != '""': - # newname = self.getreference() + self.getsep() + foldername - #else: - # newname = foldername - newname = foldername + """Create a folder on the IMAP server + + This will not update the list cached in :meth:`getfolders`. You + will need to invoke :meth:`forgetfolders` to force new caching + when you are done creating folders yourself. + + :param foldername: Full path of the folder to be created.""" + + if self.getreference(): + foldername = self.getreference() + self.getsep() + foldername + if not foldername: # Create top level folder as folder separator + foldername = self.getsep() + self.ui.makefolder(self, foldername) + if self.account.dryrun: + return imapobj = self.imapserver.acquireconnection() try: - result = imapobj.create(newname) + result = imapobj.create(foldername) if result[0] != 'OK': - raise RuntimeError, "Repository %s could not create folder %s: %s" % (self.getname(), foldername, str(result)) + raise OfflineImapError("Folder '%s'[%s] could not be created. " + "Server responded: %s"% (foldername, self, str(result)), + OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) - + class MappedIMAPRepository(IMAPRepository): def getfoldertype(self): return MappedIMAPFolder diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 92e392d..fc34a55 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -1,6 +1,5 @@ # Local status cache repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,54 +15,111 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from Base import BaseRepository -from offlineimap import folder -import offlineimap.folder.LocalStatus -import os, re +import os + +from offlineimap.folder.LocalStatus import LocalStatusFolder +from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder +from offlineimap.repository.Base import BaseRepository class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): BaseRepository.__init__(self, reposname, account) - self.directory = os.path.join(account.getaccountmeta(), 'LocalStatus') - if not os.path.exists(self.directory): - os.mkdir(self.directory, 0700) - self.folders = None + + # class and root for all backends + self.backends = {} + self.backends['sqlite'] = { + 'class': LocalStatusSQLiteFolder, + 'root': os.path.join(account.getaccountmeta(), 'LocalStatus-sqlite') + } + + self.backends['plain'] = { + 'class': LocalStatusFolder, + 'root': os.path.join(account.getaccountmeta(), 'LocalStatus') + } + + # Set class and root for the configured backend + self.setup_backend(self.account.getconf('status_backend', 'plain')) + + if not os.path.exists(self.root): + os.mkdir(self.root, 0o700) + + # self._folders is a dict of name:LocalStatusFolders() + self._folders = {} + + def setup_backend(self, backend): + if backend in self.backends.keys(): + self._backend = backend + self.root = self.backends[backend]['root'] + self.LocalStatusFolderClass = self.backends[backend]['class'] + + else: + raise SyntaxWarning("Unknown status_backend '%s' for account '%s'"% + (backend, self.account.name)) + + def import_other_backend(self, folder): + for bk, dic in self.backends.items(): + # skip folder's own type + if dic['class'] == type(folder): + continue + + repobk = LocalStatusRepository(self.name, self.account) + repobk.setup_backend(bk) # fake the backend + folderbk = dic['class'](folder.name, repobk) + + # if backend contains data, import it to folder. + if not folderbk.isnewfolder(): + self.ui._msg('Migrating LocalStatus cache from %s to %s " \ + "status folder for %s:%s'% + (bk, self._backend, self.name, folder.name)) + + folderbk.cachemessagelist() + folder.messagelist = folderbk.messagelist + folder.saveall() + break def getsep(self): return '.' - def getfolderfilename(self, foldername): - foldername = re.sub('/\.$', '/dot', foldername) - foldername = re.sub('^\.$', 'dot', foldername) - return os.path.join(self.directory, foldername) - def makefolder(self, foldername): - # "touch" the file, truncating it. - filename = self.getfolderfilename(foldername) - file = open(filename + ".tmp", "wt") - file.write(offlineimap.folder.LocalStatus.magicline + '\n') - file.flush() - os.fsync(file.fileno()) - file.close() - os.rename(filename + ".tmp", filename) - - # Invalidate the cache. - self.folders = None + """Create a LocalStatus Folder.""" - def getfolders(self): - retval = [] - for folder in os.listdir(self.directory): - retval.append(folder.LocalStatus.LocalStatusFolder(self.directory, - folder, self, self.accountname, - self.config)) - return retval + if self.account.dryrun: + return # bail out in dry-run mode + + # Create an empty StatusFolder + folder = self.LocalStatusFolderClass(foldername, self) + folder.save() + + # Invalidate the cache. + self.forgetfolders() def getfolder(self, foldername): - return folder.LocalStatus.LocalStatusFolder(self.directory, foldername, - self, self.accountname, - self.config) + """Return the Folder() object for a foldername.""" + if foldername in self._folders: + return self._folders[foldername] - + folder = self.LocalStatusFolderClass(foldername, self) - + # If folder is empty, try to import data from an other backend. + if folder.isnewfolder(): + self.import_other_backend(folder) + + self._folders[foldername] = folder + return folder + + def getfolders(self): + """Returns a list of all cached folders. + + Does nothing for this backend. We mangle the folder file names + (see getfolderfilename) so we can not derive folder names from + the file names that we have available. TODO: need to store a + list of folder names somehow?""" + + pass + + def forgetfolders(self): + """Forgets the cached list of folders, if any. Useful to run + after a sync run.""" + + self._folders = {} diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 648440a..0262ba2 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -1,6 +1,5 @@ # Maildir repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,10 +15,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from Base import BaseRepository -from offlineimap import folder, imaputil +from offlineimap import folder from offlineimap.ui import getglobalui -from mailbox import Maildir +from offlineimap.error import OfflineImapError +from offlineimap.repository.Base import BaseRepository import os from stat import * @@ -27,37 +26,45 @@ class MaildirRepository(BaseRepository): def __init__(self, reposname, account): """Initialize a MaildirRepository object. Takes a path name to the directory holding all the Maildir directories.""" + BaseRepository.__init__(self, reposname, account) self.root = self.getlocalroot() self.folders = None self.ui = getglobalui() - self.debug("MaildirRepository initialized, sep is " + repr(self.getsep())) - self.folder_atimes = [] + self.debug("MaildirRepository initialized, sep is %s"% repr(self.getsep())) + self.folder_atimes = [] # Create the top-level folder if it doesn't exist if not os.path.isdir(self.root): - os.mkdir(self.root, 0700) + os.mkdir(self.root, 0o700) def _append_folder_atimes(self, foldername): - p = os.path.join(self.root, foldername) - new = os.path.join(p, 'new') - cur = os.path.join(p, 'cur') - f = p, os.stat(new)[ST_ATIME], os.stat(cur)[ST_ATIME] - self.folder_atimes.append(f) + """Store the atimes of a folder's new|cur in self.folder_atimes""" - def restore_folder_atimes(self): - if not self.folder_atimes: - return + p = os.path.join(self.root, foldername) + new = os.path.join(p, 'new') + cur = os.path.join(p, 'cur') + atimes = (p, os.path.getatime(new), os.path.getatime(cur)) + self.folder_atimes.append(atimes) - for f in self.folder_atimes: - t = f[1], os.stat(os.path.join(f[0], 'new'))[ST_MTIME] - os.utime(os.path.join(f[0], 'new'), t) - t = f[2], os.stat(os.path.join(f[0], 'cur'))[ST_MTIME] - os.utime(os.path.join(f[0], 'cur'), t) + def restore_atime(self): + """Sets folders' atime back to their values after a sync + + Controlled by the 'restoreatime' config parameter.""" + + if not self.getconfboolean('restoreatime', False): + return # not configured to restore + + for (dirpath, new_atime, cur_atime) in self.folder_atimes: + new_dir = os.path.join(dirpath, 'new') + cur_dir = os.path.join(dirpath, 'cur') + os.utime(new_dir, (new_atime, os.path.getmtime(new_dir))) + os.utime(cur_dir, (cur_atime, os.path.getmtime(cur_dir))) def getlocalroot(self): - return os.path.expanduser(self.getconf('localfolders')) + xforms = [os.path.expanduser] + return self.getconf_xform('localfolders', xforms) def debug(self, msg): self.ui.debug('maildir', msg) @@ -66,111 +73,137 @@ class MaildirRepository(BaseRepository): return self.getconf('sep', '.').strip() def makefolder(self, foldername): - self.debug("makefolder called with arg " + repr(foldername)) - # Do the chdir thing so the call to makedirs does not make the - # self.root directory (we'd prefer to raise an error in that case), - # but will make the (relative) paths underneath it. Need to use - # makedirs to support a / separator. - self.debug("Is dir? " + repr(os.path.isdir(foldername))) - if self.getsep() == '/': - for invalid in ['new', 'cur', 'tmp', 'offlineimap.uidvalidity']: - for component in foldername.split('/'): - assert component != invalid, "When using nested folders (/ as a separator in the account config), your folder names may not contain 'new', 'cur', 'tmp', or 'offlineimap.uidvalidity'." + """Create new Maildir folder if necessary - assert foldername.find('./') == -1, "Folder names may not contain ../" + This will not update the list cached in getfolders(). You will + need to invoke :meth:`forgetfolders` to force new caching when + you are done creating folders yourself. + + :param foldername: A relative mailbox name. The maildir will be + created in self.root+'/'+foldername. All intermediate folder + levels will be created if they do not exist yet. 'cur', + 'tmp', and 'new' subfolders will be created in the maildir. + """ + + self.ui.makefolder(self, foldername) + if self.account.dryrun: + return + full_path = os.path.abspath(os.path.join(self.root, foldername)) + + # sanity tests + if self.getsep() == '/': + for component in foldername.split('/'): + assert not component in ['new', 'cur', 'tmp'],\ + "When using nested folders (/ as a Maildir separator), "\ + "folder names may not contain 'new', 'cur', 'tmp'." + assert foldername.find('../') == -1, "Folder names may not contain ../" assert not foldername.startswith('/'), "Folder names may not begin with /" - oldcwd = os.getcwd() - os.chdir(self.root) - - # If we're using hierarchical folders, it's possible that sub-folders - # may be created before higher-up ones. If this is the case, - # makedirs will fail because the higher-up dir already exists. - # So, check to see if this is indeed the case. - - if (self.getsep() == '/' or self.getconfboolean('existsok', 0) or foldername == '.') \ - and os.path.isdir(foldername): - self.debug("makefolder: %s already is a directory" % foldername) - # Already exists. Sanity-check that it's not a Maildir. - for subdir in ['cur', 'new', 'tmp']: - assert not os.path.isdir(os.path.join(foldername, subdir)), \ - "Tried to create folder %s but it already had dir %s" %\ - (foldername, subdir) - else: - self.debug("makefolder: calling makedirs %s" % foldername) - os.makedirs(foldername, 0700) - self.debug("makefolder: creating cur, new, tmp") + # If we're using hierarchical folders, it's possible that + # sub-folders may be created before higher-up ones. + self.debug("makefolder: calling makedirs '%s'"% full_path) + try: + os.makedirs(full_path, 0o700) + except OSError as e: + if e.errno == 17 and os.path.isdir(full_path): + self.debug("makefolder: '%s' already a directory"% foldername) + else: + raise for subdir in ['cur', 'new', 'tmp']: - os.mkdir(os.path.join(foldername, subdir), 0700) - # Invalidate the cache - self.folders = None - os.chdir(oldcwd) + try: + os.mkdir(os.path.join(full_path, subdir), 0o700) + except OSError as e: + if e.errno == 17 and os.path.isdir(full_path): + self.debug("makefolder: '%s' already has subdir %s"% + (foldername, subdir)) + else: + raise def deletefolder(self, foldername): - self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s" % foldername) + self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s"% foldername) def getfolder(self, foldername): - if self.config.has_option('Repository ' + self.name, 'restoreatime') and self.config.getboolean('Repository ' + self.name, 'restoreatime'): - self._append_folder_atimes(foldername) - return folder.Maildir.MaildirFolder(self.root, foldername, - self.getsep(), self, - self.accountname, self.config) - - def _getfolders_scandir(self, root, extension = None): - self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s" \ - % (root, extension)) - # extension willl only be non-None when called recursively when - # getsep() returns '/'. + """Return a Folder instance of this Maildir + + If necessary, scan and cache all foldernames to make sure that + we only return existing folders and that 2 calls with the same + name will return the same object.""" + + # getfolders() will scan and cache the values *if* necessary + folders = self.getfolders() + for f in folders: + if foldername == f.name: + return f + raise OfflineImapError("getfolder() asked for a nonexisting " + "folder '%s'."% foldername, + OfflineImapError.ERROR.FOLDER) + + def _getfolders_scandir(self, root, extension=None): + """Recursively scan folder 'root'; return a list of MailDirFolder + + :param root: (absolute) path to Maildir root + :param extension: (relative) subfolder to examine within root""" + + self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s"% + (root, extension)) retval = [] # Configure the full path to this repository -- "toppath" - - if extension == None: - toppath = root - else: + if extension: toppath = os.path.join(root, extension) + else: + toppath = root + self.debug(" toppath = %s"% toppath) - self.debug(" toppath = %s" % toppath) - - # Iterate over directories in top. - for dirname in os.listdir(toppath) + ['.']: - self.debug(" *** top of loop") - self.debug(" dirname = %s" % dirname) - if dirname in ['cur', 'new', 'tmp', 'offlineimap.uidvalidity']: - self.debug(" skipping this dir (Maildir special)") + # Iterate over directories in top & top itself. + for dirname in os.listdir(toppath) + ['']: + self.debug(" dirname = %s"% dirname) + if dirname == '' and extension is not None: + self.debug(' skip this entry (already scanned)') + continue + if dirname in ['cur', 'new', 'tmp']: + self.debug(" skip this entry (Maildir special)") # Bypass special files. continue fullname = os.path.join(toppath, dirname) - self.debug(" fullname = %s" % fullname) if not os.path.isdir(fullname): - self.debug(" skipping this entry (not a directory)") + self.debug(" skip this entry (not a directory)") # Not a directory -- not a folder. continue - foldername = dirname - if extension != None: + # extension can be None. + if extension: foldername = os.path.join(extension, dirname) + else: + foldername = dirname + if (os.path.isdir(os.path.join(fullname, 'cur')) and os.path.isdir(os.path.join(fullname, 'new')) and os.path.isdir(os.path.join(fullname, 'tmp'))): # This directory has maildir stuff -- process - self.debug(" This is a maildir folder.") + self.debug(" This is maildir folder '%s'."% foldername) + if self.getconfboolean('restoreatime', False): + self._append_folder_atimes(foldername) + fd = self.getfoldertype()(self.root, foldername, + self.getsep(), self) + retval.append(fd) - self.debug(" foldername = %s" % foldername) - - if self.config.has_option('Repository ' + self.name, 'restoreatime') and self.config.getboolean('Repository ' + self.name, 'restoreatime'): - self._append_folder_atimes(foldername) - retval.append(folder.Maildir.MaildirFolder(self.root, foldername, - self.getsep(), self, self.accountname, - self.config)) - if self.getsep() == '/' and dirname != '.': - # Check sub-directories for folders. + if self.getsep() == '/' and dirname != '': + # Recursively check sub-directories for folders too. retval.extend(self._getfolders_scandir(root, foldername)) - self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ + self.debug("_GETFOLDERS_SCANDIR RETURNING %s"% \ repr([x.getname() for x in retval])) return retval - + def getfolders(self): if self.folders == None: self.folders = self._getfolders_scandir(self.root) return self.folders - + + def getfoldertype(self): + return folder.Maildir.MaildirFolder + + def forgetfolders(self): + """Forgets the cached list of folders, if any. Useful to run + after a sync run.""" + + self.folders = None diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py index be5c29e..0fbbc13 100644 --- a/offlineimap/repository/__init__.py +++ b/offlineimap/repository/__init__.py @@ -1 +1,92 @@ -__all__ = ['Gmail', 'IMAP', 'Base', 'Maildir', 'LocalStatus'] +# Copyright (C) 2002-2007 John Goerzen +# 2010 Sebastian Spaeth and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +from sys import exc_info + +try: + from configparser import NoSectionError +except ImportError: #python2 + from ConfigParser import NoSectionError + +from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository +from offlineimap.repository.Gmail import GmailRepository +from offlineimap.repository.Maildir import MaildirRepository +from offlineimap.repository.GmailMaildir import GmailMaildirRepository +from offlineimap.repository.LocalStatus import LocalStatusRepository +from offlineimap.error import OfflineImapError + + +class Repository(object): + """Abstract class that returns the correct Repository type + instance based on 'account' and 'reqtype', e.g. a + class:`ImapRepository` instance.""" + + def __new__(cls, account, reqtype): + """ + :param account: :class:`Account` + :param regtype: 'remote', 'local', or 'status'""" + + if reqtype == 'remote': + name = account.getconf('remoterepository') + # We don't support Maildirs on the remote side. + typemap = {'IMAP': IMAPRepository, + 'Gmail': GmailRepository} + + elif reqtype == 'local': + name = account.getconf('localrepository') + typemap = {'IMAP': MappedIMAPRepository, + 'Maildir': MaildirRepository, + 'GmailMaildir': GmailMaildirRepository} + + elif reqtype == 'status': + # create and return a LocalStatusRepository. + name = account.getconf('localrepository') + return LocalStatusRepository(name, account) + + else: + errstr = "Repository type %s not supported" % reqtype + raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO) + + # Get repository type. + config = account.getconfig() + try: + repostype = config.get('Repository ' + name, 'type').strip() + except NoSectionError as e: + errstr = ("Could not find section '%s' in configuration. Required " + "for account '%s'." % ('Repository %s' % name, account)) + raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \ + None, exc_info()[2] + + try: + repo = typemap[repostype] + except KeyError: + errstr = "'%s' repository not supported for '%s' repositories."% \ + (repostype, reqtype) + raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \ + None, exc_info()[2] + + return repo(name, account) + + def __init__(self, account, reqtype): + """Load the correct Repository type and return that. The + __init__ of the corresponding Repository class will be + executed instead of this stub + + :param account: :class:`Account` + :param regtype: 'remote', 'local', or 'status' + """ + pass diff --git a/offlineimap/syncmaster.py b/offlineimap/syncmaster.py index 5a0a2fb..5fd0dee 100644 --- a/offlineimap/syncmaster.py +++ b/offlineimap/syncmaster.py @@ -16,27 +16,24 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap.threadutil import threadlist, InstanceLimitedThread, ExitNotifyThread -from offlineimap.accounts import SyncableAccount, SigListener +from offlineimap.threadutil import threadlist, InstanceLimitedThread +from offlineimap.accounts import SyncableAccount from threading import currentThread -def syncaccount(threads, config, accountname, siglisteners): +def syncaccount(threads, config, accountname): account = SyncableAccount(config, accountname) - siglistener = SigListener() thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT', target = account.syncrunner, - name = "Account sync %s" % accountname, - kwargs = {'siglistener': siglistener} ) - # the Sync Runner thread is the only one that will mutate siglisteners - siglisteners.append(siglistener) - thread.setDaemon(1) + name = "Account sync %s" % accountname) + thread.setDaemon(True) thread.start() threads.add(thread) -def syncitall(accounts, config, siglisteners): - currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE') +def syncitall(accounts, config): + # Special exit message for SyncRunner thread, so main thread can exit + currentThread().exit_message = 'SYNCRUNNER_EXITED_NORMALLY' threads = threadlist() for accountname in accounts: - syncaccount(threads, config, accountname, siglisteners) + syncaccount(threads, config, accountname) # Wait for the threads to finish. threads.reset() diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 09c742e..f69f8a6 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -1,6 +1,5 @@ -# Copyright (C) 2002, 2003 John Goerzen +# Copyright (C) 2002-2012 John Goerzen & contributors # Thread support module -# # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,40 +15,34 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import * -from StringIO import StringIO -from Queue import Queue, Empty -import sys, traceback, thread, time +from threading import Lock, Thread, BoundedSemaphore, currentThread +try: + from Queue import Queue, Empty +except ImportError: # python3 + from queue import Queue, Empty +import traceback +import os.path +import sys from offlineimap.ui import getglobalui -profiledir = None - -def setprofiledir(newdir): - global profiledir - profiledir = newdir - ###################################################################### # General utilities ###################################################################### def semaphorereset(semaphore, originalstate): - """Wait until the semaphore gets back to its original state -- all acquired - resources released.""" + """Block until `semaphore` gets back to its original state, ie all acquired + resources have been released.""" + for i in range(originalstate): semaphore.acquire() # Now release these. for i in range(originalstate): semaphore.release() - -def semaphorewait(semaphore): - semaphore.acquire() - semaphore.release() - -def threadsreset(threadlist): - for thr in threadlist: - thr.join() class threadlist: + """Store the list of all threads in the software so it can be used to find out + what's running and what's not.""" + def __init__(self): self.lock = Lock() self.list = [] @@ -83,78 +76,90 @@ class threadlist: if not thread: return thread.join() - + ###################################################################### # Exit-notify threads ###################################################################### exitthreads = Queue(100) -inited = 0 - -def initexitnotify(): - """Initialize the exit notify system. This MUST be called from the - SAME THREAD that will call monitorloop BEFORE it calls monitorloop. - This SHOULD be called before the main thread starts any other - ExitNotifyThreads, or else it may miss the ability to catch the exit - status from them!""" - pass def exitnotifymonitorloop(callback): """An infinite "monitoring" loop watching for finished ExitNotifyThread's. - :param callback: the function to call when a thread terminated. That - function is called with a single argument -- the - ExitNotifyThread that has terminated. The monitor will + This one is supposed to run in the main thread. + :param callback: the function to call when a thread terminated. That + function is called with a single argument -- the + ExitNotifyThread that has terminated. The monitor will not continue to monitor for other threads until 'callback' returns, so if it intends to perform long calculations, it should start a new thread itself -- but - NOT an ExitNotifyThread, or else an infinite loop + NOT an ExitNotifyThread, or else an infinite loop may result. - Furthermore, the monitor will hold the lock all the + Furthermore, the monitor will hold the lock all the while the other thread is waiting. :type callback: a callable function """ + global exitthreads - while 1: + do_loop = True + while do_loop: # Loop forever and call 'callback' for each thread that exited try: # we need a timeout in the get() call, so that ctrl-c can throw # a SIGINT (http://bugs.python.org/issue1360). A timeout with empty # Queue will raise `Empty`. thrd = exitthreads.get(True, 60) - callback(thrd) + # request to abort when callback returns true + do_loop = (callback(thrd) != True) except Empty: pass def threadexited(thread): - """Called when a thread exits.""" + """Called when a thread exits. + + Main thread is aborted when this returns True.""" + ui = getglobalui() - if thread.getExitCause() == 'EXCEPTION': - if isinstance(thread.getExitException(), SystemExit): + if thread.exit_exception: + if isinstance(thread.exit_exception, SystemExit): # Bring a SystemExit into the main thread. # Do not send it back to UI layer right now. # Maybe later send it to ui.terminate? raise SystemExit ui.threadException(thread) # Expected to terminate sys.exit(100) # Just in case... - os._exit(100) - elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE': - ui.terminate() - # Just in case... - sys.exit(100) - os._exit(100) + elif thread.exit_message == 'SYNCRUNNER_EXITED_NORMALLY': + return True else: ui.threadExited(thread) + return False class ExitNotifyThread(Thread): - """This class is designed to alert a "monitor" to the fact that a thread has - exited and to provide for the ability for it to find out why.""" + """This class is designed to alert a "monitor" to the fact that a + thread has exited and to provide for the ability for it to find out + why. All instances are made daemon threads (setDaemon(True), so we + bail out when the mainloop dies. + + The thread can set instance variables self.exit_message for a human + readable reason of the thread exit.""" + + profiledir = None + """Class variable that is set to the profile directory if required.""" + + def __init__(self, *args, **kwargs): + super(ExitNotifyThread, self).__init__(*args, **kwargs) + # These are all child threads that are supposed to go away when + # the main thread is killed. + self.setDaemon(True) + self.exit_message = None + self._exit_exc = None + self._exit_stacktrace = None + def run(self): - global exitthreads, profiledir - self.threadid = thread.get_ident() + global exitthreads try: - if not profiledir: # normal case + if not ExitNotifyThread.profiledir: # normal case Thread.run(self) else: try: @@ -166,54 +171,43 @@ class ExitNotifyThread(Thread): prof = prof.runctx("Thread.run(self)", globals(), locals()) except SystemExit: pass - prof.dump_stats( \ - profiledir + "/" + str(self.threadid) + "_" + \ - self.getName() + ".prof") - except: - self.setExitCause('EXCEPTION') - if sys: - self.setExitException(sys.exc_info()[1]) - sbuf = StringIO() - traceback.print_exc(file = sbuf) - self.setExitStackTrace(sbuf.getvalue()) - else: - self.setExitCause('NORMAL') - if not hasattr(self, 'exitmessage'): - self.setExitMessage(None) + prof.dump_stats(os.path.join(ExitNotifyThread.profiledir, + "%s_%s.prof"% (self.ident, self.getName()))) + except Exception as e: + # Thread exited with Exception, store it + tb = traceback.format_exc() + self.set_exit_exception(e, tb) if exitthreads: exitthreads.put(self, True) - def setExitCause(self, cause): - self.exitcause = cause - def getExitCause(self): + def set_exit_exception(self, exc, st=None): + """Sets Exception and stacktrace of a thread, so that other + threads can query its exit status""" + + self._exit_exc = exc + self._exit_stacktrace = st + + @property + def exit_exception(self): """Returns the cause of the exit, one of: - 'EXCEPTION' -- the thread aborted because of an exception - 'NORMAL' -- normal termination.""" - return self.exitcause - def setExitException(self, exc): - self.exitexception = exc - def getExitException(self): - """If getExitCause() is 'EXCEPTION', holds the value from - sys.exc_info()[1] for this exception.""" - return self.exitexception - def setExitStackTrace(self, st): - self.exitstacktrace = st - def getExitStackTrace(self): - """If getExitCause() is 'EXCEPTION', returns a string representing - the stack trace for this exception.""" - return self.exitstacktrace - def setExitMessage(self, msg): - """Sets the exit message to be fetched by a subsequent call to - getExitMessage. This message may be any object or type except - None.""" - self.exitmessage = msg - def getExitMessage(self): - """For any exit cause, returns the message previously set by - a call to setExitMessage(), or None if there was no such message - set.""" - return self.exitmessage - + Exception() -- the thread aborted with this exception + None -- normal termination.""" + + return self._exit_exc + + @property + def exit_stacktrace(self): + """Returns a string representing the stack trace if set""" + + return self._exit_stacktrace + + @classmethod + def set_profiledir(cls, directory): + """If set, will output profile information to 'directory'""" + + cls.profiledir = directory + ###################################################################### # Instance-limited threads @@ -225,21 +219,21 @@ instancelimitedlock = Lock() def initInstanceLimit(instancename, instancemax): """Initialize the instance-limited thread implementation to permit up to intancemax threads with the given instancename.""" + instancelimitedlock.acquire() - if not instancelimitedsems.has_key(instancename): + if not instancename in instancelimitedsems: instancelimitedsems[instancename] = BoundedSemaphore(instancemax) instancelimitedlock.release() class InstanceLimitedThread(ExitNotifyThread): def __init__(self, instancename, *args, **kwargs): self.instancename = instancename - - apply(ExitNotifyThread.__init__, (self,) + args, kwargs) + super(InstanceLimitedThread, self).__init__(*args, **kwargs) def start(self): instancelimitedsems[self.instancename].acquire() ExitNotifyThread.start(self) - + def run(self): try: ExitNotifyThread.run(self) diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py deleted file mode 100644 index 08303a4..0000000 --- a/offlineimap/ui/Blinkenlights.py +++ /dev/null @@ -1,147 +0,0 @@ -# Blinkenlights base classes -# Copyright (C) 2003 John Goerzen -# -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -from threading import RLock, currentThread -from offlineimap.ui.UIBase import UIBase -import thread - -class BlinkenBase: - """This is a mix-in class that should be mixed in with either UIBase - or another appropriate base class. The Tk interface, for instance, - will probably mix it in with VerboseUI.""" - - def acct(s, accountname): - s.gettf().setcolor('purple') - s.__class__.__bases__[-1].acct(s, accountname) - - def connecting(s, hostname, port): - s.gettf().setcolor('gray') - s.__class__.__bases__[-1].connecting(s, hostname, port) - - def syncfolders(s, srcrepos, destrepos): - s.gettf().setcolor('blue') - s.__class__.__bases__[-1].syncfolders(s, srcrepos, destrepos) - - def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): - s.gettf().setcolor('cyan') - s.__class__.__bases__[-1].syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder) - - def skippingfolder(s, folder): - s.gettf().setcolor('cyan') - s.__class__.__bases__[-1].skippingfolder(s, folder) - - def loadmessagelist(s, repos, folder): - s.gettf().setcolor('green') - s._msg("Scanning folder [%s/%s]" % (s.getnicename(repos), - folder.getvisiblename())) - - def syncingmessages(s, sr, sf, dr, df): - s.gettf().setcolor('blue') - s.__class__.__bases__[-1].syncingmessages(s, sr, sf, dr, df) - - def copyingmessage(s, uid, src, destlist): - s.gettf().setcolor('orange') - s.__class__.__bases__[-1].copyingmessage(s, uid, src, destlist) - - def deletingmessages(s, uidlist, destlist): - s.gettf().setcolor('red') - s.__class__.__bases__[-1].deletingmessages(s, uidlist, destlist) - - def deletingmessage(s, uid, destlist): - s.gettf().setcolor('red') - s.__class__.__bases__[-1].deletingmessage(s, uid, destlist) - - def addingflags(s, uidlist, flags, destlist): - s.gettf().setcolor('yellow') - s.__class__.__bases__[-1].addingflags(s, uidlist, flags, destlist) - - def deletingflags(s, uidlist, flags, destlist): - s.gettf().setcolor('pink') - s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, destlist) - - def warn(s, msg, minor = 0): - if minor: - s.gettf().setcolor('pink') - else: - s.gettf().setcolor('red') - s.__class__.__bases__[-1].warn(s, msg, minor) - - def init_banner(s): - s.availablethreadframes = {} - s.threadframes = {} - #tflock protects the s.threadframes manipulation to only happen from 1 thread - s.tflock = RLock() - - def threadExited(s, thread): - threadid = thread.threadid - accountname = s.getthreadaccount(thread) - s.tflock.acquire() - try: - if threadid in s.threadframes[accountname]: - tf = s.threadframes[accountname][threadid] - del s.threadframes[accountname][threadid] - s.availablethreadframes[accountname].append(tf) - tf.setthread(None) - finally: - s.tflock.release() - - UIBase.threadExited(s, thread) - - def gettf(s): - threadid = thread.get_ident() - accountname = s.getthreadaccount() - - s.tflock.acquire() - - try: - if not accountname in s.threadframes: - s.threadframes[accountname] = {} - - if threadid in s.threadframes[accountname]: - return s.threadframes[accountname][threadid] - - if not accountname in s.availablethreadframes: - s.availablethreadframes[accountname] = [] - - if len(s.availablethreadframes[accountname]): - tf = s.availablethreadframes[accountname].pop(0) - tf.setthread(currentThread()) - else: - tf = s.getaccountframe().getnewthreadframe() - s.threadframes[accountname][threadid] = tf - return tf - finally: - s.tflock.release() - - def callhook(s, msg): - s.gettf().setcolor('white') - s.__class__.__bases__[-1].callhook(s, msg) - - def sleep(s, sleepsecs, siglistener): - s.gettf().setcolor('red') - s.getaccountframe().startsleep(sleepsecs) - return UIBase.sleep(s, sleepsecs, siglistener) - - def sleeping(s, sleepsecs, remainingsecs): - if remainingsecs and s.gettf().getcolor() == 'black': - s.gettf().setcolor('red') - else: - s.gettf().setcolor('black') - return s.getaccountframe().sleeping(sleepsecs, remainingsecs) - - diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index b5a10da..ddc05ea 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -1,6 +1,5 @@ # Curses-based interfaces -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,57 +15,83 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import RLock, Lock, Event +from threading import RLock, currentThread, Lock, Event +from collections import deque import time import sys import os -import signal -import curses, curses.panel, curses.textpad, curses.wrapper -from Blinkenlights import BlinkenBase -from UIBase import UIBase +import curses +import logging + +from offlineimap.ui.UIBase import UIBase +from offlineimap.threadutil import ExitNotifyThread import offlineimap -acctkeys = '1234567890abcdefghijklmnoprstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-=;/.,' - class CursesUtil: - def __init__(self): - self.pairlock = Lock() - # iolock protects access to the + def __init__(self, *args, **kwargs): + # iolock protects access to the self.iolock = RLock() - self.start() + self.tframe_lock = RLock() + # tframe_lock protects the self.threadframes manipulation to + # only happen from 1 thread. + self.colormap = {} + """dict, translating color string to curses color pair number""" - def initpairs(self): - self.pairlock.acquire() - try: - self.pairs = {self._getpairindex(curses.COLOR_WHITE, - curses.COLOR_BLACK): 0} - self.nextpair = 1 - finally: - self.pairlock.release() + def curses_colorpair(self, col_name): + """Return the curses color pair, that corresponds to the color.""" - def lock(self): - """Locks the Curses ui thread + return curses.color_pair(self.colormap[col_name]) + + def init_colorpairs(self): + """Initialize the curses color pairs available.""" + + # set special colors 'gray' and 'banner' + self.colormap['white'] = 0 #hardcoded by curses + curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) + self.colormap['banner'] = 1 # color 'banner' for bannerwin + + bcol = curses.COLOR_BLACK + colors = ( # name, color, bold? + ('black', curses.COLOR_BLACK, False), + ('blue', curses.COLOR_BLUE,False), + ('red', curses.COLOR_RED, False), + ('purple', curses.COLOR_MAGENTA, False), + ('cyan', curses.COLOR_CYAN, False), + ('green', curses.COLOR_GREEN, False), + ('orange', curses.COLOR_YELLOW, False)) + #set the rest of all colors starting at pair 2 + i = 1 + for name, fcol, bold in colors: + i += 1 + self.colormap[name] = i + curses.init_pair(i, fcol, bcol) + + def lock(self, block=True): + """Locks the Curses ui thread. Can be invoked multiple times from the owning thread. Invoking from a non-owning thread blocks and waits until it has been unlocked by the owning thread.""" - self.iolock.acquire() + + return self.iolock.acquire(block) def unlock(self): - """Unlocks the Curses ui thread + """Unlocks the Curses ui thread. Decrease the lock counter by one and unlock the ui thread if the counter reaches 0. Only call this method when the calling thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked.""" + self.iolock.release() - - def locked(self, target, *args, **kwargs): + + def exec_locked(self, target, *args, **kwargs): """Perform an operation with full locking.""" + self.lock() try: - apply(target, args, kwargs) + target(*args, **kwargs) finally: self.unlock() @@ -74,536 +99,555 @@ class CursesUtil: def lockedstuff(): curses.panel.update_panels() curses.doupdate() - self.locked(lockedstuff) + self.exec_locked(lockedstuff) def isactive(self): return hasattr(self, 'stdscr') - def _getpairindex(self, fg, bg): - return '%d/%d' % (fg,bg) - - def getpair(self, fg, bg): - if not self.has_color: - return 0 - pindex = self._getpairindex(fg, bg) - self.pairlock.acquire() - try: - if self.pairs.has_key(pindex): - return curses.color_pair(self.pairs[pindex]) - else: - self.pairs[pindex] = self.nextpair - curses.init_pair(self.nextpair, fg, bg) - self.nextpair += 1 - return curses.color_pair(self.nextpair - 1) - finally: - self.pairlock.release() - - def start(self): - self.stdscr = curses.initscr() - curses.noecho() - curses.cbreak() - self.stdscr.keypad(1) - try: - curses.start_color() - self.has_color = curses.has_colors() - except: - self.has_color = 0 - - self.oldcursor = None - try: - self.oldcursor = curses.curs_set(0) - except: - pass - - self.stdscr.clear() - self.stdscr.refresh() - (self.height, self.width) = self.stdscr.getmaxyx() - self.initpairs() - - def stop(self): - if not hasattr(self, 'stdscr'): - return - #self.stdscr.addstr(self.height - 1, 0, "\n", - # self.getpair(curses.COLOR_WHITE, - # curses.COLOR_BLACK)) - if self.oldcursor != None: - curses.curs_set(self.oldcursor) - self.stdscr.refresh() - self.stdscr.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - del self.stdscr - - def reset(self): - # dirty walkaround for bug http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 (fixed since #83743) - if (sys.version_info[0:3] >= (2,6) and sys.version_info[0:3] <= (2,6,5)): return - self.stop() - self.start() class CursesAccountFrame: - def __init__(s, master, accountname, ui): - s.c = master - s.children = [] - s.accountname = accountname - s.ui = ui + """Notable instance variables: - def drawleadstr(s, secs = None): - if secs == None: - acctstr = '%s: [active] %13.13s: ' % (s.key, s.accountname) - else: - acctstr = '%s: [%3d:%02d] %13.13s: ' % (s.key, - secs / 60, secs % 60, - s.accountname) - s.c.locked(s.window.addstr, 0, 0, acctstr) - s.location = len(acctstr) + - account: corresponding Account() + - children + - ui + - key + - window: curses window associated with an account + """ - def setwindow(s, window, key): - s.window = window - s.key = key - s.drawleadstr() - for child in s.children: - child.update(window, 0, s.location) - s.location += 1 + def __init__(self, ui, account): + """ + :param account: An Account() or None (for eg SyncrunnerThread)""" - def getnewthreadframe(s): - tf = CursesThreadFrame(s.c, s.ui, s.window, 0, s.location) - s.location += 1 - s.children.append(tf) + self.children = [] + self.account = account if account else '*Control' + self.ui = ui + self.window = None + # Curses window associated with this acc. + self.acc_num = None + # Account number (& hotkey) associated with this acc. + self.location = 0 + # length of the account prefix string + + def drawleadstr(self, secs = 0): + """Draw the account status string. + + secs tells us how long we are going to sleep.""" + + sleepstr = '%3d:%02d'% (secs // 60, secs % 60) if secs else 'active' + accstr = '%s: [%s] %12.12s: '% (self.acc_num, sleepstr, self.account) + + self.ui.exec_locked(self.window.addstr, 0, 0, accstr) + self.location = len(accstr) + + def setwindow(self, curses_win, acc_num): + """Register an curses win and a hotkey as Account window. + + :param curses_win: the curses window associated with an account + :param acc_num: int denoting the hotkey associated with this account.""" + + self.window = curses_win + self.acc_num = acc_num + self.drawleadstr() + # Update the child ThreadFrames + for child in self.children: + child.update(curses_win, self.location, 0) + self.location += 1 + + def get_new_tframe(self): + """Create a new ThreadFrame and append it to self.children. + + :returns: The new ThreadFrame""" + + tf = CursesThreadFrame(self.ui, self.window, self.location, 0) + self.location += 1 + self.children.append(tf) return tf - def startsleep(s, sleepsecs): - s.sleeping_abort = 0 + def sleeping(self, sleepsecs, remainingsecs): + """Show how long we are going to sleep and sleep. - def sleeping(s, sleepsecs, remainingsecs): - if remainingsecs: - s.c.lock() - try: - s.drawleadstr(remainingsecs) - s.window.refresh() - finally: - s.c.unlock() - time.sleep(sleepsecs) - else: - s.c.lock() - try: - s.drawleadstr() - s.window.refresh() - finally: - s.c.unlock() - return s.sleeping_abort + :returns: Boolean, whether we want to abort the sleep""" - def syncnow(s): - s.sleeping_abort = 1 + self.drawleadstr(remainingsecs) + self.ui.exec_locked(self.window.refresh) + time.sleep(sleepsecs) + return self.account.get_abort_event() + + def syncnow(self): + """Request that we stop sleeping asap and continue to sync.""" + + # if this belongs to an Account (and not *Control), set the + # skipsleep pref + if isinstance(self.account, offlineimap.accounts.Account): + self.ui.info("Requested synchronization for acc: %s"% self.account) + self.account.config.set('Account %s'% self.account.name, + 'skipsleep', '1') class CursesThreadFrame: - def __init__(s, master, ui, window, y, x): - """master should be a CursesUtil object.""" - s.c = master - s.ui = ui - s.window = window - s.x = x - s.y = y - s.colors = [] - bg = curses.COLOR_BLACK - s.colormap = {'black': s.c.getpair(curses.COLOR_BLACK, bg), - 'gray': s.c.getpair(curses.COLOR_WHITE, bg), - 'white': curses.A_BOLD | s.c.getpair(curses.COLOR_WHITE, bg), - 'blue': s.c.getpair(curses.COLOR_BLUE, bg), - 'red': s.c.getpair(curses.COLOR_RED, bg), - 'purple': s.c.getpair(curses.COLOR_MAGENTA, bg), - 'cyan': s.c.getpair(curses.COLOR_CYAN, bg), - 'green': s.c.getpair(curses.COLOR_GREEN, bg), - 'orange': s.c.getpair(curses.COLOR_YELLOW, bg), - 'yellow': curses.A_BOLD | s.c.getpair(curses.COLOR_YELLOW, bg), - 'pink': curses.A_BOLD | s.c.getpair(curses.COLOR_RED, bg)} - #s.setcolor('gray') - s.setcolor('black') + """curses_color: current color pair for logging.""" - def setcolor(self, color): - self.color = self.colormap[color] + def __init__(self, ui, acc_win, x, y): + """ + :param ui: is a Blinkenlights() instance + :param acc_win: curses Account window""" + + self.ui = ui + self.window = acc_win + self.x = x + self.y = y + self.curses_color = curses.color_pair(0) #default color + + def setcolor(self, color, modifier=0): + """Draw the thread symbol '@' in the specified color + + :param modifier: Curses modified, such as curses.A_BOLD + """ + + self.curses_color = modifier | self.ui.curses_colorpair(color) self.colorname = color self.display() def display(self): - def lockedstuff(): - if self.getcolor() == 'black': - self.window.addstr(self.y, self.x, ' ', self.color) - else: - self.window.addstr(self.y, self.x, self.ui.config.getdefault("ui.Curses.Blinkenlights", "statuschar", '.'), self.color) - self.c.stdscr.move(self.c.height - 1, self.c.width - 1) + def locked_display(): + self.window.addch(self.y, self.x, '@', self.curses_color) self.window.refresh() - self.c.locked(lockedstuff) + # lock the curses IO while fudging stuff + self.ui.exec_locked(locked_display) - def getcolor(self): - return self.colorname + def update(self, acc_win, x, y): + """Update the xy position of the '.' (and possibly the aframe).""" - def getcolorpair(self): - return self.color - - def update(self, window, y, x): - self.window = window + self.window = acc_win self.y = y self.x = x self.display() - def setthread(self, newthread): + def std_color(self): self.setcolor('black') - #if newthread: - # self.setcolor('gray') - #else: - # self.setcolor('black') -class InputHandler: - def __init__(s, util): - s.c = util - s.bgchar = None - s.inputlock = Lock() - s.lockheld = 0 - s.statuslock = Lock() - s.startup = Event() - s.startthread() - def startthread(s): - s.thread = offlineimap.threadutil.ExitNotifyThread(target = s.bgreaderloop, - name = "InputHandler loop") - s.thread.setDaemon(1) - s.thread.start() +class InputHandler(ExitNotifyThread): + """Listens for input via the curses interfaces""" - def bgreaderloop(s): - while 1: - s.statuslock.acquire() - if s.lockheld or s.bgchar == None: - s.statuslock.release() - s.startup.wait() - else: - s.statuslock.release() - ch = s.c.stdscr.getch() - s.statuslock.acquire() - try: - if s.lockheld or s.bgchar == None: - curses.ungetch(ch) - else: - s.bgchar(ch) - finally: - s.statuslock.release() + #TODO, we need to use the ugly exitnotifythread (rather than simply + #threading.Thread here, so exiting this thread via the callback + #handler, kills off all parents too. Otherwise, they would simply + #continue. + def __init__(self, ui): + super(InputHandler, self).__init__() + self.char_handler = None + self.ui = ui + self.enabled = Event() + # We will only parse input if we are enabled. + self.inputlock = RLock() + # denotes whether we should be handling the next char. + self.start() #automatically start the thread - def set_bgchar(s, callback): - """Sets a "background" character handler. If a key is pressed - while not doing anything else, it will be passed to this handler. + def get_next_char(self): + """Return the key pressed or -1. + + Wait until `enabled` and loop internally every stdscr.timeout() + msecs, releasing the inputlock. + :returns: char or None if disabled while in here""" + + self.enabled.wait() + while self.enabled.is_set(): + with self.inputlock: + char = self.ui.stdscr.getch() + if char != -1: yield char + + def run(self): + while True: + char_gen = self.get_next_char() + for char in char_gen: + self.char_handler(char) + #curses.ungetch(char) + + def set_char_hdlr(self, callback): + """Sets a character callback handler. + + If a key is pressed it will be passed to this handler. Keys + include the curses.KEY_RESIZE key. callback is a function taking a single arg -- the char pressed. + If callback is None, input will be ignored.""" - If callback is None, clears the request.""" - s.statuslock.acquire() - oldhandler = s.bgchar - newhandler = callback - s.bgchar = callback + with self.inputlock: + self.char_handler = callback + # start or stop the parsing of things + if callback is None: + self.enabled.clear() + else: + self.enabled.set() - if oldhandler and not newhandler: - pass - if newhandler and not oldhandler: - s.startup.set() - - s.statuslock.release() - - def input_acquire(s): + def input_acquire(self): """Call this method when you want exclusive input control. - Make sure to call input_release afterwards! - """ - s.inputlock.acquire() - s.statuslock.acquire() - s.lockheld = 1 - s.statuslock.release() + Make sure to call input_release afterwards! While this lockis + held, input can go to e.g. the getpass input.""" - def input_release(s): + self.enabled.clear() + self.inputlock.acquire() + + def input_release(self): """Call this method when you are done getting input.""" - s.statuslock.acquire() - s.lockheld = 0 - s.statuslock.release() - s.inputlock.release() - s.startup.set() - -class Blinkenlights(BlinkenBase, UIBase): - def init_banner(s): - s.af = {} - s.aflock = Lock() - s.c = CursesUtil() - s.text = [] - BlinkenBase.init_banner(s) - s.setupwindows() - s.inputhandler = InputHandler(s.c) - s.gettf().setcolor('red') - s._msg(offlineimap.banner) - s.inputhandler.set_bgchar(s.keypress) - signal.signal(signal.SIGWINCH, s.resizehandler) - s.resizelock = Lock() - s.resizecount = 0 - def resizehandler(s, signum, frame): - s.resizeterm() - - def resizeterm(s, dosleep = 1): - if not s.resizelock.acquire(0): - s.resizecount += 1 - return - signal.signal(signal.SIGWINCH, signal.SIG_IGN) - s.aflock.acquire() - s.c.lock() - s.resizecount += 1 - while s.resizecount: - s.c.reset() - s.setupwindows() - s.resizecount -= 1 - s.c.unlock() - s.aflock.release() - s.resizelock.release() - signal.signal(signal.SIGWINCH, s.resizehandler) - if dosleep: - time.sleep(1) - s.resizeterm(0) - - def isusable(s): - # Not a terminal? Can't use curses. - if not sys.stdout.isatty() and sys.stdin.isatty(): - return 0 - - # No TERM specified? Can't use curses. - try: - if not len(os.environ['TERM']): - return 0 - except: return 0 - - # ncurses doesn't want to start? Can't use curses. - # This test is nasty because initscr() actually EXITS on error. - # grr. - - pid = os.fork() - if pid: - # parent - return not os.WEXITSTATUS(os.waitpid(pid, 0)[1]) - else: - # child - curses.initscr() - curses.endwin() - # If we didn't die by here, indicate success. - sys.exit(0) - - def keypress(s, key): - if key < 1 or key > 255: - return - - if chr(key) == 'q': - # Request to quit. - s.terminate() - - try: - index = acctkeys.index(chr(key)) - except ValueError: - # Key not a valid one: exit. - return - - if index >= len(s.hotkeys): - # Not in our list of valid hotkeys. - return - - # Trying to end sleep somewhere. - - s.getaccountframe(s.hotkeys[index]).syncnow() - - def getpass(s, accountname, config, errmsg = None): - s.inputhandler.input_acquire() - - # See comment on _msg for info on why both locks are obtained. - - s.tflock.acquire() - s.c.lock() - try: - s.gettf().setcolor('white') - s._addline(" *** Input Required", s.gettf().getcolorpair()) - s._addline(" *** Please enter password for account %s: " % accountname, - s.gettf().getcolorpair()) - s.logwindow.refresh() - password = s.logwindow.getstr() - finally: - s.tflock.release() - s.c.unlock() - s.inputhandler.input_release() - return password - - def setupwindows(s): - s.c.lock() - try: - s.bannerwindow = curses.newwin(1, s.c.width, 0, 0) - s.setupwindow_drawbanner() - s.logheight = s.c.height - 1 - len(s.af.keys()) - s.logwindow = curses.newwin(s.logheight, s.c.width, 1, 0) - s.logwindow.idlok(1) - s.logwindow.scrollok(1) - s.logwindow.move(s.logheight - 1, 0) - s.setupwindow_drawlog() - accounts = s.af.keys() - accounts.sort() - accounts.reverse() - - pos = s.c.height - 1 - index = 0 - s.hotkeys = [] - for account in accounts: - accountwindow = curses.newwin(1, s.c.width, pos, 0) - s.af[account].setwindow(accountwindow, acctkeys[index]) - s.hotkeys.append(account) - index += 1 - pos -= 1 - - curses.doupdate() - finally: - s.c.unlock() - - def setupwindow_drawbanner(s): - if s.c.has_color: - color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLUE) | \ - curses.A_BOLD - else: - color = curses.A_REVERSE - s.bannerwindow.bkgd(' ', color) # Fill background with that color - s.bannerwindow.addstr("%s %s" % (offlineimap.__productname__, - offlineimap.__version__)) - s.bannerwindow.addstr(0, s.bannerwindow.getmaxyx()[1] - len(offlineimap.__copyright__) - 1, - offlineimap.__copyright__) - - s.bannerwindow.noutrefresh() - - def setupwindow_drawlog(s): - if s.c.has_color: - color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLACK) - else: - color = curses.A_NORMAL - s.logwindow.bkgd(' ', color) - for line, color in s.text: - s.logwindow.addstr("\n" + line, color) - s.logwindow.noutrefresh() - - def getaccountframe(s, accountname = None): - if accountname == None: - accountname = s.getthreadaccount() - s.aflock.acquire() - try: - if accountname in s.af: - return s.af[accountname] - - # New one. - s.af[accountname] = CursesAccountFrame(s.c, accountname, s) - s.c.lock() - try: - s.c.reset() - s.setupwindows() - finally: - s.c.unlock() - finally: - s.aflock.release() - return s.af[accountname] + self.inputlock.release() + self.enabled.set() - def _display(s, msg, color = None): - if "\n" in msg: - for thisline in msg.split("\n"): - s._msg(thisline) - return +class CursesLogHandler(logging.StreamHandler): + """self.ui has been set to the UI class before anything is invoked""" + def emit(self, record): + log_str = logging.StreamHandler.format(self, record) + color = self.ui.gettf().curses_color # We must acquire both locks. Otherwise, deadlock can result. # This can happen if one thread calls _msg (locking curses, then # tf) and another tries to set the color (locking tf, then curses) # # By locking both up-front here, in this order, we prevent deadlock. - - s.tflock.acquire() - s.c.lock() + self.ui.tframe_lock.acquire() + self.ui.lock() try: - if not s.c.isactive(): - # For dumping out exceptions and stuff. - print msg - return - if color: - s.gettf().setcolor(color) - elif s.gettf().getcolor() == 'black': - s.gettf().setcolor('gray') - s._addline(msg, s.gettf().getcolorpair()) - s.logwindow.refresh() + y,x = self.ui.logwin.getyx() + if y or x: self.ui.logwin.addch(10) # no \n before 1st item + self.ui.logwin.addstr(log_str, color) finally: - s.c.unlock() - s.tflock.release() + self.ui.unlock() + self.ui.tframe_lock.release() + self.ui.logwin.noutrefresh() + self.ui.stdscr.refresh() - def _addline(s, msg, color): - s.c.lock() +class Blinkenlights(UIBase, CursesUtil): + """Curses-cased fancy UI. + + Notable instance variables self. ....: + + - stdscr: THe curses std screen + - bannerwin: The top line banner window + - width|height: The total curses screen dimensions + - logheight: Available height for the logging part + - log_con_handler: The CursesLogHandler() + - threadframes: + - accframes[account]: 'Accountframe'""" + + def __init__(self, *args, **kwargs): + super(Blinkenlights, self).__init__(*args, **kwargs) + CursesUtil.__init__(self) + + ################################################## UTILS + def setup_consolehandler(self): + """Backend specific console handler. + + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + + # create console handler with a higher log level + ch = CursesLogHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + # the handler is not usable yet. We still need all the + # intialization stuff currently done in init_banner. Move here? + return ch + + def isusable(s): + """Returns true if the backend is usable ie Curses works.""" + + # Not a terminal? Can't use curses. + if not sys.stdout.isatty() and sys.stdin.isatty(): + return False + # No TERM specified? Can't use curses. + if not os.environ.get('TERM', None): + return False + # Test if ncurses actually starts up fine. Only do so for + # python>=2.6.6 as calling initscr() twice messing things up. + # see http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 + if sys.version_info[0:3] < (2,6) or sys.version_info[0:3] >= (2,6,6): + try: + curses.initscr() + curses.endwin() + except: + return False + return True + + def init_banner(self): + self.availablethreadframes = {} + self.threadframes = {} + self.accframes = {} + self.aflock = Lock() + + self.stdscr = curses.initscr() + # turn off automatic echoing of keys to the screen + curses.noecho() + # react to keys instantly, without Enter key + curses.cbreak() + # return special key values, eg curses.KEY_LEFT + self.stdscr.keypad(1) + # wait 1s for input, so we don't block the InputHandler infinitely + self.stdscr.timeout(1000) + curses.start_color() + # turn off cursor and save original state + self.oldcursor = None try: - s.logwindow.addstr("\n" + msg, color) - s.text.append((msg, color)) - while len(s.text) > s.logheight: - s.text = s.text[1:] + self.oldcursor = curses.curs_set(0) + except: + pass + + self.stdscr.clear() + self.stdscr.refresh() + self.init_colorpairs() + # set log handlers ui to ourself + self._log_con_handler.ui = self + self.setupwindows() + # Settup keyboard handler + self.inputhandler = InputHandler(self) + self.inputhandler.set_char_hdlr(self.on_keypressed) + + self.gettf().setcolor('red') + self.info(offlineimap.banner) + + def acct(self, *args): + """Output that we start syncing an account (and start counting).""" + + self.gettf().setcolor('purple') + super(Blinkenlights, self).acct(*args) + + def connecting(self, *args): + self.gettf().setcolor('white') + super(Blinkenlights, self).connecting(*args) + + def syncfolders(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).syncfolders(*args) + + def syncingfolder(self, *args): + self.gettf().setcolor('cyan') + super(Blinkenlights, self).syncingfolder(*args) + + def skippingfolder(self, *args): + self.gettf().setcolor('cyan') + super(Blinkenlights, self).skippingfolder(*args) + + def loadmessagelist(self, *args): + self.gettf().setcolor('green') + super(Blinkenlights, self).loadmessagelist(*args) + + def syncingmessages(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).syncingmessages(*args) + + def copyingmessage(self, *args): + self.gettf().setcolor('orange') + super(Blinkenlights, self).copyingmessage(*args) + + def deletingmessages(self, *args): + self.gettf().setcolor('red') + super(Blinkenlights, self).deletingmessages(*args) + + def addingflags(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).addingflags(*args) + + def deletingflags(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).deletingflags(*args) + + def callhook(self, *args): + self.gettf().setcolor('white') + super(Blinkenlights, self).callhook(*args) + + ############ Generic logging functions ############################# + def warn(self, msg, minor=0): + self.gettf().setcolor('red', curses.A_BOLD) + super(Blinkenlights, self).warn(msg) + + def threadExited(self, thread): + acc = self.getthreadaccount(thread) + with self.tframe_lock: + if thread in self.threadframes[acc]: + tf = self.threadframes[acc][thread] + tf.setcolor('black') + self.availablethreadframes[acc].append(tf) + del self.threadframes[acc][thread] + super(Blinkenlights, self).threadExited(thread) + + def gettf(self): + """Return the ThreadFrame() of the current thread.""" + + cur_thread = currentThread() + acc = self.getthreadaccount() #Account() or None + + with self.tframe_lock: + # Ideally we already have self.threadframes[accountname][thread] + try: + if cur_thread in self.threadframes[acc]: + return self.threadframes[acc][cur_thread] + except KeyError: + # Ensure threadframes already has an account dict + self.threadframes[acc] = {} + self.availablethreadframes[acc] = deque() + + # If available, return a ThreadFrame() + if len(self.availablethreadframes[acc]): + tf = self.availablethreadframes[acc].popleft() + tf.std_color() + else: + tf = self.getaccountframe(acc).get_new_tframe() + self.threadframes[acc][cur_thread] = tf + return tf + + def on_keypressed(self, key): + # received special KEY_RESIZE, resize terminal + if key == curses.KEY_RESIZE: + self.resizeterm() + + if key < 1 or key > 255: + return + if chr(key) == 'q': + # Request to quit completely. + self.warn("Requested shutdown via 'q'") + offlineimap.accounts.Account.set_abort_event(self.config, 3) + try: + index = int(chr(key)) + except ValueError: + return # Key not a valid number: exit. + if index >= len(self.hotkeys): + # Not in our list of valid hotkeys. + return + # Trying to end sleep somewhere. + self.getaccountframe(self.hotkeys[index]).syncnow() + + def sleep(self, sleepsecs, account): + self.gettf().setcolor('red') + self.info("Next sync in %d:%02d"% (sleepsecs / 60, sleepsecs % 60)) + return super(Blinkenlights, self).sleep(sleepsecs, account) + + def sleeping(self, sleepsecs, remainingsecs): + if not sleepsecs: + # reset color to default if we are done sleeping. + self.gettf().setcolor('white') + accframe = self.getaccountframe(self.getthreadaccount()) + return accframe.sleeping(sleepsecs, remainingsecs) + + def resizeterm(self): + """Resize the current windows.""" + + self.exec_locked(self.setupwindows, True) + + def mainException(self): + UIBase.mainException(self) + + def getpass(self, accountname, config, errmsg=None): + # disable the hotkeys inputhandler + self.inputhandler.input_acquire() + + # See comment on _msg for info on why both locks are obtained. + self.lock() + try: + #s.gettf().setcolor('white') + self.warn(" *** Input Required") + self.warn(" *** Please enter password for account %s: " % \ + accountname) + self.logwin.refresh() + password = self.logwin.getstr() finally: - s.c.unlock() + self.unlock() + self.inputhandler.input_release() + return password - def terminate(s, exitstatus = 0, errortitle = None, errormsg = None): - s.c.stop() - UIBase.terminate(s, exitstatus = exitstatus, errortitle = errortitle, errormsg = errormsg) + def setupwindows(self, resize=False): + """Setup and draw bannerwin and logwin. - def threadException(s, thread): - s.c.stop() - UIBase.threadException(s, thread) + If `resize`, don't create new windows, just adapt size. This + function should be invoked with CursesUtils.locked().""" - def mainException(s): - s.c.stop() - UIBase.mainException(s) + self.height, self.width = self.stdscr.getmaxyx() + self.logheight = self.height - len(self.accframes) - 1 + if resize: + curses.resizeterm(self.height, self.width) + self.bannerwin.resize(1, self.width) + self.logwin.resize(self.logheight, self.width) + else: + self.bannerwin = curses.newwin(1, self.width, 0, 0) + self.logwin = curses.newwin(self.logheight, self.width, 1, 0) - def sleep(s, sleepsecs, siglistener): - s.gettf().setcolor('red') - s._msg("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return BlinkenBase.sleep(s, sleepsecs, siglistener) - -if __name__ == '__main__': - x = Blinkenlights(None) - x.init_banner() - import time - time.sleep(5) - x.c.stop() - fgs = {'black': curses.COLOR_BLACK, 'red': curses.COLOR_RED, - 'green': curses.COLOR_GREEN, 'yellow': curses.COLOR_YELLOW, - 'blue': curses.COLOR_BLUE, 'magenta': curses.COLOR_MAGENTA, - 'cyan': curses.COLOR_CYAN, 'white': curses.COLOR_WHITE} - - x = CursesUtil() - win1 = curses.newwin(x.height, x.width / 4 - 1, 0, 0) - win1.addstr("Black/normal\n") - for name, fg in fgs.items(): - win1.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLACK)) - win2 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1]) - win2.addstr("Blue/normal\n") - for name, fg in fgs.items(): - win2.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLUE)) - win3 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1] + - win2.getmaxyx()[1]) - win3.addstr("Black/bright\n") - for name, fg in fgs.items(): - win3.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLACK) | \ - curses.A_BOLD) - win4 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1] * 3) - win4.addstr("Blue/bright\n") - for name, fg in fgs.items(): - win4.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLUE) | \ - curses.A_BOLD) - - - win1.refresh() - win2.refresh() - win3.refresh() - win4.refresh() - x.stdscr.refresh() - import time - time.sleep(5) - x.stop() - print x.has_color - print x.height - print x.width + self.draw_bannerwin() + self.logwin.idlok(True) # needed for scrollok below + self.logwin.scrollok(True) # scroll window when too many lines added + self.draw_logwin() + self.accounts = reversed(sorted(self.accframes.keys())) + pos = self.height - 1 + index = 0 + self.hotkeys = [] + for account in self.accounts: + acc_win = curses.newwin(1, self.width, pos, 0) + self.accframes[account].setwindow(acc_win, index) + self.hotkeys.append(account) + index += 1 + pos -= 1 + curses.doupdate() + + def draw_bannerwin(self): + """Draw the top-line banner line.""" + + if curses.has_colors(): + color = curses.A_BOLD | self.curses_colorpair('banner') + else: + color = curses.A_REVERSE + self.bannerwin.clear() # Delete old content (eg before resizes) + self.bannerwin.bkgd(' ', color) # Fill background with that color + string = "%s %s"% (offlineimap.__productname__, + offlineimap.__bigversion__) + self.bannerwin.addstr(0, 0, string, color) + self.bannerwin.addstr(0, self.width -len(offlineimap.__copyright__) -1, + offlineimap.__copyright__, color) + self.bannerwin.noutrefresh() + + def draw_logwin(self): + """(Re)draw the current logwindow.""" + + if curses.has_colors(): + color = curses.color_pair(0) #default colors + else: + color = curses.A_NORMAL + self.logwin.move(0, 0) + self.logwin.erase() + self.logwin.bkgd(' ', color) + + def getaccountframe(self, acc_name): + """Return an AccountFrame() corresponding to acc_name. + + Note that the *control thread uses acc_name `None`.""" + + with self.aflock: + # 1) Return existing or 2) create a new CursesAccountFrame. + if acc_name in self.accframes: return self.accframes[acc_name] + self.accframes[acc_name] = CursesAccountFrame(self, acc_name) + # update the window layout + self.setupwindows(resize= True) + return self.accframes[acc_name] + + def terminate(self, *args, **kwargs): + curses.nocbreak(); + self.stdscr.keypad(0); + curses.echo() + curses.endwin() + # need to remove the Curses console handler now and replace with + # basic one, so exceptions and stuff are properly displayed + self.logger.removeHandler(self._log_con_handler) + UIBase.setup_consolehandler(self) + # reset the warning method, we do not have curses anymore + self.warn = super(Blinkenlights, self).warn + # finally call parent terminate which prints out exceptions etc + super(Blinkenlights, self).terminate(*args, **kwargs) + + def threadException(self, thread): + #self._log_con_handler.stop() + UIBase.threadException(self, thread) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index e5988e5..dc650c3 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -1,5 +1,4 @@ -# Copyright (C) 2007 John Goerzen -# +# Copyright (C) 2007-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,166 +13,183 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -import urllib, sys, re, time, traceback, threading, thread -from UIBase import UIBase -from threading import * +try: + from urllib import urlencode +except ImportError: # python3 + from urllib.parse import urlencode +import sys +import time +import logging +from threading import currentThread +from offlineimap.ui.UIBase import UIBase import offlineimap -protocol = '6.0.0' +protocol = '7.0.0' + +class MachineLogFormatter(logging.Formatter): + """urlencodes any outputted line, to avoid multi-line output""" + def format(s, record): + # Mapping of log levels to historic tag names + severity_map = { + 'info': 'msg', + 'warning': 'warn', + } + line = super(MachineLogFormatter, s).format(record) + severity = record.levelname.lower() + if severity in severity_map: + severity = severity_map[severity] + if hasattr(record, "machineui"): + command = record.machineui["command"] + whoami = record.machineui["id"] + else: + command = "" + whoami = currentThread().getName() + + prefix = "%s:%s"% (command, urlencode([('', whoami)])[1:]) + return "%s:%s:%s"% (severity, prefix, urlencode([('', line)])[1:]) + class MachineUI(UIBase): - def __init__(s, config, verbose = 0): - UIBase.__init__(s, config, verbose) - s.safechars=" ;,./-_=+()[]" - s.iswaiting = 0 - s.outputlock = Lock() - s._printData('__init__', protocol) + def __init__(s, config, loglevel=logging.INFO): + super(MachineUI, s).__init__(config, loglevel) + s._log_con_handler.createLock() + """lock needed to block on password input""" + # Set up the formatter that urlencodes the strings... + s._log_con_handler.setFormatter(MachineLogFormatter()) - def isusable(s): - return True + # Arguments: + # - handler: must be method from s.logger that reflects + # the severity of the passed message + # - command: command that produced this message + # - msg: the message itself + def _printData(s, handler, command, msg): + handler(msg, + extra = { + 'machineui': { + 'command': command, + 'id': currentThread().getName(), + } + }) - def _printData(s, command, data, dolock = True): - s._printDataOut('msg', command, data, dolock) + def _msg(s, msg): + s._printData(s.logger.info, '_display', msg) - def _printWarn(s, command, data, dolock = True): - s._printDataOut('warn', command, data, dolock) - - def _printDataOut(s, datatype, command, data, dolock = True): - if dolock: - s.outputlock.acquire() - try: - print "%s:%s:%s:%s" % \ - (datatype, - urllib.quote(command, s.safechars), - urllib.quote(currentThread().getName(), s.safechars), - urllib.quote(data, s.safechars)) - sys.stdout.flush() - finally: - if dolock: - s.outputlock.release() - - def _display(s, msg): - s._printData('_display', msg) - - def warn(s, msg, minor = 0): - s._printData('warn', '%s\n%d' % (msg, int(minor))) + def warn(s, msg, minor=0): + # TODO, remove and cleanup the unused minor stuff + s._printData(s.logger.warning, '', msg) def registerthread(s, account): - UIBase.registerthread(s, account) - s._printData('registerthread', account) + super(MachineUI, s).registerthread(account) + s._printData(s.logger.info, 'registerthread', account) def unregisterthread(s, thread): UIBase.unregisterthread(s, thread) - s._printData('unregisterthread', thread.getName()) + s._printData(s.logger.info, 'unregisterthread', thread.getName()) def debugging(s, debugtype): - s._printData('debugging', debugtype) + s._printData(s.logger.debug, 'debugging', debugtype) def acct(s, accountname): - s._printData('acct', accountname) + s._printData(s.logger.info, 'acct', accountname) def acctdone(s, accountname): - s._printData('acctdone', accountname) + s._printData(s.logger.info, 'acctdone', accountname) def validityproblem(s, folder): - s._printData('validityproblem', "%s\n%s\n%s\n%s" % \ + s._printData(s.logger.warning, 'validityproblem', "%s\n%s\n%s\n%s"% (folder.getname(), folder.getrepository().getname(), - folder.getsaveduidvalidity(), folder.getuidvalidity())) + folder.get_saveduidvalidity(), folder.get_uidvalidity())) def connecting(s, hostname, port): - s._printData('connecting', "%s\n%s" % (hostname, str(port))) + s._printData(s.logger.info, 'connecting', "%s\n%s"% (hostname, str(port))) def syncfolders(s, srcrepos, destrepos): - s._printData('syncfolders', "%s\n%s" % (s.getnicename(srcrepos), + s._printData(s.logger.info, 'syncfolders', "%s\n%s"% (s.getnicename(srcrepos), s.getnicename(destrepos))) def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): - s._printData('syncingfolder', "%s\n%s\n%s\n%s\n" % \ + s._printData(s.logger.info, 'syncingfolder', "%s\n%s\n%s\n%s\n"% (s.getnicename(srcrepos), srcfolder.getname(), s.getnicename(destrepos), destfolder.getname())) def loadmessagelist(s, repos, folder): - s._printData('loadmessagelist', "%s\n%s" % (s.getnicename(repos), + s._printData(s.logger.info, 'loadmessagelist', "%s\n%s"% (s.getnicename(repos), folder.getvisiblename())) def messagelistloaded(s, repos, folder, count): - s._printData('messagelistloaded', "%s\n%s\n%d" % \ + s._printData(s.logger.info, 'messagelistloaded', "%s\n%s\n%d"% (s.getnicename(repos), folder.getname(), count)) def syncingmessages(s, sr, sf, dr, df): - s._printData('syncingmessages', "%s\n%s\n%s\n%s\n" % \ + s._printData(s.logger.info, 'syncingmessages', "%s\n%s\n%s\n%s\n"% (s.getnicename(sr), sf.getname(), s.getnicename(dr), df.getname())) - def copyingmessage(s, uid, src, destlist): - ds = s.folderlist(destlist) - s._printData('copyingmessage', "%d\n%s\n%s\n%s" % \ - (uid, s.getnicename(src), src.getname(), ds)) - - def folderlist(s, list): - return ("\f".join(["%s\t%s" % (s.getnicename(x), x.getname()) for x in list])) + def copyingmessage(s, uid, num, num_to_copy, srcfolder, destfolder): + s._printData(s.logger.info, 'copyingmessage', "%d\n%s\n%s\n%s[%s]"% + (uid, s.getnicename(srcfolder), srcfolder.getname(), + s.getnicename(destfolder), destfolder)) - def deletingmessage(s, uid, destlist): - s.deletingmessages(s, [uid], destlist) + def folderlist(s, list): + return ("\f".join(["%s\t%s"% (s.getnicename(x), x.getname()) for x in list])) def uidlist(s, list): return ("\f".join([str(u) for u in list])) def deletingmessages(s, uidlist, destlist): ds = s.folderlist(destlist) - s._printData('deletingmessages', "%s\n%s" % (s.uidlist(uidlist), ds)) + s._printData(s.logger.info, 'deletingmessages', "%s\n%s"% (s.uidlist(uidlist), ds)) - def addingflags(s, uidlist, flags, destlist): - ds = s.folderlist(destlist) - s._printData("addingflags", "%s\n%s\n%s" % (s.uidlist(uidlist), + def addingflags(s, uidlist, flags, dest): + s._printData(s.logger.info, "addingflags", "%s\n%s\n%s"% (s.uidlist(uidlist), "\f".join(flags), - ds)) + dest)) - def deletingflags(s, uidlist, flags, destlist): - ds = s.folderlist(destlist) - s._printData('deletingflags', "%s\n%s\n%s" % (s.uidlist(uidlist), + def deletingflags(s, uidlist, flags, dest): + s._printData(s.logger.info, 'deletingflags', "%s\n%s\n%s"% (s.uidlist(uidlist), "\f".join(flags), - ds)) + dest)) def threadException(s, thread): - print s.getThreadExceptionString(thread) - s._printData('threadException', "%s\n%s" % \ + s._printData(s.logger.warning, 'threadException', "%s\n%s"% (thread.getName(), s.getThreadExceptionString(thread))) s.delThreadDebugLog(thread) s.terminate(100) - def terminate(s, exitstatus = 0, errortitle = '', errormsg = ''): - s._printData('terminate', "%d\n%s\n%s" % (exitstatus, errortitle, errormsg)) + def terminate(s, exitstatus=0, errortitle='', errormsg=''): + s._printData(s.logger.info, 'terminate', "%d\n%s\n%s"% (exitstatus, errortitle, errormsg)) sys.exit(exitstatus) def mainException(s): - s._printData('mainException', s.getMainExceptionString()) + s._printData(s.logger.warning, 'mainException', s.getMainExceptionString()) def threadExited(s, thread): - s._printData('threadExited', thread.getName()) + s._printData(s.logger.info, 'threadExited', thread.getName()) UIBase.threadExited(s, thread) def sleeping(s, sleepsecs, remainingsecs): - s._printData('sleeping', "%d\n%d" % (sleepsecs, remainingsecs)) + s._printData(s.logger.info, 'sleeping', "%d\n%d"% (sleepsecs, remainingsecs)) if sleepsecs > 0: time.sleep(sleepsecs) return 0 - def getpass(s, accountname, config, errmsg = None): - s.outputlock.acquire() + def getpass(s, accountname, config, errmsg=None): + if errmsg: + s._printData(s.logger.warning, + 'getpasserror', "%s\n%s"% (accountname, errmsg), + False) + + s._log_con_handler.acquire() # lock the console output try: - if errmsg: - s._printData('getpasserror', "%s\n%s" % (accountname, errmsg), - False) - s._printData('getpass', accountname, False) + s._printData(s.logger.info, 'getpass', accountname) return (sys.stdin.readline()[:-1]) finally: - s.outputlock.release() + s._log_con_handler.release() def init_banner(s): - s._printData('initbanner', offlineimap.banner) + s._printData(s.logger.info, 'protocol', protocol) + s._printData(s.logger.info, 'initbanner', offlineimap.banner) def callhook(s, msg): - s._printData('callhook', msg) + s._printData(s.logger.info, 'callhook', msg) diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index 9cd5eca..de1e8df 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -1,6 +1,5 @@ # Noninteractive UI -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2012 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,36 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import sys, time -from UIBase import UIBase +import logging +from offlineimap.ui.UIBase import UIBase class Basic(UIBase): - def getpass(s, accountname, config, errmsg = None): - raise NotImplementedError, "Prompting for a password is not supported in noninteractive mode." + """'Quiet' simply sets log level to INFO""" + def __init__(self, config, loglevel = logging.INFO): + return super(Basic, self).__init__(config, loglevel) - def _display(s, msg): - print msg - sys.stdout.flush() - - def warn(s, msg, minor = 0): - warntxt = 'WARNING' - if minor: - warntxt = 'warning' - sys.stderr.write(warntxt + ": " + str(msg) + "\n") - - def sleep(s, sleepsecs, siglistener): - if s.verbose >= 0: - s._msg("Sleeping for %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return UIBase.sleep(s, sleepsecs, siglistener) - - def sleeping(s, sleepsecs, remainingsecs): - if sleepsecs > 0: - time.sleep(sleepsecs) - return 0 - - def locked(s): - s.warn("Another OfflineIMAP is running with the same metadatadir; exiting.") - -class Quiet(Basic): - def __init__(s, config, verbose = -1): - Basic.__init__(s, config, verbose) +class Quiet(UIBase): + """'Quiet' simply sets log level to WARNING""" + def __init__(self, config, loglevel = logging.WARNING): + return super(Quiet, self).__init__(config, loglevel) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index ee18dfa..0b5aa6a 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -1,6 +1,5 @@ # TTY UI -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,55 +15,97 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from UIBase import UIBase +import logging +import sys +import time from getpass import getpass -import select, sys -from threading import * +from offlineimap import banner +from offlineimap.ui.UIBase import UIBase + +class TTYFormatter(logging.Formatter): + """Specific Formatter that adds thread information to the log output.""" + + def __init__(self, *args, **kwargs): + #super() doesn't work in py2.6 as 'logging' uses old-style class + logging.Formatter.__init__(self, *args, **kwargs) + self._last_log_thread = None + + def format(self, record): + """Override format to add thread information.""" + + #super() doesn't work in py2.6 as 'logging' uses old-style class + log_str = logging.Formatter.format(self, record) + # If msg comes from a different thread than our last, prepend + # thread info. Most look like 'Account sync foo' or 'Folder + # sync foo'. + t_name = record.threadName + if t_name == 'MainThread': + return log_str # main thread doesn't get things prepended + if t_name != self._last_log_thread: + self._last_log_thread = t_name + log_str = "%s:\n %s" % (t_name, log_str) + else: + log_str = " %s"% log_str + return log_str + class TTYUI(UIBase): - def __init__(s, config, verbose = 0): - UIBase.__init__(s, config, verbose) - s.iswaiting = 0 - s.outputlock = Lock() - s._lastThreaddisplay = None + def setup_consolehandler(self): + """Backend specific console handler + + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + + # create console handler with a higher log level + ch = logging.StreamHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = TTYFormatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(banner) + # init lock for console output + ch.createLock() + return ch + + def isusable(self): + """TTYUI is reported as usable when invoked on a terminal.""" - def isusable(s): return sys.stdout.isatty() and sys.stdin.isatty() - - def _display(s, msg): - s.outputlock.acquire() - try: - #if the next output comes from a different thread than our last one - #add the info. - #Most look like 'account sync foo' or 'Folder sync foo'. - try: - threadname = currentThread().name - except AttributeError: - threadname = currentThread().getName() - if (threadname == s._lastThreaddisplay): - print " %s" % msg - else: - print "%s:\n %s" % (threadname, msg) - s._lastThreaddisplay = threadname - sys.stdout.flush() - finally: - s.outputlock.release() + def getpass(self, accountname, config, errmsg=None): + """TTYUI backend is capable of querying the password.""" - def getpass(s, accountname, config, errmsg = None): if errmsg: - s._msg("%s: %s" % (accountname, errmsg)) - s.outputlock.acquire() + self.warn("%s: %s"% (accountname, errmsg)) + self._log_con_handler.acquire() # lock the console output try: - return getpass("%s: Enter password: " % accountname) + return getpass("Enter password for account '%s': " % accountname) finally: - s.outputlock.release() + self._log_con_handler.release() - def mainException(s): - if isinstance(sys.exc_info()[1], KeyboardInterrupt) and \ - s.iswaiting: - sys.stdout.write("Timer interrupted at user request; program terminating. \n") - s.terminate() + def mainException(self): + if isinstance(sys.exc_info()[1], KeyboardInterrupt): + self.logger.warn("Timer interrupted at user request; program " + "terminating.\n") + self.terminate() else: - UIBase.mainException(s) + UIBase.mainException(self) + def sleeping(self, sleepsecs, remainingsecs): + """Sleep for sleepsecs, display remainingsecs to go. + + Does nothing if sleepsecs <= 0. + Display a message on the screen if we pass a full minute. + + This implementation in UIBase does not support this, but some + implementations return 0 for successful sleep and 1 for an + 'abort', ie a request to sync immediately.""" + + if sleepsecs > 0: + if remainingsecs//60 != (remainingsecs-sleepsecs)//60: + self.logger.info("Next refresh in %.1f minutes" % ( + remainingsecs/60.0)) + time.sleep(sleepsecs) + return 0 diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 6621a62..3e698ab 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -1,6 +1,5 @@ # UI base class -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,353 +15,567 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import re, time, sys, traceback, threading, thread -from StringIO import StringIO -from Queue import Empty +import logging +import re +import time +import sys +import traceback +import threading +try: + from Queue import Queue +except ImportError: #python3 + from queue import Queue +from collections import deque +from offlineimap.error import OfflineImapError import offlineimap -debugtypes = {'imap': 'IMAP protocol debugging', +debugtypes = {'':'Other offlineimap related sync messages', + 'imap': 'IMAP protocol debugging', 'maildir': 'Maildir repository debugging', 'thread': 'Threading debugging'} globalui = None def setglobalui(newui): + """Set the global ui object to be used for logging.""" + global globalui globalui = newui + def getglobalui(): + """Return the current ui object.""" + global globalui return globalui -class UIBase: - def __init__(s, config, verbose = 0): - s.verbose = verbose - s.config = config - s.debuglist = [] - s.debugmessages = {} - s.debugmsglen = 50 - s.threadaccounts = {} - s.logfile = None - + +class UIBase(object): + def __init__(self, config, loglevel=logging.INFO): + self.config = config + # Is this a 'dryrun'? + self.dryrun = config.getdefaultboolean('general', 'dry-run', False) + self.debuglist = [] + # list of debugtypes we are supposed to log + self.debugmessages = {} + # debugmessages in a deque(v) per thread(k) + self.debugmsglen = 15 + self.threadaccounts = {} + # dict linking active threads (k) to account names (v) + self.acct_startimes = {} + # linking active accounts with the time.time() when sync started + self.logfile = None + self.exc_queue = Queue() + # saves all occuring exceptions, so we can output them at the end + self.uidval_problem = False + # at least one folder skipped due to UID validity problem + # create logger with 'OfflineImap' app + self.logger = logging.getLogger('OfflineImap') + self.logger.setLevel(loglevel) + self._log_con_handler = self.setup_consolehandler() + """The console handler (we need access to be able to lock it).""" + ################################################## UTILS - def _msg(s, msg): - """Generic tool called when no other works.""" - s._log(msg) - s._display(msg) + def setup_consolehandler(self): + """Backend specific console handler. - def _log(s, msg): - """Log it to disk. Returns true if it wrote something; false - otherwise.""" - if s.logfile: - s.logfile.write("%s: %s\n" % (threading.currentThread().getName(), - msg)) - return 1 - return 0 + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" - def setlogfd(s, logfd): - s.logfile = logfd - logfd.write("This is %s %s\n" % \ - (offlineimap.__productname__, - offlineimap.__version__)) - logfd.write("Python: %s\n" % sys.version) - logfd.write("Platform: %s\n" % sys.platform) - logfd.write("Args: %s\n" % sys.argv) + # create console handler with a higher log level + ch = logging.StreamHandler(sys.stdout) + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(offlineimap.banner) + return ch - def _display(s, msg): + def setlogfile(self, logfile): + """Create file handler which logs to file.""" + + fh = logging.FileHandler(logfile, 'at') + file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " + "%(message)s", '%Y-%m-%d %H:%M:%S') + fh.setFormatter(file_formatter) + self.logger.addHandler(fh) + # write out more verbose initial info blurb on the log file + p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) + msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ + "Args: %s"% (offlineimap.__bigversion__, p_ver, sys.platform, + " ".join(sys.argv)) + self.logger.info(msg) + + def _msg(self, msg): """Display a message.""" - raise NotImplementedError - def warn(s, msg, minor = 0): - if minor: - s._msg("warning: " + msg) + # TODO: legacy function, rip out. + self.info(msg) + + def info(self, msg): + """Display a message.""" + + self.logger.info(msg) + + def warn(self, msg, minor=0): + self.logger.warning(msg) + + def error(self, exc, exc_traceback=None, msg=None): + """Log a message at severity level ERROR. + + Log Exception 'exc' to error log, possibly prepended by a preceding + error "msg", detailing at what point the error occurred. + + In debug mode, we also output the full traceback that occurred + if one has been passed in via sys.info()[2]. + + Also save the Exception to a stack that can be output at the end + of the sync run when offlineiamp exits. It is recommended to + always pass in exceptions if possible, so we can give the user + the best debugging info. + + We are always pushing tracebacks to the exception queue to + make them to be output at the end of the run to allow users + pass sensible diagnostics to the developers or to solve + problems by themselves. + + One example of such a call might be: + + ui.error(exc, sys.exc_info()[2], msg="While syncing Folder %s in " + "repo %s") + """ + if msg: + self.logger.error("ERROR: %s\n %s" % (msg, exc)) else: - s._msg("WARNING: " + msg) + self.logger.error("ERROR: %s" % (exc)) - def registerthread(s, account): - """Provides a hint to UIs about which account this particular - thread is processing.""" - if s.threadaccounts.has_key(threading.currentThread()): - raise ValueError, "Thread %s already registered (old %s, new %s)" %\ - (threading.currentThread().getName(), - s.getthreadaccount(s), account) - s.threadaccounts[threading.currentThread()] = account + instant_traceback = exc_traceback + if not self.debuglist: + # only output tracebacks in debug mode + instant_traceback = None + # push exc on the queue for later output + self.exc_queue.put((msg, exc, exc_traceback)) + if instant_traceback: + self.logger.error(traceback.format_tb(instant_traceback)) - def unregisterthread(s, thr): - """Recognizes a thread has exited.""" - if s.threadaccounts.has_key(thr): - del s.threadaccounts[thr] + def registerthread(self, account): + """Register current thread as being associated with an account name.""" - def getthreadaccount(s, thr = None): - if not thr: + cur_thread = threading.currentThread() + if cur_thread in self.threadaccounts: + # was already associated with an old account, update info + self.debug('thread', "Register thread '%s' (previously '%s', now " + "'%s')" % (cur_thread.getName(), + self.getthreadaccount(cur_thread), account)) + else: + self.debug('thread', "Register new thread '%s' (account '%s')"% + (cur_thread.getName(), account)) + self.threadaccounts[cur_thread] = account + + def unregisterthread(self, thr): + """Unregister a thread as being associated with an account name.""" + + if thr in self.threadaccounts: + del self.threadaccounts[thr] + self.debug('thread', "Unregister thread '%s'" % thr.getName()) + + def getthreadaccount(self, thr=None): + """Get Account() for a thread (current if None) + + If no account has been registered with this thread, return 'None'.""" + + if thr == None: thr = threading.currentThread() - if s.threadaccounts.has_key(thr): - return s.threadaccounts[thr] - return '*Control' + if thr in self.threadaccounts: + return self.threadaccounts[thr] + return None - def debug(s, debugtype, msg): - thisthread = threading.currentThread() - if s.debugmessages.has_key(thisthread): - s.debugmessages[thisthread].append("%s: %s" % (debugtype, msg)) - else: - s.debugmessages[thisthread] = ["%s: %s" % (debugtype, msg)] + def debug(self, debugtype, msg): + cur_thread = threading.currentThread() + if not cur_thread in self.debugmessages: + # deque(..., self.debugmsglen) would be handy but was + # introduced in p2.6 only, so we'll need to work around and + # shorten our debugmsg list manually :-( + self.debugmessages[cur_thread] = deque() + self.debugmessages[cur_thread].append("%s: %s" % (debugtype, msg)) - while len(s.debugmessages[thisthread]) > s.debugmsglen: - s.debugmessages[thisthread] = s.debugmessages[thisthread][1:] + # Shorten queue if needed + if len(self.debugmessages[cur_thread]) > self.debugmsglen: + self.debugmessages[cur_thread].popleft() - if debugtype in s.debuglist: - if not s._log("DEBUG[%s]: %s" % (debugtype, msg)): - s._display("DEBUG[%s]: %s" % (debugtype, msg)) + if debugtype in self.debuglist: # log if we are supposed to do so + self.logger.debug("[%s]: %s" % (debugtype, msg)) - def add_debug(s, debugtype): + def add_debug(self, debugtype): global debugtypes if debugtype in debugtypes: - if not debugtype in s.debuglist: - s.debuglist.append(debugtype) - s.debugging(debugtype) + if not debugtype in self.debuglist: + self.debuglist.append(debugtype) + self.debugging(debugtype) else: - s.invaliddebug(debugtype) + self.invaliddebug(debugtype) - def debugging(s, debugtype): + def debugging(self, debugtype): global debugtypes - s._msg("Now debugging for %s: %s" % (debugtype, debugtypes[debugtype])) + self.logger.debug("Now debugging for %s: %s" % (debugtype, + debugtypes[debugtype])) - def invaliddebug(s, debugtype): - s.warn("Invalid debug type: %s" % debugtype) + def invaliddebug(self, debugtype): + self.warn("Invalid debug type: %s" % debugtype) - def locked(s): - raise Exception, "Another OfflineIMAP is running with the same metadatadir; exiting." + def getnicename(self, object): + """Return the type of a repository or Folder as string. - def getnicename(s, object): - prelimname = str(object.__class__).split('.')[-1] + (IMAP, Gmail, Maildir, etc...)""" + + prelimname = object.__class__.__name__.split('.')[-1] # Strip off extra stuff. return re.sub('(Folder|Repository)', '', prelimname) - def isusable(s): + def isusable(self): """Returns true if this UI object is usable in the current environment. For instance, an X GUI would return true if it's being run in X with a valid DISPLAY setting, and false otherwise.""" - return 1 + + return True ################################################## INPUT - def getpass(s, accountname, config, errmsg = None): - raise NotImplementedError + def getpass(self, accountname, config, errmsg = None): + raise NotImplementedError("Prompting for a password is not supported" + " in this UI backend.") - def folderlist(s, list): - return ', '.join(["%s[%s]" % (s.getnicename(x), x.getname()) for x in list]) + def folderlist(self, folder_list): + return ', '.join(["%s[%s]"% \ + (self.getnicename(x), x.getname()) for x in folder_list]) ################################################## WARNINGS - def msgtoreadonly(s, destfolder, uid, content, flags): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to synchronize message %d to folder %s[%s], but that folder is read-only. The message will not be copied to that folder." % \ - (uid, s.getnicename(destfolder), destfolder.getname())) + def msgtoreadonly(self, destfolder, uid, content, flags): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to synchronize message %d to folder %s[%s], " + "but that folder is read-only. The message will not be " + "copied to that folder."% ( + uid, self.getnicename(destfolder), destfolder)) - def flagstoreadonly(s, destfolder, uidlist, flags): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to modify flags for messages %s in folder %s[%s], but that folder is read-only. No flags have been modified for that message." % \ - (str(uidlist), s.getnicename(destfolder), destfolder.getname())) + def flagstoreadonly(self, destfolder, uidlist, flags): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to modify flags for messages %s in folder %s[%s], " + "but that folder is read-only. No flags have been modified " + "for that message."% ( + str(uidlist), self.getnicename(destfolder), destfolder)) - def deletereadonly(s, destfolder, uidlist): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to delete messages %s in folder %s[%s], but that folder is read-only. No messages have been deleted in that folder." % \ - (str(uidlist), s.getnicename(destfolder), destfolder.getname())) + def labelstoreadonly(self, destfolder, uidlist, labels): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to modify labels for messages %s in folder %s[%s], " + "but that folder is read-only. No labels have been modified " + "for that message."% ( + str(uidlist), self.getnicename(destfolder), destfolder)) + + def deletereadonly(self, destfolder, uidlist): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to delete messages %s in folder %s[%s], but that " + "folder is read-only. No messages have been deleted in that " + "folder."% (str(uidlist), self.getnicename(destfolder), + destfolder)) ################################################## MESSAGES - def init_banner(s): + def init_banner(self): """Called when the UI starts. Must be called before any other UI call except isusable(). Displays the copyright banner. This is where the UI should do its setup -- TK, for instance, would create the application window here.""" - if s.verbose >= 0: - s._msg(offlineimap.banner) + pass - def connecting(s, hostname, port): - if s.verbose < 0: - return - if hostname == None: - hostname = '' - if port != None: - port = ":%s" % str(port) - displaystr = ' to %s%s.' % (hostname, port) - if hostname == '' and port == None: - displaystr = '.' - s._msg("Establishing connection" + displaystr) + def connecting(self, hostname, port): + """Log 'Establishing connection to'.""" - def acct(s, accountname): - if s.verbose >= 0: - s._msg("***** Processing account %s" % accountname) + if not self.logger.isEnabledFor(logging.INFO): return + displaystr = '' + hostname = hostname if hostname else '' + port = "%s"% port if port else '' + if hostname: + displaystr = ' to %s:%s' % (hostname, port) + self.logger.info("Establishing connection%s" % displaystr) - def acctdone(s, accountname): - if s.verbose >= 0: - s._msg("***** Finished processing account " + accountname) + def acct(self, account): + """Output that we start syncing an account (and start counting).""" - def syncfolders(s, srcrepos, destrepos): - if s.verbose >= 0: - s._msg("Copying folder structure from %s to %s" % \ - (s.getnicename(srcrepos), s.getnicename(destrepos))) + self.acct_startimes[account] = time.time() + self.logger.info("*** Processing account %s" % account) + + def acctdone(self, account): + """Output that we finished syncing an account (in which time).""" + + sec = time.time() - self.acct_startimes[account] + del self.acct_startimes[account] + self.logger.info("*** Finished account '%s' in %d:%02d"% + (account, sec // 60, sec % 60)) + + def syncfolders(self, src_repo, dst_repo): + """Log 'Copying folder structure...'.""" + + if self.logger.isEnabledFor(logging.DEBUG): + self.debug('', "Copying folder structure from %s to %s" %\ + (src_repo, dst_repo)) ############################## Folder syncing - def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): + def makefolder(self, repo, foldername): + """Called when a folder is created.""" + + prefix = "[DRYRUN] " if self.dryrun else "" + self.info(("{0}Creating folder {1}[{2}]".format( + prefix, foldername, repo))) + + def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder): """Called when a folder sync operation is started.""" - if s.verbose >= 0: - s._msg("Syncing %s: %s -> %s" % (srcfolder.getname(), - s.getnicename(srcrepos), - s.getnicename(destrepos))) - def skippingfolder(s, folder): + self.logger.info("Syncing %s: %s -> %s"% (srcfolder, + self.getnicename(srcrepos), + self.getnicename(destrepos))) + + def skippingfolder(self, folder): """Called when a folder sync operation is started.""" - if s.verbose >= 0: - s._msg("Skipping %s (not changed)" % folder.getname()) + self.logger.info("Skipping %s (not changed)" % folder) - def validityproblem(s, folder): - s.warn("UID validity problem for folder %s (repo %s) (saved %d; got %d); skipping it" % \ - (folder.getname(), folder.getrepository().getname(), - folder.getsaveduidvalidity(), folder.getuidvalidity())) + def validityproblem(self, folder): + self.uidval_problem = True + self.logger.warning("UID validity problem for folder %s (repo %s) " + "(saved %d; got %d); skipping it. Please see FAQ " + "and manual on how to handle this."% \ + (folder, folder.getrepository(), + folder.get_saveduidvalidity(), folder.get_uidvalidity())) - def loadmessagelist(s, repos, folder): - if s.verbose > 0: - s._msg("Loading message list for %s[%s]" % (s.getnicename(repos), - folder.getname())) + def loadmessagelist(self, repos, folder): + self.logger.debug("Loading message list for %s[%s]"% ( + self.getnicename(repos), + folder)) - def messagelistloaded(s, repos, folder, count): - if s.verbose > 0: - s._msg("Message list for %s[%s] loaded: %d messages" % \ - (s.getnicename(repos), folder.getname(), count)) + def messagelistloaded(self, repos, folder, count): + self.logger.debug("Message list for %s[%s] loaded: %d messages" % ( + self.getnicename(repos), folder, count)) ############################## Message syncing - def syncingmessages(s, sr, sf, dr, df): - if s.verbose > 0: - s._msg("Syncing messages %s[%s] -> %s[%s]" % (s.getnicename(sr), - sf.getname(), - s.getnicename(dr), - df.getname())) + def syncingmessages(self, sr, srcfolder, dr, dstfolder): + self.logger.debug("Syncing messages %s[%s] -> %s[%s]" % ( + self.getnicename(sr), srcfolder, + self.getnicename(dr), dstfolder)) - def copyingmessage(s, uid, src, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Copy message %d %s[%s] -> %s" % (uid, s.getnicename(src), - src.getname(), ds)) + def copyingmessage(self, uid, num, num_to_copy, src, destfolder): + """Output a log line stating which message we copy""" + self.logger.info("Copy message %s (%d of %d) %s:%s -> %s" % ( + uid, num, num_to_copy, src.repository, src, + destfolder.repository)) - def deletingmessage(s, uid, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting message %d in %s" % (uid, ds)) + def deletingmessages(self, uidlist, destlist): + ds = self.folderlist(destlist) + prefix = "[DRYRUN] " if self.dryrun else "" + self.info("{0}Deleting {1} messages ({2}) in {3}".format( + prefix, len(uidlist), + offlineimap.imaputil.uid_sequence(uidlist), ds)) - def deletingmessages(s, uidlist, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting %d messages (%s) in %s" % \ - (len(uidlist), - ", ".join([str(u) for u in uidlist]), - ds)) + def addingflags(self, uidlist, flags, dest): + self.logger.info("Adding flag %s to %d messages on %s" % ( + ", ".join(flags), len(uidlist), dest)) - def addingflags(s, uidlist, flags, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Adding flags %s to %d messages on %s" % \ - (", ".join(flags), len(uidlist), ds)) + def deletingflags(self, uidlist, flags, dest): + self.logger.info("Deleting flag %s from %d messages on %s" % ( + ", ".join(flags), len(uidlist), dest)) - def deletingflags(s, uidlist, flags, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting flags %s to %d messages on %s" % \ - (", ".join(flags), len(uidlist), ds)) + def addinglabels(self, uidlist, label, dest): + self.logger.info("Adding label %s to %d messages on %s" % ( + label, len(uidlist), dest)) + + def deletinglabels(self, uidlist, label, dest): + self.logger.info("Deleting label %s from %d messages on %s" % ( + label, len(uidlist), dest)) + + def settinglabels(self, uid, num, num_to_set, labels, dest): + self.logger.info("Setting labels to message %d on %s (%d of %d): %s" % ( + uid, dest, num, num_to_set, ", ".join(labels))) + + def collectingdata(self, uidlist, source): + if uidlist: + self.logger.info("Collecting data from %d messages on %s"% ( + len(uidlist), source)) + else: + self.logger.info("Collecting data from messages on %s"% source) + + def serverdiagnostics(self, repository, type): + """Connect to repository and output useful information for debugging.""" + + conn = None + self._msg("%s repository '%s': type '%s'" % (type, repository.name, + self.getnicename(repository))) + try: + if hasattr(repository, 'gethost'): # IMAP + self._msg("Host: %s Port: %s SSL: %s"% (repository.gethost(), + repository.getport(), repository.getssl())) + try: + conn = repository.imapserver.acquireconnection() + except OfflineImapError as e: + self._msg("Failed to connect. Reason %s" % e) + else: + if 'ID' in conn.capabilities: + self._msg("Server supports ID extension.") + #TODO: Debug and make below working, it hangs Gmail + #res_type, response = conn.id(( + # 'name', offlineimap.__productname__, + # 'version', offlineimap.__bigversion__)) + #self._msg("Server ID: %s %s" % (res_type, response[0])) + self._msg("Server welcome string: %s" % str(conn.welcome)) + self._msg("Server capabilities: %s\n" % str(conn.capabilities)) + repository.imapserver.releaseconnection(conn) + if type != 'Status': + folderfilter = repository.getconf('folderfilter', None) + if folderfilter: + self._msg("folderfilter= %s\n" % folderfilter) + folderincludes = repository.getconf('folderincludes', None) + if folderincludes: + self._msg("folderincludes= %s\n" % folderincludes) + nametrans = repository.getconf('nametrans', None) + if nametrans: + self._msg("nametrans= %s\n" % nametrans) + + folders = repository.getfolders() + foldernames = [(f.name, f.getvisiblename(), f.sync_this) + for f in folders] + folders = [] + for name, visiblename, sync_this in foldernames: + syncstr = "" if sync_this else " (disabled)" + if name == visiblename: folders.append("%s%s" % (name, + syncstr)) + else: folders.append("%s -> %s%s" % (name, + visiblename, syncstr)) + self._msg("Folderlist:\n %s\n" % "\n ".join(folders)) + finally: + if conn: #release any existing IMAP connection + repository.imapserver.close() + + def savemessage(self, debugtype, uid, flags, folder): + """Output a log line stating that we save a msg.""" + + self.debug(debugtype, "Write mail '%s:%d' with flags %s"% + (folder, uid, repr(flags))) ################################################## Threads - def getThreadDebugLog(s, thread): - if s.debugmessages.has_key(thread): + def getThreadDebugLog(self, thread): + if thread in self.debugmessages: message = "\nLast %d debug messages logged for %s prior to exception:\n"\ - % (len(s.debugmessages[thread]), thread.getName()) - message += "\n".join(s.debugmessages[thread]) + % (len(self.debugmessages[thread]), thread.getName()) + message += "\n".join(self.debugmessages[thread]) else: - message = "\nNo debug messages were logged for %s." % \ - thread.getName() + message = "\nNo debug messages were logged for %s."% \ + thread.getName() return message - def delThreadDebugLog(s, thread): - if s.debugmessages.has_key(thread): - del s.debugmessages[thread] + def delThreadDebugLog(self, thread): + if thread in self.debugmessages: + del self.debugmessages[thread] - def getThreadExceptionString(s, thread): - message = "Thread '%s' terminated with exception:\n%s" % \ - (thread.getName(), thread.getExitStackTrace()) - message += "\n" + s.getThreadDebugLog(thread) + def getThreadExceptionString(self, thread): + message = "Thread '%s' terminated with exception:\n%s"% \ + (thread.getName(), thread.exit_stacktrace) + message += "\n" + self.getThreadDebugLog(thread) return message - def threadException(s, thread): + def threadException(self, thread): """Called when a thread has terminated with an exception. The argument is the ExitNotifyThread that has so terminated.""" - s._msg(s.getThreadExceptionString(thread)) - s.delThreadDebugLog(thread) - s.terminate(100) - def getMainExceptionString(s): - sbuf = StringIO() - traceback.print_exc(file = sbuf) - return "Main program terminated with exception:\n" + \ - sbuf.getvalue() + "\n" + \ - s.getThreadDebugLog(threading.currentThread()) + self.warn(self.getThreadExceptionString(thread)) + self.delThreadDebugLog(thread) + self.terminate(100) - def mainException(s): - s._msg(s.getMainExceptionString()) - - def terminate(s, exitstatus = 0, errortitle = None, errormsg = None): + def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" - if errormsg <> None: - if errortitle <> None: - sys.stderr.write('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) + + #print any exceptions that have occurred over the run + if not self.exc_queue.empty(): + self.warn("ERROR: Exceptions occurred during the run!") + if exitstatus == 0: + exitstatus = 1 + while not self.exc_queue.empty(): + msg, exc, exc_traceback = self.exc_queue.get() + if msg: + self.warn("ERROR: %s\n %s"% (msg, exc)) else: - sys.stderr.write('%s\n' % errormsg) + self.warn("ERROR: %s"% (exc)) + if exc_traceback: + self.warn("\nTraceback:\n%s"% "".join( + traceback.format_tb(exc_traceback))) + + if errormsg and errortitle: + self.warn('ERROR: %s\n\n%s\n'% (errortitle, errormsg)) + elif errormsg: + self.warn('%s\n'% errormsg) + if self.uidval_problem: + self.warn('At least one folder skipped due to UID validity problem') + if exitstatus == 0: + exitstatus = 2 sys.exit(exitstatus) - def threadExited(s, thread): - """Called when a thread has exited normally. Many UIs will - just ignore this.""" - s.delThreadDebugLog(thread) - s.unregisterthread(thread) + def threadExited(self, thread): + """Called when a thread has exited normally. + + Many UIs will just ignore this.""" + + self.delThreadDebugLog(thread) + self.unregisterthread(thread) ################################################## Hooks - def callhook(s, msg): - if s.verbose >= 0: - s._msg(msg) + def callhook(self, msg): + if self.dryrun: + self.info("[DRYRUN] {0}".format(msg)) + else: + self.info(msg) ################################################## Other - def sleep(s, sleepsecs, siglistener): + def sleep(self, sleepsecs, account): """This function does not actually output anything, but handles the overall sleep, dealing with updates as necessary. It will, however, call sleeping() which DOES output something. - Returns 0 if timeout expired, 1 if there is a request to cancel - the timer, and 2 if there is a request to abort the program.""" + :returns: 0/False if timeout expired, 1/2/True if there is a + request to cancel the timer. + """ - abortsleep = 0 + abortsleep = False while sleepsecs > 0 and not abortsleep: - try: - abortsleep = siglistener.get_nowait() - # retrieved signal while sleeping: 1 means immediately resynch, 2 means immediately die - except Empty: - # no signal - abortsleep = s.sleeping(10, sleepsecs) - sleepsecs -= 10 - s.sleeping(0, 0) # Done sleeping. + if account.get_abort_event(): + abortsleep = True + else: + abortsleep = self.sleeping(10, sleepsecs) + sleepsecs -= 10 + self.sleeping(0, 0) # Done sleeping. return abortsleep - def sleeping(s, sleepsecs, remainingsecs): + def sleeping(self, sleepsecs, remainingsecs): """Sleep for sleepsecs, display remainingsecs to go. Does nothing if sleepsecs <= 0. - Display a message on the screen every 10 seconds. + Display a message on the screen if we pass a full minute. This implementation in UIBase does not support this, but some implementations return 0 for successful sleep and 1 for an 'abort', ie a request to sync immediately. """ + if sleepsecs > 0: - if remainingsecs % 10 == 0: - s._msg("Next refresh in %d seconds" % remainingsecs) + if remainingsecs//60 != (remainingsecs-sleepsecs)//60: + self.logger.debug("Next refresh in %.1f minutes" % ( + remainingsecs/60.0)) time.sleep(sleepsecs) return 0 diff --git a/offlineimap/ui/__init__.py b/offlineimap/ui/__init__.py index 83d81c6..3da42b9 100644 --- a/offlineimap/ui/__init__.py +++ b/offlineimap/ui/__init__.py @@ -1,5 +1,5 @@ # UI module -# Copyright (C) 2010 Sebastian Spaeth +# Copyright (C) 2010-2011 Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,14 +18,14 @@ from offlineimap.ui.UIBase import getglobalui, setglobalui from offlineimap.ui import TTY, Noninteractive, Machine -UI_LIST = {'TTY.TTYUI': TTY.TTYUI, - 'Noninteractive.Basic': Noninteractive.Basic, - 'Noninteractive.Quiet': Noninteractive.Quiet, - 'Machine.MachineUI': Machine.MachineUI} +UI_LIST = {'ttyui': TTY.TTYUI, + 'basic': Noninteractive.Basic, + 'quiet': Noninteractive.Quiet, + 'machineui': Machine.MachineUI} #add Blinkenlights UI if it imports correctly (curses installed) try: from offlineimap.ui import Curses - UI_LIST['Curses.Blinkenlights'] = Curses.Blinkenlights + UI_LIST['blinkenlights'] = Curses.Blinkenlights except ImportError: pass diff --git a/offlineimap/ui/debuglock.py b/offlineimap/ui/debuglock.py index 7873a1f..673efb0 100644 --- a/offlineimap/ui/debuglock.py +++ b/offlineimap/ui/debuglock.py @@ -1,6 +1,5 @@ # Locking debugging code -- temporary -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,7 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import * +from threading import Lock, currentThread import traceback logfile = open("/tmp/logfile", "wt") loglock = Lock() @@ -25,11 +24,12 @@ class DebuggingLock: def __init__(self, name): self.lock = Lock() self.name = name - + def acquire(self, blocking = 1): self.print_tb("Acquire lock") self.lock.acquire(blocking) - self.logmsg("===== %s: Thread %s acquired lock\n" % (self.name, currentThread().getName())) + self.logmsg("===== %s: Thread %s acquired lock\n"% + (self.name, currentThread().getName())) def release(self): self.print_tb("Release lock") @@ -42,8 +42,8 @@ class DebuggingLock: loglock.release() def print_tb(self, msg): - self.logmsg(".... %s: Thread %s attempting to %s\n" % \ + self.logmsg(".... %s: Thread %s attempting to %s\n"% \ (self.name, currentThread().getName(), msg) + \ "\n".join(traceback.format_list(traceback.extract_stack()))) - + diff --git a/offlineimap/utils/__init__.py b/offlineimap/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/offlineimap/utils/const.py b/offlineimap/utils/const.py new file mode 100644 index 0000000..f4584bc --- /dev/null +++ b/offlineimap/utils/const.py @@ -0,0 +1,37 @@ +# Copyright (C) 2013-2014 Eygene A. Ryabinkin and contributors +# +# Collection of classes that implement const-like behaviour +# for various objects. + +import copy + +class ConstProxy(object): + """Implements read-only access to a given object + that can be attached to each instance only once.""" + + def __init__(self): + self.__dict__['__source'] = None + + + def __getattr__(self, name): + src = self.__dict__['__source'] + if src == None: + raise ValueError("using non-initialized ConstProxy() object") + return copy.deepcopy(getattr(src, name)) + + + def __setattr__(self, name, value): + raise AttributeError("tried to set '%s' to '%s' for constant object"% \ + (name, value)) + + + def __delattr__(self, name): + raise RuntimeError("tried to delete field '%s' from constant object"% \ + (name)) + + + def set_source(self, source): + """ Sets source object for this instance. """ + if (self.__dict__['__source'] != None): + raise ValueError("source object is already set") + self.__dict__['__source'] = source diff --git a/offlineimap/utils/distro.py b/offlineimap/utils/distro.py new file mode 100644 index 0000000..8cd2b79 --- /dev/null +++ b/offlineimap/utils/distro.py @@ -0,0 +1,91 @@ +# Copyright 2014 Eygene A. Ryabinkin. +# +# Module that supports distribution-specific functions. + +import platform +import os + + +# Each dictionary value is either string or some iterable. +# +# For the former we will just return the value, for an iterable +# we will walk through the values and will return the first +# one that corresponds to the existing file. +__DEF_OS_LOCATIONS = { + 'freebsd': '/usr/local/share/certs/ca-root-nss.crt', + 'openbsd': '/etc/ssl/cert.pem', + 'dragonfly': '/etc/ssl/cert.pem', + 'darwin': [ + # MacPorts, port curl-ca-bundle + '/opt/local/share/curl/curl-ca-bundle.crt', + ], + 'linux-ubuntu': '/etc/ssl/certs/ca-certificates.crt', + 'linux-debian': '/etc/ssl/certs/ca-certificates.crt', + 'linux-fedora': '/etc/pki/tls/certs/ca-bundle.crt', + 'linux-redhat': '/etc/pki/tls/certs/ca-bundle.crt', + 'linux-suse': '/etc/ssl/ca-bundle.pem', +} + + +def get_os_name(): + """ + Finds out OS name. For non-Linux system it will be just a plain + OS name (like FreeBSD), for Linux it will be "linux-", + where is the name of the distribution, as returned by + the first component of platform.linux_distribution. + + Return value will be all-lowercase to avoid confusion about + proper name capitalisation. + + """ + OS = platform.system().lower() + + if OS.startswith('linux'): + DISTRO = platform.linux_distribution()[0] + if DISTRO: + OS = OS + "-%s" % DISTRO.lower() + + return OS + +def get_os_sslcertfile_searchpath(): + """Returns search path for CA bundle for the current OS. + + We will return an iterable even if configuration has just + a single value: it is easier for our callers to be sure + that they can iterate over result. + + Returned value of None means that there is no search path + at all. + """ + + OS = get_os_name() + + l = None + if OS in __DEF_OS_LOCATIONS: + l = __DEF_OS_LOCATIONS[OS] + if not hasattr(l, '__iter__'): + l = (l, ) + return l + + +def get_os_sslcertfile(): + """ + Finds out the location for the distribution-specific + CA certificate file bundle. + + Returns the location of the file or None if there is + no known CA certificate file or all known locations + correspond to non-existing filesystem objects. + """ + + l = get_os_sslcertfile_searchpath() + if l == None: + return None + + for f in l: + assert (type(f) == type("")) + if os.path.exists(f) and \ + (os.path.isfile(f) or os.path.islink(f)): + return f + + return None diff --git a/offlineimap/utils/stacktrace.py b/offlineimap/utils/stacktrace.py new file mode 100644 index 0000000..7c885b0 --- /dev/null +++ b/offlineimap/utils/stacktrace.py @@ -0,0 +1,25 @@ +# Copyright 2013 Eygene A. Ryabinkin +# Functions to perform stack tracing (for multithreaded programs +# as well as for single-threaded ones). + +import sys +import threading +import traceback + + +def dump(out): + """ Dumps current stack trace into I/O object 'out' """ + id2name = {} + for th in threading.enumerate(): + id2name[th.ident] = th.name + n = 0 + for i, stack in sys._current_frames().items(): + out.write ("\n# Thread #%d (id=%d), %s\n" % \ + (n, i, id2name[i])) + n = n + 1 + for f, lno, name, line in traceback.extract_stack (stack): + out.write ('File: "%s", line %d, in %s' % \ + (f, lno, name)) + if line: + out.write (" %s" % (line.strip())) + out.write ("\n") diff --git a/scripts/get-repository.sh b/scripts/get-repository.sh new file mode 100755 index 0000000..a6dad95 --- /dev/null +++ b/scripts/get-repository.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Licence: this file is in the public deomain. +# +# Download and configure the repositories of the website or wiki. + +repository=$1 +github_remote=$2 + +# +# TODO +# +function final_note () { + cat < + $ cd ./$1 + $ git remote add myfork https://github.com//.git +EOF +} + +function setup () { + target_dir=$1 + remote_url=$2 + + # Adjust $PWD if necessary. + test -d scripts || cd .. + if test ! -d scripts + then + echo "cannot figure the correct workdir..." + exit 2 + fi + + if test -d $target_dir + then + echo "Directory '$target_dir' already exists..." + exit 3 + fi + + git clone "${remote_url}.git" "$1" + echo '' + if test $? -gt 0 + then + echo "Cannot fork $remote_url to $1" + exit 4 + fi +} + +function configure_website () { + renderer='./render.sh' + + echo "Found Github username: '$1'" + echo "If it's wrong, please fix the script ./website/render.sh" + + cd ./website + if test $? -eq 0 + then + sed -r -i -e "s,{{USERNAME}},$1," "$renderer" + cd .. + else + echo "ERROR: could not enter ./website. (?)" + fi +} + +function configure_wiki () { + : # noop +} + +test n$github_remote = 'n' && github_remote='origin' + +# Get Github username. +#offlineimap_url="$(git config --local --get remote.origin.url)" +offlineimap_url="$(git config --local --get remote.nicolas33.url)" +username=$(echo $offlineimap_url | sed -r -e 's,.*github.com.([^/]+)/.*,\1,') + + +case n$repository in + nwebsite) + upstream=https://github.com/OfflineIMAP/offlineimap.github.io + setup website "$upstream" + configure_website "$username" + final_note website "$upstream" + ;; + nwiki) + upstream=https://github.com/OfflineIMAP/offlineimap.wiki + setup wiki "$upstream" + configure_wiki + final_note wiki "$upstream" + ;; + *) + cat <] + +: The name of the Git repository of YOUR fork at Github. + Default: origin +EOF + exit 1 + ;; +esac + diff --git a/setup.py b/setup.py index c87077b..73db977 100644 --- a/setup.py +++ b/setup.py @@ -20,11 +20,35 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# END OF COPYRIGHT # - -from distutils.core import setup +import os +from distutils.core import setup, Command import offlineimap +import logging +from test.OLItest import TextTestRunner, TestLoader, OLITestLib + +class TestCommand(Command): + """runs the OLI testsuite""" + description = """Runs the test suite. In order to execute only a single + test, you could also issue e.g. 'python -m unittest + test.tests.test_01_basic.TestBasicFunctions.test_01_olistartup' on the + command line.""" + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + logging.basicConfig(format='%(message)s') + # set credentials and OfflineImap command to be executed: + OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') + suite = TestLoader().discover('./test/tests') + #TODO: failfast does not seem to exist in python2.6? + TextTestRunner(verbosity=2,failfast=True).run(suite) + setup(name = "offlineimap", version = offlineimap.__version__, @@ -33,9 +57,11 @@ setup(name = "offlineimap", author_email = offlineimap.__author_email__, url = offlineimap.__homepage__, packages = ['offlineimap', 'offlineimap.folder', - 'offlineimap.repository', 'offlineimap.ui'], + 'offlineimap.repository', 'offlineimap.ui', + 'offlineimap.utils'], scripts = ['bin/offlineimap'], license = offlineimap.__copyright__ + \ - ", Licensed under the GPL version 2" + ", Licensed under the GPL version 2", + cmdclass = { 'test': TestCommand} ) diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..08ea7df --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,2 @@ +credentials.conf +tmp_* \ No newline at end of file diff --git a/test/OLItest/TestRunner.py b/test/OLItest/TestRunner.py new file mode 100644 index 0000000..e5fc030 --- /dev/null +++ b/test/OLItest/TestRunner.py @@ -0,0 +1,256 @@ +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import imaplib +import unittest +import logging +import os +import re +import sys +import shutil +import subprocess +import tempfile +import random +random.seed() + +from offlineimap.CustomConfig import CustomConfigParser +from . import default_conf + + +class OLITestLib(): + cred_file = None + testdir = None + """Absolute path of the current temporary test directory""" + cmd = None + """command that will be executed to invoke offlineimap""" + + def __init__(self, cred_file = None, cmd='offlineimap'): + """ + + :param cred_file: file of the configuration + snippet for authenticating against the test IMAP server(s). + :param cmd: command that will be executed to invoke offlineimap""" + OLITestLib.cred_file = cred_file + if not os.path.isfile(cred_file): + raise UserWarning("Please copy 'credentials.conf.sample' to '%s' " + "and set your credentials there." % cred_file) + OLITestLib.cmd = cmd + + @classmethod + def create_test_dir(cls, suffix=''): + """Creates a test directory and places OLI config there + + Note that this is a class method. There can only be one test + directory at a time. OLITestLib is not suited for running + several tests in parallel. The user is responsible for + cleaning that up herself.""" + assert cls.cred_file != None + # creating temporary dir for testing in same dir as credentials.conf + cls.testdir = os.path.abspath( + tempfile.mkdtemp(prefix='tmp_%s_'%suffix, + dir=os.path.dirname(cls.cred_file))) + cls.write_config_file() + return cls.testdir + + @classmethod + def get_default_config(cls): + """Creates a default ConfigParser file and returns it + + The returned config can be manipulated and then saved with + write_config_file()""" + #TODO, only do first time and cache then for subsequent calls? + assert cls.cred_file != None + assert cls.testdir != None + config = CustomConfigParser() + config.readfp(default_conf) + default_conf.seek(0) # rewind config_file to start + config.read(cls.cred_file) + config.set("general", "metadata", cls.testdir) + return config + + @classmethod + def write_config_file(cls, config=None): + """Creates a OLI configuration file + + It is created in testdir (so create_test_dir has to be called + earlier) using the credentials information given (so they had + to be set earlier). Failure to do either of them will raise an + AssertionException. If config is None, a default one will be + used via get_default_config, otherwise it needs to be a config + object derived from that.""" + if config is None: + config = cls.get_default_config() + localfolders = os.path.join(cls.testdir, 'mail') + config.set("Repository Maildir", "localfolders", localfolders) + with open(os.path.join(cls.testdir, 'offlineimap.conf'), "wt") as f: + config.write(f) + + @classmethod + def delete_test_dir(cls): + """Deletes the current test directory + + The users is responsible for cleaning that up herself.""" + if os.path.isdir(cls.testdir): + shutil.rmtree(cls.testdir) + + @classmethod + def run_OLI(cls): + """Runs OfflineImap + + :returns: (rescode, stdout (as unicode)) + """ + try: + output = subprocess.check_output( + [cls.cmd, + "-c%s" % os.path.join(cls.testdir, 'offlineimap.conf')], + shell=False) + except subprocess.CalledProcessError as e: + return (e.returncode, e.output.decode('utf-8')) + return (0, output.decode('utf-8')) + + @classmethod + def delete_remote_testfolders(cls, reponame=None): + """Delete all INBOX.OLITEST* folders on the remote IMAP repository + + reponame: All on `reponame` or all IMAP-type repositories if None""" + config = cls.get_default_config() + if reponame: + sections = ['Repository {0}'.format(reponame)] + else: + sections = [r for r in config.sections() \ + if r.startswith('Repository')] + sections = filter(lambda s: \ + config.get(s, 'Type').lower() == 'imap', + sections) + for sec in sections: + # Connect to each IMAP repo and delete all folders + # matching the folderfilter setting. We only allow basic + # settings and no fancy password getting here... + # 1) connect and get dir listing + host = config.get(sec, 'remotehost') + user = config.get(sec, 'remoteuser') + passwd = config.get(sec, 'remotepass') + imapobj = imaplib.IMAP4(host) + imapobj.login(user, passwd) + res_t, data = imapobj.list() + assert res_t == 'OK' + dirs = [] + for d in data: + if d == '': + continue + if isinstance(d, tuple): + # literal (unquoted) + folder = b'"%s"' % d[1].replace('"', '\\"') + else: + m = re.search(br''' + [ ] # space + (?P + (?P"?) # starting quote + ([^"]|\\")* # a non-quote or a backslashded quote + (?P=quote))$ # ending quote + ''', d, flags=re.VERBOSE) + folder = bytearray(m.group('dir')) + if not m.group('quote'): + folder = '"%s"' % folder + #folder = folder.replace(br'\"', b'"') # remove quoting + dirs.append(folder) + # 2) filter out those not starting with INBOX.OLItest and del... + dirs = [d for d in dirs if d.startswith(b'"INBOX.OLItest') or d.startswith(b'"INBOX/OLItest')] + for folder in dirs: + res_t, data = imapobj.delete(folder) + assert res_t == 'OK', "Folder deletion of {0} failed with error"\ + ":\n{1} {2}".format(folder.decode('utf-8'), res_t, data) + imapobj.logout() + + @classmethod + def create_maildir(cls, folder): + """Create empty maildir 'folder' in our test maildir + + Does not fail if it already exists""" + assert cls.testdir != None + maildir = os.path.join(cls.testdir, 'mail', folder) + for subdir in ('','tmp','cur','new'): + try: + os.makedirs(os.path.join(maildir, subdir)) + except OSError as e: + if e.errno != 17: # 'already exists' is ok. + raise + + @classmethod + def delete_maildir(cls, folder): + """Delete maildir 'folder' in our test maildir + + Does not fail if not existing""" + assert cls.testdir != None + maildir = os.path.join(cls.testdir, 'mail', folder) + shutil.rmtree(maildir, ignore_errors=True) + + @classmethod + def create_mail(cls, folder, mailfile=None, content=None): + """Create a mail in maildir 'folder'/new + + Use default mailfilename if not given. + Use some default content if not given""" + assert cls.testdir != None + while True: # Loop till we found a unique filename + mailfile = '{0}:2,'.format(random.randint(0,999999999)) + mailfilepath = os.path.join(cls.testdir, 'mail', + folder, 'new', mailfile) + if not os.path.isfile(mailfilepath): + break + with open(mailfilepath,"wb") as mailf: + mailf.write(b'''From: test +Subject: Boo +Date: 1 Jan 1980 +To: test@offlineimap.org + +Content here.''') + + @classmethod + def count_maildir_mails(cls, folder): + """Returns the number of mails in maildir 'folder' + + Counting only those in cur&new (ignoring tmp).""" + assert cls.testdir != None + maildir = os.path.join(cls.testdir, 'mail', folder) + + boxes, mails = 0, 0 + for dirpath, dirs, files in os.walk(maildir, False): + if set(dirs) == set(['cur', 'new', 'tmp']): + # New maildir folder + boxes += 1 + #raise RuntimeError("%s is not Maildir" % maildir) + if dirpath.endswith(('/cur', '/new')): + mails += len(files) + return boxes, mails + + # find UID in a maildir filename + re_uidmatch = re.compile(',U=(\d+)') + + @classmethod + def get_maildir_uids(cls, folder): + """Returns a list of maildir mail uids, 'None' if no valid uid""" + assert cls.testdir != None + mailfilepath = os.path.join(cls.testdir, 'mail', folder) + assert os.path.isdir(mailfilepath) + ret = [] + for dirpath, dirs, files in os.walk(mailfilepath): + if not dirpath.endswith((os.path.sep + 'new', os.path.sep + 'cur')): + continue # only /new /cur are interesting + for file in files: + m = cls.re_uidmatch.search(file) + uid = m.group(1) if m else None + ret.append(uid) + return ret diff --git a/test/OLItest/__init__.py b/test/OLItest/__init__.py new file mode 100644 index 0000000..ca6ef61 --- /dev/null +++ b/test/OLItest/__init__.py @@ -0,0 +1,34 @@ +# OfflineImap test library +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +__all__ = ['OLITestLib', 'TextTestRunner','TestLoader'] + +__productname__ = 'OfflineIMAP Test suite' +__version__ = '0' +__copyright__ = "Copyright 2012- Sebastian Spaeth & contributors" +__author__ = 'Sebastian Spaeth' +__author_email__= 'Sebastian@SSpaeth.de' +__description__ = 'Moo' +__license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" +__homepage__ = "http://offlineimap.org" +banner = """%(__productname__)s %(__version__)s + %(__license__)s""" % locals() + +import unittest +from unittest import TestLoader, TextTestRunner +from .globals import default_conf +from .TestRunner import OLITestLib diff --git a/test/OLItest/globals.py b/test/OLItest/globals.py new file mode 100644 index 0000000..ba072e2 --- /dev/null +++ b/test/OLItest/globals.py @@ -0,0 +1,42 @@ +#Constants, that don't rely on anything else in the module +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +try: + from cStringIO import StringIO +except ImportError: #python3 + from io import StringIO + +default_conf=StringIO("""[general] +#will be set automatically +metadata = +accounts = test +ui = quiet + +[Account test] +localrepository = Maildir +remoterepository = IMAP + +[Repository Maildir] +Type = Maildir +# will be set automatically during tests +localfolders = + +[Repository IMAP] +type=IMAP +# Don't hammer the server with too many connection attempts: +maxconnections=1 +folderfilter= lambda f: f.startswith('INBOX.OLItest') or f.startswith('INBOX/OLItest') +""") diff --git a/test/README b/test/README new file mode 100644 index 0000000..a02e3b9 --- /dev/null +++ b/test/README @@ -0,0 +1,18 @@ +Documentation for the OfflineImap Test suite. + +How to run the tests +==================== + +- Copy the credentials.conf.sample to credentials.conf and insert + credentials for an IMAP account and a Gmail account. Delete the Gmail + section if you don't have a Gmail account. Do note, that the tests + will change the account and upload/delete/modify it's contents and + folder structure. So don't use a real used account here... + +- go to the top level dir (one above this one) and execute: + 'python setup.py test' + +System requirements +=================== + +This test suite depend on python>=2.7 to run out of the box. If you want to run this with python 2.6 you will need to install the backport from http://pypi.python.org/pypi/unittest2 instead. \ No newline at end of file diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/credentials.conf.sample b/test/credentials.conf.sample new file mode 100644 index 0000000..cd22f77 --- /dev/null +++ b/test/credentials.conf.sample @@ -0,0 +1,13 @@ +[Repository IMAP] +type = IMAP +remotehost = localhost +ssl = no +#sslcacertfile = +#cert_fingerprint = +remoteuser = user@domain +remotepass = SeKr3t + +[Repository Gmail] +type = Gmail +remoteuser = user@domain +remotepass = SeKr3t diff --git a/test/tests/__init__.py b/test/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/test/tests/test_00_globals.py b/test/tests/test_00_globals.py new file mode 100755 index 0000000..b4572f9 --- /dev/null +++ b/test/tests/test_00_globals.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# Copyright 2013 Eygene A. Ryabinkin + +from offlineimap import globals +import unittest + +class Opt: + def __init__(self): + self.one = "baz" + self.two = 42 + self.three = True + + +class TestOfflineimapGlobals(unittest.TestCase): + + @classmethod + def setUpClass(klass): + klass.o = Opt() + globals.set_options (klass.o) + + def test_initial_state(self): + for k in self.o.__dict__.keys(): + self.assertTrue(getattr(self.o, k) == + getattr(globals.options, k)) + + def test_object_changes(self): + self.o.one = "one" + self.o.two = 119 + self.o.three = False + return self.test_initial_state() + + def test_modification(self): + with self.assertRaises(AttributeError): + globals.options.two = True + + def test_deletion(self): + with self.assertRaises(RuntimeError): + del globals.options.three + + def test_nonexistent_key(self): + with self.assertRaises(AttributeError): + a = globals.options.nosuchoption + + def test_double_init(self): + with self.assertRaises(ValueError): + globals.set_options (True) + + +if __name__ == "__main__": + suite = unittest.TestLoader().loadTestsFromTestCase(TestOfflineimapGlobals) + unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/test/tests/test_00_imaputil.py b/test/tests/test_00_imaputil.py new file mode 100644 index 0000000..4e9d142 --- /dev/null +++ b/test/tests/test_00_imaputil.py @@ -0,0 +1,94 @@ +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import unittest +import logging + +from offlineimap import imaputil +from offlineimap.ui import UI_LIST, setglobalui +from offlineimap.CustomConfig import CustomConfigParser + +from test.OLItest import OLITestLib + +# Things need to be setup first, usually setup.py initializes everything. +# but if e.g. called from command line, we take care of default values here: +if not OLITestLib.cred_file: + OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') + +def setUpModule(): + logging.info("Set Up test module %s" % __name__) + tdir = OLITestLib.create_test_dir(suffix=__name__) + +def tearDownModule(): + logging.info("Tear Down test module") + # comment out next line to keep testdir after test runs. TODO: make nicer + OLITestLib.delete_test_dir() + +#Stuff that can be used +#self.assertEqual(self.seq, range(10)) +# should raise an exception for an immutable sequence +#self.assertRaises(TypeError, random.shuffle, (1,2,3)) +#self.assertTrue(element in self.seq) +#self.assertFalse(element in self.seq) + +class TestInternalFunctions(unittest.TestCase): + """While the other test files test OfflineImap as a program, these + tests directly invoke internal helper functions to guarantee that + they deliver results as expected""" + + @classmethod + def setUpClass(cls): + #This is run before all tests in this class + config= OLITestLib.get_default_config() + setglobalui(UI_LIST['quiet'](config)) + + def test_01_imapsplit(self): + """Test imaputil.imapsplit()""" + res = imaputil.imapsplit(b'(\\HasNoChildren) "." "INBOX.Sent"') + self.assertEqual(res, [b'(\\HasNoChildren)', b'"."', b'"INBOX.Sent"']) + + res = imaputil.imapsplit(b'"mo\\" o" sdfsdf') + self.assertEqual(res, [b'"mo\\" o"', b'sdfsdf']) + + def test_02_flagsplit(self): + """Test imaputil.flagsplit()""" + res = imaputil.flagsplit(b'(\\Draft \\Deleted)') + self.assertEqual(res, [b'\\Draft', b'\\Deleted']) + + res = imaputil.flagsplit(b'(FLAGS (\\Seen Old) UID 4807)') + self.assertEqual(res, [b'FLAGS', b'(\\Seen Old)', b'UID', b'4807']) + + def test_04_flags2hash(self): + """Test imaputil.flags2hash()""" + res = imaputil.flags2hash(b'(FLAGS (\\Seen Old) UID 4807)') + self.assertEqual(res, {b'FLAGS': b'(\\Seen Old)', b'UID': b'4807'}) + + def test_05_flagsimap2maildir(self): + """Test imaputil.flagsimap2maildir()""" + res = imaputil.flagsimap2maildir(b'(\\Draft \\Deleted)') + self.assertEqual(res, set(b'DT')) + + def test_06_flagsmaildir2imap(self): + """Test imaputil.flagsmaildir2imap()""" + res = imaputil.flagsmaildir2imap(set(b'DR')) + self.assertEqual(res, b'(\\Answered \\Draft)') + # test all possible flags + res = imaputil.flagsmaildir2imap(set(b'SRFTD')) + self.assertEqual(res, b'(\\Answered \\Deleted \\Draft \\Flagged \\Seen)') + + def test_07_uid_sequence(self): + """Test imaputil.uid_sequence()""" + res = imaputil.uid_sequence([1,2,3,4,5,10,12,13]) + self.assertEqual(res, b'1:5,10,12:13') diff --git a/test/tests/test_01_basic.py b/test/tests/test_01_basic.py new file mode 100644 index 0000000..629131c --- /dev/null +++ b/test/tests/test_01_basic.py @@ -0,0 +1,160 @@ +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import random +import unittest +import logging +import os, sys +from test.OLItest import OLITestLib + +# Things need to be setup first, usually setup.py initializes everything. +# but if e.g. called from command line, we take care of default values here: +if not OLITestLib.cred_file: + OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') + + +def setUpModule(): + logging.info("Set Up test module %s" % __name__) + tdir = OLITestLib.create_test_dir(suffix=__name__) + +def tearDownModule(): + logging.info("Tear Down test module") + OLITestLib.delete_test_dir() + +#Stuff that can be used +#self.assertEqual(self.seq, range(10)) +# should raise an exception for an immutable sequence +#self.assertRaises(TypeError, random.shuffle, (1,2,3)) +#self.assertTrue(element in self.seq) +#self.assertFalse(element in self.seq) + +class TestBasicFunctions(unittest.TestCase): + def setUp(self): + OLITestLib.delete_remote_testfolders() + + def tearDown(self): + OLITestLib.delete_remote_testfolders() + + def test_01_olistartup(self): + """Tests if OLI can be invoked without exceptions + + Cleans existing remote test folders. Then syncs all "OLItest* + (specified in the default config) to our local Maildir. The + result should be 0 folders and 0 mails.""" + code, res = OLITestLib.run_OLI() + self.assertEqual(res, "") + boxes, mails = OLITestLib.count_maildir_mails('') + self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0 " + "mails, but sync led to {0} folders and {1} mails".format( + boxes, mails)) + + def test_02_createdir(self): + """Create local 'OLItest 1', sync""" + OLITestLib.delete_maildir('') #Delete all local maildir folders + OLITestLib.create_maildir('INBOX.OLItest 1') + code, res = OLITestLib.run_OLI() + self.assertEqual(res, "") + boxes, mails = OLITestLib.count_maildir_mails('') + self.assertTrue((boxes, mails)==(1,0), msg="Expected 1 folders and 0 " + "mails, but sync led to {0} folders and {1} mails".format( + boxes, mails)) + + def test_03_createdir_quote(self): + """Create local 'OLItest "1"' maildir, sync + + Folder names with quotes used to fail and have been fixed, so + one is included here as a small challenge.""" + OLITestLib.delete_maildir('') #Delete all local maildir folders + OLITestLib.create_maildir('INBOX.OLItest "1"') + code, res = OLITestLib.run_OLI() + if 'unallowed folder' in res: + raise unittest.SkipTest("remote server doesn't handle quote") + self.assertEqual(res, "") + boxes, mails = OLITestLib.count_maildir_mails('') + self.assertTrue((boxes, mails)==(1,0), msg="Expected 1 folders and 0 " + "mails, but sync led to {0} folders and {1} mails".format( + boxes, mails)) + + def test_04_nametransmismatch(self): + """Create mismatching remote and local nametrans rules + + This should raise an error.""" + config = OLITestLib.get_default_config() + config.set('Repository IMAP', 'nametrans', + 'lambda f: f' ) + config.set('Repository Maildir', 'nametrans', + 'lambda f: f + "moo"' ) + OLITestLib.write_config_file(config) + code, res = OLITestLib.run_OLI() + #logging.warn("%s %s "% (code, res)) + # We expect an INFINITE FOLDER CREATION WARNING HERE.... + mismatch = "ERROR: INFINITE FOLDER CREATION DETECTED!" in res + self.assertEqual(mismatch, True, msg="Mismatching nametrans rules did " + "NOT trigger an 'infinite folder generation' error. Output was:\n" + "{0}".format(res)) + # Write out default config file again + OLITestLib.write_config_file() + + + def test_05_createmail(self): + """Create mail in OLItest 1, sync, wipe folder sync + + Currently, this will mean the folder will be recreated + locally. At some point when remote folder deletion is + implemented, this behavior will change.""" + OLITestLib.delete_remote_testfolders() + OLITestLib.delete_maildir('') #Delete all local maildir folders + OLITestLib.create_maildir('INBOX.OLItest') + OLITestLib.create_mail('INBOX.OLItest') + code, res = OLITestLib.run_OLI() + #logging.warn("%s %s "% (code, res)) + self.assertEqual(res, "") + boxes, mails = OLITestLib.count_maildir_mails('') + self.assertTrue((boxes, mails)==(1,1), msg="Expected 1 folders and 1 " + "mails, but sync led to {0} folders and {1} mails".format( + boxes, mails)) + # The local Mail should have been assigned a proper UID now, check! + uids = OLITestLib.get_maildir_uids('INBOX.OLItest') + self.assertFalse (None in uids, msg = "All mails should have been "+ \ + "assigned the IMAP's UID number, but {0} messages had no valid ID "\ + .format(len([None for x in uids if x==None]))) + + def test_06_createfolders(self): + """Test if createfolders works as expected + + Create a local Maildir, then sync with remote "createfolders" + disabled. Delete local Maildir and sync. We should have no new + local maildir then. TODO: Rewrite this test to directly test + and count the remote folders when the helper functions have + been written""" + config = OLITestLib.get_default_config() + config.set('Repository IMAP', 'createfolders', + 'False' ) + OLITestLib.write_config_file(config) + + # delete all remote and local testfolders + OLITestLib.delete_remote_testfolders() + OLITestLib.delete_maildir('') + OLITestLib.create_maildir('INBOX.OLItest') + code, res = OLITestLib.run_OLI() + #logging.warn("%s %s "% (code, res)) + self.assertEqual(res, "") + OLITestLib.delete_maildir('INBOX.OLItest') + code, res = OLITestLib.run_OLI() + self.assertEqual(res, "") + boxes, mails = OLITestLib.count_maildir_mails('') + self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0 " + "mails, but sync led to {} folders and {} mails".format( + boxes, mails)) diff --git a/test/tests/test_02_MappedIMAP.py b/test/tests/test_02_MappedIMAP.py new file mode 100644 index 0000000..05aa394 --- /dev/null +++ b/test/tests/test_02_MappedIMAP.py @@ -0,0 +1,71 @@ +# Copyright (C) 2012- Sebastian Spaeth & contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import random +import unittest +import logging +import os, sys +from test.OLItest import OLITestLib + +# Things need to be setup first, usually setup.py initializes everything. +# but if e.g. called from command line, we take care of default values here: +if not OLITestLib.cred_file: + OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') + + +def setUpModule(): + logging.info("Set Up test module %s" % __name__) + tdir = OLITestLib.create_test_dir(suffix=__name__) + +def tearDownModule(): + logging.info("Tear Down test module") + OLITestLib.delete_test_dir() + +#Stuff that can be used +#self.assertEqual(self.seq, range(10)) +# should raise an exception for an immutable sequence +#self.assertRaises(TypeError, random.shuffle, (1,2,3)) +#self.assertTrue(element in self.seq) +#self.assertFalse(element in self.seq) + +class TestBasicFunctions(unittest.TestCase): + #@classmethod + #def setUpClass(cls): + #This is run before all tests in this class + # cls._connection = createExpensiveConnectionObject() + + #@classmethod + #This is run after all tests in this class + #def tearDownClass(cls): + # cls._connection.destroy() + + # This will be run before each test + #def setUp(self): + # self.seq = range(10) + + def test_01_MappedImap(self): + """Tests if a MappedIMAP sync can be invoked without exceptions + + Cleans existing remote test folders. Then syncs all "OLItest* + (specified in the default config) to our local IMAP (Gmail). The + result should be 0 folders and 0 mails.""" + pass #TODO + #OLITestLib.delete_remote_testfolders() + #code, res = OLITestLib.run_OLI() + #self.assertEqual(res, "") + #boxes, mails = OLITestLib.count_maildir_mails('') + #self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0" + # "mails, but sync led to {} folders and {} mails".format( + # boxes, mails))