diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..b819e0a --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,28 @@ +> This v1.0 template stands in `.github/`. + +### General informations + +- OfflineIMAP version: +- server name or domain: +- CLI options: + +``` +Configuration file offlineimaprc goes here. REMOVE PRIVATE DATA. +``` + +``` +The pythonfile file goes here (if any). REMOVE PRIVATE DATA. +``` + + +### Log error + +``` +Logs go here. REMOVE PRIVATE DATA. +``` + +### Steps to reproduce the error + +- +- + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..f41dae2 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,29 @@ +> This v1.0 template stands in `.github/`. + +### Peer reviews + +Trick to [fetch the pull +request](https://help.github.com/articles/checking-out-pull-requests-locally): +there is a (read-only) `refs/pull/` namespace. + +``` bash +git fetch OFFICIAL_REPOSITORY_NAME pull/PULL_ID/head:LOCAL_BRANCH_NAME +``` + +### This PR + +> Add character x `[x]`. + +- [] I've read the [DCO](http://www.offlineimap.org/doc/dco.html). +- [] I've read the [Coding Guidelines](http://www.offlineimap.org/doc/CodingGuidelines.html) +- [] The relevant informations about the changes stands in the commit message, not here in the message of the pull request. +- [] Code changes follow the style of the files they change. +- [] Code is tested (provide details). + +### References + +- Issue #no_space + +### Additional information + + diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..995a1b9 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,18 @@ + +# Realistic Code of Conduct + +1. We mostly care about making our softwares better. + +2. Everybody is free to decide how to contribute. + +3. Free speech owns to anyone of us. + +4. Feel offended? This might be very well-deserved. + +5. We don't need a code of conduct imposed on us, thanks. + +6. Ignoring this Realistic Code of Conduct is welcome. + + diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9bc6f7f..ac0beff 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -7,11 +7,11 @@ .. _maintainers: https://github.com/OfflineIMAP/offlineimap/blob/next/MAINTAINERS.rst .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst -.. _Community's website: https://offlineimap.org -.. _APIs in OfflineIMAP: http://offlineimap.org/documentation.html#available-apis -.. _documentation: https://offlineimap.org/documentation.html -.. _Coding Guidelines: http://offlineimap.org/doc/CodingGuidelines.html -.. _Know the status of your patches: http://offlineimap.org/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission +.. _Community's website: http://www.offlineimap.org +.. _APIs in OfflineIMAP: http://www.offlineimap.org/documentation.html#available-apis +.. _documentation: http://www.offlineimap.org/documentation.html +.. _Coding Guidelines: http://www.offlineimap.org/doc/CodingGuidelines.html +.. _Know the status of your patches: http://www.offlineimap.org/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission ================= @@ -27,6 +27,15 @@ contributions. .. contents:: :depth: 3 +Submit issues +============= + +Issues are welcome to both Github_ and the `mailing list`_, at your own +convenience. + +You might help closing some issues, too. :-) + + For the imaptients ================== @@ -36,13 +45,6 @@ For the imaptients - All the `documentation`_ -Submit issues -============= - -Issues are welcome to both Github_ and the `mailing list`_, at your own -convenience. - - Community ========= diff --git a/Changelog.md b/Changelog.md index bda0b3d..8d49115 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,272 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} +### OfflineIMAP v6.7.0 (2016-03-10) + +#### Notes + +New stable release out! + +With the work of Ilias, maintainer at Debian, OfflineIMAP is learning a new CLI +option to help fixing filenames for the users using nametrans and updating from +versions prior to v6.3.5. Distribution maintainers might want to backport this +feature for their packaged versions out after v6.3.5. Have a look at commit +c84d23b65670f to know more. + +OfflineIMAP earns the slogan "Get the emails where you need them", authored by +Norbert Preining. + +Julien Danjou, the author of the book _The Hacker’s Guide To Python_, shared us +his screenshot of a running session of OfflineIMAP. + +I recently created rooms for chat sessions at Gitter. It appears to be really +cool, supports seamless authentication with a github account, persistent logs, +desktop/mobile clients and many more usefull features. Join us at Gitter! + +- https://gitter.im/OfflineIMAP/offlineimap [NEW] +- https://gitter.im/OfflineIMAP/imapfw [NEW] + +Now, the OfflineIMAP community has 2 official websites: + +- http://www.offlineimap.org (for offlineimap) +- http://imapfw.offlineimap.org (for imapfw) [NEW] + +The Twitter account was resurrected, too. Feel free to join us: + + https://twitter.com/OfflineIMAP + +Finally, the teams of the OfflineIMAP organization at Github were renewed to +facilitate the integration of new contributors and directly improve both the +documentation and the websites. + +As a side note, the [imapfw repository](https://github.com/OfflineIMAP/imapfw) +has now more than 50 stargazers. This is very encouraging. + +Thank you much everybody for your various contributions into OfflineIMAP! + +#### Authors + +- Ben Boeckel (1) +- Ebben Aries (1) +- Ilias Tsitsimpis (1) + +#### Features + +- Introduce a code of conduct. +- Add github templates. +- Change hard coding of AF_UNSPEC to user-defined address-families per repository. [Ebben Aries] +- Add documentation for the ipv6 configuration option. + +#### Fixes + +- Identify and fix messages with FMD5 inconsistencies. [Ilias Tsitsimpis] +- Curses, UIBase: remove references to __bigversion__. [Ben Boeckel] +- Sphinx doc: remove usage of __bigversion__. +- MANIFEST: exclude rfcs (used for Pypi packages). +- Changelog: fix typo. + +#### Changes + +- release.sh: move the authors section up. +- release.sh: add pypi instructions. +- MAINTAINERS: update. + + + + +### OfflineIMAP v6.7.0-rc2 (2016-02-22) + +#### Notes + +Learn to abruptly abort on multiple Ctrl+C. + +Some bugs got fixed. XOAUTH2 now honors the proxy configuration option. Error +message was improved when it fails to write a new mail in a local Maildir. + +I've enabled the hook for integration with Github. You'll get notifications on +updates of the master branch of the repository (mostly for new releases). I may +write some tweets about OfflineIMAP sometimes. + +#### Features + +- Abort after three Ctrl-C keystrokes. + +#### Fixes + +- Fix year of copyright. +- Versioning: avoid confusing pip by spliting out __version__ with __revision__. +- Fix: exceptions.OSError might not have attribute EEXIST defined. +- XOAUTH2 handler: urlopen with proxied socket. +- Manual: small grammar fix. +- Fix typos in offlineimap(1) manpage. + +#### Changes + +- Update links to the new URL www.offlineimap.org. + + +### OfflineIMAP v6.7.0-rc1 (2016-01-24) + +#### Notes + +Starting a new cycle with all EXPERIMENTAL and TESTING stuff marked stable. +Otherwise, not much exciting yet. There's pending work that would need some +love by contributors: + +- https://github.com/OfflineIMAP/offlineimap/issues/211 +- https://github.com/OfflineIMAP/offlineimap/pull/111 +- https://github.com/OfflineIMAP/offlineimap/issues/184 + +#### Features + +- Allow authorization via XOAUTH2 using access token. + +#### Fixes + +- Revert "Don't output initial blurb in "quiet" mode". +- Fix Changelog. + +#### Changes + +- Declare newmail_hook option stable. +- Declare utime_from_header option stable. +- Decode foldernames is removed EXPERIMENTAL flag. +- Declare XOAUTH2 stable. +- Declare tls_level option stable. +- Declare IMAP Keywords option stable. + + +### OfflineIMAP v6.6.1 (2015-12-28) + +#### Notes + +This is a very small new stable release for two fixes. + +Amending support for BINARY APPEND which is not correctly implemented. Also, +remove potential harms from dot files in a local maildir. + +#### Fixes + +- Bump imaplib2 from 2.53 to 2.52. Remove support for binary send. +- Ignore aloo dot files in the Maildir while scanning for mails. + + +### OfflineIMAP v6.6.0 (2015-12-05) + +#### Features + +- Maildir learns to mimic Dovecot's format of lower-case letters (a,b,c..) for + "custom flags" or user keywords. + +#### Fixes + +- Broken retry loop would break connection management. +- Replace rogue `print` statement by `self.ui.debug`. + +#### Changes + +- Bump imaplib2 from v2.52 to v2.53. +- Code cleanups. +- Add a full stack of all thread dump upon EXIT or KILL signal in thread debug + mode. + + +### OfflineIMAP v6.6.0-rc3 (2015-11-05) + +#### Notes + +Changes are slowing down and the code is under serious testing by some new +contributors. Everything expected at this time in the release cycle. Thanks to +them. + +SSL is now enabled by default to prevent from sending private data in clear +stream to the wild. + +#### Features + +- Add new config option `filename_use_mail_timestamp`. + +#### Fixes + +- Bump from imaplib2 v2.51 to v2.52. +- Minor fixes. + +#### Changes + +- Enable SSL by default. +- Fix: avoid writing password to log. +- offlineimap.conf: improve namtrans doc a bit. + + +### OfflineIMAP v6.6.0-rc2 (2015-10-15) + +#### Notes + +Interesting job was done in this release with 3 new features: + +- Support for XOAUTH2; +- New 'tls_level' configuration option to automatically discard insecure SSL protocols; +- New interface 'syslog' comes in, next to the -s CLI option. This allows better + integration with systemd. + +I won't merge big changes until the stable is out. IOW, you can seriously start +testing this rc2. + +#### Features + +- Add a new syslog ui. +- Introduce the 'tls_level' configuration option. +- Learn XOAUTH2 authentication (used by Gmail servers). +- Manual IDLE section improved (minor). + +#### Fixes + +- Configuration option utime_from_header handles out-of-bounds dates. +- offlineimap.conf: fix erroneous assumption about ssl23. +- Fix status code to reflect success or failure of a sync. +- contrib/release.sh: fix changelog edition. + +#### Changes + +- Bump imaplib2 from v2.48 to v2.51. +- README: new section status and future. +- Minor code cleanups. +- Makefile: improve building of targz. +- systemd: log to syslog rather than stderr for better integration. + + +### OfflineIMAP v6.6.0-rc1 (2015-09-28) + +#### Notes + +Let's go with a new release. + +Basic UTF support was implemented while it is still exeprimental. Use this with +care. OfflineIMAP can now send the logs to syslog and notify on new mail. + + +#### Features + +- logging: add a switch to log to syslog. +- Added the newmail_hook. +- utf-7 feature is set experimental. + +#### Fixes + +- offlineimap.conf: fix a typo in the new mail hook example. +- Fix language. +- Fix spelling inconsistency. +- offlineimap.conf: don't use quotes for sep option. +- man page: fingerprint can be used with SSL. +- fix #225 « Runonce (offlineimap -o) does not stop if autorefresh is declared in DEFAULT section ». +- CONTRIBUTING: fix links to offlineimap.org. + +#### Changes + +- Bump imaplib2 from 2.43 to 2.48 +- README: small improvements + + ### OfflineIMAP v6.5.7 (2015-05-15) diff --git a/MAINTAINERS.rst b/MAINTAINERS.rst index 2cff161..3341287 100644 --- a/MAINTAINERS.rst +++ b/MAINTAINERS.rst @@ -1,7 +1,7 @@ .. -*- coding: utf-8 -*- -Official maintainers -==================== +Maintainers +=========== Eygene Ryabinkin email: rea at freebsd.org @@ -15,15 +15,31 @@ Nicolas Sebrecht email: nicolas.s-dev at laposte.net github: nicolas33 -Mailing List maintainers -======================== -Eygene Ryabinkin - email: rea at freebsd.org +Github +------ -Sebastian Spaeth - email: sebastian at sspaeth.de +- Eygene Ryabinkin +- Sebastian Spaeth +- Nicolas Sebrecht -Nicolas Sebrecht - email: nicolas.s-dev at laposte.net +Mailing List +------------ + +- Eygene Ryabinkin +- Sebastian Spaeth +- Nicolas Sebrecht + + +Twitter +------- + +- Nicolas Sebrecht + + +Pypi +---- + +- Nicolas Sebrecht +- Sebastian Spaeth diff --git a/MANIFEST.in b/MANIFEST.in index 3a403a3..aac4358 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,9 @@ include Makefile include README.md include offlineimap.conf* include offlineimap.py +recursive-include contrib * recursive-include offlineimap *.py recursive-include bin * recursive-include docs * recursive-include test * +prune docs/rfcs diff --git a/Makefile b/Makefile index da5c660..5de3e00 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -VERSION=`./offlineimap.py --version` -TARGZ=offlineimap_$(VERSION).tar.gz +VERSION=$(shell ./offlineimap.py --version) +ABBREV=$(shell git log --format='%h' HEAD~1..) +TARGZ=offlineimap-$(VERSION)-$(ABBREV) SHELL=/bin/bash RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` @@ -30,12 +31,12 @@ build: clean: -python setup.py clean --all - -rm -f bin/offlineimapc + -rm -f bin/offlineimapc 2>/dev/null -find . -name '*.pyc' -exec rm -f {} \; -find . -name '*.pygc' -exec rm -f {} \; -find . -name '*.class' -exec rm -f {} \; -find . -name '.cache*' -exec rm -f {} \; - -rm -f manpage.links manpage.refs + -rm -f manpage.links manpage.refs 2>/dev/null -find . -name auth -exec rm -vf {}/password {}/username \; @$(MAKE) -C clean @@ -47,11 +48,7 @@ websitedoc: targz: ../$(TARGZ) ../$(TARGZ): - if ! pwd | grep -q "/offlineimap-$(VERSION)$$"; then \ - echo "Containing directory must be called offlineimap-$(VERSION)"; \ - exit 1; \ - fi; \ - pwd && cd .. && pwd && tar -zhcv --exclude '.git' --exclude 'website' --exclude 'wiki' -f $(TARGZ) offlineimap-$(VERSION) + cd .. && tar -zhcv --transform s,^offlineimap,$(TARGZ), -f $(TARGZ).tar.gz --exclude '*.pyc' offlineimap/{bin,Changelog.md,contrib,CONTRIBUTING.rst,COPYING,docs,MAINTAINERS.rst,MANIFEST.in,offlineimap,offlineimap.conf,offlineimap.conf.minimal,offlineimap.py,README.md,scripts,setup.py,test,TODO.rst} rpm: targz cd .. && sudo rpmbuild -ta $(TARGZ) diff --git a/README.md b/README.md index ae9c6b6..1dc84e0 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,46 @@ -[offlineimap]: https://github.com/OfflineIMAP/offlineimap -[website]: http://offlineimap.org +[offlineimap]: http://github.com/OfflineIMAP/offlineimap +[website]: http://www.offlineimap.org [wiki]: http://github.com/OfflineIMAP/offlineimap/wiki +[blog]: http://www.offlineimap.org/posts.html -# OfflineImap +# OfflineIMAP + +***Get the emails where you need them.*** ## Description OfflineIMAP is a software to dispose your e-mail mailbox(es) as a **local Maildir**. OfflineIMAP will synchronize both sides via *IMAP*. -The main downside about IMAP is that you have to **trust** your MAIL provider to -not loose your mails. This is not something impossible while not very common. +The main downside about IMAP is that you have to **trust** your email provider to +not lose your mails. This is not something impossible while not very common. With OfflineIMAP, you can download your Mailboxes and make you own backups of -the Maildir. +the [Maildir](https://en.wikipedia.org/wiki/Maildir). -This allows reading your mails while offline without the need for the mail -reader (MUA) to support IMAP disconnected operations. Need an attachement from a -message without internet? It's fine, the message is still there. +This allows reading your email while offline without the need for the mail +reader (MUA) to support IMAP disconnected operations. Need an attachment from a +message without internet connection? It's fine, the message is still there. + + +## Project status and future + +> As one of the maintainer of OfflineIMAP, I'd like to put my efforts into +> [imapfw](http://github.com/OfflineIMAP/imapfw). **imapfw** is a software in +> development that I intend to replace OfflineIMAP in the long term. +> +> That's why I'm not going to do development in OfflineIMAP. I continue to do +> the maintenance job in OfflineIMAP: fixing small bugs, (quick) +> reviewing/merging patches and rolling out new releases, but that's all. +> +> While I keep tracking issues for OfflineIMAP, you should not expect support +> much from me anymore. +> +> You won't be left at the side. OfflineIMAP's community is large enough so that +> you'll find people for most of your issues. +> +> Get news from the [blog][blog]. +> +> Nicolas Sebrecht. ,-) ## License @@ -31,17 +55,16 @@ GNU General Public License v2. * It is **flexible**. * It is **safe**. - ## Downloads -You should first check if your distribution already package OfflineIMAP for you. +You should first check if your distribution already packages OfflineIMAP for you. Downloads releases as [tarball or zipball](https://github.com/OfflineIMAP/offlineimap/tags). ## Feedbacks and contributions -**The user discussions, development, announces and all the exciting stuff take -place in the mailing list.** While not mandatory to send emails, you can +**The user discussions, development, announcements and all the exciting stuff take +place on the mailing list.** While not mandatory to send emails, you can [subscribe here](http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project). Bugs, issues and contributions can be requested to both the mailing list or the @@ -59,20 +82,21 @@ Bugs, issues and contributions can be requested to both the mailing list or the * Python v2.7 * Python SQlite (optional while recommended) +* Python json and urllib (used for XOAuth2 authentication) ## Documentation All the current and updated documentation is at the [community's website][website]. -### Dispose locally +### Read documentation locally -You might want to dispose the documentation locally. Get the sources of the website. -For the other documentations, run the approppriate make target: +You might want to read the documentation locally. Get the sources of the website. +For the other documentation, run the appropriate make target: ``` $ ./scripts/get-repository.sh website $ cd docs -$ make html # Require rst2html -$ make man # Require a2x -$ make api # Require sphinx +$ make html # Requires rst2html +$ make man # Requires a2x +$ make api # Requires sphinx ``` diff --git a/TODO.rst b/TODO.rst index 73f80da..cb10f2b 100644 --- a/TODO.rst +++ b/TODO.rst @@ -120,8 +120,4 @@ TODO list so don't matter much about that if you don't get the point or what could be done. - -* Support Python 3. - - * Support Unicode. diff --git a/contrib/release.sh b/contrib/release.sh index 72e9936..14d80f8 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -16,7 +16,7 @@ # TODO: move configuration out and source it. # TODO: implement rollback. -__VERSION__='v0.2' +__VERSION__='v0.3' SPHINXBUILD=sphinx-build @@ -29,6 +29,7 @@ CHANGELOG='Changelog.md' CACHEDIR='.git/offlineimap-release' WEBSITE='website' WEBSITE_LATEST="${WEBSITE}/_data/latest.yml" +ME='Nicolas Sebrecht' TMP_CHANGELOG_EXCERPT="${CACHEDIR}/changelog.excerpt.md" TMP_CHANGELOG_EXCERPT_OLD="${TMP_CHANGELOG_EXCERPT}.old" @@ -154,7 +155,19 @@ function update_offlineimap_version () { # function get_git_history () { debug 'in get_git_history' - git log --oneline "${1}.." | sed -r -e 's,^(.),\- \1,' + git log --format='- %h %s. [%aN]' --no-merges "${1}.." | \ + sed -r -e "s, \[${ME}\]$,," +} + + +# +# $1: previous version +# +function get_git_who () { + debug 'in get_git_who' + echo + git shortlog --no-merges -sn "${1}.." | \ + sed -r -e 's, +([0-9]+)\t(.*),- \2 (\1),' } @@ -178,8 +191,15 @@ function changelog_template () { #### Notes -// Add some notes. Good notes are about what was done in this release. -// HINT: explain big changes. +// Add some notes. Good notes are about what was done in this release from the +// bigger perspective. +// HINT: explain most important changes. + +#### Authors + +The authors of this release. + +// Use list syntax with '- ' #### Features @@ -193,8 +213,8 @@ function changelog_template () { // Use list syntax with '- ' -// The preformatted shortlog was added below. -// Make use of this to fill the sections 'Features' and 'Fixes' above. +// The preformatted log was added below. Make use of this to fill the sections +// above. EOF } @@ -213,6 +233,7 @@ function update_changelog () { then changelog_template "$1" > "$TMP_CHANGELOG_EXCERPT" get_git_history "$2" >> "$TMP_CHANGELOG_EXCERPT" + get_git_who "$2" >> "$TMP_CHANGELOG_EXCERPT" edit_file "the Changelog excerpt" $TMP_CHANGELOG_EXCERPT # Remove comments. @@ -231,12 +252,13 @@ function update_changelog () { # Check and edit Changelog. ask "Next step: you'll be asked to review the diff of $CHANGELOG" - action=$No - while test ! $action -eq $Yes + while true do git diff -- "$CHANGELOG" | less ask 'edit Changelog?' $CHANGELOG - action=$? + test ! $? -eq $Yes && break + # Asked to edit the Changelog; will loop again. + $EDITOR "$CHANGELOG" done } @@ -352,6 +374,9 @@ OfflineIMAP $1 is out. Downloads: http://github.com/OfflineIMAP/offlineimap/archive/${1}.tar.gz http://github.com/OfflineIMAP/offlineimap/archive/${1}.zip + +Pip: + pip install --user git+https://github.com/OfflineIMAP/offlineimap.git@${1} EOF } @@ -429,6 +454,17 @@ cat < master:master +- git push next:next +- git push $new_version +- python setup.py sdist && twine upload dist/* && rm -rf dist MANIFEST +- cd website +- git checkout master +- git merge $branch_name +- git push master:master +- cd .. +- git send-email $TMP_ANNOUNCE Have fun! ,-) EOF diff --git a/contrib/systemd/offlineimap.service b/contrib/systemd/offlineimap.service index f29f93c..8b77bc4 100644 --- a/contrib/systemd/offlineimap.service +++ b/contrib/systemd/offlineimap.service @@ -3,7 +3,7 @@ Description=Offlineimap Service [Service] Type=oneshot -ExecStart=/usr/bin/offlineimap -o +ExecStart=/usr/bin/offlineimap -o -u syslog [Install] WantedBy=mail.target diff --git a/contrib/systemd/offlineimap@.service b/contrib/systemd/offlineimap@.service index 7be965a..6730652 100644 --- a/contrib/systemd/offlineimap@.service +++ b/contrib/systemd/offlineimap@.service @@ -3,7 +3,7 @@ Description=Offlineimap Service for account %i [Service] Type=oneshot -ExecStart=/usr/bin/offlineimap -o -a %i +ExecStart=/usr/bin/offlineimap -o -a %i -u syslog [Install] WantedBy=mail.target diff --git a/docs/doc-src/conf.py b/docs/doc-src/conf.py index 0886d4a..95cfb51 100644 --- a/docs/doc-src/conf.py +++ b/docs/doc-src/conf.py @@ -18,7 +18,7 @@ import sys, os # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('../..')) -from offlineimap import __version__, __bigversion__, __author__, __copyright__ +from offlineimap import __version__, __author__, __copyright__ # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions @@ -50,7 +50,7 @@ copyright = __copyright__ # The short X.Y version. version = __version__ # The full version, including alpha/beta/rc tags. -release = __bigversion__ +release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index e923a64..5915936 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -1,5 +1,5 @@ .. OfflineImap documentation master file -.. _OfflineIMAP: http://offlineimap.org +.. _OfflineIMAP: http://www.offlineimap.org Welcome to OfflineIMAP's developer documentation diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 618f2ab..0c66489 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -77,7 +77,7 @@ amounts of data. This option implies the -1 option. Overrides the accounts section in the config file. + -Allows to specify a particular account or set of accounts to sync without +Allows one to specify a particular account or set of accounts to sync without having to edit the config file. @@ -105,6 +105,10 @@ included), implies the single-thread option -1. Send logs to . +-s:: + + Send logs to syslog. + -f :: @@ -145,7 +149,7 @@ option is ignored if maxage is set. + This overrides the default specified in the configuration file. The UI specified with -u will be forced to be used, even if checks determine that it -is not usable. Possible interface choices are: quiet, basic, ttyui, +is not usable. Possible interface choices are: quiet, basic, syslog, ttyui, blinkenlights, machineui. @@ -159,6 +163,20 @@ blinkenlights, machineui. This option is only applicable in non-verbose mode. +--migrate-fmd5-using-nametrans:: + Migrate FMD5 hashes from versions prior to 6.3.5. ++ +The way that FMD5 hashes are calculated was changed in version 6.3.5 (now using +the nametrans folder name) introducing a regression which may lead to +re-uploading all messages. Try and fix the above regression by calculating the +correct FMD5 values and renaming the corresponding messages. + +CAUTION: Since the FMD5 part of the filename changes, this may lead to UID +conflicts. Ensure to dispose a proper backup of both the cache and the Maildir +before running this fix as well as verify the results using the `--dry-run' +flag first. + + Synchronization Performance --------------------------- @@ -207,7 +225,7 @@ in between. 5. Turn off fsync. + In the [general] section you can set fsync to True or False. If you want to -play 110% safe and wait for all operations to hit the disk before continueing, +play 110% safe and wait for all operations to hit the disk before continuing, you can set this to True. If you set it to False, you lose some of that safety, trading it for speed. @@ -215,7 +233,7 @@ safety, trading it for speed. Upgrading from plain text to SQLite cache format ------------------------------------------------ -OfflineImap uses a cache to store the last know status of mails (flags etc). +OfflineImap uses a cache to store the last known status of mails (flags etc). Historically that has meant plain text files, but recently we introduced sqlite-based cache, which helps with performance and CPU usage on large @@ -259,8 +277,8 @@ out the connection that is used by default. + Unfortunately, by default we will not verify the certificate of an IMAP TLS/SSL server we connect to, so connecting by SSL is no guarantee against -man-in-the-middle attacks. While verifying a server certificate fingerprint is -being planned, it is not implemented yet. There is currently only one safe way +man-in-the-middle attacks. While verifying a server certificate checking the +fingerprint is recommended. There is currently only one safe way to ensure that you connect to the correct server in an encrypted manner: you can specify a 'sslcacertfile' setting in your repository section of offlineimap.conf pointing to a file that contains (among others) a CA @@ -340,6 +358,8 @@ Email will show up, but may not be processed until the next refresh cycle. - IMAP IDLE <-> IMAP IDLE doesn't work yet. + - IDLE might stop syncing on a system suspend/resume. + - IDLE may only work "once" per refresh. + If you encounter this bug, please send a report to the list! @@ -376,7 +396,7 @@ You should enable this option with a value like 10. * OfflineIMAP confused when mails change while in a sync. + -When OfflineIMAP is syncing, some events happening since the invokation on +When OfflineIMAP is syncing, some events happening since the invocation on remote or local side are badly handled. OfflineIMAP won't track for changes during the sync. @@ -422,4 +442,4 @@ See Also -------- offlineimapui(7), openssl(1), signal(7), sqlite3(1). - http://offlineimap.org + http://www.offlineimap.org diff --git a/docs/offlineimapui.txt b/docs/offlineimapui.txt index c087ece..2ea661b 100644 --- a/docs/offlineimapui.txt +++ b/docs/offlineimapui.txt @@ -127,6 +127,17 @@ It will output nothing except errors and serious warnings. Like Basic, this user interface is not capable of reading a password from the keyboard; account passwords must be specified using one of the configuration file options. + +Syslog +------ + +Syslog is designed for situations where OfflineIMAP is run as a daemon (e.g., +as a systemd --user service), but errors should be forwarded to the system log. +Like Basic, this user interface is not capable of reading a password from the +keyboard; account passwords must be specified using one of the configuration +file options. + + MachineUI --------- diff --git a/offlineimap.conf b/offlineimap.conf index 69ede92..6f05dcc 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -2,7 +2,7 @@ # This file documents *all* possible options and can be quite scary. # Looking for a quick start? Take a look at offlineimap.conf.minimal. -# More details can be found at http://offlineimap.org . +# More details can be found at http://www.offlineimap.org . ################################################## # Overview @@ -295,6 +295,17 @@ remoterepository = RemoteExample #postsynchook = notifysync.sh +# This option stands in the [Account Test] section. +# +# You can specify a newmail hook to execute an external command upon receipt +# of new mail in the INBOX. +# +# This example plays a sound file of your chosing when new mail arrives. +# +#newmail_hook = lambda: os.system("cvlc --play-and-stop --play-and-exit /path/to/sound/file.mp3" + +# " > /dev/null 2>&1") + + # This option stands in the [Account Test] section. # # OfflineImap caches the state of the synchronisation to e.g. be able to @@ -452,7 +463,9 @@ localfolders = ~/Test # ignored for IMAP repositories, as it is queried automatically. # Otherwise, default value is ".". # -#sep = "." +# Don't use quotes. +# +#sep = . # This option stands in the [Repository LocalExample] section. @@ -492,13 +505,58 @@ localfolders = ~/Test # file/message content. # # If enabled, this forbid the -q (quick mode) CLI option to work correctly. -# This option is still "TESTING" feature. # # Default: no. # #utime_from_header = no +# This option stands in the [Repository LocalExample] section. +# +# This option is similar to "utime_from_header" and could be use as a +# complementary feature to keep track of a message date. This option only +# makes sense for the Maildir type. +# +# By default each message is stored in a file which prefix is the fetch +# timestamp and an order rank such as "1446590057_0". In a multithreading +# environment message are fetched in a random order, then you can't trust +# the file name to sort your boxes. +# +# If set to "yes" the file name prefix if build on the message "Date" header +# (which should be present) or the "Received-date" if "Date" is not +# found. If neither "Received-date" nor "Date" is found, the current system +# date is used. Now you can quickly sort your messages using their file +# names. +# +# Used in combination with "utime_from_header" all your message would be in +# order with the correct mtime attribute. +# +#filename_use_mail_timestamp = no + +# This option stands in the [Repository LocalExample] section. +# +# Map IMAP [user-defined] keywords to lowercase letters, similar to Dovecot's +# format described in http://wiki2.dovecot.org/MailboxFormat/Maildir . This +# option makes sense for the Maildir type, only. +# +# Configuration example: +# customflag_x = some_keyword +# +# With the configuration example above enabled, all IMAP messages that have +# 'some_keyword' in their FLAGS field will have an 'x' in the flags part of the +# maildir filename: +# 1234567890.M20046P2137.mailserver,S=4542,W=4642:2,Sx +# +# Valid fields are customflag_[a-z], valid values are whatever the IMAP server +# allows. +# +# Comparison in offlineimap is case-sensitive. +# +#customflag_a = some_keyword +#customflag_b = $OtherKeyword +#customflag_c = NonJunk +#customflag_d = ToDo + [Repository GmailLocalExample] # This type of repository enables syncing of Gmail. All Maildir @@ -522,6 +580,18 @@ type = GmailMaildir type = IMAP +# This option stands in the [Repository RemoteExample] section. +# +# Configure which address family to use for the connection. If not specified, +# AF_UNSPEC is used as a fallback (default). +# +# AF_INET6: +#ipv6 = True +# +# AF_INET: +#ipv6 = False + + # These options stands in the [Repository RemoteExample] section. # # The following can fetch the account credentials via a python expression that @@ -622,15 +692,35 @@ remotehost = examplehost # This option stands in the [Repository RemoteExample] section. # -# SSL version (optional). +# Set SSL version to use (optional). # # It is best to leave this unset, in which case the correct version will be # automatically detected. In rare cases, it may be necessary to specify a -# particular version from: tls1, ssl2, ssl3, ssl23 (SSLv2 or SSLv3) +# particular version from: tls1, ssl2, ssl3, ssl23. +# +# ssl23 is the highest protocol version that both the client and server support. +# Despite the name, this option can select “TLS” protocols as well as “SSL”. +# +# See the configuration option tls_level to automatically disable insecure +# protocols. # #ssl_version = ssl23 +# This option stands in the [Repository RemoteExample] section. +# +# TLS support level (optional). +# +# Specify the level of support that should be allowed for this repository. +# Can be used to disallow insecure SSL versions as defined by IETF +# (see https://tools.ietf.org/html/rfc6176). +# +# Supported values are: +# tls_secure, tls_no_ssl, tls_compat (the default). +# +#tls_level = tls_compat + + # This option stands in the [Repository RemoteExample] section. # # Specify the port. If not specified, use a default port. @@ -673,9 +763,47 @@ remoteuser = username # limitations, if GSSAPI is set, it will be tried first, no matter where it was # specified in the list. # -#auth_mechanisms = GSSAPI, CRAM-MD5, PLAIN, LOGIN +#auth_mechanisms = GSSAPI, CRAM-MD5, XOAUTH2, PLAIN, LOGIN +# This option stands in the [Repository RemoteExample] section. +# +# XOAuth2 authentication (for instance, to use with Gmail). +# +# This option was tested on Gmail only, but should work +# with type = IMAP for compatible servers. +# +# Mandatory parameters are "oauth2_client_id", "oauth2_client_secret" and +# either "oauth2_refresh_token" or "oauth2_access_token". +# See below to learn how to get those. +# +# Specify the OAuth2 client id and secret to use for the connection.. +# Here's how to register an OAuth2 client for Gmail, as of 10-2-2016: +# - Go to the Google developer console +# https://console.developers.google.com/project +# - Create a new project +# - In API & Auth, select Credentials +# - Setup the OAuth Consent Screen +# - Then add Credentials of type OAuth 2.0 Client ID +# - Choose application type Other; type in a name for your client +# - You now have a client ID and client secret +# +#oauth2_client_id = YOUR_CLIENT_ID +#oauth2_client_secret = YOUR_CLIENT_SECRET + +# Specify the refresh token to use for the connection to the mail server. +# Here's an example of a way to get a refresh token: +# - Clone this project: https://github.com/google/gmail-oauth2-tools +# - Type the following command-line in a terminal and follow the instructions +# python python/oauth2.py --generate_oauth2_token \ +# --client_id=YOUR_CLIENT_ID --client_secret=YOUR_CLIENT_SECRET +# - Access token can be obtained using refresh token with command +# python python/oauth2.py --user=YOUR_EMAIL --client_id=YOUR_CLIENT_ID +# --client_secret=YOUR_CLIENT_SECRET --refresh_token=REFRESH_TOKEN +# +#oauth2_refresh_token = REFRESH_TOKEN +#oauth2_access_token = ACCESS_TOKEN + ########## Passwords # There are six ways to specify the password for the IMAP server: @@ -762,6 +890,21 @@ remoteuser = username #reference = Mail +# This option stands in the [Repository RemoteExample] section. +# +# IMAP defines an encoding for non-ASCII ("international") characters. Enable +# this option if you want to decode them to the nowadays ubiquitous UTF-8. +# +# Note that the IMAP 4rev1 specification (RFC 3501) allows both UTF-8 and +# modified UTF-7 folder names. +# +# WARNING: with this option enabled: +# - compatibility with any other version is NOT GUARANTED (including newer); +# - no support is provided. +# +#decodefoldernames = no + + # This option stands in the [Repository RemoteExample] section. # # In between synchronisations, OfflineIMAP can monitor mailboxes for new @@ -855,16 +998,17 @@ remoteuser = username # folders, UNLESS the second values are filtered out by folderfilter below. # Failure to follow this rule will result in undefined behavior. # -# See the user documentation for details and use cases. They are also online at: -# http://docs.offlineimap.org/en/latest/nametrans.html +# If you enable nametrans, you will likely need to set the reversed nametrans on +# the other side. See the user documentation for details and use cases. They +# are also online at: http://www.offlineimap.org/doc/nametrans.html # # This example below will remove "INBOX." from the leading edge of folders # (great for Courier IMAP users). # #nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) # -# Using Courier remotely and want to duplicate its mailbox naming -# locally? Try this: +# Using Courier remotely and want to duplicate its mailbox naming locally? Try +# this: # #nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 8b93c96..369a477 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,17 +1,16 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.7" -__revision__ = "" -__bigversion__ = __version__ + __revision__ -__copyright__ = "Copyright 2002-2015 John Goerzen & contributors" +# Expecting trailing "-rcN" or "" for stable releases. +__version__ = "6.7.0" +__copyright__ = "Copyright 2002-2016 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "offlineimap-project@lists.alioth.debian.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" __license__ = "Licensed under the GNU GPL v2 or any later version (with an OpenSSL exception)" -__bigcopyright__ = """%(__productname__)s %(__bigversion__)s +__bigcopyright__ = """%(__productname__)s %(__version__)s %(__license__)s""" % locals() -__homepage__ = "http://offlineimap.org" +__homepage__ = "http://www.offlineimap.org" banner = __bigcopyright__ diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 7957b83..14b0867 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -40,6 +40,11 @@ class BaseFolder(object): # Top level dir name is always '' self.root = None self.name = name if not name == self.getsep() else '' + self.newmail_hook = None + # Only set the newmail_hook if the IMAP folder is named 'INBOX' + if self.name == 'INBOX': + self.newmail_hook = repository.newmail_hook + self.have_newmail = False self.repository = repository self.visiblename = repository.nametrans(name) # In case the visiblename becomes '.' or '/' (top-level) we use @@ -55,6 +60,13 @@ class BaseFolder(object): self._utime_from_header = self.config.getdefaultboolean(repo, "utime_from_header", utime_from_header_global) + # Do we need to use mail timestamp for filename prefix? + filename_use_mail_timestamp_global = self.config.getdefaultboolean( + "general", "filename_use_mail_timestamp", False) + repo = "Repository " + repository.name + self._filename_use_mail_timestamp = self.config.getdefaultboolean(repo, + "filename_use_mail_timestamp", filename_use_mail_timestamp_global) + # Determine if we're running static or dynamic folder filtering # and check filtering status self._dynamic_folderfilter = self.config.getdefaultboolean( @@ -408,6 +420,11 @@ class BaseFolder(object): raise NotImplementedError + def getmessagekeywords(self, uid): + """Returns the keywords for the specified message.""" + + raise NotImplementedError + def savemessageflags(self, uid, flags): """Sets the specified message's flags to the given set. @@ -781,6 +798,9 @@ class BaseFolder(object): # Got new UID, change the local uid. # Save uploaded status in the statusfolder statusfolder.savemessage(new_uid, message, flags, rtime) + # Check whether the mail has been seen + if 'S' not in flags: + self.have_newmail = True elif new_uid == 0: # Message was stored to dstfolder, but we can't find it's UID # This means we can't link current message to the one created @@ -817,6 +837,9 @@ class BaseFolder(object): This function checks and protects us from action in dryrun mode.""" + # We have no new mail yet + self.have_newmail = False + threads = [] copylist = filter(lambda uid: not statusfolder.uidexists(uid), @@ -854,6 +877,11 @@ class BaseFolder(object): for thread in threads: thread.join() + # Execute new mail hook if we have new mail + if self.have_newmail: + if self.newmail_hook != None: + self.newmail_hook(); + def __syncmessagesto_delete(self, dstfolder, statusfolder): """Pass 2: Remove locally deleted messages on dst. @@ -880,6 +908,45 @@ class BaseFolder(object): return #don't delete messages in dry-run mode dstfolder.deletemessages(deletelist) + def combine_flags_and_keywords(self, uid, dstfolder): + """Combine the message's flags and keywords using the mapping for the + destination folder.""" + + # Take a copy of the message flag set, otherwise + # __syncmessagesto_flags() will fail because statusflags is actually a + # reference to selfflags (which it should not, but I don't have time to + # debug THAT). + selfflags = set(self.getmessageflags(uid)) + + try: + keywordmap = dstfolder.getrepository().getkeywordmap() + if keywordmap is None: + return selfflags + + knownkeywords = set(keywordmap.keys()) + + selfkeywords = self.getmessagekeywords(uid) + + if not knownkeywords >= selfkeywords: + #some of the message's keywords are not in the mapping, so + #skip them + + skipped_keywords = list(selfkeywords - knownkeywords) + selfkeywords &= knownkeywords + + self.ui.warn("Unknown keywords skipped: %s\n" + "You may want to change your configuration to include " + "those\n" % (skipped_keywords)) + + keywordletterset = set([keywordmap[keyw] for keyw in selfkeywords]) + + #add the mapped keywords to the list of message flags + selfflags |= keywordletterset + except NotImplementedError: + pass + + return selfflags + def __syncmessagesto_flags(self, dstfolder, statusfolder): """Pass 3: Flag synchronization. @@ -902,13 +969,13 @@ class BaseFolder(object): if uid < 0 or not dstfolder.uidexists(uid): continue - selfflags = self.getmessageflags(uid) - if statusfolder.uidexists(uid): statusflags = statusfolder.getmessageflags(uid) else: statusflags = set() + selfflags = self.combine_flags_and_keywords(uid, dstfolder) + addflags = selfflags - statusflags delflags = statusflags - selfflags diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 3d83b91..ee52aed 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -72,11 +72,7 @@ class GmailFolder(IMAPFolder): (probably severity MESSAGE) if e.g. no message with this UID could be found. """ - imapobj = self.imapserver.acquireconnection() - try: - data = self._fetch_from_imap(imapobj, str(uid), 2) - finally: - self.imapserver.releaseconnection(imapobj) + data = self._fetch_from_imap(str(uid), 2) # data looks now e.g. #[('320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....')] diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 9ed7fec..5a26051 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -251,13 +251,22 @@ class IMAPFolder(BaseFolder): uid = long(options['UID']) self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) + keywords = imaputil.flagsimap2keywords(options['FLAGS']) rtime = imaplibutil.Internaldate2epoch(messagestr) - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, + 'keywords': keywords} self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) def dropmessagelistcache(self): self.messagelist = {} + # Interface from BaseFolder + def getvisiblename(self): + vname = super(IMAPFolder, self).getvisiblename() + if self.repository.getdecodefoldernames(): + return imaputil.decode_mailbox_name(vname) + return vname + # Interface from BaseFolder def getmessagelist(self): return self.messagelist @@ -266,18 +275,14 @@ class IMAPFolder(BaseFolder): def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body). - After this function all CRLFs will be transformed to '\n'. + After this function all CRLFs will be transformed to '\n'. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with this UID could be found. """ - imapobj = self.imapserver.acquireconnection() - try: - data = self._fetch_from_imap(imapobj, str(uid), 2) - finally: - self.imapserver.releaseconnection(imapobj) + data = self._fetch_from_imap(str(uid), 2) # data looks now e.g. [('320 (UID 17061 BODY[] # {2565}','msgbody....')] we only asked for one message, @@ -302,6 +307,10 @@ class IMAPFolder(BaseFolder): def getmessageflags(self, uid): return self.messagelist[uid]['flags'] + # Interface from BaseFolder + def getmessagekeywords(self, uid): + return self.messagelist[uid]['keywords'] + def __generate_randomheader(self, content): """Returns a unique X-OfflineIMAP header @@ -667,7 +676,7 @@ class IMAPFolder(BaseFolder): return uid - def _fetch_from_imap(self, imapobj, uids, retry_num=1): + def _fetch_from_imap(self, uids, retry_num=1): """Fetches data from IMAP server. Arguments: @@ -677,22 +686,37 @@ class IMAPFolder(BaseFolder): Returns: data obtained by this query.""" - query = "(%s)"% (" ".join(self.imap_query)) - fails_left = retry_num # retry on dropped connection - while fails_left: - try: - imapobj.select(self.getfullname(), readonly = True) - res_type, data = imapobj.uid('fetch', uids, query) - fails_left = 0 - except imapobj.abort as e: - # Release dropped connection, and get a new one - self.imapserver.releaseconnection(imapobj, True) - imapobj = self.imapserver.acquireconnection() - self.ui.error(e, exc_info()[2]) - fails_left -= 1 - # self.ui.error() will show the original traceback - if not fails_left: - raise e + imapobj = self.imapserver.acquireconnection() + try: + query = "(%s)"% (" ".join(self.imap_query)) + fails_left = retry_num ## retry on dropped connection + while fails_left: + try: + imapobj.select(self.getfullname(), readonly = True) + res_type, data = imapobj.uid('fetch', uids, query) + break + except imapobj.abort as e: + fails_left -= 1 + # self.ui.error() will show the original traceback + if fails_left <= 0: + message = ("%s, while fetching msg %r in folder %r." + " Max retry reached (%d)"% + (e, uids, self.name, retry_num)) + severity = OfflineImapError.ERROR.MESSAGE + raise OfflineImapError(message, + OfflineImapError.ERROR.MESSAGE) + # Release dropped connection, and get a new one + self.imapserver.releaseconnection(imapobj, True) + imapobj = self.imapserver.acquireconnection() + self.ui.error("%s. While fetching msg %r in folder %r." + " Retrying (%d/%d)"% + (e, uids, self.name, retry_num - fails_left, retry_num)) + finally: + # The imapobj here might be different than the one created before + # the ``try`` clause. So please avoid transforming this to a nice + # ``with`` without taking this into account. + self.imapserver.releaseconnection(imapobj) + if data == [None] or res_type != 'OK': #IMAP server says bad request or UID does not exist severity = OfflineImapError.ERROR.MESSAGE diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 3f5c071..77a774b 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -38,22 +38,20 @@ re_uidmatch = re.compile(',U=(\d+)') # Find a numeric timestamp in a string (filename prefix) re_timestampmatch = re.compile('(\d+)'); -timeseq = 0 -lasttime = 0 +timehash = {} timelock = Lock() -def _gettimeseq(): - global lasttime, timeseq, timelock +def _gettimeseq(date=None): + global timehash, timelock timelock.acquire() try: - thistime = long(time.time()) - if thistime == lasttime: - timeseq += 1 - return (thistime, timeseq) + if date is None: + date = long(time.time()) + if timehash.has_key(date): + timehash[date] += 1 else: - lasttime = thistime - timeseq = 0 - return (thistime, timeseq) + timehash[date] = 0 + return (date, timehash[date]) finally: timelock.release() @@ -137,9 +135,7 @@ class MaildirFolder(BaseFolder): uid = long(uidmatch.group(1)) flagmatch = self.re_flagmatch.search(filename) if flagmatch: - # Filter out all lowercase (custom maildir) flags. We don't - # handle them yet. - flags = set((c for c in flagmatch.group(1) if not c.islower())) + flags = set((c for c in flagmatch.group(1))) return prefix, uid, fmd5, flags def _scanfolder(self, min_date=None, min_uid=None): @@ -151,7 +147,7 @@ class MaildirFolder(BaseFolder): with similar UID's (e.g. the UID was reassigned much later). Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F - (flagged). + (flagged), plus lower-case letters for custom flags. :returns: dict that can be used as self.messagelist. """ @@ -167,6 +163,8 @@ class MaildirFolder(BaseFolder): date_excludees = {} for dirannex, filename in files: + if filename.startswith('.'): + continue # Ignore dot files. # We store just dirannex and filename, ie 'cur/123...' filepath = os.path.join(dirannex, filename) # Check maxsize if this message should be considered. @@ -269,14 +267,14 @@ class MaildirFolder(BaseFolder): filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) - def new_message_filename(self, uid, flags=set()): + def new_message_filename(self, uid, flags=set(), date=None): """Creates a new unique Maildir filename :param uid: The UID`None`, or a set of maildir flags :param flags: A set of maildir flags :returns: String containing unique message filename""" - timeval, timeseq = _gettimeseq() + timeval, timeseq = _gettimeseq(date) return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \ (timeval, timeseq, os.getpid(), socket.gethostname(), uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) @@ -294,7 +292,8 @@ class MaildirFolder(BaseFolder): that was created.""" tmpname = os.path.join('tmp', filename) - # open file and write it out + # Open file and write it out. + # XXX: why do we need to loop 7 times? tries = 7 while tries: tries = tries - 1 @@ -303,6 +302,8 @@ class MaildirFolder(BaseFolder): os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666) break except OSError as e: + if not hasattr(e, 'EEXIST'): + raise if e.errno == e.EEXIST: if tries: time.sleep(0.23) @@ -346,13 +347,43 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - messagename = self.new_message_filename(uid, flags) + + # use the mail timestamp given by either Date or Delivery-date mail + # headers. + message_timestamp = None + if self._filename_use_mail_timestamp: + try: + message_timestamp = emailutil.get_message_date(content, 'Date') + if message_timestamp is None: + # Give a try with Delivery-date + date = emailutil.get_message_date(content, 'Delivery-date') + except: + # This should never happen + from email.Parser import Parser + from offlineimap.ui import getglobalui + datestr = Parser().parsestr(content, True).get("Date") + ui = getglobalui() + ui.warn("UID %d has invalid date %s: %s\n" + "Not using message timestamp as file prefix" % (uid, datestr, e)) + # No need to check if date is None here since it would + # be overridden by _gettimeseq. + messagename = self.new_message_filename(uid, flags, date=message_timestamp) tmpname = self.save_to_tmp_file(messagename, content) if self.utime_from_header: - date = emailutil.get_message_date(content, 'Date') - if date != None: - os.utime(os.path.join(self.getfullname(), tmpname), (date, date)) + try: + date = emailutil.get_message_date(content, 'Date') + if date is not None: + os.utime(os.path.join(self.getfullname(), tmpname), + (date, date)) + # In case date is wrongly so far into the future as to be > max int32 + except Exception as e: + from email.Parser import Parser + from offlineimap.ui import getglobalui + datestr = Parser().parsestr(content, True).get("Date") + ui = getglobalui() + ui.warn("UID %d has invalid date %s: %s\n" + "Not changing file modification time" % (uid, datestr, e)) self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags @@ -386,8 +417,7 @@ class MaildirFolder(BaseFolder): if flags != self.messagelist[uid]['flags']: # Flags have actually changed, construct new filename Strip - # off existing infostring (possibly discarding small letter - # flags that dovecot uses TODO) + # off existing infostring infomatch = self.re_flagmatch.search(filename) if infomatch: filename = filename[:-len(infomatch.group())] #strip off @@ -455,3 +485,37 @@ class MaildirFolder(BaseFolder): os.unlink(filepath) # Yep -- return. del(self.messagelist[uid]) + + def migratefmd5(self, dryrun=False): + """Migrate FMD5 hashes from versions prior to 6.3.5 + + :param dryrun: Run in dry run mode + :type fix: Boolean + :return: None + """ + oldfmd5 = md5(self.name).hexdigest() + msglist = self._scanfolder() + for mkey, mvalue in msglist.iteritems(): + filename = os.path.join(self.getfullname(), mvalue['filename']) + match = re.search("FMD5=([a-fA-F0-9]+)", filename) + if match is None: + self.ui.debug("maildir", + "File `%s' doesn't have an FMD5 assigned" + % filename) + elif match.group(1) == oldfmd5: + self.ui.info("Migrating file `%s' to FMD5 `%s'" + % (filename, self._foldermd5)) + if not dryrun: + newfilename = filename.replace( + "FMD5=" + match.group(1), "FMD5=" + self._foldermd5) + try: + os.rename(filename, newfilename) + except OSError as e: + raise OfflineImapError( + "Can't rename file '%s' to '%s': %s" % ( + filename, newfilename, e[1]), + OfflineImapError.ERROR.FOLDER), None, exc_info()[2] + elif match.group(1) != self._foldermd5: + self.ui.warn(("Inconsistent FMD5 for file `%s':" + " Neither `%s' nor `%s' found") + % (filename, oldfmd5, self._foldermd5)) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py old mode 100644 new mode 100755 index eeb4792..61878f6 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.43" +__version__ = "2.52" __release__ = "2" -__revision__ = "43" +__revision__ = "52" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -46,20 +46,27 @@ Fix for offlineimap "indexerror: string index out of range" bug provided by Eyge Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014. Conversion to Python3 provided by F. Malina February 2015. Fix for READ-ONLY error from multiple EXAMINE/SELECT calls by Pierre-Louis Bonicoli March 2015. -Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli March 2015.""" +Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli March 2015. +Fix for correct byte encoding for _CRAM_MD5_AUTH taken from python3.5 imaplib.py June 2015. +Fix for correct Python 3 exception handling by Tobias Brink August 2015. +Fix to allow interruptible IDLE command by Tim Peoples September 2015. +Add support for TLS levels by Ben Boeckel September 2015. +Fix for shutown exception by Sebastien Gross November 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" import binascii, errno, os, random, re, select, socket, sys, time, threading, zlib -try: - import queue # py3 +if bytes != str: + # Python 3, but NB assumes strings in all I/O + # for backwards compatibility with python 2 usage. + import queue string_types = str -except ImportError: - import Queue as queue # py2 +else: + import Queue as queue string_types = basestring - + threading.TIMEOUT_MAX = 9223372036854.0 select_module = select @@ -77,6 +84,10 @@ READ_SIZE = 32768 # Consume all available in socke DFLT_DEBUG_BUF_LVL = 3 # Level above which the logging output goes directly to stderr +TLS_SECURE = "tls_secure" # Recognised TLS levels +TLS_NO_SSL = "tls_no_ssl" +TLS_COMPAT = "tls_compat" + AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first # Commands @@ -179,7 +190,7 @@ class Request(object): def get_response(self, exc_fmt=None): self.callback = None if __debug__: self.parent._log(3, '%s:%s.ready.wait' % (self.name, self.tag)) - self.ready.wait() + self.ready.wait(threading.TIMEOUT_MAX) if self.aborted is not None: typ, val = self.aborted @@ -319,6 +330,7 @@ class IMAP4(object): self.compressor = None # COMPRESS/DEFLATE if not None self.decompressor = None + self._tls_established = False # Create unique tag for this session, # and compile tagged response matcher. @@ -380,7 +392,7 @@ class IMAP4(object): # request and store CAPABILITY response. try: - self.welcome = self._request_push(tag='continuation').get_response('IMAP4 protocol error: %s')[1] + self.welcome = self._request_push(name='welcome', tag='continuation').get_response('IMAP4 protocol error: %s')[1] if self._get_untagged_response('PREAUTH'): self.state = AUTH @@ -441,19 +453,22 @@ class IMAP4(object): af, socktype, proto, canonname, sa = res try: s = socket.socket(af, socktype, proto) - except socket.error as msg: + except socket.error as m: + msg = m continue try: for i in (0, 1): try: s.connect(sa) break - except socket.error as msg: + except socket.error as m: + msg = m if len(msg.args) < 2 or msg.args[0] != errno.EINTR: raise else: raise socket.error(msg) - except socket.error as msg: + except socket.error as m: + msg = m s.close() continue break @@ -465,40 +480,60 @@ class IMAP4(object): def ssl_wrap_socket(self): - # Allow sending of keep-alive messages - seems to prevent some servers - # from closing SSL, leading to deadlocks. - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - try: import ssl + + TLS_MAP = {} + if hasattr(ssl, "PROTOCOL_TLSv1_2"): # py3 + TLS_MAP[TLS_SECURE] = { + "tls1_2": ssl.PROTOCOL_TLSv1_2, + "tls1_1": ssl.PROTOCOL_TLSv1_1, + } + else: + TLS_MAP[TLS_SECURE] = {} + TLS_MAP[TLS_NO_SSL] = TLS_MAP[TLS_SECURE].copy() + TLS_MAP[TLS_NO_SSL].update({ + "tls1": ssl.PROTOCOL_TLSv1, + }) + TLS_MAP[TLS_COMPAT] = TLS_MAP[TLS_NO_SSL].copy() + TLS_MAP[TLS_COMPAT].update({ + "ssl23": ssl.PROTOCOL_SSLv23, + None: ssl.PROTOCOL_SSLv23, + }) + if hasattr(ssl, "PROTOCOL_SSLv3"): # Might not be available. + TLS_MAP[TLS_COMPAT].update({ + "ssl3": ssl.PROTOCOL_SSLv3 + }) + if self.ca_certs is not None: cert_reqs = ssl.CERT_REQUIRED else: cert_reqs = ssl.CERT_NONE - if self.ssl_version == "tls1": - ssl_version = ssl.PROTOCOL_TLSv1 - elif self.ssl_version == "ssl2": - ssl_version = ssl.PROTOCOL_SSLv2 - elif self.ssl_version == "ssl3": - ssl_version = ssl.PROTOCOL_SSLv3 - elif self.ssl_version == "ssl23" or self.ssl_version is None: - ssl_version = ssl.PROTOCOL_SSLv23 - else: - raise socket.sslerror("Invalid SSL version requested: %s", self.ssl_version) + if self.tls_level not in TLS_MAP: + raise RuntimeError("unknown tls_level: %s" % self.tls_level) + + if self.ssl_version not in TLS_MAP[self.tls_level]: + raise socket.sslerror("Invalid SSL version '%s' requested for tls_version '%s'" % (self.ssl_version, self.tls_level)) + + ssl_version = TLS_MAP[self.tls_level][self.ssl_version] self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version) ssl_exc = ssl.SSLError self.read_fd = self.sock.fileno() except ImportError: # No ssl module, and socket.ssl has no fileno(), and does not allow certificate verification - raise socket.sslerror("imaplib2 SSL mode does not work without ssl module") + raise socket.sslerror("imaplib SSL mode does not work without ssl module") if self.cert_verify_cb is not None: cert_err = self.cert_verify_cb(self.sock.getpeercert(), self.host) if cert_err: raise ssl_exc(cert_err) + # Allow sending of keep-alive messages - seems to prevent some servers + # from closing SSL, leading to deadlocks. + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + def start_compressing(self): @@ -534,16 +569,23 @@ class IMAP4(object): data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: - self.sock.sendall(bytes(data, 'utf8')) - else: - self.sock.sendall(data) + data = bytes(data, 'ASCII') + + self.sock.sendall(data) def shutdown(self): """shutdown() Close I/O established in "open".""" - self.sock.close() + try: + self.sock.shutdown(socket.SHUT_RDWR) + except Exception as e: + # The server might already have closed the connection + if e.errno != errno.ENOTCONN: + raise + finally: + self.sock.close() def socket(self): @@ -881,7 +923,9 @@ class IMAP4(object): def _CRAM_MD5_AUTH(self, challenge): """Authobject to use with CRAM-MD5 authentication.""" import hmac - return self.user + " " + hmac.HMAC(self.password, challenge).hexdigest() + pwd = (self.password.encode('ASCII') if isinstance(self.password, str) + else self.password) + return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest() def logout(self, **kw): @@ -1065,8 +1109,8 @@ class IMAP4(object): return self._simple_command(name, sort_criteria, charset, *search_criteria, **kw) - def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", **kw): - """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23") + def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", tls_level=TLS_COMPAT, **kw): + """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", tls_level="tls_compat") Start TLS negotiation as per RFC 2595.""" name = 'STARTTLS' @@ -1074,7 +1118,7 @@ class IMAP4(object): if name not in self.capabilities: raise self.abort('TLS not supported by server') - if hasattr(self, '_tls_established') and self._tls_established: + if self._tls_established: raise self.abort('TLS session already established') # Must now shutdown reader thread after next response, and restart after changing read_fd @@ -1102,6 +1146,7 @@ class IMAP4(object): self.ca_certs = ca_certs self.cert_verify_cb = cert_verify_cb self.ssl_version = ssl_version + self.tls_level = tls_level try: self.ssl_wrap_socket() @@ -1229,14 +1274,17 @@ class IMAP4(object): self.commands_lock.release() - if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%s"]' % (typ, len(urd)-1, dat)) + if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%.80s"]' % (typ, len(urd)-1, dat)) def _check_bye(self): bye = self._get_untagged_response('BYE', leave=True) if bye: - raise self.abort(bye[-1]) + if str != bytes: + raise self.abort(bye[-1].decode('ASCII', 'replace')) + else: + raise self.abort(bye[-1]) def _checkquote(self, arg): @@ -1297,13 +1345,13 @@ class IMAP4(object): self.commands_lock.release() if need_event: if __debug__: self._log(3, 'sync command %s waiting for empty commands Q' % name) - self.state_change_free.wait() + self.state_change_free.wait(threading.TIMEOUT_MAX) if __debug__: self._log(3, 'sync command %s proceeding' % name) if self.state not in Commands[name][CMD_VAL_STATES]: self.literal = None - raise self.error('command %s illegal in state %s' - % (name, self.state)) + raise self.error('command %s illegal in state %s, only allowed in states %s' + % (name, self.state, ', '.join(Commands[name][CMD_VAL_STATES]))) self._check_bye() @@ -1316,7 +1364,7 @@ class IMAP4(object): while self._get_untagged_response(typ): continue - if self._get_untagged_response('READ-ONLY', leave=True) and not self.is_readonly: + if not self.is_readonly and self._get_untagged_response('READ-ONLY', leave=True): self.literal = None raise self.readonly('mailbox status changed to READ-ONLY') @@ -1348,7 +1396,7 @@ class IMAP4(object): return rqb # Must setup continuation expectancy *before* ouq.put - crqb = self._request_push(tag='continuation') + crqb = self._request_push(name=name, tag='continuation') self.ouq.put(rqb) @@ -1373,7 +1421,7 @@ class IMAP4(object): if literator is not None: # Need new request for next continuation response - crqb = self._request_push(tag='continuation') + crqb = self._request_push(name=name, tag='continuation') if __debug__: self._log(4, 'write literal size %s' % len(literal)) crqb.data = '%s%s' % (literal, CRLF) @@ -1402,7 +1450,7 @@ class IMAP4(object): def _command_completer(self, cb_arg_list): # Called for callback commands - (response, cb_arg, error) = cb_arg_list + response, cb_arg, error = cb_arg_list rqb, kw = cb_arg rqb.callback = kw['callback'] rqb.callback_arg = kw.get('cb_arg') @@ -1413,13 +1461,17 @@ class IMAP4(object): return bye = self._get_untagged_response('BYE', leave=True) if bye: - rqb.abort(self.abort, bye[-1]) + if str != bytes: + rqb.abort(self.abort, bye[-1].decode('ASCII', 'replace')) + else: + rqb.abort(self.abort, bye[-1]) return typ, dat = response if typ == 'BAD': if __debug__: self._print_log() rqb.abort(self.error, '%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) return + if __debug__: self._log(4, '_command_completer(%s, %s, None) = %s' % (response, cb_arg, rqb.tag)) if 'untagged_response' in kw: response = self._untagged_response(typ, dat, kw['untagged_response']) rqb.deliver(response) @@ -1463,7 +1515,7 @@ class IMAP4(object): if not leave: del self.untagged_responses[i] self.commands_lock.release() - if __debug__: self._log(5, '_get_untagged_response(%s) => %s' % (name, dat)) + if __debug__: self._log(5, '_get_untagged_response(%s) => %.80s' % (name, dat)) return dat self.commands_lock.release() @@ -1605,11 +1657,17 @@ class IMAP4(object): self.commands_lock.acquire() rqb = self.tagged_commands.pop(name) if not self.tagged_commands: + need_event = True + else: + need_event = False + self.commands_lock.release() + + if __debug__: self._log(4, '_request_pop(%s, %s) [%d] = %s' % (name, data, len(self.tagged_commands), rqb.tag)) + rqb.deliver(data) + + if need_event: if __debug__: self._log(3, 'state_change_free.set') self.state_change_free.set() - self.commands_lock.release() - if __debug__: self._log(4, '_request_pop(%s, %s) = %s' % (name, data, rqb.tag)) - rqb.deliver(data) def _request_push(self, tag=None, name=None, **kw): @@ -1645,7 +1703,7 @@ class IMAP4(object): if not dat: break data += dat - if __debug__: self._log(4, '_untagged_response(%s, ?, %s) => %s' % (typ, name, data)) + if __debug__: self._log(4, '_untagged_response(%s, ?, %s) => %.80s' % (typ, name, data)) return typ, data @@ -1762,7 +1820,10 @@ class IMAP4(object): } return ' '.join([PollErrors[s] for s in PollErrors.keys() if (s & state)]) - line_part = '' + if bytes != str: + line_part = b'' + else: + line_part = '' poll = select.poll() @@ -1774,7 +1835,7 @@ class IMAP4(object): while not (terminate or self.Terminate): if self.state == LOGOUT: - timeout = 1 + timeout = 10 else: timeout = read_poll_timeout try: @@ -1802,11 +1863,11 @@ class IMAP4(object): if bytes != str: stop = data.find(b'\n', start) if stop < 0: - line_part += data[start:].decode() + line_part += data[start:] break stop += 1 line_part, start, line = \ - '', stop, line_part + data[start:stop].decode() + b'', stop, (line_part + data[start:stop]).decode(errors='ignore') else: stop = data.find('\n', start) if stop < 0: @@ -1846,7 +1907,10 @@ class IMAP4(object): if __debug__: self._log(1, 'starting using select') - line_part = '' + if bytes != str: + line_part = b'' + else: + line_part = '' rxzero = 0 terminate = False @@ -1878,11 +1942,11 @@ class IMAP4(object): if bytes != str: stop = data.find(b'\n', start) if stop < 0: - line_part += data[start:].decode() + line_part += data[start:] break stop += 1 line_part, start, line = \ - '', stop, line_part + data[start:stop].decode() + b'', stop, (line_part + data[start:stop]).decode(errors='ignore') else: stop = data.find('\n', start) if stop < 0: @@ -2035,7 +2099,7 @@ class IMAP4_SSL(IMAP4): """IMAP4 client class over SSL connection Instantiate with: - IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None) + IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None, tls_level="tls_compat") host - host's name (default: localhost); port - port number (default: standard IMAP4 SSL port); @@ -2043,23 +2107,30 @@ class IMAP4_SSL(IMAP4): certfile - PEM formatted certificate chain file (default: None); ca_certs - PEM formatted certificate chain file used to validate server certificates (default: None); cert_verify_cb - function to verify authenticity of server certificates (default: None); - ssl_version - SSL version to use (default: "ssl23", choose from: "tls1","ssl2","ssl3","ssl23"); + ssl_version - SSL version to use (default: "ssl23", choose from: "tls1","ssl3","ssl23"); debug - debug level (default: 0 - no debug); debug_file - debug stream (default: sys.stderr); identifier - thread identifier prefix (default: host); timeout - timeout in seconds when expecting a command response. debug_buf_lvl - debug level at which buffering is turned off. + tls_level - TLS security level (default: "tls_compat"). + + The recognized values for tls_level are: + tls_secure: accept only TLS protocols recognized as "secure" + tls_no_ssl: disable SSLv2 and SSLv3 support + tls_compat: accept all SSL/TLS versions For more documentation see the docstring of the parent class IMAP4. """ - def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): + def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None, tls_level=TLS_COMPAT): self.keyfile = keyfile self.certfile = certfile self.ca_certs = ca_certs self.cert_verify_cb = cert_verify_cb self.ssl_version = ssl_version + self.tls_level = tls_level IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout, debug_buf_lvl) @@ -2100,27 +2171,18 @@ class IMAP4_SSL(IMAP4): data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: - if hasattr(self.sock, "sendall"): - self.sock.sendall(bytes(data, 'utf8')) - else: - dlen = len(data) - while dlen > 0: - sent = self.sock.write(bytes(data, 'utf8')) - if sent == dlen: - break # avoid copy - data = data[sent:] - dlen = dlen - sent + data = bytes(data, 'utf8') + + if hasattr(self.sock, "sendall"): + self.sock.sendall(data) else: - if hasattr(self.sock, "sendall"): - self.sock.sendall(data) - else: - dlen = len(data) - while dlen > 0: - sent = self.sock.write(data) - if sent == dlen: - break # avoid copy - data = data[sent:] - dlen = dlen - sent + dlen = len(data) + while dlen > 0: + sent = self.sock.write(data) + if sent == dlen: + break # avoid copy + data = data[sent:] + dlen = dlen - sent def ssl(self): @@ -2195,9 +2257,9 @@ class IMAP4_stream(IMAP4): data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: - self.writefile.write(bytes(data, 'utf8')) - else: - self.writefile.write(data) + data = bytes(data, 'utf8') + + self.writefile.write(data) self.writefile.flush() @@ -2372,8 +2434,14 @@ if __name__ == '__main__': # To test: invoke either as 'python imaplib2.py [IMAP4_server_hostname]', # or as 'python imaplib2.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' - # or as 'python imaplib2.py -l "keyfile[:certfile]" [IMAP4_SSL_server_hostname]' + # or as 'python imaplib2.py -l keyfile[:certfile]|: [IMAP4_SSL_server_hostname]' + # + # Option "-d " turns on debugging (use "-d 5" for everything) # Option "-i" tests that IDLE is interruptible + # Option "-p " allows alternate ports + + if not __debug__: + raise ValueError('Please run without -O') import getopt, getpass @@ -2446,10 +2514,10 @@ if __name__ == '__main__': ) - AsyncError = None + AsyncError, M = None, None def responder(cb_arg_list): - (response, cb_arg, error) = cb_arg_list + response, cb_arg, error = cb_arg_list global AsyncError cmd, args = cb_arg if error is not None: @@ -2491,7 +2559,7 @@ if __name__ == '__main__': if keyfile is not None: if not keyfile: keyfile = None if not certfile: certfile = None - M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) + M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, ssl_version="tls1", debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl, tls_level="tls_no_ssl") elif stream_command: M = IMAP4_stream(stream_command, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) else: @@ -2569,7 +2637,7 @@ if __name__ == '__main__': print('All tests OK.') except: - if not idle_intr or not 'IDLE' in M.capabilities: + if not idle_intr or M is None or not 'IDLE' in M.capabilities: print('Tests failed.') if not debug: diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 952404a..9b1095b 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -74,7 +74,7 @@ class UsefulIMAPMixIn(object): """open_socket() Open socket choosing first address family available.""" msg = (-1, 'could not open socket') - for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): + for res in socket.getaddrinfo(self.host, self.port, self.af, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: # use socket of our own, possiblly socksified socket. @@ -175,6 +175,9 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): """Improved version of imaplib.IMAP4_SSL overriding select().""" def __init__(self, *args, **kwargs): + if "af" in kwargs: + self.af = kwargs['af'] + del kwargs['af'] if "use_socket" in kwargs: self.socket = kwargs['use_socket'] del kwargs['use_socket'] @@ -209,6 +212,9 @@ class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): """Improved version of imaplib.IMAP4 overriding select().""" def __init__(self, *args, **kwargs): + if "af" in kwargs: + self.af = kwargs['af'] + del kwargs['af'] if "use_socket" in kwargs: self.socket = kwargs['use_socket'] del kwargs['use_socket'] diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index f0b2248..a604993 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -19,6 +19,11 @@ from threading import Lock, BoundedSemaphore, Thread, Event, currentThread import hmac import socket import base64 + +import json +import urllib + +import socket import time import errno from sys import exc_info @@ -76,6 +81,13 @@ class IMAPServer: self.goodpassword = None self.usessl = repos.getssl() + self.useipv6 = repos.getipv6() + if self.useipv6 == True: + self.af = socket.AF_INET6 + elif self.useipv6 == False: + self.af = socket.AF_INET + else: + self.af = socket.AF_UNSPEC self.hostname = \ None if self.preauth_tunnel else repos.gethost() self.port = repos.getport() @@ -88,6 +100,13 @@ class IMAPServer: self.__verifycert = None # disable cert verification self.fingerprint = repos.get_ssl_fingerprint() self.sslversion = repos.getsslversion() + self.tlslevel = repos.gettlslevel() + + self.oauth2_refresh_token = repos.getoauth2_refresh_token() + self.oauth2_access_token = repos.getoauth2_access_token() + self.oauth2_client_id = repos.getoauth2_client_id() + self.oauth2_client_secret = repos.getoauth2_client_secret() + self.oauth2_request_url = repos.getoauth2_request_url() self.delim = None self.root = None @@ -195,11 +214,44 @@ class IMAPServer: authz = self.user_identity NULL = u'\x00' retval = NULL.join((authz, authc, passwd)).encode('utf-8') - self.ui.debug('imap', '__plainhandler: returning %s' % retval) + logsafe_retval = NULL.join((authz, authc, "(passwd hidden for log)")).encode('utf-8') + self.ui.debug('imap', '__plainhandler: returning %s' % logsafe_retval) return retval - # XXX: describe function + def __xoauth2handler(self, response): + if self.oauth2_refresh_token is None and self.oauth2_access_token is None: + return None + + if self.oauth2_access_token is None: + # need to move these to config + # generate new access token + params = {} + params['client_id'] = self.oauth2_client_id + params['client_secret'] = self.oauth2_client_secret + params['refresh_token'] = self.oauth2_refresh_token + params['grant_type'] = 'refresh_token' + + self.ui.debug('imap', 'xoauth2handler: url "%s"' % self.oauth2_request_url) + self.ui.debug('imap', 'xoauth2handler: params "%s"' % params) + + original_socket = socket.socket + socket.socket = self.proxied_socket + try: + response = urllib.urlopen(self.oauth2_request_url, urllib.urlencode(params)).read() + finally: + socket.socket = original_socket + + resp = json.loads(response) + self.ui.debug('imap', 'xoauth2handler: response "%s"' % resp) + self.oauth2_access_token = resp['access_token'] + + self.ui.debug('imap', 'xoauth2handler: access_token "%s"' % self.oauth2_access_token) + auth_string = 'user=%s\1auth=Bearer %s\1\1' % (self.username, self.oauth2_access_token) + #auth_string = base64.b64encode(auth_string) + self.ui.debug('imap', 'xoauth2handler: returning "%s"' % auth_string) + return auth_string + def __gssauth(self, response): data = base64.b64encode(response) try: @@ -283,6 +335,10 @@ class IMAPServer: imapobj.authenticate('PLAIN', self.__plainhandler) return True + def __authn_xoauth2(self, imapobj): + imapobj.authenticate('XOAUTH2', self.__xoauth2handler) + return True + def __authn_login(self, imapobj): # Use LOGIN command, unless LOGINDISABLED is advertized # (per RFC 2595) @@ -314,6 +370,7 @@ class IMAPServer: auth_methods = { "GSSAPI": (self.__authn_gssapi, False, True), "CRAM-MD5": (self.__authn_cram_md5, True, True), + "XOAUTH2": (self.__authn_xoauth2, True, True), "PLAIN": (self.__authn_plain, True, True), "LOGIN": (self.__authn_login, True, False), } @@ -437,6 +494,8 @@ class IMAPServer: timeout=socket.getdefaulttimeout(), fingerprint=self.fingerprint, use_socket=self.proxied_socket, + tls_level=self.tlslevel, + af=self.af, ) else: self.ui.connecting(self.hostname, self.port) @@ -444,6 +503,7 @@ class IMAPServer: self.hostname, self.port, timeout=socket.getdefaulttimeout(), use_socket=self.proxied_socket, + af=self.af, ) if not self.preauth_tunnel: diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index f1f287b..6a18732 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -25,6 +25,9 @@ from offlineimap.ui import getglobalui # Message headers that use space as the separator (for label storage) SPACE_SEPARATED_LABEL_HEADERS = ('X-Label', 'Keywords') +# Find the modified UTF-7 shifts of an international mailbox name. +MUTF7_SHIFT_RE = re.compile(r'&[^-]*-|\+') + def __debug(*args): msg = [] @@ -192,6 +195,14 @@ def flagsimap2maildir(flagstring): retval.add(maildirflag) return retval +def flagsimap2keywords(flagstring): + """Convert string '(\\Draft \\Deleted somekeyword otherkeyword)' into a + keyword set (somekeyword otherkeyword).""" + + imapflagset = set(flagstring[1:-1].split()) + serverflagset = set([flag for (flag, c) in flagmap]) + return imapflagset - serverflagset + def flagsmaildir2imap(maildirflaglist): """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'.""" @@ -328,3 +339,28 @@ def labels_from_header(header_name, header_value): return labels + +def decode_mailbox_name(name): + """Decodes a modified UTF-7 mailbox name. + + If the string cannot be decoded, it is returned unmodified. + + See RFC 3501, sec. 5.1.3. + + Arguments: + - name: string, possibly encoded with modified UTF-7 + + Returns: decoded UTF-8 string. + """ + def demodify(m): + s = m.group() + if s == '+': + return '+-' + return '+' + s[1:-1].replace(',', '/') + '-' + + ret = MUTF7_SHIFT_RE.sub(demodify, name) + + try: + return ret.decode('utf-7').encode('utf-8') + except UnicodeEncodeError: + return name diff --git a/offlineimap/init.py b/offlineimap/init.py index a48c152..636db5d 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -25,11 +25,15 @@ import logging from optparse import OptionParser import offlineimap -from offlineimap import accounts, threadutil, syncmaster +from offlineimap import accounts, threadutil, syncmaster, folder from offlineimap import globals from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser from offlineimap.utils import stacktrace +from offlineimap.repository import Repository + +import traceback +import collections class OfflineImap: @@ -47,11 +51,13 @@ class OfflineImap: options, args = self.__parse_cmd_options() if options.diagnostics: self.__serverdiagnostics(options) + elif options.migrate_fmd5: + self.__migratefmd5(options) else: - self.__sync(options) + return self.__sync(options) def __parse_cmd_options(self): - parser = OptionParser(version=offlineimap.__bigversion__, + parser = OptionParser(version=offlineimap.__version__, description="%s.\n\n%s" % (offlineimap.__copyright__, offlineimap.__license__)) @@ -89,6 +95,11 @@ class OfflineImap: parser.add_option("-l", dest="logfile", metavar="FILE", help="log to FILE") + parser.add_option("-s", + action="store_true", dest="syslog", + default=False, + help="log to syslog") + parser.add_option("-f", dest="folders", metavar="folder1[,folder2[,...]]", help="only sync the specified folders") @@ -110,7 +121,11 @@ class OfflineImap: parser.add_option("-u", dest="interface", help="specifies an alternative user interface" - " (quiet, basic, ttyui, blinkenlights, machineui)") + " (quiet, basic, syslog, ttyui, blinkenlights, machineui)") + + parser.add_option("--migrate-fmd5-using-nametrans", + action="store_true", dest="migrate_fmd5", default=False, + help="migrate FMD5 hashes from versions prior to 6.3.5") (options, args) = parser.parse_args() globals.set_options (options) @@ -196,6 +211,10 @@ class OfflineImap: if options.logfile: self.ui.setlogfile(options.logfile) + #set up syslog + if options.syslog: + self.ui.setup_sysloghandler() + #welcome blurb self.ui.init_banner() @@ -217,6 +236,9 @@ class OfflineImap: imaplib.Debug = 5 if options.runonce: + # Must kill the possible default option + if config.has_option('DEFAULT', 'autorefresh'): + config.remove_option('DEFAULT', 'autorefresh') # FIXME: spaghetti code alert! for section in accounts.getaccountlist(config): config.remove_option('Account ' + section, "autorefresh") @@ -260,6 +282,42 @@ class OfflineImap: self.config = config return (options, args) + def __dumpstacks(self, context=1, sighandler_deep=2): + """ Signal handler: dump a stack trace for each existing thread.""" + + currentThreadId = threading.currentThread().ident + + def unique_count(l): + d = collections.defaultdict(lambda: 0) + for v in l: + d[tuple(v)] += 1 + return list((k, v) for k, v in d.iteritems()) + + stack_displays = [] + for threadId, stack in sys._current_frames().items(): + stack_display = [] + for filename, lineno, name, line in traceback.extract_stack(stack): + stack_display.append(' File: "%s", line %d, in %s' + % (filename, lineno, name)) + if line: + stack_display.append(" %s" % (line.strip())) + if currentThreadId == threadId: + stack_display = stack_display[:- (sighandler_deep * 2)] + stack_display.append(' => Stopped to handle current signal. ') + stack_displays.append(stack_display) + stacks = unique_count(stack_displays) + self.ui.debug('thread', "** Thread List:\n") + for stack, times in stacks: + if times == 1: + msg = "%s Thread is at:\n%s\n" + else: + msg = "%s Threads are at:\n%s\n" + self.ui.debug('thread', msg % (times, '\n'.join(stack[- (context * 2):]))) + + self.ui.debug('thread', "Dumped a total of %d Threads." % + len(sys._current_frames().keys())) + + def __sync(self, options): """Invoke the correct single/multithread syncing @@ -309,10 +367,19 @@ class OfflineImap: getglobalui().warn("Terminating NOW (this may "\ "take a few seconds)...") accounts.Account.set_abort_event(self.config, 3) + if 'thread' in self.ui.debuglist: + self.__dumpstacks(5) + + # Abort after three Ctrl-C keystrokes + self.num_sigterm += 1 + if self.num_sigterm >= 3: + getglobalui().warn("Signaled thrice. Aborting!") + sys.exit(1) elif sig == signal.SIGQUIT: stacktrace.dump(sys.stderr) os.abort() + self.num_sigterm = 0 signal.signal(signal.SIGHUP, sig_handler) signal.signal(signal.SIGUSR1, sig_handler) signal.signal(signal.SIGUSR2, sig_handler) @@ -339,11 +406,13 @@ class OfflineImap: offlineimap.mbnames.write(True) self.ui.terminate() + return 0 except (SystemExit): raise except Exception as e: self.ui.error(e) self.ui.terminate() + return 1 def __sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread @@ -365,3 +434,21 @@ class OfflineImap: for account in allaccounts: if account.name not in activeaccounts: continue account.serverdiagnostics() + + def __migratefmd5(self, options): + activeaccounts = self.config.get("general", "accounts") + if options.accounts: + activeaccounts = options.accounts + activeaccounts = activeaccounts.replace(" ", "") + activeaccounts = activeaccounts.split(",") + allaccounts = accounts.AccountListGenerator(self.config) + + for account in allaccounts: + if account.name not in activeaccounts: + continue + localrepo = Repository(account, 'local') + if localrepo.getfoldertype() != folder.Maildir.MaildirFolder: + continue + folders = localrepo.getfolders() + for f in folders: + f.migratefmd5(options.dryrun) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 0cf44f8..8634628 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -48,6 +48,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): self.folderfilter = lambda foldername: 1 self.folderincludes = [] self.foldersort = None + self.newmail_hook = None if self.config.has_option(self.getsection(), 'nametrans'): self.nametrans = self.localeval.eval( self.getconf('nametrans'), {'re': re}) @@ -132,6 +133,9 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getsep(self): raise NotImplementedError + def getkeywordmap(self): + raise NotImplementedError + def should_sync_folder(self, fname): """Should this folder be synced?""" diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 2e23e62..a45d274 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -29,6 +29,8 @@ class GmailRepository(IMAPRepository): # Gmail IMAP server port PORT = 993 + OAUTH2_URL = 'https://accounts.google.com/o/oauth2/token' + def __init__(self, reposname, account): """Initialize a GmailRepository object.""" # Enforce SSL usage @@ -49,6 +51,20 @@ class GmailRepository(IMAPRepository): self._host = GmailRepository.HOSTNAME return self._host + def getoauth2_request_url(self): + """Return the server name to connect to. + + Gmail implementation first checks for the usual IMAP settings + and falls back to imap.gmail.com if not specified.""" + + url = super(GmailRepository, self).getoauth2_request_url() + if url is None: + # Nothing was configured, cache and return hardcoded one. + self._oauth2_request_url = GmailRepository.OAUTH2_URL + else: + self._oauth2_request_url = url + return self._oauth2_request_url + def getport(self): return GmailRepository.PORT diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 68c8e33..60d5a08 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -34,8 +34,14 @@ class IMAPRepository(BaseRepository): BaseRepository.__init__(self, reposname, account) # self.ui is being set by the BaseRepository self._host = None + self._oauth2_request_url = None self.imapserver = imapserver.IMAPServer(self) self.folders = None + # Only set the newmail_hook in an IMAP repository. + if self.config.has_option(self.getsection(), 'newmail_hook'): + self.newmail_hook = self.localeval.eval( + self.getconf('newmail_hook')) + if self.getconf('sep', None): self.ui.info("The 'sep' setting is being ignored for IMAP " "repository '%s' (it's autodetected)"% self) @@ -125,12 +131,12 @@ class IMAPRepository(BaseRepository): return self.getconf('remote_identity', default=None) def get_auth_mechanisms(self): - supported = ["GSSAPI", "CRAM-MD5", "PLAIN", "LOGIN"] + supported = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"] # Mechanisms are ranged from the strongest to the # weakest ones. # TODO: we need DIGEST-MD5, it must come before CRAM-MD5 # TODO: due to the chosen-plaintext resistance. - default = ["GSSAPI", "CRAM-MD5", "PLAIN", "LOGIN"] + default = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"] mechs = self.getconflist('auth_mechanisms', r',\s*', default) @@ -188,8 +194,11 @@ class IMAPRepository(BaseRepository): return self.getconfint('remoteport', None) + def getipv6(self): + return self.getconfboolean('ipv6', None) + def getssl(self): - return self.getconfboolean('ssl', 0) + return self.getconfboolean('ssl', 1) def getsslclientcert(self): xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] @@ -240,6 +249,9 @@ class IMAPRepository(BaseRepository): raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) return cacertfile + def gettlslevel(self): + return self.getconf('tls_level', 'tls_compat') + def getsslversion(self): return self.getconf('ssl_version', None) @@ -252,6 +264,30 @@ class IMAPRepository(BaseRepository): value = self.getconf('cert_fingerprint', "") return [f.strip().lower() for f in value.split(',') if f] + def getoauth2_request_url(self): + if self._oauth2_request_url: # Use cached value if possible. + return self._oauth2_request_url + + oauth2_request_url = self.getconf('oauth2_request_url', None) + if oauth2_request_url != None: + self._oauth2_request_url = oauth2_request_url + return self._oauth2_request_url + + #raise OfflineImapError("No remote oauth2_request_url for repository " + #"'%s' specified."% self, OfflineImapError.ERROR.REPO) + + def getoauth2_refresh_token(self): + return self.getconf('oauth2_refresh_token', None) + + def getoauth2_access_token(self): + return self.getconf('oauth2_access_token', None) + + def getoauth2_client_id(self): + return self.getconf('oauth2_client_id', None) + + def getoauth2_client_secret(self): + return self.getconf('oauth2_client_secret', None) + def getpreauthtunnel(self): return self.getconf('preauthtunnel', None) @@ -261,6 +297,9 @@ class IMAPRepository(BaseRepository): def getreference(self): return self.getconf('reference', '') + def getdecodefoldernames(self): + return self.getconfboolean('decodefoldernames', 0) + def getidlefolders(self): localeval = self.localeval return localeval.eval(self.getconf('idlefolders', '[]')) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 0262ba2..10085e7 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -39,6 +39,14 @@ class MaildirRepository(BaseRepository): if not os.path.isdir(self.root): os.mkdir(self.root, 0o700) + # Create the keyword->char mapping + self.keyword2char = dict() + for c in 'abcdefghijklmnopqrstuvwxyz': + confkey = 'customflag_' + c + keyword = self.getconf(confkey, None) + if keyword is not None: + self.keyword2char[keyword] = c + def _append_folder_atimes(self, foldername): """Store the atimes of a folder's new|cur in self.folder_atimes""" @@ -72,6 +80,9 @@ class MaildirRepository(BaseRepository): def getsep(self): return self.getconf('sep', '.').strip() + def getkeywordmap(self): + return self.keyword2char if len(self.keyword2char) > 0 else None + def makefolder(self, foldername): """Create new Maildir folder if necessary diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index ddc05ea..d5b148d 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -603,7 +603,7 @@ class Blinkenlights(UIBase, CursesUtil): self.bannerwin.clear() # Delete old content (eg before resizes) self.bannerwin.bkgd(' ', color) # Fill background with that color string = "%s %s"% (offlineimap.__productname__, - offlineimap.__bigversion__) + offlineimap.__version__) self.bannerwin.addstr(0, 0, string, color) self.bannerwin.addstr(0, self.width -len(offlineimap.__copyright__) -1, offlineimap.__copyright__, color) diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index de1e8df..0e23a93 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -17,9 +17,10 @@ import logging from offlineimap.ui.UIBase import UIBase +import offlineimap class Basic(UIBase): - """'Quiet' simply sets log level to INFO""" + """'Basic' simply sets log level to INFO""" def __init__(self, config, loglevel = logging.INFO): return super(Basic, self).__init__(config, loglevel) @@ -27,3 +28,22 @@ class Quiet(UIBase): """'Quiet' simply sets log level to WARNING""" def __init__(self, config, loglevel = logging.WARNING): return super(Quiet, self).__init__(config, loglevel) + +class Syslog(UIBase): + """'Syslog' sets log level to INFO and outputs to syslog instead of stdout""" + def __init__(self, config, loglevel = logging.INFO): + return super(Syslog, self).__init__(config, loglevel) + + def setup_consolehandler(self): + # create syslog handler + ch = logging.handlers.SysLogHandler('/dev/log') + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(offlineimap.banner) + return ch + + def setup_sysloghandler(self): + pass # Do not honor -s (log to syslog) CLI option. diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 3e698ab..62e42d1 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import logging +import logging.handlers import re import time import sys @@ -91,10 +92,22 @@ class UIBase(object): self.logger.info(offlineimap.banner) return ch + def setup_sysloghandler(self): + """Backend specific syslog handler.""" + + # create syslog handler + ch = logging.handlers.SysLogHandler('/dev/log') + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + def setlogfile(self, logfile): """Create file handler which logs to file.""" fh = logging.FileHandler(logfile, 'at') + #fh.setLevel(logging.DEBUG) file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " "%(message)s", '%Y-%m-%d %H:%M:%S') fh.setFormatter(file_formatter) @@ -102,9 +115,11 @@ class UIBase(object): # write out more verbose initial info blurb on the log file p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ - "Args: %s"% (offlineimap.__bigversion__, p_ver, sys.platform, + "Args: %s"% (offlineimap.__version__, p_ver, sys.platform, " ".join(sys.argv)) - self.logger.info(msg) + record = logging.LogRecord('OfflineImap', logging.INFO, __file__, + None, msg, None, None) + fh.emit(record) def _msg(self, msg): """Display a message.""" @@ -430,7 +445,7 @@ class UIBase(object): #TODO: Debug and make below working, it hangs Gmail #res_type, response = conn.id(( # 'name', offlineimap.__productname__, - # 'version', offlineimap.__bigversion__)) + # 'version', offlineimap.__version__)) #self._msg("Server ID: %s %s" % (res_type, response[0])) self._msg("Server welcome string: %s" % str(conn.welcome)) self._msg("Server capabilities: %s\n" % str(conn.capabilities)) diff --git a/offlineimap/ui/__init__.py b/offlineimap/ui/__init__.py index 3da42b9..258b5bb 100644 --- a/offlineimap/ui/__init__.py +++ b/offlineimap/ui/__init__.py @@ -21,6 +21,7 @@ from offlineimap.ui import TTY, Noninteractive, Machine UI_LIST = {'ttyui': TTY.TTYUI, 'basic': Noninteractive.Basic, 'quiet': Noninteractive.Quiet, + 'syslog': Noninteractive.Syslog, 'machineui': Machine.MachineUI} #add Blinkenlights UI if it imports correctly (curses installed) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..9e99ac2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ + +[metadata] +description-file = README.md diff --git a/test/OLItest/__init__.py b/test/OLItest/__init__.py index ca6ef61..e6dc341 100644 --- a/test/OLItest/__init__.py +++ b/test/OLItest/__init__.py @@ -24,7 +24,7 @@ __author__ = 'Sebastian Spaeth' __author_email__= 'Sebastian@SSpaeth.de' __description__ = 'Moo' __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" -__homepage__ = "http://offlineimap.org" +__homepage__ = "http://www.offlineimap.org" banner = """%(__productname__)s %(__version__)s %(__license__)s""" % locals()