From fc4c7549d6d03415edc2537b29c83fd6baea1bc6 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:03:15 +0100 Subject: [PATCH 01/19] Do not ignore gmail labels if header appears multiple times There should be just one header storing gmail labels, but due to a bug, multiple X-Keywords (or equivalent) headers may be found on the local messages. Now we, when extracting the labels from a message, we read all label headers, instead of just the first one. This has the consequence that some old labels stored locally in a second X-Keywords (or third...) header, which effectively was rendered invisible to offlineimap until now, may pop back up again and be pushed to gmail. No labels will be removed by the changes in this commit, though. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Base.py | 25 +++++++++++++++++++++++-- offlineimap/folder/Gmail.py | 5 +++-- offlineimap/folder/GmailMaildir.py | 19 +++++++++++-------- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index b56cd1b..4b30d8c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -512,8 +512,8 @@ class BaseFolder(object): def getmessageheader(self, content, name): """ - Searches for the given header and returns its value. - Header name is case-insensitive. + Searches for the first occurence of the given header and returns + its value. Header name is case-insensitive. Arguments: - contents: message itself @@ -535,6 +535,27 @@ class BaseFolder(object): return None + def getmessageheaderlist(self, content, name): + """ + Searches for the given header and returns a list of values for + that header. + + Arguments: + - contents: message itself + - name: name of the header to be searched + + Returns: list of header values or emptylist if no such header was found + + """ + self.ui.debug('', 'getmessageheaderlist: called to get %s' % name) + eoh = self.__find_eoh(content) + self.ui.debug('', 'getmessageheaderlist: eoh = %d' % eoh) + headers = content[0:eoh] + self.ui.debug('', 'getmessageheaderlist: headers = %s' % repr(headers)) + + return re.findall('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE) + + def deletemessageheaders(self, content, header_list): """ Deletes headers in the given list from the message content. diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index e3ef92a..2a8afd2 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -189,8 +189,9 @@ class GmailFolder(IMAPFolder): if not self.synclabels: return super(GmailFolder, self).savemessage(uid, content, flags, rtime) - labels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + labels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) self.savemessagelabels(ret, labels) diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 2cab213..7a67c87 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -87,9 +87,10 @@ class GmailMaildirFolder(MaildirFolder): content = file.read() file.close() - self.messagelist[uid]['labels'] = \ - imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + self.messagelist[uid]['labels'] = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + self.messagelist[uid]['labels'].update( + imaputil.labels_from_header(self.labelsheader, hstr)) self.messagelist[uid]['labels_cached'] = True return self.messagelist[uid]['labels'] @@ -111,8 +112,10 @@ class GmailMaildirFolder(MaildirFolder): if not self.synclabels: return super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) - labels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + labels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) + ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) # Update the mtime and labels @@ -135,9 +138,9 @@ class GmailMaildirFolder(MaildirFolder): content = file.read() file.close() - oldlabels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) - + oldlabels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + oldlabels.update(imaputil.labels_from_header(self.labelsheader, hstr)) labels = labels - ignorelabels ignoredlabels = oldlabels & ignorelabels From 2a5ef8c2ef2d6d64ef9ab8c6216ef68d6f07937c Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:16:48 +0100 Subject: [PATCH 02/19] Delete gmail labels header before adding a new one This fixes a bug in which a message ended up with multiple gmail labels header (X-Keywords or so). Fix fix it by removing all labels headers before adding the updated one. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Gmail.py | 4 ++++ offlineimap/folder/GmailMaildir.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 2a8afd2..69f0075 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -93,6 +93,10 @@ class GmailFolder(IMAPFolder): labels = set() labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) + + # First remove old label headers that may be in the message content retrieved + # from gmail Then add a labels header with current gmail labels. + body = self.deletemessageheaders(body, self.labelsheader) body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) if len(body)>200: diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 7a67c87..3b127d2 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -153,7 +153,11 @@ class GmailMaildirFolder(MaildirFolder): # Change labels into content labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels | ignoredlabels)) + + # First remove old labels header, and then add the new one + content = self.deletemessageheaders(content, self.labelsheader) content = self.addmessageheader(content, '\n', self.labelsheader, labels_str) + rtime = self.messagelist[uid].get('rtime', None) # write file with new labels to a unique file in tmp From e70607e3e33701cc3b7fcc4cc952f57097a0ac12 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:18:35 +0100 Subject: [PATCH 03/19] Warn about a tricky piece of code in addmessageheader As requested by Nicholas Sebrecht. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 4b30d8c..106c390 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -401,6 +401,9 @@ class BaseFolder(object): """ Adds new header to the provided message. + WARNING: This function is a bit tricky, and modifying it in the wrong way, + may easily lead to data-loss. + Arguments: - content: message content, headers and body as a single string - linebreak: string that carries line ending From 15e8e089133741a7e2e74c2cb223fa137eb63781 Mon Sep 17 00:00:00 2001 From: Eygene Ryabinkin Date: Wed, 26 Nov 2014 23:17:43 +0300 Subject: [PATCH 04/19] Properly generate package via "sdist" Include tests, configuration examples and other stuff we usually include to the tarballs. GitHub issue: https://github.com/OfflineIMAP/offlineimap/issues/137 Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 ++ MANIFEST.in | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 MANIFEST.in diff --git a/Changelog.rst b/Changelog.rst index 321bf87..9a98d29 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,8 @@ ChangeLog OfflineIMAP v6.5.6.1 (YYYY-MM-DD) ================================= +* Properly generate tarball from "sdist" command (GitHub #137) + * Expand environment variables in the following configuration items: - general.pythonfile; diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..3a403a3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,14 @@ +global-exclude .gitignore .git *.bak *.orig *.rej +include setup.py +include COPYING +include Changelog* +include MAINTAINERS +include MANIFEST.in +include Makefile +include README.md +include offlineimap.conf* +include offlineimap.py +recursive-include offlineimap *.py +recursive-include bin * +recursive-include docs * +recursive-include test * From 0521aa270653cb0e42d27c01d7f8723ec24f336c Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 27 Nov 2014 00:56:38 -0800 Subject: [PATCH 05/19] Fix typo in apply_xforms invocation Fixes a bug introduced in e51ed80ecc1830ca3184761ccbfa63cf22cc2d42 since apply_xform (without the 's') doesn't exist. Signed-off-by: Eygene Ryabinkin --- offlineimap/mbnames.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 436a7e6..176a760 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -55,7 +55,7 @@ def __genmbnames(): localeval = config.getlocaleval() if not config.getdefaultboolean("mbnames", "enabled", 0): return - path = config.apply_xform(config.get("mbnames", "filename"), xforms) + path = config.apply_xforms(config.get("mbnames", "filename"), xforms) file = open(path, "wt") file.write(localeval.eval(config.get("mbnames", "header"))) folderfilter = lambda accountname, foldername: 1 From 54cff7f7860914b2c2d7cb42129acb3e67cf3449 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:23:35 +0100 Subject: [PATCH 06/19] imaplibutil.py: remove unused imports Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 2869623..05aaf7d 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -17,9 +17,6 @@ import os import fcntl -import re -import socket -import ssl import time import subprocess import threading @@ -27,7 +24,7 @@ from hashlib import sha1 from offlineimap.ui import getglobalui from offlineimap import OfflineImapError -from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, IMAP4_PORT, InternalDate, Mon2num +from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, InternalDate, Mon2num class UsefulIMAPMixIn(object): From de5f22a23af72273107a7ea2ada8003eb891cb7d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:18:45 +0100 Subject: [PATCH 07/19] CustomConfig.py: remove unused imports Signed-off-by: Nicolas Sebrecht --- offlineimap/CustomConfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 1f4bcd0..61bf639 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -15,9 +15,9 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA try: - from ConfigParser import SafeConfigParser, Error, NoOptionError + from ConfigParser import SafeConfigParser, Error except ImportError: #python3 - from configparser import SafeConfigParser, Error, NoOptionError + from configparser import SafeConfigParser, Error from offlineimap.localeval import LocalEval import os import re From 2785e779e2820edb4deae570e9f713d0545c89ea Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:19:16 +0100 Subject: [PATCH 08/19] init.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 5bb0438..d9425d5 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -26,7 +26,6 @@ from optparse import OptionParser import offlineimap from offlineimap import accounts, threadutil, syncmaster from offlineimap import globals -from offlineimap.error import OfflineImapError from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser from offlineimap.utils import stacktrace From 62afde18250db2f956d8f37c7818f6a9d5a2ef4c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:19:39 +0100 Subject: [PATCH 09/19] repository/Base.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index cc6abdd..d7a6866 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -17,7 +17,6 @@ import re import os.path -import traceback from sys import exc_info from offlineimap import CustomConfig from offlineimap.ui import getglobalui From 7f1419a40ac7cead70b71fecc5ab855f785f1190 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:20:13 +0100 Subject: [PATCH 10/19] repository/GmailMaildir.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/GmailMaildir.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/repository/GmailMaildir.py b/offlineimap/repository/GmailMaildir.py index 9072b7c..b790c73 100644 --- a/offlineimap/repository/GmailMaildir.py +++ b/offlineimap/repository/GmailMaildir.py @@ -18,7 +18,6 @@ from offlineimap.repository.Maildir import MaildirRepository from offlineimap.folder.GmailMaildir import GmailMaildirFolder -from offlineimap.error import OfflineImapError class GmailMaildirRepository(MaildirRepository): def __init__(self, reposname, account): From 24a4ab3e167ec2315b0c918f746c7ff64459eb34 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:20:43 +0100 Subject: [PATCH 11/19] repository/LocalStatus.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/LocalStatus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index 0375d0d..ba29d37 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -16,11 +16,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os + from offlineimap.folder.LocalStatus import LocalStatusFolder from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder from offlineimap.repository.Base import BaseRepository -import os -import re class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): From 7b453efcce7be684d687b094744ce8cc910ba9fd Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:21:13 +0100 Subject: [PATCH 12/19] ui/Curses.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Curses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 4150066..6cc1855 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -20,7 +20,6 @@ from collections import deque import time import sys import os -import signal import curses import logging from offlineimap.ui.UIBase import UIBase From e613f6992dfefa0881136a91ef289da4fc9e48a3 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 22:21:44 +0100 Subject: [PATCH 13/19] ui/UIBase.py: remove unused import Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index dace655..1013a98 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -19,7 +19,6 @@ import logging import re import time import sys -import os import traceback import threading try: From 4589cfeff2fad8db016f54ba05e57ef2fe0aa51f Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 23 Dec 2014 10:12:23 +0100 Subject: [PATCH 14/19] localeval: comment on security issues Minor syntax fixes. Signed-off-by: Nicolas Sebrecht --- offlineimap/localeval.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/offlineimap/localeval.py b/offlineimap/localeval.py index 22014e6..e7d656f 100644 --- a/offlineimap/localeval.py +++ b/offlineimap/localeval.py @@ -1,7 +1,6 @@ """Eval python code with global namespace of a python source file.""" -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2014 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,18 +23,24 @@ except: pass class LocalEval: + """Here is a powerfull but very dangerous option, of course. + + Assume source file to be ASCII encoded.""" + def __init__(self, path=None): - self.namespace={} + self.namespace = {} if path is not None: - file=open(path, 'r') - module=imp.load_module( + # FIXME: limit opening files owned by current user with rights set + # to fixed mode 644. + file = open(path, 'r') + module = imp.load_module( '', file, path, ('', 'r', imp.PY_SOURCE)) for attr in dir(module): - self.namespace[attr]=getattr(module, attr) + self.namespace[attr] = getattr(module, attr) def eval(self, text, namespace=None): names = {} From 532278b4dd01fc487ff3e5f549ead1b5a3de3fac Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 30 Dec 2014 01:04:00 +0100 Subject: [PATCH 15/19] docs: remove obsolete comment about SubmittingPatches.rst Signed-off-by: Nicolas Sebrecht --- docs/doc-src/API.rst | 7 ++-- docs/doc-src/advanced_config.rst | 6 ++- docs/doc-src/nametrans.rst | 65 ++++++++++++++++++++++++-------- docs/doc-src/ui.rst | 5 ++- 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/docs/doc-src/API.rst b/docs/doc-src/API.rst index 1eb9adb..d47fcf4 100644 --- a/docs/doc-src/API.rst +++ b/docs/doc-src/API.rst @@ -15,10 +15,9 @@ OfflineImap can be imported as:: from offlineimap import OfflineImap -The file ``SubmittingPatches.rst`` in the source distribution documents a -number of resources and conventions you may find useful. It will eventually -be merged into the main documentation. -.. TODO: merge SubmittingPatches.rst to the main documentation +The file ``HACKING.rst`` in the source distribution documents a +number of resources and conventions you may find useful. + :mod:`offlineimap` -- The OfflineImap module ============================================= diff --git a/docs/doc-src/advanced_config.rst b/docs/doc-src/advanced_config.rst index 56e6f18..49f2ea4 100644 --- a/docs/doc-src/advanced_config.rst +++ b/docs/doc-src/advanced_config.rst @@ -1,9 +1,11 @@ Message filtering ================= -There are two ways to selectively filter messages out of a folder, using `maxsize` and `maxage`. Setting each option will basically ignore all messages that are on the server by pretending they don't exist. +There are two ways to selectively filter messages out of a folder, using +`maxsize` and `maxage`. Setting each option will basically ignore all messages +that are on the server by pretending they don't exist. -:todo: explain them and give tipps on how to use and not use them. Use cases! +:todo: explain them and give tips on how to use and not use them. Use cases! maxage ------ diff --git a/docs/doc-src/nametrans.rst b/docs/doc-src/nametrans.rst index f6b9924..030ead2 100644 --- a/docs/doc-src/nametrans.rst +++ b/docs/doc-src/nametrans.rst @@ -13,7 +13,10 @@ safely skip this section. folderfilter ------------ -If you do not want to synchronize all your folders, you can specify a `folderfilter`_ function that determines which folders to include in a sync and which to exclude. Typically, you would set a folderfilter option on the remote repository only, and it would be a lambda or any other python function. +If you do not want to synchronize all your folders, you can specify a +`folderfilter`_ function that determines which folders to include in a sync and +which to exclude. Typically, you would set a folderfilter option on the remote +repository only, and it would be a lambda or any other python function. The only parameter to that function is the folder name. If the filter function returns True, the folder will be synced, if it returns False, @@ -45,12 +48,18 @@ at the end when required by Python syntax) For instance:: ['INBOX', 'Sent Mail', 'Deleted Items', 'Received'] -Usually it suffices to put a `folderfilter`_ setting in the remote repository section. You might want to put a folderfilter option on the local repository if you want to prevent some folders on the local repository to be created on the remote one. (Even in this case, folder filters on the remote repository will prevent that) +Usually it suffices to put a `folderfilter`_ setting in the remote repository +section. You might want to put a folderfilter option on the local repository if +you want to prevent some folders on the local repository to be created on the +remote one. (Even in this case, folder filters on the remote repository will +prevent that) folderincludes -------------- -You can specify `folderincludes`_ to manually include additional folders to be synced, even if they had been filtered out by a folderfilter setting. `folderincludes`_ should return a Python list. +You can specify `folderincludes`_ to manually include additional folders to be +synced, even if they had been filtered out by a folderfilter setting. +`folderincludes`_ should return a Python list. This can be used to 1) add a folder that was excluded by your folderfilter rule, 2) to include a folder that your server does not specify @@ -84,11 +93,14 @@ nametrans`_ rules on the LOCAL repository. nametrans ---------- -Sometimes, folders need to have different names on the remote and the -local repositories. To achieve this you can specify a folder name -translator. This must be a eval-able Python expression that takes a -foldername arg and returns the new value. We suggest a lambda function, -but it could be any python function really. If you use nametrans rules, you will need to set them both on the remote and the local repository, see `Reverse nametrans`_ just below for details. The following examples are thought to be put in the remote repository section. +Sometimes, folders need to have different names on the remote and the local +repositories. To achieve this you can specify a folder name translator. This +must be a eval-able Python expression that takes a foldername arg and returns +the new value. We suggest a lambda function, but it could be any python +function really. If you use nametrans rules, you will need to set them both on +the remote and the local repository, see `Reverse nametrans`_ just below for +details. The following examples are thought to be put in the remote repository +section. The below will remove "INBOX." from the leading edge of folders (great for Courier IMAP users):: @@ -112,38 +124,59 @@ locally? Try this:: Reverse nametrans +++++++++++++++++ -Since 6.4.0, OfflineImap supports the creation of folders on the remote repository and that complicates things. Previously, only one nametrans setting on the remote repository was needed and that transformed a remote to a local name. However, nametrans transformations are one-way, and OfflineImap has no way using those rules on the remote repository to back local names to remote names. +Since 6.4.0, OfflineImap supports the creation of folders on the remote +repository and that complicates things. Previously, only one nametrans setting +on the remote repository was needed and that transformed a remote to a local +name. However, nametrans transformations are one-way, and OfflineImap has no way +using those rules on the remote repository to back local names to remote names. -Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts off any existing INBOX prefix. Now, if we parse a list of local folders, finding e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We have no way of knowing. This is why **every nametrans setting on a remote repository requires an equivalent nametrans rule on the local repository that reverses the transformation**. +Take a remote nametrans rule `lambda f: re.sub('^INBOX/','',f)` which cuts off +any existing INBOX prefix. Now, if we parse a list of local folders, finding +e.g. a folder "Sent", is it supposed to map to "INBOX/Sent" or to "Sent"? We +have no way of knowing. This is why **every nametrans setting on a remote +repository requires an equivalent nametrans rule on the local repository that +reverses the transformation**. Take the above examples. If your remote nametrans setting was:: nametrans = lambda folder: re.sub('^INBOX\.', '', folder) -then you will want to have this in your local repository, prepending "INBOX" to any local folder name:: +then you will want to have this in your local repository, prepending "INBOX" to +any local folder name:: nametrans = lambda folder: 'INBOX.' + folder -Failure to set the local nametrans rule will lead to weird-looking error messages of -for instance- this type:: +Failure to set the local nametrans rule will lead to weird-looking error +messages of -for instance- this type:: ERROR: Creating folder moo.foo on repository remote Folder 'moo.foo'[remote] could not be created. Server responded: ('NO', ['Unknown namespace.']) -(This indicates that you attempted to create a folder "Sent" when all remote folders needed to be under the prefix of "INBOX."). +(This indicates that you attempted to create a folder "Sent" when all remote +folders needed to be under the prefix of "INBOX."). OfflineImap will make some sanity checks if it needs to create a new folder on the remote side and a back-and-forth nametrans-lation does not yield the original foldername (as that could potentially lead to infinite folder creation cycles). -You can probably already see now that creating nametrans rules can be a pretty daunting and complex endeavour. Check out the Use cases in the manual. If you have some interesting use cases that we can present as examples here, please let us know. +You can probably already see now that creating nametrans rules can be a pretty +daunting and complex endeavour. Check out the Use cases in the manual. If you +have some interesting use cases that we can present as examples here, please let +us know. Debugging folderfilter and nametrans ------------------------------------ -Given the complexity of the functions and regexes involved, it is easy to misconfigure things. One way to test your configuration without danger to corrupt anything or to create unwanted folders is to invoke offlineimap with the `--info` option. +Given the complexity of the functions and regexes involved, it is easy to +misconfigure things. One way to test your configuration without danger to +corrupt anything or to create unwanted folders is to invoke offlineimap with the +`--info` option. -It will output a list of folders and their transformations on the screen (save them to a file with -l info.log), and will help you to tweak your rules as well as to understand your configuration. It also provides good output for bug reporting. +It will output a list of folders and their transformations on the screen (save +them to a file with -l info.log), and will help you to tweak your rules as well +as to understand your configuration. It also provides good output for bug +reporting. FAQ on nametrans ---------------- diff --git a/docs/doc-src/ui.rst b/docs/doc-src/ui.rst index 27be221..2931010 100644 --- a/docs/doc-src/ui.rst +++ b/docs/doc-src/ui.rst @@ -3,7 +3,10 @@ .. currentmodule:: offlineimap.ui -OfflineImap has various ui systems, that can be selected. They offer various functionalities. They must implement all functions that the :class:`offlineimap.ui.UIBase` offers. Early on, the ui must be set using :meth:`getglobalui` +OfflineImap has various ui systems, that can be selected. They offer various +functionalities. They must implement all functions that the +:class:`offlineimap.ui.UIBase` offers. Early on, the ui must be set using +:meth:`getglobalui` .. automethod:: offlineimap.ui.setglobalui .. automethod:: offlineimap.ui.getglobalui From a35c432671143cff2a6d5d1006aa4d351fe6c200 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 30 Dec 2014 01:05:20 +0100 Subject: [PATCH 16/19] utils/const.py: fix ident Signed-off-by: Nicolas Sebrecht --- offlineimap/utils/const.py | 47 ++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/offlineimap/utils/const.py b/offlineimap/utils/const.py index a62b6a6..f4584bc 100644 --- a/offlineimap/utils/const.py +++ b/offlineimap/utils/const.py @@ -1,40 +1,37 @@ -# Copyright 2013 Eygene A. Ryabinkin. +# Copyright (C) 2013-2014 Eygene A. Ryabinkin and contributors # # Collection of classes that implement const-like behaviour # for various objects. import copy -class ConstProxy (object): - """ - Implements read-only access to a given object - that can be attached to each instance only once. +class ConstProxy(object): + """Implements read-only access to a given object + that can be attached to each instance only once.""" - """ - - def __init__ (self): - self.__dict__['__source'] = None + def __init__(self): + self.__dict__['__source'] = None - def __getattr__ (self, name): - src = self.__dict__['__source'] - if src == None: - raise ValueError ("using non-initialized ConstProxy() object") - return copy.deepcopy (getattr (src, name)) + def __getattr__(self, name): + src = self.__dict__['__source'] + if src == None: + raise ValueError("using non-initialized ConstProxy() object") + return copy.deepcopy(getattr(src, name)) - def __setattr__ (self, name, value): - raise AttributeError ("tried to set '%s' to '%s' for constant object" % \ - (name, value)) + def __setattr__(self, name, value): + raise AttributeError("tried to set '%s' to '%s' for constant object"% \ + (name, value)) - def __delattr__ (self, name): - raise RuntimeError ("tried to delete field '%s' from constant object" % \ - (name)) + def __delattr__(self, name): + raise RuntimeError("tried to delete field '%s' from constant object"% \ + (name)) - def set_source (self, source): - """ Sets source object for this instance. """ - if (self.__dict__['__source'] != None): - raise ValueError ("source object is already set") - self.__dict__['__source'] = source + def set_source(self, source): + """ Sets source object for this instance. """ + if (self.__dict__['__source'] != None): + raise ValueError("source object is already set") + self.__dict__['__source'] = source From 11a28fb0cb2bddf6e8f6d322f2d6f73b543f4933 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 21:55:24 +0100 Subject: [PATCH 17/19] ui/UIBase: folderlist(): avoid built-in list() redefinition Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 1013a98..0edfa18 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -230,9 +230,9 @@ class UIBase(object): raise NotImplementedError("Prompting for a password is not supported"\ " in this UI backend.") - def folderlist(self, list): - return ', '.join(["%s[%s]" % \ - (self.getnicename(x), x.getname()) for x in list]) + def folderlist(self, folder_list): + return ', '.join(["%s[%s]"% \ + (self.getnicename(x), x.getname()) for x in folder_list]) ################################################## WARNINGS def msgtoreadonly(self, destfolder, uid, content, flags): From 61021260cb6676447979b5f6247295409b023418 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 1 Jan 2015 21:41:11 +0100 Subject: [PATCH 18/19] more consistent style Signed-off-by: Nicolas Sebrecht --- offlineimap/CustomConfig.py | 147 +++++++++++--------------- offlineimap/accounts.py | 16 +-- offlineimap/error.py | 2 + offlineimap/folder/Base.py | 14 ++- offlineimap/folder/LocalStatus.py | 29 ++--- offlineimap/folder/Maildir.py | 25 +++-- offlineimap/folder/UIDMaps.py | 8 +- offlineimap/imaplibutil.py | 42 ++++---- offlineimap/imaputil.py | 45 ++++---- offlineimap/init.py | 24 ++--- offlineimap/mbnames.py | 5 +- offlineimap/repository/Base.py | 9 +- offlineimap/repository/IMAP.py | 35 +++--- offlineimap/repository/LocalStatus.py | 7 +- offlineimap/repository/Maildir.py | 10 +- offlineimap/ui/Curses.py | 8 +- offlineimap/ui/Machine.py | 2 +- offlineimap/ui/TTY.py | 10 +- offlineimap/ui/UIBase.py | 81 ++++++++------ offlineimap/ui/debuglock.py | 3 +- 20 files changed, 277 insertions(+), 245 deletions(-) diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py index 61bf639..ae27d41 100644 --- a/offlineimap/CustomConfig.py +++ b/offlineimap/CustomConfig.py @@ -1,4 +1,4 @@ -# Copyright (C) 2003-2012 John Goerzen & contributors +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,21 +14,20 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import os +import re + try: from ConfigParser import SafeConfigParser, Error except ImportError: #python3 from configparser import SafeConfigParser, Error from offlineimap.localeval import LocalEval -import os -import re class CustomConfigParser(SafeConfigParser): def getdefault(self, section, option, default, *args, **kwargs): - """ - Same as config.get, but returns the value of `default` - if there is no such option specified. - - """ + """Same as config.get, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): return self.get(*(section, option) + args, **kwargs) else: @@ -36,45 +35,37 @@ class CustomConfigParser(SafeConfigParser): def getdefaultint(self, section, option, default, *args, **kwargs): - """ - Same as config.getint, but returns the value of `default` - if there is no such option specified. - - """ + """Same as config.getint, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): - return self.getint (*(section, option) + args, **kwargs) + return self.getint(*(section, option) + args, **kwargs) else: return default def getdefaultfloat(self, section, option, default, *args, **kwargs): - """ - Same as config.getfloat, but returns the value of `default` - if there is no such option specified. - - """ + """Same as config.getfloat, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): return self.getfloat(*(section, option) + args, **kwargs) else: return default def getdefaultboolean(self, section, option, default, *args, **kwargs): - """ - Same as config.getboolean, but returns the value of `default` - if there is no such option specified. - - """ + """Same as config.getboolean, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): return self.getboolean(*(section, option) + args, **kwargs) else: return default def getlist(self, section, option, separator_re): - """ - Parses option as the list of values separated - by the given regexp. + """Parses option as the list of values separated + by the given regexp.""" - """ try: val = self.get(section, option).strip() return re.split(separator_re, val) @@ -83,11 +74,9 @@ class CustomConfigParser(SafeConfigParser): (separator_re, e)) def getdefaultlist(self, section, option, default, separator_re): - """ - Same as getlist, but returns the value of `default` - if there is no such option specified. - - """ + """Same as getlist, but returns the value of `default` + if there is no such option specified.""" + if self.has_option(section, option): return self.getlist(*(section, option, separator_re)) else: @@ -104,40 +93,48 @@ class CustomConfigParser(SafeConfigParser): def getlocaleval(self): xforms = [os.path.expanduser, os.path.expandvars] if self.has_option("general", "pythonfile"): - path = self.apply_xforms(self.get("general", "pythonfile"), xforms) + if globals.options.use_unicode: + path = uni.fsEncode(self.get("general", "pythonfile"), + exception_msg="cannot convert character for pythonfile") + else: + path = self.get("general", "pythonfile") + path = self.apply_xforms(path, xforms) else: path = None return LocalEval(path) def getsectionlist(self, key): - """ - Returns a list of sections that start with key + " ". + """Returns a list of sections that start with (str) key + " ". That is, if key is "Account", returns all section names that start with "Account ", but strips off the "Account ". - For instance, for "Account Test", returns "Test". + For instance, for "Account Test", returns "Test".""" - """ key = key + ' ' - return [x[len(key):] for x in self.sections() \ + if globals.options.use_unicode: + sections = [] + for section in self.sections(): + sections.append(uni.uni2str(section, exception_msg= + "non ASCII character in section %s"% section)) + return [x[len(key):] for x in sections \ + if x.startswith(key)] + else: + return [x[len(key):] for x in self.sections() \ if x.startswith(key)] def set_if_not_exists(self, section, option, value): - """ - Set a value if it does not exist yet + """Set a value if it does not exist yet. This allows to set default if the user has not explicitly - configured anything. - - """ + configured anything.""" + if not self.has_option(section, option): self.set(section, option, value) def apply_xforms(self, string, transforms): - """ - Applies set of transformations to a string. + """Applies set of transformations to a string. Arguments: - string: source string; if None, then no processing will @@ -145,9 +142,8 @@ class CustomConfigParser(SafeConfigParser): - transforms: iterable that returns transformation function on each turn. - Returns transformed string. + Returns transformed string.""" - """ if string == None: return None for f in transforms: @@ -157,21 +153,18 @@ class CustomConfigParser(SafeConfigParser): def CustomConfigDefault(): - """ - Just a constant that won't occur anywhere else. + """Just a constant that won't occur anywhere else. This allows us to differentiate if the user has passed in any default value to the getconf* functions in ConfigHelperMixin - derived classes. + derived classes.""" - """ pass class ConfigHelperMixin: - """ - Allow comfortable retrieving of config values pertaining + """Allow comfortable retrieving of config values pertaining to a section. If a class inherits from cls:`ConfigHelperMixin`, it needs @@ -181,13 +174,10 @@ class ConfigHelperMixin: the section to look up). All calls to getconf* will then return the configuration values for the CustomConfigParser object in the specific section. - """ - def _confighelper_runner(self, option, default, defaultfunc, mainfunc, *args): - """ - Returns configuration or default value for option + """Returns configuration or default value for option that contains in section identified by getsection(). Arguments: @@ -201,8 +191,8 @@ class ConfigHelperMixin: - defaultfunc and mainfunc: processing helpers. - args: additional trailing arguments that will be passed to all processing helpers. - """ + lst = [self.getsection(), option] if default == CustomConfigDefault: return mainfunc(*(lst + list(args))) @@ -210,50 +200,43 @@ class ConfigHelperMixin: lst.append(default) return defaultfunc(*(lst + list(args))) - def getconfig(self): - """ - Returns CustomConfigParser object that we will use + """Returns CustomConfigParser object that we will use for all our actions. - Must be overriden in all classes that use this mix-in. + Must be overriden in all classes that use this mix-in.""" - """ raise NotImplementedError("ConfigHelperMixin.getconfig() " "is to be overriden") def getsection(self): - """ - Returns name of configuration section in which our + """Returns name of configuration section in which our class keeps its configuration. - Must be overriden in all classes that use this mix-in. + Must be overriden in all classes that use this mix-in.""" - """ raise NotImplementedError("ConfigHelperMixin.getsection() " "is to be overriden") def getconf(self, option, default = CustomConfigDefault): - """ - Retrieves string from the configuration. + """Retrieves string from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. - """ + return self._confighelper_runner(option, default, self.getconfig().getdefault, self.getconfig().get) def getconf_xform(self, option, xforms, default = CustomConfigDefault): - """ - Retrieves string from the configuration transforming the result. + """Retrieves string from the configuration transforming the result. Arguments: - option: option name whose value is to be retrieved; @@ -262,22 +245,21 @@ class ConfigHelperMixin: both retrieved and default one; - default: default value for string if no such option exists. - """ + value = self.getconf(option, default) return self.getconfig().apply_xforms(value, xforms) def getconfboolean(self, option, default = CustomConfigDefault): - """ - Retrieves boolean value from the configuration. + """Retrieves boolean value from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. - """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultboolean, self.getconfig().getboolean) @@ -293,21 +275,21 @@ class ConfigHelperMixin: exists. """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultint, self.getconfig().getint) def getconffloat(self, option, default = CustomConfigDefault): - """ - Retrieves floating-point value from the configuration. + """Retrieves floating-point value from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. - """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultfloat, self.getconfig().getfloat) @@ -315,8 +297,7 @@ class ConfigHelperMixin: def getconflist(self, option, separator_re, default = CustomConfigDefault): - """ - Retrieves strings from the configuration and splits it + """Retrieves strings from the configuration and splits it into the list of strings. Arguments: @@ -325,8 +306,8 @@ class ConfigHelperMixin: to be used for split operation; - default: default return value if no such option exists. - """ + return self._confighelper_runner(option, default, self.getconfig().getdefaultlist, self.getconfig().getlist, separator_re) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index eabbd23..190f6de 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2003-2011 John Goerzen & contributors +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,29 +14,33 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap import mbnames, CustomConfig, OfflineImapError -from offlineimap import globals -from offlineimap.repository import Repository -from offlineimap.ui import getglobalui -from offlineimap.threadutil import InstanceLimitedThread from subprocess import Popen, PIPE from threading import Event import os from sys import exc_info import traceback +from offlineimap import mbnames, CustomConfig, OfflineImapError +from offlineimap import globals +from offlineimap.repository import Repository +from offlineimap.ui import getglobalui +from offlineimap.threadutil import InstanceLimitedThread + try: import fcntl except: pass # ok if this fails, we can do without +# FIXME: spaghetti code alert! def getaccountlist(customconfig): return customconfig.getsectionlist('Account') +# FIXME: spaghetti code alert! def AccountListGenerator(customconfig): return [Account(customconfig, accountname) for accountname in getaccountlist(customconfig)] +# FIXME: spaghetti code alert! def AccountHashGenerator(customconfig): retval = {} for item in AccountListGenerator(customconfig): diff --git a/offlineimap/error.py b/offlineimap/error.py index aa7f535..1be64ac 100644 --- a/offlineimap/error.py +++ b/offlineimap/error.py @@ -10,6 +10,7 @@ class OfflineImapError(Exception): * **REPO**: Abort repository sync, continue with next account * **CRITICAL**: Immediately exit offlineimap """ + MESSAGE, FOLDER_RETRY, FOLDER, REPO, CRITICAL = 0, 10, 15, 20, 30 def __init__(self, reason, severity, errcode=None): @@ -26,6 +27,7 @@ class OfflineImapError(Exception): value). So far, no errcodes have been defined yet. :type severity: OfflineImapError.ERROR value""" + self.errcode = errcode self.severity = severity diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 106c390..2e1b95f 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -1,5 +1,5 @@ # Base folder support -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,7 +23,6 @@ import offlineimap.accounts import os.path import re from sys import exc_info -import traceback class BaseFolder(object): @@ -113,6 +112,7 @@ class BaseFolder(object): def quickchanged(self, statusfolder): """ Runs quick check for folder changes and returns changed status: True -- changed, False -- not changed. + :param statusfolder: keeps track of the last known folder state. """ return True @@ -129,11 +129,13 @@ class BaseFolder(object): return 1 def getvisiblename(self): - """The nametrans-transposed name of the folder's name""" + """The nametrans-transposed name of the folder's name.""" + return self.visiblename def getexplainedname(self): - """ Name that shows both real and nametrans-mangled values""" + """Name that shows both real and nametrans-mangled values.""" + if self.name == self.visiblename: return self.name else: @@ -603,6 +605,7 @@ class BaseFolder(object): :param new_uid: (optional) If given, the old UID will be changed to a new UID. This allows backends efficient renaming of messages if the UID has changed.""" + raise NotImplementedError def deletemessage(self, uid): @@ -610,6 +613,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + raise NotImplementedError def deletemessages(self, uidlist): @@ -617,6 +621,7 @@ class BaseFolder(object): Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" + for uid in uidlist: self.deletemessage(uid) @@ -632,6 +637,7 @@ class BaseFolder(object): :param statusfolder: A LocalStatusFolder instance :param register: whether we should register a new thread." :returns: Nothing on success, or raises an Exception.""" + # Sometimes, it could be the case that if a sync takes awhile, # a message might be deleted from the maildir before it can be # synced to the status cache. This is only a problem with diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 1dccf90..5f3d32d 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -1,5 +1,5 @@ # Local status cache virtual folder -# Copyright (C) 2002 - 2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,10 +15,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from .Base import BaseFolder import os import threading +from .Base import BaseFolder class LocalStatusFolder(BaseFolder): """LocalStatus backend implemented as a plain text file""" @@ -33,9 +33,9 @@ class LocalStatusFolder(BaseFolder): self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self.messagelist = {} self.savelock = threading.Lock() - self.doautosave = self.config.getdefaultboolean("general", "fsync", - False) - """Should we perform fsyncs as often as possible?""" + # Should we perform fsyncs as often as possible? + self.doautosave = self.config.getdefaultboolean( + "general", "fsync", False) # Interface from BaseFolder def storesmessages(self): @@ -63,13 +63,12 @@ class LocalStatusFolder(BaseFolder): def readstatus_v1(self, fp): - """ - Read status folder in format version 1. + """Read status folder in format version 1. Arguments: - fp: I/O object that points to the opened database file. - """ + for line in fp.xreadlines(): line = line.strip() try: @@ -86,13 +85,12 @@ class LocalStatusFolder(BaseFolder): def readstatus(self, fp): - """ - Read status file in the current format. + """Read status file in the current format. Arguments: - fp: I/O object that points to the opened database file. - """ + for line in fp.xreadlines(): line = line.strip() try: @@ -164,11 +162,13 @@ class LocalStatusFolder(BaseFolder): def save(self): - """Save changed data to disk. For this backend it is the same as saveall""" + """Save changed data to disk. For this backend it is the same as saveall.""" + self.saveall() def saveall(self): - """Saves the entire messagelist to disk""" + """Saves the entire messagelist to disk.""" + with self.savelock: file = open(self.filename + ".tmp", "wt") file.write((self.magicline % self.cur_version) + "\n") @@ -198,6 +198,7 @@ class LocalStatusFolder(BaseFolder): See folder/Base for detail. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode.""" + if uid < 0: # We cannot assign a uid. return uid @@ -235,6 +236,7 @@ class LocalStatusFolder(BaseFolder): def savemessageslabelsbulk(self, labels): """Saves labels from a dictionary in a single database operation.""" + for uid, lb in labels.items(): self.messagelist[uid]['labels'] = lb self.save() @@ -254,6 +256,7 @@ class LocalStatusFolder(BaseFolder): def savemessagesmtimebulk(self, mtimes): """Saves mtimes from the mtimes dictionary in a single database operation.""" + for uid, mt in mtimes.items(): self.messagelist[uid]['mtime'] = mt self.save() diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 156b39e..88ece8f 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -1,5 +1,5 @@ # Maildir folder support -# Copyright (C) 2002 - 2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,15 +19,12 @@ import socket import time import re import os -import tempfile from .Base import BaseFolder from threading import Lock - try: from hashlib import md5 except ImportError: from md5 import md5 - try: # python 2.6 has set() built in set except NameError: @@ -131,6 +128,7 @@ class MaildirFolder(BaseFolder): :returns: (prefix, UID, FMD5, flags). UID is a numeric "long" type. flags is a set() of Maildir flags""" + prefix, uid, fmd5, flags = None, None, None, set() prefixmatch = self.re_prefixmatch.match(filename) if prefixmatch: @@ -227,7 +225,8 @@ class MaildirFolder(BaseFolder): # Interface from BaseFolder def getmessage(self, uid): - """Return the content of the message""" + """Return the content of the message.""" + filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) file = open(filepath, 'rt') @@ -249,6 +248,7 @@ class MaildirFolder(BaseFolder): :param uid: The UID`None`, or a set of maildir flags :param flags: A set of maildir flags :returns: String containing unique message filename""" + timeval, timeseq = _gettimeseq() return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s' % \ (timeval, timeseq, os.getpid(), socket.gethostname(), @@ -256,8 +256,7 @@ class MaildirFolder(BaseFolder): def save_to_tmp_file(self, filename, content): - """ - Saves given content to the named temporary file in the + """Saves given content to the named temporary file in the 'tmp' subdirectory of $CWD. Arguments: @@ -265,9 +264,7 @@ class MaildirFolder(BaseFolder): - content: data to be saved. Returns: relative path to the temporary file - that was created. - - """ + that was created.""" tmpname = os.path.join('tmp', filename) # open file and write it out @@ -364,7 +361,7 @@ class MaildirFolder(BaseFolder): infomatch = self.re_flagmatch.search(filename) if infomatch: filename = filename[:-len(infomatch.group())] #strip off - infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) + infostr = '%s2,%s'% (self.infosep, ''.join(sorted(flags))) filename += infostr newfilename = os.path.join(dir_prefix, filename) @@ -386,8 +383,10 @@ class MaildirFolder(BaseFolder): This will not update the statusfolder UID, you need to do that yourself. :param new_uid: (optional) If given, the old UID will be changed - to a new UID. The Maildir backend can implement this as an efficient - rename.""" + to a new UID. The Maildir backend can implement this as + an efficient rename. + """ + if not uid in self.messagelist: raise OfflineImapError("Cannot change unknown Maildir UID %s" % uid) if uid == new_uid: return diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 10173f7..7815793 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -1,5 +1,5 @@ # Base folder support -# Copyright (C) 2002-2012 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -59,8 +59,8 @@ class MappedIMAPFolder(IMAPFolder): try: line = line.strip() except ValueError: - raise Exception("Corrupt line '%s' in UID mapping file '%s'" \ - %(line, mapfilename)) + raise Exception("Corrupt line '%s' in UID mapping file '%s'"% + (line, mapfilename)) (str1, str2) = line.split(':') loc = long(str1) rem = long(str2) @@ -89,7 +89,7 @@ class MappedIMAPFolder(IMAPFolder): raise OfflineImapError("Could not find UID for msg '{0}' (f:'{1}'." " This is usually a bad thing and should be reported on the ma" "iling list.".format(e.args[0], self), - OfflineImapError.ERROR.MESSAGE) + OfflineImapError.ERROR.MESSAGE) # Interface from BaseFolder def cachemessagelist(self): diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 05aaf7d..e012a01 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -1,6 +1,5 @@ # imaplib utilities -# Copyright (C) 2002-2007 John Goerzen -# 2012-2012 Sebastian Spaeth +# Copyright (C) 2002-2015 John Goerzen & contributors # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -33,11 +32,12 @@ class UsefulIMAPMixIn(object): return self.mailbox return None - def select(self, mailbox='INBOX', readonly=False, force = False): + def select(self, mailbox='INBOX', readonly=False, force=False): """Selects a mailbox on the IMAP server :returns: 'OK' on success, nothing if the folder was already - selected or raises an :exc:`OfflineImapError`""" + selected or raises an :exc:`OfflineImapError`.""" + if self.__getselectedfolder() == mailbox and self.is_readonly == readonly \ and not force: # No change; return. @@ -67,6 +67,7 @@ class UsefulIMAPMixIn(object): def _mesg(self, s, tn=None, secs=None): new_mesg(self, s, tn, secs) + class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): """IMAP4 client class over a tunnel @@ -80,6 +81,7 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): def open(self, host, port): """The tunnelcmd comes in on host!""" + self.host = host self.process = subprocess.Popen(host, shell=True, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) @@ -90,7 +92,8 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): self.set_nonblocking(self.read_fd) def set_nonblocking(self, fd): - "Mark fd as nonblocking" + """Mark fd as nonblocking""" + # get the file's current flag settings fl = fcntl.fcntl(fd, fcntl.F_GETFL) # clear non-blocking mode from flags @@ -115,10 +118,8 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): if self.compressor is not None: data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) - self.outfd.write(data) - def shutdown(self): self.infd.close() self.outfd.close() @@ -135,7 +136,8 @@ def new_mesg(self, s, tn=None, secs=None): class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): - """Improved version of imaplib.IMAP4_SSL overriding select()""" + """Improved version of imaplib.IMAP4_SSL overriding select().""" + def __init__(self, *args, **kwargs): self._fingerprint = kwargs.get('fingerprint', None) if type(self._fingerprint) != type([]): @@ -146,32 +148,34 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): def open(self, host=None, port=None): if not self.ca_certs and not self._fingerprint: - raise OfflineImapError("No CA certificates " + \ - "and no server fingerprints configured. " + \ - "You must configure at least something, otherwise " + \ + raise OfflineImapError("No CA certificates " + "and no server fingerprints configured. " + "You must configure at least something, otherwise " "having SSL helps nothing.", OfflineImapError.ERROR.REPO) super(WrappedIMAP4_SSL, self).open(host, port) if self._fingerprint: # compare fingerprints fingerprint = sha1(self.sock.getpeercert(True)).hexdigest() if fingerprint not in self._fingerprint: - raise OfflineImapError("Server SSL fingerprint '%s' " % fingerprint + \ - "for hostname '%s' " % host + \ - "does not match configured fingerprint(s) %s. " % self._fingerprint + \ - "Please verify and set 'cert_fingerprint' accordingly " + \ - "if not set yet.", OfflineImapError.ERROR.REPO) + raise OfflineImapError("Server SSL fingerprint '%s' " + "for hostname '%s' " + "does not match configured fingerprint(s) %s. " + "Please verify and set 'cert_fingerprint' accordingly " + "if not set yet."% + (fingerprint, host, self._fingerprint), + OfflineImapError.ERROR.REPO) class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): - """Improved version of imaplib.IMAP4 overriding select()""" + """Improved version of imaplib.IMAP4 overriding select().""" + pass def Internaldate2epoch(resp): """Convert IMAP4 INTERNALDATE to UT. - Returns seconds since the epoch. - """ + Returns seconds since the epoch.""" mo = InternalDate.match(resp) if not mo: diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index c8e3f76..24fcfea 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -1,6 +1,5 @@ # IMAP utility module -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,8 +36,8 @@ def dequote(string): """Takes string which may or may not be quoted and unquotes it. It only considers double quotes. This function does NOT consider - parenthised lists to be quoted. - """ + parenthised lists to be quoted.""" + if string and string.startswith('"') and string.endswith('"'): string = string[1:-1] # Strip off the surrounding quotes. string = string.replace('\\"', '"') @@ -49,8 +48,8 @@ def quote(string): """Takes an unquoted string and quotes it. It only adds double quotes. This function does NOT consider - parenthised lists to be quoted. - """ + parenthised lists to be quoted.""" + string = string.replace('"', '\\"') string = string.replace('\\', '\\\\') return '"%s"' % string @@ -62,12 +61,14 @@ def flagsplit(string): (FLAGS (\\Seen Old) UID 4807) returns ['FLAGS,'(\\Seen Old)','UID', '4807'] """ + if string[0] != '(' or string[-1] != ')': raise ValueError("Passed string '%s' is not a flag list" % string) return imapsplit(string[1:-1]) def __options2hash(list): """convert list [1,2,3,4,5,6] to {1:2, 3:4, 5:6}""" + # effectively this does dict(zip(l[::2],l[1::2])), however # measurements seemed to have indicated that the manual variant is # faster for mosly small lists. @@ -84,6 +85,7 @@ def flags2hash(flags): E.g. '(FLAGS (\\Seen Old) UID 4807)' leads to {'FLAGS': '(\\Seen Old)', 'UID': '4807'}""" + return __options2hash(flagsplit(flags)) def imapsplit(imapstring): @@ -182,7 +184,8 @@ flagmap = [('\\Seen', 'S'), ('\\Draft', 'D')] def flagsimap2maildir(flagstring): - """Convert string '(\\Draft \\Deleted)' into a flags set(DR)""" + """Convert string '(\\Draft \\Deleted)' into a flags set(DR).""" + retval = set() imapflaglist = flagstring[1:-1].split() for imapflag, maildirflag in flagmap: @@ -191,7 +194,8 @@ def flagsimap2maildir(flagstring): return retval def flagsmaildir2imap(maildirflaglist): - """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'""" + """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'.""" + retval = [] for imapflag, maildirflag in flagmap: if maildirflag in maildirflaglist: @@ -203,7 +207,8 @@ def uid_sequence(uidlist): [1,2,3,4,5,10,12,13] will return "1:5,10,12:13". This function sorts the list, and only collapses if subsequent entries form a range. - :returns: The collapsed UID list as string""" + :returns: The collapsed UID list as string.""" + def getrange(start, end): if start == end: return(str(start)) @@ -230,8 +235,7 @@ def uid_sequence(uidlist): def __split_quoted(string): - """ - Looks for the ending quote character in the string that starts + """Looks for the ending quote character in the string that starts with quote character, splitting out quoted component and the rest of the string (without possible space between these two parts. @@ -241,7 +245,6 @@ def __split_quoted(string): Examples: - "this is \" a test" (\\None) => ("this is \" a test", (\\None)) - "\\" => ("\\", ) - """ if len(string) == 0: @@ -269,17 +272,15 @@ def __split_quoted(string): def format_labels_string(header, labels): - """ - Formats labels for embedding into a message, + """Formats labels for embedding into a message, with format according to header name. Headers from SPACE_SEPARATED_LABEL_HEADERS keep space-separated list of labels, the rest uses comma (',') as the separator. Also see parse_labels_string() and modify it accordingly - if logics here gets changed. + if logics here gets changed.""" - """ if header in SPACE_SEPARATED_LABEL_HEADERS: sep = ' ' else: @@ -289,18 +290,16 @@ def format_labels_string(header, labels): def parse_labels_string(header, labels_str): - """ - Parses a string into a set of labels, with a format according to + """Parses a string into a set of labels, with a format according to the name of the header. See __format_labels_string() for explanation on header handling and keep these two functions synced with each other. TODO: add test to ensure that - format_labels_string * parse_labels_string is unity + - format_labels_string * parse_labels_string is unity and - parse_labels_string * format_labels_string is unity - + - parse_labels_string * format_labels_string is unity """ if header in SPACE_SEPARATED_LABEL_HEADERS: @@ -314,15 +313,13 @@ def parse_labels_string(header, labels_str): def labels_from_header(header_name, header_value): - """ - Helper that builds label set from the corresponding header value. + """Helper that builds label set from the corresponding header value. Arguments: - header_name: name of the header that keeps labels; - header_value: value of the said header, can be None Returns: set of labels parsed from the header (or empty set). - """ if header_value: diff --git a/offlineimap/init.py b/offlineimap/init.py index d9425d5..143287b 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -1,5 +1,5 @@ # OfflineIMAP initialization code -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -214,7 +214,7 @@ class OfflineImap: config.set(section, key, value) #which ui to use? cmd line option overrides config file - ui_type = config.getdefault('general','ui', 'ttyui') + ui_type = config.getdefault('general', 'ui', 'ttyui') if options.interface != None: ui_type = options.interface if '.' in ui_type: @@ -222,13 +222,13 @@ class OfflineImap: ui_type = ui_type.split('.')[-1] # TODO, make use of chosen ui for logging logging.warning('Using old interface name, consider using one ' - 'of %s' % ', '.join(UI_LIST.keys())) + 'of %s'% ', '.join(UI_LIST.keys())) if options.diagnostics: ui_type = 'basic' # enforce basic UI for --info #dry-run? Set [general]dry-run=True if options.dryrun: - dryrun = config.set('general','dry-run', "True") - config.set_if_not_exists('general','dry-run','False') + dryrun = config.set('general', 'dry-run', 'True') + config.set_if_not_exists('general', 'dry-run', 'False') try: # create the ui class @@ -264,7 +264,7 @@ class OfflineImap: imaplib.Debug = 5 if options.runonce: - # FIXME: maybe need a better + # FIXME: spaghetti code alert! for section in accounts.getaccountlist(config): config.remove_option('Account ' + section, "autorefresh") @@ -275,7 +275,7 @@ class OfflineImap: #custom folder list specified? if options.folders: foldernames = options.folders.split(",") - folderfilter = "lambda f: f in %s" % foldernames + folderfilter = "lambda f: f in %s"% foldernames folderincludes = "[]" for accountname in accounts.getaccountlist(config): account_section = 'Account ' + accountname @@ -355,12 +355,12 @@ class OfflineImap: "take a few seconds)...") accounts.Account.set_abort_event(self.config, 3) elif sig == signal.SIGQUIT: - stacktrace.dump (sys.stderr) + stacktrace.dump(sys.stderr) os.abort() - signal.signal(signal.SIGHUP,sig_handler) - signal.signal(signal.SIGUSR1,sig_handler) - signal.signal(signal.SIGUSR2,sig_handler) + signal.signal(signal.SIGHUP, sig_handler) + signal.signal(signal.SIGUSR1, sig_handler) + signal.signal(signal.SIGUSR2, sig_handler) signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) signal.signal(signal.SIGQUIT, sig_handler) @@ -394,7 +394,7 @@ class OfflineImap: for accountname in accs: account = offlineimap.accounts.SyncableAccount(self.config, accountname) - threading.currentThread().name = "Account sync %s" % accountname + threading.currentThread().name = "Account sync %s"% accountname account.syncrunner() def __serverdiagnostics(self, options): diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py index 176a760..936a110 100644 --- a/offlineimap/mbnames.py +++ b/offlineimap/mbnames.py @@ -1,6 +1,6 @@ # Mailbox name generator -# Copyright (C) 2002 John Goerzen -# +# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -49,6 +49,7 @@ def write(): def __genmbnames(): """Takes a configparser object and a boxlist, which is a list of hashes containing 'accountname' and 'foldername' keys.""" + xforms = [os.path.expanduser, os.path.expandvars] mblock.acquire() try: diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index d7a6866..4edd6c5 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -1,5 +1,5 @@ # Base repository support -# Copyright (C) 2002-2012 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ import re import os.path from sys import exc_info + from offlineimap import CustomConfig from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError @@ -113,6 +114,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): @property def readonly(self): """Is the repository readonly?""" + return self._readonly def getlocaleval(self): @@ -120,11 +122,13 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getfolders(self): """Returns a list of ALL folders on this server.""" + return [] def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" + pass def getsep(self): @@ -132,6 +136,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def should_sync_folder(self, fname): """Should this folder be synced?""" + return fname in self.folderincludes or self.folderfilter(fname) def get_create_folders(self): @@ -139,11 +144,13 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): It is disabled by either setting the whole repository 'readonly' or by using the 'createfolders' setting.""" + return (not self._readonly) and \ self.getconfboolean('createfolders', True) def makefolder(self, foldername): """Create a new folder""" + raise NotImplementedError def deletefolder(self, foldername): diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 32cf3ac..d664259 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -1,5 +1,5 @@ # IMAP repository support -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,17 +15,19 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap.repository.Base import BaseRepository -from offlineimap import folder, imaputil, imapserver, OfflineImapError -from offlineimap.folder.UIDMaps import MappedIMAPFolder -from offlineimap.threadutil import ExitNotifyThread -from offlineimap.utils.distro import get_os_sslcertfile from threading import Event import os from sys import exc_info import netrc import errno +from offlineimap.repository.Base import BaseRepository +from offlineimap import folder, imaputil, imapserver, OfflineImapError +from offlineimap.folder.UIDMaps import MappedIMAPFolder +from offlineimap.threadutil import ExitNotifyThread +from offlineimap.utils.distro import get_os_sslcertfile + + class IMAPRepository(BaseRepository): def __init__(self, reposname, account): """Initialize an IMAPRepository object.""" @@ -116,14 +118,10 @@ class IMAPRepository(BaseRepository): "'%s' specified." % self, OfflineImapError.ERROR.REPO) - def get_remote_identity(self): - """ - Remote identity is used for certain SASL mechanisms + """Remote identity is used for certain SASL mechanisms (currently -- PLAIN) to inform server about the ID - we want to authorize as instead of our login name. - - """ + we want to authorize as instead of our login name.""" return self.getconf('remote_identity', default=None) @@ -218,13 +216,10 @@ class IMAPRepository(BaseRepository): return self.getconf('ssl_version', None) def get_ssl_fingerprint(self): - """ - Return array of possible certificate fingerprints. + """Return array of possible certificate fingerprints. Configuration item cert_fingerprint can contain multiple - comma-separated fingerprints in hex form. - - """ + comma-separated fingerprints in hex form.""" value = self.getconf('cert_fingerprint', "") return [f.strip().lower() for f in value.split(',') if f] @@ -262,8 +257,8 @@ class IMAPRepository(BaseRepository): 5. read password from /etc/netrc On success we return the password. - If all strategies fail we return None. - """ + If all strategies fail we return None.""" + # 1. evaluate Repository 'remotepasseval' passwd = self.getconf('remotepasseval', None) if passwd != None: @@ -304,7 +299,6 @@ class IMAPRepository(BaseRepository): # no strategy yielded a password! return None - def getfolder(self, foldername): return self.getfoldertype()(self.imapserver, foldername, self) @@ -392,6 +386,7 @@ class IMAPRepository(BaseRepository): when you are done creating folders yourself. :param foldername: Full path of the folder to be created.""" + if self.getreference(): foldername = self.getreference() + self.getsep() + foldername if not foldername: # Create top level folder as folder separator diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index ba29d37..52ba714 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -1,6 +1,5 @@ # Local status cache repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -81,7 +80,7 @@ class LocalStatusRepository(BaseRepository): return '.' def makefolder(self, foldername): - """Create a LocalStatus Folder""" + """Create a LocalStatus Folder.""" if self.account.dryrun: return # bail out in dry-run mode @@ -114,9 +113,11 @@ class LocalStatusRepository(BaseRepository): (see getfolderfilename) so we can not derive folder names from the file names that we have available. TODO: need to store a list of folder names somehow?""" + pass def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" + self._folders = {} diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index dae811d..f0495fa 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -1,6 +1,5 @@ # Maildir repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,6 +26,7 @@ class MaildirRepository(BaseRepository): def __init__(self, reposname, account): """Initialize a MaildirRepository object. Takes a path name to the directory holding all the Maildir directories.""" + BaseRepository.__init__(self, reposname, account) self.root = self.getlocalroot() @@ -41,6 +41,7 @@ class MaildirRepository(BaseRepository): def _append_folder_atimes(self, foldername): """Store the atimes of a folder's new|cur in self.folder_atimes""" + p = os.path.join(self.root, foldername) new = os.path.join(p, 'new') cur = os.path.join(p, 'cur') @@ -51,6 +52,7 @@ class MaildirRepository(BaseRepository): """Sets folders' atime back to their values after a sync Controlled by the 'restoreatime' config parameter.""" + if not self.getconfboolean('restoreatime', False): return # not configured to restore @@ -82,6 +84,7 @@ class MaildirRepository(BaseRepository): levels will be created if they do not exist yet. 'cur', 'tmp', and 'new' subfolders will be created in the maildir. """ + self.ui.makefolder(self, foldername) if self.account.dryrun: return @@ -134,7 +137,7 @@ class MaildirRepository(BaseRepository): "folder '%s'." % foldername, OfflineImapError.ERROR.FOLDER) - def _getfolders_scandir(self, root, extension = None): + def _getfolders_scandir(self, root, extension=None): """Recursively scan folder 'root'; return a list of MailDirFolder :param root: (absolute) path to Maildir root @@ -200,4 +203,5 @@ class MaildirRepository(BaseRepository): def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" + self.folders = None diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 6cc1855..fb3da80 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -1,5 +1,5 @@ # Curses-based interfaces -# Copyright (C) 2003-2011 John Goerzen & contributors +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,12 +22,13 @@ import sys import os import curses import logging + from offlineimap.ui.UIBase import UIBase from offlineimap.threadutil import ExitNotifyThread import offlineimap -class CursesUtil: +class CursesUtil: def __init__(self, *args, **kwargs): # iolock protects access to the self.iolock = RLock() @@ -322,6 +323,7 @@ class Blinkenlights(UIBase, CursesUtil): Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" + # create console handler with a higher log level ch = CursesLogHandler() #ch.setLevel(logging.DEBUG) @@ -336,6 +338,7 @@ class Blinkenlights(UIBase, CursesUtil): def isusable(s): """Returns true if the backend is usable ie Curses works""" + # Not a terminal? Can't use curses. if not sys.stdout.isatty() and sys.stdin.isatty(): return False @@ -391,6 +394,7 @@ class Blinkenlights(UIBase, CursesUtil): def acct(self, *args): """Output that we start syncing an account (and start counting)""" + self.gettf().setcolor('purple') super(Blinkenlights, self).acct(*args) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 01abd6f..5cf7449 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -1,4 +1,4 @@ -# Copyright (C) 2007-2011 John Goerzen & contributors +# Copyright (C) 2007-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index efde74f..5fa4dde 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -1,5 +1,5 @@ # TTY UI -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +24,7 @@ from offlineimap.ui.UIBase import UIBase class TTYFormatter(logging.Formatter): """Specific Formatter that adds thread information to the log output""" + def __init__(self, *args, **kwargs): #super() doesn't work in py2.6 as 'logging' uses old-style class logging.Formatter.__init__(self, *args, **kwargs) @@ -46,12 +47,14 @@ class TTYFormatter(logging.Formatter): log_str = " %s" % log_str return log_str + class TTYUI(UIBase): def setup_consolehandler(self): """Backend specific console handler Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" + # create console handler with a higher log level ch = logging.StreamHandler() #ch.setLevel(logging.DEBUG) @@ -67,10 +70,12 @@ class TTYUI(UIBase): def isusable(self): """TTYUI is reported as usable when invoked on a terminal""" + return sys.stdout.isatty() and sys.stdin.isatty() - def getpass(self, accountname, config, errmsg = None): + def getpass(self, accountname, config, errmsg=None): """TTYUI backend is capable of querying the password""" + if errmsg: self.warn("%s: %s" % (accountname, errmsg)) self._log_con_handler.acquire() # lock the console output @@ -97,6 +102,7 @@ class TTYUI(UIBase): implementations return 0 for successful sleep and 1 for an 'abort', ie a request to sync immediately. """ + if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: self.logger.info("Next refresh in %.1f minutes" % ( diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 0edfa18..f6007da 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -1,5 +1,5 @@ # UI base class -# Copyright (C) 2002-2011 John Goerzen & contributors +# Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -45,22 +45,22 @@ def getglobalui(): return globalui class UIBase(object): - def __init__(self, config, loglevel = logging.INFO): + def __init__(self, config, loglevel=logging.INFO): self.config = config # Is this a 'dryrun'? self.dryrun = config.getdefaultboolean('general', 'dry-run', False) self.debuglist = [] - """list of debugtypes we are supposed to log""" + # list of debugtypes we are supposed to log self.debugmessages = {} - """debugmessages in a deque(v) per thread(k)""" + # debugmessages in a deque(v) per thread(k) self.debugmsglen = 15 self.threadaccounts = {} - """dict linking active threads (k) to account names (v)""" + # dict linking active threads (k) to account names (v) self.acct_startimes = {} - """linking active accounts with the time.time() when sync started""" + # linking active accounts with the time.time() when sync started self.logfile = None self.exc_queue = Queue() - """saves all occuring exceptions, so we can output them at the end""" + # saves all occuring exceptions, so we can output them at the end # create logger with 'OfflineImap' app self.logger = logging.getLogger('OfflineImap') self.logger.setLevel(loglevel) @@ -73,6 +73,7 @@ class UIBase(object): Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" + # create console handler with a higher log level ch = logging.StreamHandler(sys.stdout) #ch.setLevel(logging.DEBUG) @@ -94,12 +95,13 @@ class UIBase(object): # write out more verbose initial info blurb on the log file p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ - "Args: %s" % (offlineimap.__bigversion__, p_ver, sys.platform, + "Args: %s"% (offlineimap.__bigversion__, p_ver, sys.platform, " ".join(sys.argv)) self.logger.info(msg) def _msg(self, msg): """Display a message.""" + # TODO: legacy function, rip out. self.info(msg) @@ -149,7 +151,8 @@ class UIBase(object): self._msg(traceback.format_tb(instant_traceback)) def registerthread(self, account): - """Register current thread as being associated with an account name""" + """Register current thread as being associated with an account name.""" + cur_thread = threading.currentThread() if cur_thread in self.threadaccounts: # was already associated with an old account, update info @@ -162,15 +165,17 @@ class UIBase(object): self.threadaccounts[cur_thread] = account def unregisterthread(self, thr): - """Unregister a thread as being associated with an account name""" + """Unregister a thread as being associated with an account name.""" + if thr in self.threadaccounts: del self.threadaccounts[thr] self.debug('thread', "Unregister thread '%s'" % thr.getName()) - def getthreadaccount(self, thr = None): + def getthreadaccount(self, thr=None): """Get Account() for a thread (current if None) - If no account has been registered with this thread, return 'None'""" + If no account has been registered with this thread, return 'None'.""" + if thr == None: thr = threading.currentThread() if thr in self.threadaccounts: @@ -214,6 +219,7 @@ class UIBase(object): """Return the type of a repository or Folder as string (IMAP, Gmail, Maildir, etc...)""" + prelimname = object.__class__.__name__.split('.')[-1] # Strip off extra stuff. return re.sub('(Folder|Repository)', '', prelimname) @@ -222,6 +228,7 @@ class UIBase(object): """Returns true if this UI object is usable in the current environment. For instance, an X GUI would return true if it's being run in X with a valid DISPLAY setting, and false otherwise.""" + return True ################################################## INPUT @@ -281,7 +288,8 @@ class UIBase(object): pass def connecting(self, hostname, port): - """Log 'Establishing connection to'""" + """Log 'Establishing connection to'.""" + if not self.logger.isEnabledFor(logging.INFO): return displaystr = '' hostname = hostname if hostname else '' @@ -291,19 +299,22 @@ class UIBase(object): self.logger.info("Establishing connection%s" % displaystr) def acct(self, account): - """Output that we start syncing an account (and start counting)""" + """Output that we start syncing an account (and start counting).""" + self.acct_startimes[account] = time.time() self.logger.info("*** Processing account %s" % account) def acctdone(self, account): - """Output that we finished syncing an account (in which time)""" + """Output that we finished syncing an account (in which time).""" + sec = time.time() - self.acct_startimes[account] del self.acct_startimes[account] self.logger.info("*** Finished account '%s' in %d:%02d" % (account, sec // 60, sec % 60)) def syncfolders(self, src_repo, dst_repo): - """Log 'Copying folder structure...'""" + """Log 'Copying folder structure...'.""" + if self.logger.isEnabledFor(logging.DEBUG): self.debug('', "Copying folder structure from %s to %s" %\ (src_repo, dst_repo)) @@ -328,12 +339,12 @@ class UIBase(object): def validityproblem(self, folder): self.logger.warning("UID validity problem for folder %s (repo %s) " "(saved %d; got %d); skipping it. Please see FAQ " - "and manual on how to handle this." % \ + "and manual on how to handle this."% \ (folder, folder.getrepository(), folder.get_saveduidvalidity(), folder.get_uidvalidity())) def loadmessagelist(self, repos, folder): - self.logger.debug("Loading message list for %s[%s]" % ( + self.logger.debug(u"Loading message list for %s[%s]"% ( self.getnicename(repos), folder)) @@ -389,7 +400,8 @@ class UIBase(object): self.logger.info("Collecting data from messages on %s" % source) def serverdiagnostics(self, repository, type): - """Connect to repository and output useful information for debugging""" + """Connect to repository and output useful information for debugging.""" + conn = None self._msg("%s repository '%s': type '%s'" % (type, repository.name, self.getnicename(repository))) @@ -440,8 +452,9 @@ class UIBase(object): repository.imapserver.close() def savemessage(self, debugtype, uid, flags, folder): - """Output a log line stating that we save a msg""" - self.debug(debugtype, "Write mail '%s:%d' with flags %s" % + """Output a log line stating that we save a msg.""" + + self.debug(debugtype, u"Write mail '%s:%d' with flags %s"% (folder, uid, repr(flags))) ################################################## Threads @@ -461,42 +474,46 @@ class UIBase(object): del self.debugmessages[thread] def getThreadExceptionString(self, thread): - message = "Thread '%s' terminated with exception:\n%s" % \ + message = u"Thread '%s' terminated with exception:\n%s"% \ (thread.getName(), thread.exit_stacktrace) - message += "\n" + self.getThreadDebugLog(thread) + message += u"\n" + self.getThreadDebugLog(thread) return message def threadException(self, thread): """Called when a thread has terminated with an exception. The argument is the ExitNotifyThread that has so terminated.""" + self.warn(self.getThreadExceptionString(thread)) self.delThreadDebugLog(thread) self.terminate(100) def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" + #print any exceptions that have occurred over the run if not self.exc_queue.empty(): - self.warn("ERROR: Exceptions occurred during the run!") + self.warn(u"ERROR: Exceptions occurred during the run!") while not self.exc_queue.empty(): msg, exc, exc_traceback = self.exc_queue.get() if msg: - self.warn("ERROR: %s\n %s" % (msg, exc)) + self.warn(u"ERROR: %s\n %s"% (msg, exc)) else: - self.warn("ERROR: %s" % (exc)) + self.warn(u"ERROR: %s"% (exc)) if exc_traceback: - self.warn("\nTraceback:\n%s" %"".join( + self.warn(u"\nTraceback:\n%s"% "".join( traceback.format_tb(exc_traceback))) if errormsg and errortitle: - self.warn('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) + self.warn(u'ERROR: %s\n\n%s\n'% (errortitle, errormsg)) elif errormsg: - self.warn('%s\n' % errormsg) + self.warn(u'%s\n' % errormsg) sys.exit(exitstatus) def threadExited(self, thread): - """Called when a thread has exited normally. Many UIs will - just ignore this.""" + """Called when a thread has exited normally. + + Many UIs will just ignore this.""" + self.delThreadDebugLog(thread) self.unregisterthread(thread) @@ -518,6 +535,7 @@ class UIBase(object): :returns: 0/False if timeout expired, 1/2/True if there is a request to cancel the timer. """ + abortsleep = False while sleepsecs > 0 and not abortsleep: if account.get_abort_event(): @@ -538,6 +556,7 @@ class UIBase(object): implementations return 0 for successful sleep and 1 for an 'abort', ie a request to sync immediately. """ + if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: self.logger.debug("Next refresh in %.1f minutes" % ( diff --git a/offlineimap/ui/debuglock.py b/offlineimap/ui/debuglock.py index 4756b08..ef6e825 100644 --- a/offlineimap/ui/debuglock.py +++ b/offlineimap/ui/debuglock.py @@ -1,6 +1,5 @@ # Locking debugging code -- temporary -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From dbb632275e75a9d0ab9319b92c589c1cce943191 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 7 Jan 2015 22:20:14 +0100 Subject: [PATCH 19/19] v6.5.7-rc1 Signed-off-by: Nicolas Sebrecht --- Changelog.rst | 111 +++++++++++++++++++++++++++++----------- offlineimap/__init__.py | 6 +-- 2 files changed, 85 insertions(+), 32 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 9a98d29..5580e69 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,10 +5,26 @@ ChangeLog :website: http://offlineimap.org -OfflineIMAP v6.5.6.1 (YYYY-MM-DD) -================================= +OfflineIMAP v6.5.7-rc1 (2015-01-07) +=================================== -* Properly generate tarball from "sdist" command (GitHub #137) +Notes +----- + +I think it's time for a new release candidate. Our release cycle are long +enough and users are asked to use the current TIP of the next branch to test +our recent patches. + +The current version makes better support for environment variable expansion and +improves OS portability. Gmail should be better supported: we are still +expecting feedbacks. Embedded library imaplib2 is updated to v2.37. +Debugging messages are added and polished. + +There's some code cleanups and refactoring, also. + + +Features +-------- * Expand environment variables in the following configuration items: @@ -21,39 +37,73 @@ OfflineIMAP v6.5.6.1 (YYYY-MM-DD) configuration items: - Repository.sslclientcert; - Repository.sslclientkey. - -* Updated bundled imaplib2 to 2.37: - - add missing idle_lock in _handler() - +* Support default CA bundle locations for a couple of + known Unix systems (Michael Vogt, GutHub pull #19) * Added default CA bundle location for OpenBSD (GitHub pull #120) and DragonFlyBSD. +Fixes +----- + +* Fix unbounded recursion during flag update (Josh Berry). +* Do not ignore gmail labels if header appears multiple times +* Delete gmail labels header before adding a new one +* Fix improper header separator for X-OfflineIMAP header +* Match header names case-insensitively +* Create SQLite database directory if it doesn't exist + yet; warn if path is not a directory (Nick Farrell, + GutHub pull #102) +* Properly manipulate contents of messagelist for folder +* Fix label processing in GmailMaildir +* Properly capitalize OpenSSL +* Fix warning-level message processing by MachineUI + (GitHub pull #64, GitHub pull #118). +* Properly generate tarball from "sdist" command (GitHub #137) +* Fix Markdown formatting +* Fix typo in apply_xforms invocation +* Merge pull request #136 from aroig/gh/label-fix +* Fix mangled message headers for servers without UIDPLUS: + X-OfflineIMAP was added with preceeding '\n' instead of + '\r\n' just before message was uploaded to the IMAP server. +* Add missing version bump for 6.5.6 (it was released with + 6.5.5 in setup.py and other places). + +Changes +------- + +* Warn about a tricky piece of code in addmessageheader +* Rename addmessageheader()'s crlf parameter to linebreak +* addmessageheader: fix case #2 and flesh out docstring +* addmessageheader(): add debug for header insertion +* Add version qualifier to differentiate releases and development ones +* More clearly show results of folder name translation +* IMAP: provide message-id in error messages +* Trade recursion by plain old cycle +* Avoid copying array every time, just slice it * Added OpenSSL exception clause to our main GPL to allow people to link with OpenSSL in run-time. It is needed at least for Debian, see https://lists.debian.org/debian-legal/2002/10/msg00113.html for details. +* Brought CustomConfig.py into more proper shape +* Updated bundled imaplib2 to 2.37: + - add missing idle_lock in _handler() +* Imaplib2: trade backticks to repr() +* Introduce CustomConfig method that applies set of transforms +* imaplibutil.py: remove unused imports +* CustomConfig.py: remove unused imports +* init.py: remove unused import +* repository/Base.py: remove unused import +* repository/GmailMaildir.py: remove unused import +* repository/LocalStatus.py: remove unused import +* ui/Curses.py: remove unused import +* ui/UIBase.py: remove unused import +* localeval: comment on security issues +* docs: remove obsolete comment about SubmittingPatches.rst +* utils/const.py: fix ident +* ui/UIBase: folderlist(): avoid built-in list() redefinition +* more consistent style -* Fix warning-level message processing by MachineUI - (GitHub pull #64, GitHub pull #118). - -* Support default CA bundle locations for a couple of - known Unix systems (Michael Vogt, GutHub pull #19) - -* Create SQLite database directory if it doesn't exist - yet; warn if path is not a directory (Nick Farrell, - GutHub pull #102) - -* Fix mangled message headers for servers without UIDPLUS: - X-OfflineIMAP was added with preceeding '\n' instead of - '\r\n' just before message was uploaded to the IMAP server. - -* Add missing version bump for 6.5.6 (it was released with - 6.5.5 in setup.py and other places). - -* Various fixes in documentation. - -* Fix unbounded recursion during flag update (Josh Berry). OfflineIMAP v6.5.6 (2014-05-14) @@ -272,7 +322,9 @@ OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" OfflineIMAP v6.5.0 (2012-01-06) =============================== -This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. Please upgrade to avoid potential data loss! The version has been bumped to 6.5.0, please let everyone know that the 6.4.x series is problematic. +This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. +Please upgrade to avoid potential data loss! The version has been bumped to +6.5.0, please let everyone know that the 6.4.x series is problematic. * Uploading multiple emails to an IMAP server would lead to wrong UIDs being returned (ie the same for all), which confused offlineimap and @@ -368,7 +420,8 @@ Bug Fixes OfflineIMAP v6.4.0 (2011-09-29) =============================== -This is the first stable release to support the forward-compatible per-account locks and remote folder creation that has been introduced in the 6.3.5 series. +This is the first stable release to support the forward-compatible per-account +locks and remote folder creation that has been introduced in the 6.3.5 series. * Various regression and bug fixes from the last couple of RCs diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 6a4e0e2..ff6acd9 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,10 +1,10 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.6.1" -__revision__ = "-devel" +__version__ = "6.5.7" +__revision__ = "-rc1" __bigversion__ = __version__ + __revision__ -__copyright__ = "Copyright 2002-2013 John Goerzen & contributors" +__copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support"