From 6b3f429c81f985b23df144053c9f767cd0137a1f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 30 Jun 2011 14:04:18 +0200 Subject: [PATCH 01/44] Split offlineimap.run() Was getting too large, split into an parse_cmd_options and a sync() function. Moving config and ui to self.config and self.ui to make them available through the OfflineImap instance. Signed-off-by: Sebastian Spaeth --- offlineimap/init.py | 108 ++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 27aeebd..065807f 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -44,9 +44,13 @@ class OfflineImap: """ def run(self): """Parse the commandline and invoke everything""" + # next line also sets self.config + options = self.parse_cmd_options() + self.sync(options) + def parse_cmd_options(self): parser = OptionParser(version=offlineimap.__version__, - description="%s.\n\n%s" % + description="%s.\n\n%s" % (offlineimap.__copyright__, offlineimap.__license__)) parser.add_option("-1", @@ -103,7 +107,7 @@ class OfflineImap: "Only sync the specified folders. The folder names " "are the *untranslated* foldernames. This " "command-line option overrides any 'folderfilter' " - "and 'folderincludes' options in the configuration " + "and 'folderincludes' options in the configuration " "file.") parser.add_option("-k", dest="configoverride", @@ -187,19 +191,19 @@ class OfflineImap: 'of %s' % ', '.join(UI_LIST.keys())) try: # create the ui class - ui = UI_LIST[ui_type.lower()](config) + self.ui = UI_LIST[ui_type.lower()](config) except KeyError: logging.error("UI '%s' does not exist, choose one of: %s" % \ (ui_type,', '.join(UI_LIST.keys()))) sys.exit(1) - setglobalui(ui) + setglobalui(self.ui) #set up additional log files if options.logfile: - ui.setlogfd(open(options.logfile, 'wt')) + self.ui.setlogfd(open(options.logfile, 'wt')) #welcome blurb - ui.init_banner() + self.ui.init_banner() if options.debugtype: if options.debugtype.lower() == 'all': @@ -207,13 +211,13 @@ class OfflineImap: #force single threading? if not ('thread' in options.debugtype.split(',') \ and not options.singlethreading): - ui._msg("Debug mode: Forcing to singlethreaded.") + self.ui._msg("Debug mode: Forcing to singlethreaded.") options.singlethreading = True debugtypes = options.debugtype.split(',') + [''] for type in debugtypes: type = type.strip() - ui.add_debug(type) + self.ui.add_debug(type) if type.lower() == 'imap': imaplib.Debug = 5 @@ -241,47 +245,64 @@ class OfflineImap: config.set(section, "folderfilter", folderfilter) config.set(section, "folderincludes", folderincludes) - self.config = config + if options.logfile: + sys.stderr = self.ui.logfile + socktimeout = config.getdefaultint("general", "socktimeout", 0) + if socktimeout > 0: + socket.setdefaulttimeout(socktimeout) + + threadutil.initInstanceLimit('ACCOUNTLIMIT', + config.getdefaultint('general', 'maxsyncaccounts', 1)) + + for reposname in config.getsectionlist('Repository'): + for instancename in ["FOLDER_" + reposname, + "MSGCOPY_" + reposname]: + if options.singlethreading: + threadutil.initInstanceLimit(instancename, 1) + else: + threadutil.initInstanceLimit(instancename, + config.getdefaultint('Repository ' + reposname, + 'maxconnections', 2)) + self.config = config + return options + + def sync(self, options): + """Invoke the correct single/multithread syncing + + self.config is supposed to have been correctly initialized + already.""" def sigterm_handler(signum, frame): # die immediately - ui = getglobalui() - ui.terminate(errormsg="terminating...") - - signal.signal(signal.SIGTERM,sigterm_handler) + self.ui.terminate(errormsg="terminating...") + signal.signal(signal.SIGTERM, sigterm_handler) try: - pidfd = open(config.getmetadatadir() + "/pid", "w") + pidfd = open(self.config.getmetadatadir() + "/pid", "w") pidfd.write(str(os.getpid()) + "\n") pidfd.close() except: pass - try: - if options.logfile: - sys.stderr = ui.logfile - - socktimeout = config.getdefaultint("general", "socktimeout", 0) - if socktimeout > 0: - socket.setdefaulttimeout(socktimeout) - - activeaccounts = config.get("general", "accounts") + try: + activeaccounts = self.config.get("general", "accounts") if options.accounts: activeaccounts = options.accounts activeaccounts = activeaccounts.replace(" ", "") activeaccounts = activeaccounts.split(",") - allaccounts = accounts.AccountHashGenerator(config) + allaccounts = accounts.AccountHashGenerator(self.config) syncaccounts = [] for account in activeaccounts: if account not in allaccounts: if len(allaccounts) == 0: - errormsg = 'The account "%s" does not exist because no accounts are defined!'%account + errormsg = "The account '%s' does not exist because no"\ + " accounts are defined!" % account else: - errormsg = 'The account "%s" does not exist. Valid accounts are:'%account - for name in allaccounts.keys(): - errormsg += '\n%s'%name - ui.terminate(1, errortitle = 'Unknown Account "%s"'%account, errormsg = errormsg) + errormsg = "The account '%s' does not exist. Valid ac"\ + "counts are: " % account + errormsg += ", ".join(allaccounts.keys()) + self.ui.terminate(1, errormsg = errormsg) if account not in syncaccounts: syncaccounts.append(account) @@ -289,19 +310,6 @@ class OfflineImap: remoterepos = None localrepos = None - threadutil.initInstanceLimit('ACCOUNTLIMIT', - config.getdefaultint('general', - 'maxsyncaccounts', 1)) - - for reposname in config.getsectionlist('Repository'): - for instancename in ["FOLDER_" + reposname, - "MSGCOPY_" + reposname]: - if options.singlethreading: - threadutil.initInstanceLimit(instancename, 1) - else: - threadutil.initInstanceLimit(instancename, - config.getdefaultint('Repository ' + reposname, - 'maxconnections', 2)) def sig_handler(sig, frame): if sig == signal.SIGUSR1 or sig == signal.SIGHUP: # tell each account to stop sleeping @@ -315,7 +323,7 @@ class OfflineImap: signal.signal(signal.SIGUSR2,sig_handler) #various initializations that need to be performed: - offlineimap.mbnames.init(config, syncaccounts) + offlineimap.mbnames.init(self.config, syncaccounts) #TODO: keep legacy lock for a few versions, then remove. self._legacy_lock = open(self.config.getmetadatadir() + "/lock", @@ -331,20 +339,20 @@ class OfflineImap: if options.singlethreading: #singlethreaded - self.sync_singlethreaded(syncaccounts, config) + self.sync_singlethreaded(syncaccounts) else: # multithreaded t = threadutil.ExitNotifyThread(target=syncmaster.syncitall, name='Sync Runner', kwargs = {'accounts': syncaccounts, - 'config': config}) + 'config': self.config}) t.setDaemon(1) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) - ui.terminate() + self.ui.terminate() except KeyboardInterrupt: - ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') + self.ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') return except (SystemExit): raise @@ -352,13 +360,13 @@ class OfflineImap: ui.error(e) ui.terminate() - def sync_singlethreaded(self, accs, config): + def sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread :param accs: A list of accounts that should be synced - :param config: The CustomConfig object """ for accountname in accs: - account = offlineimap.accounts.SyncableAccount(config, accountname) + account = offlineimap.accounts.SyncableAccount(self.config, + accountname) threading.currentThread().name = "Account sync %s" % accountname account.syncrunner() From 3885acf87dd4e14ad28ca63c4818114a1dc95e15 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 30 Jun 2011 15:18:03 +0200 Subject: [PATCH 02/44] Implement server diagnostics This outputs a handy summary of your server configuration and version strings etc, which is useful for bug reporting. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/accounts.py | 18 ++++++++++----- offlineimap/init.py | 36 +++++++++++++++++++++--------- offlineimap/ui/UIBase.py | 47 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 15 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5095b2e..7982706 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,6 +13,9 @@ others. New Features ------------ +* add a --info command line switch that outputs useful information about + the server and the configuration for all enabled accounts. + Changes ------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 71f08fe..1c5dd38 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -82,6 +82,9 @@ class Account(CustomConfig.ConfigHelperMixin): def __str__(self): return self.name + def getaccountmeta(self): + return os.path.join(self.metadatadir, 'Account-' + self.name) + def getsection(self): return 'Account ' + self.getname() @@ -154,8 +157,16 @@ class Account(CustomConfig.ConfigHelperMixin): self.quicknum = 0 return 1 return 0 - - + + def serverdiagnostics(self): + """Output diagnostics for all involved repositories""" + remote_repo = Repository(self, 'remote') + local_repo = Repository(self, 'local') + #status_repo = Repository(self, 'status') + self.ui.serverdiagnostics(remote_repo, 'Remote') + self.ui.serverdiagnostics(local_repo, 'Local') + #self.ui.serverdiagnostics(statusrepos, 'Status') + class SyncableAccount(Account): """A syncable email account connecting 2 repositories @@ -233,9 +244,6 @@ class SyncableAccount(Account): if looping and self.sleeper() >= 2: looping = 0 - def getaccountmeta(self): - return os.path.join(self.metadatadir, 'Account-' + self.name) - def sync(self): """Synchronize the account once, then return diff --git a/offlineimap/init.py b/offlineimap/init.py index 065807f..a7f82ca 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -44,9 +44,12 @@ class OfflineImap: """ def run(self): """Parse the commandline and invoke everything""" - # next line also sets self.config - options = self.parse_cmd_options() - self.sync(options) + # next line also sets self.config and self.ui + options, args = self.parse_cmd_options() + if options.diagnostics: + self.serverdiagnostics(options) + else: + self.sync(options) def parse_cmd_options(self): parser = OptionParser(version=offlineimap.__version__, @@ -143,6 +146,13 @@ class OfflineImap: "not usable. Possible interface choices are: %s " % ", ".join(UI_LIST.keys())) + parser.add_option("--info", + action="store_true", dest="diagnostics", + default=False, + help="Output information on the configured email repositories" + ". Useful for debugging and bug reporting. Use in conjunction wit" + "h the -a option to limit the output to a single account") + (options, args) = parser.parse_args() #read in configuration file @@ -265,7 +275,7 @@ class OfflineImap: config.getdefaultint('Repository ' + reposname, 'maxconnections', 2)) self.config = config - return options + return (options, args) def sync(self, options): """Invoke the correct single/multithread syncing @@ -276,7 +286,7 @@ class OfflineImap: # die immediately self.ui.terminate(errormsg="terminating...") signal.signal(signal.SIGTERM, sigterm_handler) - + try: pidfd = open(self.config.getmetadatadir() + "/pid", "w") pidfd.write(str(os.getpid()) + "\n") @@ -305,11 +315,7 @@ class OfflineImap: self.ui.terminate(1, errormsg = errormsg) if account not in syncaccounts: syncaccounts.append(account) - - server = None - remoterepos = None - localrepos = None - + def sig_handler(sig, frame): if sig == signal.SIGUSR1 or sig == signal.SIGHUP: # tell each account to stop sleeping @@ -370,3 +376,13 @@ class OfflineImap: accountname) threading.currentThread().name = "Account sync %s" % accountname account.syncrunner() + + def serverdiagnostics(self, options): + activeaccounts = self.config.get("general", "accounts") + if options.accounts: + activeaccounts = options.accounts + activeaccounts = activeaccounts.split(",") + allaccounts = accounts.AccountListGenerator(self.config) + for account in allaccounts: + if account.name not in activeaccounts: continue + account.serverdiagnostics() diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 6d92152..caf5efd 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -322,6 +322,53 @@ class UIBase: s._msg("Deleting flag %s from %d messages on %s" % \ (", ".join(flags), len(uidlist), dest)) + def serverdiagnostics(self, repository, type): + """Connect to repository and output useful information for debugging""" + conn = None + self._msg("%s repository '%s': type '%s'" % (type, repository.name, + self.getnicename(repository))) + try: + if hasattr(repository, 'gethost'): # IMAP + self._msg("Host: %s Port: %s SSL: %s" % (repository.gethost(), + repository.getport(), + repository.getssl())) + try: + conn = repository.imapserver.acquireconnection() + except OfflineImapError, e: + self._msg("Failed to connect. Reason %s" % e) + else: + if 'ID' in conn.capabilities: + self._msg("Server supports ID extension.") + #TODO: Debug and make below working, it hangs Gmail + #res_type, response = conn.id(( + # 'name', offlineimap.__productname__, + # 'version', offlineimap.__version__)) + #self._msg("Server ID: %s %s" % (res_type, response[0])) + self._msg("Server welcome string: %s" % str(conn.welcome)) + self._msg("Server capabilities: %s\n" % str(conn.capabilities)) + repository.imapserver.releaseconnection(conn) + if type != 'Status': + folderfilter = repository.getconf('folderfilter', None) + if folderfilter: + self._msg("folderfilter= %s\n" % folderfilter) + folderincludes = repository.getconf('folderincludes', None) + if folderincludes: + self._msg("folderincludes= %s\n" % folderincludes) + nametrans = repository.getconf('nametrans', None) + if nametrans: + self._msg("nametrans= %s\n" % nametrans) + + folders = repository.getfolders() + foldernames = [(f.name, f.getvisiblename()) for f in folders] + folders = [] + for name, visiblename in foldernames: + if name == visiblename: folders.append(name) + else: folders.append("%s -> %s" % (name, visiblename)) + self._msg("Folderlist: %s\n" % str(folders)) + finally: + if conn: #release any existing IMAP connection + repository.imapserver.close() + ################################################## Threads def getThreadDebugLog(s, thread): From 0ba9634205c005fba3915a76b2857dcb836afc9e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 10:59:07 +0200 Subject: [PATCH 03/44] Fix string formatting when port is not given. If port is None, we would try to format an empty string with %d wich fails. Fix it by using %s. Reported-by: Iain Dalton Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index caf5efd..b5dcdaa 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -232,7 +232,7 @@ class UIBase: if s.verbose < 0: return displaystr = '' hostname = hostname if hostname else '' - port = "%d" % port if port else '' + port = "%s" % port if port else '' if hostname: displaystr = ' to %s:%s' % (hostname, port) s._msg("Establishing connection%s" % displaystr) From 9578f291959a5762ed157a16a461853f2d51ed58 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 16:53:18 +0200 Subject: [PATCH 04/44] Use 'reference' value when creating an IMAP directory A repositories 'reference value is always prefixed to the full folder path, so we should do so when creating a new one. The code had existed but was commented out since 2003, I guess the "reference" option is not too often used. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 3 +-- offlineimap/repository/IMAP.py | 16 +++++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8dbd7aa..58ff633 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -1,6 +1,5 @@ # IMAP folder support -# Copyright (C) 2002-2007 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 2d2962f..4c2cadd 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -1,6 +1,5 @@ # IMAP repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -171,7 +170,7 @@ class IMAPRepository(BaseRepository): return self.getconf('preauthtunnel', None) def getreference(self): - return self.getconf('reference', '""') + return self.getconf('reference', '') def getidlefolders(self): localeval = self.localeval @@ -316,14 +315,9 @@ class IMAPRepository(BaseRepository): when you are done creating folders yourself. :param foldername: Full path of the folder to be created.""" - #TODO: IMHO this existing commented out code is correct and - #should be enabled, but this would change the behavior for - #existing configurations who have a 'reference' set on a Mapped - #IMAP server....: - #if self.getreference() != '""': - # newname = self.getreference() + self.getsep() + foldername - #else: - # newname = foldername + if self.getreference(): + foldername = self.getreference() + self.getsep() + foldername + imapobj = self.imapserver.acquireconnection() try: self.ui._msg("Creating new IMAP folder '%s' on server %s" %\ From e707bc530b52c5e8dd5f4862e40f996286ac179a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 3 Oct 2011 14:20:13 +0200 Subject: [PATCH 05/44] Fix quotation marks I guess they looked identical to whoever wrote that, but here the the first had a space with a combining grave accent where the second has a plain space and a plain backquote instead. Reported-by: Daniel Shahaf Signed-off-by: Sebastian Spaeth --- docs/MANUAL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index bb13478..49dedd6 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -478,7 +478,7 @@ or:: folder separators be replaced with the destination repositories' folder separator. -So if ̀f` was "Sent", the first nametrans yields the translated name +So if 'f' was "Sent", the first nametrans yields the translated name "INBOX/Sent" to be used on the other side. As that repository uses the folder separator '.' rather than '/', the ultimate name to be used will be "INBOX.Sent". From 76b0d7cf25ac3049529b3afda906b1207bd708db Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 5 Oct 2011 18:18:55 +0200 Subject: [PATCH 06/44] imapserver: Make noop() more resitent against dropped connections. Drop a connection, if the NOOP to keep a connection open fails due to broken connections. Note that I believe this function is not working as intended. We grab one random connection and send a NOOP. This is not enough to keep all connections open, and if we invoke this function multiple times, we might well always get the same connection to send a NOOP through. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ offlineimap/imapserver.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 7982706..1f6b46b 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -32,3 +32,6 @@ Bug Fixes * New folders on the remote would be skipped on the very sync run they are created and only by synced in subsequent runs. Fixed. + +* Make NOOPs to keep a server connection open more resistant against dropped + connections. diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index ef54e31..7006ef9 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -491,10 +491,24 @@ class IdleThread(object): self.thread.join() def noop(self): + #TODO: AFAIK this is not optimal, we will send a NOOP on one + #random connection (ie not enough to keep all connections + #open). In case we do the noop multiple times, we can well use + #the same connection every time, as we get a random one. This + #function should IMHO send a noop on ALL available connections + #to the server. imapobj = self.parent.acquireconnection() - imapobj.noop() - self.stop_sig.wait() - self.parent.releaseconnection(imapobj) + try: + imapobj.noop() + except imapobj.abort: + self.ui.warn('Attempting NOOP on dropped connection %s' % \ + imapobj.identifier) + self.parent.releaseconnection(imapobj, True) + imapobj = None + finally: + if imapobj: + self.parent.releaseconnection(imapobj) + self.stop_sig.wait() # wait until we are supposed to quit def dosync(self): remoterepos = self.parent.repos From b0fd6c6ab8caf8cb0f30af4c9df595a5fbddfbaf Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 21 Sep 2011 00:46:46 +0200 Subject: [PATCH 07/44] Do not import threading.* Only import the lock, that we actually need. Also import the with statement for use with python 2.5. We'll need it for sure in this file. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/UIDMaps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index fa1924d..5da8e00 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -15,8 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -from threading import * +from __future__ import with_statement # needed for python 2.5 +from threading import Lock from IMAP import IMAPFolder import os.path From 097b7ab2b03aaceba818e25d12856886333b10fd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 5 Oct 2011 18:41:38 +0200 Subject: [PATCH 08/44] createfolder: Fix wrong error output If folder creation failed, we would output the wrong repository and folder name (copy'n paste error). Fix this so we actually output the correct values. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 972aefb..8f400f0 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -207,9 +207,8 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): src_repo.makefolder(newsrc_name) src_haschanged = True # Need to refresh list except OfflineImapError, e: - self.ui.error(e, exc_info()[2], - "Creating folder %s on repository %s" %\ - (src_name, dst_repo)) + self.ui.error(e, exc_info()[2], "Creating folder %s on " + "repository %s" % (newsrc_name, src_repo)) raise status_repo.makefolder(newsrc_name.replace( src_repo.getsep(), status_repo.getsep())) From d4a11c62eaeff9726e7989d5fa2c589449957dce Mon Sep 17 00:00:00 2001 From: dtk Date: Wed, 12 Oct 2011 11:03:27 +0200 Subject: [PATCH 09/44] Fix typos in documentary comments in exemplary config file I stumbled upon a few typos in the config file coming with master and just patched them. Apply as you like. Signed-off-by: dtk Signed-off-by: Sebastian Spaeth --- offlineimap.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 224e933..b6ff6e6 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -55,7 +55,7 @@ metadata = ~/.offlineimap accounts = Test -# Offlineimap can synchronize more the one account at a time. If you +# Offlineimap can synchronize more than one account at a time. If you # want to enable this feature, set the below value to something # greater than 1. To force it to synchronize only one account at a # time, set it to 1. @@ -200,7 +200,7 @@ remoterepository = RemoteExample # quick = 10 # You can specify a pre and post sync hook to execute a external command. -# in this case a call to imapfilter to filter mail before the sync process +# In this case a call to imapfilter to filter mail before the sync process # starts and a custom shell script after the sync completes. # The pre sync script has to complete before a sync to the account will # start. @@ -445,7 +445,7 @@ holdconnectionopen = no # mark them deleted on the server, but not actually delete them. # You must use some other IMAP client to delete them if you use this # setting; otherwise, the messgaes will just pile up there forever. -# Therefore, this setting is definately NOT recommended. +# Therefore, this setting is definitely NOT recommended. # # expunge = no From cbec8bb5b20eb8051aaea2b20f81de9158dab975 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 26 Oct 2011 16:47:21 +0200 Subject: [PATCH 10/44] Rework UI system to make use of the logging module Logging was flawed as the output was e.g. heavily buffered and people complained about missing log entries. Fix this by making use of the standard logging facilities that offlineimap offers. This is one big ugly patch that does many things. It fixes the Blinkenlights backend to work again with the logging facilities. Resize windows and hotkeys are still not handled absolut correctly, this is left for future fixing. THe rest of the backends should be working fine. Signed-off-by: Sebastian Spaeth --- offlineimap/__init__.py | 6 +- offlineimap/accounts.py | 13 +- offlineimap/init.py | 31 +- offlineimap/syncmaster.py | 3 +- offlineimap/threadutil.py | 31 +- offlineimap/ui/Blinkenlights.py | 148 ----- offlineimap/ui/Curses.py | 964 ++++++++++++++++--------------- offlineimap/ui/Machine.py | 78 +-- offlineimap/ui/Noninteractive.py | 41 +- offlineimap/ui/TTY.py | 99 ++-- offlineimap/ui/UIBase.py | 358 ++++++------ offlineimap/ui/__init__.py | 2 +- 12 files changed, 829 insertions(+), 945 deletions(-) delete mode 100644 offlineimap/ui/Blinkenlights.py diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 46ff1d7..ea9aa0d 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -8,15 +8,11 @@ __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" __bigcopyright__ = """%(__productname__)s %(__version__)s -%(__copyright__)s. -%(__license__)s. -""" % locals() + %(__license__)s""" % locals() __homepage__ = "http://github.com/nicolas33/offlineimap" - banner = __bigcopyright__ - from offlineimap.error import OfflineImapError # put this last, so we don't run into circular dependencies using # e.g. offlineimap.__version__. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 1c5dd38..7387c51 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -143,7 +143,7 @@ class Account(CustomConfig.ConfigHelperMixin): for item in kaobjs: item.startkeepalive() - + refreshperiod = int(self.refreshperiod * 60) sleepresult = self.ui.sleep(refreshperiod, self) @@ -166,7 +166,8 @@ class Account(CustomConfig.ConfigHelperMixin): self.ui.serverdiagnostics(remote_repo, 'Remote') self.ui.serverdiagnostics(local_repo, 'Local') #self.ui.serverdiagnostics(statusrepos, 'Status') - + + class SyncableAccount(Account): """A syncable email account connecting 2 repositories @@ -208,7 +209,7 @@ class SyncableAccount(Account): self.ui.registerthread(self.name) accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): - os.mkdir(accountmetadata, 0700) + os.mkdir(accountmetadata, 0700) self.remoterepos = Repository(self, 'remote') self.localrepos = Repository(self, 'local') @@ -223,7 +224,7 @@ class SyncableAccount(Account): self.sync() except (KeyboardInterrupt, SystemExit): raise - except OfflineImapError, e: + except OfflineImapError, e: # Stop looping and bubble up Exception if needed. if e.severity >= OfflineImapError.ERROR.REPO: if looping: @@ -242,7 +243,7 @@ class SyncableAccount(Account): self.ui.acctdone(self) self.unlock() if looping and self.sleeper() >= 2: - looping = 0 + looping = 0 def sync(self): """Synchronize the account once, then return @@ -408,7 +409,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, else: ui.debug('imap', "Not syncing to read-only repository '%s'" \ % localrepos.getname()) - + # Synchronize local changes if not remoterepos.getconfboolean('readonly', False): ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) diff --git a/offlineimap/init.py b/offlineimap/init.py index a7f82ca..0e5b08d 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -157,9 +157,10 @@ class OfflineImap: #read in configuration file configfilename = os.path.expanduser(options.configfile) - + config = CustomConfigParser() if not os.path.exists(configfilename): + # TODO, initialize and make use of chosen ui for logging logging.error(" *** Config file '%s' does not exist; aborting!" % configfilename) sys.exit(1) @@ -168,14 +169,17 @@ class OfflineImap: #profile mode chosen? if options.profiledir: if not options.singlethreading: + # TODO, make use of chosen ui for logging logging.warn("Profile mode: Forcing to singlethreaded.") options.singlethreading = True if os.path.exists(options.profiledir): - logging.warn("Profile mode: Directory '%s' already exists!" % + # TODO, make use of chosen ui for logging + logging.warn("Profile mode: Directory '%s' already exists!" % options.profiledir) else: os.mkdir(options.profiledir) threadutil.ExitNotifyThread.set_profiledir(options.profiledir) + # TODO, make use of chosen ui for logging logging.warn("Profile mode: Potentially large data will be " "created in '%s'" % options.profiledir) @@ -197,6 +201,7 @@ class OfflineImap: if '.' in ui_type: #transform Curses.Blinkenlights -> Blinkenlights ui_type = ui_type.split('.')[-1] + # TODO, make use of chosen ui for logging logging.warning('Using old interface name, consider using one ' 'of %s' % ', '.join(UI_LIST.keys())) try: @@ -210,12 +215,13 @@ class OfflineImap: #set up additional log files if options.logfile: - self.ui.setlogfd(open(options.logfile, 'wt')) - + self.ui.setlogfile(options.logfile) + #welcome blurb self.ui.init_banner() if options.debugtype: + self.ui.logger.setLevel(logging.DEBUG) if options.debugtype.lower() == 'all': options.debugtype = 'imap,maildir,thread' #force single threading? @@ -293,15 +299,15 @@ class OfflineImap: pidfd.close() except: pass - - try: + + try: activeaccounts = self.config.get("general", "accounts") if options.accounts: activeaccounts = options.accounts activeaccounts = activeaccounts.replace(" ", "") activeaccounts = activeaccounts.split(",") allaccounts = accounts.AccountHashGenerator(self.config) - + syncaccounts = [] for account in activeaccounts: if account not in allaccounts: @@ -323,11 +329,11 @@ class OfflineImap: elif sig == signal.SIGUSR2: # tell each account to stop looping accounts.Account.set_abort_event(self.config, 2) - + signal.signal(signal.SIGHUP,sig_handler) signal.signal(signal.SIGUSR1,sig_handler) signal.signal(signal.SIGUSR2,sig_handler) - + #various initializations that need to be performed: offlineimap.mbnames.init(self.config, syncaccounts) @@ -352,10 +358,9 @@ class OfflineImap: name='Sync Runner', kwargs = {'accounts': syncaccounts, 'config': self.config}) - t.setDaemon(1) + t.setDaemon(True) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) - self.ui.terminate() except KeyboardInterrupt: self.ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') @@ -363,8 +368,8 @@ class OfflineImap: except (SystemExit): raise except Exception, e: - ui.error(e) - ui.terminate() + self.ui.error(e) + self.ui.terminate() def sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread diff --git a/offlineimap/syncmaster.py b/offlineimap/syncmaster.py index 3aea6d2..0d139dd 100644 --- a/offlineimap/syncmaster.py +++ b/offlineimap/syncmaster.py @@ -25,12 +25,11 @@ def syncaccount(threads, config, accountname): thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT', target = account.syncrunner, name = "Account sync %s" % accountname) - thread.setDaemon(1) + thread.setDaemon(True) thread.start() threads.add(thread) def syncitall(accounts, config): - currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE') threads = threadlist() for accountname in accounts: syncaccount(threads, config, accountname) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 463752f..39d3ae7 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -35,7 +35,7 @@ def semaphorereset(semaphore, originalstate): # Now release these. for i in range(originalstate): semaphore.release() - + class threadlist: def __init__(self): self.lock = Lock() @@ -70,7 +70,7 @@ class threadlist: if not thread: return thread.join() - + ###################################################################### # Exit-notify threads @@ -81,20 +81,21 @@ exitthreads = Queue(100) def exitnotifymonitorloop(callback): """An infinite "monitoring" loop watching for finished ExitNotifyThread's. - :param callback: the function to call when a thread terminated. That - function is called with a single argument -- the - ExitNotifyThread that has terminated. The monitor will + This one is supposed to run in the main thread. + :param callback: the function to call when a thread terminated. That + function is called with a single argument -- the + ExitNotifyThread that has terminated. The monitor will not continue to monitor for other threads until 'callback' returns, so if it intends to perform long calculations, it should start a new thread itself -- but - NOT an ExitNotifyThread, or else an infinite loop + NOT an ExitNotifyThread, or else an infinite loop may result. - Furthermore, the monitor will hold the lock all the + Furthermore, the monitor will hold the lock all the while the other thread is waiting. :type callback: a callable function """ global exitthreads - while 1: + while 1: # Loop forever and call 'callback' for each thread that exited try: # we need a timeout in the get() call, so that ctrl-c can throw @@ -124,11 +125,19 @@ def threadexited(thread): ui.threadExited(thread) class ExitNotifyThread(Thread): - """This class is designed to alert a "monitor" to the fact that a thread has - exited and to provide for the ability for it to find out why.""" + """This class is designed to alert a "monitor" to the fact that a + thread has exited and to provide for the ability for it to find out + why. All instances are made daemon threads (setDaemon(True), so we + bail out when the mainloop dies.""" profiledir = None """class variable that is set to the profile directory if required""" + def __init__(self, *args, **kwargs): + super(ExitNotifyThread, self).__init__(*args, **kwargs) + # These are all child threads that are supposed to go away when + # the main thread is killed. + self.setDaemon(True) + def run(self): global exitthreads self.threadid = get_ident() @@ -220,7 +229,7 @@ class InstanceLimitedThread(ExitNotifyThread): def start(self): instancelimitedsems[self.instancename].acquire() ExitNotifyThread.start(self) - + def run(self): try: ExitNotifyThread.run(self) diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py deleted file mode 100644 index 330b9a8..0000000 --- a/offlineimap/ui/Blinkenlights.py +++ /dev/null @@ -1,148 +0,0 @@ -# Blinkenlights base classes -# Copyright (C) 2003 John Goerzen -# -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -from threading import RLock, currentThread -from offlineimap.ui.UIBase import UIBase -from thread import get_ident # python < 2.6 support - -class BlinkenBase: - """This is a mix-in class that should be mixed in with either UIBase - or another appropriate base class. The Tk interface, for instance, - will probably mix it in with VerboseUI.""" - - def acct(s, accountname): - s.gettf().setcolor('purple') - s.__class__.__bases__[-1].acct(s, accountname) - - def connecting(s, hostname, port): - s.gettf().setcolor('gray') - s.__class__.__bases__[-1].connecting(s, hostname, port) - - def syncfolders(s, srcrepos, destrepos): - s.gettf().setcolor('blue') - s.__class__.__bases__[-1].syncfolders(s, srcrepos, destrepos) - - def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): - s.gettf().setcolor('cyan') - s.__class__.__bases__[-1].syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder) - - def skippingfolder(s, folder): - s.gettf().setcolor('cyan') - s.__class__.__bases__[-1].skippingfolder(s, folder) - - def loadmessagelist(s, repos, folder): - s.gettf().setcolor('green') - s._msg("Scanning folder [%s/%s]" % (s.getnicename(repos), - folder.getvisiblename())) - - def syncingmessages(s, sr, sf, dr, df): - s.gettf().setcolor('blue') - s.__class__.__bases__[-1].syncingmessages(s, sr, sf, dr, df) - - def copyingmessage(s, uid, num, num_to_copy, src, destfolder): - s.gettf().setcolor('orange') - s.__class__.__bases__[-1].copyingmessage(s, uid, num, num_to_copy, src, - destfolder) - - def deletingmessages(s, uidlist, destlist): - s.gettf().setcolor('red') - s.__class__.__bases__[-1].deletingmessages(s, uidlist, destlist) - - def deletingmessage(s, uid, destlist): - s.gettf().setcolor('red') - s.__class__.__bases__[-1].deletingmessage(s, uid, destlist) - - def addingflags(s, uidlist, flags, dest): - s.gettf().setcolor('yellow') - s.__class__.__bases__[-1].addingflags(s, uidlist, flags, dest) - - def deletingflags(s, uidlist, flags, dest): - s.gettf().setcolor('pink') - s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, dest) - - def warn(s, msg, minor = 0): - if minor: - s.gettf().setcolor('pink') - else: - s.gettf().setcolor('red') - s.__class__.__bases__[-1].warn(s, msg, minor) - - def init_banner(s): - s.availablethreadframes = {} - s.threadframes = {} - #tflock protects the s.threadframes manipulation to only happen from 1 thread - s.tflock = RLock() - - def threadExited(s, thread): - threadid = thread.threadid - accountname = s.getthreadaccount(thread) - s.tflock.acquire() - try: - if threadid in s.threadframes[accountname]: - tf = s.threadframes[accountname][threadid] - del s.threadframes[accountname][threadid] - s.availablethreadframes[accountname].append(tf) - tf.setthread(None) - finally: - s.tflock.release() - - UIBase.threadExited(s, thread) - - def gettf(s): - threadid = get_ident() - accountname = s.getthreadaccount() - - s.tflock.acquire() - - try: - if not accountname in s.threadframes: - s.threadframes[accountname] = {} - - if threadid in s.threadframes[accountname]: - return s.threadframes[accountname][threadid] - - if not accountname in s.availablethreadframes: - s.availablethreadframes[accountname] = [] - - if len(s.availablethreadframes[accountname]): - tf = s.availablethreadframes[accountname].pop(0) - tf.setthread(currentThread()) - else: - tf = s.getaccountframe().getnewthreadframe() - s.threadframes[accountname][threadid] = tf - return tf - finally: - s.tflock.release() - - def callhook(s, msg): - s.gettf().setcolor('white') - s.__class__.__bases__[-1].callhook(s, msg) - - def sleep(s, sleepsecs, account): - s.gettf().setcolor('red') - s.getaccountframe().startsleep(sleepsecs) - return UIBase.sleep(s, sleepsecs, account) - - def sleeping(s, sleepsecs, remainingsecs): - if remainingsecs and s.gettf().getcolor() == 'black': - s.gettf().setcolor('red') - else: - s.gettf().setcolor('black') - return s.getaccountframe().sleeping(sleepsecs, remainingsecs) - - diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 84b0e9a..3dc32f5 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -16,42 +16,67 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import RLock, Lock, Event +from __future__ import with_statement # needed for python 2.5 +from threading import RLock, currentThread, Lock, Event, Thread +from thread import get_ident # python < 2.6 support +from collections import deque import time import sys import os import signal import curses -from Blinkenlights import BlinkenBase -from UIBase import UIBase +import logging +from offlineimap.ui.UIBase import UIBase +from offlineimap.threadutil import ExitNotifyThread import offlineimap - acctkeys = '1234567890abcdefghijklmnoprstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-=;/.,' class CursesUtil: - def __init__(self): - self.pairlock = Lock() - # iolock protects access to the + + def __init__(self, *args, **kwargs): + # iolock protects access to the self.iolock = RLock() - self.start() + self.tframe_lock = RLock() + """tframe_lock protects the self.threadframes manipulation to + only happen from 1 thread""" + self.colormap = {} + """dict, translating color string to curses color pair number""" - def initpairs(self): - self.pairlock.acquire() - try: - self.pairs = {self._getpairindex(curses.COLOR_WHITE, - curses.COLOR_BLACK): 0} - self.nextpair = 1 - finally: - self.pairlock.release() + def curses_colorpair(self, col_name): + """Return the curses color pair, that corresponds to the color""" + return curses.color_pair(self.colormap[col_name]) - def lock(self): + def init_colorpairs(self): + """initialize the curses color pairs available""" + # set special colors 'gray' and 'banner' + self.colormap['white'] = 0 #hardcoded by curses + curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) + self.colormap['banner'] = 1 # color 'banner' for bannerwin + + bcol = curses.COLOR_BLACK + colors = ( # name, color, bold? + ('black', curses.COLOR_BLACK, False), + ('blue', curses.COLOR_BLUE,False), + ('red', curses.COLOR_RED, False), + ('purple', curses.COLOR_MAGENTA, False), + ('cyan', curses.COLOR_CYAN, False), + ('green', curses.COLOR_GREEN, False), + ('orange', curses.COLOR_YELLOW, False)) + #set the rest of all colors starting at pair 2 + i = 1 + for name, fcol, bold in colors: + i += 1 + self.colormap[name] = i + curses.init_pair(i, fcol, bcol) + + def lock(self, block=True): """Locks the Curses ui thread Can be invoked multiple times from the owning thread. Invoking from a non-owning thread blocks and waits until it has been unlocked by the owning thread.""" - self.iolock.acquire() + return self.iolock.acquire(block) def unlock(self): """Unlocks the Curses ui thread @@ -61,8 +86,8 @@ class CursesUtil: thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked.""" self.iolock.release() - - def locked(self, target, *args, **kwargs): + + def exec_locked(self, target, *args, **kwargs): """Perform an operation with full locking.""" self.lock() try: @@ -74,102 +99,56 @@ class CursesUtil: def lockedstuff(): curses.panel.update_panels() curses.doupdate() - self.locked(lockedstuff) + self.exec_locked(lockedstuff) def isactive(self): return hasattr(self, 'stdscr') - def _getpairindex(self, fg, bg): - return '%d/%d' % (fg,bg) - - def getpair(self, fg, bg): - if not self.has_color: - return 0 - pindex = self._getpairindex(fg, bg) - self.pairlock.acquire() - try: - if self.pairs.has_key(pindex): - return curses.color_pair(self.pairs[pindex]) - else: - self.pairs[pindex] = self.nextpair - curses.init_pair(self.nextpair, fg, bg) - self.nextpair += 1 - return curses.color_pair(self.nextpair - 1) - finally: - self.pairlock.release() - - def start(self): - self.stdscr = curses.initscr() - curses.noecho() - curses.cbreak() - self.stdscr.keypad(1) - try: - curses.start_color() - self.has_color = curses.has_colors() - except: - self.has_color = 0 - - self.oldcursor = None - try: - self.oldcursor = curses.curs_set(0) - except: - pass - - self.stdscr.clear() - self.stdscr.refresh() - (self.height, self.width) = self.stdscr.getmaxyx() - self.initpairs() - - def stop(self): - if not hasattr(self, 'stdscr'): - return - #self.stdscr.addstr(self.height - 1, 0, "\n", - # self.getpair(curses.COLOR_WHITE, - # curses.COLOR_BLACK)) - if self.oldcursor != None: - curses.curs_set(self.oldcursor) - self.stdscr.refresh() - self.stdscr.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - del self.stdscr - - def reset(self): - # dirty walkaround for bug http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 (fixed since #83743) - if (sys.version_info[0:3] >= (2,6) and sys.version_info[0:3] <= (2,6,5)): return - self.stop() - self.start() class CursesAccountFrame: - def __init__(s, master, accountname, ui): - s.c = master - s.children = [] - s.accountname = accountname - s.ui = ui + """Notable instance variables: - def drawleadstr(s, secs = None): + - accountname: String with associated account name + - children + - ui + - key + - window: curses window associated with an account + """ + + def __init__(self, ui, accountname): + self.children = [] + self.accountname = accountname + self.ui = ui + + def drawleadstr(self, secs = None): + #TODO: does what? if secs == None: - acctstr = '%s: [active] %13.13s: ' % (s.key, s.accountname) + acctstr = '%s: [active] %13.13s: ' % (self.key, self.accountname) else: - acctstr = '%s: [%3d:%02d] %13.13s: ' % (s.key, + acctstr = '%s: [%3d:%02d] %13.13s: ' % (self.key, secs / 60, secs % 60, - s.accountname) - s.c.locked(s.window.addstr, 0, 0, acctstr) - s.location = len(acctstr) + self.accountname) + self.ui.exec_locked(self.window.addstr, 0, 0, acctstr) + self.location = len(acctstr) - def setwindow(s, window, key): - s.window = window - s.key = key - s.drawleadstr() - for child in s.children: - child.update(window, 0, s.location) - s.location += 1 + def setwindow(self, curses_win, key): + #TODO: does what? + # the curses window associated with an account + self.window = curses_win + self.key = key + self.drawleadstr() + # Update the child ThreadFrames + for child in self.children: + child.update(curses_win, self.location, 0) + self.location += 1 - def getnewthreadframe(s): - tf = CursesThreadFrame(s.c, s.ui, s.window, 0, s.location) - s.location += 1 - s.children.append(tf) + def get_new_tframe(self): + """Create a new ThreadFrame and append it to self.children + + :returns: The new ThreadFrame""" + tf = CursesThreadFrame(self.ui, self.window, self.location, 0) + self.location += 1 + self.children.append(tf) return tf def startsleep(s, sleepsecs): @@ -197,413 +176,456 @@ class CursesAccountFrame: s.sleeping_abort = 1 class CursesThreadFrame: - def __init__(s, master, ui, window, y, x): - """master should be a CursesUtil object.""" - s.c = master - s.ui = ui - s.window = window - s.x = x - s.y = y - s.colors = [] - bg = curses.COLOR_BLACK - s.colormap = {'black': s.c.getpair(curses.COLOR_BLACK, bg), - 'gray': s.c.getpair(curses.COLOR_WHITE, bg), - 'white': curses.A_BOLD | s.c.getpair(curses.COLOR_WHITE, bg), - 'blue': s.c.getpair(curses.COLOR_BLUE, bg), - 'red': s.c.getpair(curses.COLOR_RED, bg), - 'purple': s.c.getpair(curses.COLOR_MAGENTA, bg), - 'cyan': s.c.getpair(curses.COLOR_CYAN, bg), - 'green': s.c.getpair(curses.COLOR_GREEN, bg), - 'orange': s.c.getpair(curses.COLOR_YELLOW, bg), - 'yellow': curses.A_BOLD | s.c.getpair(curses.COLOR_YELLOW, bg), - 'pink': curses.A_BOLD | s.c.getpair(curses.COLOR_RED, bg)} - #s.setcolor('gray') - s.setcolor('black') + """ + curses_color: current color pair for logging""" + def __init__(self, ui, acc_win, x, y): + """ + :param ui: is a Blinkenlights() instance + :param acc_win: curses Account window""" + self.ui = ui + self.window = acc_win + self.x = x + self.y = y + self.curses_color = curses.color_pair(0) #default color - def setcolor(self, color): - self.color = self.colormap[color] + def setcolor(self, color, modifier=0): + """Draw the thread symbol '.' in the specified color + :param modifier: Curses modified, such as curses.A_BOLD""" + self.curses_color = modifier | self.ui.curses_colorpair(color) self.colorname = color self.display() def display(self): - def lockedstuff(): - if self.getcolor() == 'black': - self.window.addstr(self.y, self.x, ' ', self.color) - else: - self.window.addstr(self.y, self.x, '.', self.color) - self.c.stdscr.move(self.c.height - 1, self.c.width - 1) + def locked_display(): + self.window.addch(self.y, self.x, '.', self.curses_color) self.window.refresh() - self.c.locked(lockedstuff) + # lock the curses IO while fudging stuff + self.ui.exec_locked(locked_display) - def getcolor(self): - return self.colorname - - def getcolorpair(self): - return self.color - - def update(self, window, y, x): - self.window = window + def update(self, acc_win, x, y): + """Update the xy position of the '.' (and possibly the aframe)""" + self.window = acc_win self.y = y self.x = x self.display() - def setthread(self, newthread): + def std_color(self): self.setcolor('black') - #if newthread: - # self.setcolor('gray') - #else: - # self.setcolor('black') -class InputHandler: - def __init__(s, util): - s.c = util - s.bgchar = None - s.inputlock = Lock() - s.lockheld = 0 - s.statuslock = Lock() - s.startup = Event() - s.startthread() - def startthread(s): - s.thread = offlineimap.threadutil.ExitNotifyThread(target = s.bgreaderloop, - name = "InputHandler loop") - s.thread.setDaemon(1) - s.thread.start() +class InputHandler(ExitNotifyThread): + """Listens for input via the curses interfaces""" + #TODO, we need to use the ugly exitnotifythread (rather than simply + #threading.Thread here, so exiting this thread via the callback + #handler, kills off all parents too. Otherwise, they would simply + #continue. + def __init__(self, ui): + super(InputHandler, self).__init__() + self.char_handler = None + self.ui = ui + self.enabled = Event() + """We will only parse input if we are enabled""" + self.inputlock = RLock() + """denotes whether we should be handling the next char.""" + self.start() #automatically start the thread - def bgreaderloop(s): - while 1: - s.statuslock.acquire() - if s.lockheld or s.bgchar == None: - s.statuslock.release() - s.startup.wait() - else: - s.statuslock.release() - ch = s.c.stdscr.getch() - s.statuslock.acquire() - try: - if s.lockheld or s.bgchar == None: - curses.ungetch(ch) - else: - s.bgchar(ch) - finally: - s.statuslock.release() + def get_next_char(self): + """return the key pressed or -1 - def set_bgchar(s, callback): - """Sets a "background" character handler. If a key is pressed - while not doing anything else, it will be passed to this handler. + Wait until `enabled` and loop internally every stdscr.timeout() + msecs, releasing the inputlock. + :returns: char or None if disabled while in here""" + self.enabled.wait() + while self.enabled.is_set(): + with self.inputlock: + char = self.ui.stdscr.getch() + if char != -1: yield char + + def run(self): + while True: + char_gen = self.get_next_char() + for char in char_gen: + self.char_handler(char) + #curses.ungetch(char) + + def set_char_hdlr(self, callback): + """Sets a character callback handler + + If a key is pressed it will be passed to this handler. Keys + include the curses.KEY_RESIZE key. callback is a function taking a single arg -- the char pressed. + If callback is None, input will be ignored.""" + with self.inputlock: + self.char_handler = callback + # start or stop the parsing of things + if callback is None: + self.enabled.clear() + else: + self.enabled.set() - If callback is None, clears the request.""" - s.statuslock.acquire() - oldhandler = s.bgchar - newhandler = callback - s.bgchar = callback - - if oldhandler and not newhandler: - pass - if newhandler and not oldhandler: - s.startup.set() - - s.statuslock.release() - - def input_acquire(s): + def input_acquire(self): """Call this method when you want exclusive input control. - Make sure to call input_release afterwards! + + Make sure to call input_release afterwards! While this lockis + held, input can go to e.g. the getpass input. """ + self.enabled.clear() + self.inputlock.acquire() - s.inputlock.acquire() - s.statuslock.acquire() - s.lockheld = 1 - s.statuslock.release() - - def input_release(s): + def input_release(self): """Call this method when you are done getting input.""" - s.statuslock.acquire() - s.lockheld = 0 - s.statuslock.release() - s.inputlock.release() - s.startup.set() - -class Blinkenlights(BlinkenBase, UIBase): - def init_banner(s): - s.af = {} - s.aflock = Lock() - s.c = CursesUtil() - s.text = [] - BlinkenBase.init_banner(s) - s.setupwindows() - s.inputhandler = InputHandler(s.c) - s.gettf().setcolor('red') - s._msg(offlineimap.banner) - s.inputhandler.set_bgchar(s.keypress) - signal.signal(signal.SIGWINCH, s.resizehandler) - s.resizelock = Lock() - s.resizecount = 0 - - def resizehandler(s, signum, frame): - s.resizeterm() - - def resizeterm(s, dosleep = 1): - if not s.resizelock.acquire(0): - s.resizecount += 1 - return - signal.signal(signal.SIGWINCH, signal.SIG_IGN) - s.aflock.acquire() - s.c.lock() - s.resizecount += 1 - while s.resizecount: - s.c.reset() - s.setupwindows() - s.resizecount -= 1 - s.c.unlock() - s.aflock.release() - s.resizelock.release() - signal.signal(signal.SIGWINCH, s.resizehandler) - if dosleep: - time.sleep(1) - s.resizeterm(0) - - def isusable(s): - # Not a terminal? Can't use curses. - if not sys.stdout.isatty() and sys.stdin.isatty(): - return 0 - - # No TERM specified? Can't use curses. - try: - if not len(os.environ['TERM']): - return 0 - except: return 0 - - # ncurses doesn't want to start? Can't use curses. - # This test is nasty because initscr() actually EXITS on error. - # grr. - - pid = os.fork() - if pid: - # parent - return not os.WEXITSTATUS(os.waitpid(pid, 0)[1]) - else: - # child - curses.initscr() - curses.endwin() - # If we didn't die by here, indicate success. - sys.exit(0) - - def keypress(s, key): - if key < 1 or key > 255: - return - - if chr(key) == 'q': - # Request to quit. - s.terminate() - - try: - index = acctkeys.index(chr(key)) - except ValueError: - # Key not a valid one: exit. - return - - if index >= len(s.hotkeys): - # Not in our list of valid hotkeys. - return - - # Trying to end sleep somewhere. - - s.getaccountframe(s.hotkeys[index]).syncnow() - - def getpass(s, accountname, config, errmsg = None): - s.inputhandler.input_acquire() - - # See comment on _msg for info on why both locks are obtained. - - s.tflock.acquire() - s.c.lock() - try: - s.gettf().setcolor('white') - s._addline(" *** Input Required", s.gettf().getcolorpair()) - s._addline(" *** Please enter password for account %s: " % accountname, - s.gettf().getcolorpair()) - s.logwindow.refresh() - password = s.logwindow.getstr() - finally: - s.tflock.release() - s.c.unlock() - s.inputhandler.input_release() - return password - - def setupwindows(s): - s.c.lock() - try: - s.bannerwindow = curses.newwin(1, s.c.width, 0, 0) - s.setupwindow_drawbanner() - s.logheight = s.c.height - 1 - len(s.af.keys()) - s.logwindow = curses.newwin(s.logheight, s.c.width, 1, 0) - s.logwindow.idlok(1) - s.logwindow.scrollok(1) - s.logwindow.move(s.logheight - 1, 0) - s.setupwindow_drawlog() - accounts = s.af.keys() - accounts.sort() - accounts.reverse() - - pos = s.c.height - 1 - index = 0 - s.hotkeys = [] - for account in accounts: - accountwindow = curses.newwin(1, s.c.width, pos, 0) - s.af[account].setwindow(accountwindow, acctkeys[index]) - s.hotkeys.append(account) - index += 1 - pos -= 1 - - curses.doupdate() - finally: - s.c.unlock() - - def setupwindow_drawbanner(s): - if s.c.has_color: - color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLUE) | \ - curses.A_BOLD - else: - color = curses.A_REVERSE - s.bannerwindow.bkgd(' ', color) # Fill background with that color - s.bannerwindow.addstr("%s %s" % (offlineimap.__productname__, - offlineimap.__version__)) - s.bannerwindow.addstr(0, s.bannerwindow.getmaxyx()[1] - len(offlineimap.__copyright__) - 1, - offlineimap.__copyright__) - - s.bannerwindow.noutrefresh() - - def setupwindow_drawlog(s): - if s.c.has_color: - color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLACK) - else: - color = curses.A_NORMAL - s.logwindow.bkgd(' ', color) - for line, color in s.text: - s.logwindow.addstr("\n" + line, color) - s.logwindow.noutrefresh() - - def getaccountframe(s, accountname = None): - if accountname == None: - accountname = s.getthreadaccount() - s.aflock.acquire() - try: - if accountname in s.af: - return s.af[accountname] - - # New one. - s.af[accountname] = CursesAccountFrame(s.c, accountname, s) - s.c.lock() - try: - s.c.reset() - s.setupwindows() - finally: - s.c.unlock() - finally: - s.aflock.release() - return s.af[accountname] + self.inputlock.release() + self.enabled.set() - def _display(s, msg, color = None): - if "\n" in msg: - for thisline in msg.split("\n"): - s._msg(thisline) - return +class CursesLogHandler(logging.StreamHandler): + def emit(self, record): + log_str = super(CursesLogHandler, self).format(record) + color = self.ui.gettf().curses_color # We must acquire both locks. Otherwise, deadlock can result. # This can happen if one thread calls _msg (locking curses, then # tf) and another tries to set the color (locking tf, then curses) # # By locking both up-front here, in this order, we prevent deadlock. - - s.tflock.acquire() - s.c.lock() + self.ui.tframe_lock.acquire() + self.ui.lock() try: - if not s.c.isactive(): - # For dumping out exceptions and stuff. - print msg - return - if color: - s.gettf().setcolor(color) - elif s.gettf().getcolor() == 'black': - s.gettf().setcolor('gray') - s._addline(msg, s.gettf().getcolorpair()) - s.logwindow.refresh() + for line in log_str.split("\n"): + self.ui.logwin.addstr("\n" + line, color) + self.ui.text.append((line, color)) + while len(self.ui.text) > self.ui.logheight: + self.ui.text.popleft() finally: - s.c.unlock() - s.tflock.release() + self.ui.unlock() + self.ui.tframe_lock.release() + self.ui.logwin.refresh() + self.ui.stdscr.refresh() - def _addline(s, msg, color): - s.c.lock() +class Blinkenlights(UIBase, CursesUtil): + """Curses-cased fancy UI + + Notable instance variables self. ....: + + - stdscr: THe curses std screen + - bannerwin: The top line banner window + - width|height: The total curses screen dimensions + - logheight: Available height for the logging part + - log_con_handler: The CursesLogHandler() + - threadframes: + - accframes[account]: 'Accountframe'""" + + def __init__(self, *args, **kwargs): + super(Blinkenlights, self).__init__(*args, **kwargs) + CursesUtil.__init__(self) + + ################################################## UTILS + def setup_consolehandler(self): + """Backend specific console handler + + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = CursesLogHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + # the handler is not usable yet. We still need all the + # intialization stuff currently done in init_banner. Move here? + return ch + + def isusable(s): + """Returns true if the backend is usable ie Curses works""" + # Not a terminal? Can't use curses. + if not sys.stdout.isatty() and sys.stdin.isatty(): + return False + # No TERM specified? Can't use curses. + if not os.environ.get('TERM', None): + return False + # Test if ncurses actually starts up fine. Only do so for + # python>=2.6.6 as calling initscr() twice messing things up. + # see http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 + if sys.version_info[0:3] < (2,6) or sys.version_info[0:3] >= (2,6,6): + try: + curses.initscr() + curses.endwin() + except: + return False + return True + + def init_banner(self): + self.availablethreadframes = {} + self.threadframes = {} + self.accframes = {} + self.aflock = Lock() + self.text = deque() + + self.stdscr = curses.initscr() + # turn off automatic echoing of keys to the screen + curses.noecho() + # react to keys instantly, without Enter key + curses.cbreak() + # return special key values, eg curses.KEY_LEFT + self.stdscr.keypad(1) + # wait 1s for input, so we don't block the InputHandler infinitely + self.stdscr.timeout(1000) + curses.start_color() + # turn off cursor and save original state + self.oldcursor = None try: - s.logwindow.addstr("\n" + msg, color) - s.text.append((msg, color)) - while len(s.text) > s.logheight: - s.text = s.text[1:] - finally: - s.c.unlock() + self.oldcursor = curses.curs_set(0) + except: + pass - def terminate(s, exitstatus = 0, errortitle = None, errormsg = None): - s.c.stop() - UIBase.terminate(s, exitstatus = exitstatus, errortitle = errortitle, errormsg = errormsg) + self.stdscr.clear() + self.stdscr.refresh() + self.init_colorpairs() + # set log handlers ui to ourself + self._log_con_handler.ui = self + self.setupwindows() + # Settup keyboard handler + self.inputhandler = InputHandler(self) + self.inputhandler.set_char_hdlr(self.on_keypressed) - def threadException(s, thread): - s.c.stop() - UIBase.threadException(s, thread) + self.gettf().setcolor('red') + self.info(offlineimap.banner) - def mainException(s): - s.c.stop() - UIBase.mainException(s) + def acct(self, *args): + """Output that we start syncing an account (and start counting)""" + self.gettf().setcolor('purple') + super(Blinkenlights, self).acct(*args) + + def connecting(self, *args): + self.gettf().setcolor('white') + super(Blinkenlights, self).connecting(*args) + + def syncfolders(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).syncfolders(*args) + + def syncingfolder(self, *args): + self.gettf().setcolor('cyan') + super(Blinkenlights, self).syncingfolder(*args) + + def skippingfolder(self, *args): + self.gettf().setcolor('cyan') + super(Blinkenlights, self).skippingfolder(*args) + + def loadmessagelist(self, *args): + self.gettf().setcolor('green') + super(Blinkenlights, self).loadmessagelist(*args) + + def syncingmessages(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).syncingmessages(*args) + + def copyingmessage(self, *args): + self.gettf().setcolor('orange') + super(Blinkenlights, self).copyingmessage(*args) + + def deletingmessages(self, *args): + self.gettf().setcolor('red') + super(Blinkenlights, self).deletingmessages(*args) + + def addingflags(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).addingflags(*args) + + def deletingflags(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).deletingflags(*args) + + def callhook(self, *args): + self.gettf().setcolor('white') + super(Blinkenlights, self).callhook(*args) + + ############ Generic logging functions ############################# + def warn(self, msg, minor=0): + self.gettf().setcolor('red', curses.A_BOLD) + super(Blinkenlights, self).warn(msg) + + def threadExited(self, thread): + acc_name = self.getthreadaccount(thread) + with self.tframe_lock: + if thread in self.threadframes[acc_name]: + tf = self.threadframes[acc_name][thread] + tf.setcolor('black') + self.availablethreadframes[acc_name].append(tf) + del self.threadframes[acc_name][thread] + super(Blinkenlights, self).threadExited(thread) + + def gettf(self): + """Return the ThreadFrame() of the current thread""" + cur_thread = currentThread() + acc_name = self.getthreadaccount() + + with self.tframe_lock: + # Ideally we already have self.threadframes[accountname][thread] + try: + if cur_thread in self.threadframes[acc_name]: + return self.threadframes[acc_name][cur_thread] + except KeyError: + # Ensure threadframes already has an account dict + self.threadframes[acc_name] = {} + self.availablethreadframes[acc_name] = deque() + + # If available, return a ThreadFrame() + if len(self.availablethreadframes[acc_name]): + tf = self.availablethreadframes[acc_name].popleft() + tf.std_color() + else: + tf = self.getaccountframe(acc_name).get_new_tframe() + self.threadframes[acc_name][cur_thread] = tf + return tf + + def on_keypressed(self, key): + # received special KEY_RESIZE, resize terminal + if key == curses.KEY_RESIZE: + self.resizeterm() + + if key < 1 or key > 255: + return + if chr(key) == 'q': + # Request to quit. + #TODO: this causes us to bail out in main loop when the thread exits + #TODO: review and rework this mechanism. + currentThread().setExitCause('EXCEPTION') + self.terminate() + try: + index = acctkeys.index(chr(key)) + except ValueError: + # Key not a valid one: exit. + return + if index >= len(self.hotkeys): + # Not in our list of valid hotkeys. + return + # Trying to end sleep somewhere. + self.getaccountframe(self.hotkeys[index]).syncnow() def sleep(s, sleepsecs, account): s.gettf().setcolor('red') s._msg("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return BlinkenBase.sleep(s, sleepsecs, account) - -if __name__ == '__main__': - x = Blinkenlights(None) - x.init_banner() - import time - time.sleep(5) - x.c.stop() - fgs = {'black': curses.COLOR_BLACK, 'red': curses.COLOR_RED, - 'green': curses.COLOR_GREEN, 'yellow': curses.COLOR_YELLOW, - 'blue': curses.COLOR_BLUE, 'magenta': curses.COLOR_MAGENTA, - 'cyan': curses.COLOR_CYAN, 'white': curses.COLOR_WHITE} - - x = CursesUtil() - win1 = curses.newwin(x.height, x.width / 4 - 1, 0, 0) - win1.addstr("Black/normal\n") - for name, fg in fgs.items(): - win1.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLACK)) - win2 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1]) - win2.addstr("Blue/normal\n") - for name, fg in fgs.items(): - win2.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLUE)) - win3 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1] + - win2.getmaxyx()[1]) - win3.addstr("Black/bright\n") - for name, fg in fgs.items(): - win3.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLACK) | \ - curses.A_BOLD) - win4 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1] * 3) - win4.addstr("Blue/bright\n") - for name, fg in fgs.items(): - win4.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLUE) | \ - curses.A_BOLD) - - - win1.refresh() - win2.refresh() - win3.refresh() - win4.refresh() - x.stdscr.refresh() - import time - time.sleep(5) - x.stop() - print x.has_color - print x.height - print x.width + s.getaccountframe().startsleep(sleepsecs) + return UIBase.sleep(s, sleepsecs, account) + + def sleeping(self, sleepsecs, remainingsecs): + if remainingsecs and s.gettf().getcolor() == 'black': + self.gettf().setcolor('red') + else: + self.gettf().setcolor('black') + return self.getaccountframe().sleeping(sleepsecs, remainingsecs) + + def resizeterm(self): + """Resize the current windows""" + self.exec_locked(self.setupwindows(True)) + + def mainException(self): + UIBase.mainException(self) + + def getpass(self, accountname, config, errmsg = None): + # disable the hotkeys inputhandler + self.inputhandler.input_acquire() + + # See comment on _msg for info on why both locks are obtained. + self.lock() + try: + #s.gettf().setcolor('white') + self.warn(" *** Input Required") + self.warn(" *** Please enter password for account %s: " % \ + accountname) + self.logwin.refresh() + password = self.logwin.getstr() + finally: + self.unlock() + self.inputhandler.input_release() + return password + + def setupwindows(self, resize=False): + """Setup and draw bannerwin and logwin + + If `resize`, don't create new windows, just adapt size""" + self.height, self.width = self.stdscr.getmaxyx() + if resize: + raise Exception("resizehandler %d" % self.width) + + self.logheight = self.height - len(self.accframes) - 1 + if resize: + curses.resizeterm(self.height, self.width) + self.bannerwin.resize(1, self.width) + else: + self.bannerwin = curses.newwin(1, self.width, 0, 0) + self.logwin = curses.newwin(self.logheight, self.width, 1, 0) + + self.draw_bannerwin() + self.logwin.idlok(1) + self.logwin.scrollok(1) + self.logwin.move(self.logheight - 1, 0) + self.draw_logwin() + self.accounts = reversed(sorted(self.accframes.keys())) + + pos = self.height - 1 + index = 0 + self.hotkeys = [] + for account in self.accounts: + acc_win = curses.newwin(1, self.width, pos, 0) + self.accframes[account].setwindow(acc_win, acctkeys[index]) + self.hotkeys.append(account) + index += 1 + pos -= 1 + curses.doupdate() + + def draw_bannerwin(self): + """Draw the top-line banner line""" + if curses.has_colors(): + color = curses.A_BOLD | self.curses_colorpair('banner') + else: + color = curses.A_REVERSE + self.bannerwin.bkgd(' ', color) # Fill background with that color + string = "%s %s" % (offlineimap.__productname__, + offlineimap.__version__) + self.bannerwin.addstr(0, 0, string, color) + self.bannerwin.addstr(0, self.width -len(offlineimap.__copyright__) -1, + offlineimap.__copyright__, color) + self.bannerwin.noutrefresh() + + def draw_logwin(self): + #if curses.has_colors(): + # color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLACK) + #else: + color = curses.A_NORMAL + self.logwin.bkgd(' ', color) + for line, color in self.text: + self.logwin.addstr("\n" + line, color) + self.logwin.noutrefresh() + + def getaccountframe(self, acc_name = None): + """Return an AccountFrame()""" + if acc_name == None: + acc_name = self.getthreadaccount() + with self.aflock: + # 1) Return existing or 2) create a new CursesAccountFrame. + if acc_name in self.accframes: return self.accframes[acc_name] + self.accframes[acc_name] = CursesAccountFrame(self, acc_name) + self.setupwindows() + return self.accframes[acc_name] + + def terminate(self, *args, **kwargs): + curses.nocbreak(); + self.stdscr.keypad(0); + curses.echo() + curses.endwin() + # need to remove the Curses console handler now and replace with + # basic one, so exceptions and stuff are properly displayed + self.logger.removeHandler(self._log_con_handler) + UIBase.setup_consolehandler(self) + # finally call parent terminate which prints out exceptions etc + super(Blinkenlights, self).terminate(*args, **kwargs) + + def threadException(s, thread): + #self._log_con_handler.stop() + UIBase.threadException(s, thread) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 50a7f99..334121d 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -1,5 +1,4 @@ -# Copyright (C) 2007 John Goerzen -# +# Copyright (C) 2007-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,48 +17,30 @@ import urllib import sys import time +import logging from UIBase import UIBase -from threading import currentThread, Lock +from threading import currentThread import offlineimap -protocol = '6.0.0' +protocol = '7.0.0' class MachineUI(UIBase): - def __init__(s, config, verbose = 0): - UIBase.__init__(s, config, verbose) - s.safechars=" ;,./-_=+()[]" - s.iswaiting = 0 - s.outputlock = Lock() - s._printData('__init__', protocol) + def __init__(self, config, loglevel = logging.INFO): + super(MachineUI, self).__init__(config, loglevel) + self._log_con_handler.createLock() + """lock needed to block on password input""" - def isusable(s): - return True + def _printData(self, command, msg): + self.logger.info("%s:%s:%s:%s" % ( + 'msg', command, currentThread().getName(), msg)) - def _printData(s, command, data, dolock = True): - s._printDataOut('msg', command, data, dolock) - - def _printWarn(s, command, data, dolock = True): - s._printDataOut('warn', command, data, dolock) - - def _printDataOut(s, datatype, command, data, dolock = True): - if dolock: - s.outputlock.acquire() - try: - print "%s:%s:%s:%s" % \ - (datatype, - urllib.quote(command, s.safechars), - urllib.quote(currentThread().getName(), s.safechars), - urllib.quote(data, s.safechars)) - sys.stdout.flush() - finally: - if dolock: - s.outputlock.release() - - def _display(s, msg): + def _msg(s, msg): s._printData('_display', msg) def warn(s, msg, minor = 0): - s._printData('warn', '%s\n%d' % (msg, int(minor))) + # TODO, remove and cleanup the unused minor stuff + self.logger.warning("%s:%s:%s:%s" % ( + 'warn', '', currentThread().getName(), msg)) def registerthread(s, account): UIBase.registerthread(s, account) @@ -112,13 +93,10 @@ class MachineUI(UIBase): self._printData('copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ (uid, self.getnicename(srcfolder), srcfolder.getname(), self.getnicename(destfolder), destfolder)) - + def folderlist(s, list): return ("\f".join(["%s\t%s" % (s.getnicename(x), x.getname()) for x in list])) - def deletingmessage(s, uid, destlist): - s.deletingmessages(s, [uid], destlist) - def uidlist(s, list): return ("\f".join([str(u) for u in list])) @@ -161,19 +139,21 @@ class MachineUI(UIBase): return 0 - def getpass(s, accountname, config, errmsg = None): - s.outputlock.acquire() + def getpass(self, accountname, config, errmsg = None): + if errmsg: + self._printData('getpasserror', "%s\n%s" % (accountname, errmsg), + False) + + self._log_con_handler.acquire() # lock the console output try: - if errmsg: - s._printData('getpasserror', "%s\n%s" % (accountname, errmsg), - False) - s._printData('getpass', accountname, False) + self._printData('getpass', accountname, False) return (sys.stdin.readline()[:-1]) finally: - s.outputlock.release() + self._log_con_handler.release() - def init_banner(s): - s._printData('initbanner', offlineimap.banner) + def init_banner(self): + self._printData('protocol', protocol) + self._printData('initbanner', offlineimap.banner) - def callhook(s, msg): - s._printData('callhook', msg) + def callhook(self, msg): + self._printData('callhook', msg) diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index 33248dd..36bb92b 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -1,6 +1,5 @@ # Noninteractive UI -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,37 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import sys -import time +import logging from UIBase import UIBase class Basic(UIBase): - def getpass(s, accountname, config, errmsg = None): - raise NotImplementedError, "Prompting for a password is not supported in noninteractive mode." + """'Quiet' simply sets log level to INFO""" + def __init__(self, config, loglevel = logging.INFO): + return super(Basic, self).__init__(config, loglevel) - def _display(s, msg): - print msg - sys.stdout.flush() - - def warn(s, msg, minor = 0): - warntxt = 'WARNING' - if minor: - warntxt = 'warning' - sys.stderr.write(warntxt + ": " + str(msg) + "\n") - - def sleep(s, sleepsecs, siglistener): - if s.verbose >= 0: - s._msg("Sleeping for %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return UIBase.sleep(s, sleepsecs, siglistener) - - def sleeping(s, sleepsecs, remainingsecs): - if sleepsecs > 0: - time.sleep(sleepsecs) - return 0 - - def locked(s): - s.warn("Another OfflineIMAP is running with the same metadatadir; exiting.") - -class Quiet(Basic): - def __init__(s, config, verbose = -1): - Basic.__init__(s, config, verbose) +class Quiet(UIBase): + """'Quiet' simply sets log level to WARNING""" + def __init__(self, config, loglevel = logging.WARNING): + return super(Quiet, self).__init__(config, loglevel) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 48c468f..cedd8a6 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -1,6 +1,5 @@ # TTY UI -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,53 +14,73 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from UIBase import UIBase -from getpass import getpass + +import logging import sys -from threading import Lock, currentThread +from getpass import getpass +from offlineimap import banner +from offlineimap.ui.UIBase import UIBase + +class TTYFormatter(logging.Formatter): + """Specific Formatter that adds thread information to the log output""" + def __init__(self, *args, **kwargs): + super(TTYFormatter, self).__init__(*args, **kwargs) + self._last_log_thread = None + + def format(self, record): + """Override format to add thread information""" + log_str = super(TTYFormatter, self).format(record) + # If msg comes from a different thread than our last, prepend + # thread info. Most look like 'Account sync foo' or 'Folder + # sync foo'. + t_name = record.threadName + if t_name == 'MainThread': + return log_str # main thread doesn't get things prepended + if t_name != self._last_log_thread: + self._last_log_thread = t_name + log_str = "%s:\n %s" % (t_name, log_str) + else: + log_str = " %s" % log_str + return log_str class TTYUI(UIBase): - def __init__(s, config, verbose = 0): - UIBase.__init__(s, config, verbose) - s.iswaiting = 0 - s.outputlock = Lock() - s._lastThreaddisplay = None + def setup_consolehandler(self): + """Backend specific console handler - def isusable(s): + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = logging.StreamHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = TTYFormatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(banner) + # init lock for console output + ch.createLock() + return ch + + def isusable(self): + """TTYUI is reported as usable when invoked on a terminal""" return sys.stdout.isatty() and sys.stdin.isatty() - - def _display(s, msg): - s.outputlock.acquire() - try: - #if the next output comes from a different thread than our last one - #add the info. - #Most look like 'account sync foo' or 'Folder sync foo'. - threadname = currentThread().getName() - if (threadname == s._lastThreaddisplay \ - or threadname == 'MainThread'): - print " %s" % msg - else: - print "%s:\n %s" % (threadname, msg) - s._lastThreaddisplay = threadname - sys.stdout.flush() - finally: - s.outputlock.release() - - def getpass(s, accountname, config, errmsg = None): + def getpass(self, accountname, config, errmsg = None): + """TTYUI backend is capable of querying the password""" if errmsg: - s._msg("%s: %s" % (accountname, errmsg)) - s.outputlock.acquire() + self.warn("%s: %s" % (accountname, errmsg)) + self._log_con_handler.acquire() # lock the console output try: - return getpass("%s: Enter password: " % accountname) + return getpass("Enter password for account '%s': " % accountname) finally: - s.outputlock.release() + self._log_con_handler.release() - def mainException(s): - if isinstance(sys.exc_info()[1], KeyboardInterrupt) and \ - s.iswaiting: - sys.stdout.write("Timer interrupted at user request; program terminating. \n") - s.terminate() + def mainException(self): + if isinstance(sys.exc_info()[1], KeyboardInterrupt): + self.logger.warn("Timer interrupted at user request; program " + "terminating.\n") + self.terminate() else: - UIBase.mainException(s) + UIBase.mainException(self) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index b5dcdaa..9f5f4bc 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -15,12 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import logging import re import time import sys +import os import traceback import threading from Queue import Queue +from collections import deque import offlineimap debugtypes = {'':'Other offlineimap related sync messages', @@ -38,54 +41,72 @@ def getglobalui(): global globalui return globalui -class UIBase: - def __init__(s, config, verbose = 0): - s.verbose = verbose - s.config = config - s.debuglist = [] - s.debugmessages = {} - s.debugmsglen = 50 - s.threadaccounts = {} +class UIBase(object): + def __init__(self, config, loglevel = logging.INFO): + self.config = config + self.debuglist = [] + """list of debugtypes we are supposed to log""" + self.debugmessages = {} + """debugmessages in a deque(v) per thread(k)""" + self.debugmsglen = 50 + self.threadaccounts = {} """dict linking active threads (k) to account names (v)""" - s.acct_startimes = {} + self.acct_startimes = {} """linking active accounts with the time.time() when sync started""" - s.logfile = None - s.exc_queue = Queue() + self.logfile = None + self.exc_queue = Queue() """saves all occuring exceptions, so we can output them at the end""" + # create logger with 'OfflineImap' app + self.logger = logging.getLogger('OfflineImap') + self.logger.setLevel(loglevel) + self._log_con_handler = self.setup_consolehandler() + """The console handler (we need access to be able to lock it)""" ################################################## UTILS - def _msg(s, msg): - """Generic tool called when no other works.""" - s._log(msg) - s._display(msg) + def setup_consolehandler(self): + """Backend specific console handler - def _log(s, msg): - """Log it to disk. Returns true if it wrote something; false - otherwise.""" - if s.logfile: - s.logfile.write("%s: %s\n" % (threading.currentThread().getName(), - msg)) - return 1 - return 0 + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = logging.StreamHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(offlineimap.banner) + return ch - def setlogfd(s, logfd): - s.logfile = logfd - logfd.write("This is %s %s\n" % \ - (offlineimap.__productname__, - offlineimap.__version__)) - logfd.write("Python: %s\n" % sys.version) - logfd.write("Platform: %s\n" % sys.platform) - logfd.write("Args: %s\n" % sys.argv) + def setlogfile(self, logfile): + """Create file handler which logs to file""" + fh = logging.FileHandler(logfile, 'wt') + #fh.setLevel(logging.DEBUG) + file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " + "%(message)s", '%Y-%m-%d %H:%M:%S') + fh.setFormatter(file_formatter) + self.logger.addHandler(fh) + # write out more verbose initial info blurb on the log file + p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) + msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ + "Args: %s" % (offlineimap.__version__, p_ver, sys.platform, + " ".join(sys.argv)) + record = logging.LogRecord('OfflineImap', logging.INFO, __file__, + None, msg, None, None) + fh.emit(record) - def _display(s, msg): + def _msg(self, msg): """Display a message.""" - raise NotImplementedError + # TODO: legacy function, rip out. + self.info(msg) - def warn(s, msg, minor = 0): - if minor: - s._msg("warning: " + msg) - else: - s._msg("WARNING: " + msg) + def info(self, msg): + """Display a message.""" + self.logger.info(msg) + + def warn(self, msg, minor = 0): + self.logger.warning(msg) def error(self, exc, exc_traceback=None, msg=None): """Log a message at severity level ERROR @@ -146,40 +167,43 @@ class UIBase: return self.threadaccounts[thr] return '*Control' # unregistered thread is '*Control' - def debug(s, debugtype, msg): - thisthread = threading.currentThread() - if s.debugmessages.has_key(thisthread): - s.debugmessages[thisthread].append("%s: %s" % (debugtype, msg)) - else: - s.debugmessages[thisthread] = ["%s: %s" % (debugtype, msg)] + def debug(self, debugtype, msg): + cur_thread = threading.currentThread() + if not self.debugmessages.has_key(cur_thread): + # deque(..., self.debugmsglen) would be handy but was + # introduced in p2.6 only, so we'll need to work around and + # shorten our debugmsg list manually :-( + self.debugmessages[cur_thread] = deque() + self.debugmessages[cur_thread].append("%s: %s" % (debugtype, msg)) - while len(s.debugmessages[thisthread]) > s.debugmsglen: - s.debugmessages[thisthread] = s.debugmessages[thisthread][1:] + # Shorten queue if needed + if len(self.debugmessages[cur_thread]) > self.debugmsglen: + self.debugmessages[cur_thread].popleft() - if debugtype in s.debuglist: - if not s._log("DEBUG[%s]: %s" % (debugtype, msg)): - s._display("DEBUG[%s]: %s" % (debugtype, msg)) + if debugtype in self.debuglist: # log if we are supposed to do so + self.logger.debug("[%s]: %s" % (debugtype, msg)) - def add_debug(s, debugtype): + def add_debug(self, debugtype): global debugtypes if debugtype in debugtypes: - if not debugtype in s.debuglist: - s.debuglist.append(debugtype) - s.debugging(debugtype) + if not debugtype in self.debuglist: + self.debuglist.append(debugtype) + self.debugging(debugtype) else: - s.invaliddebug(debugtype) + self.invaliddebug(debugtype) - def debugging(s, debugtype): + def debugging(self, debugtype): global debugtypes - s._msg("Now debugging for %s: %s" % (debugtype, debugtypes[debugtype])) + self.logger.debug("Now debugging for %s: %s" % (debugtype, + debugtypes[debugtype])) - def invaliddebug(s, debugtype): - s.warn("Invalid debug type: %s" % debugtype) + def invaliddebug(self, debugtype): + self.warn("Invalid debug type: %s" % debugtype) def locked(s): raise Exception, "Another OfflineIMAP is running with the same metadatadir; exiting." - def getnicename(s, object): + def getnicename(self, object): """Return the type of a repository or Folder as string (IMAP, Gmail, Maildir, etc...)""" @@ -187,61 +211,73 @@ class UIBase: # Strip off extra stuff. return re.sub('(Folder|Repository)', '', prelimname) - def isusable(s): + def isusable(self): """Returns true if this UI object is usable in the current environment. For instance, an X GUI would return true if it's being run in X with a valid DISPLAY setting, and false otherwise.""" - return 1 + return True ################################################## INPUT - def getpass(s, accountname, config, errmsg = None): - raise NotImplementedError + def getpass(self, accountname, config, errmsg = None): + raise NotImplementedError("Prompting for a password is not supported"\ + " in this UI backend.") - def folderlist(s, list): - return ', '.join(["%s[%s]" % (s.getnicename(x), x.getname()) for x in list]) + def folderlist(self, list): + return ', '.join(["%s[%s]" % \ + (self.getnicename(x), x.getname()) for x in list]) ################################################## WARNINGS - def msgtoreadonly(s, destfolder, uid, content, flags): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to synchronize message %d to folder %s[%s], but that folder is read-only. The message will not be copied to that folder." % \ - (uid, s.getnicename(destfolder), destfolder.getname())) + def msgtoreadonly(self, destfolder, uid, content, flags): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to synchronize message %d to folder %s[%s], " + "but that folder is read-only. The message will not be " + "copied to that folder." % ( + uid, self.getnicename(destfolder), destfolder)) - def flagstoreadonly(s, destfolder, uidlist, flags): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to modify flags for messages %s in folder %s[%s], but that folder is read-only. No flags have been modified for that message." % \ - (str(uidlist), s.getnicename(destfolder), destfolder.getname())) + def flagstoreadonly(self, destfolder, uidlist, flags): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to modify flags for messages %s in folder %s[%s], " + "but that folder is read-only. No flags have been modified " + "for that message." % ( + str(uidlist), self.getnicename(destfolder), destfolder)) - def deletereadonly(s, destfolder, uidlist): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to delete messages %s in folder %s[%s], but that folder is read-only. No messages have been deleted in that folder." % \ - (str(uidlist), s.getnicename(destfolder), destfolder.getname())) + def deletereadonly(self, destfolder, uidlist): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to delete messages %s in folder %s[%s], but that " + "folder is read-only. No messages have been deleted in that " + "folder." % (str(uidlist), self.getnicename(destfolder), + destfolder)) ################################################## MESSAGES - def init_banner(s): + def init_banner(self): """Called when the UI starts. Must be called before any other UI call except isusable(). Displays the copyright banner. This is where the UI should do its setup -- TK, for instance, would create the application window here.""" - if s.verbose >= 0: - s._msg(offlineimap.banner) + pass - def connecting(s, hostname, port): + def connecting(self, hostname, port): """Log 'Establishing connection to'""" - if s.verbose < 0: return + if not self.logger.isEnabledFor(logging.info): return displaystr = '' hostname = hostname if hostname else '' port = "%s" % port if port else '' if hostname: displaystr = ' to %s:%s' % (hostname, port) - s._msg("Establishing connection%s" % displaystr) + self.logger.info("Establishing connection%s" % displaystr) def acct(self, account): """Output that we start syncing an account (and start counting)""" self.acct_startimes[account] = time.time() - if self.verbose >= 0: - self._msg("*** Processing account %s" % account) + self.logger.info("*** Processing account %s" % account) def acctdone(self, account): """Output that we finished syncing an account (in which time)""" @@ -252,75 +288,63 @@ class UIBase: def syncfolders(self, src_repo, dst_repo): """Log 'Copying folder structure...'""" - if self.verbose < 0: return - self.debug('', "Copying folder structure from %s to %s" % \ - (src_repo, dst_repo)) + if self.logger.isEnabledFor(logging.DEBUG): + self.debug('', "Copying folder structure from %s to %s" %\ + (src_repo, dst_repo)) ############################## Folder syncing - def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): + def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder): """Called when a folder sync operation is started.""" - if s.verbose >= 0: - s._msg("Syncing %s: %s -> %s" % (srcfolder.getname(), - s.getnicename(srcrepos), - s.getnicename(destrepos))) + self.logger.info("Syncing %s: %s -> %s" % (srcfolder, + self.getnicename(srcrepos), + self.getnicename(destrepos))) - def skippingfolder(s, folder): + def skippingfolder(self, folder): """Called when a folder sync operation is started.""" - if s.verbose >= 0: - s._msg("Skipping %s (not changed)" % folder.getname()) + self.logger.info("Skipping %s (not changed)" % folder) - def validityproblem(s, folder): - s.warn("UID validity problem for folder %s (repo %s) (saved %d; got %d); skipping it" % \ - (folder.getname(), folder.getrepository().getname(), + def validityproblem(self, folder): + self.logger.warning("UID validity problem for folder %s (repo %s) " + "(saved %d; got %d); skipping it. Please see FAQ " + "and manual how to handle this." % \ + (folder, folder.getrepository(), folder.getsaveduidvalidity(), folder.getuidvalidity())) - def loadmessagelist(s, repos, folder): - if s.verbose > 0: - s._msg("Loading message list for %s[%s]" % (s.getnicename(repos), - folder.getname())) + def loadmessagelist(self, repos, folder): + self.logger.debug("Loading message list for %s[%s]" % ( + self.getnicename(repos), + folder)) - def messagelistloaded(s, repos, folder, count): - if s.verbose > 0: - s._msg("Message list for %s[%s] loaded: %d messages" % \ - (s.getnicename(repos), folder.getname(), count)) + def messagelistloaded(self, repos, folder, count): + self.logger.debug("Message list for %s[%s] loaded: %d messages" % ( + self.getnicename(repos), folder, count)) ############################## Message syncing - def syncingmessages(s, sr, sf, dr, df): - if s.verbose > 0: - s._msg("Syncing messages %s[%s] -> %s[%s]" % (s.getnicename(sr), - sf.getname(), - s.getnicename(dr), - df.getname())) + def syncingmessages(self, sr, srcfolder, dr, dstfolder): + self.logger.debug("Syncing messages %s[%s] -> %s[%s]" % ( + self.getnicename(sr), srcfolder, + self.getnicename(dr), dstfolder)) def copyingmessage(self, uid, num, num_to_copy, src, destfolder): """Output a log line stating which message we copy""" - if self.verbose < 0: return - self._msg("Copy message %s (%d of %d) %s:%s -> %s" % (uid, num, - num_to_copy, src.repository, src, destfolder.repository)) + self.logger.info("Copy message %s (%d of %d) %s:%s -> %s" % ( + uid, num, num_to_copy, src.repository, src, + destfolder.repository)) - def deletingmessage(s, uid, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting message %d in %s" % (uid, ds)) + def deletingmessages(self, uidlist, destlist): + ds = self.folderlist(destlist) + self.logger.info("Deleting %d messages (%s) in %s" % ( + len(uidlist), + offlineimap.imaputil.uid_sequence(uidlist), ds)) - def deletingmessages(s, uidlist, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting %d messages (%s) in %s" % \ - (len(uidlist), - offlineimap.imaputil.uid_sequence(uidlist), - ds)) + def addingflags(self, uidlist, flags, dest): + self.logger.info("Adding flag %s to %d messages on %s" % ( + ", ".join(flags), len(uidlist), dest)) - def addingflags(s, uidlist, flags, dest): - if s.verbose >= 0: - s._msg("Adding flag %s to %d messages on %s" % \ - (", ".join(flags), len(uidlist), dest)) - - def deletingflags(s, uidlist, flags, dest): - if s.verbose >= 0: - s._msg("Deleting flag %s from %d messages on %s" % \ - (", ".join(flags), len(uidlist), dest)) + def deletingflags(self, uidlist, flags, dest): + self.logger.info("Deleting flag %s from %d messages on %s" % ( + ", ".join(flags), len(uidlist), dest)) def serverdiagnostics(self, repository, type): """Connect to repository and output useful information for debugging""" @@ -371,69 +395,68 @@ class UIBase: ################################################## Threads - def getThreadDebugLog(s, thread): - if s.debugmessages.has_key(thread): + def getThreadDebugLog(self, thread): + if self.debugmessages.has_key(thread): message = "\nLast %d debug messages logged for %s prior to exception:\n"\ - % (len(s.debugmessages[thread]), thread.getName()) - message += "\n".join(s.debugmessages[thread]) + % (len(self.debugmessages[thread]), thread.getName()) + message += "\n".join(self.debugmessages[thread]) else: message = "\nNo debug messages were logged for %s." % \ thread.getName() return message - def delThreadDebugLog(s, thread): - if s.debugmessages.has_key(thread): - del s.debugmessages[thread] + def delThreadDebugLog(self, thread): + if thread in self.debugmessages: + del self.debugmessages[thread] - def getThreadExceptionString(s, thread): + def getThreadExceptionString(self, thread): message = "Thread '%s' terminated with exception:\n%s" % \ (thread.getName(), thread.getExitStackTrace()) - message += "\n" + s.getThreadDebugLog(thread) + message += "\n" + self.getThreadDebugLog(thread) return message - def threadException(s, thread): + def threadException(self, thread): """Called when a thread has terminated with an exception. The argument is the ExitNotifyThread that has so terminated.""" - s._msg(s.getThreadExceptionString(thread)) - s.delThreadDebugLog(thread) - s.terminate(100) + self.warn(self.getThreadExceptionString(thread)) + self.delThreadDebugLog(thread) + self.terminate(100) def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" #print any exceptions that have occurred over the run if not self.exc_queue.empty(): - self._msg("\nERROR: Exceptions occurred during the run!") + self.warn("ERROR: Exceptions occurred during the run!") while not self.exc_queue.empty(): msg, exc, exc_traceback = self.exc_queue.get() if msg: - self._msg("ERROR: %s\n %s" % (msg, exc)) + self.warn("ERROR: %s\n %s" % (msg, exc)) else: - self._msg("ERROR: %s" % (exc)) + self.warn("ERROR: %s" % (exc)) if exc_traceback: - self._msg("\nTraceback:\n%s" %"".join( + self.warn("\nTraceback:\n%s" %"".join( traceback.format_tb(exc_traceback))) if errormsg and errortitle: - sys.stderr.write('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) + self.warn('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) elif errormsg: - sys.stderr.write('%s\n' % errormsg) + self.warn('%s\n' % errormsg) sys.exit(exitstatus) - def threadExited(s, thread): + def threadExited(self, thread): """Called when a thread has exited normally. Many UIs will just ignore this.""" - s.delThreadDebugLog(thread) - s.unregisterthread(thread) + self.delThreadDebugLog(thread) + self.unregisterthread(thread) ################################################## Hooks - def callhook(s, msg): - if s.verbose >= 0: - s._msg(msg) + def callhook(self, msg): + self.info(msg) ################################################## Other - def sleep(s, sleepsecs, account): + def sleep(self, sleepsecs, account): """This function does not actually output anything, but handles the overall sleep, dealing with updates as necessary. It will, however, call sleeping() which DOES output something. @@ -446,12 +469,12 @@ class UIBase: if account.get_abort_event(): abortsleep = True else: - abortsleep = s.sleeping(10, sleepsecs) - sleepsecs -= 10 - s.sleeping(0, 0) # Done sleeping. + abortsleep = self.sleeping(10, sleepsecs) + sleepsecs -= 10 + self.sleeping(0, 0) # Done sleeping. return abortsleep - def sleeping(s, sleepsecs, remainingsecs): + def sleeping(self, sleepsecs, remainingsecs): """Sleep for sleepsecs, display remainingsecs to go. Does nothing if sleepsecs <= 0. @@ -463,6 +486,7 @@ class UIBase: """ if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: - s._msg("Next refresh in %.1f minutes" % (remainingsecs/60.0)) + self.logger.info("Next refresh in %.1f minutes" % ( + remainingsecs/60.0)) time.sleep(sleepsecs) return 0 diff --git a/offlineimap/ui/__init__.py b/offlineimap/ui/__init__.py index 102cbfd..6ddbaf6 100644 --- a/offlineimap/ui/__init__.py +++ b/offlineimap/ui/__init__.py @@ -1,5 +1,5 @@ # UI module -# Copyright (C) 2010 Sebastian Spaeth +# Copyright (C) 2010-2011 Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From f6f8fc852809798823793d9800577321765e3be4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 27 Oct 2011 16:56:18 +0200 Subject: [PATCH 11/44] Simplify the keepalive code a bit Should not change behavior. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 25 +++++++++---------------- offlineimap/repository/Base.py | 2 +- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 7006ef9..e709125 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -389,47 +389,40 @@ class IMAPServer: to be invoked in a separate thread, which should be join()'d after the event is set.""" self.ui.debug('imap', 'keepalive thread started') - while 1: - self.ui.debug('imap', 'keepalive: top of loop') - if event.isSet(): - self.ui.debug('imap', 'keepalive: event is set; exiting') - return - self.ui.debug('imap', 'keepalive: acquiring connectionlock') + while not event.isSet(): self.connectionlock.acquire() numconnections = len(self.assignedconnections) + \ len(self.availableconnections) self.connectionlock.release() - self.ui.debug('imap', 'keepalive: connectionlock released') + threads = [] - for i in range(numconnections): self.ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections)) if len(self.idlefolders) > i: + # IDLE thread idler = IdleThread(self, self.idlefolders[i]) else: + # NOOP thread idler = IdleThread(self) idler.start() threads.append(idler) - self.ui.debug('imap', 'keepalive: thread started') self.ui.debug('imap', 'keepalive: waiting for timeout') event.wait(timeout) self.ui.debug('imap', 'keepalive: after wait') - self.ui.debug('imap', 'keepalive: joining threads') - for idler in threads: # Make sure all the commands have completed. idler.stop() idler.join() - - self.ui.debug('imap', 'keepalive: bottom of loop') - + self.ui.debug('imap', 'keepalive: all threads joined') + self.ui.debug('imap', 'keepalive: event is set; exiting') + return def verifycert(self, cert, hostname): '''Verify that cert (in socket.getpeercert() format) matches hostname. CRLs are not handled. - + Returns error message if any problems are found and None on success. ''' errstr = "CA Cert verifying failed: " @@ -508,7 +501,7 @@ class IdleThread(object): finally: if imapobj: self.parent.releaseconnection(imapobj) - self.stop_sig.wait() # wait until we are supposed to quit + self.stop_sig.wait() # wait until we are supposed to quit def dosync(self): remoterepos = self.parent.repos diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 8f400f0..8af1f03 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -229,4 +229,4 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): """Stop keep alive, but don't bother waiting for the threads to terminate.""" pass - + From 8195d1410f94de4956047f8dc004c0acc68ee386 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 27 Oct 2011 16:58:44 +0200 Subject: [PATCH 12/44] Prettify Blinkenlights.sleep() Prettify the function. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 3dc32f5..46b5157 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -506,11 +506,11 @@ class Blinkenlights(UIBase, CursesUtil): # Trying to end sleep somewhere. self.getaccountframe(self.hotkeys[index]).syncnow() - def sleep(s, sleepsecs, account): - s.gettf().setcolor('red') - s._msg("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - s.getaccountframe().startsleep(sleepsecs) - return UIBase.sleep(s, sleepsecs, account) + def sleep(self, sleepsecs, account): + self.gettf().setcolor('red') + self.info("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) + self.getaccountframe().startsleep(sleepsecs) + return super(Blinkenlights, self).sleep(sleepsecs, account) def sleeping(self, sleepsecs, remainingsecs): if remainingsecs and s.gettf().getcolor() == 'black': From d992c66156ccd15eac3d9b92a0a49a375e36b40d Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 27 Oct 2011 17:23:43 +0200 Subject: [PATCH 13/44] Rework the whole unused get/setExitCause machinery It is basically unused by now. Rework to be able to make use of it later, no functional changes. Signed-off-by: Sebastian Spaeth --- offlineimap/threadutil.py | 72 ++++++++++++++++----------------------- offlineimap/ui/Curses.py | 2 +- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 39d3ae7..2dfb194 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -109,15 +109,15 @@ def exitnotifymonitorloop(callback): def threadexited(thread): """Called when a thread exits.""" ui = getglobalui() - if thread.getExitCause() == 'EXCEPTION': - if isinstance(thread.getExitException(), SystemExit): + if thread.exit_exception: + if isinstance(thread.exit_exception, SystemExit): # Bring a SystemExit into the main thread. # Do not send it back to UI layer right now. # Maybe later send it to ui.terminate? raise SystemExit ui.threadException(thread) # Expected to terminate sys.exit(100) # Just in case... - elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE': + elif thread.exit_message == 'SYNC_WITH_TIMER_TERMINATE': ui.terminate() # Just in case... sys.exit(100) @@ -128,7 +128,10 @@ class ExitNotifyThread(Thread): """This class is designed to alert a "monitor" to the fact that a thread has exited and to provide for the ability for it to find out why. All instances are made daemon threads (setDaemon(True), so we - bail out when the mainloop dies.""" + bail out when the mainloop dies. + + The thread can set instance variables self.exit_message for a human + readable reason of the thread exit.""" profiledir = None """class variable that is set to the profile directory if required""" @@ -137,6 +140,9 @@ class ExitNotifyThread(Thread): # These are all child threads that are supposed to go away when # the main thread is killed. self.setDaemon(True) + self.exit_message = None + self._exit_exc = None + self._exit_stacktrace = None def run(self): global exitthreads @@ -156,49 +162,31 @@ class ExitNotifyThread(Thread): pass prof.dump_stats(os.path.join(ExitNotifyThread.profiledir, "%s_%s.prof" % (self.threadid, self.getName()))) - except: - self.setExitCause('EXCEPTION') - if sys: - self.setExitException(sys.exc_info()[1]) - tb = traceback.format_exc() - self.setExitStackTrace(tb) - else: - self.setExitCause('NORMAL') - if not hasattr(self, 'exitmessage'): - self.setExitMessage(None) + except Exception, e: + # Thread exited with Exception, store it + tb = traceback.format_exc() + self.set_exit_exception(e, tb) if exitthreads: exitthreads.put(self, True) - def setExitCause(self, cause): - self.exitcause = cause - def getExitCause(self): + def set_exit_exception(self, exc, st=None): + """Sets Exception and stacktrace of a thread, so that other + threads can query its exit status""" + self._exit_exc = exc + self._exit_stacktrace = st + + @property + def exit_exception(self): """Returns the cause of the exit, one of: - 'EXCEPTION' -- the thread aborted because of an exception - 'NORMAL' -- normal termination.""" - return self.exitcause - def setExitException(self, exc): - self.exitexception = exc - def getExitException(self): - """If getExitCause() is 'EXCEPTION', holds the value from - sys.exc_info()[1] for this exception.""" - return self.exitexception - def setExitStackTrace(self, st): - self.exitstacktrace = st - def getExitStackTrace(self): - """If getExitCause() is 'EXCEPTION', returns a string representing - the stack trace for this exception.""" - return self.exitstacktrace - def setExitMessage(self, msg): - """Sets the exit message to be fetched by a subsequent call to - getExitMessage. This message may be any object or type except - None.""" - self.exitmessage = msg - def getExitMessage(self): - """For any exit cause, returns the message previously set by - a call to setExitMessage(), or None if there was no such message - set.""" - return self.exitmessage + Exception() -- the thread aborted with this exception + None -- normal termination.""" + return self._exit_exc + + @property + def exit_stacktrace(self): + """Returns a string representing the stack trace if set""" + return self._exit_stacktrace @classmethod def set_profiledir(cls, directory): diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 46b5157..2afb6cd 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -493,7 +493,7 @@ class Blinkenlights(UIBase, CursesUtil): # Request to quit. #TODO: this causes us to bail out in main loop when the thread exits #TODO: review and rework this mechanism. - currentThread().setExitCause('EXCEPTION') + currentThread().set_exit_exception(SystemExit("User requested shutdown")) self.terminate() try: index = acctkeys.index(chr(key)) From 7c45e05428e6e6a9437c2f3dc6597e19e864b111 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 27 Oct 2011 17:45:00 +0200 Subject: [PATCH 14/44] Fix getExitStackTrace call It was changed --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 9f5f4bc..44c920f 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -411,7 +411,7 @@ class UIBase(object): def getThreadExceptionString(self, thread): message = "Thread '%s' terminated with exception:\n%s" % \ - (thread.getName(), thread.getExitStackTrace()) + (thread.getName(), thread.exit_stacktrace) message += "\n" + self.getThreadDebugLog(thread) return message From d981d66305d3e8342e74cf5a130339aab1eb5396 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 28 Oct 2011 09:35:23 +0200 Subject: [PATCH 15/44] Add changelog entry for reworked UI backends Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 1f6b46b..778b52c 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -15,7 +15,7 @@ New Features * add a --info command line switch that outputs useful information about the server and the configuration for all enabled accounts. - + Changes ------- @@ -23,6 +23,14 @@ Changes * Output how long an account sync took (min:sec). +* Reworked logging which was reported to e.g. not flush output to files + often enough. User-visible changes: + a) console output goes to stderr (for now). + b) file output has timestamps and looks identical in the basic and + ttyui UIs. + c) File output should be flushed after logging by default (do + report if not). + Bug Fixes --------- From 33b4a16dac98e303322d2a0e56404e865c762d28 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 08:40:03 +0100 Subject: [PATCH 16/44] Fix mbox.select(foldername) readonly parameter comparison The default parameter value was "None", and we were comparing that directly to the imaplib2 value of is_readonly which is False or True, so the comparison always returned "False". Fix this by setting the default parameter to "False" and not "None". Also convert all users of that function to use False/True. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ offlineimap/folder/IMAP.py | 4 ++-- offlineimap/imaplibutil.py | 14 +++++++------- offlineimap/repository/IMAP.py | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 778b52c..ff9fc5b 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -43,3 +43,7 @@ Bug Fixes * Make NOOPs to keep a server connection open more resistant against dropped connections. + +* a readonly parameter to select() was not always treated correctly, + which could result in some folders being opened read-only when we + really needed read-write. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 58ff633..a7b8762 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -54,7 +54,7 @@ class IMAPFolder(BaseFolder): try: imapobj.select(self.getfullname()) except imapobj.readonly: - imapobj.select(self.getfullname(), readonly = 1) + imapobj.select(self.getfullname(), readonly = True) def suggeststhreads(self): return 1 @@ -204,7 +204,7 @@ class IMAPFolder(BaseFolder): fails_left = 2 # retry on dropped connection while fails_left: try: - imapobj.select(self.getfullname(), readonly = 1) + imapobj.select(self.getfullname(), readonly = True) res_type, data = imapobj.uid('fetch', str(uid), '(BODY.PEEK[])') fails_left = 0 diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 5c94592..c9a7715 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -34,37 +34,37 @@ except ImportError: #fails on python <2.6 pass -class UsefulIMAPMixIn: +class UsefulIMAPMixIn(object): def getselectedfolder(self): if self.state == 'SELECTED': return self.mailbox return None - def select(self, mailbox='INBOX', readonly=None, force = 0): + def select(self, mailbox='INBOX', readonly=False, force = 0): """Selects a mailbox on the IMAP server :returns: 'OK' on success, nothing if the folder was already selected or raises an :exc:`OfflineImapError`""" - if (not force) and self.getselectedfolder() == mailbox \ - and self.is_readonly == readonly: + if self.getselectedfolder() == mailbox and self.is_readonly == readonly \ + and not force: # No change; return. return # Wipe out all old responses, to maintain semantics with old imaplib2 del self.untagged_responses[:] try: - result = self.__class__.__bases__[1].select(self, mailbox, readonly) + result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) except self.abort, e: # self.abort is raised when we are supposed to retry errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ "ver said: %s" % (self.host, mailbox, e.args[0]) severity = OfflineImapError.ERROR.FOLDER_RETRY - raise OfflineImapError(errstr, severity) + raise OfflineImapError(errstr, severity) if result[0] != 'OK': #in case of error, bail out with OfflineImapError errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\ (mailbox, result) severity = OfflineImapError.ERROR.FOLDER - raise OfflineImapError(errstr, severity) + raise OfflineImapError(errstr, severity) return result def _mesg(self, s, tn=None, secs=None): diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 4c2cadd..8c29ee2 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -289,7 +289,7 @@ class IMAPRepository(BaseRepository): try: for foldername in self.folderincludes: try: - imapobj.select(foldername, readonly = 1) + imapobj.select(foldername, readonly = True) except OfflineImapError, e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: From 006f19aba90640d931f475635a8a69694d498e72 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:34:45 +0100 Subject: [PATCH 17/44] Make releaseconnection a NOOP when conn is None During cleanup we often call releaseconnection in a finally: block. But in cases of error, we might have dropped the connection earlier already and set it to "None". In this case don't fail releaseconnection() but make it a NOOP. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index e709125..de1cf95 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -113,6 +113,7 @@ class IMAPServer: :param drop_conn: If True, the connection will be released and not be reused. This can be used to indicate broken connections.""" + if connection is None: return #noop on bad connection self.connectionlock.acquire() self.assignedconnections.remove(connection) # Don't reuse broken connections From 656405616d77b75264a23a86e725884bfeb68632 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:27:08 +0100 Subject: [PATCH 18/44] Drop connection if it might be bad on APPEND 1) Differentiate error messages between imaplib.abort and imaplib.error exceptions in the log. 2) Drop connections in the case of imapobj.error, it also might denote a broken connection. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index a7b8762..4471387 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -540,17 +540,19 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() if not retry_left: raise OfflineImapError("Saving msg in folder '%s', " - "repository '%s' failed. Server reponded: %s\n" + "repository '%s' failed (abort). Server reponded: %s\n" "Message content was: %s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) self.ui.error(e, exc_info()[2]) - except imapobj.error, e: # APPEND failed # If the server responds with 'BAD', append() # raise()s directly. So we catch that too. + # drop conn, it might be bad. + self.imapserver.releaseconnection(imapobj, True) + imapobj = None raise OfflineImapError("Saving msg folder '%s', repo '%s'" - "failed. Server reponded: %s\nMessage content was: " + "failed (error). Server reponded: %s\nMessage content was: " "%s" % (self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE) # Checkpoint. Let it write out stuff, etc. Eg searches for From 54117e702d78e549df82c0233a3a23a248a78349 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:55:42 +0100 Subject: [PATCH 19/44] Bump bundled imaplib2 to 2.29 It has fixed some bugs, so update to current upstream. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/imaplib2.py | 27 +++++++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index ff9fc5b..13b97e8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -31,6 +31,8 @@ Changes c) File output should be flushed after logging by default (do report if not). +* Bumped bundled imaplib2 to release 2.29 + Bug Fixes --------- diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index cf3c480..ffa2676 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.28" +__version__ = "2.29" __release__ = "2" -__revision__ = "28" +__revision__ = "29" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -1234,11 +1234,13 @@ class IMAP4(object): def _choose_nonull_or_dflt(self, dflt, *args): dflttyp = type(dflt) + if isinstance(dflttyp, basestring): + dflttyp = basestring # Allow any string type for arg in args: if arg is not None: - if type(arg) is dflttyp: + if isinstance(arg, dflttyp): return arg - if __debug__: self._log(1, 'bad arg type is %s, expecting %s' % (type(arg), dflttyp)) + if __debug__: self._log(0, 'bad arg is %s, expecting %s' % (type(arg), dflttyp)) return dflt @@ -2323,6 +2325,7 @@ if __name__ == '__main__': data = open(os.path.exists("test.data") and "test.data" or __file__).read(1000) test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)s%(data)s' \ % {'user':USER, 'lf':'\n', 'data':data} + test_seq1 = [ ('list', ('""', '%')), ('create', ('/tmp/imaplib2_test.0',)), @@ -2342,7 +2345,7 @@ if __name__ == '__main__': test_seq2 = ( ('select', ()), - ('response',('UIDVALIDITY',)), + ('response', ('UIDVALIDITY',)), ('response', ('EXISTS',)), ('append', (None, None, None, test_mesg)), ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')), @@ -2351,6 +2354,7 @@ if __name__ == '__main__': ('recent', ()), ) + AsyncError = None def responder((response, cb_arg, error)): @@ -2436,10 +2440,21 @@ if __name__ == '__main__': if 'IDLE' in M.capabilities: run('idle', (2,), cb=False) - run('idle', (99,), cb=True) # Asynchronous, to test interruption of 'idle' by 'noop' + run('idle', (99,)) # Asynchronous, to test interruption of 'idle' by 'noop' time.sleep(1) run('noop', (), cb=False) + run('append', (None, None, None, test_mesg), cb=False) + num = run('search', (None, 'ALL'), cb=False)[0].split()[0] + dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) + M._mesg('fetch %s => %s' % (num, `dat`)) + run('idle', (2,)) + run('store', (num, '-FLAGS', '(\Seen)'), cb=False), + dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) + M._mesg('fetch %s => %s' % (num, `dat`)) + run('uid', ('STORE', num, 'FLAGS', '(\Deleted)')) + run('expunge', ()) + run('logout', (), cb=False) if debug: From 4eeb88dd8fff60f11321192cf5ca264e8ddcb3e9 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 10:56:42 +0100 Subject: [PATCH 20/44] Append, don't truncate, log file The new logging handler was truncating the log file on each start. Append by default. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/UIBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 44c920f..75cb313 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -81,7 +81,7 @@ class UIBase(object): def setlogfile(self, logfile): """Create file handler which logs to file""" - fh = logging.FileHandler(logfile, 'wt') + fh = logging.FileHandler(logfile, 'at') #fh.setLevel(logging.DEBUG) file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " "%(message)s", '%Y-%m-%d %H:%M:%S') From d54859a9318baf6d6eb000a82793d3beff20f547 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:29:23 +0100 Subject: [PATCH 21/44] Don't setDaemon explicitly, it's done inherently All ExitNotifyThreads and InstanceLimitThreads are setDaemon(True) in their constructor, so there is no need to do that again in the code. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 1 - offlineimap/folder/Base.py | 1 - offlineimap/init.py | 1 - offlineimap/threadutil.py | 4 ++-- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 7387c51..7a87791 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -291,7 +291,6 @@ class SyncableAccount(Account): name = "Folder %s [acc: %s]" % (remotefolder, self), args = (self.name, remoterepos, remotefolder, localrepos, statusrepos, quick)) - thread.setDaemon(1) thread.start() folderthreads.append(thread) # wait for all threads to finish diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 91ee08a..f8cea25 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -341,7 +341,6 @@ class BaseFolder(object): target = self.copymessageto, name = "Copy message from %s:%s" % (self.repository, self), args = (uid, dstfolder, statusfolder)) - thread.setDaemon(1) thread.start() threads.append(thread) else: diff --git a/offlineimap/init.py b/offlineimap/init.py index 0e5b08d..36240ac 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -358,7 +358,6 @@ class OfflineImap: name='Sync Runner', kwargs = {'accounts': syncaccounts, 'config': self.config}) - t.setDaemon(True) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) self.ui.terminate() diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 2dfb194..e94910b 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -28,8 +28,8 @@ from offlineimap.ui import getglobalui ###################################################################### def semaphorereset(semaphore, originalstate): - """Wait until the semaphore gets back to its original state -- all acquired - resources released.""" + """Block until `semaphore` gets back to its original state, ie all acquired + resources have been released.""" for i in range(originalstate): semaphore.acquire() # Now release these. From ccfa747ce65583c2352e6546885a9a47ff59e5cb Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:30:16 +0100 Subject: [PATCH 22/44] Use lock with 'with' statement To make sure, the lock gets released even if we raise an exception between acquire() and release() Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index de1cf95..721c646 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +from __future__ import with_statement # needed for python 2.5 from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui from threading import Lock, BoundedSemaphore, Thread, Event, currentThread @@ -355,7 +356,7 @@ class IMAPServer: else: # re-raise all other errors raise - + def connectionwait(self): """Waits until there is a connection available. Note that between the time that a connection becomes available and the time it is @@ -370,18 +371,18 @@ class IMAPServer: def close(self): # Make sure I own all the semaphores. Let the threads finish # their stuff. This is a blocking method. - self.connectionlock.acquire() - threadutil.semaphorereset(self.semaphore, self.maxconnections) - for imapobj in self.assignedconnections + self.availableconnections: - imapobj.logout() - self.assignedconnections = [] - self.availableconnections = [] - self.lastowner = {} - # reset kerberos state - self.gss_step = self.GSS_STATE_STEP - self.gss_vc = None - self.gssapi = False - self.connectionlock.release() + with self.connectionlock: + # first, wait till all + threadutil.semaphorereset(self.semaphore, self.maxconnections) + for imapobj in self.assignedconnections + self.availableconnections: + imapobj.logout() + self.assignedconnections = [] + self.availableconnections = [] + self.lastowner = {} + # reset kerberos state + self.gss_step = self.GSS_STATE_STEP + self.gss_vc = None + self.gssapi = False def keepalive(self, timeout, event): """Sends a NOOP to each connection recorded. It will wait a maximum From 52ca66f055f5f65a5533b5ede616f140d90b4b51 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:37:29 +0100 Subject: [PATCH 23/44] Reduce log verbosity while scanning Maildir no need to be that verbose here... Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Maildir.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index ae09486..6e93839 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -149,14 +149,12 @@ class MaildirRepository(BaseRepository): # Iterate over directories in top & top itself. for dirname in os.listdir(toppath) + ['']: - self.debug(" *** top of loop") self.debug(" dirname = %s" % dirname) if dirname in ['cur', 'new', 'tmp']: self.debug(" skipping this dir (Maildir special)") # Bypass special files. continue fullname = os.path.join(toppath, dirname) - self.debug(" fullname = %s" % fullname) if not os.path.isdir(fullname): self.debug(" skipping this entry (not a directory)") # Not a directory -- not a folder. From 74d580bc6892f0a22bbbc573c2418b8fbf839933 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:55:05 +0100 Subject: [PATCH 24/44] Exit "infinite" monitorloop when SyncRunner thread exits The huge UI rework patch removed some obscure logic and special handling of thread exit messages. It turns out that this was in fact still needed as a specific exit message of the SyncRunner thread signified the threatmonitor to quit. We will want a nicer machinery for this in the future I guess, but fix the eternal hang on exit by reintroducing a special exit message for the SyncRunner thread, and return from the infinite monitor loop if SyncRunner finishes. Signed-off-by: Sebastian Spaeth --- offlineimap/syncmaster.py | 2 ++ offlineimap/threadutil.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/offlineimap/syncmaster.py b/offlineimap/syncmaster.py index 0d139dd..5fd0dee 100644 --- a/offlineimap/syncmaster.py +++ b/offlineimap/syncmaster.py @@ -30,6 +30,8 @@ def syncaccount(threads, config, accountname): threads.add(thread) def syncitall(accounts, config): + # Special exit message for SyncRunner thread, so main thread can exit + currentThread().exit_message = 'SYNCRUNNER_EXITED_NORMALLY' threads = threadlist() for accountname in accounts: syncaccount(threads, config, accountname) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index e94910b..c102446 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -95,19 +95,23 @@ def exitnotifymonitorloop(callback): :type callback: a callable function """ global exitthreads - while 1: + do_loop = True + while do_loop: # Loop forever and call 'callback' for each thread that exited try: # we need a timeout in the get() call, so that ctrl-c can throw # a SIGINT (http://bugs.python.org/issue1360). A timeout with empty # Queue will raise `Empty`. thrd = exitthreads.get(True, 60) - callback(thrd) + # request to abort when callback returns true + do_loop = (callback(thrd) != True) except Empty: pass def threadexited(thread): - """Called when a thread exits.""" + """Called when a thread exits. + + Main thread is aborted when this returns True.""" ui = getglobalui() if thread.exit_exception: if isinstance(thread.exit_exception, SystemExit): @@ -117,12 +121,11 @@ def threadexited(thread): raise SystemExit ui.threadException(thread) # Expected to terminate sys.exit(100) # Just in case... - elif thread.exit_message == 'SYNC_WITH_TIMER_TERMINATE': - ui.terminate() - # Just in case... - sys.exit(100) + elif thread.exit_message == 'SYNCRUNNER_EXITED_NORMALLY': + return True else: ui.threadExited(thread) + return False class ExitNotifyThread(Thread): """This class is designed to alert a "monitor" to the fact that a From 8dbf62cfdba9924e5e82b041e118f6fe059637da Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 11:56:04 +0100 Subject: [PATCH 25/44] Add a TODO comment This function can IMHO lead to possible deadlocks when waiting for the connectionlock. Do add a comment to that regard, this will need to audit. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 721c646..2f41bde 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -372,7 +372,10 @@ class IMAPServer: # Make sure I own all the semaphores. Let the threads finish # their stuff. This is a blocking method. with self.connectionlock: - # first, wait till all + # first, wait till all connections had been released. + # TODO: won't work IMHO, as releaseconnection() also + # requires the connectionlock, leading to a potential + # deadlock! Audit & check! threadutil.semaphorereset(self.semaphore, self.maxconnections) for imapobj in self.assignedconnections + self.availableconnections: imapobj.logout() From 212e50eb4b53055e247feb29d34e8a007610c5cc Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 12:48:22 +0100 Subject: [PATCH 26/44] Fix sleeping in the Blinkenlights UI We were still referring to s.gettf() in sleeping(self, ...) causing each attempt to sleep to crash. Fix this, and the CursesAccountFrame.sleeping() method. I am sure, there is still wrong and broken but we are getting there. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 2afb6cd..42a8f01 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -151,26 +151,15 @@ class CursesAccountFrame: self.children.append(tf) return tf - def startsleep(s, sleepsecs): - s.sleeping_abort = 0 + def startsleep(self, sleepsecs): + self.sleeping_abort = 0 - def sleeping(s, sleepsecs, remainingsecs): - if remainingsecs: - s.c.lock() - try: - s.drawleadstr(remainingsecs) - s.window.refresh() - finally: - s.c.unlock() - time.sleep(sleepsecs) - else: - s.c.lock() - try: - s.drawleadstr() - s.window.refresh() - finally: - s.c.unlock() - return s.sleeping_abort + def sleeping(self, sleepsecs, remainingsecs): + # show how long we are going to sleep and sleep + self.drawleadstr(remainingsecs) + self.ui.exec_locked(self.window.refresh) + time.sleep(sleepsecs) + return 0 def syncnow(s): s.sleeping_abort = 1 @@ -513,10 +502,9 @@ class Blinkenlights(UIBase, CursesUtil): return super(Blinkenlights, self).sleep(sleepsecs, account) def sleeping(self, sleepsecs, remainingsecs): - if remainingsecs and s.gettf().getcolor() == 'black': - self.gettf().setcolor('red') - else: - self.gettf().setcolor('black') + if not sleepsecs: + # reset color to default if we are done sleeping. + self.gettf().setcolor('white') return self.getaccountframe().sleeping(sleepsecs, remainingsecs) def resizeterm(self): From bfbd37802525279fbfbd540a6e06eafa60b90921 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 15:56:33 +0100 Subject: [PATCH 27/44] Add FAQ item on "Mailbox already exists" error IMAP servers treating folder names as case insensitive can lead to that error. Signed-off-by: Sebastian Spaeth --- docs/FAQ.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/FAQ.rst b/docs/FAQ.rst index e3a586b..29b53d6 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -219,6 +219,27 @@ as follows:: 2) while in sleep mode, you can also send a SIGUSR1. See the `Signals on UNIX`_ section in the MANUAL for details. +I get a "Mailbox already exists" error +-------------------------------------- +**Q:** When synchronizing, I receive errors such as:: + + Folder 'sent'[main-remote] could not be created. Server responded: + ('NO', ['Mailbox already exists.']) + +**A:** IMAP folders are usually case sensitive. But some IMAP servers seem + to treat "special" folders as case insensitive (e.g. the initial + INBOX. part, or folders such as "Sent" or "Trash"). If you happen to + have a folder "sent" on one side of things and a folder called "Sent" + on the other side, offlineimap will try to create those folders on + both sides. If you server happens to treat those folders as + case-insensitive you can then see this warning. + + You can solve this by excluding the "sent" folder by filtering it from + the repository settings:: + + folderfilter= lambda f: f not in ['sent'] + + Configuration Questions ======================= From 70125d58e67de1fff83a3e59d9498668153a2d55 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 2 Nov 2011 17:03:40 +0100 Subject: [PATCH 28/44] Curses UI: make resize behave better Resizing a Blinkenlights terminal doesn't crash anymore, and actually seems to be changing the size, with this patch. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 42a8f01..341d4e8 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -119,6 +119,7 @@ class CursesAccountFrame: self.children = [] self.accountname = accountname self.ui = ui + self.window = None def drawleadstr(self, secs = None): #TODO: does what? @@ -178,7 +179,7 @@ class CursesThreadFrame: self.curses_color = curses.color_pair(0) #default color def setcolor(self, color, modifier=0): - """Draw the thread symbol '.' in the specified color + """Draw the thread symbol '@' in the specified color :param modifier: Curses modified, such as curses.A_BOLD""" self.curses_color = modifier | self.ui.curses_colorpair(color) self.colorname = color @@ -186,7 +187,7 @@ class CursesThreadFrame: def display(self): def locked_display(): - self.window.addch(self.y, self.x, '.', self.curses_color) + self.window.addch(self.y, self.x, '@', self.curses_color) self.window.refresh() # lock the curses IO while fudging stuff self.ui.exec_locked(locked_display) @@ -289,7 +290,7 @@ class CursesLogHandler(logging.StreamHandler): finally: self.ui.unlock() self.ui.tframe_lock.release() - self.ui.logwin.refresh() + self.ui.logwin.noutrefresh() self.ui.stdscr.refresh() class Blinkenlights(UIBase, CursesUtil): @@ -509,7 +510,7 @@ class Blinkenlights(UIBase, CursesUtil): def resizeterm(self): """Resize the current windows""" - self.exec_locked(self.setupwindows(True)) + self.exec_locked(self.setupwindows, True) def mainException(self): UIBase.mainException(self) @@ -537,33 +538,30 @@ class Blinkenlights(UIBase, CursesUtil): If `resize`, don't create new windows, just adapt size""" self.height, self.width = self.stdscr.getmaxyx() - if resize: - raise Exception("resizehandler %d" % self.width) - self.logheight = self.height - len(self.accframes) - 1 if resize: curses.resizeterm(self.height, self.width) self.bannerwin.resize(1, self.width) + self.logwin.resize(self.logheight, self.width) else: self.bannerwin = curses.newwin(1, self.width, 0, 0) self.logwin = curses.newwin(self.logheight, self.width, 1, 0) self.draw_bannerwin() - self.logwin.idlok(1) - self.logwin.scrollok(1) + self.logwin.idlok(True) # needed for scrollok below + self.logwin.scrollok(True) # scroll window when too many lines added self.logwin.move(self.logheight - 1, 0) self.draw_logwin() self.accounts = reversed(sorted(self.accframes.keys())) - pos = self.height - 1 index = 0 self.hotkeys = [] for account in self.accounts: - acc_win = curses.newwin(1, self.width, pos, 0) - self.accframes[account].setwindow(acc_win, acctkeys[index]) - self.hotkeys.append(account) - index += 1 - pos -= 1 + acc_win = curses.newwin(1, self.width, pos, 0) + self.accframes[account].setwindow(acc_win, acctkeys[index]) + self.hotkeys.append(account) + index += 1 + pos -= 1 curses.doupdate() def draw_bannerwin(self): @@ -572,6 +570,7 @@ class Blinkenlights(UIBase, CursesUtil): color = curses.A_BOLD | self.curses_colorpair('banner') else: color = curses.A_REVERSE + self.bannerwin.clear() # Delete old content (eg before resizes) self.bannerwin.bkgd(' ', color) # Fill background with that color string = "%s %s" % (offlineimap.__productname__, offlineimap.__version__) @@ -581,10 +580,12 @@ class Blinkenlights(UIBase, CursesUtil): self.bannerwin.noutrefresh() def draw_logwin(self): - #if curses.has_colors(): - # color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLACK) - #else: - color = curses.A_NORMAL + """(Re)draw the current logwindow""" + if curses.has_colors(): + color = curses.color_pair(0) #default colors + else: + color = curses.A_NORMAL + self.logwin.clear() self.logwin.bkgd(' ', color) for line, color in self.text: self.logwin.addstr("\n" + line, color) @@ -598,7 +599,8 @@ class Blinkenlights(UIBase, CursesUtil): # 1) Return existing or 2) create a new CursesAccountFrame. if acc_name in self.accframes: return self.accframes[acc_name] self.accframes[acc_name] = CursesAccountFrame(self, acc_name) - self.setupwindows() + # update the window layout + self.setupwindows(resize= True) return self.accframes[acc_name] def terminate(self, *args, **kwargs): From ef28d5dac0efebcfaea6776e4f5988f1fb48e65b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Nov 2011 12:25:55 +0100 Subject: [PATCH 29/44] More Blinkenlights UI cleanup Rename some variables, simplify the hotkeys treatment. Refresh/exit signals still don't work as of yet, but will come. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 51 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 341d4e8..bc63f7e 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -1,6 +1,5 @@ # Curses-based interfaces -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from __future__ import with_statement # needed for python 2.5 -from threading import RLock, currentThread, Lock, Event, Thread +from threading import RLock, currentThread, Lock, Event from thread import get_ident # python < 2.6 support from collections import deque import time @@ -30,8 +29,6 @@ from offlineimap.ui.UIBase import UIBase from offlineimap.threadutil import ExitNotifyThread import offlineimap -acctkeys = '1234567890abcdefghijklmnoprstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-=;/.,' - class CursesUtil: def __init__(self, *args, **kwargs): @@ -115,28 +112,34 @@ class CursesAccountFrame: - window: curses window associated with an account """ - def __init__(self, ui, accountname): + def __init__(self, ui, acc_name): self.children = [] - self.accountname = accountname + self.acc_name = acc_name self.ui = ui self.window = None + """Curses window associated with this acc""" + self.acc_num = None + """Account number (& hotkey) associated with this acc""" + self.location = 0 + """length of the account prefix string""" - def drawleadstr(self, secs = None): - #TODO: does what? - if secs == None: - acctstr = '%s: [active] %13.13s: ' % (self.key, self.accountname) - else: - acctstr = '%s: [%3d:%02d] %13.13s: ' % (self.key, - secs / 60, secs % 60, - self.accountname) - self.ui.exec_locked(self.window.addstr, 0, 0, acctstr) - self.location = len(acctstr) + def drawleadstr(self, secs = 0): + """Draw the account status string - def setwindow(self, curses_win, key): - #TODO: does what? - # the curses window associated with an account + secs tells us how long we are going to sleep.""" + sleepstr = '%3d:%02d' % (secs // 60, secs % 60) if secs else 'active' + accstr = '%s: [%s] %12.12s: ' % (self.acc_num, sleepstr, self.acc_name) + + self.ui.exec_locked(self.window.addstr, 0, 0, accstr) + self.location = len(accstr) + + def setwindow(self, curses_win, acc_num): + """Register an curses win and a hotkey as Account window + + :param curses_win: the curses window associated with an account + :param acc_num: int denoting the hotkey associated with this account.""" self.window = curses_win - self.key = key + self.acc_num = acc_num self.drawleadstr() # Update the child ThreadFrames for child in self.children: @@ -486,9 +489,9 @@ class Blinkenlights(UIBase, CursesUtil): currentThread().set_exit_exception(SystemExit("User requested shutdown")) self.terminate() try: - index = acctkeys.index(chr(key)) + index = int(chr(key)) except ValueError: - # Key not a valid one: exit. + # Key not a valid number: exit. return if index >= len(self.hotkeys): # Not in our list of valid hotkeys. @@ -558,7 +561,7 @@ class Blinkenlights(UIBase, CursesUtil): self.hotkeys = [] for account in self.accounts: acc_win = curses.newwin(1, self.width, pos, 0) - self.accframes[account].setwindow(acc_win, acctkeys[index]) + self.accframes[account].setwindow(acc_win, index) self.hotkeys.append(account) index += 1 pos -= 1 From ab184d84e23a2280d6c501159759027180e07683 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Nov 2011 13:27:35 +0100 Subject: [PATCH 30/44] Reduce parameter list to account.syncfolder call The remote|local|statusrepo is an anttribute of each SyncableAccount() anyway, so we don't need to pass it in, we can simply get it from the Account(). Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 19 ++++++++++--------- offlineimap/imapserver.py | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 7a87791..51a3ab9 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -289,8 +289,7 @@ class SyncableAccount(Account): instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, name = "Folder %s [acc: %s]" % (remotefolder, self), - args = (self.name, remoterepos, remotefolder, localrepos, - statusrepos, quick)) + args = (self, remotefolder, quick)) thread.start() folderthreads.append(thread) # wait for all threads to finish @@ -329,15 +328,17 @@ class SyncableAccount(Account): except Exception, e: self.ui.error(e, exc_info()[2], msg = "Calling hook") - -def syncfolder(accountname, remoterepos, remotefolder, localrepos, - statusrepos, quick): +def syncfolder(account, remotefolder, quick): """This function is called as target for the InstanceLimitedThread invokation in SyncableAccount. Filtered folders on the remote side will not invoke this function.""" + remoterepos = account.remoterepos + localrepos = account.localrepos + statusrepos = account.statusrepos + ui = getglobalui() - ui.registerthread(accountname) + ui.registerthread(account.name) try: # Load local folder. localfolder = localrepos.\ @@ -352,7 +353,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, % localfolder) return # Write the mailboxes - mbnames.add(accountname, localfolder.getvisiblename()) + mbnames.add(account.name, localfolder.getvisiblename()) # Load status folder. statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ @@ -431,11 +432,11 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, "[acc: '%s']" % ( remotefolder.getvisiblename().\ replace(remoterepos.getsep(), localrepos.getsep()), - accountname)) + account)) # we reconstruct foldername above rather than using # localfolder, as the localfolder var is not # available if assignment fails. except Exception, e: ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ - (accountname,remotefolder.getvisiblename(), + (account, remotefolder.getvisiblename(), traceback.format_exc())) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 2f41bde..b8446d3 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -515,9 +515,9 @@ class IdleThread(object): remoterepos = account.remoterepos statusrepos = account.statusrepos remotefolder = remoterepos.getfolder(self.folder) - offlineimap.accounts.syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False) + offlineimap.accounts.syncfolder(account, remotefolder, quick=False) ui = getglobalui() - ui.unregisterthread(currentThread()) + ui.unregisterthread(currentThread()) #syncfolder registered the thread def idle(self): """Invoke IDLE mode until timeout or self.stop() is invoked""" From f4a32bafd64ee80050ee3aa85aab5a5d369693b6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Nov 2011 13:45:44 +0100 Subject: [PATCH 31/44] Pass ui.registerthread an Account() and not a name as string This way, we can use all the account functions such as set_abort_event() from the ui if needed. Signed-off-by: Sebastian Spaeth --- offlineimap/accounts.py | 4 ++-- offlineimap/folder/Base.py | 2 +- offlineimap/ui/Curses.py | 11 +++++------ offlineimap/ui/Machine.py | 6 +++--- offlineimap/ui/UIBase.py | 8 +++++--- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 51a3ab9..bf3f327 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -206,7 +206,7 @@ class SyncableAccount(Account): pass #Failed to delete for some reason. def syncrunner(self): - self.ui.registerthread(self.name) + self.ui.registerthread(self) accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): os.mkdir(accountmetadata, 0700) @@ -338,7 +338,7 @@ def syncfolder(account, remotefolder, quick): statusrepos = account.statusrepos ui = getglobalui() - ui.registerthread(account.name) + ui.registerthread(account) try: # Load local folder. localfolder = localrepos.\ diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index f8cea25..53bc71c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -254,7 +254,7 @@ class BaseFolder(object): # self.getmessage(). So, don't call self.getmessage unless # really needed. if register: # output that we start a new thread - self.ui.registerthread(self.accountname) + self.ui.registerthread(self.repository.account) try: message = None diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index bc63f7e..71ffe3a 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -105,16 +105,16 @@ class CursesUtil: class CursesAccountFrame: """Notable instance variables: - - accountname: String with associated account name + - account: corresponding Account() - children - ui - key - window: curses window associated with an account """ - def __init__(self, ui, acc_name): + def __init__(self, ui, account): self.children = [] - self.acc_name = acc_name + self.account = account self.ui = ui self.window = None """Curses window associated with this acc""" @@ -128,7 +128,7 @@ class CursesAccountFrame: secs tells us how long we are going to sleep.""" sleepstr = '%3d:%02d' % (secs // 60, secs % 60) if secs else 'active' - accstr = '%s: [%s] %12.12s: ' % (self.acc_num, sleepstr, self.acc_name) + accstr = '%s: [%s] %12.12s: ' % (self.acc_num, sleepstr, self.account) self.ui.exec_locked(self.window.addstr, 0, 0, accstr) self.location = len(accstr) @@ -491,8 +491,7 @@ class Blinkenlights(UIBase, CursesUtil): try: index = int(chr(key)) except ValueError: - # Key not a valid number: exit. - return + return # Key not a valid number: exit. if index >= len(self.hotkeys): # Not in our list of valid hotkeys. return diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 334121d..868433a 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -42,9 +42,9 @@ class MachineUI(UIBase): self.logger.warning("%s:%s:%s:%s" % ( 'warn', '', currentThread().getName(), msg)) - def registerthread(s, account): - UIBase.registerthread(s, account) - s._printData('registerthread', account) + def registerthread(self, account): + super(MachineUI, self).registerthread(self, account) + self._printData('registerthread', account) def unregisterthread(s, thread): UIBase.unregisterthread(s, thread) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 75cb313..0176422 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -160,12 +160,14 @@ class UIBase(object): self.debug('thread', "Unregister thread '%s'" % thr.getName()) def getthreadaccount(self, thr = None): - """Get name of account for a thread (current if None)""" - if not thr: + """Get Account() for a thread (current if None) + + If no account has been registered with this thread, return 'None'""" + if thr == None: thr = threading.currentThread() if thr in self.threadaccounts: return self.threadaccounts[thr] - return '*Control' # unregistered thread is '*Control' + return None def debug(self, debugtype, msg): cur_thread = threading.currentThread() From f8d5f1890ca7725df629967a6ace80030f1c9dbf Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 3 Nov 2011 14:21:25 +0100 Subject: [PATCH 32/44] Make sleep abort request working again for Curses UI 1) Rework the sleep abort request to set the skipsleep configuration setting that the sleep() code checks. 2) Only output 15 rather than 50 debug messages on abort... Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 63 ++++++++++++++++++++++------------------ offlineimap/ui/UIBase.py | 2 +- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 71ffe3a..35a17fb 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -113,8 +113,10 @@ class CursesAccountFrame: """ def __init__(self, ui, account): + """ + :param account: An Account() or None (for eg SyncrunnerThread)""" self.children = [] - self.account = account + self.account = account if account else '*Control' self.ui = ui self.window = None """Curses window associated with this acc""" @@ -155,18 +157,21 @@ class CursesAccountFrame: self.children.append(tf) return tf - def startsleep(self, sleepsecs): - self.sleeping_abort = 0 - def sleeping(self, sleepsecs, remainingsecs): # show how long we are going to sleep and sleep self.drawleadstr(remainingsecs) self.ui.exec_locked(self.window.refresh) time.sleep(sleepsecs) - return 0 + return self.account.abort_signal.is_set() - def syncnow(s): - s.sleeping_abort = 1 + def syncnow(self): + """Request that we stop sleeping asap and continue to sync""" + # if this belongs to an Account (and not *Control), set the + # skipsleep pref + if isinstance(self.account, offlineimap.accounts.Account): + self.ui.info("Requested synchronization for acc: %s" % self.account) + self.account.config.set('Account %s' % self.account.name, + 'skipsleep', '1') class CursesThreadFrame: """ @@ -442,37 +447,37 @@ class Blinkenlights(UIBase, CursesUtil): super(Blinkenlights, self).warn(msg) def threadExited(self, thread): - acc_name = self.getthreadaccount(thread) + acc = self.getthreadaccount(thread) with self.tframe_lock: - if thread in self.threadframes[acc_name]: - tf = self.threadframes[acc_name][thread] + if thread in self.threadframes[acc]: + tf = self.threadframes[acc][thread] tf.setcolor('black') - self.availablethreadframes[acc_name].append(tf) - del self.threadframes[acc_name][thread] + self.availablethreadframes[acc].append(tf) + del self.threadframes[acc][thread] super(Blinkenlights, self).threadExited(thread) def gettf(self): """Return the ThreadFrame() of the current thread""" cur_thread = currentThread() - acc_name = self.getthreadaccount() + acc = self.getthreadaccount() #Account() or None with self.tframe_lock: # Ideally we already have self.threadframes[accountname][thread] try: - if cur_thread in self.threadframes[acc_name]: - return self.threadframes[acc_name][cur_thread] + if cur_thread in self.threadframes[acc]: + return self.threadframes[acc][cur_thread] except KeyError: # Ensure threadframes already has an account dict - self.threadframes[acc_name] = {} - self.availablethreadframes[acc_name] = deque() + self.threadframes[acc] = {} + self.availablethreadframes[acc] = deque() # If available, return a ThreadFrame() - if len(self.availablethreadframes[acc_name]): - tf = self.availablethreadframes[acc_name].popleft() + if len(self.availablethreadframes[acc]): + tf = self.availablethreadframes[acc].popleft() tf.std_color() else: - tf = self.getaccountframe(acc_name).get_new_tframe() - self.threadframes[acc_name][cur_thread] = tf + tf = self.getaccountframe(acc).get_new_tframe() + self.threadframes[acc][cur_thread] = tf return tf def on_keypressed(self, key): @@ -501,14 +506,14 @@ class Blinkenlights(UIBase, CursesUtil): def sleep(self, sleepsecs, account): self.gettf().setcolor('red') self.info("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - self.getaccountframe().startsleep(sleepsecs) return super(Blinkenlights, self).sleep(sleepsecs, account) def sleeping(self, sleepsecs, remainingsecs): if not sleepsecs: # reset color to default if we are done sleeping. self.gettf().setcolor('white') - return self.getaccountframe().sleeping(sleepsecs, remainingsecs) + accframe = self.getaccountframe(self.getthreadaccount()) + return accframe.sleeping(sleepsecs, remainingsecs) def resizeterm(self): """Resize the current windows""" @@ -593,10 +598,10 @@ class Blinkenlights(UIBase, CursesUtil): self.logwin.addstr("\n" + line, color) self.logwin.noutrefresh() - def getaccountframe(self, acc_name = None): - """Return an AccountFrame()""" - if acc_name == None: - acc_name = self.getthreadaccount() + def getaccountframe(self, acc_name): + """Return an AccountFrame() corresponding to acc_name + + Note that the *control thread uses acc_name `None`.""" with self.aflock: # 1) Return existing or 2) create a new CursesAccountFrame. if acc_name in self.accframes: return self.accframes[acc_name] @@ -617,7 +622,7 @@ class Blinkenlights(UIBase, CursesUtil): # finally call parent terminate which prints out exceptions etc super(Blinkenlights, self).terminate(*args, **kwargs) - def threadException(s, thread): + def threadException(self, thread): #self._log_con_handler.stop() - UIBase.threadException(s, thread) + UIBase.threadException(self, thread) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 0176422..7dc23e5 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -48,7 +48,7 @@ class UIBase(object): """list of debugtypes we are supposed to log""" self.debugmessages = {} """debugmessages in a deque(v) per thread(k)""" - self.debugmsglen = 50 + self.debugmsglen = 15 self.threadaccounts = {} """dict linking active threads (k) to account names (v)""" self.acct_startimes = {} From a93c80292d0474420654eb9cfa873ce20a6fbdc8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 8 Nov 2011 13:51:36 +0100 Subject: [PATCH 33/44] Curses UI: Simplify text buffer handling Rather than keeping a separate queue of all logged lines in memory, we rely on the curses window scrolling functionality to scroll lines. On resizing the terminal this means, we'll clear the screen and start filling it afresh, but that should be acceptable. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Curses.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 35a17fb..f1626c4 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -278,6 +278,7 @@ class InputHandler(ExitNotifyThread): class CursesLogHandler(logging.StreamHandler): + """self.ui has been set to the UI class before anything is invoked""" def emit(self, record): log_str = super(CursesLogHandler, self).format(record) @@ -290,11 +291,9 @@ class CursesLogHandler(logging.StreamHandler): self.ui.tframe_lock.acquire() self.ui.lock() try: - for line in log_str.split("\n"): - self.ui.logwin.addstr("\n" + line, color) - self.ui.text.append((line, color)) - while len(self.ui.text) > self.ui.logheight: - self.ui.text.popleft() + y,x = self.ui.logwin.getyx() + if y or x: self.ui.logwin.addch(10) # no \n before 1st item + self.ui.logwin.addstr(log_str, color) finally: self.ui.unlock() self.ui.tframe_lock.release() @@ -360,7 +359,6 @@ class Blinkenlights(UIBase, CursesUtil): self.threadframes = {} self.accframes = {} self.aflock = Lock() - self.text = deque() self.stdscr = curses.initscr() # turn off automatic echoing of keys to the screen @@ -543,7 +541,8 @@ class Blinkenlights(UIBase, CursesUtil): def setupwindows(self, resize=False): """Setup and draw bannerwin and logwin - If `resize`, don't create new windows, just adapt size""" + If `resize`, don't create new windows, just adapt size. This + function should be invoked with CursesUtils.locked().""" self.height, self.width = self.stdscr.getmaxyx() self.logheight = self.height - len(self.accframes) - 1 if resize: @@ -555,9 +554,8 @@ class Blinkenlights(UIBase, CursesUtil): self.logwin = curses.newwin(self.logheight, self.width, 1, 0) self.draw_bannerwin() - self.logwin.idlok(True) # needed for scrollok below + self.logwin.idlok(True) # needed for scrollok below self.logwin.scrollok(True) # scroll window when too many lines added - self.logwin.move(self.logheight - 1, 0) self.draw_logwin() self.accounts = reversed(sorted(self.accframes.keys())) pos = self.height - 1 @@ -592,11 +590,9 @@ class Blinkenlights(UIBase, CursesUtil): color = curses.color_pair(0) #default colors else: color = curses.A_NORMAL - self.logwin.clear() + self.logwin.move(0, 0) + self.logwin.erase() self.logwin.bkgd(' ', color) - for line, color in self.text: - self.logwin.addstr("\n" + line, color) - self.logwin.noutrefresh() def getaccountframe(self, acc_name): """Return an AccountFrame() corresponding to acc_name From 98e82db505bf65db5fd33259f5a42d911647c716 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 17 Nov 2011 08:55:32 +0100 Subject: [PATCH 34/44] Release current master as 6.4.1 Signed-off-by: Sebastian Spaeth Conflicts: Changelog.draft.rst --- Changelog.draft.rst | 18 ------------------ Changelog.rst | 24 ++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 13b97e8..a9b7de4 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,10 +19,6 @@ New Features Changes ------- -* Indicate progress when copying many messages (slightly change log format) - -* Output how long an account sync took (min:sec). - * Reworked logging which was reported to e.g. not flush output to files often enough. User-visible changes: a) console output goes to stderr (for now). @@ -35,17 +31,3 @@ Changes Bug Fixes --------- - -* Syncing multiple accounts in single-threaded mode would fail as we try - to "register" a thread as belonging to two accounts which was - fatal. Make it non-fatal (it can be legitimate). - -* New folders on the remote would be skipped on the very sync run they - are created and only by synced in subsequent runs. Fixed. - -* Make NOOPs to keep a server connection open more resistant against dropped - connections. - -* a readonly parameter to select() was not always treated correctly, - which could result in some folders being opened read-only when we - really needed read-write. diff --git a/Changelog.rst b/Changelog.rst index b301c20..4c17c39 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,30 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.0 (2011-11-17) +=============================== + +Changes +------- + +* Indicate progress when copying many messages (slightly change log format) + +* Output how long an account sync took (min:sec). + +Bug Fixes +--------- + +* Syncing multiple accounts in single-threaded mode would fail as we try + to "register" a thread as belonging to two accounts which was + fatal. Make it non-fatal (it can be legitimate). + +* New folders on the remote would be skipped on the very sync run they + are created and only by synced in subsequent runs. Fixed. + +* a readonly parameter to select() was not always treated correctly, + which could result in some folders being opened read-only when we + really needed read-write. + OfflineIMAP v6.4.0 (2011-09-29) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index ea9aa0d..13a6f28 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.0" +__version__ = "6.4.1" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 51164c4974974d78ae3482b0517d4e2a2ce750a0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 17 Nov 2011 09:04:03 +0100 Subject: [PATCH 35/44] Fix version number typo in Changelog Signed-off-by: Sebastian Spaeth --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 4c17c39..7fe4d4b 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,7 +11,7 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. -OfflineIMAP v6.4.0 (2011-11-17) +OfflineIMAP v6.4.1 (2011-11-17) =============================== Changes From 20f2edfcecfbb14e27a1a9445751ccb0c451eb45 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 09:49:09 +0100 Subject: [PATCH 36/44] Sanity check to notify us when we call IMAPRepository.getsep() too early The folder delimiter is only initialized after a call to acquireconnection(), so we must never call this function too early. Include an assert() to make sure we get notified when we do. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/IMAP.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 8c29ee2..8635f7c 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -74,6 +74,13 @@ class IMAPRepository(BaseRepository): return num def getsep(self): + """Return the folder separator for the IMAP repository + + This requires that self.imapserver has been initialized with an + acquireconnection() or it will still be `None`""" + assert self.imapserver.delim != None, "'%s' " \ + "repository called getsep() before the folder separator was " \ + "queried from the server" % self return self.imapserver.delim def gethost(self): From bf4127c2d63dee6b87727a5eeeaed3f12c81ca02 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 10:12:54 +0100 Subject: [PATCH 37/44] Remove unused imapserver getdelim() imapserver.getdelim() was not used at all, so remove this function. The folder delimiter is available via the repository.getsep() call. Signed-off-by: Sebastian Spaeth --- offlineimap/imapserver.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index b8446d3..0106782 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -47,7 +47,11 @@ class IMAPServer: """Initializes all variables from an IMAPRepository() instance Various functions, such as acquireconnection() return an IMAP4 - object on which we can operate.""" + object on which we can operate. + + Public instance variables are: self.: + delim The server's folder delimiter. Only valid after acquireconnection() + """ GSS_STATE_STEP = 0 GSS_STATE_WRAP = 1 def __init__(self, repos): @@ -98,11 +102,6 @@ class IMAPServer: self.passworderror = None return self.password - def getdelim(self): - """Returns this server's folder delimiter. Can only be called - after one or more calls to acquireconnection.""" - return self.delim - def getroot(self): """Returns this server's folder root. Can only be called after one or more calls to acquireconnection.""" From c93f8710a38cec1135b03b034bbc69e477782bd2 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 10:27:12 +0100 Subject: [PATCH 38/44] Init folder list early enough We need the list of folders and the folder delimiter, but it was not always retrieved early enough. E.g. when doing IMAP<->IMAP sync and the local IMAP being readonly, we would bunk out with a mysterious error message become repository.getsel() would still return None. This commit fixes this error. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ offlineimap/accounts.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index a9b7de4..92c57a1 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -31,3 +31,8 @@ Changes Bug Fixes --------- + +* IMAP<->IMAP sync with a readonly local IMAP repository failed with a + rather mysterious "TypeError: expected a character buffer object" + error. Fix this my retrieving the list of folders early enough even + for readonly repositories. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index bf3f327..6c1d7f2 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -274,6 +274,13 @@ class SyncableAccount(Account): remoterepos = self.remoterepos localrepos = self.localrepos statusrepos = self.statusrepos + + # init repos with list of folders, so we have them (and the + # folder delimiter etc) + remoterepos.getfolders() + localrepos.getfolders() + statusrepos.getfolders() + # replicate the folderstructure between REMOTE to LOCAL if not localrepos.getconfboolean('readonly', False): self.ui.syncfolders(remoterepos, localrepos) From 9a654968fc6d73d3bc45b8191e5bae28ce51ac47 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 17:01:43 +0100 Subject: [PATCH 39/44] Don't append trailing slash to maildir foldernames When sep='/' in a Maildir, we were doing a os.path.join(dirname,'') on the top level maildir, which results in a "dirname/", so all our maildir folder names had slashes appended. Which is pretty much wrong, so this fixes it by only using os.path.join when we actually have something to append. Signed-off-by: Sebastian Spaeth --- offlineimap/repository/Maildir.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 6e93839..70a5ca3 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -160,7 +160,7 @@ class MaildirRepository(BaseRepository): # Not a directory -- not a folder. continue foldername = dirname - if extension != None: + if extension and dirname != '': foldername = os.path.join(extension, dirname) if (os.path.isdir(os.path.join(fullname, 'cur')) and os.path.isdir(os.path.join(fullname, 'new')) and @@ -185,7 +185,7 @@ class MaildirRepository(BaseRepository): self.debug("_GETFOLDERS_SCANDIR RETURNING %s" % \ repr([x.getname() for x in retval])) return retval - + def getfolders(self): if self.folders == None: self.folders = self._getfolders_scandir(self.root) From 416b1fe55145cc64de3432938a7ef87004768db8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 17:06:26 +0100 Subject: [PATCH 40/44] Add changelog entry for bug fixed in previous commit Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 92c57a1..3261465 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -36,3 +36,7 @@ Bug Fixes rather mysterious "TypeError: expected a character buffer object" error. Fix this my retrieving the list of folders early enough even for readonly repositories. + +* Fix regression from 6.4.0. When using local Maildirs with "/" as a + folder separator, all folder names would get a trailing slash + appended, which is plain wrong. From 50e78a8d41ea01c1f077438d9c2d6cf7367575b5 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 17:14:52 +0100 Subject: [PATCH 41/44] Release v6.4.2 This is a bugfix release over 6.4.1. Upgrading is recommended. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 9 --------- Changelog.rst | 12 ++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 3261465..a9b7de4 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -31,12 +31,3 @@ Changes Bug Fixes --------- - -* IMAP<->IMAP sync with a readonly local IMAP repository failed with a - rather mysterious "TypeError: expected a character buffer object" - error. Fix this my retrieving the list of folders early enough even - for readonly repositories. - -* Fix regression from 6.4.0. When using local Maildirs with "/" as a - folder separator, all folder names would get a trailing slash - appended, which is plain wrong. diff --git a/Changelog.rst b/Changelog.rst index 7fe4d4b..f369aaa 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,18 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.2 (2011-12-01) +=============================== + +* IMAP<->IMAP sync with a readonly local IMAP repository failed with a + rather mysterious "TypeError: expected a character buffer object" + error. Fix this my retrieving the list of folders early enough even + for readonly repositories. + +* Fix regression from 6.4.0. When using local Maildirs with "/" as a + folder separator, all folder names would get a trailing slash + appended, which is plain wrong. + OfflineIMAP v6.4.1 (2011-11-17) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 13a6f28..5e7b1c9 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.1" +__version__ = "6.4.2" __copyright__ = "Copyright 2002-2011 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" From 8ec6980c96313380609f2cc8af7c9daf0cc110e4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 1 Dec 2011 23:57:54 +0100 Subject: [PATCH 42/44] Don't fail on empty LocalStatus cache files As reported in https://github.com/spaetz/offlineimap/pull/2, we would fail when files are empty because file.read() would throw attribute errors. Fix this by removing the superfluous read() check and additionally log some warning message. Reported-by: Ralf Schmitt Signed-off-by: Sebastian Spaeth --- offlineimap/folder/LocalStatus.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index fbe2e24..fd4228c 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -65,9 +65,11 @@ class LocalStatusFolder(BaseFolder): file = open(self.filename, "rt") self.messagelist = {} line = file.readline().strip() - if not line and not line.read(): + if not line: # The status file is empty - should not have happened, # but somehow did. + errstr = "Cache file '%s' is empty. Closing..." % self.filename + self.ui.warn(errstr) file.close() return assert(line == magicline) From 0ccf06d5e631db2a5e139d63335134d983591487 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 Jan 2012 19:24:19 +0100 Subject: [PATCH 43/44] Implement clean CTRL-C termination Previously, we would simply bail out in an ugly way, potentially leaving temporary files around etc, or while writing status files. Hand SIGINT and SIGTERM as an event to the Account class, and make that bail out cleanly at predefined points. Stopping on ctrl-c can take a few seconds (it will e.g. finish to transfer the ongoing message), but it will shut down cleanly. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 ++++ offlineimap/accounts.py | 23 ++++++++++++++++++----- offlineimap/folder/Base.py | 7 +++++++ offlineimap/init.py | 16 ++++++++-------- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index a9b7de4..8b9b8bd 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -29,5 +29,9 @@ Changes * Bumped bundled imaplib2 to release 2.29 +* Make ctrl-c exit cleanly rather aborting brutally (which could leave + around temporary files, half-written cache files, etc). Exiting on + SIGTERM and CTRL-C can take a little longer, but will be clean. + Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 6c1d7f2..857fbfb 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -50,7 +50,9 @@ class Account(CustomConfig.ConfigHelperMixin): :class:`accounts.SyncableAccount` which contains all functions used for syncing an account.""" #signal gets set when we should stop looping - abort_signal = Event() + abort_soon_signal = Event() + #signal gets set on CTRL-C/SIGTERM + abort_NOW_signal = Event() def __init__(self, config, name): """ @@ -97,7 +99,8 @@ class Account(CustomConfig.ConfigHelperMixin): set_abort_event() to send the corresponding signal. Signum = 1 implies that we want all accounts to abort or skip the current or next sleep phase. Signum = 2 will end the autorefresh loop, - ie all accounts will return after they finished a sync. + ie all accounts will return after they finished a sync. signum=3 + means, abort NOW, e.g. on SIGINT or SIGTERM. This is a class method, it will send the signal to all accounts. """ @@ -107,7 +110,10 @@ class Account(CustomConfig.ConfigHelperMixin): config.set('Account ' + acctsection, "skipsleep", '1') elif signum == 2: # don't autorefresh anymore - cls.abort_signal.set() + cls.abort_soon_signal.set() + elif signum == 3: + # abort ASAP + cls.abort_NOW_signal.set() def get_abort_event(self): """Checks if an abort signal had been sent @@ -122,7 +128,8 @@ class Account(CustomConfig.ConfigHelperMixin): skipsleep = self.getconfboolean("skipsleep", 0) if skipsleep: self.config.set(self.getsection(), "skipsleep", '0') - return skipsleep or Account.abort_signal.is_set() + return skipsleep or Account.abort_soon_signal.is_set() or \ + Account.abort_NOW_signal.is_set() def sleeper(self): """Sleep if the account is set to autorefresh @@ -152,7 +159,8 @@ class Account(CustomConfig.ConfigHelperMixin): item.stopkeepalive() if sleepresult: - if Account.abort_signal.is_set(): + if Account.abort_soon_signal.is_set() or \ + Account.abort_NOW_signal.is_set(): return 2 self.quicknum = 0 return 1 @@ -288,6 +296,8 @@ class SyncableAccount(Account): # iterate through all folders on the remote repo and sync for remotefolder in remoterepos.getfolders(): + # check for CTRL-C or SIGTERM + if Account.abort_NOW_signal.is_set(): break if not remotefolder.sync_this: self.ui.debug('', "Not syncing filtered remote folder '%s'" "[%s]" % (remotefolder, remoterepos)) @@ -320,6 +330,9 @@ class SyncableAccount(Account): self.callhook(hook) def callhook(self, cmd): + # check for CTRL-C or SIGTERM and run postsynchook + if Account.abort_NOW_signal.is_set(): + return if not cmd: return try: diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 53bc71c..f61e89d 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -18,6 +18,7 @@ from offlineimap import threadutil from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError +import offlineimap.accounts import os.path import re from sys import exc_info @@ -332,6 +333,9 @@ class BaseFolder(object): self.getmessageuidlist()) num_to_copy = len(copylist) for num, uid in enumerate(copylist): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # exceptions are caught in copymessageto() if self.suggeststhreads(): @@ -447,6 +451,9 @@ class BaseFolder(object): ('syncing flags' , self.syncmessagesto_flags)] for (passdesc, action) in passes: + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break try: action(dstfolder, statusfolder) except (KeyboardInterrupt): diff --git a/offlineimap/init.py b/offlineimap/init.py index 36240ac..4287b5e 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -288,11 +288,6 @@ class OfflineImap: self.config is supposed to have been correctly initialized already.""" - def sigterm_handler(signum, frame): - # die immediately - self.ui.terminate(errormsg="terminating...") - signal.signal(signal.SIGTERM, sigterm_handler) - try: pidfd = open(self.config.getmetadatadir() + "/pid", "w") pidfd.write(str(os.getpid()) + "\n") @@ -328,11 +323,19 @@ class OfflineImap: accounts.Account.set_abort_event(self.config, 1) elif sig == signal.SIGUSR2: # tell each account to stop looping + getglobalui().warn("Terminating after this sync...") accounts.Account.set_abort_event(self.config, 2) + elif sig == signal.SIGTERM or sig == signal.SIGINT: + # tell each account to ABORT ASAP (ctrl-c) + getglobalui().warn("Terminating NOW (this may "\ + "take a few seconds)...") + accounts.Account.set_abort_event(self.config, 3) signal.signal(signal.SIGHUP,sig_handler) signal.signal(signal.SIGUSR1,sig_handler) signal.signal(signal.SIGUSR2,sig_handler) + signal.signal(signal.SIGTERM, sig_handler) + signal.signal(signal.SIGINT, sig_handler) #various initializations that need to be performed: offlineimap.mbnames.init(self.config, syncaccounts) @@ -361,9 +364,6 @@ class OfflineImap: t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) self.ui.terminate() - except KeyboardInterrupt: - self.ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') - return except (SystemExit): raise except Exception, e: From 9a7c700248c7f8deaddb798d438787f5af19cd05 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Wed, 4 Jan 2012 19:34:13 +0100 Subject: [PATCH 44/44] Release 6.4.3 Merge in the new logging mechanism, and provide an --info feature that will help with debugging. There is still some unstableness, so there will be another release soon, but this should be no worse than 6.4.2 at least... Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 17 ----------------- Changelog.rst | 27 +++++++++++++++++++++++++++ offlineimap/__init__.py | 4 ++-- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 8b9b8bd..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -13,25 +13,8 @@ others. New Features ------------ -* add a --info command line switch that outputs useful information about - the server and the configuration for all enabled accounts. - Changes ------- -* Reworked logging which was reported to e.g. not flush output to files - often enough. User-visible changes: - a) console output goes to stderr (for now). - b) file output has timestamps and looks identical in the basic and - ttyui UIs. - c) File output should be flushed after logging by default (do - report if not). - -* Bumped bundled imaplib2 to release 2.29 - -* Make ctrl-c exit cleanly rather aborting brutally (which could leave - around temporary files, half-written cache files, etc). Exiting on - SIGTERM and CTRL-C can take a little longer, but will be clean. - Bug Fixes --------- diff --git a/Changelog.rst b/Changelog.rst index f369aaa..b713c40 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,33 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.4.3 (2012-01-04) +=============================== + +New Features +------------ + +* add a --info command line switch that outputs useful information about + the server and the configuration for all enabled accounts. + +Changes +------- + +* Reworked logging which was reported to e.g. not flush output to files + often enough. User-visible changes: + a) console output goes to stderr (for now). + b) file output has timestamps and looks identical in the basic and + ttyui UIs. + c) File output should be flushed after logging by default (do + report if not). + +* Bumped bundled imaplib2 to release 2.29 + +* Make ctrl-c exit cleanly rather aborting brutally (which could leave + around temporary files, half-written cache files, etc). Exiting on + SIGTERM and CTRL-C can take a little longer, but will be clean. + + OfflineIMAP v6.4.2 (2011-12-01) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 5e7b1c9..ea857e6 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,8 +1,8 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.2" -__copyright__ = "Copyright 2002-2011 John Goerzen & contributors" +__version__ = "6.4.3" +__copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support"