diff --git a/Changelog.draft.rst b/Changelog.draft.rst index d9aa0bf..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,8 +18,3 @@ Changes Bug Fixes --------- - -* IMAP<->IMAP sync with a readonly local IMAP repository failed with a - rather mysterious "TypeError: expected a character buffer object" - error. Fix this my retrieving the list of folders early enough even - for readonly repositories. diff --git a/Changelog.rst b/Changelog.rst index f369aaa..049a362 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,34 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +======= +OfflineIMAP v6.4.3 (2012-01-04) +=============================== + +New Features +------------ + +* add a --info command line switch that outputs useful information about + the server and the configuration for all enabled accounts. + +Changes +------- + +* Reworked logging which was reported to e.g. not flush output to files + often enough. User-visible changes: + a) console output goes to stderr (for now). + b) file output has timestamps and looks identical in the basic and + ttyui UIs. + c) File output should be flushed after logging by default (do + report if not). + +* Bumped bundled imaplib2 to release 2.29 + +* Make ctrl-c exit cleanly rather aborting brutally (which could leave + around temporary files, half-written cache files, etc). Exiting on + SIGTERM and CTRL-C can take a little longer, but will be clean. + + OfflineIMAP v6.4.2 (2011-12-01) =============================== diff --git a/docs/FAQ.rst b/docs/FAQ.rst index e3a586b..29b53d6 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -219,6 +219,27 @@ as follows:: 2) while in sleep mode, you can also send a SIGUSR1. See the `Signals on UNIX`_ section in the MANUAL for details. +I get a "Mailbox already exists" error +-------------------------------------- +**Q:** When synchronizing, I receive errors such as:: + + Folder 'sent'[main-remote] could not be created. Server responded: + ('NO', ['Mailbox already exists.']) + +**A:** IMAP folders are usually case sensitive. But some IMAP servers seem + to treat "special" folders as case insensitive (e.g. the initial + INBOX. part, or folders such as "Sent" or "Trash"). If you happen to + have a folder "sent" on one side of things and a folder called "Sent" + on the other side, offlineimap will try to create those folders on + both sides. If you server happens to treat those folders as + case-insensitive you can then see this warning. + + You can solve this by excluding the "sent" folder by filtering it from + the repository settings:: + + folderfilter= lambda f: f not in ['sent'] + + Configuration Questions ======================= diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index bb13478..49dedd6 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -478,7 +478,7 @@ or:: folder separators be replaced with the destination repositories' folder separator. -So if ̀f` was "Sent", the first nametrans yields the translated name +So if 'f' was "Sent", the first nametrans yields the translated name "INBOX/Sent" to be used on the other side. As that repository uses the folder separator '.' rather than '/', the ultimate name to be used will be "INBOX.Sent". diff --git a/offlineimap.conf b/offlineimap.conf index 224e933..b6ff6e6 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -55,7 +55,7 @@ metadata = ~/.offlineimap accounts = Test -# Offlineimap can synchronize more the one account at a time. If you +# Offlineimap can synchronize more than one account at a time. If you # want to enable this feature, set the below value to something # greater than 1. To force it to synchronize only one account at a # time, set it to 1. @@ -200,7 +200,7 @@ remoterepository = RemoteExample # quick = 10 # You can specify a pre and post sync hook to execute a external command. -# in this case a call to imapfilter to filter mail before the sync process +# In this case a call to imapfilter to filter mail before the sync process # starts and a custom shell script after the sync completes. # The pre sync script has to complete before a sync to the account will # start. @@ -445,7 +445,7 @@ holdconnectionopen = no # mark them deleted on the server, but not actually delete them. # You must use some other IMAP client to delete them if you use this # setting; otherwise, the messgaes will just pile up there forever. -# Therefore, this setting is definately NOT recommended. +# Therefore, this setting is definitely NOT recommended. # # expunge = no diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index b7b3f53..ea857e6 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,22 +1,18 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.2" -__copyright__ = "Copyright 2002-2011 John Goerzen & contributors" +__version__ = "6.4.3" +__copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" __bigcopyright__ = """%(__productname__)s %(__version__)s -%(__copyright__)s. -%(__license__)s. -""" % locals() + %(__license__)s""" % locals() __homepage__ = "http://github.com/nicolas33/offlineimap" - banner = __bigcopyright__ - from offlineimap.error import OfflineImapError # put this last, so we don't run into circular dependencies using # e.g. offlineimap.__version__. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index d476b3e..857fbfb 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -50,7 +50,9 @@ class Account(CustomConfig.ConfigHelperMixin): :class:`accounts.SyncableAccount` which contains all functions used for syncing an account.""" #signal gets set when we should stop looping - abort_signal = Event() + abort_soon_signal = Event() + #signal gets set on CTRL-C/SIGTERM + abort_NOW_signal = Event() def __init__(self, config, name): """ @@ -82,6 +84,9 @@ class Account(CustomConfig.ConfigHelperMixin): def __str__(self): return self.name + def getaccountmeta(self): + return os.path.join(self.metadatadir, 'Account-' + self.name) + def getsection(self): return 'Account ' + self.getname() @@ -94,7 +99,8 @@ class Account(CustomConfig.ConfigHelperMixin): set_abort_event() to send the corresponding signal. Signum = 1 implies that we want all accounts to abort or skip the current or next sleep phase. Signum = 2 will end the autorefresh loop, - ie all accounts will return after they finished a sync. + ie all accounts will return after they finished a sync. signum=3 + means, abort NOW, e.g. on SIGINT or SIGTERM. This is a class method, it will send the signal to all accounts. """ @@ -104,7 +110,10 @@ class Account(CustomConfig.ConfigHelperMixin): config.set('Account ' + acctsection, "skipsleep", '1') elif signum == 2: # don't autorefresh anymore - cls.abort_signal.set() + cls.abort_soon_signal.set() + elif signum == 3: + # abort ASAP + cls.abort_NOW_signal.set() def get_abort_event(self): """Checks if an abort signal had been sent @@ -119,7 +128,8 @@ class Account(CustomConfig.ConfigHelperMixin): skipsleep = self.getconfboolean("skipsleep", 0) if skipsleep: self.config.set(self.getsection(), "skipsleep", '0') - return skipsleep or Account.abort_signal.is_set() + return skipsleep or Account.abort_soon_signal.is_set() or \ + Account.abort_NOW_signal.is_set() def sleeper(self): """Sleep if the account is set to autorefresh @@ -140,7 +150,7 @@ class Account(CustomConfig.ConfigHelperMixin): for item in kaobjs: item.startkeepalive() - + refreshperiod = int(self.refreshperiod * 60) sleepresult = self.ui.sleep(refreshperiod, self) @@ -149,13 +159,23 @@ class Account(CustomConfig.ConfigHelperMixin): item.stopkeepalive() if sleepresult: - if Account.abort_signal.is_set(): + if Account.abort_soon_signal.is_set() or \ + Account.abort_NOW_signal.is_set(): return 2 self.quicknum = 0 return 1 return 0 - - + + def serverdiagnostics(self): + """Output diagnostics for all involved repositories""" + remote_repo = Repository(self, 'remote') + local_repo = Repository(self, 'local') + #status_repo = Repository(self, 'status') + self.ui.serverdiagnostics(remote_repo, 'Remote') + self.ui.serverdiagnostics(local_repo, 'Local') + #self.ui.serverdiagnostics(statusrepos, 'Status') + + class SyncableAccount(Account): """A syncable email account connecting 2 repositories @@ -194,10 +214,10 @@ class SyncableAccount(Account): pass #Failed to delete for some reason. def syncrunner(self): - self.ui.registerthread(self.name) + self.ui.registerthread(self) accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): - os.mkdir(accountmetadata, 0700) + os.mkdir(accountmetadata, 0700) self.remoterepos = Repository(self, 'remote') self.localrepos = Repository(self, 'local') @@ -212,7 +232,7 @@ class SyncableAccount(Account): self.sync() except (KeyboardInterrupt, SystemExit): raise - except OfflineImapError, e: + except OfflineImapError, e: # Stop looping and bubble up Exception if needed. if e.severity >= OfflineImapError.ERROR.REPO: if looping: @@ -231,10 +251,7 @@ class SyncableAccount(Account): self.ui.acctdone(self) self.unlock() if looping and self.sleeper() >= 2: - looping = 0 - - def getaccountmeta(self): - return os.path.join(self.metadatadir, 'Account-' + self.name) + looping = 0 def sync(self): """Synchronize the account once, then return @@ -279,6 +296,8 @@ class SyncableAccount(Account): # iterate through all folders on the remote repo and sync for remotefolder in remoterepos.getfolders(): + # check for CTRL-C or SIGTERM + if Account.abort_NOW_signal.is_set(): break if not remotefolder.sync_this: self.ui.debug('', "Not syncing filtered remote folder '%s'" "[%s]" % (remotefolder, remoterepos)) @@ -287,9 +306,7 @@ class SyncableAccount(Account): instancename = 'FOLDER_' + self.remoterepos.getname(), target = syncfolder, name = "Folder %s [acc: %s]" % (remotefolder, self), - args = (self.name, remoterepos, remotefolder, localrepos, - statusrepos, quick)) - thread.setDaemon(1) + args = (self, remotefolder, quick)) thread.start() folderthreads.append(thread) # wait for all threads to finish @@ -313,6 +330,9 @@ class SyncableAccount(Account): self.callhook(hook) def callhook(self, cmd): + # check for CTRL-C or SIGTERM and run postsynchook + if Account.abort_NOW_signal.is_set(): + return if not cmd: return try: @@ -328,15 +348,17 @@ class SyncableAccount(Account): except Exception, e: self.ui.error(e, exc_info()[2], msg = "Calling hook") - -def syncfolder(accountname, remoterepos, remotefolder, localrepos, - statusrepos, quick): +def syncfolder(account, remotefolder, quick): """This function is called as target for the InstanceLimitedThread invokation in SyncableAccount. Filtered folders on the remote side will not invoke this function.""" + remoterepos = account.remoterepos + localrepos = account.localrepos + statusrepos = account.statusrepos + ui = getglobalui() - ui.registerthread(accountname) + ui.registerthread(account) try: # Load local folder. localfolder = localrepos.\ @@ -351,7 +373,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, % localfolder) return # Write the mailboxes - mbnames.add(accountname, localfolder.getvisiblename()) + mbnames.add(account.name, localfolder.getvisiblename()) # Load status folder. statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ @@ -407,7 +429,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, else: ui.debug('imap', "Not syncing to read-only repository '%s'" \ % localrepos.getname()) - + # Synchronize local changes if not remoterepos.getconfboolean('readonly', False): ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) @@ -430,11 +452,11 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, "[acc: '%s']" % ( remotefolder.getvisiblename().\ replace(remoterepos.getsep(), localrepos.getsep()), - accountname)) + account)) # we reconstruct foldername above rather than using # localfolder, as the localfolder var is not # available if assignment fails. except Exception, e: ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s" % \ - (accountname,remotefolder.getvisiblename(), + (account, remotefolder.getvisiblename(), traceback.format_exc())) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 91ee08a..f61e89d 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -18,6 +18,7 @@ from offlineimap import threadutil from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError +import offlineimap.accounts import os.path import re from sys import exc_info @@ -254,7 +255,7 @@ class BaseFolder(object): # self.getmessage(). So, don't call self.getmessage unless # really needed. if register: # output that we start a new thread - self.ui.registerthread(self.accountname) + self.ui.registerthread(self.repository.account) try: message = None @@ -332,6 +333,9 @@ class BaseFolder(object): self.getmessageuidlist()) num_to_copy = len(copylist) for num, uid in enumerate(copylist): + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # exceptions are caught in copymessageto() if self.suggeststhreads(): @@ -341,7 +345,6 @@ class BaseFolder(object): target = self.copymessageto, name = "Copy message from %s:%s" % (self.repository, self), args = (uid, dstfolder, statusfolder)) - thread.setDaemon(1) thread.start() threads.append(thread) else: @@ -448,6 +451,9 @@ class BaseFolder(object): ('syncing flags' , self.syncmessagesto_flags)] for (passdesc, action) in passes: + # bail out on CTRL-C or SIGTERM + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break try: action(dstfolder, statusfolder) except (KeyboardInterrupt): diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 2fdea30..4471387 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -1,6 +1,5 @@ # IMAP folder support -# Copyright (C) 2002-2007 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index fbe2e24..fd4228c 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -65,9 +65,11 @@ class LocalStatusFolder(BaseFolder): file = open(self.filename, "rt") self.messagelist = {} line = file.readline().strip() - if not line and not line.read(): + if not line: # The status file is empty - should not have happened, # but somehow did. + errstr = "Cache file '%s' is empty. Closing..." % self.filename + self.ui.warn(errstr) file.close() return assert(line == magicline) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index fa1924d..5da8e00 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -15,8 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -from threading import * +from __future__ import with_statement # needed for python 2.5 +from threading import Lock from IMAP import IMAPFolder import os.path diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index cf3c480..ffa2676 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.28" +__version__ = "2.29" __release__ = "2" -__revision__ = "28" +__revision__ = "29" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -1234,11 +1234,13 @@ class IMAP4(object): def _choose_nonull_or_dflt(self, dflt, *args): dflttyp = type(dflt) + if isinstance(dflttyp, basestring): + dflttyp = basestring # Allow any string type for arg in args: if arg is not None: - if type(arg) is dflttyp: + if isinstance(arg, dflttyp): return arg - if __debug__: self._log(1, 'bad arg type is %s, expecting %s' % (type(arg), dflttyp)) + if __debug__: self._log(0, 'bad arg is %s, expecting %s' % (type(arg), dflttyp)) return dflt @@ -2323,6 +2325,7 @@ if __name__ == '__main__': data = open(os.path.exists("test.data") and "test.data" or __file__).read(1000) test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)s%(data)s' \ % {'user':USER, 'lf':'\n', 'data':data} + test_seq1 = [ ('list', ('""', '%')), ('create', ('/tmp/imaplib2_test.0',)), @@ -2342,7 +2345,7 @@ if __name__ == '__main__': test_seq2 = ( ('select', ()), - ('response',('UIDVALIDITY',)), + ('response', ('UIDVALIDITY',)), ('response', ('EXISTS',)), ('append', (None, None, None, test_mesg)), ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')), @@ -2351,6 +2354,7 @@ if __name__ == '__main__': ('recent', ()), ) + AsyncError = None def responder((response, cb_arg, error)): @@ -2436,10 +2440,21 @@ if __name__ == '__main__': if 'IDLE' in M.capabilities: run('idle', (2,), cb=False) - run('idle', (99,), cb=True) # Asynchronous, to test interruption of 'idle' by 'noop' + run('idle', (99,)) # Asynchronous, to test interruption of 'idle' by 'noop' time.sleep(1) run('noop', (), cb=False) + run('append', (None, None, None, test_mesg), cb=False) + num = run('search', (None, 'ALL'), cb=False)[0].split()[0] + dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) + M._mesg('fetch %s => %s' % (num, `dat`)) + run('idle', (2,)) + run('store', (num, '-FLAGS', '(\Seen)'), cb=False), + dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) + M._mesg('fetch %s => %s' % (num, `dat`)) + run('uid', ('STORE', num, 'FLAGS', '(\Deleted)')) + run('expunge', ()) + run('logout', (), cb=False) if debug: diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index b7f062d..0106782 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +from __future__ import with_statement # needed for python 2.5 from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui from threading import Lock, BoundedSemaphore, Thread, Event, currentThread @@ -46,7 +47,11 @@ class IMAPServer: """Initializes all variables from an IMAPRepository() instance Various functions, such as acquireconnection() return an IMAP4 - object on which we can operate.""" + object on which we can operate. + + Public instance variables are: self.: + delim The server's folder delimiter. Only valid after acquireconnection() + """ GSS_STATE_STEP = 0 GSS_STATE_WRAP = 1 def __init__(self, repos): @@ -97,11 +102,6 @@ class IMAPServer: self.passworderror = None return self.password - def getdelim(self): - """Returns this server's folder delimiter. Can only be called - after one or more calls to acquireconnection.""" - return self.delim - def getroot(self): """Returns this server's folder root. Can only be called after one or more calls to acquireconnection.""" @@ -355,7 +355,7 @@ class IMAPServer: else: # re-raise all other errors raise - + def connectionwait(self): """Waits until there is a connection available. Note that between the time that a connection becomes available and the time it is @@ -370,18 +370,21 @@ class IMAPServer: def close(self): # Make sure I own all the semaphores. Let the threads finish # their stuff. This is a blocking method. - self.connectionlock.acquire() - threadutil.semaphorereset(self.semaphore, self.maxconnections) - for imapobj in self.assignedconnections + self.availableconnections: - imapobj.logout() - self.assignedconnections = [] - self.availableconnections = [] - self.lastowner = {} - # reset kerberos state - self.gss_step = self.GSS_STATE_STEP - self.gss_vc = None - self.gssapi = False - self.connectionlock.release() + with self.connectionlock: + # first, wait till all connections had been released. + # TODO: won't work IMHO, as releaseconnection() also + # requires the connectionlock, leading to a potential + # deadlock! Audit & check! + threadutil.semaphorereset(self.semaphore, self.maxconnections) + for imapobj in self.assignedconnections + self.availableconnections: + imapobj.logout() + self.assignedconnections = [] + self.availableconnections = [] + self.lastowner = {} + # reset kerberos state + self.gss_step = self.GSS_STATE_STEP + self.gss_vc = None + self.gssapi = False def keepalive(self, timeout, event): """Sends a NOOP to each connection recorded. It will wait a maximum @@ -390,47 +393,40 @@ class IMAPServer: to be invoked in a separate thread, which should be join()'d after the event is set.""" self.ui.debug('imap', 'keepalive thread started') - while 1: - self.ui.debug('imap', 'keepalive: top of loop') - if event.isSet(): - self.ui.debug('imap', 'keepalive: event is set; exiting') - return - self.ui.debug('imap', 'keepalive: acquiring connectionlock') + while not event.isSet(): self.connectionlock.acquire() numconnections = len(self.assignedconnections) + \ len(self.availableconnections) self.connectionlock.release() - self.ui.debug('imap', 'keepalive: connectionlock released') + threads = [] - for i in range(numconnections): self.ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections)) if len(self.idlefolders) > i: + # IDLE thread idler = IdleThread(self, self.idlefolders[i]) else: + # NOOP thread idler = IdleThread(self) idler.start() threads.append(idler) - self.ui.debug('imap', 'keepalive: thread started') self.ui.debug('imap', 'keepalive: waiting for timeout') event.wait(timeout) self.ui.debug('imap', 'keepalive: after wait') - self.ui.debug('imap', 'keepalive: joining threads') - for idler in threads: # Make sure all the commands have completed. idler.stop() idler.join() - - self.ui.debug('imap', 'keepalive: bottom of loop') - + self.ui.debug('imap', 'keepalive: all threads joined') + self.ui.debug('imap', 'keepalive: event is set; exiting') + return def verifycert(self, cert, hostname): '''Verify that cert (in socket.getpeercert() format) matches hostname. CRLs are not handled. - + Returns error message if any problems are found and None on success. ''' errstr = "CA Cert verifying failed: " @@ -492,10 +488,24 @@ class IdleThread(object): self.thread.join() def noop(self): + #TODO: AFAIK this is not optimal, we will send a NOOP on one + #random connection (ie not enough to keep all connections + #open). In case we do the noop multiple times, we can well use + #the same connection every time, as we get a random one. This + #function should IMHO send a noop on ALL available connections + #to the server. imapobj = self.parent.acquireconnection() - imapobj.noop() - self.stop_sig.wait() - self.parent.releaseconnection(imapobj) + try: + imapobj.noop() + except imapobj.abort: + self.ui.warn('Attempting NOOP on dropped connection %s' % \ + imapobj.identifier) + self.parent.releaseconnection(imapobj, True) + imapobj = None + finally: + if imapobj: + self.parent.releaseconnection(imapobj) + self.stop_sig.wait() # wait until we are supposed to quit def dosync(self): remoterepos = self.parent.repos @@ -504,9 +514,9 @@ class IdleThread(object): remoterepos = account.remoterepos statusrepos = account.statusrepos remotefolder = remoterepos.getfolder(self.folder) - offlineimap.accounts.syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False) + offlineimap.accounts.syncfolder(account, remotefolder, quick=False) ui = getglobalui() - ui.unregisterthread(currentThread()) + ui.unregisterthread(currentThread()) #syncfolder registered the thread def idle(self): """Invoke IDLE mode until timeout or self.stop() is invoked""" diff --git a/offlineimap/init.py b/offlineimap/init.py index 27aeebd..4287b5e 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -44,9 +44,16 @@ class OfflineImap: """ def run(self): """Parse the commandline and invoke everything""" + # next line also sets self.config and self.ui + options, args = self.parse_cmd_options() + if options.diagnostics: + self.serverdiagnostics(options) + else: + self.sync(options) + def parse_cmd_options(self): parser = OptionParser(version=offlineimap.__version__, - description="%s.\n\n%s" % + description="%s.\n\n%s" % (offlineimap.__copyright__, offlineimap.__license__)) parser.add_option("-1", @@ -103,7 +110,7 @@ class OfflineImap: "Only sync the specified folders. The folder names " "are the *untranslated* foldernames. This " "command-line option overrides any 'folderfilter' " - "and 'folderincludes' options in the configuration " + "and 'folderincludes' options in the configuration " "file.") parser.add_option("-k", dest="configoverride", @@ -139,13 +146,21 @@ class OfflineImap: "not usable. Possible interface choices are: %s " % ", ".join(UI_LIST.keys())) + parser.add_option("--info", + action="store_true", dest="diagnostics", + default=False, + help="Output information on the configured email repositories" + ". Useful for debugging and bug reporting. Use in conjunction wit" + "h the -a option to limit the output to a single account") + (options, args) = parser.parse_args() #read in configuration file configfilename = os.path.expanduser(options.configfile) - + config = CustomConfigParser() if not os.path.exists(configfilename): + # TODO, initialize and make use of chosen ui for logging logging.error(" *** Config file '%s' does not exist; aborting!" % configfilename) sys.exit(1) @@ -154,14 +169,17 @@ class OfflineImap: #profile mode chosen? if options.profiledir: if not options.singlethreading: + # TODO, make use of chosen ui for logging logging.warn("Profile mode: Forcing to singlethreaded.") options.singlethreading = True if os.path.exists(options.profiledir): - logging.warn("Profile mode: Directory '%s' already exists!" % + # TODO, make use of chosen ui for logging + logging.warn("Profile mode: Directory '%s' already exists!" % options.profiledir) else: os.mkdir(options.profiledir) threadutil.ExitNotifyThread.set_profiledir(options.profiledir) + # TODO, make use of chosen ui for logging logging.warn("Profile mode: Potentially large data will be " "created in '%s'" % options.profiledir) @@ -183,37 +201,39 @@ class OfflineImap: if '.' in ui_type: #transform Curses.Blinkenlights -> Blinkenlights ui_type = ui_type.split('.')[-1] + # TODO, make use of chosen ui for logging logging.warning('Using old interface name, consider using one ' 'of %s' % ', '.join(UI_LIST.keys())) try: # create the ui class - ui = UI_LIST[ui_type.lower()](config) + self.ui = UI_LIST[ui_type.lower()](config) except KeyError: logging.error("UI '%s' does not exist, choose one of: %s" % \ (ui_type,', '.join(UI_LIST.keys()))) sys.exit(1) - setglobalui(ui) + setglobalui(self.ui) #set up additional log files if options.logfile: - ui.setlogfd(open(options.logfile, 'wt')) - + self.ui.setlogfile(options.logfile) + #welcome blurb - ui.init_banner() + self.ui.init_banner() if options.debugtype: + self.ui.logger.setLevel(logging.DEBUG) if options.debugtype.lower() == 'all': options.debugtype = 'imap,maildir,thread' #force single threading? if not ('thread' in options.debugtype.split(',') \ and not options.singlethreading): - ui._msg("Debug mode: Forcing to singlethreaded.") + self.ui._msg("Debug mode: Forcing to singlethreaded.") options.singlethreading = True debugtypes = options.debugtype.split(',') + [''] for type in debugtypes: type = type.strip() - ui.add_debug(type) + self.ui.add_debug(type) if type.lower() == 'imap': imaplib.Debug = 5 @@ -241,81 +261,84 @@ class OfflineImap: config.set(section, "folderfilter", folderfilter) config.set(section, "folderincludes", folderincludes) - self.config = config + if options.logfile: + sys.stderr = self.ui.logfile - def sigterm_handler(signum, frame): - # die immediately - ui = getglobalui() - ui.terminate(errormsg="terminating...") + socktimeout = config.getdefaultint("general", "socktimeout", 0) + if socktimeout > 0: + socket.setdefaulttimeout(socktimeout) - signal.signal(signal.SIGTERM,sigterm_handler) - + threadutil.initInstanceLimit('ACCOUNTLIMIT', + config.getdefaultint('general', 'maxsyncaccounts', 1)) + + for reposname in config.getsectionlist('Repository'): + for instancename in ["FOLDER_" + reposname, + "MSGCOPY_" + reposname]: + if options.singlethreading: + threadutil.initInstanceLimit(instancename, 1) + else: + threadutil.initInstanceLimit(instancename, + config.getdefaultint('Repository ' + reposname, + 'maxconnections', 2)) + self.config = config + return (options, args) + + def sync(self, options): + """Invoke the correct single/multithread syncing + + self.config is supposed to have been correctly initialized + already.""" try: - pidfd = open(config.getmetadatadir() + "/pid", "w") + pidfd = open(self.config.getmetadatadir() + "/pid", "w") pidfd.write(str(os.getpid()) + "\n") pidfd.close() except: pass - + try: - if options.logfile: - sys.stderr = ui.logfile - - socktimeout = config.getdefaultint("general", "socktimeout", 0) - if socktimeout > 0: - socket.setdefaulttimeout(socktimeout) - - activeaccounts = config.get("general", "accounts") + activeaccounts = self.config.get("general", "accounts") if options.accounts: activeaccounts = options.accounts activeaccounts = activeaccounts.replace(" ", "") activeaccounts = activeaccounts.split(",") - allaccounts = accounts.AccountHashGenerator(config) - + allaccounts = accounts.AccountHashGenerator(self.config) + syncaccounts = [] for account in activeaccounts: if account not in allaccounts: if len(allaccounts) == 0: - errormsg = 'The account "%s" does not exist because no accounts are defined!'%account + errormsg = "The account '%s' does not exist because no"\ + " accounts are defined!" % account else: - errormsg = 'The account "%s" does not exist. Valid accounts are:'%account - for name in allaccounts.keys(): - errormsg += '\n%s'%name - ui.terminate(1, errortitle = 'Unknown Account "%s"'%account, errormsg = errormsg) + errormsg = "The account '%s' does not exist. Valid ac"\ + "counts are: " % account + errormsg += ", ".join(allaccounts.keys()) + self.ui.terminate(1, errormsg = errormsg) if account not in syncaccounts: syncaccounts.append(account) - - server = None - remoterepos = None - localrepos = None - - threadutil.initInstanceLimit('ACCOUNTLIMIT', - config.getdefaultint('general', - 'maxsyncaccounts', 1)) - - for reposname in config.getsectionlist('Repository'): - for instancename in ["FOLDER_" + reposname, - "MSGCOPY_" + reposname]: - if options.singlethreading: - threadutil.initInstanceLimit(instancename, 1) - else: - threadutil.initInstanceLimit(instancename, - config.getdefaultint('Repository ' + reposname, - 'maxconnections', 2)) + def sig_handler(sig, frame): if sig == signal.SIGUSR1 or sig == signal.SIGHUP: # tell each account to stop sleeping accounts.Account.set_abort_event(self.config, 1) elif sig == signal.SIGUSR2: # tell each account to stop looping + getglobalui().warn("Terminating after this sync...") accounts.Account.set_abort_event(self.config, 2) - + elif sig == signal.SIGTERM or sig == signal.SIGINT: + # tell each account to ABORT ASAP (ctrl-c) + getglobalui().warn("Terminating NOW (this may "\ + "take a few seconds)...") + accounts.Account.set_abort_event(self.config, 3) + signal.signal(signal.SIGHUP,sig_handler) signal.signal(signal.SIGUSR1,sig_handler) signal.signal(signal.SIGUSR2,sig_handler) - + signal.signal(signal.SIGTERM, sig_handler) + signal.signal(signal.SIGINT, sig_handler) + #various initializations that need to be performed: - offlineimap.mbnames.init(config, syncaccounts) + offlineimap.mbnames.init(self.config, syncaccounts) #TODO: keep legacy lock for a few versions, then remove. self._legacy_lock = open(self.config.getmetadatadir() + "/lock", @@ -331,34 +354,39 @@ class OfflineImap: if options.singlethreading: #singlethreaded - self.sync_singlethreaded(syncaccounts, config) + self.sync_singlethreaded(syncaccounts) else: # multithreaded t = threadutil.ExitNotifyThread(target=syncmaster.syncitall, name='Sync Runner', kwargs = {'accounts': syncaccounts, - 'config': config}) - t.setDaemon(1) + 'config': self.config}) t.start() threadutil.exitnotifymonitorloop(threadutil.threadexited) - - ui.terminate() - except KeyboardInterrupt: - ui.terminate(1, errormsg = 'CTRL-C pressed, aborting...') - return + self.ui.terminate() except (SystemExit): raise except Exception, e: - ui.error(e) - ui.terminate() + self.ui.error(e) + self.ui.terminate() - def sync_singlethreaded(self, accs, config): + def sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread :param accs: A list of accounts that should be synced - :param config: The CustomConfig object """ for accountname in accs: - account = offlineimap.accounts.SyncableAccount(config, accountname) + account = offlineimap.accounts.SyncableAccount(self.config, + accountname) threading.currentThread().name = "Account sync %s" % accountname account.syncrunner() + + def serverdiagnostics(self, options): + activeaccounts = self.config.get("general", "accounts") + if options.accounts: + activeaccounts = options.accounts + activeaccounts = activeaccounts.split(",") + allaccounts = accounts.AccountListGenerator(self.config) + for account in allaccounts: + if account.name not in activeaccounts: continue + account.serverdiagnostics() diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 972aefb..8af1f03 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -207,9 +207,8 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): src_repo.makefolder(newsrc_name) src_haschanged = True # Need to refresh list except OfflineImapError, e: - self.ui.error(e, exc_info()[2], - "Creating folder %s on repository %s" %\ - (src_name, dst_repo)) + self.ui.error(e, exc_info()[2], "Creating folder %s on " + "repository %s" % (newsrc_name, src_repo)) raise status_repo.makefolder(newsrc_name.replace( src_repo.getsep(), status_repo.getsep())) @@ -230,4 +229,4 @@ class BaseRepository(object, CustomConfig.ConfigHelperMixin): """Stop keep alive, but don't bother waiting for the threads to terminate.""" pass - + diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index f93510f..8635f7c 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -1,6 +1,5 @@ # IMAP repository support -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -75,6 +74,13 @@ class IMAPRepository(BaseRepository): return num def getsep(self): + """Return the folder separator for the IMAP repository + + This requires that self.imapserver has been initialized with an + acquireconnection() or it will still be `None`""" + assert self.imapserver.delim != None, "'%s' " \ + "repository called getsep() before the folder separator was " \ + "queried from the server" % self return self.imapserver.delim def gethost(self): @@ -171,7 +177,7 @@ class IMAPRepository(BaseRepository): return self.getconf('preauthtunnel', None) def getreference(self): - return self.getconf('reference', '""') + return self.getconf('reference', '') def getidlefolders(self): localeval = self.localeval @@ -316,14 +322,9 @@ class IMAPRepository(BaseRepository): when you are done creating folders yourself. :param foldername: Full path of the folder to be created.""" - #TODO: IMHO this existing commented out code is correct and - #should be enabled, but this would change the behavior for - #existing configurations who have a 'reference' set on a Mapped - #IMAP server....: - #if self.getreference() != '""': - # newname = self.getreference() + self.getsep() + foldername - #else: - # newname = foldername + if self.getreference(): + foldername = self.getreference() + self.getsep() + foldername + imapobj = self.imapserver.acquireconnection() try: self.ui._msg("Creating new IMAP folder '%s' on server %s" %\ diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 0dd79a2..70a5ca3 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -149,14 +149,12 @@ class MaildirRepository(BaseRepository): # Iterate over directories in top & top itself. for dirname in os.listdir(toppath) + ['']: - self.debug(" *** top of loop") self.debug(" dirname = %s" % dirname) if dirname in ['cur', 'new', 'tmp']: self.debug(" skipping this dir (Maildir special)") # Bypass special files. continue fullname = os.path.join(toppath, dirname) - self.debug(" fullname = %s" % fullname) if not os.path.isdir(fullname): self.debug(" skipping this entry (not a directory)") # Not a directory -- not a folder. diff --git a/offlineimap/syncmaster.py b/offlineimap/syncmaster.py index 3aea6d2..5fd0dee 100644 --- a/offlineimap/syncmaster.py +++ b/offlineimap/syncmaster.py @@ -25,12 +25,13 @@ def syncaccount(threads, config, accountname): thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT', target = account.syncrunner, name = "Account sync %s" % accountname) - thread.setDaemon(1) + thread.setDaemon(True) thread.start() threads.add(thread) def syncitall(accounts, config): - currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE') + # Special exit message for SyncRunner thread, so main thread can exit + currentThread().exit_message = 'SYNCRUNNER_EXITED_NORMALLY' threads = threadlist() for accountname in accounts: syncaccount(threads, config, accountname) diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py index 463752f..c102446 100644 --- a/offlineimap/threadutil.py +++ b/offlineimap/threadutil.py @@ -28,14 +28,14 @@ from offlineimap.ui import getglobalui ###################################################################### def semaphorereset(semaphore, originalstate): - """Wait until the semaphore gets back to its original state -- all acquired - resources released.""" + """Block until `semaphore` gets back to its original state, ie all acquired + resources have been released.""" for i in range(originalstate): semaphore.acquire() # Now release these. for i in range(originalstate): semaphore.release() - + class threadlist: def __init__(self): self.lock = Lock() @@ -70,7 +70,7 @@ class threadlist: if not thread: return thread.join() - + ###################################################################### # Exit-notify threads @@ -81,54 +81,72 @@ exitthreads = Queue(100) def exitnotifymonitorloop(callback): """An infinite "monitoring" loop watching for finished ExitNotifyThread's. - :param callback: the function to call when a thread terminated. That - function is called with a single argument -- the - ExitNotifyThread that has terminated. The monitor will + This one is supposed to run in the main thread. + :param callback: the function to call when a thread terminated. That + function is called with a single argument -- the + ExitNotifyThread that has terminated. The monitor will not continue to monitor for other threads until 'callback' returns, so if it intends to perform long calculations, it should start a new thread itself -- but - NOT an ExitNotifyThread, or else an infinite loop + NOT an ExitNotifyThread, or else an infinite loop may result. - Furthermore, the monitor will hold the lock all the + Furthermore, the monitor will hold the lock all the while the other thread is waiting. :type callback: a callable function """ global exitthreads - while 1: + do_loop = True + while do_loop: # Loop forever and call 'callback' for each thread that exited try: # we need a timeout in the get() call, so that ctrl-c can throw # a SIGINT (http://bugs.python.org/issue1360). A timeout with empty # Queue will raise `Empty`. thrd = exitthreads.get(True, 60) - callback(thrd) + # request to abort when callback returns true + do_loop = (callback(thrd) != True) except Empty: pass def threadexited(thread): - """Called when a thread exits.""" + """Called when a thread exits. + + Main thread is aborted when this returns True.""" ui = getglobalui() - if thread.getExitCause() == 'EXCEPTION': - if isinstance(thread.getExitException(), SystemExit): + if thread.exit_exception: + if isinstance(thread.exit_exception, SystemExit): # Bring a SystemExit into the main thread. # Do not send it back to UI layer right now. # Maybe later send it to ui.terminate? raise SystemExit ui.threadException(thread) # Expected to terminate sys.exit(100) # Just in case... - elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE': - ui.terminate() - # Just in case... - sys.exit(100) + elif thread.exit_message == 'SYNCRUNNER_EXITED_NORMALLY': + return True else: ui.threadExited(thread) + return False class ExitNotifyThread(Thread): - """This class is designed to alert a "monitor" to the fact that a thread has - exited and to provide for the ability for it to find out why.""" + """This class is designed to alert a "monitor" to the fact that a + thread has exited and to provide for the ability for it to find out + why. All instances are made daemon threads (setDaemon(True), so we + bail out when the mainloop dies. + + The thread can set instance variables self.exit_message for a human + readable reason of the thread exit.""" profiledir = None """class variable that is set to the profile directory if required""" + def __init__(self, *args, **kwargs): + super(ExitNotifyThread, self).__init__(*args, **kwargs) + # These are all child threads that are supposed to go away when + # the main thread is killed. + self.setDaemon(True) + self.exit_message = None + self._exit_exc = None + self._exit_stacktrace = None + def run(self): global exitthreads self.threadid = get_ident() @@ -147,49 +165,31 @@ class ExitNotifyThread(Thread): pass prof.dump_stats(os.path.join(ExitNotifyThread.profiledir, "%s_%s.prof" % (self.threadid, self.getName()))) - except: - self.setExitCause('EXCEPTION') - if sys: - self.setExitException(sys.exc_info()[1]) - tb = traceback.format_exc() - self.setExitStackTrace(tb) - else: - self.setExitCause('NORMAL') - if not hasattr(self, 'exitmessage'): - self.setExitMessage(None) + except Exception, e: + # Thread exited with Exception, store it + tb = traceback.format_exc() + self.set_exit_exception(e, tb) if exitthreads: exitthreads.put(self, True) - def setExitCause(self, cause): - self.exitcause = cause - def getExitCause(self): + def set_exit_exception(self, exc, st=None): + """Sets Exception and stacktrace of a thread, so that other + threads can query its exit status""" + self._exit_exc = exc + self._exit_stacktrace = st + + @property + def exit_exception(self): """Returns the cause of the exit, one of: - 'EXCEPTION' -- the thread aborted because of an exception - 'NORMAL' -- normal termination.""" - return self.exitcause - def setExitException(self, exc): - self.exitexception = exc - def getExitException(self): - """If getExitCause() is 'EXCEPTION', holds the value from - sys.exc_info()[1] for this exception.""" - return self.exitexception - def setExitStackTrace(self, st): - self.exitstacktrace = st - def getExitStackTrace(self): - """If getExitCause() is 'EXCEPTION', returns a string representing - the stack trace for this exception.""" - return self.exitstacktrace - def setExitMessage(self, msg): - """Sets the exit message to be fetched by a subsequent call to - getExitMessage. This message may be any object or type except - None.""" - self.exitmessage = msg - def getExitMessage(self): - """For any exit cause, returns the message previously set by - a call to setExitMessage(), or None if there was no such message - set.""" - return self.exitmessage + Exception() -- the thread aborted with this exception + None -- normal termination.""" + return self._exit_exc + + @property + def exit_stacktrace(self): + """Returns a string representing the stack trace if set""" + return self._exit_stacktrace @classmethod def set_profiledir(cls, directory): @@ -220,7 +220,7 @@ class InstanceLimitedThread(ExitNotifyThread): def start(self): instancelimitedsems[self.instancename].acquire() ExitNotifyThread.start(self) - + def run(self): try: ExitNotifyThread.run(self) diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py deleted file mode 100644 index 330b9a8..0000000 --- a/offlineimap/ui/Blinkenlights.py +++ /dev/null @@ -1,148 +0,0 @@ -# Blinkenlights base classes -# Copyright (C) 2003 John Goerzen -# -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -from threading import RLock, currentThread -from offlineimap.ui.UIBase import UIBase -from thread import get_ident # python < 2.6 support - -class BlinkenBase: - """This is a mix-in class that should be mixed in with either UIBase - or another appropriate base class. The Tk interface, for instance, - will probably mix it in with VerboseUI.""" - - def acct(s, accountname): - s.gettf().setcolor('purple') - s.__class__.__bases__[-1].acct(s, accountname) - - def connecting(s, hostname, port): - s.gettf().setcolor('gray') - s.__class__.__bases__[-1].connecting(s, hostname, port) - - def syncfolders(s, srcrepos, destrepos): - s.gettf().setcolor('blue') - s.__class__.__bases__[-1].syncfolders(s, srcrepos, destrepos) - - def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): - s.gettf().setcolor('cyan') - s.__class__.__bases__[-1].syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder) - - def skippingfolder(s, folder): - s.gettf().setcolor('cyan') - s.__class__.__bases__[-1].skippingfolder(s, folder) - - def loadmessagelist(s, repos, folder): - s.gettf().setcolor('green') - s._msg("Scanning folder [%s/%s]" % (s.getnicename(repos), - folder.getvisiblename())) - - def syncingmessages(s, sr, sf, dr, df): - s.gettf().setcolor('blue') - s.__class__.__bases__[-1].syncingmessages(s, sr, sf, dr, df) - - def copyingmessage(s, uid, num, num_to_copy, src, destfolder): - s.gettf().setcolor('orange') - s.__class__.__bases__[-1].copyingmessage(s, uid, num, num_to_copy, src, - destfolder) - - def deletingmessages(s, uidlist, destlist): - s.gettf().setcolor('red') - s.__class__.__bases__[-1].deletingmessages(s, uidlist, destlist) - - def deletingmessage(s, uid, destlist): - s.gettf().setcolor('red') - s.__class__.__bases__[-1].deletingmessage(s, uid, destlist) - - def addingflags(s, uidlist, flags, dest): - s.gettf().setcolor('yellow') - s.__class__.__bases__[-1].addingflags(s, uidlist, flags, dest) - - def deletingflags(s, uidlist, flags, dest): - s.gettf().setcolor('pink') - s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, dest) - - def warn(s, msg, minor = 0): - if minor: - s.gettf().setcolor('pink') - else: - s.gettf().setcolor('red') - s.__class__.__bases__[-1].warn(s, msg, minor) - - def init_banner(s): - s.availablethreadframes = {} - s.threadframes = {} - #tflock protects the s.threadframes manipulation to only happen from 1 thread - s.tflock = RLock() - - def threadExited(s, thread): - threadid = thread.threadid - accountname = s.getthreadaccount(thread) - s.tflock.acquire() - try: - if threadid in s.threadframes[accountname]: - tf = s.threadframes[accountname][threadid] - del s.threadframes[accountname][threadid] - s.availablethreadframes[accountname].append(tf) - tf.setthread(None) - finally: - s.tflock.release() - - UIBase.threadExited(s, thread) - - def gettf(s): - threadid = get_ident() - accountname = s.getthreadaccount() - - s.tflock.acquire() - - try: - if not accountname in s.threadframes: - s.threadframes[accountname] = {} - - if threadid in s.threadframes[accountname]: - return s.threadframes[accountname][threadid] - - if not accountname in s.availablethreadframes: - s.availablethreadframes[accountname] = [] - - if len(s.availablethreadframes[accountname]): - tf = s.availablethreadframes[accountname].pop(0) - tf.setthread(currentThread()) - else: - tf = s.getaccountframe().getnewthreadframe() - s.threadframes[accountname][threadid] = tf - return tf - finally: - s.tflock.release() - - def callhook(s, msg): - s.gettf().setcolor('white') - s.__class__.__bases__[-1].callhook(s, msg) - - def sleep(s, sleepsecs, account): - s.gettf().setcolor('red') - s.getaccountframe().startsleep(sleepsecs) - return UIBase.sleep(s, sleepsecs, account) - - def sleeping(s, sleepsecs, remainingsecs): - if remainingsecs and s.gettf().getcolor() == 'black': - s.gettf().setcolor('red') - else: - s.gettf().setcolor('black') - return s.getaccountframe().sleeping(sleepsecs, remainingsecs) - - diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 84b0e9a..f1626c4 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -1,6 +1,5 @@ # Curses-based interfaces -# Copyright (C) 2003 John Goerzen -# +# Copyright (C) 2003-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,42 +15,65 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from threading import RLock, Lock, Event +from __future__ import with_statement # needed for python 2.5 +from threading import RLock, currentThread, Lock, Event +from thread import get_ident # python < 2.6 support +from collections import deque import time import sys import os import signal import curses -from Blinkenlights import BlinkenBase -from UIBase import UIBase +import logging +from offlineimap.ui.UIBase import UIBase +from offlineimap.threadutil import ExitNotifyThread import offlineimap - -acctkeys = '1234567890abcdefghijklmnoprstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-=;/.,' - class CursesUtil: - def __init__(self): - self.pairlock = Lock() - # iolock protects access to the + + def __init__(self, *args, **kwargs): + # iolock protects access to the self.iolock = RLock() - self.start() + self.tframe_lock = RLock() + """tframe_lock protects the self.threadframes manipulation to + only happen from 1 thread""" + self.colormap = {} + """dict, translating color string to curses color pair number""" - def initpairs(self): - self.pairlock.acquire() - try: - self.pairs = {self._getpairindex(curses.COLOR_WHITE, - curses.COLOR_BLACK): 0} - self.nextpair = 1 - finally: - self.pairlock.release() + def curses_colorpair(self, col_name): + """Return the curses color pair, that corresponds to the color""" + return curses.color_pair(self.colormap[col_name]) - def lock(self): + def init_colorpairs(self): + """initialize the curses color pairs available""" + # set special colors 'gray' and 'banner' + self.colormap['white'] = 0 #hardcoded by curses + curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) + self.colormap['banner'] = 1 # color 'banner' for bannerwin + + bcol = curses.COLOR_BLACK + colors = ( # name, color, bold? + ('black', curses.COLOR_BLACK, False), + ('blue', curses.COLOR_BLUE,False), + ('red', curses.COLOR_RED, False), + ('purple', curses.COLOR_MAGENTA, False), + ('cyan', curses.COLOR_CYAN, False), + ('green', curses.COLOR_GREEN, False), + ('orange', curses.COLOR_YELLOW, False)) + #set the rest of all colors starting at pair 2 + i = 1 + for name, fcol, bold in colors: + i += 1 + self.colormap[name] = i + curses.init_pair(i, fcol, bcol) + + def lock(self, block=True): """Locks the Curses ui thread Can be invoked multiple times from the owning thread. Invoking from a non-owning thread blocks and waits until it has been unlocked by the owning thread.""" - self.iolock.acquire() + return self.iolock.acquire(block) def unlock(self): """Unlocks the Curses ui thread @@ -61,8 +83,8 @@ class CursesUtil: thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked.""" self.iolock.release() - - def locked(self, target, *args, **kwargs): + + def exec_locked(self, target, *args, **kwargs): """Perform an operation with full locking.""" self.lock() try: @@ -74,536 +96,529 @@ class CursesUtil: def lockedstuff(): curses.panel.update_panels() curses.doupdate() - self.locked(lockedstuff) + self.exec_locked(lockedstuff) def isactive(self): return hasattr(self, 'stdscr') - def _getpairindex(self, fg, bg): - return '%d/%d' % (fg,bg) - - def getpair(self, fg, bg): - if not self.has_color: - return 0 - pindex = self._getpairindex(fg, bg) - self.pairlock.acquire() - try: - if self.pairs.has_key(pindex): - return curses.color_pair(self.pairs[pindex]) - else: - self.pairs[pindex] = self.nextpair - curses.init_pair(self.nextpair, fg, bg) - self.nextpair += 1 - return curses.color_pair(self.nextpair - 1) - finally: - self.pairlock.release() - - def start(self): - self.stdscr = curses.initscr() - curses.noecho() - curses.cbreak() - self.stdscr.keypad(1) - try: - curses.start_color() - self.has_color = curses.has_colors() - except: - self.has_color = 0 - - self.oldcursor = None - try: - self.oldcursor = curses.curs_set(0) - except: - pass - - self.stdscr.clear() - self.stdscr.refresh() - (self.height, self.width) = self.stdscr.getmaxyx() - self.initpairs() - - def stop(self): - if not hasattr(self, 'stdscr'): - return - #self.stdscr.addstr(self.height - 1, 0, "\n", - # self.getpair(curses.COLOR_WHITE, - # curses.COLOR_BLACK)) - if self.oldcursor != None: - curses.curs_set(self.oldcursor) - self.stdscr.refresh() - self.stdscr.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - del self.stdscr - - def reset(self): - # dirty walkaround for bug http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 (fixed since #83743) - if (sys.version_info[0:3] >= (2,6) and sys.version_info[0:3] <= (2,6,5)): return - self.stop() - self.start() class CursesAccountFrame: - def __init__(s, master, accountname, ui): - s.c = master - s.children = [] - s.accountname = accountname - s.ui = ui + """Notable instance variables: - def drawleadstr(s, secs = None): - if secs == None: - acctstr = '%s: [active] %13.13s: ' % (s.key, s.accountname) - else: - acctstr = '%s: [%3d:%02d] %13.13s: ' % (s.key, - secs / 60, secs % 60, - s.accountname) - s.c.locked(s.window.addstr, 0, 0, acctstr) - s.location = len(acctstr) + - account: corresponding Account() + - children + - ui + - key + - window: curses window associated with an account + """ - def setwindow(s, window, key): - s.window = window - s.key = key - s.drawleadstr() - for child in s.children: - child.update(window, 0, s.location) - s.location += 1 + def __init__(self, ui, account): + """ + :param account: An Account() or None (for eg SyncrunnerThread)""" + self.children = [] + self.account = account if account else '*Control' + self.ui = ui + self.window = None + """Curses window associated with this acc""" + self.acc_num = None + """Account number (& hotkey) associated with this acc""" + self.location = 0 + """length of the account prefix string""" - def getnewthreadframe(s): - tf = CursesThreadFrame(s.c, s.ui, s.window, 0, s.location) - s.location += 1 - s.children.append(tf) + def drawleadstr(self, secs = 0): + """Draw the account status string + + secs tells us how long we are going to sleep.""" + sleepstr = '%3d:%02d' % (secs // 60, secs % 60) if secs else 'active' + accstr = '%s: [%s] %12.12s: ' % (self.acc_num, sleepstr, self.account) + + self.ui.exec_locked(self.window.addstr, 0, 0, accstr) + self.location = len(accstr) + + def setwindow(self, curses_win, acc_num): + """Register an curses win and a hotkey as Account window + + :param curses_win: the curses window associated with an account + :param acc_num: int denoting the hotkey associated with this account.""" + self.window = curses_win + self.acc_num = acc_num + self.drawleadstr() + # Update the child ThreadFrames + for child in self.children: + child.update(curses_win, self.location, 0) + self.location += 1 + + def get_new_tframe(self): + """Create a new ThreadFrame and append it to self.children + + :returns: The new ThreadFrame""" + tf = CursesThreadFrame(self.ui, self.window, self.location, 0) + self.location += 1 + self.children.append(tf) return tf - def startsleep(s, sleepsecs): - s.sleeping_abort = 0 + def sleeping(self, sleepsecs, remainingsecs): + # show how long we are going to sleep and sleep + self.drawleadstr(remainingsecs) + self.ui.exec_locked(self.window.refresh) + time.sleep(sleepsecs) + return self.account.abort_signal.is_set() - def sleeping(s, sleepsecs, remainingsecs): - if remainingsecs: - s.c.lock() - try: - s.drawleadstr(remainingsecs) - s.window.refresh() - finally: - s.c.unlock() - time.sleep(sleepsecs) - else: - s.c.lock() - try: - s.drawleadstr() - s.window.refresh() - finally: - s.c.unlock() - return s.sleeping_abort - - def syncnow(s): - s.sleeping_abort = 1 + def syncnow(self): + """Request that we stop sleeping asap and continue to sync""" + # if this belongs to an Account (and not *Control), set the + # skipsleep pref + if isinstance(self.account, offlineimap.accounts.Account): + self.ui.info("Requested synchronization for acc: %s" % self.account) + self.account.config.set('Account %s' % self.account.name, + 'skipsleep', '1') class CursesThreadFrame: - def __init__(s, master, ui, window, y, x): - """master should be a CursesUtil object.""" - s.c = master - s.ui = ui - s.window = window - s.x = x - s.y = y - s.colors = [] - bg = curses.COLOR_BLACK - s.colormap = {'black': s.c.getpair(curses.COLOR_BLACK, bg), - 'gray': s.c.getpair(curses.COLOR_WHITE, bg), - 'white': curses.A_BOLD | s.c.getpair(curses.COLOR_WHITE, bg), - 'blue': s.c.getpair(curses.COLOR_BLUE, bg), - 'red': s.c.getpair(curses.COLOR_RED, bg), - 'purple': s.c.getpair(curses.COLOR_MAGENTA, bg), - 'cyan': s.c.getpair(curses.COLOR_CYAN, bg), - 'green': s.c.getpair(curses.COLOR_GREEN, bg), - 'orange': s.c.getpair(curses.COLOR_YELLOW, bg), - 'yellow': curses.A_BOLD | s.c.getpair(curses.COLOR_YELLOW, bg), - 'pink': curses.A_BOLD | s.c.getpair(curses.COLOR_RED, bg)} - #s.setcolor('gray') - s.setcolor('black') + """ + curses_color: current color pair for logging""" + def __init__(self, ui, acc_win, x, y): + """ + :param ui: is a Blinkenlights() instance + :param acc_win: curses Account window""" + self.ui = ui + self.window = acc_win + self.x = x + self.y = y + self.curses_color = curses.color_pair(0) #default color - def setcolor(self, color): - self.color = self.colormap[color] + def setcolor(self, color, modifier=0): + """Draw the thread symbol '@' in the specified color + :param modifier: Curses modified, such as curses.A_BOLD""" + self.curses_color = modifier | self.ui.curses_colorpair(color) self.colorname = color self.display() def display(self): - def lockedstuff(): - if self.getcolor() == 'black': - self.window.addstr(self.y, self.x, ' ', self.color) - else: - self.window.addstr(self.y, self.x, '.', self.color) - self.c.stdscr.move(self.c.height - 1, self.c.width - 1) + def locked_display(): + self.window.addch(self.y, self.x, '@', self.curses_color) self.window.refresh() - self.c.locked(lockedstuff) + # lock the curses IO while fudging stuff + self.ui.exec_locked(locked_display) - def getcolor(self): - return self.colorname - - def getcolorpair(self): - return self.color - - def update(self, window, y, x): - self.window = window + def update(self, acc_win, x, y): + """Update the xy position of the '.' (and possibly the aframe)""" + self.window = acc_win self.y = y self.x = x self.display() - def setthread(self, newthread): + def std_color(self): self.setcolor('black') - #if newthread: - # self.setcolor('gray') - #else: - # self.setcolor('black') -class InputHandler: - def __init__(s, util): - s.c = util - s.bgchar = None - s.inputlock = Lock() - s.lockheld = 0 - s.statuslock = Lock() - s.startup = Event() - s.startthread() - def startthread(s): - s.thread = offlineimap.threadutil.ExitNotifyThread(target = s.bgreaderloop, - name = "InputHandler loop") - s.thread.setDaemon(1) - s.thread.start() +class InputHandler(ExitNotifyThread): + """Listens for input via the curses interfaces""" + #TODO, we need to use the ugly exitnotifythread (rather than simply + #threading.Thread here, so exiting this thread via the callback + #handler, kills off all parents too. Otherwise, they would simply + #continue. + def __init__(self, ui): + super(InputHandler, self).__init__() + self.char_handler = None + self.ui = ui + self.enabled = Event() + """We will only parse input if we are enabled""" + self.inputlock = RLock() + """denotes whether we should be handling the next char.""" + self.start() #automatically start the thread - def bgreaderloop(s): - while 1: - s.statuslock.acquire() - if s.lockheld or s.bgchar == None: - s.statuslock.release() - s.startup.wait() - else: - s.statuslock.release() - ch = s.c.stdscr.getch() - s.statuslock.acquire() - try: - if s.lockheld or s.bgchar == None: - curses.ungetch(ch) - else: - s.bgchar(ch) - finally: - s.statuslock.release() + def get_next_char(self): + """return the key pressed or -1 - def set_bgchar(s, callback): - """Sets a "background" character handler. If a key is pressed - while not doing anything else, it will be passed to this handler. + Wait until `enabled` and loop internally every stdscr.timeout() + msecs, releasing the inputlock. + :returns: char or None if disabled while in here""" + self.enabled.wait() + while self.enabled.is_set(): + with self.inputlock: + char = self.ui.stdscr.getch() + if char != -1: yield char + + def run(self): + while True: + char_gen = self.get_next_char() + for char in char_gen: + self.char_handler(char) + #curses.ungetch(char) + + def set_char_hdlr(self, callback): + """Sets a character callback handler + + If a key is pressed it will be passed to this handler. Keys + include the curses.KEY_RESIZE key. callback is a function taking a single arg -- the char pressed. + If callback is None, input will be ignored.""" + with self.inputlock: + self.char_handler = callback + # start or stop the parsing of things + if callback is None: + self.enabled.clear() + else: + self.enabled.set() - If callback is None, clears the request.""" - s.statuslock.acquire() - oldhandler = s.bgchar - newhandler = callback - s.bgchar = callback - - if oldhandler and not newhandler: - pass - if newhandler and not oldhandler: - s.startup.set() - - s.statuslock.release() - - def input_acquire(s): + def input_acquire(self): """Call this method when you want exclusive input control. - Make sure to call input_release afterwards! + + Make sure to call input_release afterwards! While this lockis + held, input can go to e.g. the getpass input. """ + self.enabled.clear() + self.inputlock.acquire() - s.inputlock.acquire() - s.statuslock.acquire() - s.lockheld = 1 - s.statuslock.release() - - def input_release(s): + def input_release(self): """Call this method when you are done getting input.""" - s.statuslock.acquire() - s.lockheld = 0 - s.statuslock.release() - s.inputlock.release() - s.startup.set() - -class Blinkenlights(BlinkenBase, UIBase): - def init_banner(s): - s.af = {} - s.aflock = Lock() - s.c = CursesUtil() - s.text = [] - BlinkenBase.init_banner(s) - s.setupwindows() - s.inputhandler = InputHandler(s.c) - s.gettf().setcolor('red') - s._msg(offlineimap.banner) - s.inputhandler.set_bgchar(s.keypress) - signal.signal(signal.SIGWINCH, s.resizehandler) - s.resizelock = Lock() - s.resizecount = 0 - - def resizehandler(s, signum, frame): - s.resizeterm() - - def resizeterm(s, dosleep = 1): - if not s.resizelock.acquire(0): - s.resizecount += 1 - return - signal.signal(signal.SIGWINCH, signal.SIG_IGN) - s.aflock.acquire() - s.c.lock() - s.resizecount += 1 - while s.resizecount: - s.c.reset() - s.setupwindows() - s.resizecount -= 1 - s.c.unlock() - s.aflock.release() - s.resizelock.release() - signal.signal(signal.SIGWINCH, s.resizehandler) - if dosleep: - time.sleep(1) - s.resizeterm(0) - - def isusable(s): - # Not a terminal? Can't use curses. - if not sys.stdout.isatty() and sys.stdin.isatty(): - return 0 - - # No TERM specified? Can't use curses. - try: - if not len(os.environ['TERM']): - return 0 - except: return 0 - - # ncurses doesn't want to start? Can't use curses. - # This test is nasty because initscr() actually EXITS on error. - # grr. - - pid = os.fork() - if pid: - # parent - return not os.WEXITSTATUS(os.waitpid(pid, 0)[1]) - else: - # child - curses.initscr() - curses.endwin() - # If we didn't die by here, indicate success. - sys.exit(0) - - def keypress(s, key): - if key < 1 or key > 255: - return - - if chr(key) == 'q': - # Request to quit. - s.terminate() - - try: - index = acctkeys.index(chr(key)) - except ValueError: - # Key not a valid one: exit. - return - - if index >= len(s.hotkeys): - # Not in our list of valid hotkeys. - return - - # Trying to end sleep somewhere. - - s.getaccountframe(s.hotkeys[index]).syncnow() - - def getpass(s, accountname, config, errmsg = None): - s.inputhandler.input_acquire() - - # See comment on _msg for info on why both locks are obtained. - - s.tflock.acquire() - s.c.lock() - try: - s.gettf().setcolor('white') - s._addline(" *** Input Required", s.gettf().getcolorpair()) - s._addline(" *** Please enter password for account %s: " % accountname, - s.gettf().getcolorpair()) - s.logwindow.refresh() - password = s.logwindow.getstr() - finally: - s.tflock.release() - s.c.unlock() - s.inputhandler.input_release() - return password - - def setupwindows(s): - s.c.lock() - try: - s.bannerwindow = curses.newwin(1, s.c.width, 0, 0) - s.setupwindow_drawbanner() - s.logheight = s.c.height - 1 - len(s.af.keys()) - s.logwindow = curses.newwin(s.logheight, s.c.width, 1, 0) - s.logwindow.idlok(1) - s.logwindow.scrollok(1) - s.logwindow.move(s.logheight - 1, 0) - s.setupwindow_drawlog() - accounts = s.af.keys() - accounts.sort() - accounts.reverse() - - pos = s.c.height - 1 - index = 0 - s.hotkeys = [] - for account in accounts: - accountwindow = curses.newwin(1, s.c.width, pos, 0) - s.af[account].setwindow(accountwindow, acctkeys[index]) - s.hotkeys.append(account) - index += 1 - pos -= 1 - - curses.doupdate() - finally: - s.c.unlock() - - def setupwindow_drawbanner(s): - if s.c.has_color: - color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLUE) | \ - curses.A_BOLD - else: - color = curses.A_REVERSE - s.bannerwindow.bkgd(' ', color) # Fill background with that color - s.bannerwindow.addstr("%s %s" % (offlineimap.__productname__, - offlineimap.__version__)) - s.bannerwindow.addstr(0, s.bannerwindow.getmaxyx()[1] - len(offlineimap.__copyright__) - 1, - offlineimap.__copyright__) - - s.bannerwindow.noutrefresh() - - def setupwindow_drawlog(s): - if s.c.has_color: - color = s.c.getpair(curses.COLOR_WHITE, curses.COLOR_BLACK) - else: - color = curses.A_NORMAL - s.logwindow.bkgd(' ', color) - for line, color in s.text: - s.logwindow.addstr("\n" + line, color) - s.logwindow.noutrefresh() - - def getaccountframe(s, accountname = None): - if accountname == None: - accountname = s.getthreadaccount() - s.aflock.acquire() - try: - if accountname in s.af: - return s.af[accountname] - - # New one. - s.af[accountname] = CursesAccountFrame(s.c, accountname, s) - s.c.lock() - try: - s.c.reset() - s.setupwindows() - finally: - s.c.unlock() - finally: - s.aflock.release() - return s.af[accountname] + self.inputlock.release() + self.enabled.set() - def _display(s, msg, color = None): - if "\n" in msg: - for thisline in msg.split("\n"): - s._msg(thisline) - return +class CursesLogHandler(logging.StreamHandler): + """self.ui has been set to the UI class before anything is invoked""" + def emit(self, record): + log_str = super(CursesLogHandler, self).format(record) + color = self.ui.gettf().curses_color # We must acquire both locks. Otherwise, deadlock can result. # This can happen if one thread calls _msg (locking curses, then # tf) and another tries to set the color (locking tf, then curses) # # By locking both up-front here, in this order, we prevent deadlock. - - s.tflock.acquire() - s.c.lock() + self.ui.tframe_lock.acquire() + self.ui.lock() try: - if not s.c.isactive(): - # For dumping out exceptions and stuff. - print msg - return - if color: - s.gettf().setcolor(color) - elif s.gettf().getcolor() == 'black': - s.gettf().setcolor('gray') - s._addline(msg, s.gettf().getcolorpair()) - s.logwindow.refresh() + y,x = self.ui.logwin.getyx() + if y or x: self.ui.logwin.addch(10) # no \n before 1st item + self.ui.logwin.addstr(log_str, color) finally: - s.c.unlock() - s.tflock.release() + self.ui.unlock() + self.ui.tframe_lock.release() + self.ui.logwin.noutrefresh() + self.ui.stdscr.refresh() - def _addline(s, msg, color): - s.c.lock() +class Blinkenlights(UIBase, CursesUtil): + """Curses-cased fancy UI + + Notable instance variables self. ....: + + - stdscr: THe curses std screen + - bannerwin: The top line banner window + - width|height: The total curses screen dimensions + - logheight: Available height for the logging part + - log_con_handler: The CursesLogHandler() + - threadframes: + - accframes[account]: 'Accountframe'""" + + def __init__(self, *args, **kwargs): + super(Blinkenlights, self).__init__(*args, **kwargs) + CursesUtil.__init__(self) + + ################################################## UTILS + def setup_consolehandler(self): + """Backend specific console handler + + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = CursesLogHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + # the handler is not usable yet. We still need all the + # intialization stuff currently done in init_banner. Move here? + return ch + + def isusable(s): + """Returns true if the backend is usable ie Curses works""" + # Not a terminal? Can't use curses. + if not sys.stdout.isatty() and sys.stdin.isatty(): + return False + # No TERM specified? Can't use curses. + if not os.environ.get('TERM', None): + return False + # Test if ncurses actually starts up fine. Only do so for + # python>=2.6.6 as calling initscr() twice messing things up. + # see http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 + if sys.version_info[0:3] < (2,6) or sys.version_info[0:3] >= (2,6,6): + try: + curses.initscr() + curses.endwin() + except: + return False + return True + + def init_banner(self): + self.availablethreadframes = {} + self.threadframes = {} + self.accframes = {} + self.aflock = Lock() + + self.stdscr = curses.initscr() + # turn off automatic echoing of keys to the screen + curses.noecho() + # react to keys instantly, without Enter key + curses.cbreak() + # return special key values, eg curses.KEY_LEFT + self.stdscr.keypad(1) + # wait 1s for input, so we don't block the InputHandler infinitely + self.stdscr.timeout(1000) + curses.start_color() + # turn off cursor and save original state + self.oldcursor = None try: - s.logwindow.addstr("\n" + msg, color) - s.text.append((msg, color)) - while len(s.text) > s.logheight: - s.text = s.text[1:] + self.oldcursor = curses.curs_set(0) + except: + pass + + self.stdscr.clear() + self.stdscr.refresh() + self.init_colorpairs() + # set log handlers ui to ourself + self._log_con_handler.ui = self + self.setupwindows() + # Settup keyboard handler + self.inputhandler = InputHandler(self) + self.inputhandler.set_char_hdlr(self.on_keypressed) + + self.gettf().setcolor('red') + self.info(offlineimap.banner) + + def acct(self, *args): + """Output that we start syncing an account (and start counting)""" + self.gettf().setcolor('purple') + super(Blinkenlights, self).acct(*args) + + def connecting(self, *args): + self.gettf().setcolor('white') + super(Blinkenlights, self).connecting(*args) + + def syncfolders(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).syncfolders(*args) + + def syncingfolder(self, *args): + self.gettf().setcolor('cyan') + super(Blinkenlights, self).syncingfolder(*args) + + def skippingfolder(self, *args): + self.gettf().setcolor('cyan') + super(Blinkenlights, self).skippingfolder(*args) + + def loadmessagelist(self, *args): + self.gettf().setcolor('green') + super(Blinkenlights, self).loadmessagelist(*args) + + def syncingmessages(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).syncingmessages(*args) + + def copyingmessage(self, *args): + self.gettf().setcolor('orange') + super(Blinkenlights, self).copyingmessage(*args) + + def deletingmessages(self, *args): + self.gettf().setcolor('red') + super(Blinkenlights, self).deletingmessages(*args) + + def addingflags(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).addingflags(*args) + + def deletingflags(self, *args): + self.gettf().setcolor('blue') + super(Blinkenlights, self).deletingflags(*args) + + def callhook(self, *args): + self.gettf().setcolor('white') + super(Blinkenlights, self).callhook(*args) + + ############ Generic logging functions ############################# + def warn(self, msg, minor=0): + self.gettf().setcolor('red', curses.A_BOLD) + super(Blinkenlights, self).warn(msg) + + def threadExited(self, thread): + acc = self.getthreadaccount(thread) + with self.tframe_lock: + if thread in self.threadframes[acc]: + tf = self.threadframes[acc][thread] + tf.setcolor('black') + self.availablethreadframes[acc].append(tf) + del self.threadframes[acc][thread] + super(Blinkenlights, self).threadExited(thread) + + def gettf(self): + """Return the ThreadFrame() of the current thread""" + cur_thread = currentThread() + acc = self.getthreadaccount() #Account() or None + + with self.tframe_lock: + # Ideally we already have self.threadframes[accountname][thread] + try: + if cur_thread in self.threadframes[acc]: + return self.threadframes[acc][cur_thread] + except KeyError: + # Ensure threadframes already has an account dict + self.threadframes[acc] = {} + self.availablethreadframes[acc] = deque() + + # If available, return a ThreadFrame() + if len(self.availablethreadframes[acc]): + tf = self.availablethreadframes[acc].popleft() + tf.std_color() + else: + tf = self.getaccountframe(acc).get_new_tframe() + self.threadframes[acc][cur_thread] = tf + return tf + + def on_keypressed(self, key): + # received special KEY_RESIZE, resize terminal + if key == curses.KEY_RESIZE: + self.resizeterm() + + if key < 1 or key > 255: + return + if chr(key) == 'q': + # Request to quit. + #TODO: this causes us to bail out in main loop when the thread exits + #TODO: review and rework this mechanism. + currentThread().set_exit_exception(SystemExit("User requested shutdown")) + self.terminate() + try: + index = int(chr(key)) + except ValueError: + return # Key not a valid number: exit. + if index >= len(self.hotkeys): + # Not in our list of valid hotkeys. + return + # Trying to end sleep somewhere. + self.getaccountframe(self.hotkeys[index]).syncnow() + + def sleep(self, sleepsecs, account): + self.gettf().setcolor('red') + self.info("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) + return super(Blinkenlights, self).sleep(sleepsecs, account) + + def sleeping(self, sleepsecs, remainingsecs): + if not sleepsecs: + # reset color to default if we are done sleeping. + self.gettf().setcolor('white') + accframe = self.getaccountframe(self.getthreadaccount()) + return accframe.sleeping(sleepsecs, remainingsecs) + + def resizeterm(self): + """Resize the current windows""" + self.exec_locked(self.setupwindows, True) + + def mainException(self): + UIBase.mainException(self) + + def getpass(self, accountname, config, errmsg = None): + # disable the hotkeys inputhandler + self.inputhandler.input_acquire() + + # See comment on _msg for info on why both locks are obtained. + self.lock() + try: + #s.gettf().setcolor('white') + self.warn(" *** Input Required") + self.warn(" *** Please enter password for account %s: " % \ + accountname) + self.logwin.refresh() + password = self.logwin.getstr() finally: - s.c.unlock() + self.unlock() + self.inputhandler.input_release() + return password - def terminate(s, exitstatus = 0, errortitle = None, errormsg = None): - s.c.stop() - UIBase.terminate(s, exitstatus = exitstatus, errortitle = errortitle, errormsg = errormsg) + def setupwindows(self, resize=False): + """Setup and draw bannerwin and logwin - def threadException(s, thread): - s.c.stop() - UIBase.threadException(s, thread) + If `resize`, don't create new windows, just adapt size. This + function should be invoked with CursesUtils.locked().""" + self.height, self.width = self.stdscr.getmaxyx() + self.logheight = self.height - len(self.accframes) - 1 + if resize: + curses.resizeterm(self.height, self.width) + self.bannerwin.resize(1, self.width) + self.logwin.resize(self.logheight, self.width) + else: + self.bannerwin = curses.newwin(1, self.width, 0, 0) + self.logwin = curses.newwin(self.logheight, self.width, 1, 0) - def mainException(s): - s.c.stop() - UIBase.mainException(s) + self.draw_bannerwin() + self.logwin.idlok(True) # needed for scrollok below + self.logwin.scrollok(True) # scroll window when too many lines added + self.draw_logwin() + self.accounts = reversed(sorted(self.accframes.keys())) + pos = self.height - 1 + index = 0 + self.hotkeys = [] + for account in self.accounts: + acc_win = curses.newwin(1, self.width, pos, 0) + self.accframes[account].setwindow(acc_win, index) + self.hotkeys.append(account) + index += 1 + pos -= 1 + curses.doupdate() - def sleep(s, sleepsecs, account): - s.gettf().setcolor('red') - s._msg("Next sync in %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return BlinkenBase.sleep(s, sleepsecs, account) - -if __name__ == '__main__': - x = Blinkenlights(None) - x.init_banner() - import time - time.sleep(5) - x.c.stop() - fgs = {'black': curses.COLOR_BLACK, 'red': curses.COLOR_RED, - 'green': curses.COLOR_GREEN, 'yellow': curses.COLOR_YELLOW, - 'blue': curses.COLOR_BLUE, 'magenta': curses.COLOR_MAGENTA, - 'cyan': curses.COLOR_CYAN, 'white': curses.COLOR_WHITE} - - x = CursesUtil() - win1 = curses.newwin(x.height, x.width / 4 - 1, 0, 0) - win1.addstr("Black/normal\n") - for name, fg in fgs.items(): - win1.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLACK)) - win2 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1]) - win2.addstr("Blue/normal\n") - for name, fg in fgs.items(): - win2.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLUE)) - win3 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1] + - win2.getmaxyx()[1]) - win3.addstr("Black/bright\n") - for name, fg in fgs.items(): - win3.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLACK) | \ - curses.A_BOLD) - win4 = curses.newwin(x.height, x.width / 4 - 1, 0, win1.getmaxyx()[1] * 3) - win4.addstr("Blue/bright\n") - for name, fg in fgs.items(): - win4.addstr("%s\n" % name, x.getpair(fg, curses.COLOR_BLUE) | \ - curses.A_BOLD) - - - win1.refresh() - win2.refresh() - win3.refresh() - win4.refresh() - x.stdscr.refresh() - import time - time.sleep(5) - x.stop() - print x.has_color - print x.height - print x.width + def draw_bannerwin(self): + """Draw the top-line banner line""" + if curses.has_colors(): + color = curses.A_BOLD | self.curses_colorpair('banner') + else: + color = curses.A_REVERSE + self.bannerwin.clear() # Delete old content (eg before resizes) + self.bannerwin.bkgd(' ', color) # Fill background with that color + string = "%s %s" % (offlineimap.__productname__, + offlineimap.__version__) + self.bannerwin.addstr(0, 0, string, color) + self.bannerwin.addstr(0, self.width -len(offlineimap.__copyright__) -1, + offlineimap.__copyright__, color) + self.bannerwin.noutrefresh() + + def draw_logwin(self): + """(Re)draw the current logwindow""" + if curses.has_colors(): + color = curses.color_pair(0) #default colors + else: + color = curses.A_NORMAL + self.logwin.move(0, 0) + self.logwin.erase() + self.logwin.bkgd(' ', color) + + def getaccountframe(self, acc_name): + """Return an AccountFrame() corresponding to acc_name + + Note that the *control thread uses acc_name `None`.""" + with self.aflock: + # 1) Return existing or 2) create a new CursesAccountFrame. + if acc_name in self.accframes: return self.accframes[acc_name] + self.accframes[acc_name] = CursesAccountFrame(self, acc_name) + # update the window layout + self.setupwindows(resize= True) + return self.accframes[acc_name] + + def terminate(self, *args, **kwargs): + curses.nocbreak(); + self.stdscr.keypad(0); + curses.echo() + curses.endwin() + # need to remove the Curses console handler now and replace with + # basic one, so exceptions and stuff are properly displayed + self.logger.removeHandler(self._log_con_handler) + UIBase.setup_consolehandler(self) + # finally call parent terminate which prints out exceptions etc + super(Blinkenlights, self).terminate(*args, **kwargs) + + def threadException(self, thread): + #self._log_con_handler.stop() + UIBase.threadException(self, thread) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 50a7f99..868433a 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -1,5 +1,4 @@ -# Copyright (C) 2007 John Goerzen -# +# Copyright (C) 2007-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,52 +17,34 @@ import urllib import sys import time +import logging from UIBase import UIBase -from threading import currentThread, Lock +from threading import currentThread import offlineimap -protocol = '6.0.0' +protocol = '7.0.0' class MachineUI(UIBase): - def __init__(s, config, verbose = 0): - UIBase.__init__(s, config, verbose) - s.safechars=" ;,./-_=+()[]" - s.iswaiting = 0 - s.outputlock = Lock() - s._printData('__init__', protocol) + def __init__(self, config, loglevel = logging.INFO): + super(MachineUI, self).__init__(config, loglevel) + self._log_con_handler.createLock() + """lock needed to block on password input""" - def isusable(s): - return True + def _printData(self, command, msg): + self.logger.info("%s:%s:%s:%s" % ( + 'msg', command, currentThread().getName(), msg)) - def _printData(s, command, data, dolock = True): - s._printDataOut('msg', command, data, dolock) - - def _printWarn(s, command, data, dolock = True): - s._printDataOut('warn', command, data, dolock) - - def _printDataOut(s, datatype, command, data, dolock = True): - if dolock: - s.outputlock.acquire() - try: - print "%s:%s:%s:%s" % \ - (datatype, - urllib.quote(command, s.safechars), - urllib.quote(currentThread().getName(), s.safechars), - urllib.quote(data, s.safechars)) - sys.stdout.flush() - finally: - if dolock: - s.outputlock.release() - - def _display(s, msg): + def _msg(s, msg): s._printData('_display', msg) def warn(s, msg, minor = 0): - s._printData('warn', '%s\n%d' % (msg, int(minor))) + # TODO, remove and cleanup the unused minor stuff + self.logger.warning("%s:%s:%s:%s" % ( + 'warn', '', currentThread().getName(), msg)) - def registerthread(s, account): - UIBase.registerthread(s, account) - s._printData('registerthread', account) + def registerthread(self, account): + super(MachineUI, self).registerthread(self, account) + self._printData('registerthread', account) def unregisterthread(s, thread): UIBase.unregisterthread(s, thread) @@ -112,13 +93,10 @@ class MachineUI(UIBase): self._printData('copyingmessage', "%d\n%s\n%s\n%s[%s]" % \ (uid, self.getnicename(srcfolder), srcfolder.getname(), self.getnicename(destfolder), destfolder)) - + def folderlist(s, list): return ("\f".join(["%s\t%s" % (s.getnicename(x), x.getname()) for x in list])) - def deletingmessage(s, uid, destlist): - s.deletingmessages(s, [uid], destlist) - def uidlist(s, list): return ("\f".join([str(u) for u in list])) @@ -161,19 +139,21 @@ class MachineUI(UIBase): return 0 - def getpass(s, accountname, config, errmsg = None): - s.outputlock.acquire() + def getpass(self, accountname, config, errmsg = None): + if errmsg: + self._printData('getpasserror', "%s\n%s" % (accountname, errmsg), + False) + + self._log_con_handler.acquire() # lock the console output try: - if errmsg: - s._printData('getpasserror', "%s\n%s" % (accountname, errmsg), - False) - s._printData('getpass', accountname, False) + self._printData('getpass', accountname, False) return (sys.stdin.readline()[:-1]) finally: - s.outputlock.release() + self._log_con_handler.release() - def init_banner(s): - s._printData('initbanner', offlineimap.banner) + def init_banner(self): + self._printData('protocol', protocol) + self._printData('initbanner', offlineimap.banner) - def callhook(s, msg): - s._printData('callhook', msg) + def callhook(self, msg): + self._printData('callhook', msg) diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index 33248dd..36bb92b 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -1,6 +1,5 @@ # Noninteractive UI -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,37 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import sys -import time +import logging from UIBase import UIBase class Basic(UIBase): - def getpass(s, accountname, config, errmsg = None): - raise NotImplementedError, "Prompting for a password is not supported in noninteractive mode." + """'Quiet' simply sets log level to INFO""" + def __init__(self, config, loglevel = logging.INFO): + return super(Basic, self).__init__(config, loglevel) - def _display(s, msg): - print msg - sys.stdout.flush() - - def warn(s, msg, minor = 0): - warntxt = 'WARNING' - if minor: - warntxt = 'warning' - sys.stderr.write(warntxt + ": " + str(msg) + "\n") - - def sleep(s, sleepsecs, siglistener): - if s.verbose >= 0: - s._msg("Sleeping for %d:%02d" % (sleepsecs / 60, sleepsecs % 60)) - return UIBase.sleep(s, sleepsecs, siglistener) - - def sleeping(s, sleepsecs, remainingsecs): - if sleepsecs > 0: - time.sleep(sleepsecs) - return 0 - - def locked(s): - s.warn("Another OfflineIMAP is running with the same metadatadir; exiting.") - -class Quiet(Basic): - def __init__(s, config, verbose = -1): - Basic.__init__(s, config, verbose) +class Quiet(UIBase): + """'Quiet' simply sets log level to WARNING""" + def __init__(self, config, loglevel = logging.WARNING): + return super(Quiet, self).__init__(config, loglevel) diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py index 48c468f..cedd8a6 100644 --- a/offlineimap/ui/TTY.py +++ b/offlineimap/ui/TTY.py @@ -1,6 +1,5 @@ # TTY UI -# Copyright (C) 2002 John Goerzen -# +# Copyright (C) 2002-2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,53 +14,73 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from UIBase import UIBase -from getpass import getpass + +import logging import sys -from threading import Lock, currentThread +from getpass import getpass +from offlineimap import banner +from offlineimap.ui.UIBase import UIBase + +class TTYFormatter(logging.Formatter): + """Specific Formatter that adds thread information to the log output""" + def __init__(self, *args, **kwargs): + super(TTYFormatter, self).__init__(*args, **kwargs) + self._last_log_thread = None + + def format(self, record): + """Override format to add thread information""" + log_str = super(TTYFormatter, self).format(record) + # If msg comes from a different thread than our last, prepend + # thread info. Most look like 'Account sync foo' or 'Folder + # sync foo'. + t_name = record.threadName + if t_name == 'MainThread': + return log_str # main thread doesn't get things prepended + if t_name != self._last_log_thread: + self._last_log_thread = t_name + log_str = "%s:\n %s" % (t_name, log_str) + else: + log_str = " %s" % log_str + return log_str class TTYUI(UIBase): - def __init__(s, config, verbose = 0): - UIBase.__init__(s, config, verbose) - s.iswaiting = 0 - s.outputlock = Lock() - s._lastThreaddisplay = None + def setup_consolehandler(self): + """Backend specific console handler - def isusable(s): + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = logging.StreamHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = TTYFormatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(banner) + # init lock for console output + ch.createLock() + return ch + + def isusable(self): + """TTYUI is reported as usable when invoked on a terminal""" return sys.stdout.isatty() and sys.stdin.isatty() - - def _display(s, msg): - s.outputlock.acquire() - try: - #if the next output comes from a different thread than our last one - #add the info. - #Most look like 'account sync foo' or 'Folder sync foo'. - threadname = currentThread().getName() - if (threadname == s._lastThreaddisplay \ - or threadname == 'MainThread'): - print " %s" % msg - else: - print "%s:\n %s" % (threadname, msg) - s._lastThreaddisplay = threadname - sys.stdout.flush() - finally: - s.outputlock.release() - - def getpass(s, accountname, config, errmsg = None): + def getpass(self, accountname, config, errmsg = None): + """TTYUI backend is capable of querying the password""" if errmsg: - s._msg("%s: %s" % (accountname, errmsg)) - s.outputlock.acquire() + self.warn("%s: %s" % (accountname, errmsg)) + self._log_con_handler.acquire() # lock the console output try: - return getpass("%s: Enter password: " % accountname) + return getpass("Enter password for account '%s': " % accountname) finally: - s.outputlock.release() + self._log_con_handler.release() - def mainException(s): - if isinstance(sys.exc_info()[1], KeyboardInterrupt) and \ - s.iswaiting: - sys.stdout.write("Timer interrupted at user request; program terminating. \n") - s.terminate() + def mainException(self): + if isinstance(sys.exc_info()[1], KeyboardInterrupt): + self.logger.warn("Timer interrupted at user request; program " + "terminating.\n") + self.terminate() else: - UIBase.mainException(s) + UIBase.mainException(self) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index b5094eb..7dc23e5 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -15,12 +15,15 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +import logging import re import time import sys +import os import traceback import threading from Queue import Queue +from collections import deque import offlineimap debugtypes = {'':'Other offlineimap related sync messages', @@ -38,54 +41,72 @@ def getglobalui(): global globalui return globalui -class UIBase: - def __init__(s, config, verbose = 0): - s.verbose = verbose - s.config = config - s.debuglist = [] - s.debugmessages = {} - s.debugmsglen = 50 - s.threadaccounts = {} +class UIBase(object): + def __init__(self, config, loglevel = logging.INFO): + self.config = config + self.debuglist = [] + """list of debugtypes we are supposed to log""" + self.debugmessages = {} + """debugmessages in a deque(v) per thread(k)""" + self.debugmsglen = 15 + self.threadaccounts = {} """dict linking active threads (k) to account names (v)""" - s.acct_startimes = {} + self.acct_startimes = {} """linking active accounts with the time.time() when sync started""" - s.logfile = None - s.exc_queue = Queue() + self.logfile = None + self.exc_queue = Queue() """saves all occuring exceptions, so we can output them at the end""" + # create logger with 'OfflineImap' app + self.logger = logging.getLogger('OfflineImap') + self.logger.setLevel(loglevel) + self._log_con_handler = self.setup_consolehandler() + """The console handler (we need access to be able to lock it)""" ################################################## UTILS - def _msg(s, msg): - """Generic tool called when no other works.""" - s._log(msg) - s._display(msg) + def setup_consolehandler(self): + """Backend specific console handler - def _log(s, msg): - """Log it to disk. Returns true if it wrote something; false - otherwise.""" - if s.logfile: - s.logfile.write("%s: %s\n" % (threading.currentThread().getName(), - msg)) - return 1 - return 0 + Sets up things and adds them to self.logger. + :returns: The logging.Handler() for console output""" + # create console handler with a higher log level + ch = logging.StreamHandler() + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(offlineimap.banner) + return ch - def setlogfd(s, logfd): - s.logfile = logfd - logfd.write("This is %s %s\n" % \ - (offlineimap.__productname__, - offlineimap.__version__)) - logfd.write("Python: %s\n" % sys.version) - logfd.write("Platform: %s\n" % sys.platform) - logfd.write("Args: %s\n" % sys.argv) + def setlogfile(self, logfile): + """Create file handler which logs to file""" + fh = logging.FileHandler(logfile, 'at') + #fh.setLevel(logging.DEBUG) + file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " + "%(message)s", '%Y-%m-%d %H:%M:%S') + fh.setFormatter(file_formatter) + self.logger.addHandler(fh) + # write out more verbose initial info blurb on the log file + p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) + msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ + "Args: %s" % (offlineimap.__version__, p_ver, sys.platform, + " ".join(sys.argv)) + record = logging.LogRecord('OfflineImap', logging.INFO, __file__, + None, msg, None, None) + fh.emit(record) - def _display(s, msg): + def _msg(self, msg): """Display a message.""" - raise NotImplementedError + # TODO: legacy function, rip out. + self.info(msg) - def warn(s, msg, minor = 0): - if minor: - s._msg("warning: " + msg) - else: - s._msg("WARNING: " + msg) + def info(self, msg): + """Display a message.""" + self.logger.info(msg) + + def warn(self, msg, minor = 0): + self.logger.warning(msg) def error(self, exc, exc_traceback=None, msg=None): """Log a message at severity level ERROR @@ -139,47 +160,52 @@ class UIBase: self.debug('thread', "Unregister thread '%s'" % thr.getName()) def getthreadaccount(self, thr = None): - """Get name of account for a thread (current if None)""" - if not thr: + """Get Account() for a thread (current if None) + + If no account has been registered with this thread, return 'None'""" + if thr == None: thr = threading.currentThread() if thr in self.threadaccounts: return self.threadaccounts[thr] - return '*Control' # unregistered thread is '*Control' + return None - def debug(s, debugtype, msg): - thisthread = threading.currentThread() - if s.debugmessages.has_key(thisthread): - s.debugmessages[thisthread].append("%s: %s" % (debugtype, msg)) - else: - s.debugmessages[thisthread] = ["%s: %s" % (debugtype, msg)] + def debug(self, debugtype, msg): + cur_thread = threading.currentThread() + if not self.debugmessages.has_key(cur_thread): + # deque(..., self.debugmsglen) would be handy but was + # introduced in p2.6 only, so we'll need to work around and + # shorten our debugmsg list manually :-( + self.debugmessages[cur_thread] = deque() + self.debugmessages[cur_thread].append("%s: %s" % (debugtype, msg)) - while len(s.debugmessages[thisthread]) > s.debugmsglen: - s.debugmessages[thisthread] = s.debugmessages[thisthread][1:] + # Shorten queue if needed + if len(self.debugmessages[cur_thread]) > self.debugmsglen: + self.debugmessages[cur_thread].popleft() - if debugtype in s.debuglist: - if not s._log("DEBUG[%s]: %s" % (debugtype, msg)): - s._display("DEBUG[%s]: %s" % (debugtype, msg)) + if debugtype in self.debuglist: # log if we are supposed to do so + self.logger.debug("[%s]: %s" % (debugtype, msg)) - def add_debug(s, debugtype): + def add_debug(self, debugtype): global debugtypes if debugtype in debugtypes: - if not debugtype in s.debuglist: - s.debuglist.append(debugtype) - s.debugging(debugtype) + if not debugtype in self.debuglist: + self.debuglist.append(debugtype) + self.debugging(debugtype) else: - s.invaliddebug(debugtype) + self.invaliddebug(debugtype) - def debugging(s, debugtype): + def debugging(self, debugtype): global debugtypes - s._msg("Now debugging for %s: %s" % (debugtype, debugtypes[debugtype])) + self.logger.debug("Now debugging for %s: %s" % (debugtype, + debugtypes[debugtype])) - def invaliddebug(s, debugtype): - s.warn("Invalid debug type: %s" % debugtype) + def invaliddebug(self, debugtype): + self.warn("Invalid debug type: %s" % debugtype) def locked(s): raise Exception, "Another OfflineIMAP is running with the same metadatadir; exiting." - def getnicename(s, object): + def getnicename(self, object): """Return the type of a repository or Folder as string (IMAP, Gmail, Maildir, etc...)""" @@ -187,61 +213,73 @@ class UIBase: # Strip off extra stuff. return re.sub('(Folder|Repository)', '', prelimname) - def isusable(s): + def isusable(self): """Returns true if this UI object is usable in the current environment. For instance, an X GUI would return true if it's being run in X with a valid DISPLAY setting, and false otherwise.""" - return 1 + return True ################################################## INPUT - def getpass(s, accountname, config, errmsg = None): - raise NotImplementedError + def getpass(self, accountname, config, errmsg = None): + raise NotImplementedError("Prompting for a password is not supported"\ + " in this UI backend.") - def folderlist(s, list): - return ', '.join(["%s[%s]" % (s.getnicename(x), x.getname()) for x in list]) + def folderlist(self, list): + return ', '.join(["%s[%s]" % \ + (self.getnicename(x), x.getname()) for x in list]) ################################################## WARNINGS - def msgtoreadonly(s, destfolder, uid, content, flags): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to synchronize message %d to folder %s[%s], but that folder is read-only. The message will not be copied to that folder." % \ - (uid, s.getnicename(destfolder), destfolder.getname())) + def msgtoreadonly(self, destfolder, uid, content, flags): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to synchronize message %d to folder %s[%s], " + "but that folder is read-only. The message will not be " + "copied to that folder." % ( + uid, self.getnicename(destfolder), destfolder)) - def flagstoreadonly(s, destfolder, uidlist, flags): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to modify flags for messages %s in folder %s[%s], but that folder is read-only. No flags have been modified for that message." % \ - (str(uidlist), s.getnicename(destfolder), destfolder.getname())) + def flagstoreadonly(self, destfolder, uidlist, flags): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to modify flags for messages %s in folder %s[%s], " + "but that folder is read-only. No flags have been modified " + "for that message." % ( + str(uidlist), self.getnicename(destfolder), destfolder)) - def deletereadonly(s, destfolder, uidlist): - if not (s.config.has_option('general', 'ignore-readonly') and s.config.getboolean("general", "ignore-readonly")): - s.warn("Attempted to delete messages %s in folder %s[%s], but that folder is read-only. No messages have been deleted in that folder." % \ - (str(uidlist), s.getnicename(destfolder), destfolder.getname())) + def deletereadonly(self, destfolder, uidlist): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return + self.warn("Attempted to delete messages %s in folder %s[%s], but that " + "folder is read-only. No messages have been deleted in that " + "folder." % (str(uidlist), self.getnicename(destfolder), + destfolder)) ################################################## MESSAGES - def init_banner(s): + def init_banner(self): """Called when the UI starts. Must be called before any other UI call except isusable(). Displays the copyright banner. This is where the UI should do its setup -- TK, for instance, would create the application window here.""" - if s.verbose >= 0: - s._msg(offlineimap.banner) + pass - def connecting(s, hostname, port): + def connecting(self, hostname, port): """Log 'Establishing connection to'""" - if s.verbose < 0: return + if not self.logger.isEnabledFor(logging.info): return displaystr = '' hostname = hostname if hostname else '' port = "%s" % port if port else '' if hostname: displaystr = ' to %s:%s' % (hostname, port) - s._msg("Establishing connection%s" % displaystr) + self.logger.info("Establishing connection%s" % displaystr) def acct(self, account): """Output that we start syncing an account (and start counting)""" self.acct_startimes[account] = time.time() - if self.verbose >= 0: - self._msg("*** Processing account %s" % account) + self.logger.info("*** Processing account %s" % account) def acctdone(self, account): """Output that we finished syncing an account (in which time)""" @@ -252,141 +290,175 @@ class UIBase: def syncfolders(self, src_repo, dst_repo): """Log 'Copying folder structure...'""" - if self.verbose < 0: return - self.debug('', "Copying folder structure from %s to %s" % \ - (src_repo, dst_repo)) + if self.logger.isEnabledFor(logging.DEBUG): + self.debug('', "Copying folder structure from %s to %s" %\ + (src_repo, dst_repo)) ############################## Folder syncing - def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): + def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder): """Called when a folder sync operation is started.""" - if s.verbose >= 0: - s._msg("Syncing %s: %s -> %s" % (srcfolder.getname(), - s.getnicename(srcrepos), - s.getnicename(destrepos))) + self.logger.info("Syncing %s: %s -> %s" % (srcfolder, + self.getnicename(srcrepos), + self.getnicename(destrepos))) - def skippingfolder(s, folder): + def skippingfolder(self, folder): """Called when a folder sync operation is started.""" - if s.verbose >= 0: - s._msg("Skipping %s (not changed)" % folder.getname()) + self.logger.info("Skipping %s (not changed)" % folder) - def validityproblem(s, folder): - s.warn("UID validity problem for folder %s (repo %s) (saved %d; got %d); skipping it" % \ - (folder.getname(), folder.getrepository().getname(), + def validityproblem(self, folder): + self.logger.warning("UID validity problem for folder %s (repo %s) " + "(saved %d; got %d); skipping it. Please see FAQ " + "and manual how to handle this." % \ + (folder, folder.getrepository(), folder.getsaveduidvalidity(), folder.getuidvalidity())) - def loadmessagelist(s, repos, folder): - if s.verbose > 0: - s._msg("Loading message list for %s[%s]" % (s.getnicename(repos), - folder.getname())) + def loadmessagelist(self, repos, folder): + self.logger.debug("Loading message list for %s[%s]" % ( + self.getnicename(repos), + folder)) - def messagelistloaded(s, repos, folder, count): - if s.verbose > 0: - s._msg("Message list for %s[%s] loaded: %d messages" % \ - (s.getnicename(repos), folder.getname(), count)) + def messagelistloaded(self, repos, folder, count): + self.logger.debug("Message list for %s[%s] loaded: %d messages" % ( + self.getnicename(repos), folder, count)) ############################## Message syncing - def syncingmessages(s, sr, sf, dr, df): - if s.verbose > 0: - s._msg("Syncing messages %s[%s] -> %s[%s]" % (s.getnicename(sr), - sf.getname(), - s.getnicename(dr), - df.getname())) + def syncingmessages(self, sr, srcfolder, dr, dstfolder): + self.logger.debug("Syncing messages %s[%s] -> %s[%s]" % ( + self.getnicename(sr), srcfolder, + self.getnicename(dr), dstfolder)) def copyingmessage(self, uid, num, num_to_copy, src, destfolder): """Output a log line stating which message we copy""" - if self.verbose < 0: return - self._msg("Copy message %s (%d of %d) %s:%s -> %s" % (uid, num, - num_to_copy, src.repository, src, destfolder.repository)) + self.logger.info("Copy message %s (%d of %d) %s:%s -> %s" % ( + uid, num, num_to_copy, src.repository, src, + destfolder.repository)) - def deletingmessage(s, uid, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting message %d in %s" % (uid, ds)) + def deletingmessages(self, uidlist, destlist): + ds = self.folderlist(destlist) + self.logger.info("Deleting %d messages (%s) in %s" % ( + len(uidlist), + offlineimap.imaputil.uid_sequence(uidlist), ds)) - def deletingmessages(s, uidlist, destlist): - if s.verbose >= 0: - ds = s.folderlist(destlist) - s._msg("Deleting %d messages (%s) in %s" % \ - (len(uidlist), - offlineimap.imaputil.uid_sequence(uidlist), - ds)) + def addingflags(self, uidlist, flags, dest): + self.logger.info("Adding flag %s to %d messages on %s" % ( + ", ".join(flags), len(uidlist), dest)) - def addingflags(s, uidlist, flags, dest): - if s.verbose >= 0: - s._msg("Adding flag %s to %d messages on %s" % \ - (", ".join(flags), len(uidlist), dest)) + def deletingflags(self, uidlist, flags, dest): + self.logger.info("Deleting flag %s from %d messages on %s" % ( + ", ".join(flags), len(uidlist), dest)) - def deletingflags(s, uidlist, flags, dest): - if s.verbose >= 0: - s._msg("Deleting flag %s from %d messages on %s" % \ - (", ".join(flags), len(uidlist), dest)) + def serverdiagnostics(self, repository, type): + """Connect to repository and output useful information for debugging""" + conn = None + self._msg("%s repository '%s': type '%s'" % (type, repository.name, + self.getnicename(repository))) + try: + if hasattr(repository, 'gethost'): # IMAP + self._msg("Host: %s Port: %s SSL: %s" % (repository.gethost(), + repository.getport(), + repository.getssl())) + try: + conn = repository.imapserver.acquireconnection() + except OfflineImapError, e: + self._msg("Failed to connect. Reason %s" % e) + else: + if 'ID' in conn.capabilities: + self._msg("Server supports ID extension.") + #TODO: Debug and make below working, it hangs Gmail + #res_type, response = conn.id(( + # 'name', offlineimap.__productname__, + # 'version', offlineimap.__version__)) + #self._msg("Server ID: %s %s" % (res_type, response[0])) + self._msg("Server welcome string: %s" % str(conn.welcome)) + self._msg("Server capabilities: %s\n" % str(conn.capabilities)) + repository.imapserver.releaseconnection(conn) + if type != 'Status': + folderfilter = repository.getconf('folderfilter', None) + if folderfilter: + self._msg("folderfilter= %s\n" % folderfilter) + folderincludes = repository.getconf('folderincludes', None) + if folderincludes: + self._msg("folderincludes= %s\n" % folderincludes) + nametrans = repository.getconf('nametrans', None) + if nametrans: + self._msg("nametrans= %s\n" % nametrans) + + folders = repository.getfolders() + foldernames = [(f.name, f.getvisiblename()) for f in folders] + folders = [] + for name, visiblename in foldernames: + if name == visiblename: folders.append(name) + else: folders.append("%s -> %s" % (name, visiblename)) + self._msg("Folderlist: %s\n" % str(folders)) + finally: + if conn: #release any existing IMAP connection + repository.imapserver.close() ################################################## Threads - def getThreadDebugLog(s, thread): - if s.debugmessages.has_key(thread): + def getThreadDebugLog(self, thread): + if self.debugmessages.has_key(thread): message = "\nLast %d debug messages logged for %s prior to exception:\n"\ - % (len(s.debugmessages[thread]), thread.getName()) - message += "\n".join(s.debugmessages[thread]) + % (len(self.debugmessages[thread]), thread.getName()) + message += "\n".join(self.debugmessages[thread]) else: message = "\nNo debug messages were logged for %s." % \ thread.getName() return message - def delThreadDebugLog(s, thread): - if s.debugmessages.has_key(thread): - del s.debugmessages[thread] + def delThreadDebugLog(self, thread): + if thread in self.debugmessages: + del self.debugmessages[thread] - def getThreadExceptionString(s, thread): + def getThreadExceptionString(self, thread): message = "Thread '%s' terminated with exception:\n%s" % \ - (thread.getName(), thread.getExitStackTrace()) - message += "\n" + s.getThreadDebugLog(thread) + (thread.getName(), thread.exit_stacktrace) + message += "\n" + self.getThreadDebugLog(thread) return message - def threadException(s, thread): + def threadException(self, thread): """Called when a thread has terminated with an exception. The argument is the ExitNotifyThread that has so terminated.""" - s._msg(s.getThreadExceptionString(thread)) - s.delThreadDebugLog(thread) - s.terminate(100) + self.warn(self.getThreadExceptionString(thread)) + self.delThreadDebugLog(thread) + self.terminate(100) def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" #print any exceptions that have occurred over the run if not self.exc_queue.empty(): - self._msg("\nERROR: Exceptions occurred during the run!") + self.warn("ERROR: Exceptions occurred during the run!") while not self.exc_queue.empty(): msg, exc, exc_traceback = self.exc_queue.get() if msg: - self._msg("ERROR: %s\n %s" % (msg, exc)) + self.warn("ERROR: %s\n %s" % (msg, exc)) else: - self._msg("ERROR: %s" % (exc)) + self.warn("ERROR: %s" % (exc)) if exc_traceback: - self._msg("\nTraceback:\n%s" %"".join( + self.warn("\nTraceback:\n%s" %"".join( traceback.format_tb(exc_traceback))) if errormsg and errortitle: - sys.stderr.write('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) + self.warn('ERROR: %s\n\n%s\n'%(errortitle, errormsg)) elif errormsg: - sys.stderr.write('%s\n' % errormsg) + self.warn('%s\n' % errormsg) sys.exit(exitstatus) - def threadExited(s, thread): + def threadExited(self, thread): """Called when a thread has exited normally. Many UIs will just ignore this.""" - s.delThreadDebugLog(thread) - s.unregisterthread(thread) + self.delThreadDebugLog(thread) + self.unregisterthread(thread) ################################################## Hooks - def callhook(s, msg): - if s.verbose >= 0: - s._msg(msg) + def callhook(self, msg): + self.info(msg) ################################################## Other - def sleep(s, sleepsecs, account): + def sleep(self, sleepsecs, account): """This function does not actually output anything, but handles the overall sleep, dealing with updates as necessary. It will, however, call sleeping() which DOES output something. @@ -399,12 +471,12 @@ class UIBase: if account.get_abort_event(): abortsleep = True else: - abortsleep = s.sleeping(10, sleepsecs) - sleepsecs -= 10 - s.sleeping(0, 0) # Done sleeping. + abortsleep = self.sleeping(10, sleepsecs) + sleepsecs -= 10 + self.sleeping(0, 0) # Done sleeping. return abortsleep - def sleeping(s, sleepsecs, remainingsecs): + def sleeping(self, sleepsecs, remainingsecs): """Sleep for sleepsecs, display remainingsecs to go. Does nothing if sleepsecs <= 0. @@ -416,6 +488,7 @@ class UIBase: """ if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: - s._msg("Next refresh in %.1f minutes" % (remainingsecs/60.0)) + self.logger.info("Next refresh in %.1f minutes" % ( + remainingsecs/60.0)) time.sleep(sleepsecs) return 0 diff --git a/offlineimap/ui/__init__.py b/offlineimap/ui/__init__.py index 102cbfd..6ddbaf6 100644 --- a/offlineimap/ui/__init__.py +++ b/offlineimap/ui/__init__.py @@ -1,5 +1,5 @@ # UI module -# Copyright (C) 2010 Sebastian Spaeth +# Copyright (C) 2010-2011 Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by