From 854eaf3055b84a6c0072365c0bd3fab35c44d6b9 Mon Sep 17 00:00:00 2001 From: jgoerzen Date: Sat, 4 Jan 2003 05:57:46 +0100 Subject: [PATCH] /offlineimap/head: changeset 297 Changed to a more account-centric behavior. The refresh time is now a per-account variable. Implemented new account classes. User interfaces must now be updated to take advantage of this. --- offlineimap/head/debian/changelog | 443 ++++++++++++++++++ offlineimap/head/offlineimap/CustomConfig.py | 46 ++ offlineimap/head/offlineimap/accounts.py | 211 +++++++++ offlineimap/head/offlineimap/init.py | 31 +- offlineimap/head/offlineimap/mbnames.py | 3 +- offlineimap/head/offlineimap/syncmaster.py | 196 +------- offlineimap/head/offlineimap/test.py | 17 + offlineimap/head/offlineimap/threadutil.py | 36 ++ .../head/offlineimap/ui/Noninteractive.py | 2 +- offlineimap/head/offlineimap/ui/UIBase.py | 4 + offlineimap/head/offlineimap/ui/detector.py | 4 +- 11 files changed, 789 insertions(+), 204 deletions(-) create mode 100644 offlineimap/head/debian/changelog create mode 100644 offlineimap/head/offlineimap/CustomConfig.py create mode 100644 offlineimap/head/offlineimap/accounts.py create mode 100644 offlineimap/head/offlineimap/test.py diff --git a/offlineimap/head/debian/changelog b/offlineimap/head/debian/changelog new file mode 100644 index 0000000..642b59f --- /dev/null +++ b/offlineimap/head/debian/changelog @@ -0,0 +1,443 @@ +offlineimap (3.99.6) unstable; urgency=low + + * Beginnings of work to make it work with a threaded Tcl/Tk Tkinter. + * Now properly handles folder names that contain parenthesis. Used + patch from Kyler Laird in + http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=173895. + Closes: #173895. + * Changed to a more account-centric behavior. The refresh time is now + a per-account variable. Implemented new account classes. User + interfaces must now be updated to take advantage of this. + + -- John Goerzen Thu, 2 Jan 2003 13:59:44 -0600 + +offlineimap (3.99.5) unstable; urgency=low + + * Added ability to disable expunging on the server. + * Fixed infinite loop with preauth. Closes: #169514, #171485. + + -- John Goerzen Tue, 03 Dec 2002 06:22:37 -0600 + +offlineimap (3.99.4) unstable; urgency=low + + * Fixed setup.py installation instructions. + * Added more debugging to the CRAM-MD5 authentication module. + * CRAM-MD5 *really* fixed this time. Thanks to MJ for the patch. + * Adding missing import os to imapserver.py. Thanks to John Wiegley + for catching that. + + -- John Goerzen Tue, 5 Nov 2002 08:06:45 -0600 + +offlineimap (3.99.3) unstable; urgency=low + + * Moved password promting into imapserver.py. Passwords are now asked + for on-demand and typos will no longer crash the program (the user + will be re-prompted). Closes: #162672. + * Falls back to plain-text auth if CRAM-MD5 fails. Fixes [complete.org #42] + * Fixed CRAM-MD5 auth so it actually works now. + + -- John Goerzen Mon, 4 Nov 2002 06:16:11 -0600 + +offlineimap (3.99.2) unstable; urgency=low + + * Further attempts to fix imapsplit problems. + * When an exception occurs, OfflineIMAP will attempt to print the last + 50 debug messages, whether or not debugging was enabled for this + session. This way, even unexpected and non-repeatable errors stand + a chance of getting a more detailed log. + * Handle uidvalidity file in an atomic fashion. CLoses: #165600. + * Supports CRAM-MD5 authentication. Fixes [complete.org #38], and for + Debian, Closes: #154165. + * Noted CRAM-MD5 support in the "CONFORMING TO" section of the manual. + * Certain servers may not always return the UID flag for new messages. + This causes an OfflineIMAP exception, though rerunning will fix it. + Now, OfflineIMAP will detect the condition and ignore the offending + messages (without an exception) until the next sync. + + -- John Goerzen Sat, 02 Nov 2002 10:23:10 -0600 + +offlineimap (3.99.1) unstable; urgency=low + + * Fixed some syntax errors in imaputil.py + * Fixed a syntax error with mbnames + + -- John Goerzen Wed, 9 Oct 2002 19:34:37 -0500 + +offlineimap (3.99.0) unstable; urgency=low + + * The next few releases are adding features and reorganizing + code in preparation for 4.0.0. + * imaputil.py now logs information with IMAP debugging is enabled. + * Added folderfilter capability to mbnames recorder. You can now omit + specified folders from the mbnames output. + * Added a workaround to imaputil.py to deal with a bug in imaplib.py's + tuple when a response contains a literal in certain cases. + * Split out the code in bin/offlineimap into offlineimap/init.py. + Retaining bin/offlineimap as a skeletal piece only. Contains + about three lines of code now. This will make many things + easier, including debugging. + * Added library version check to bin/offlineimap and + offlineimap/init.py. + * Moved __main__.ui to functions in UIBase: getglobalui() and + setglobalui(). + * Added license comments to some source files that were missing them. + * Moved some code from offlineimap/init.py to new file + offlineimap/syncmaster.py to help dileneate between code that + performs different functions. + * Moved threadexited from offlineimap/init.py to + offlineimap/threadutil.py. + * offlineimap.py is back to ease the use of OfflineIMAP in single-user + installations. + + -- John Goerzen Mon, 07 Oct 2002 05:08:08 -0500 + +offlineimap (3.2.8) unstable; urgency=low + + * Added a work-around for some IMAP servers that respond poorly + to LIST "" "". It will now do LIST "" "*", for them only. + + -- John Goerzen Mon, 30 Sep 2002 10:48:01 -0500 + +offlineimap (3.2.7) unstable; urgency=low + + * Moved executable to bin/offlineimap. This will allow setup.py to + properly install it as offlineimap instead of offlineimap.py. + * Made sure executables use /usr/bin/env in bangpath. + * Font size for Blinkenlights interface is now configurable. + + -- John Goerzen Thu, 19 Sep 2002 06:46:56 -0500 + +offlineimap (3.2.6) unstable; urgency=low + + * Changed indentation in debian/control. Closes: #156327. + * Removed calls to folder object deletions. None have been implemented + anyway. + * folder/Maildir.py: unlink throws OSError, not IOError; fixed. + Now handles message deleting race condition properly. + Closes: #154497. + + -- John Goerzen Fri, 16 Aug 2002 17:43:19 -0500 + +offlineimap (3.2.5) unstable; urgency=low + + * Now handles uploading messages without Message-Id headers. + Closes: #156022. + * Applied patch from Tommi Virtanen that adds two new config file + options: pythonfile and foldersort. Fixes [complete.org #29], and + for Debian, Closes: #155637. + * Added documentation for the above features. + * Even more resiliant in the face of invalid Date headers. Closes: #155994. + + -- John Goerzen Fri, 9 Aug 2002 17:54:01 -0500 + +offlineimap (3.2.4) unstable; urgency=low + + * When using nested folders, the Maildir repository handler now properly + deals with folders that are nested inside "noselect" folders -- ones + that do not actually contain messages and are not provided in the + server's LIST response. Fixes [complete.org #32] and, for Debian, + Closes: #155866. + + -- John Goerzen Thu, 8 Aug 2002 17:54:44 -0500 + +offlineimap (3.2.3) unstable; urgency=low + + * -d now takes a parameter: imap or maildir (or both) to specify + what type of debugging to do. + + -- John Goerzen Thu, 8 Aug 2002 10:02:36 -0500 + +offlineimap (3.2.2) unstable; urgency=low + + * Updated manual to show new Gray color. + * Scrolling behavior is better now; sometimes, with fast-scrolling text, + the log would stop scrolling. + * Better handling of read-only folders. We will now warn if there is + a change, but not propogate it. New config variable ignore-readonly + can suppress the warnings. This fixes [complete.org #10] and, + for Debian, Closes: #154769. + * If a given Maildir folder is new, remove the associated local status + cache file, if any. That way, there will not be any chance of + propogating hordes of deletes and adds based on old status data. + * Added support for /-separated Maildirs -- that is, hierarchical + Maildir trees. Fixes [complete.org #28] and, for Debian, + Closes: #155460. + * Preventitive security: Folder names may not contain ./ or start with /. + + -- John Goerzen Wed, 07 Aug 2002 20:22:25 -0500 + +offlineimap (3.2.1) unstable; urgency=low + + * There is a new "connecting" event that will appear in all but the + Quiet UIs. It has a gray color in Blinkenlights. This event indicates + the the program is connecting to a remote server. + * Blinkenlights UI log window is now scrolled and has a new + config file option "bufferlines" to specify the size of the scroll + buffer. + * The Blinkenlights window is now non-resizable when the log is disabled. + When the log is enabled, the window is resizable, and the changes in + size are reflected in the log widget. Therefore, the Bigger Log + and Smaller Log items can disappear, and the Log menu now becomes + a Show Log or a Hide Log menu option. No sub-menus necessary anymore. + This presents a much cleaner feel, more intuitive operation, and + faster navigation. + * Fix for account name interpolation in dot warning from 3.2.0 from + Martijn Pieters. + * Backed out check for . in account names for now. Will put it back in + when we have a consensus on what exactly to do. Doubt that anyone + has a foldername that would conflict with Blinkenlights anyway. + * Fix reading the ui.Tk.Blinkenlights bufferlines option. + + -- John Goerzen Wed, 24 Jul 2002 17:04:04 -0500 + +offlineimap (3.2.0) unstable; urgency=low + + * New BLINKENLIGHTS interface! Mesmerising, isn't it? + * New ui.Tk.Blinkenlights section in offlineimap.conf. + * New USER INTERFACES section in the manual. + * TTYUI isusable() now checks to see if stdout and stdin are TTYs. + * Added build-dependency on python2.2-dev. Closes: #154167. + + -- John Goerzen Wed, 24 Jul 2002 17:53:20 -0500 + +offlineimap (3.1.1) unstable; urgency=low + + * Modified imaputil.py and folder/Maildir.py to run faster. Eliminated + many regular expressions; pre-compiled many others. + * Fixed threadutil's exitnotifyloop to always handle threads in the order + they exited, rather than sometimes in the inverse order. This way, + make sure to handle thread's exception messages before a thread exited. + * Replaced imaplib.py's braindead readline() with a more efficient one. + * More optimizations to imaputil and folders for faster operation. + * These optimizations, all together, have resulted in OfflineIMAP + using approximately half the CPU time of previous versions, fixing + [complete.org #6], and... Closes: #153503. + + -- John Goerzen Wed, 24 Jul 2002 06:53:16 -0500 + +offlineimap (3.1.0) unstable; urgency=low + + * When uploading messages from a Maildir, now convert \r\n to \n in case + the message is stored weirdly. That way, everything is uniform. + Fixes [complete.org #11]. + * Manual: added UW IMAPD example with references from docwhat@gerf.org. + * New UI modules: Noninteractive.Basic and Noninteractive.Quiet. + Fixes [complete.org #14]. + * Added per-thread profiling support to aid in debugging. + + -- John Goerzen Sun, 21 Jul 2002 16:09:42 -0500 + +offlineimap (3.0.3) unstable; urgency=low + + * No longer throws an exception when updating messages with strange + Date headers; will just set IMAP Internaldate to the current date. + Closes: #153425. + * No longer doubles-up reference names for mailboxes. Closes: #153515. + * Noted new bug-tracking system in manual and rebuilt manual files. + * Now stores incoming messages in 'cur' instead of 'new' if they have + the S flag. Closes: #152482. + + -- John Goerzen Sun, 21 Jul 2002 13:46:13 -0500 + +offlineimap (3.0.2) unstable; urgency=low + + * Fixed mailbox name recorder to use localfolder.getvisiblename() rather + than remotefolder.getvisiblename() + * Fixed remotepassfile option. Closes: #153119. Used 1-line patch from + Tommi Virtanen. + * Now handles cases of not being able to get UID for an uploaded message + more gracefully. This could occur if the server doesn't support + SEARCH, can't find the message ID, or finds multiple message IDs. + Closes: #153241. + * Now source is in Subversion. Make version.py log the Subversion + revision number. + + -- John Goerzen Wed, 15 Jul 2002 06:43:36 -0500 + +offlineimap (3.0.1) unstable; urgency=low + + * Detabified the source. + * Added UI list to the manpage. + * Added -o (run only once) option with patch sent in by Martijn Pieters. + * Optimized folder/IMAP.py addmessagesflags() with new listjoin() in + imaputil. Now, send the server 1:5,7 instead of 1,2,3,4,5,7. + * Made folder/Maildir.py/deletemessage() more tolerant if a message + asked to be deleted already has been. + * In Base.py/copymessageto(), no longer bother calling getmessage() + unless a folder's storemessages() returns true. This will also help + with syncing to LocalStatus if the user deleted messages in the + Maildir since the cachemessagelist() was called. + + -- John Goerzen Fri, 12 Jul 2002 07:28:24 -0500 + +offlineimap (3.0.0) unstable; urgency=low + + * Introduced a new graphical user interface written with Tkinter. + It features a nice view of multi-threaded displays. + * The TTY user interface now also displays thread names. + * Program-wide, new threads are given descriptive names to aid in + debugging and status messages. + * Added new module offlineimap/ui/detect.py that is used to detect + which user interface to select for a given session. Its operation + is governed by the ui config option and the -u command-line option. + * Made IMAP folder addmessagesflags() resiliant to a server refusing + to return a full set of new message flags. Closes: #152587. + * Completely rewrote documentation. OfflineIMAP now has an + exhaustive manpage, which is really a manual. It is also shipped + in plain text, HTML, PDF, and PostScript formats. + * New command-line options: + -1 to force no multi-threaded operation + -u to force a particular UI + -a to specify which accounts to sync + -h to print help + -c to specify an alternate config file + * Added a workaround for UW IMAP problem wherein the server loses + uidvalidity whenever a folder is emptied. Now, the program + will not consider it a problem if uidvalidity is lost when a folder + and the local status cache are both completely empty, since we do + not really need to preserve uidvalidity in that case anyway. + Closes: #152079. + + -- John Goerzen Thu, 11 Jul 2002 22:35:42 -0500 + +offlineimap (2.0.8) unstable; urgency=low + + * Modified the IMAP folder to use SELECT rather than STATUS more often. + Makes the code more robust; handles better with read-only folders; + and runs faster, especially for non-threaded useres, where it + may eliminate up to 2-3 commands per folder. + * Made sure IMAP folder savemessage() does a select. This was a possible + bug. + * Modified Maildir folder to unlink messages with T flag in + cachemessagelist() + * My own box now syncs in 3 seconds. + * Optimized acquireconnection() to try to give a thread back the + connection that it last used, if possible. + + -- John Goerzen Tue, 9 Jul 2002 23:29:30 -0500 + +offlineimap (2.0.7) unstable; urgency=low + + * Fixed imaplib.py to work better with read-only folders. + + -- John Goerzen Tue, 9 Jul 2002 20:24:21 -0500 + +offlineimap (2.0.6) unstable; urgency=low + + * Added support for holdconnectionopen and keepalive. This feature + allows for an IMAP server connection(s) to be held open until + the next sync process, permitting faster restart times. + * Another try at read-only folder support. This is nasty because I + have no way to test it and imaplib's read-only support is weird. + * Closing out old bug; fixed in 1.0.2. Closes: #150803. + * Optimized algorithm so that a SELECT is never issued for folders + that contain no messages. + + -- John Goerzen Tue, 9 Jul 2002 20:05:24 -0500 + +offlineimap (2.0.5) unstable; urgency=low + + * Fixed a folderfilter example. Partially fixes #152079. + * Added folderincludes capability. Partially fixes #152079. + * More fixes for read-only folders. + + -- John Goerzen Fri, 5 Jul 2002 09:21:52 -0500 + +offlineimap (2.0.4) unstable; urgency=low + + * Made OfflineIMAP at least rudimentarily compatible with read-only + folders. It will still fail if they get modified locally, though. + * Flags are handled case-insensitively. Closes: #151993. + + -- John Goerzen Fri, 5 Jul 2002 09:10:29 -0500 + +offlineimap (2.0.3) unstable; urgency=low + + * Added support for specifying references. Closes: #151960. + * Added -d command-line option to enable imaplib debugging. + + -- John Goerzen Thu, 4 Jul 2002 20:39:29 -0500 + +offlineimap (2.0.2) unstable; urgency=low + + * Added support for remotepassfile. Closes: #151943. + * Added support for preauth tunnels. + + -- John Goerzen Thu, 4 Jul 2002 14:46:23 -0500 + +offlineimap (2.0.1) unstable; urgency=low + + * Fixed a bug with not properly propogating foldersep changes. + Now, local folders and status folders properly use the foldersep + mechanism. This corrects a problem with Exchange servers. + * Wrote a major new thread montiring subsystem, defined a new + ExitNotifyThread. Handling of Ctrl-C now occurs within 1 second + rather than after the whole program terminates. Exceptions that + occur in a thread are now caught by the main thread and marshalled + over into the UI side of things for dispatch. The entire program will + now abort when one thread dies with an exception. + + -- John Goerzen Thu, 4 Jul 2002 09:07:06 -0500 + +offlineimap (2.0.0) unstable; urgency=low + + * This code is now multithreaded. New config file options control the + behavior. This can make synchronizing several times faster. + * Fixed the STATUS call to be compatible with Exchange. + * Added the ability to exclude folders. + * If upgrading from 1.0.x, you will need to add maxsyncaccounts to the + general section and maxconnections to each account sections. + There is also a new folderfilter option. + You can find examples of all of these in the new offlineimap.conf + example file packaged with the distribution. + * The Debian package now properly installs the example offlineimap.conf + file. + * There is a new mailing list available. To join, send SUBSCRIBE + to offlineimap-request@complete.org. The posting address is + offlineimap@complete.org. + + -- John Goerzen Wed, 3 Jul 2002 19:21:32 -0500 + +offlineimap (1.0.4) unstable; urgency=low + + * Deletion of more than one message has been optimized. This could make + deleting large numbers of messages far faster -- several orders of + magnitude. + * Moved more sleep code into ui layer. Fancier sleep actions are now + possible. Better handling of Ctrl-C in TTY handler. + + -- John Goerzen Tue, 2 Jul 2002 19:16:04 -0500 + +offlineimap (1.0.3) unstable; urgency=low + + * Fixed a bug when a message was deleted on the IMAP side and modified + on the local side. + + -- John Goerzen Mon, 24 Jun 2002 19:08:21 -0500 + +offlineimap (1.0.2) unstable; urgency=low + + * Made sure that LocalStatus does writing atomically. If the program + is interrupted during save(), there will always be a complete copy of + either the old or the new data. + + -- John Goerzen Mon, 24 Jun 2002 06:57:28 -0500 + +offlineimap (1.0.1) unstable; urgency=low + + * Fixed a bug with writing messages to some IMAP servers. Turns + out we need to issue CHECK between APPEND and SEARCH for some. + Thanks to Donovan Lange for reporting this bug and helping track it + down. + + -- John Goerzen Fri, 21 Jun 2002 22:03:12 -0500 + +offlineimap (1.0.0) unstable; urgency=low + + * Initial Release. Closes: #150571. + + -- John Goerzen Fri, 21 Jun 2002 18:54:56 -0500 + +Local variables: +mode: debian-changelog +End: diff --git a/offlineimap/head/offlineimap/CustomConfig.py b/offlineimap/head/offlineimap/CustomConfig.py new file mode 100644 index 0000000..fbfd65f --- /dev/null +++ b/offlineimap/head/offlineimap/CustomConfig.py @@ -0,0 +1,46 @@ +# Copyright (C) 2003 John Goerzen +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +from ConfigParser import ConfigParser +from offlineimap.localeval import LocalEval +import os + +class CustomConfigParser(ConfigParser): + def getdefault(self, section, option, default, *args, **kwargs): + """Same as config.get, but returns the "default" option if there + is no such option specified.""" + if self.has_option(section, option): + return apply(self.get, [section, option] + list(args), kwargs) + else: + return default + + def getmetadatadir(self): + metadatadir = os.path.expanduser(self.getdefault("general", "metadata", "~/.offlineimap")) + if not os.path.exists(metadatadir): + os.mkdir(metadatadir, 0700) + return metadatadir + + def getlocaleval(self): + if self.has_option("general", "pythonfile"): + path = os.path.expanduser(self.get("general", "pythonfile")) + else: + path = None + return LocalEval(path) + + def getaccountlist(self): + return [x for x in self.sections() if x != 'general'] + diff --git a/offlineimap/head/offlineimap/accounts.py b/offlineimap/head/offlineimap/accounts.py new file mode 100644 index 0000000..40a1611 --- /dev/null +++ b/offlineimap/head/offlineimap/accounts.py @@ -0,0 +1,211 @@ +# Copyright (C) 2003 John Goerzen +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +from offlineimap import imapserver, repository, threadutil +from offlineimap.ui import UIBase +from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread +from threading import Event +import os + +mailboxes = [] + +class Account: + def __init__(self, config, name): + self.config = config + self.name = name + self.metadatadir = config.getmetadatadir() + self.localeval = config.getlocaleval() + self.server = imapserver.ConfigedIMAPServer(config, self.name) + self.ui = UIBase.getglobalui() + if self.config.has_option(self.name, 'autorefresh'): + self.refreshperiod = self.config.getint(self.name, 'autorefresh') + else: + self.refreshperiod = None + self.hold = self.config.has_option(self.name, 'holdconnectionopen') \ + and self.config.getboolean(self.name, 'holdconnectionopen') + if self.config.has_option(self.name, 'keepalive'): + self.keepalive = self.config.getint(self.name, 'keepalive') + else: + self.keepalive = None + + def getconf(self, option, default = None): + if default != None: + return self.config.get(self.name, option) + else: + return self.config.getdefault(self.name, option, + default) + + def sleeper(self): + """Sleep handler. Returns same value as UIBase.sleep: + 0 if timeout expired, 1 if there was a request to cancel the timer, + and 2 if there is a request to abort the program. + + Also, returns 100 if configured to not sleep at all.""" + + if not self.refreshperiod: + return 100 + refreshperiod = self.refreshperiod * 60 + if self.keepalive: + kaevent = Event() + kathread = ExitNotifyThread(target = self.server.keepalive, + name = "Keep alive " + self.name, + args = (self.keepalive, kaevent)) + kathread.setDaemon(1) + kathread.start() + sleepresult = self.ui.sleep(refreshperiod) + if sleepresult == 2: + # Cancel keep-alive, but don't bother terminating threads + if self.keepalive: + kaevent.set() + return sleepresult + else: + # Cancel keep-alive and wait for thread to terminate. + if self.keepalive: + kaevent.set() + kathread.join() + return sleepresult + +class AccountSynchronizationMixin: + def syncrunner(self): + self.ui.acct(self.name) + if not self.refreshperiod: + self.sync() + self.ui.acctdone(self.name) + return + looping = 1 + while looping: + self.sync() + looping = self.sleeper() != 2 + self.ui.acctdone(self.name) + + def sync(self): + # We don't need an account lock because syncitall() goes through + # each account once, then waits for all to finish. + try: + accountmetadata = os.path.join(self.metadatadir, self.name) + if not os.path.exists(accountmetadata): + os.mkdir(accountmetadata, 0700) + + remoterepos = repository.IMAP.IMAPRepository(self.config, + self.localeval, + self.name, + self.server) + + # Connect to the Maildirs. + localrepos = repository.Maildir.MaildirRepository(os.path.expanduser(self.config.get(self.name, "localfolders")), self.name, self.config) + + # Connect to the local cache. + statusrepos = repository.LocalStatus.LocalStatusRepository(accountmetadata) + + self.ui.syncfolders(remoterepos, localrepos) + remoterepos.syncfoldersto(localrepos) + + folderthreads = [] + for remotefolder in remoterepos.getfolders(): + thread = InstanceLimitedThread(\ + instancename = 'FOLDER_' + self.name, + target = syncfolder, + name = "Folder sync %s[%s]" % \ + (self.name, remotefolder.getvisiblename()), + args = (self.name, remoterepos, remotefolder, localrepos, + statusrepos)) + thread.setDaemon(1) + thread.start() + folderthreads.append(thread) + threadutil.threadsreset(folderthreads) + if not self.hold: + server.close() + finally: + pass + +class SyncableAccount(Account, AccountSynchronizationMixin): + pass + +def syncfolder(accountname, remoterepos, remotefolder, localrepos, + statusrepos): + global mailboxes + ui = UIBase.getglobalui() + # Load local folder. + localfolder = localrepos.\ + getfolder(remotefolder.getvisiblename().\ + replace(remoterepos.getsep(), localrepos.getsep())) + # Write the mailboxes + mailboxes.append({'accountname': accountname, + 'foldername': localfolder.getvisiblename()}) + # Load local folder + ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) + ui.loadmessagelist(localrepos, localfolder) + localfolder.cachemessagelist() + ui.messagelistloaded(localrepos, localfolder, len(localfolder.getmessagelist().keys())) + + + # Load status folder. + statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ + replace(remoterepos.getsep(), + statusrepos.getsep())) + if localfolder.getuidvalidity() == None: + # This is a new folder, so delete the status cache to be sure + # we don't have a conflict. + statusfolder.deletemessagelist() + + statusfolder.cachemessagelist() + + + # If either the local or the status folder has messages and + # there is a UID validity problem, warn and abort. + # If there are no messages, UW IMAPd loses UIDVALIDITY. + # But we don't really need it if both local folders are empty. + # So, in that case, save it off. + if (len(localfolder.getmessagelist()) or \ + len(statusfolder.getmessagelist())) and \ + not localfolder.isuidvalidityok(remotefolder): + ui.validityproblem(remotefolder) + return + else: + localfolder.saveuidvalidity(remotefolder.getuidvalidity()) + + # Load remote folder. + ui.loadmessagelist(remoterepos, remotefolder) + remotefolder.cachemessagelist() + ui.messagelistloaded(remoterepos, remotefolder, + len(remotefolder.getmessagelist().keys())) + + + # + + if not statusfolder.isnewfolder(): + # Delete local copies of remote messages. This way, + # if a message's flag is modified locally but it has been + # deleted remotely, we'll delete it locally. Otherwise, we + # try to modify a deleted message's flags! This step + # need only be taken if a statusfolder is present; otherwise, + # there is no action taken *to* the remote repository. + + remotefolder.syncmessagesto_delete(localfolder, [localfolder, + statusfolder]) + ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) + localfolder.syncmessagesto(statusfolder, [remotefolder, statusfolder]) + + # Synchronize remote changes. + ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) + remotefolder.syncmessagesto(localfolder) + + # Make sure the status folder is up-to-date. + ui.syncingmessages(localrepos, localfolder, statusrepos, statusfolder) + localfolder.syncmessagesto(statusfolder) + statusfolder.save() + diff --git a/offlineimap/head/offlineimap/init.py b/offlineimap/head/offlineimap/init.py index 7e4d18f..bb3d8e9 100644 --- a/offlineimap/head/offlineimap/init.py +++ b/offlineimap/head/offlineimap/init.py @@ -21,7 +21,7 @@ from offlineimap.localeval import LocalEval from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread from offlineimap.ui import UIBase import re, os, os.path, offlineimap, sys -from ConfigParser import ConfigParser +from offlineimap.CustomConfig import CustomConfigParser from threading import * from getopt import getopt @@ -51,20 +51,14 @@ def startup(versionno): threadutil.setprofiledir(profiledir) sys.stderr.write("WARNING: profile mode engaged;\nPotentially large data will be created in " + profiledir + "\n") - config = ConfigParser() + config = CustomConfigParser() if not os.path.exists(configfilename): sys.stderr.write(" *** Config file %s does not exist; aborting!\n" % configfilename) sys.exit(1) config.read(configfilename) - if config.has_option("general", "pythonfile"): - path=os.path.expanduser(config.get("general", "pythonfile")) - else: - path=None - localeval = LocalEval(path) - - ui = offlineimap.ui.detector.findUI(config, localeval, options.get('-u')) + ui = offlineimap.ui.detector.findUI(config, options.get('-u')) ui.init_banner() UIBase.setglobalui(ui) @@ -74,12 +68,9 @@ def startup(versionno): if debugtype == 'imap': imaplib.Debug = 5 - if '-o' in options and config.has_option("general", "autorefresh"): - config.remove_option("general", "autorefresh") - - metadatadir = os.path.expanduser(config.get("general", "metadata")) - if not os.path.exists(metadatadir): - os.mkdir(metadatadir, 0700) + if '-o' in options: + for section in config.getaccountlist(): + config.remove_option(section, "autorefresh") accounts = config.get("general", "accounts") if '-a' in options: @@ -105,17 +96,11 @@ def startup(versionno): threadutil.initInstanceLimit(instancename, config.getint(account, "maxconnections")) - mailboxes = [] - servers = {} - threadutil.initexitnotify() - t = ExitNotifyThread(target=syncmaster.sync_with_timer, + t = ExitNotifyThread(target=syncmaster.syncitall, name='Sync Runner', kwargs = {'accounts': accounts, - 'metadatadir': metadatadir, - 'servers': servers, - 'config': config, - 'localeval': localeval}) + 'config': config}) t.setDaemon(1) t.start() try: diff --git a/offlineimap/head/offlineimap/mbnames.py b/offlineimap/head/offlineimap/mbnames.py index 48c2f30..0d510be 100644 --- a/offlineimap/head/offlineimap/mbnames.py +++ b/offlineimap/head/offlineimap/mbnames.py @@ -19,9 +19,10 @@ import os.path import re # for folderfilter -def genmbnames(config, localeval, boxlist): +def genmbnames(config, boxlist): """Takes a configparser object and a boxlist, which is a list of hashes containing 'accountname' and 'foldername' keys.""" + localeval = config.getlocaleval() if not config.getboolean("mbnames", "enabled"): return file = open(os.path.expanduser(config.get("mbnames", "filename")), "wt") diff --git a/offlineimap/head/offlineimap/syncmaster.py b/offlineimap/head/offlineimap/syncmaster.py index 7911977..8aa65dc 100644 --- a/offlineimap/head/offlineimap/syncmaster.py +++ b/offlineimap/head/offlineimap/syncmaster.py @@ -18,187 +18,29 @@ from offlineimap import imaplib, imapserver, repository, folder, mbnames, threadutil, version from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread +import offlineimap.accounts +from offlineimap.accounts import SyncableAccount from offlineimap.ui import UIBase import re, os, os.path, offlineimap, sys from ConfigParser import ConfigParser from threading import * -def syncaccount(accountname, metadatadir, servers, config, - localeval, *args): - ui = UIBase.getglobalui() - # We don't need an account lock because syncitall() goes through - # each account once, then waits for all to finish. - try: - ui.acct(accountname) - accountmetadata = os.path.join(metadatadir, accountname) - if not os.path.exists(accountmetadata): - os.mkdir(accountmetadata, 0700) - - server = None - if accountname in servers: - server = servers[accountname] - else: - server = imapserver.ConfigedIMAPServer(config, accountname) - servers[accountname] = server - - remoterepos = repository.IMAP.IMAPRepository(config, localeval, accountname, server) - - # Connect to the Maildirs. - localrepos = repository.Maildir.MaildirRepository(os.path.expanduser(config.get(accountname, "localfolders")), accountname, config) - - # Connect to the local cache. - statusrepos = repository.LocalStatus.LocalStatusRepository(accountmetadata) - - ui.syncfolders(remoterepos, localrepos) - remoterepos.syncfoldersto(localrepos) - ui.acct(accountname) - - folderthreads = [] - for remotefolder in remoterepos.getfolders(): - thread = InstanceLimitedThread(\ - instancename = 'FOLDER_' + accountname, - target = syncfolder, - name = "Folder sync %s[%s]" % \ - (accountname, remotefolder.getvisiblename()), - args = (accountname, remoterepos, remotefolder, localrepos, - statusrepos)) - thread.setDaemon(1) - thread.start() - folderthreads.append(thread) - threadutil.threadsreset(folderthreads) - if not (config.has_option(accountname, 'holdconnectionopen') and \ - config.getboolean(accountname, 'holdconnectionopen')): - server.close() - finally: - pass - -def syncfolder(accountname, remoterepos, remotefolder, localrepos, - statusrepos): - ui = UIBase.getglobalui() - # Load local folder. - localfolder = localrepos.\ - getfolder(remotefolder.getvisiblename().\ - replace(remoterepos.getsep(), localrepos.getsep())) - # Write the mailboxes - mailboxes.append({'accountname': accountname, - 'foldername': localfolder.getvisiblename()}) - # Load local folder - ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) - ui.loadmessagelist(localrepos, localfolder) - localfolder.cachemessagelist() - ui.messagelistloaded(localrepos, localfolder, len(localfolder.getmessagelist().keys())) - - - # Load status folder. - statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ - replace(remoterepos.getsep(), - statusrepos.getsep())) - if localfolder.getuidvalidity() == None: - # This is a new folder, so delete the status cache to be sure - # we don't have a conflict. - statusfolder.deletemessagelist() - - statusfolder.cachemessagelist() - +def syncaccount(threads, config, accountname): + account = SyncableAccount(config, accountname) + thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT', + target = account.syncrunner, + name = "Account sync %s" % accountname) + thread.setDaemon(1) + thread.start() + threads.add(thread) - # If either the local or the status folder has messages and - # there is a UID validity problem, warn and abort. - # If there are no messages, UW IMAPd loses UIDVALIDITY. - # But we don't really need it if both local folders are empty. - # So, in that case, save it off. - if (len(localfolder.getmessagelist()) or \ - len(statusfolder.getmessagelist())) and \ - not localfolder.isuidvalidityok(remotefolder): - ui.validityproblem(remotefolder) - return - else: - localfolder.saveuidvalidity(remotefolder.getuidvalidity()) - - # Load remote folder. - ui.loadmessagelist(remoterepos, remotefolder) - remotefolder.cachemessagelist() - ui.messagelistloaded(remoterepos, remotefolder, - len(remotefolder.getmessagelist().keys())) - - - # - - if not statusfolder.isnewfolder(): - # Delete local copies of remote messages. This way, - # if a message's flag is modified locally but it has been - # deleted remotely, we'll delete it locally. Otherwise, we - # try to modify a deleted message's flags! This step - # need only be taken if a statusfolder is present; otherwise, - # there is no action taken *to* the remote repository. - - remotefolder.syncmessagesto_delete(localfolder, [localfolder, - statusfolder]) - ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) - localfolder.syncmessagesto(statusfolder, [remotefolder, statusfolder]) - - # Synchronize remote changes. - ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) - remotefolder.syncmessagesto(localfolder) - - # Make sure the status folder is up-to-date. - ui.syncingmessages(localrepos, localfolder, statusrepos, statusfolder) - localfolder.syncmessagesto(statusfolder) - statusfolder.save() - - - -def syncitall(accounts, metadatadir, servers, config, localeval): - ui = UIBase.getglobalui() - global mailboxes - mailboxes = [] # Reset. - threads = [] - for accountname in accounts: - thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT', - target = syncaccount, - name = "Account sync %s" % accountname, - args = (accountname, metadatadir, - servers, config, - localeval)) - thread.setDaemon(1) - thread.start() - threads.append(thread) - # Wait for the threads to finish. - threadutil.threadsreset(threads) - mbnames.genmbnames(config, localeval, mailboxes) - -def sync_with_timer(accounts, metadatadir, servers, config, - localeval): - ui = UIBase.getglobalui() +def syncitall(accounts, config): currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE') - syncitall(accounts, metadatadir, servers, config, localeval) - if config.has_option('general', 'autorefresh'): - refreshperiod = config.getint('general', 'autorefresh') * 60 - while 1: - # Set up keep-alives. - kaevents = {} - kathreads = {} - for accountname in accounts: - if config.has_option(accountname, 'holdconnectionopen') and \ - config.getboolean(accountname, 'holdconnectionopen') and \ - config.has_option(accountname, 'keepalive'): - event = Event() - kaevents[accountname] = event - thread = ExitNotifyThread(target = servers[accountname].keepalive, - name = "Keep alive " + accountname, - args = (config.getint(accountname, 'keepalive'), event)) - thread.setDaemon(1) - thread.start() - kathreads[accountname] = thread - if ui.sleep(refreshperiod) == 2: - # Cancel keep-alives, but don't bother terminating threads - for event in kaevents.values(): - event.set() - break - else: - # Cancel keep-alives and wait for threads to terminate. - for event in kaevents.values(): - event.set() - for thread in kathreads.values(): - thread.join() - syncitall(accounts, metadatadir, servers, config, - localeval) + ui = UIBase.getglobalui() + threads = threadutil.threadlist() + offlineimap.accounts.mailboxes = [] # Reset. + for accountname in accounts: + syncaccount(threads, config, accountname) + # Wait for the threads to finish. + threads.reset() + mbnames.genmbnames(config, offlineimap.accounts.mailboxes) diff --git a/offlineimap/head/offlineimap/test.py b/offlineimap/head/offlineimap/test.py new file mode 100644 index 0000000..60b2f8f --- /dev/null +++ b/offlineimap/head/offlineimap/test.py @@ -0,0 +1,17 @@ +#!/usr/bin/python2.2 -i +import hmac +def getpassword(): + return 'tanstaaftanstaaf' + +def md5handler(response): + challenge = response.strip() + print "challenge is", challenge + msg = getpassword() + reply = hmac.new(challenge, msg) + retval = 'tim' + ' ' + \ + reply.hexdigest() + while len(retval) < 64: + retval += "\0" + + print "md5handler returning", retval + return retval diff --git a/offlineimap/head/offlineimap/threadutil.py b/offlineimap/head/offlineimap/threadutil.py index e536c05..48fab97 100644 --- a/offlineimap/head/offlineimap/threadutil.py +++ b/offlineimap/head/offlineimap/threadutil.py @@ -48,6 +48,42 @@ def threadsreset(threadlist): for thr in threadlist: thr.join() +class threadlist: + def __init__(self): + self.lock = Lock() + self.list = [] + + def add(self, thread): + self.lock.acquire() + try: + self.list.append(thread) + finally: + self.lock.release() + + def remove(self, thread): + self.lock.acquire() + try: + self.list.remove(thread) + finally: + self.lock.release() + + def pop(self): + self.lock.acquire() + try: + if not len(self.list): + return None + return self.list.pop() + finally: + self.lock.release() + + def reset(self): + while 1: + thread = self.pop() + if not thread: + return + thread.join() + + ###################################################################### # Exit-notify threads ###################################################################### diff --git a/offlineimap/head/offlineimap/ui/Noninteractive.py b/offlineimap/head/offlineimap/ui/Noninteractive.py index 65b6937..abda18d 100644 --- a/offlineimap/head/offlineimap/ui/Noninteractive.py +++ b/offlineimap/head/offlineimap/ui/Noninteractive.py @@ -20,7 +20,7 @@ import sys, time from UIBase import UIBase class Basic(UIBase): - def getpass(s, accountname, config): + def getpass(s, accountname, config, errmsg = None): raise NotImplementedError, "Prompting for a password is not supported in noninteractive mode." def _msg(s, msg): diff --git a/offlineimap/head/offlineimap/ui/UIBase.py b/offlineimap/head/offlineimap/ui/UIBase.py index d48b162..2b37532 100644 --- a/offlineimap/head/offlineimap/ui/UIBase.py +++ b/offlineimap/head/offlineimap/ui/UIBase.py @@ -140,6 +140,10 @@ class UIBase: if s.verbose >= 0: s._msg("***** Processing account %s" % accountname) + def acctdone(s, accountname): + if s.verbose >= 0: + s._msg("***** Finished processing account " + accountname) + def syncfolders(s, srcrepos, destrepos): if s.verbose >= 0: s._msg("Copying folder structure from %s to %s" % \ diff --git a/offlineimap/head/offlineimap/ui/detector.py b/offlineimap/head/offlineimap/ui/detector.py index f33552d..c561ada 100644 --- a/offlineimap/head/offlineimap/ui/detector.py +++ b/offlineimap/head/offlineimap/ui/detector.py @@ -19,7 +19,7 @@ import offlineimap.ui import sys -def findUI(config, localeval, chosenUI=None): +def findUI(config, chosenUI=None): uistrlist = ['Tk.Blinkenlights', 'Tk.VerboseUI', 'TTY.TTYUI', 'Noninteractive.Basic', 'Noninteractive.Quiet'] namespace={} @@ -34,7 +34,7 @@ def findUI(config, localeval, chosenUI=None): uistrlist = config.get("general", "ui").replace(" ", "").split(",") for uistr in uistrlist: - uimod = getUImod(uistr, localeval, namespace) + uimod = getUImod(uistr, config.getlocaleval(), namespace) if uimod: uiinstance = uimod(config) if uiinstance.isusable():