/offlineimap/head: changeset 442

Moved account-sep branch to head

offlineimap/head/offlineimap/CustomConfig.py

@@ -54,6 +54,40 @@ class CustomConfigParser(ConfigParser):
             path = None
         return LocalEval(path)
 
-    def getaccountlist(self):
-        return [x for x in self.sections() if x != 'general']
+    def getsectionlist(self, key):
+        """Returns a list of sections that start with key + " ".  That is,
+        if key is "Account", returns all section names that start with
+        "Account ", but strips off the "Account ".  For instance, for
+        "Account Test", returns "Test"."""
+
+        key = key + ' '
+        return [x[len(key):] for x in self.sections() \
+                if x.startswith(key)]
+
+def CustomConfigDefault():
+    """Just a sample constant that won't occur anywhere else to use for the
+    default."""
+    pass
+
+class ConfigHelperMixin:
+    def _confighelper_runner(self, option, default, defaultfunc, mainfunc):
+        if default != CustomConfigDefault:
+            return apply(defaultfunc, [self.getsection(), option, default])
+        else:
+            return apply(mainfunc, [self.getsection(), option])
+
+    def getconf(self, option, default = CustomConfigDefault):
+        return self._confighelper_runner(option, default,
+                                         self.getconfig().getdefault,
+                                         self.getconfig().get)
+
+    def getconfboolean(self, option, default = CustomConfigDefault):
+        return self._confighelper_runner(option, default,
+                                         self.getconfig().getdefaultboolean,
+                                         self.getconfig().getboolean)
+
+    def getconfint(self, option, default = CustomConfigDefault):
+        return self._confighelper_runner(option, default,
+                                         self.getconfig().getdefaultint,
+                                         self.getconfig().getint)
     

offlineimap/head/offlineimap/accounts.py

@@ -15,39 +15,49 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-from offlineimap import imapserver, repository, threadutil, mbnames
+from offlineimap import repository, threadutil, mbnames, CustomConfig
 from offlineimap.ui import UIBase
 from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
 from threading import Event
 import os
 
+def getaccountlist(customconfig):
+    return customconfig.getsectionlist('Account')
+
+def AccountListGenerator(customconfig):
+    return [Account(customconfig, accountname)
+            for accountname in getaccountlist(customconfig)]
+
+def AccountHashGenerator(customconfig):
+    retval = {}
+    for item in AccountListGenerator(customconfig):
+        retval[item.getname()] = item
+    return retval
+
 mailboxes = []
 
-class Account:
+class Account(CustomConfig.ConfigHelperMixin):
     def __init__(self, config, name):
         self.config = config
         self.name = name
         self.metadatadir = config.getmetadatadir()
         self.localeval = config.getlocaleval()
-        self.server = imapserver.ConfigedIMAPServer(config, self.name)
         self.ui = UIBase.getglobalui()
-        if self.config.has_option(self.name, 'autorefresh'):
-            self.refreshperiod = self.config.getint(self.name, 'autorefresh')
-        else:
+        self.refreshperiod = self.getconfint('autorefresh', 0)
+        if self.refreshperiod == 0:
             self.refreshperiod = None
-        self.hold = self.config.has_option(self.name, 'holdconnectionopen') \
-                    and self.config.getboolean(self.name, 'holdconnectionopen')
-        if self.config.has_option(self.name, 'keepalive'):
-            self.keepalive = self.config.getint(self.name, 'keepalive')
-        else:
-            self.keepalive = None
 
-    def getconf(self, option, default = None):
-        if default != None:
-            return self.config.get(self.name, option)
-        else:
-            return self.config.getdefault(self.name, option,
-                                          default)
+    def getlocaleval(self):
+        return self.localeval
+
+    def getconfig(self):
+        return self.config
+
+    def getname(self):
+        return self.name
+
+    def getsection(self):
+        return 'Account ' + self.getname()
 
     def sleeper(self):
         """Sleep handler.  Returns same value as UIBase.sleep:
@@ -58,31 +68,46 @@ class Account:
         
         if not self.refreshperiod:
             return 100
+
+        kaobjs = []
+
+        if hasattr(self, 'localrepos'):
+            kaobjs.append(self.localrepos)
+        if hasattr(self, 'remoterepos'):
+            kaobjs.append(self.remoterepos)
+
+        for item in kaobjs:
+            item.startkeepalive()
+        
         refreshperiod = self.refreshperiod * 60
-        if self.keepalive:
-            kaevent = Event()
-            kathread = ExitNotifyThread(target = self.server.keepalive,
-                                      name = "Keep alive " + self.name,
-                                      args = (self.keepalive, kaevent))
-            kathread.setDaemon(1)
-            kathread.start()
         sleepresult = self.ui.sleep(refreshperiod)
         if sleepresult == 2:
             # Cancel keep-alive, but don't bother terminating threads
-            if self.keepalive:
-                kaevent.set()
+            for item in kaobjs:
+                item.stopkeepalive(abrupt = 1)
             return sleepresult
         else:
             # Cancel keep-alive and wait for thread to terminate.
-            if self.keepalive:
-                kaevent.set()
-                kathread.join()
+            for item in kaobjs:
+                item.stopkeepalive(abrupt = 0)
             return sleepresult
             
 class AccountSynchronizationMixin:
     def syncrunner(self):
         self.ui.registerthread(self.name)
         self.ui.acct(self.name)
+        accountmetadata = self.getaccountmeta()
+        if not os.path.exists(accountmetadata):
+            os.mkdir(accountmetadata, 0700)            
+
+        self.remoterepos = repository.Base.LoadRepository(self.getconf('remoterepository'), self, 'remote')
+
+        # Connect to the local repository.
+        self.localrepos = repository.Base.LoadRepository(self.getconf('localrepository'), self, 'local')
+
+        # Connect to the local cache.
+        self.statusrepos = repository.LocalStatus.LocalStatusRepository(self.getconf('localrepository'), self)
+            
         if not self.refreshperiod:
             self.sync()
             self.ui.acctdone(self.name)
@@ -93,32 +118,23 @@ class AccountSynchronizationMixin:
             looping = self.sleeper() != 2
         self.ui.acctdone(self.name)
 
+    def getaccountmeta(self):
+        return os.path.join(self.metadatadir, 'Account-' + self.name)
+
     def sync(self):
         # We don't need an account lock because syncitall() goes through
         # each account once, then waits for all to finish.
         try:
-            accountmetadata = os.path.join(self.metadatadir, self.name)
-            if not os.path.exists(accountmetadata):
-                os.mkdir(accountmetadata, 0700)
-
-            remoterepos = repository.IMAP.IMAPRepository(self.config,
-                                                         self.localeval,
-                                                         self.name,
-                                                         self.server)
-
-            # Connect to the Maildirs.
-            localrepos = repository.Maildir.MaildirRepository(os.path.expanduser(self.config.get(self.name, "localfolders")), self.name, self.config)
-
-            # Connect to the local cache.
-            statusrepos = repository.LocalStatus.LocalStatusRepository(accountmetadata, self.name)
-
+            remoterepos = self.remoterepos
+            localrepos = self.localrepos
+            statusrepos = self.statusrepos
             self.ui.syncfolders(remoterepos, localrepos)
             remoterepos.syncfoldersto(localrepos)
 
             folderthreads = []
             for remotefolder in remoterepos.getfolders():
                 thread = InstanceLimitedThread(\
-                    instancename = 'FOLDER_' + self.name,
+                    instancename = 'FOLDER_' + self.remoterepos.getname(),
                     target = syncfolder,
                     name = "Folder sync %s[%s]" % \
                     (self.name, remotefolder.getvisiblename()),
@@ -129,8 +145,8 @@ class AccountSynchronizationMixin:
                 folderthreads.append(thread)
             threadutil.threadsreset(folderthreads)
             mbnames.write()
-            if not self.hold:
-                self.server.close()
+            localrepos.holdordropconnections()
+            remoterepos.holdordropconnections()
         finally:
             pass
     
@@ -166,19 +182,22 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos,
         
     statusfolder.cachemessagelist()
 
-    
-    # If either the local or the status folder has messages and
-    # there is a UID validity problem, warn and abort.
-    # If there are no messages, UW IMAPd loses UIDVALIDITY.
-    # But we don't really need it if both local folders are empty.
-    # So, in that case, save it off.
-    if (len(localfolder.getmessagelist()) or \
-        len(statusfolder.getmessagelist())) and \
-        not localfolder.isuidvalidityok(remotefolder):
-        ui.validityproblem(remotefolder)
-        return
+    # If either the local or the status folder has messages and there is a UID
+    # validity problem, warn and abort.  If there are no messages, UW IMAPd
+    # loses UIDVALIDITY.  But we don't really need it if both local folders are
+    # empty.  So, in that case, just save it off.
+    if len(localfolder.getmessagelist()) or len(statusfolder.getmessagelist()):
+        if not localfolder.isuidvalidityok():
+            ui.validityproblem(localfolder, localfolder.getsaveduidvalidity(),
+                               localfolder.getuidvalidity())
+            return
+        if not remotefolder.isuidvalidityok():
+            ui.validityproblem(remotefolder, remotefolder.getsaveduidvalidity(),
+                               remotefolder.getuidvalidity())
+            return
     else:
-        localfolder.saveuidvalidity(remotefolder.getuidvalidity())
+        localfolder.saveuidvalidity()
+        remotefolder.saveuidvalidity()
 
     # Load remote folder.
     ui.loadmessagelist(remoterepos, remotefolder)

offlineimap/head/offlineimap/folder/Base.py

@@ -20,8 +20,12 @@ from threading import *
 from offlineimap import threadutil
 from offlineimap.threadutil import InstanceLimitedThread
 from offlineimap.ui import UIBase
+import os.path, re
 
 class BaseFolder:
+    def __init__(self):
+        self.uidlock = Lock()
+        
     def getname(self):
         """Returns name"""
         return self.name
@@ -68,15 +72,52 @@ class BaseFolder:
         else:
             return self.getname()
     
-    def isuidvalidityok(self, remotefolder):
-        raise NotImplementedException
+    def getfolderbasename(self):
+        foldername = self.getname()
+        foldername = foldername.replace(self.repository.getsep(), '.')
+        foldername = re.sub('/\.$', '/dot', foldername)
+        foldername = re.sub('^\.$', 'dot', foldername)
+        return foldername
+
+    def isuidvalidityok(self):
+        if self.getsaveduidvalidity() != None:
+            return self.getsaveduidvalidity() == self.getuidvalidity()
+        else:
+            self.saveuidvalidity()
+            return 1
+
+    def _getuidfilename(self):
+        return os.path.join(self.repository.getuiddir(),
+                            self.getfolderbasename())
+            
+    def getsaveduidvalidity(self):
+        if hasattr(self, '_base_saved_uidvalidity'):
+            return self._base_saved_uidvalidity
+        uidfilename = self._getuidfilename()
+        if not os.path.exists(uidfilename):
+            self._base_saved_uidvalidity = None
+        else:
+            file = open(uidfilename, "rt")
+            self._base_saved_uidvalidity = long(file.readline().strip())
+            file.close()
+        return self._base_saved_uidvalidity
+
+    def saveuidvalidity(self):
+        newval = self.getuidvalidity()
+        uidfilename = self._getuidfilename()
+        self.uidlock.acquire()
+        try:
+            file = open(uidfilename + ".tmp", "wt")
+            file.write("%d\n" % newval)
+            file.close()
+            os.rename(uidfilename + ".tmp", uidfilename)
+            self._base_saved_uidvalidity = newval
+        finally:
+            self.uidlock.release()
 
     def getuidvalidity(self):
         raise NotImplementedException
 
-    def saveuidvalidity(self, newval):
-        raise NotImplementedException
-
     def cachemessagelist(self):
         """Reads the message list from disk or network and stores it in
         memory for later use.  This list will not be re-read from disk or

offlineimap/head/offlineimap/folder/IMAP.py

@@ -19,7 +19,7 @@
 from Base import BaseFolder
 from offlineimap import imaputil, imaplib
 from offlineimap.ui import UIBase
-import rfc822, time, string
+import rfc822, time, string, random, binascii
 from StringIO import StringIO
 from copy import copy
 
@@ -27,9 +27,7 @@ from copy import copy
 class IMAPFolder(BaseFolder):
     def __init__(self, imapserver, name, visiblename, accountname, repository):
         self.config = imapserver.config
-        self.expunge = 1
-        if self.config.has_option(accountname, 'expunge'):
-           self.expunge = self.config.getboolean(accountname, 'expunge')
+        self.expunge = repository.getexpunge()
         self.name = imaputil.dequote(name)
         self.root = None # imapserver.root
         self.sep = imapserver.delim
@@ -38,6 +36,8 @@ class IMAPFolder(BaseFolder):
         self.visiblename = visiblename
         self.accountname = accountname
         self.repository = repository
+        self.randomgenerator = random.Random()
+        BaseFolder.__init__(self)
 
     def getaccountname(self):
         return self.accountname
@@ -49,7 +49,7 @@ class IMAPFolder(BaseFolder):
         self.imapserver.connectionwait()
 
     def getcopyinstancelimit(self):
-        return 'MSGCOPY_' + self.accountname
+        return 'MSGCOPY_' + self.repository.getname()
 
     def getvisiblename(self):
         return self.visiblename
@@ -70,7 +70,12 @@ class IMAPFolder(BaseFolder):
         try:
             # Primes untagged_responses
             imapobj.select(self.getfullname(), readonly = 1)
-            maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
+            try:
+                # Some mail servers do not return an EXISTS response if
+                # the folder is empty.
+                maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
+            except KeyError:
+                return
             if maxmsgid < 1:
                 # No messages; return
                 return
@@ -107,7 +112,37 @@ class IMAPFolder(BaseFolder):
     
     def getmessageflags(self, uid):
         return self.messagelist[uid]['flags']
-    
+
+    def savemessage_getnewheader(self, content):
+        headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
+        headername += binascii.hexlify(self.repository.getname()) + '-'
+        headername += binascii.hexlify(self.getname())
+        headervalue= '%d-' % long(time.time())
+        headervalue += str(self.randomgenerator.random()).replace('.', '')
+        return (headername, headervalue)
+
+    def savemessage_addheader(self, content, headername, headervalue):
+        insertionpoint = content.find("\r\n")
+        leader = content[0:insertionpoint]
+        newline = "\r\n%s: %s" % (headername, headervalue)
+        trailer = content[insertionpoint:]
+        return leader + newline + trailer
+
+    def savemessage_searchforheader(self, imapobj, headername, headervalue):
+        # Now find the UID it got.
+        headervalue = imapobj._quote(headervalue)
+        try:
+            matchinguids = imapobj.uid('search', None,
+                                       '(HEADER %s %s)' % (headername, headervalue))[1][0]
+        except imapobj.error:
+            # IMAP server doesn't implement search or had a problem.
+            return 0
+        matchinguids = matchinguids.split(' ')
+        if len(matchinguids) != 1 or matchinguids[0] == None:
+            raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
+        matchinguids.sort()
+        return long(matchinguids[0])
+
     def savemessage(self, uid, content, flags):
         imapobj = self.imapserver.acquireconnection()
         try:
@@ -123,9 +158,6 @@ class IMAPFolder(BaseFolder):
             # In order to get the new uid, we need to save off the message ID.
 
             message = rfc822.Message(StringIO(content))
-            mid = message.getheader('Message-Id')
-            if mid != None:
-                mid = imapobj._quote(mid)
             datetuple = rfc822.parsedate(message.getheader('Date'))
             # Will be None if missing or not in a valid format.
             if datetuple == None:
@@ -145,35 +177,31 @@ class IMAPFolder(BaseFolder):
             if content.find("\r\n") == -1:  # Convert line endings if not already
                 content = content.replace("\n", "\r\n")
 
+            (headername, headervalue) = self.savemessage_getnewheader(content)
+            content = self.savemessage_addheader(content, headername,
+                                                 headervalue)
+
             assert(imapobj.append(self.getfullname(),
                                        imaputil.flagsmaildir2imap(flags),
                                        date, content)[0] == 'OK')
+
             # Checkpoint.  Let it write out the messages, etc.
             assert(imapobj.check()[0] == 'OK')
-            if mid == None:
-                # No message ID in original message -- no sense trying to
-                # search for it.
-                return 0
-            # Now find the UID it got.
+
+            # Keep trying until we get the UID.
             try:
-                matchinguids = imapobj.uid('search', None,
-                                           '(HEADER Message-Id %s)' % mid)[1][0]
-            except imapobj.error:
-                # IMAP server doesn't implement search or had a problem.
-                return 0
-            matchinguids = matchinguids.split(' ')
-            if len(matchinguids) != 1 or matchinguids[0] == None:
-                return 0
-            matchinguids.sort()
-            try:
-                uid = long(matchinguids[-1])
+                uid = self.savemessage_searchforheader(imapobj, headername,
+                                                       headervalue)
             except ValueError:
-                return 0
-            self.messagelist[uid] = {'uid': uid, 'flags': flags}
-            return uid
+                assert(imapobj.noop()[0] == 'OK')
+                uid = self.savemessage_searchforheader(imapobj, headername,
+                                                       headervalue)
         finally:
             self.imapserver.releaseconnection(imapobj)
 
+        self.messagelist[uid] = {'uid': uid, 'flags': flags}
+        return uid
+
     def savemessageflags(self, uid, flags):
         imapobj = self.imapserver.acquireconnection()
         try:
@@ -197,9 +225,15 @@ class IMAPFolder(BaseFolder):
     def addmessageflags(self, uid, flags):
         self.addmessagesflags([uid], flags)
 
-    def addmessagesflags(self, uidlist, flags):
+    def addmessagesflags_noconvert(self, uidlist, flags):
         self.processmessagesflags('+', uidlist, flags)
 
+    def addmessagesflags(self, uidlist, flags):
+        """This is here for the sake of UIDMaps.py -- deletemessages must
+        add flags and get a converted UID, and if we don't have noconvert,
+        then UIDMaps will try to convert it twice."""
+        self.addmessagesflags_noconvert(uidlist, flags)
+
     def deletemessageflags(self, uid, flags):
         self.deletemessagesflags([uid], flags)
 
@@ -254,15 +288,18 @@ class IMAPFolder(BaseFolder):
                         self.messagelist[uid]['flags'].remove(flag)
 
     def deletemessage(self, uid):
-        self.deletemessages([uid])
+        self.deletemessages_noconvert([uid])
 
     def deletemessages(self, uidlist):
+        self.deletemessages_noconvert(uidlist)
+
+    def deletemessages_noconvert(self, uidlist):
         # Weed out ones not in self.messagelist
         uidlist = [uid for uid in uidlist if uid in self.messagelist]
         if not len(uidlist):
             return        
 
-        self.addmessagesflags(uidlist, ['T'])
+        self.addmessagesflags_noconvert(uidlist, ['T'])
         imapobj = self.imapserver.acquireconnection()
         try:
             try:

offlineimap/head/offlineimap/folder/LocalStatus.py

@@ -27,11 +27,13 @@ class LocalStatusFolder(BaseFolder):
         self.root = root
         self.sep = '.'
         self.filename = os.path.join(root, name)
+        self.filename = repository.getfolderfilename(name)
         self.messagelist = None
         self.repository = repository
         self.savelock = threading.Lock()
         self.doautosave = 1
         self.accountname = accountname
+        BaseFolder.__init__(self)
 
     def getaccountname(self):
         return self.accountname

offlineimap/head/offlineimap/folder/Maildir.py

@@ -43,10 +43,10 @@ class MaildirFolder(BaseFolder):
         self.name = name
         self.root = root
         self.sep = sep
-        self.uidfilename = os.path.join(self.getfullname(), "offlineimap.uidvalidity")
         self.messagelist = None
         self.repository = repository
         self.accountname = accountname
+        BaseFolder.__init__(self)
 
     def getaccountname(self):
         return self.accountname
@@ -55,31 +55,10 @@ class MaildirFolder(BaseFolder):
         return os.path.join(self.getroot(), self.getname())
 
     def getuidvalidity(self):
-        if hasattr(self, 'uidvalidity'):
-            return self.uidvalidity
-        if not os.path.exists(self.uidfilename):
-            self.uidvalidity = None
-        else:
-            file = open(self.uidfilename, "rt")
-            self.uidvalidity = long(file.readline().strip())
-            file.close()
-        return self.uidvalidity
+        """Maildirs have no notion of uidvalidity, so we just return a magic
+        token."""
+        return 42
 
-    def saveuidvalidity(self, newval):
-        file = open(self.uidfilename + ".tmp", "wt")
-        file.write("%d\n" % newval)
-        file.close()
-        os.rename(self.uidfilename + ".tmp", self.uidfilename)
-        self.uidvalidity = newval
-
-    def isuidvalidityok(self, remotefolder):
-        myval = self.getuidvalidity()
-        if myval != None:
-            return myval == remotefolder.getuidvalidity()
-        else:
-            self.saveuidvalidity(remotefolder.getuidvalidity())
-            return 1
-            
     def _scanfolder(self):
         """Cache the message list.  Maildir flags are:
         R (replied)

offlineimap/head/offlineimap/imapserver.py

@@ -1,5 +1,5 @@
 # IMAP server support
-# Copyright (C) 2002 John Goerzen
+# Copyright (C) 2002, 2003 John Goerzen
 # <jgoerzen@complete.org>
 #
 #    This program is free software; you can redistribute it and/or modify
@@ -48,11 +48,11 @@ class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplib.IMAP4_SSL): pass
 class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplib.IMAP4_Tunnel): pass
 
 class IMAPServer:
-    def __init__(self, config, accountname,
+    def __init__(self, config, reposname,
                  username = None, password = None, hostname = None,
                  port = None, ssl = 1, maxconnections = 1, tunnel = None,
                  reference = '""'):
-        self.account = accountname
+        self.reposname = reposname
         self.config = config
         self.username = username
         self.password = password
@@ -80,7 +80,8 @@ class IMAPServer:
         if self.password != None and self.passworderror == None:
             return self.password
 
-        self.password = UIBase.getglobalui().getpass(self.account, self.config,
+        self.password = UIBase.getglobalui().getpass(self.reposname,
+                                                     self.config,
                                                      self.passworderror)
         self.passworderror = None
 
@@ -152,17 +153,18 @@ class IMAPServer:
         
         self.connectionlock.release()   # Release until need to modify data
 
-        UIBase.getglobalui().connecting(self.hostname, self.port)
-
         success = 0
         while not success:
             # Generate a new connection.
             if self.tunnel:
+                UIBase.getglobalui().connecting('tunnel', self.tunnel)
                 imapobj = UsefulIMAP4_Tunnel(self.tunnel)
                 success = 1
             elif self.usessl:
+                UIBase.getglobalui().connecting(self.hostname, self.port)
                 imapobj = UsefulIMAP4_SSL(self.hostname, self.port)
             else:
+                UIBase.getglobalui().connecting(self.hostname, self.port)
                 imapobj = UsefulIMAP4(self.hostname, self.port)
 
             if not self.tunnel:
@@ -258,39 +260,34 @@ class ConfigedIMAPServer(IMAPServer):
     object and an account name.  The passwordhash is used if
     passwords for certain accounts are known.  If the password for this
     account is listed, it will be obtained from there."""
-    def __init__(self, config, accountname, passwordhash = {}):
+    def __init__(self, repository, passwordhash = {}):
         """Initialize the object.  If the account is not a tunnel,
         the password is required."""
-        host = config.get(accountname, "remotehost")
-        user = config.get(accountname, "remoteuser")
-        port = None
-        if config.has_option(accountname, "remoteport"):
-            port = config.getint(accountname, "remoteport")
-        ssl = config.getdefaultboolean(accountname, "ssl", 0)
-        usetunnel = config.has_option(accountname, "preauthtunnel")
-        reference = '""'
-        if config.has_option(accountname, "reference"):
-            reference = config.get(accountname, "reference")
+        self.repos = repository
+        self.config = self.repos.getconfig()
+        usetunnel = self.repos.getpreauthtunnel()
+        if not usetunnel:
+            host = self.repos.gethost()
+            user = self.repos.getuser()
+            port = self.repos.getport()
+            ssl = self.repos.getssl()
+        reference = self.repos.getreference()
         server = None
         password = None
-        if accountname in passwordhash:
-            password = passwordhash[accountname]
+        
+        if repository.getname() in passwordhash:
+            password = passwordhash[repository.getname()]
 
         # Connect to the remote server.
         if usetunnel:
-            IMAPServer.__init__(self, config, accountname,
-                                tunnel = config.get(accountname, "preauthtunnel"),
+            IMAPServer.__init__(self, self.config, self.repos.getname(),
+                                tunnel = usetunnel,
                                 reference = reference,
-                                maxconnections = config.getint(accountname, "maxconnections"))
+                                maxconnections = self.repos.getmaxconnections())
         else:
             if not password:
-                if config.has_option(accountname, 'remotepass'):
-                    password = config.get(accountname, 'remotepass')
-                elif config.has_option(accountname, 'remotepassfile'):
-                    passfile = open(os.path.expanduser(config.get(accountname, "remotepassfile")))
-                    password = passfile.readline().strip()
-                    passfile.close()
-            IMAPServer.__init__(self, config, accountname,
+                password = self.repos.getpassword()
+            IMAPServer.__init__(self, self.config, self.repos.getname(),
                                 user, password, host, port, ssl,
-                                config.getdefaultint(accountname, "maxconnections", 1),
+                                self.repos.getmaxconnections(),
                                 reference = reference)

offlineimap/head/offlineimap/init.py

@@ -1,5 +1,5 @@
 # OfflineIMAP initialization code
-# Copyright (C) 2002 John Goerzen
+# Copyright (C) 2002, 2003 John Goerzen
 # <jgoerzen@complete.org>
 #
 #    This program is free software; you can redistribute it and/or modify
@@ -16,13 +16,14 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-from offlineimap import imaplib, imapserver, repository, folder, mbnames, threadutil, version, syncmaster
+from offlineimap import imaplib, imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
 from offlineimap.localeval import LocalEval
 from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
 from offlineimap.ui import UIBase
 import re, os, os.path, offlineimap, sys, fcntl
 from offlineimap.CustomConfig import CustomConfigParser
 from threading import *
+import threading
 from getopt import getopt
 
 lockfd = None
@@ -78,18 +79,26 @@ def startup(versionno):
             ui.add_debug(debugtype.strip())
             if debugtype == 'imap':
                 imaplib.Debug = 5
+            if debugtype == 'thread':
+                threading._VERBOSE = 1
 
     if '-o' in options:
-        for section in config.getaccountlist():
-            config.remove_option(section, "autorefresh")
+        # FIXME: maybe need a better
+        for section in accounts.getaccountlist(config):
+            config.remove_option('Account ' + section, "autorefresh")
 
     lock(config, ui)
 
-    accounts = config.get("general", "accounts")
+    activeaccounts = config.get("general", "accounts")
     if '-a' in options:
-        accounts = options['-a']
-    accounts = accounts.replace(" ", "")
-    accounts = accounts.split(",")
+        activeaccounts = options['-a']
+    activeaccounts = activeaccounts.replace(" ", "")
+    activeaccounts = activeaccounts.split(",")
+    allaccounts = accounts.AccountHashGenerator(config)
+
+    syncaccounts = {}
+    for account in activeaccounts:
+        syncaccounts[account] = allaccounts[account]
 
     server = None
     remoterepos = None
@@ -101,18 +110,19 @@ def startup(versionno):
         threadutil.initInstanceLimit("ACCOUNTLIMIT",
                                      config.getdefaultint("general", "maxsyncaccounts", 1))
 
-    for account in accounts:
-        for instancename in ["FOLDER_" + account, "MSGCOPY_" + account]:
+    for reposname in config.getsectionlist('Repository'):
+        for instancename in ["FOLDER_" + reposname,
+                             "MSGCOPY_" + reposname]:
             if '-1' in options:
                 threadutil.initInstanceLimit(instancename, 1)
             else:
                 threadutil.initInstanceLimit(instancename,
-                                             config.getdefaultint(account, "maxconnections", 1))
+                                             config.getdefaultint('Repository ' + reposname, "maxconnections", 1))
 
     threadutil.initexitnotify()
     t = ExitNotifyThread(target=syncmaster.syncitall,
                          name='Sync Runner',
-                         kwargs = {'accounts': accounts,
+                         kwargs = {'accounts': syncaccounts,
                                    'config': config})
     t.setDaemon(1)
     t.start()

offlineimap/head/offlineimap/repository/Base.py

@@ -1,5 +1,5 @@
 # Base repository support
-# Copyright (C) 2002 John Goerzen
+# Copyright (C) 2002, 2003 John Goerzen
 # <jgoerzen@complete.org>
 #
 #    This program is free software; you can redistribute it and/or modify
@@ -16,7 +16,71 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-class BaseRepository:
+from offlineimap import CustomConfig
+import os.path
+
+def LoadRepository(name, account, reqtype):
+    from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository
+    from offlineimap.repository.Maildir import MaildirRepository
+    if reqtype == 'remote':
+        # For now, we don't support Maildirs on the remote side.
+        typemap = {'IMAP': IMAPRepository}
+    elif reqtype == 'local':
+        typemap = {'IMAP': MappedIMAPRepository,
+                   'Maildir': MaildirRepository}
+    else:
+        raise ValueError, "Request type %s not supported" % reqtype
+    config = account.getconfig()
+    repostype = config.get('Repository ' + name, 'type').strip()
+    return typemap[repostype](name, account)
+
+class BaseRepository(CustomConfig.ConfigHelperMixin):
+    def __init__(self, reposname, account):
+        self.account = account
+        self.config = account.getconfig()
+        self.name = reposname
+        self.localeval = account.getlocaleval()
+        self.accountname = self.account.getname()
+        self.uiddir = os.path.join(self.config.getmetadatadir(), 'Repository-' + self.name)
+        if not os.path.exists(self.uiddir):
+            os.mkdir(self.uiddir, 0700)
+        self.mapdir = os.path.join(self.uiddir, 'UIDMapping')
+        if not os.path.exists(self.mapdir):
+            os.mkdir(self.mapdir, 0700)
+        self.uiddir = os.path.join(self.uiddir, 'FolderValidity')
+        if not os.path.exists(self.uiddir):
+            os.mkdir(self.uiddir, 0700)
+
+    def holdordropconnections(self):
+        pass
+
+    def dropconnections(self):
+        pass
+
+    def getaccount(self):
+        return self.account
+
+    def getname(self):
+        return self.name
+
+    def getuiddir(self):
+        return self.uiddir
+
+    def getmapdir(self):
+        return self.mapdir
+
+    def getaccountname(self):
+        return self.accountname
+
+    def getsection(self):
+        return 'Repository ' + self.name
+
+    def getconfig(self):
+        return self.config
+
+    def getlocaleval(self):
+        return self.account.getlocaleval()
+    
     def getfolders(self):
         """Returns a list of ALL folders on this server."""
         return []
@@ -68,3 +132,14 @@ class BaseRepository:
         #    if not key in srchash:
         #        dest.deletefolder(key)
         
+    ##### Keepalive
+
+    def startkeepalive(self):
+        """The default implementation will do nothing."""
+        pass
+
+    def stopkeepalive(self, abrupt = 0):
+        """Stop keep alive.  If abrupt is 1, stop it but don't bother waiting
+        for the threads to terminate."""
+        pass
+    

offlineimap/head/offlineimap/repository/IMAP.py

@@ -17,38 +17,116 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 from Base import BaseRepository
-from offlineimap import folder, imaputil
+from offlineimap import folder, imaputil, imapserver
+from offlineimap.folder.UIDMaps import MappedIMAPFolder
+from offlineimap.threadutil import ExitNotifyThread
 import re, types
 from threading import *
 
 class IMAPRepository(BaseRepository):
-    def __init__(self, config, localeval, accountname, imapserver):
-        """Initialize an IMAPRepository object.  Takes an IMAPServer
-        object."""
-        self.imapserver = imapserver
-        self.config = config
-        self.accountname = accountname
+    def __init__(self, reposname, account):
+        """Initialize an IMAPRepository object."""
+        BaseRepository.__init__(self, reposname, account)
+        self.imapserver = imapserver.ConfigedIMAPServer(self)
         self.folders = None
         self.nametrans = lambda foldername: foldername
         self.folderfilter = lambda foldername: 1
         self.folderincludes = []
         self.foldersort = cmp
-        if config.has_option(accountname, 'nametrans'):
-            self.nametrans = localeval.eval(config.get(accountname, 'nametrans'), {'re': re})
-        if config.has_option(accountname, 'folderfilter'):
-            self.folderfilter = localeval.eval(config.get(accountname, 'folderfilter'), {'re': re})
-        if config.has_option(accountname, 'folderincludes'):
-            self.folderincludes = localeval.eval(config.get(accountname, 'folderincludes'), {'re': re})
-        if config.has_option(accountname, 'foldersort'):
-            self.foldersort = localeval.eval(config.get(accountname, 'foldersort'), {'re': re})
+        localeval = self.localeval
+        if self.config.has_option(self.getsection(), 'nametrans'):
+            self.nametrans = localeval.eval(self.getconf('nametrans'),
+                                            {'re': re})
+        if self.config.has_option(self.getsection(), 'folderfilter'):
+            self.folderfilter = localeval.eval(self.getconf('folderfilter'),
+                                               {'re': re})
+        if self.config.has_option(self.getsection(), 'folderincludes'):
+            self.folderincludes = localeval.eval(self.getconf('folderincludes'),
+                                                 {'re': re})
+        if self.config.has_option(self.getsection(), 'foldersort'):
+            self.foldersort = localeval.eval(self.getconf('foldersort'),
+                                             {'re': re})
+
+    def startkeepalive(self):
+        keepalivetime = self.getkeepalive()
+        if not keepalivetime: return
+        self.kaevent = Event()
+        self.kathread = ExitNotifyThread(target = self.imapserver.keepalive,
+                                         name = "Keep alive " + self.getname(),
+                                         args = (keepalivetime, self.kaevent))
+        self.kathread.setDaemon(1)
+        self.kathread.start()
+
+    def stopkeepalive(self, abrupt = 0):
+        if not hasattr(self, 'kaevent'):
+            # Keepalive is not active.
+            return
+
+        self.kaevent.set()
+        if not abrupt:
+            self.kathread.join()
+        del self.kathread
+        del self.kaevent
+
+    def holdordropconnections(self):
+        if not self.getholdconnectionopen():
+            self.dropconnections()
+
+    def dropconnections(self):
+        self.imapserver.close()
+
+    def getholdconnectionopen(self):
+        return self.getconfboolean("holdconnectionopen", 0)
+
+    def getkeepalive(self):
+        return self.getconfint("keepalive", 0)
 
     def getsep(self):
         return self.imapserver.delim
 
+    def gethost(self):
+        return self.getconf('remotehost')
+
+    def getuser(self):
+        return self.getconf('remoteuser')
+
+    def getport(self):
+        return self.getconfint('remoteport', None)
+
+    def getssl(self):
+        return self.getconfboolean('ssl', 0)
+
+    def getpreauthtunnel(self):
+        return self.getconf('preauthtunnel', None)
+
+    def getreference(self):
+        return self.getconf('reference', '""')
+
+    def getmaxconnections(self):
+        return self.getconfint('maxconnections', 1)
+
+    def getexpunge(self):
+        return self.getconfboolean('expunge', 1)
+
+    def getpassword(self):
+        password = self.getconf('remotepass', None)
+        if password != None:
+            return password
+        passfile = self.getconf('remotepassfile', None)
+        if passfile != None:
+            fd = open(os.path.expanduser(passfile))
+            password = passfile.readline().strip()
+            passfile.close()
+            return password
+        return None
+
     def getfolder(self, foldername):
-        return folder.IMAP.IMAPFolder(self.imapserver, foldername,
-                                      self.nametrans(foldername),
-                                      accountname, self)
+        return self.getfoldertype()(self.imapserver, foldername,
+                                    self.nametrans(foldername),
+                                    self.accountname, self)
+
+    def getfoldertype(self):
+        return folder.IMAP.IMAPFolder
 
     def getfolders(self):
         if self.folders != None:
@@ -60,7 +138,8 @@ class IMAPRepository(BaseRepository):
         finally:
             self.imapserver.releaseconnection(imapobj)
         for string in listresult:
-            if type(string) == types.StringType and string == '':
+            if string == None or \
+                   (type(string) == types.StringType and string == ''):
                 # Bug in imaplib: empty strings in results from
                 # literals.
                 continue
@@ -71,13 +150,31 @@ class IMAPRepository(BaseRepository):
             foldername = imaputil.dequote(name)
             if not self.folderfilter(foldername):
                 continue
-            retval.append(folder.IMAP.IMAPFolder(self.imapserver, foldername,
-                                                 self.nametrans(foldername),
-                                                 self.accountname, self))
+            retval.append(self.getfoldertype()(self.imapserver, foldername,
+                                               self.nametrans(foldername),
+                                               self.accountname, self))
         for foldername in self.folderincludes:
-            retval.append(folder.IMAP.IMAPFolder(self.imapserver, foldername,
-                                                 self.nametrans(foldername),
-                                                 self.accountname, self))
+            retval.append(self.getfoldertype()(self.imapserver, foldername,
+                                               self.nametrans(foldername),
+                                               self.accountname, self))
         retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename()))
         self.folders = retval
         return retval
+
+    def makefolder(self, foldername):
+        #if self.getreference() != '""':
+        #    newname = self.getreference() + self.getsep() + foldername
+        #else:
+        #    newname = foldername
+        newname = foldername
+        imapobj = self.imapserver.acquireconnection()
+        try:
+            result = imapobj.create(newname)
+            if result[0] != 'OK':
+                raise RuntimeError, "Repository %s could not create folder %s: %s" % (self.getname(), foldername, str(result))
+        finally:
+            self.imapserver.releaseconnection(imapobj)
+            
+class MappedIMAPRepository(IMAPRepository):
+    def getfoldertype(self):
+        return MappedIMAPFolder

offlineimap/head/offlineimap/repository/LocalStatus.py

@@ -18,18 +18,22 @@
 
 from Base import BaseRepository
 from offlineimap import folder
-import os
+import os, re
 
 class LocalStatusRepository(BaseRepository):
-    def __init__(self, directory, accountname):
-        self.directory = directory
+    def __init__(self, reposname, account):
+        BaseRepository.__init__(self, reposname, account)
+        self.directory = os.path.join(account.getaccountmeta(), 'LocalStatus')
+        if not os.path.exists(self.directory):
+            os.mkdir(self.directory, 0700)
         self.folders = None
-        self.accountname = accountname
 
     def getsep(self):
         return '.'
 
     def getfolderfilename(self, foldername):
+        foldername = re.sub('/\.$', '/dot', foldername)
+        foldername = re.sub('^\.$', 'dot', foldername)
         return os.path.join(self.directory, foldername)
 
     def makefolder(self, foldername):

offlineimap/head/offlineimap/repository/Maildir.py

@@ -23,25 +23,24 @@ from mailbox import Maildir
 import os
 
 class MaildirRepository(BaseRepository):
-    def __init__(self, root, accountname, config):
+    def __init__(self, reposname, account):
         """Initialize a MaildirRepository object.  Takes a path name
         to the directory holding all the Maildir directories."""
+        BaseRepository.__init__(self, reposname, account)
 
-        self.root = root
+        self.root = self.getlocalroot()
         self.folders = None
-        self.accountname = accountname
-        self.config = config
         self.ui = UIBase.getglobalui()
         self.debug("MaildirRepository initialized, sep is " + repr(self.getsep()))
 
+    def getlocalroot(self):
+        return os.path.expanduser(self.getconf('localfolders'))
+
     def debug(self, msg):
         self.ui.debug('maildir', msg)
 
     def getsep(self):
-        if self.config.has_option(self.accountname, 'sep'):
-            return self.config.get(self.accountname, 'sep').strip()
-        else:
-            return '.'
+        return self.getconf('sep', '.').strip()
 
     def makefolder(self, foldername):
         self.debug("makefolder called with arg " + repr(foldername))
@@ -65,7 +64,8 @@ class MaildirRepository(BaseRepository):
         # makedirs will fail because the higher-up dir already exists.
         # So, check to see if this is indeed the case.
 
-        if self.getsep() == '/' and os.path.isdir(foldername):
+        if (self.getsep() == '/' or self.getconfboolean('existsok', 0)) \
+            and os.path.isdir(foldername):
             self.debug("makefolder: %s already is a directory" % foldername)
             # Already exists.  Sanity-check that it's not a Maildir.
             for subdir in ['cur', 'new', 'tmp']:

offlineimap/head/offlineimap/syncmaster.py

@@ -1,5 +1,5 @@
 # OfflineIMAP synchronization master code
-# Copyright (C) 2002 John Goerzen
+# Copyright (C) 2002, 2003 John Goerzen
 # <jgoerzen@complete.org>
 #
 #    This program is free software; you can redistribute it and/or modify
@@ -25,21 +25,16 @@ import re, os, os.path, offlineimap, sys
 from ConfigParser import ConfigParser
 from threading import *
 
-def syncaccount(threads, config, accountname):
+def syncaccount(config, accountname):
     account = SyncableAccount(config, accountname)
     thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT',
                                    target = account.syncrunner,
                                    name = "Account sync %s" % accountname)
     thread.setDaemon(1)
     thread.start()
-    threads.add(thread)
     
 def syncitall(accounts, config):
-    currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE')
     ui = UIBase.getglobalui()
-    threads = threadutil.threadlist()
     mbnames.init(config, accounts)
     for accountname in accounts:
-        syncaccount(threads, config, accountname)
-    # Wait for the threads to finish.
-    threads.reset()
+        syncaccount(config, accountname)

offlineimap/head/offlineimap/threadutil.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2002 John Goerzen
+# Copyright (C) 2002, 2003 John Goerzen
 # Thread support module
 # <jgoerzen@complete.org>
 #
@@ -113,12 +113,14 @@ def exitnotifymonitorloop(callback):
     global exitcondition, exitthreads
     while 1:                            # Loop forever.
         exitcondition.acquire()
-        while not len(exitthreads):
-            exitcondition.wait(1)
+        try:
+            while not len(exitthreads):
+                exitcondition.wait(1)
 
-        while len(exitthreads):
-            callback(exitthreads.pop(0)) # Pull off in order added!
-        exitcondition.release()
+            while len(exitthreads):
+                callback(exitthreads.pop(0)) # Pull off in order added!
+        finally:
+            exitcondition.release()
 
 def threadexited(thread):
     """Called when a thread exits."""
@@ -132,11 +134,6 @@ def threadexited(thread):
         ui.threadException(thread)      # Expected to terminate
         sys.exit(100)                   # Just in case...
         os._exit(100)
-    elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE':
-        ui.terminate()
-        # Just in case...
-        sys.exit(100)
-        os._exit(100)
     else:
         ui.threadExited(thread)
 

offlineimap/head/offlineimap/ui/UIBase.py

@@ -155,7 +155,7 @@ class UIBase:
         if hostname == None:
             hostname = ''
         if port != None:
-            port = ":%d" % port
+            port = ":%s" % str(port)
         displaystr = ' to %s%s.' % (hostname, port)
         if hostname == '' and port == None:
             displaystr = '.'
@@ -182,9 +182,9 @@ class UIBase:
                                              s.getnicename(srcrepos),
                                              s.getnicename(destrepos)))
 
-    def validityproblem(s, folder):
-        s.warn("UID validity problem for folder %s; skipping it" % \
-               folder.getname())
+    def validityproblem(s, folder, saved, new):
+        s.warn("UID validity problem for folder %s (saved %d; got %d); skipping it" % \
+               (folder.getname(), saved, new))
 
     def loadmessagelist(s, repos, folder):
         if s.verbose > 0:

offlineimap/head/offlineimap/version.py

@@ -1,8 +1,8 @@
 productname = 'OfflineIMAP'
-versionstr = "3.99.10"
-revno = long('$Rev: 367 $'[6:-2])
+versionstr = "3.99.12"
+revno = long('$Rev: 439 $'[6:-2])
 revstr = "Rev %d" % revno
-datestr = '$Date: 2003-04-16 09:23:45 -0500 (Wed, 16 Apr 2003) $'
+datestr = '$Date: 2003-04-17 16:16:00 -0500 (Thu, 17 Apr 2003) $'
 
 versionlist = versionstr.split(".")
 major = versionlist[0]