Step 2 of SVN to arch tree conversion

offlineimap/folder/Base.py

@@ -0,0 +1,390 @@
+# Base folder support
+# Copyright (C) 2002 John Goerzen
+# <jgoerzen@complete.org>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from threading import *
+from offlineimap import threadutil
+from offlineimap.threadutil import InstanceLimitedThread
+from offlineimap.ui import UIBase
+import os.path, re
+
+class BaseFolder:
+    def __init__(self):
+        self.uidlock = Lock()
+        
+    def getname(self):
+        """Returns name"""
+        return self.name
+
+    def suggeststhreads(self):
+        """Returns true if this folder suggests using threads for actions;
+        false otherwise.  Probably only IMAP will return true."""
+        return 0
+
+    def waitforthread(self):
+        """For threading folders, waits until there is a resource available
+        before firing off a thread.  For all others, returns immediately."""
+        pass
+
+    def getcopyinstancelimit(self):
+        """For threading folders, returns the instancelimitname for
+        InstanceLimitedThreads."""
+        raise NotImplementedException
+
+    def storesmessages(self):
+        """Should be true for any backend that actually saves message bodies.
+        (Almost all of them).  False for the LocalStatus backend.  Saves
+        us from having to slurp up messages just for localstatus purposes."""
+        return 1
+
+    def getvisiblename(self):
+        return self.name
+
+    def getrepository(self):
+        """Returns the repository object that this folder is within."""
+        return self.repository
+
+    def getroot(self):
+        """Returns the root of the folder, in a folder-specific fashion."""
+        return self.root
+
+    def getsep(self):
+        """Returns the separator for this folder type."""
+        return self.sep
+
+    def getfullname(self):
+        if self.getroot():
+            return self.getroot() + self.getsep() + self.getname()
+        else:
+            return self.getname()
+    
+    def getfolderbasename(self):
+        foldername = self.getname()
+        foldername = foldername.replace(self.repository.getsep(), '.')
+        foldername = re.sub('/\.$', '/dot', foldername)
+        foldername = re.sub('^\.$', 'dot', foldername)
+        return foldername
+
+    def isuidvalidityok(self):
+        if self.getsaveduidvalidity() != None:
+            return self.getsaveduidvalidity() == self.getuidvalidity()
+        else:
+            self.saveuidvalidity()
+            return 1
+
+    def _getuidfilename(self):
+        return os.path.join(self.repository.getuiddir(),
+                            self.getfolderbasename())
+            
+    def getsaveduidvalidity(self):
+        if hasattr(self, '_base_saved_uidvalidity'):
+            return self._base_saved_uidvalidity
+        uidfilename = self._getuidfilename()
+        if not os.path.exists(uidfilename):
+            self._base_saved_uidvalidity = None
+        else:
+            file = open(uidfilename, "rt")
+            self._base_saved_uidvalidity = long(file.readline().strip())
+            file.close()
+        return self._base_saved_uidvalidity
+
+    def saveuidvalidity(self):
+        newval = self.getuidvalidity()
+        uidfilename = self._getuidfilename()
+        self.uidlock.acquire()
+        try:
+            file = open(uidfilename + ".tmp", "wt")
+            file.write("%d\n" % newval)
+            file.close()
+            os.rename(uidfilename + ".tmp", uidfilename)
+            self._base_saved_uidvalidity = newval
+        finally:
+            self.uidlock.release()
+
+    def getuidvalidity(self):
+        raise NotImplementedException
+
+    def cachemessagelist(self):
+        """Reads the message list from disk or network and stores it in
+        memory for later use.  This list will not be re-read from disk or
+        memory unless this function is called again."""
+        raise NotImplementedException
+
+    def getmessagelist(self):
+        """Gets the current message list.
+        You must call cachemessagelist() before calling this function!"""
+        raise NotImplementedException
+
+    def getmessage(self, uid):
+        """Returns the content of the specified message."""
+        raise NotImplementedException
+
+    def savemessage(self, uid, content, flags):
+        """Writes a new message, with the specified uid.
+        If the uid is < 0, the backend should assign a new uid and return it.
+
+        If the backend cannot assign a new uid, it returns the uid passed in
+        WITHOUT saving the message.
+
+        If the backend CAN assign a new uid, but cannot find out what this UID
+        is (as is the case with many IMAP servers), it returns 0 but DOES save
+        the message.
+        
+        IMAP backend should be the only one that can assign a new uid.
+
+        If the uid is > 0, the backend should set the uid to this, if it can.
+        If it cannot set the uid to that, it will save it anyway.
+        It will return the uid assigned in any case.
+        """
+        raise NotImplementedException
+
+    def getmessageflags(self, uid):
+        """Returns the flags for the specified message."""
+        raise NotImplementedException
+
+    def savemessageflags(self, uid, flags):
+        """Sets the specified message's flags to the given set."""
+        raise NotImplementedException
+
+    def addmessageflags(self, uid, flags):
+        """Adds the specified flags to the message's flag set.  If a given
+        flag is already present, it will not be duplicated."""
+        newflags = self.getmessageflags(uid)
+        for flag in flags:
+            if not flag in newflags:
+                newflags.append(flag)
+        newflags.sort()
+        self.savemessageflags(uid, newflags)
+
+    def addmessagesflags(self, uidlist, flags):
+        for uid in uidlist:
+            self.addmessageflags(uid, flags)
+
+    def deletemessageflags(self, uid, flags):
+        """Removes each flag given from the message's flag set.  If a given
+        flag is already removed, no action will be taken for that flag."""
+        newflags = self.getmessageflags(uid)
+        for flag in flags:
+            if flag in newflags:
+                newflags.remove(flag)
+        newflags.sort()
+        self.savemessageflags(uid, newflags)
+
+    def deletemessagesflags(self, uidlist, flags):
+        for uid in uidlist:
+            self.deletemessageflags(uid, flags)
+
+    def deletemessage(self, uid):
+        raise NotImplementedException
+
+    def deletemessages(self, uidlist):
+        for uid in uidlist:
+            self.deletemessage(uid)
+
+    def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1):
+        if register:
+            UIBase.getglobalui().registerthread(self.getaccountname())
+        UIBase.getglobalui().copyingmessage(uid, self, applyto)
+        successobject = None
+        successuid = None
+        message = self.getmessage(uid)
+        flags = self.getmessageflags(uid)
+        for tryappend in applyto:
+            successuid = tryappend.savemessage(uid, message, flags)
+            if successuid >= 0:
+                successobject = tryappend
+                break
+        # Did we succeed?
+        if successobject != None:
+            if successuid:       # Only if IMAP actually assigned a UID
+                # Copy the message to the other remote servers.
+                for appendserver in \
+                        [x for x in applyto if x != successobject]:
+                    appendserver.savemessage(successuid, message, flags)
+                    # Copy to its new name on the local server and delete
+                    # the one without a UID.
+                    self.savemessage(successuid, message, flags)
+            self.deletemessage(uid) # It'll be re-downloaded.
+        else:
+            # Did not find any server to take this message.  Ignore.
+            pass
+        
+
+    def syncmessagesto_neguid(self, dest, applyto):
+        """Pass 1 of folder synchronization.
+
+        Look for messages in self with a negative uid.  These are messages in
+        Maildirs that were not added by us.  Try to add them to the dests,
+        and once that succeeds, get the UID, add it to the others for real,
+        add it to local for real, and delete the fake one."""
+
+        uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0]
+        threads = []
+
+        usethread = None
+        if applyto != None:
+            usethread = applyto[0]
+        
+        for uid in uidlist:
+            if usethread and usethread.suggeststhreads():
+                usethread.waitforthread()
+                thread = InstanceLimitedThread(\
+                    usethread.getcopyinstancelimit(),
+                    target = self.syncmessagesto_neguid_msg,
+                    name = "New msg sync from %s" % self.getvisiblename(),
+                    args = (uid, dest, applyto))
+                thread.setDaemon(1)
+                thread.start()
+                threads.append(thread)
+            else:
+                self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0)
+        for thread in threads:
+            thread.join()
+
+    def copymessageto(self, uid, applyto, register = 1):
+        # Sometimes, it could be the case that if a sync takes awhile,
+        # a message might be deleted from the maildir before it can be
+        # synced to the status cache.  This is only a problem with
+        # self.getmessage().  So, don't call self.getmessage unless
+        # really needed.
+        if register:
+            UIBase.getglobalui().registerthread(self.getaccountname())
+        UIBase.getglobalui().copyingmessage(uid, self, applyto)
+        message = ''
+        # If any of the destinations actually stores the message body,
+        # load it up.
+        for object in applyto:
+            if object.storesmessages():
+                message = self.getmessage(uid)
+                break
+        flags = self.getmessageflags(uid)
+        for object in applyto:
+            newuid = object.savemessage(uid, message, flags)
+            if newuid > 0 and newuid != uid:
+                # Change the local uid.
+                self.savemessage(newuid, message, flags)
+                self.deletemessage(uid)
+                uid = newuid
+        
+
+    def syncmessagesto_copy(self, dest, applyto):
+        """Pass 2 of folder synchronization.
+
+        Look for messages present in self but not in dest.  If any, add
+        them to dest."""
+        threads = []
+        
+        for uid in self.getmessagelist().keys():
+            if uid < 0:                 # Ignore messages that pass 1 missed.
+                continue
+            if not uid in dest.getmessagelist():
+                if self.suggeststhreads():
+                    self.waitforthread()
+                    thread = InstanceLimitedThread(\
+                        self.getcopyinstancelimit(),
+                        target = self.copymessageto,
+                        name = "Copy message %d from %s" % (uid,
+                                                            self.getvisiblename()),
+                        args = (uid, applyto))
+                    thread.setDaemon(1)
+                    thread.start()
+                    threads.append(thread)
+                else:
+                    self.copymessageto(uid, applyto, register = 0)
+        for thread in threads:
+            thread.join()
+
+    def syncmessagesto_delete(self, dest, applyto):
+        """Pass 3 of folder synchronization.
+
+        Look for message present in dest but not in self.
+        If any, delete them."""
+        deletelist = []
+        for uid in dest.getmessagelist().keys():
+            if uid < 0:
+                continue
+            if not uid in self.getmessagelist():
+                deletelist.append(uid)
+        if len(deletelist):
+            UIBase.getglobalui().deletingmessages(deletelist, applyto)
+            for object in applyto:
+                object.deletemessages(deletelist)
+
+    def syncmessagesto_flags(self, dest, applyto):
+        """Pass 4 of folder synchronization.
+
+        Look for any flag matching issues -- set dest message to have the
+        same flags that we have."""
+
+        # As an optimization over previous versions, we store up which flags
+        # are being used for an add or a delete.  For each flag, we store
+        # a list of uids to which it should be added.  Then, we can call
+        # addmessagesflags() to apply them in bulk, rather than one
+        # call per message as before.  This should result in some significant
+        # performance improvements.
+
+        addflaglist = {}
+        delflaglist = {}
+        
+        for uid in self.getmessagelist().keys():
+            if uid < 0:                 # Ignore messages missed by pass 1
+                continue
+            selfflags = self.getmessageflags(uid)
+            destflags = dest.getmessageflags(uid)
+
+            addflags = [x for x in selfflags if x not in destflags]
+
+            for flag in addflags:
+                if not flag in addflaglist:
+                    addflaglist[flag] = []
+                addflaglist[flag].append(uid)
+
+            delflags = [x for x in destflags if x not in selfflags]
+            for flag in delflags:
+                if not flag in delflaglist:
+                    delflaglist[flag] = []
+                delflaglist[flag].append(uid)
+
+        for object in applyto:
+            for flag in addflaglist.keys():
+                UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object])
+                object.addmessagesflags(addflaglist[flag], [flag])
+            for flag in delflaglist.keys():
+                UIBase.getglobalui().deletingflags(delflaglist[flag], flag, [object])
+                object.deletemessagesflags(delflaglist[flag], [flag])
+                
+    def syncmessagesto(self, dest, applyto = None):
+        """Syncs messages in this folder to the destination.
+        If applyto is specified, it should be a list of folders (don't forget
+        to include dest!) to which all write actions should be applied.
+        It defaults to [dest] if not specified.  It is important that
+        the UID generator be listed first in applyto; that is, the other
+        applyto ones should be the ones that "copy" the main action."""
+        if applyto == None:
+            applyto = [dest]
+            
+        self.syncmessagesto_neguid(dest, applyto)
+        self.syncmessagesto_copy(dest, applyto)
+        self.syncmessagesto_delete(dest, applyto)
+        
+        # Now, the message lists should be identical wrt the uids present.
+        # (except for potential negative uids that couldn't be placed
+        # anywhere)
+
+        self.syncmessagesto_flags(dest, applyto)
+        
+            

offlineimap/folder/IMAP.py

@@ -0,0 +1,358 @@
+# IMAP folder support
+# Copyright (C) 2002-2004 John Goerzen
+# <jgoerzen@complete.org>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from Base import BaseFolder
+from offlineimap import imaputil, imaplib
+from offlineimap.ui import UIBase
+from offlineimap.version import versionstr
+import rfc822, time, string, random, binascii, re
+from StringIO import StringIO
+from copy import copy
+
+
+class IMAPFolder(BaseFolder):
+    def __init__(self, imapserver, name, visiblename, accountname, repository):
+        self.config = imapserver.config
+        self.expunge = repository.getexpunge()
+        self.name = imaputil.dequote(name)
+        self.root = None # imapserver.root
+        self.sep = imapserver.delim
+        self.imapserver = imapserver
+        self.messagelist = None
+        self.visiblename = visiblename
+        self.accountname = accountname
+        self.repository = repository
+        self.randomgenerator = random.Random()
+        BaseFolder.__init__(self)
+
+    def getaccountname(self):
+        return self.accountname
+
+    def suggeststhreads(self):
+        return 1
+
+    def waitforthread(self):
+        self.imapserver.connectionwait()
+
+    def getcopyinstancelimit(self):
+        return 'MSGCOPY_' + self.repository.getname()
+
+    def getvisiblename(self):
+        return self.visiblename
+
+    def getuidvalidity(self):
+        imapobj = self.imapserver.acquireconnection()
+        try:
+            # Primes untagged_responses
+            imapobj.select(self.getfullname(), readonly = 1)
+            return long(imapobj.untagged_responses['UIDVALIDITY'][0])
+        finally:
+            self.imapserver.releaseconnection(imapobj)
+    
+    def cachemessagelist(self):
+        imapobj = self.imapserver.acquireconnection()
+        self.messagelist = {}
+
+        try:
+            # Primes untagged_responses
+            imapobj.select(self.getfullname(), readonly = 1, force = 1)
+            try:
+                # Some mail servers do not return an EXISTS response if
+                # the folder is empty.
+                maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
+            except KeyError:
+                return
+            if maxmsgid < 1:
+                # No messages; return
+                return
+
+            # Now, get the flags and UIDs for these.
+            # We could conceivably get rid of maxmsgid and just say
+            # '1:*' here.
+            response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
+        finally:
+            self.imapserver.releaseconnection(imapobj)
+        for messagestr in response:
+            # Discard the message number.
+            messagestr = string.split(messagestr, maxsplit = 1)[1]
+            options = imaputil.flags2hash(messagestr)
+            if not options.has_key('UID'):
+                UIBase.getglobalui().warn('No UID in message with options %s' %\
+                                          str(options),
+                                          minor = 1)
+            else:
+                uid = long(options['UID'])
+                flags = imaputil.flagsimap2maildir(options['FLAGS'])
+                self.messagelist[uid] = {'uid': uid, 'flags': flags}
+
+    def getmessagelist(self):
+        return self.messagelist
+
+    def getmessage(self, uid):
+        ui = UIBase.getglobalui()
+        imapobj = self.imapserver.acquireconnection()
+        try:
+            imapobj.select(self.getfullname(), readonly = 1)
+            initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
+            ui.debug('imap', 'Returned object from fetching %d: %s' % \
+                     (uid, str(initialresult)))
+            return initialresult[1][0][1].replace("\r\n", "\n")
+                
+        finally:
+            self.imapserver.releaseconnection(imapobj)
+    
+    def getmessageflags(self, uid):
+        return self.messagelist[uid]['flags']
+
+    def savemessage_getnewheader(self, content):
+        headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
+        headername += binascii.hexlify(self.repository.getname()) + '-'
+        headername += binascii.hexlify(self.getname())
+        headervalue= '%d-' % long(time.time())
+        headervalue += str(self.randomgenerator.random()).replace('.', '')
+        headervalue += '-v' + versionstr
+        return (headername, headervalue)
+
+    def savemessage_addheader(self, content, headername, headervalue):
+        ui = UIBase.getglobalui()
+        ui.debug('imap',
+                 'savemessage_addheader: called to add %s: %s' % (headername,
+                                                                  headervalue))
+        insertionpoint = content.find("\r\n")
+        ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
+        leader = content[0:insertionpoint]
+        ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
+        if insertionpoint == 0 or insertionpoint == -1:
+            newline = ''
+            insertionpoint = 0
+        else:
+            newline = "\r\n"
+        newline += "%s: %s" % (headername, headervalue)
+        ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
+        trailer = content[insertionpoint:]
+        ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
+        return leader + newline + trailer
+
+    def savemessage_searchforheader(self, imapobj, headername, headervalue):
+        if imapobj.untagged_responses.has_key('APPENDUID'):
+            return long(imapobj.untagged_responses['APPENDUID'][0].split(' ')[1])
+
+        ui = UIBase.getglobalui()
+        ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
+                 (headername, headervalue))
+        # Now find the UID it got.
+        headervalue = imapobj._quote(headervalue)
+        try:
+            matchinguids = imapobj.uid('search', None,
+                                       '(HEADER %s %s)' % (headername, headervalue))[1][0]
+        except imapobj.error:
+            # IMAP server doesn't implement search or had a problem.
+            return 0
+        ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
+               
+        matchinguids = matchinguids.split(' ')
+        ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
+                 repr(matchinguids))
+        if len(matchinguids) != 1 or matchinguids[0] == None:
+            raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
+        matchinguids.sort()
+        return long(matchinguids[0])
+
+    def savemessage(self, uid, content, flags):
+        imapobj = self.imapserver.acquireconnection()
+        ui = UIBase.getglobalui()
+        ui.debug('imap', 'savemessage: called')
+        try:
+            try:
+                imapobj.select(self.getfullname()) # Needed for search
+            except imapobj.readonly:
+                ui.msgtoreadonly(self, uid, content, flags)
+                # Return indicating message taken, but no UID assigned.
+                # Fudge it.
+                return 0
+            
+            # This backend always assigns a new uid, so the uid arg is ignored.
+            # In order to get the new uid, we need to save off the message ID.
+
+            message = rfc822.Message(StringIO(content))
+            datetuple = rfc822.parsedate(message.getheader('Date'))
+            # Will be None if missing or not in a valid format.
+            if datetuple == None:
+                datetuple = time.localtime()
+            try:
+                if datetuple[0] < 1981:
+                    raise ValueError
+                # This could raise a value error if it's not a valid format.
+                date = imaplib.Time2Internaldate(datetuple) 
+            except ValueError:
+                # Argh, sometimes it's a valid format but year is 0102
+                # or something.  Argh.  It seems that Time2Internaldate
+                # will rause a ValueError if the year is 0102 but not 1902,
+                # but some IMAP servers nonetheless choke on 1902.
+                date = imaplib.Time2Internaldate(time.localtime())
+
+            ui.debug('imap', 'savemessage: using date ' + str(date))
+            content = re.sub("(?<!\r)\n", "\r\n", content)
+            ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
+
+            (headername, headervalue) = self.savemessage_getnewheader(content)
+            ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
+                     (headername, headervalue))
+            content = self.savemessage_addheader(content, headername,
+                                                 headervalue)
+            ui.debug('imap', 'savemessage: new content is: ' + repr(content))
+            ui.debug('imap', 'savemessage: new content length is ' + \
+                     str(len(content)))
+
+            assert(imapobj.append(self.getfullname(),
+                                       imaputil.flagsmaildir2imap(flags),
+                                       date, content)[0] == 'OK')
+
+            # Checkpoint.  Let it write out the messages, etc.
+            assert(imapobj.check()[0] == 'OK')
+
+            # Keep trying until we get the UID.
+            try:
+                ui.debug('imap', 'savemessage: first attempt to get new UID')
+                uid = self.savemessage_searchforheader(imapobj, headername,
+                                                       headervalue)
+            except ValueError:
+                ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
+                assert(imapobj.noop()[0] == 'OK')
+                uid = self.savemessage_searchforheader(imapobj, headername,
+                                                       headervalue)
+        finally:
+            self.imapserver.releaseconnection(imapobj)
+
+        self.messagelist[uid] = {'uid': uid, 'flags': flags}
+        ui.debug('imap', 'savemessage: returning %d' % uid)
+        return uid
+
+    def savemessageflags(self, uid, flags):
+        imapobj = self.imapserver.acquireconnection()
+        try:
+            try:
+                imapobj.select(self.getfullname())
+            except imapobj.readonly:
+                UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
+                return
+            result = imapobj.uid('store', '%d' % uid, 'FLAGS',
+                                 imaputil.flagsmaildir2imap(flags))
+            assert result[0] == 'OK', 'Error with store: ' + r[1]
+        finally:
+            self.imapserver.releaseconnection(imapobj)
+        result = result[1][0]
+        if not result:
+            self.messagelist[uid]['flags'] = flags
+        else:
+            flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
+            self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
+
+    def addmessageflags(self, uid, flags):
+        self.addmessagesflags([uid], flags)
+
+    def addmessagesflags_noconvert(self, uidlist, flags):
+        self.processmessagesflags('+', uidlist, flags)
+
+    def addmessagesflags(self, uidlist, flags):
+        """This is here for the sake of UIDMaps.py -- deletemessages must
+        add flags and get a converted UID, and if we don't have noconvert,
+        then UIDMaps will try to convert it twice."""
+        self.addmessagesflags_noconvert(uidlist, flags)
+
+    def deletemessageflags(self, uid, flags):
+        self.deletemessagesflags([uid], flags)
+
+    def deletemessagesflags(self, uidlist, flags):
+        self.processmessagesflags('-', uidlist, flags)
+
+    def processmessagesflags(self, operation, uidlist, flags):
+        imapobj = self.imapserver.acquireconnection()
+        try:
+            try:
+                imapobj.select(self.getfullname())
+            except imapobj.readonly:
+                UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
+                return
+            r = imapobj.uid('store',
+                            imaputil.listjoin(uidlist),
+                            operation + 'FLAGS',
+                            imaputil.flagsmaildir2imap(flags))
+            assert r[0] == 'OK', 'Error with store: ' + r[1]
+            r = r[1]
+        finally:
+            self.imapserver.releaseconnection(imapobj)
+        # Some IMAP servers do not always return a result.  Therefore,
+        # only update the ones that it talks about, and manually fix
+        # the others.
+        needupdate = copy(uidlist)
+        for result in r:
+            if result == None:
+                # Compensate for servers that don't return anything from
+                # STORE.
+                continue
+            attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
+            if not ('UID' in attributehash and 'FLAGS' in attributehash):
+                # Compensate for servers that don't return a UID attribute.
+                continue
+            flags = attributehash['FLAGS']
+            uid = long(attributehash['UID'])
+            self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
+            try:
+                needupdate.remove(uid)
+            except ValueError:          # Let it slide if it's not in the list
+                pass
+        for uid in needupdate:
+            if operation == '+':
+                for flag in flags:
+                    if not flag in self.messagelist[uid]['flags']:
+                        self.messagelist[uid]['flags'].append(flag)
+                    self.messagelist[uid]['flags'].sort()
+            elif operation == '-':
+                for flag in flags:
+                    if flag in self.messagelist[uid]['flags']:
+                        self.messagelist[uid]['flags'].remove(flag)
+
+    def deletemessage(self, uid):
+        self.deletemessages_noconvert([uid])
+
+    def deletemessages(self, uidlist):
+        self.deletemessages_noconvert(uidlist)
+
+    def deletemessages_noconvert(self, uidlist):
+        # Weed out ones not in self.messagelist
+        uidlist = [uid for uid in uidlist if uid in self.messagelist]
+        if not len(uidlist):
+            return        
+
+        self.addmessagesflags_noconvert(uidlist, ['T'])
+        imapobj = self.imapserver.acquireconnection()
+        try:
+            try:
+                imapobj.select(self.getfullname())
+            except imapobj.readonly:
+                UIBase.getglobalui().deletereadonly(self, uidlist)
+                return
+            if self.expunge:
+                assert(imapobj.expunge()[0] == 'OK')
+        finally:
+            self.imapserver.releaseconnection(imapobj)
+        for uid in uidlist:
+            del self.messagelist[uid]
+        
+        

offlineimap/folder/LocalStatus.py

@@ -0,0 +1,125 @@
+# Local status cache virtual folder
+# Copyright (C) 2002 - 2003 John Goerzen
+# <jgoerzen@complete.org>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from Base import BaseFolder
+import os, threading
+
+magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1"
+
+class LocalStatusFolder(BaseFolder):
+    def __init__(self, root, name, repository, accountname):
+        self.name = name
+        self.root = root
+        self.sep = '.'
+        self.filename = os.path.join(root, name)
+        self.filename = repository.getfolderfilename(name)
+        self.messagelist = None
+        self.repository = repository
+        self.savelock = threading.Lock()
+        self.doautosave = 1
+        self.accountname = accountname
+        BaseFolder.__init__(self)
+
+    def getaccountname(self):
+        return self.accountname
+
+    def storesmessages(self):
+        return 0
+
+    def isnewfolder(self):
+        return not os.path.exists(self.filename)
+
+    def getname(self):
+        return self.name
+
+    def getroot(self):
+        return self.root
+
+    def getsep(self):
+        return self.sep
+
+    def getfullname(self):
+        return self.filename
+
+    def deletemessagelist(self):
+        if not self.isnewfolder():
+            os.unlink(self.filename)
+
+    def cachemessagelist(self):
+        if self.isnewfolder():
+            self.messagelist = {}
+            return
+        file = open(self.filename, "rt")
+        self.messagelist = {}
+        line = file.readline().strip()
+        assert(line == magicline)
+        for line in file.xreadlines():
+            line = line.strip()
+            uid, flags = line.split(':')
+            uid = long(uid)
+            flags = [x for x in flags]
+            self.messagelist[uid] = {'uid': uid, 'flags': flags}
+        file.close()
+
+    def autosave(self):
+        if self.doautosave:
+            self.save()
+
+    def save(self):
+        self.savelock.acquire()
+        try:
+            file = open(self.filename + ".tmp", "wt")
+            file.write(magicline + "\n")
+            for msg in self.messagelist.values():
+                flags = msg['flags']
+                flags.sort()
+                flags = ''.join(flags)
+                file.write("%s:%s\n" % (msg['uid'], flags))
+            file.close()
+            os.rename(self.filename + ".tmp", self.filename)
+        finally:
+            self.savelock.release()
+
+    def getmessagelist(self):
+        return self.messagelist
+
+    def savemessage(self, uid, content, flags):
+        if uid < 0:
+            # We cannot assign a uid.
+            return uid
+
+        if uid in self.messagelist:     # already have it
+            self.savemessageflags(uid, flags)
+            return uid
+
+        self.messagelist[uid] = {'uid': uid, 'flags': flags}
+        self.autosave()
+        return uid
+
+    def getmessageflags(self, uid):
+        return self.messagelist[uid]['flags']
+
+    def savemessageflags(self, uid, flags):
+        self.messagelist[uid]['flags'] = flags
+        self.autosave()
+
+    def deletemessage(self, uid):
+        if not uid in self.messagelist:
+            return
+        del(self.messagelist[uid])
+        self.autosave()

offlineimap/folder/Maildir.py

@@ -0,0 +1,220 @@
+# Maildir folder support
+# Copyright (C) 2002 John Goerzen
+# <jgoerzen@complete.org>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from Base import BaseFolder
+from offlineimap import imaputil
+from offlineimap.ui import UIBase
+from threading import Lock
+import os.path, os, re, time, socket, md5
+
+foldermatchre = re.compile(',FMD5=([0-9a-f]{32})')
+uidmatchre = re.compile(',U=(\d+)')
+flagmatchre = re.compile(':.*2,([A-Z]+)')
+
+timeseq = 0
+lasttime = long(0)
+timelock = Lock()
+
+def gettimeseq():
+    global lasttime, timeseq, timelock
+    timelock.acquire()
+    try:
+        thistime = long(time.time())
+        if thistime == lasttime:
+            timeseq += 1
+            return (thistime, timeseq)
+        else:
+            lasttime = thistime
+            timeseq = 0
+            return (thistime, timeseq)
+    finally:
+        timelock.release()
+
+class MaildirFolder(BaseFolder):
+    def __init__(self, root, name, sep, repository, accountname):
+        self.name = name
+        self.root = root
+        self.sep = sep
+        self.messagelist = None
+        self.repository = repository
+        self.accountname = accountname
+        BaseFolder.__init__(self)
+
+    def getaccountname(self):
+        return self.accountname
+
+    def getfullname(self):
+        return os.path.join(self.getroot(), self.getname())
+
+    def getuidvalidity(self):
+        """Maildirs have no notion of uidvalidity, so we just return a magic
+        token."""
+        return 42
+
+    def _scanfolder(self):
+        """Cache the message list.  Maildir flags are:
+        R (replied)
+        S (seen)
+        T (trashed)
+        D (draft)
+        F (flagged)
+        and must occur in ASCII order."""
+        retval = {}
+        files = []
+        nouidcounter = -1               # Messages without UIDs get
+                                        # negative UID numbers.
+        for dirannex in ['new', 'cur']:
+            fulldirname = os.path.join(self.getfullname(), dirannex)
+            files.extend([os.path.join(fulldirname, filename) for
+                          filename in os.listdir(fulldirname)])
+        for file in files:
+            messagename = os.path.basename(file)
+            foldermatch = foldermatchre.search(messagename)
+            if (not foldermatch) or \
+               md5.new(self.getvisiblename()).hexdigest() \
+               != foldermatch.group(1):
+                # If there is no folder MD5 specified, or if it mismatches,
+                # assume it is a foreign (new) message and generate a
+                # negative uid for it
+                uid = nouidcounter
+                nouidcounter -= 1
+            else:                       # It comes from our folder.
+                uidmatch = uidmatchre.search(messagename)
+                uid = None
+                if not uidmatch:
+                    uid = nouidcounter
+                    nouidcounter -= 1
+                else:
+                    uid = long(uidmatch.group(1))
+            flagmatch = flagmatchre.search(messagename)
+            flags = []
+            if flagmatch:
+                flags = [x for x in flagmatch.group(1)]
+            flags.sort()
+            retval[uid] = {'uid': uid,
+                           'flags': flags,
+                           'filename': file}
+        return retval
+
+    def cachemessagelist(self):
+        self.messagelist = self._scanfolder()
+            
+    def getmessagelist(self):
+        return self.messagelist
+
+    def getmessage(self, uid):
+        filename = self.messagelist[uid]['filename']
+        file = open(filename, 'rt')
+        retval = file.read()
+        file.close()
+        return retval.replace("\r\n", "\n")
+
+    def savemessage(self, uid, content, flags):
+        ui = UIBase.getglobalui()
+        ui.debug('maildir', 'savemessage: called to write with flags %s and content %s' % \
+                 (repr(flags), repr(content)))
+        if uid < 0:
+            # We cannot assign a new uid.
+            return uid
+        if uid in self.messagelist:
+            # We already have it.
+            self.savemessageflags(uid, flags)
+            return uid
+        if 'S' in flags:
+            # If a message has been seen, it goes into the cur
+            # directory.  CR debian#152482, [complete.org #4]
+            newdir = os.path.join(self.getfullname(), 'cur')
+        else:
+            newdir = os.path.join(self.getfullname(), 'new')
+        tmpdir = os.path.join(self.getfullname(), 'tmp')
+        messagename = None
+        attempts = 0
+        while 1:
+            if attempts > 15:
+                raise IOError, "Couldn't write to file %s" % messagename
+            timeval, timeseq = gettimeseq()
+            messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \
+                          (timeval,
+                           timeseq,
+                           os.getpid(),
+                           socket.gethostname(),
+                           uid,
+                           md5.new(self.getvisiblename()).hexdigest())
+            if os.path.exists(os.path.join(tmpdir, messagename)):
+                time.sleep(2)
+                attempts += 1
+            else:
+                break
+        tmpmessagename = messagename.split(',')[0]
+        ui.debug('maildir', 'savemessage: using temporary name %s' % tmpmessagename)
+        file = open(os.path.join(tmpdir, tmpmessagename), "wt")
+        file.write(content)
+        file.close()
+        ui.debug('maildir', 'savemessage: moving from %s to %s' % \
+                 (tmpmessagename, messagename))
+        os.link(os.path.join(tmpdir, tmpmessagename),
+                os.path.join(newdir, messagename))
+        os.unlink(os.path.join(tmpdir, tmpmessagename))
+        self.messagelist[uid] = {'uid': uid, 'flags': [],
+                                 'filename': os.path.join(newdir, messagename)}
+        self.savemessageflags(uid, flags)
+        ui.debug('maildir', 'savemessage: returning uid %d' % uid)
+        return uid
+        
+    def getmessageflags(self, uid):
+        return self.messagelist[uid]['flags']
+
+    def savemessageflags(self, uid, flags):
+        oldfilename = self.messagelist[uid]['filename']
+        newpath, newname = os.path.split(oldfilename)
+        if 'S' in flags:
+            # If a message has been seen, it goes into the cur
+            # directory.  CR debian#152482, [complete.org #4]
+            newpath = os.path.join(self.getfullname(), 'cur')
+        else:
+            newpath = os.path.join(self.getfullname(), 'new')
+        infostr = ':'
+        infomatch = re.search('(:.*)$', newname)
+        if infomatch:                   # If the info string is present..
+            infostr = infomatch.group(1)
+            newname = newname.split(':')[0] # Strip off the info string.
+        infostr = re.sub('2,[A-Z]*', '', infostr)
+        flags.sort()
+        infostr += '2,' + ''.join(flags)
+        newname += infostr
+        
+        newfilename = os.path.join(newpath, newname)
+        if (newfilename != oldfilename):
+            os.rename(oldfilename, newfilename)
+            self.messagelist[uid]['flags'] = flags
+            self.messagelist[uid]['filename'] = newfilename
+
+    def deletemessage(self, uid):
+        if not uid in self.messagelist:
+            return
+        filename = self.messagelist[uid]['filename']
+        try:
+            os.unlink(filename)
+        except OSError:
+            # Can't find the file -- maybe already deleted?
+            newmsglist = self._scanfolder()
+            if uid in newmsglist:       # Nope, try new filename.
+                os.unlink(newmsglist[uid]['filename'])
+            # Yep -- return.
+        del(self.messagelist[uid])
+        

offlineimap/folder/__init__.py

@@ -0,0 +1,2 @@
+import Base, IMAP, Maildir, LocalStatus
+