Merge branches 'master' and 'master' of ssh://jpgarch@complete.org/~jpgarch/git/offlineimap

COPYRIGHT

@@ -1,5 +1,5 @@
 offlineimap Mail syncing software
-Copyright (C) 2002 - 2008 John Goerzen
+Copyright (C) 2002 - 2009 John Goerzen
 <jgoerzen@complete.org>
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by

bin/offlineimap

@@ -18,4 +18,4 @@
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
 from offlineimap import init
-init.startup('6.1.2')
+init.startup('6.2.0')

debian/changelog

@@ -1,3 +1,12 @@
+offlineimap (6.2.0) unstable; urgency=low
+
+  * FIX: Removed imaplib2 entirely.  It was the cause of numerous
+    crash/hang bugs.
+  * REMOVAL: IDLE support had to be removed because it depended on
+    imaplib2.
+
+ -- John Goerzen <jgoerzen@complete.org>  Wed, 12 Aug 2009 00:40:31 -0500
+
 offlineimap (6.1.2) unstable; urgency=low
 
   * Applied patch from Peter Colberg to remove usage of hard linking.

debian/copyright

@@ -4,7 +4,7 @@ on Fri, 21 Jun 2002 14:54:56 -0500.
 The original source can always be found at:
         http://software.complete.org/offlineimap/
 
-Copyright (C) 2002 - 2008 John Goerzen
+Copyright (C) 2002 - 2009 John Goerzen
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

offlineimap.conf

@@ -307,19 +307,6 @@ remoteuser = username
 #
 # reference = Mail
 
-# In between synchronisations, OfflineIMAP can monitor mailboxes for new
-# messages using the IDLE command. If you want to enable this, specify here
-# the folders you wish to monitor. Note that the IMAP protocol requires a
-# separate connection for each folder monitored in this way, so setting
-# this option will force settings for:
-#     maxconnections - to be at least the number of folders you give
-#     holdconnectionopen - to be true
-#     keepalive - to be 29 minutes unless you specify otherwise
-# This option should return a Python list. For example
-#
-# idlefolders = ['INBOX', 'INBOX.Alerts']
-#
-
 # OfflineIMAP can use multiple connections to the server in order
 # to perform multiple synchronization actions simultaneously.
 # This may place a higher burden on the server.  In most cases,

offlineimap.py

@@ -18,4 +18,4 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 from offlineimap import init
-init.startup('6.1.2')
+init.startup('6.2.0')

offlineimap/accounts.py

@@ -109,7 +109,6 @@ class Account(CustomConfig.ConfigHelperMixin):
         self.localeval = config.getlocaleval()
         self.ui = UIBase.getglobalui()
         self.refreshperiod = self.getconffloat('autorefresh', 0.0)
-        self.quickrefreshcount = self.getconfint('quick', 0)
         self.quicknum = 0
         if self.refreshperiod == 0.0:
             self.refreshperiod = None
@@ -146,10 +145,7 @@ class Account(CustomConfig.ConfigHelperMixin):
         for item in kaobjs:
             item.startkeepalive()
         
-        sleeptime = int(self.refreshperiod * 60)
+        refreshperiod = int(self.refreshperiod * 60)
-        if (self.quickrefreshcount > 0):
-            sleeptime = int(sleeptime / self.quickrefreshcount)
-
 #         try:
 #             sleepresult = siglistener.get_nowait()
 #             # retrieved signal before sleep started
@@ -157,8 +153,8 @@ class Account(CustomConfig.ConfigHelperMixin):
 #                 # catching signal 1 here means folders were cleared before signal was posted
 #                 pass
 #         except Empty:
-#             sleepresult = self.ui.sleep(sleeptime, siglistener)
+#             sleepresult = self.ui.sleep(refreshperiod, siglistener)
-        sleepresult = self.ui.sleep(sleeptime, siglistener)
+        sleepresult = self.ui.sleep(refreshperiod, siglistener)
         if sleepresult == 1:
             self.quicknum = 0
 

offlineimap/folder/Gmail.py

@@ -20,7 +20,8 @@
 """
 
 from IMAP import IMAPFolder
-from offlineimap import imaplib2, imaputil, imaplibutil
+import imaplib
+from offlineimap import imaputil, imaplibutil
 from offlineimap.ui import UIBase
 from copy import copy
 

offlineimap/folder/IMAP.py

@@ -17,7 +17,8 @@
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
 from Base import BaseFolder
-from offlineimap import imaplib2, imaputil, imaplibutil
+import imaplib
+from offlineimap import imaputil, imaplibutil
 from offlineimap.ui import UIBase
 from offlineimap.version import versionstr
 import rfc822, time, string, random, binascii, re
@@ -270,13 +271,13 @@ class IMAPFolder(BaseFolder):
                     raise ValueError
 
                 # This could raise a value error if it's not a valid format.
-                date = imaplib2.Time2Internaldate(datetuple) 
+                date = imaplib.Time2Internaldate(datetuple) 
             except (ValueError, OverflowError):
                 # Argh, sometimes it's a valid format but year is 0102
                 # or something.  Argh.  It seems that Time2Internaldate
                 # will rause a ValueError if the year is 0102 but not 1902,
                 # but some IMAP servers nonetheless choke on 1902.
-                date = imaplib2.Time2Internaldate(time.localtime())
+                date = imaplib.Time2Internaldate(time.localtime())
 
             ui.debug('imap', 'savemessage: using date ' + str(date))
             content = re.sub("(?<!\r)\n", "\r\n", content)

offlineimap/imaplibutil.py

@@ -16,12 +16,12 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-import os, re, socket, time, subprocess, sys, threading
+import re, string, types, binascii, socket, time, random, subprocess, sys, os
 from offlineimap.ui import UIBase
-from offlineimap.imaplib2 import *
+from imaplib import *
 
 # Import the symbols we need that aren't exported by default
-from offlineimap.imaplib2 import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num
+from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num
 
 # ssl is new in python 2.6
 if (sys.version_info[0] == 2 and sys.version_info[1] >= 6) or sys.version_info[0] >= 3:
@@ -43,10 +43,12 @@ class IMAP4_Tunnel(IMAP4):
         self.process = subprocess.Popen(host, shell=True, close_fds=True,
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
         (self.outfd, self.infd) = (self.process.stdin, self.process.stdout)
-        self.read_fd = self.infd.fileno()
 
     def read(self, size):
-        return os.read(self.read_fd, size)
+        retval = ''
+        while len(retval) < size:
+            retval += self.infd.read(size - len(retval))
+        return retval
 
     def readline(self):
         return self.infd.readline()
@@ -98,22 +100,78 @@ class sslwrapper:
             else:
                 retval += linebuf
 
-def new_mesg(self, s, tn=None, secs=None):
+def new_mesg(self, s, secs=None):
             if secs is None:
                 secs = time.time()
-            if tn is None:
-                tn = threading.currentThread().getName()
             tm = time.strftime('%M:%S', time.localtime(secs))
-            UIBase.getglobalui().debug('imap', '  %s.%02d %s %s' % (tm, (secs*100)%100, tn, s))
+            UIBase.getglobalui().debug('imap', '  %s.%02d %s' % (tm, (secs*100)%100, s))
 
 class WrappedIMAP4_SSL(IMAP4_SSL):
-    def open(self, host=None, port=None):
+    def open(self, host = '', port = IMAP4_SSL_PORT):
         IMAP4_SSL.open(self, host, port)
         self.sslobj = sslwrapper(self.sslobj)
 
     def readline(self):
         return self.sslobj.readline()
 
+def new_open(self, host = '', port = IMAP4_PORT):
+        """Setup connection to remote server on "host:port"
+            (default: localhost:standard IMAP4 port).
+        This connection will be used by the routines:
+            read, readline, send, shutdown.
+        """
+        self.host = host
+        self.port = port
+        res = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
+                                 socket.SOCK_STREAM)
+
+        # Try each address returned by getaddrinfo in turn until we
+        # manage to connect to one.
+        # Try all the addresses in turn until we connect()
+        last_error = 0
+        for remote in res:
+            af, socktype, proto, canonname, sa = remote
+            self.sock = socket.socket(af, socktype, proto)
+            last_error = self.sock.connect_ex(sa)
+            if last_error == 0:
+                break
+            else:
+                self.sock.close()
+        if last_error != 0:
+            # FIXME
+            raise socket.error(last_error)
+        self.file = self.sock.makefile('rb')
+
+def new_open_ssl(self, host = '', port = IMAP4_SSL_PORT):
+        """Setup connection to remote server on "host:port".
+            (default: localhost:standard IMAP4 SSL port).
+        This connection will be used by the routines:
+            read, readline, send, shutdown.
+        """
+        self.host = host
+        self.port = port
+        #This connects to the first ip found ipv4/ipv6
+        #Added by Adriaan Peeters <apeeters@lashout.net> based on a socket
+        #example from the python documentation:
+        #http://www.python.org/doc/lib/socket-example.html
+        res = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
+                                 socket.SOCK_STREAM)
+        # Try all the addresses in turn until we connect()
+        last_error = 0
+        for remote in res:
+            af, socktype, proto, canonname, sa = remote
+            self.sock = socket.socket(af, socktype, proto)
+            last_error = self.sock.connect_ex(sa)
+            if last_error == 0:
+                break
+            else:
+                self.sock.close()
+        if last_error != 0:
+            # FIXME
+            raise socket.error(last_error)
+        self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile)
+        self.sslobj = sslwrapper(self.sslobj)
+
 mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]")
 
 def Internaldate2epoch(resp):

offlineimap/imapserver.py

@@ -16,9 +16,9 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-from offlineimap import imaplib2, imaplibutil, imaputil, threadutil
+import imaplib
+from offlineimap import imaplibutil, imaputil, threadutil
 from offlineimap.ui import UIBase
-from offlineimap.accounts import syncfolder
 from threading import *
 import thread, hmac, os, time
 import base64
@@ -56,10 +56,13 @@ class UsefulIMAPMixIn:
         else:
             self.selectedfolder = None
 
-    def _mesg(self, s, tn=None, secs=None):
+    def _mesg(self, s, secs=None):
-        imaplibutil.new_mesg(self, s, tn, secs)
+        imaplibutil.new_mesg(self, s, secs)
+
+class UsefulIMAP4(UsefulIMAPMixIn, imaplib.IMAP4):
+    def open(self, host = '', port = imaplib.IMAP4_PORT):
+        imaplibutil.new_open(self, host, port)
 
-class UsefulIMAP4(UsefulIMAPMixIn, imaplib2.IMAP4):
     # This is a hack around Darwin's implementation of realloc() (which
     # Python uses inside the socket code). On Darwin, we split the
     # message into 100k chunks, which should be small enough - smaller
@@ -70,17 +73,17 @@ class UsefulIMAP4(UsefulIMAPMixIn, imaplib2.IMAP4):
             read = 0
             io = StringIO()
             while read < size:
-                sz = min(size-read, 8192)
+                data = imaplib.IMAP4.read (self, min(size-read,8192))
-                data = imaplib2.IMAP4.read (self, sz)
                 read += len(data)
                 io.write(data)
-                if len(data) < sz:
-                    break
             return io.getvalue()
         else:
-            return imaplib2.IMAP4.read (self, size)
+            return imaplib.IMAP4.read (self, size)
 
 class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
+    def open(self, host = '', port = imaplib.IMAP4_SSL_PORT):
+        imaplibutil.new_open_ssl(self, host, port)
+
     # This is the same hack as above, to be used in the case of an SSL
     # connexion.
 
@@ -89,12 +92,9 @@ class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
             read = 0
             io = StringIO()
             while read < size:
-                sz = min(size-read,8192)
+                data = imaplibutil.WrappedIMAP4_SSL.read (self, min(size-read,8192))
-                data = imaplibutil.WrappedIMAP4_SSL.read (self, sz)
                 read += len(data)
                 io.write(data)
-                if len(data) < sz:
-                    break
             return io.getvalue()
         else:
             return imaplibutil.WrappedIMAP4_SSL.read (self,size)
@@ -107,8 +107,7 @@ class IMAPServer:
     def __init__(self, config, reposname,
                  username = None, password = None, hostname = None,
                  port = None, ssl = 1, maxconnections = 1, tunnel = None,
-                 reference = '""', sslclientcert = None, sslclientkey = None,
+                 reference = '""', sslclientcert = None, sslclientkey = None):
-                 idlefolders = []):
         self.reposname = reposname
         self.config = config
         self.username = username
@@ -135,7 +134,6 @@ class IMAPServer:
         self.semaphore = BoundedSemaphore(self.maxconnections)
         self.connectionlock = Lock()
         self.reference = reference
-        self.idlefolders = idlefolders
         self.gss_step = self.GSS_STATE_STEP
         self.gss_vc = None
         self.gssapi = False
@@ -351,6 +349,8 @@ class IMAPServer:
         ui.debug('imap', 'keepalive thread started')
         while 1:
             ui.debug('imap', 'keepalive: top of loop')
+            time.sleep(timeout)
+            ui.debug('imap', 'keepalive: after wait')
             if event.isSet():
                 ui.debug('imap', 'keepalive: event is set; exiting')
                 return
@@ -361,91 +361,32 @@ class IMAPServer:
             self.connectionlock.release()
             ui.debug('imap', 'keepalive: connectionlock released')
             threads = []
+            imapobjs = []
         
             for i in range(numconnections):
                 ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
-                if len(self.idlefolders) > i:
+                imapobj = self.acquireconnection()
-                    idler = IdleThread(self, self.idlefolders[i])
+                ui.debug('imap', 'keepalive: connection %d acquired' % i)
-                else:
+                imapobjs.append(imapobj)
-                    idler = IdleThread(self)
+                thr = threadutil.ExitNotifyThread(target = imapobj.noop)
-                idler.start()
+                thr.setDaemon(1)
-                threads.append(idler)
+                thr.start()
+                threads.append(thr)
                 ui.debug('imap', 'keepalive: thread started')
 
-            ui.debug('imap', 'keepalive: waiting for timeout')
-            event.wait(timeout)
-
             ui.debug('imap', 'keepalive: joining threads')
 
-            for idler in threads:
+            for thr in threads:
                 # Make sure all the commands have completed.
-                idler.stop()
+                thr.join()
-                idler.join()
+
+            ui.debug('imap', 'keepalive: releasing connections')
+
+            for imapobj in imapobjs:
+                self.releaseconnection(imapobj)
 
             ui.debug('imap', 'keepalive: bottom of loop')
 
-class IdleThread(object):
-    def __init__(self, parent, folder=None):
-        self.parent = parent
-        self.folder = folder
-        self.event = Event()
-        if folder is None:
-            self.thread = Thread(target=self.noop)
-        else:
-            self.thread = Thread(target=self.idle)
-        self.thread.setDaemon(1)
-
-    def start(self):
-        self.thread.start()
-
-    def stop(self):
-        self.event.set()
-
-    def join(self):
-        self.thread.join()
-
-    def noop(self):
-        imapobj = self.parent.acquireconnection()
-        imapobj.noop()
-        self.event.wait()
-        self.parent.releaseconnection(imapobj)
-
-    def dosync(self):
-        remoterepos = self.parent.repos
-        account = remoterepos.account
-        localrepos = account.localrepos
-        remoterepos = account.remoterepos
-        statusrepos = account.statusrepos
-        remotefolder = remoterepos.getfolder(self.folder)
-        syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False)
-        ui = UIBase.getglobalui()
-        ui.unregisterthread(currentThread())
-
-    def idle(self):
-        imapobj = self.parent.acquireconnection()
-        imapobj.select(self.folder)
-        self.parent.releaseconnection(imapobj)
-        while True:
-            if self.event.isSet():
-                return
-            self.needsync = False
-            def callback(args):
-                if not self.event.isSet():
-                    self.needsync = True
-                    self.event.set()
-            imapobj = self.parent.acquireconnection()
-            if "IDLE" in imapobj.capabilities:
-                imapobj.idle(callback=callback)
-            else:
-                imapobj.noop()
-            self.event.wait()
-            if self.event.isSet():
-                imapobj.noop()
-            self.parent.releaseconnection(imapobj)
-            if self.needsync:
-                self.event.clear()
-                self.dosync()
-
 class ConfigedIMAPServer(IMAPServer):
     """This class is designed for easier initialization given a ConfigParser
     object and an account name.  The passwordhash is used if
@@ -465,7 +406,6 @@ class ConfigedIMAPServer(IMAPServer):
             sslclientcert = self.repos.getsslclientcert()
             sslclientkey = self.repos.getsslclientkey()
         reference = self.repos.getreference()
-        idlefolders = self.repos.getidlefolders()
         server = None
         password = None
         
@@ -477,7 +417,6 @@ class ConfigedIMAPServer(IMAPServer):
             IMAPServer.__init__(self, self.config, self.repos.getname(),
                                 tunnel = usetunnel,
                                 reference = reference,
-                                idlefolders = idlefolders,
                                 maxconnections = self.repos.getmaxconnections())
         else:
             if not password:
@@ -486,6 +425,5 @@ class ConfigedIMAPServer(IMAPServer):
                                 user, password, host, port, ssl,
                                 self.repos.getmaxconnections(),
                                 reference = reference,
-                                idlefolders = idlefolders,
                                 sslclientcert = sslclientcert,
                                 sslclientkey = sslclientkey)

offlineimap/init.py

@@ -16,7 +16,8 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-from offlineimap import imaplib2, imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
+import imaplib
+from offlineimap import imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
 from offlineimap.localeval import LocalEval
 from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
 from offlineimap.ui import UIBase
@@ -102,7 +103,7 @@ def startup(versionno):
         for debugtype in options['-d'].split(','):
             ui.add_debug(debugtype.strip())
             if debugtype == 'imap':
-                imaplib2.Debug = 5
+                imaplib.Debug = 5
             if debugtype == 'thread':
                 threading._VERBOSE = 1
 

offlineimap/repository/IMAP.py

@@ -74,16 +74,10 @@ class IMAPRepository(BaseRepository):
         self.imapserver.close()
 
     def getholdconnectionopen(self):
-        if self.getidlefolders():
-            return 1
         return self.getconfboolean("holdconnectionopen", 0)
 
     def getkeepalive(self):
-        num = self.getconfint("keepalive", 0)
+        return self.getconfint("keepalive", 0)
-        if num == 0 and self.getidlefolders():
-            return 29*60
-        else:
-            return num
 
     def getsep(self):
         return self.imapserver.delim
@@ -151,14 +145,8 @@ class IMAPRepository(BaseRepository):
     def getreference(self):
         return self.getconf('reference', '""')
 
-    def getidlefolders(self):
-        localeval = self.localeval
-        return localeval.eval(self.getconf('idlefolders', '[]'))
-
     def getmaxconnections(self):
-        num1 = len(self.getidlefolders())
+        return self.getconfint('maxconnections', 1)
-        num2 = self.getconfint('maxconnections', 1)
-        return max(num1, num2)
 
     def getexpunge(self):
         return self.getconfboolean('expunge', 1)

offlineimap/syncmaster.py

@@ -16,7 +16,8 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-from offlineimap import imaplib2, imapserver, repository, folder, mbnames, threadutil, version
+import imaplib
+from offlineimap import imapserver, repository, folder, mbnames, threadutil, version
 from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
 import offlineimap.accounts
 from offlineimap.accounts import SyncableAccount, SigListener

offlineimap/version.py

@@ -1,5 +1,5 @@
 productname = 'OfflineIMAP'
-versionstr = "6.1.2"
+versionstr = "6.2.0"
 
 versionlist = versionstr.split(".")
 major = versionlist[0]