Merge branch 'egc/imaplib2' into next

Rebased-For-Conflicts-Resolution-By: Sebastian Spaeth <Sebastian@SSpaeth.de>

offlineimap/folder/Gmail.py

@@ -20,8 +20,7 @@
 """
 
 from IMAP import IMAPFolder
-import imaplib
+from offlineimap import imaputil
-from offlineimap import imaputil, imaplibutil
 from copy import copy
 
 

offlineimap/folder/IMAP.py

@@ -16,7 +16,6 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-import imaplib
 import email
 import random
 import binascii
@@ -72,7 +71,7 @@ class IMAPFolder(BaseFolder):
         try:
             # Primes untagged_responses
             self.selectro(imapobj)
-            return long(imapobj.untagged_responses['UIDVALIDITY'][0])
+            return long(imapobj._get_untagged_response('UIDVALIDITY', True)[0])
         finally:
             self.imapserver.releaseconnection(imapobj)
     
@@ -83,17 +82,16 @@ class IMAPFolder(BaseFolder):
         imapobj = self.imapserver.acquireconnection()
         try:
             # Primes untagged_responses
-            imapobj.select(self.getfullname(), readonly = 1, force = 1)
+            imaptype, imapdata = imapobj.select(self.getfullname(), readonly = 1, force = 1)
-            try:
+            # 1. Some mail servers do not return an EXISTS response
-                # 1. Some mail servers do not return an EXISTS response
+            # if the folder is empty.  2. ZIMBRA servers can return
-                # if the folder is empty.  2. ZIMBRA servers can return
+            # multiple EXISTS replies in the form 500, 1000, 1500,
-                # multiple EXISTS replies in the form 500, 1000, 1500,
+            # 1623 so check for potentially multiple replies.
-                # 1623 so check for potentially multiple replies.
+            if imapdata == [None]:
-                maxmsgid = 0
-                for msgid in imapobj.untagged_responses['EXISTS']:
-                    maxmsgid = max(long(msgid), maxmsgid)
-            except KeyError:
                 return True
+            maxmsgid = 0
+            for msgid in imapdata:
+                maxmsgid = max(long(msgid), maxmsgid)
 
             # Different number of messages than last time?
             if maxmsgid != len(statusfolder.getmessagelist()):
@@ -128,7 +126,7 @@ class IMAPFolder(BaseFolder):
 
         try:
             # Primes untagged_responses
-            imapobj.select(self.getfullname(), readonly = 1, force = 1)
+            imaptype, imapdata = imapobj.select(self.getfullname(), readonly = 1, force = 1)
 
             maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1)
             maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1)
@@ -170,17 +168,20 @@ class IMAPFolder(BaseFolder):
                     # No messages; return
                     return
             else:
-                try:
+                # 1. Some mail servers do not return an EXISTS response
-                    # 1. Some mail servers do not return an EXISTS response
+                # if the folder is empty.  2. ZIMBRA servers can return
-                    # if the folder is empty.  2. ZIMBRA servers can return
+                # multiple EXISTS replies in the form 500, 1000, 1500,
-                    # multiple EXISTS replies in the form 500, 1000, 1500,
+                # 1623 so check for potentially multiple replies.
-                    # 1623 so check for potentially multiple replies.
+                if imapdata == [None]:
-                    maxmsgid = 0
-                    for msgid in imapobj.untagged_responses['EXISTS']:
-                        maxmsgid = max(long(msgid), maxmsgid)
-                    messagesToFetch = '1:%d' % maxmsgid;
-                except KeyError:
                     return
+
+                maxmsgid = 0
+                for msgid in imapdata:
+                    maxmsgid = max(long(msgid), maxmsgid)
+
+                maxmsgid = long(imapdata[0])
+                messagesToFetch = '1:%d' % maxmsgid;
+
                 if maxmsgid < 1:
                     #no messages; return
                     return
@@ -438,10 +439,11 @@ class IMAPFolder(BaseFolder):
                 # get the new UID from the APPENDUID response, it could look like
                 # OK [APPENDUID 38505 3955] APPEND completed
                 # with 38505 bein folder UIDvalidity and 3955 the new UID
-                if not imapobj.untagged_responses.has_key('APPENDUID'):
+                if not imapobj._get_untagged_response('APPENDUID', True):
-                    self.ui.warn("Server supports UIDPLUS but got no APPENDUID appending a message.")
+                    self.ui.warn("Server supports UIDPLUS but got no APPENDUID "
+                                 "appending a message.")
                     return 0
-                uid = long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
+                uid = long(imapobj._get_untagged_response('APPENDUID', True)[-1].split(' ')[1])
 
             else:
                 # we don't support UIDPLUS

offlineimap/imaplibutil.py

@@ -17,9 +17,11 @@
 
 import re, socket, time, subprocess
 from offlineimap.ui import getglobalui
+import threading
+from offlineimap.imaplib2 import *
 
 # Import the symbols we need that aren't exported by default
-from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num, IMAP4, IMAP4_SSL
+from offlineimap.imaplib2 import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num
 
 try:
     import ssl
@@ -43,6 +45,8 @@ class IMAP4_Tunnel(IMAP4):
         self.process = subprocess.Popen(host, shell=True, close_fds=True,
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
         (self.outfd, self.infd) = (self.process.stdin, self.process.stdout)
+        # imaplib2 polls on this fd
+        self.read_fd = self.infd.fileno()
 
     def read(self, size):
         retval = ''
@@ -65,11 +69,13 @@ class IMAP4_Tunnel(IMAP4):
         self.process.wait()
 
 
-def new_mesg(self, s, secs=None):
+def new_mesg(self, s, tn=None, secs=None):
             if secs is None:
                 secs = time.time()
+            if tn is None:
+                tn = threading.currentThread().getName()
             tm = time.strftime('%M:%S', time.localtime(secs))
-            getglobalui().debug('imap', '  %s.%02d %s' % (tm, (secs*100)%100, s))
+            getglobalui().debug('imap', '  %s.%02d %s %s' % (tm, (secs*100)%100, tn, s))
 
 class WrappedIMAP4_SSL(IMAP4_SSL):
     """Provides an improved version of the standard IMAP4_SSL
@@ -84,7 +90,7 @@ class WrappedIMAP4_SSL(IMAP4_SSL):
             del kwargs['cacertfile']
         IMAP4_SSL.__init__(self, *args, **kwargs)
 
-    def open(self, host = '', port = IMAP4_SSL_PORT):
+    def open(self, host=None, port=None):
         """Do whatever IMAP4_SSL would do in open, but call sslwrap
         with cert verification"""
         #IMAP4_SSL.open(self, host, port) uses the below 2 lines:
@@ -147,6 +153,9 @@ class WrappedIMAP4_SSL(IMAP4_SSL):
                 if error:
                     raise ssl.SSLError("SSL Certificate host name mismatch: %s" % error)
 
+        # imaplib2 uses this to poll()
+        self.read_fd = self.sock.fileno()
+
         #TODO: Done for now. We should implement a mutt-like behavior
         #that offers the users to accept a certificate (presenting a
         #fingerprint of it) (get via self.sslobj.getpeercert()), and
@@ -185,53 +194,6 @@ class WrappedIMAP4_SSL(IMAP4_SSL):
 
         return ('no matching domain name found in certificate')
 
-    def _read_upto (self, n):
-        """Read up to n bytes, emptying existing _readbuffer first"""
-        bytesfrombuf = min(n, len(self._readbuf))
-        if bytesfrombuf:
-            # Return the stuff in readbuf, even if less than n.
-            # It might contain the rest of the line, and if we try to
-            # read more, might block waiting for data that is not
-            # coming to arrive.
-            retval = self._readbuf[:bytesfrombuf]
-            self._readbuf = self._readbuf[bytesfrombuf:]
-            return retval
-        return self.sslobj.read(min(n, 16384))
-
-    def read(self, n):
-        """Read exactly n bytes
-
-        As done in IMAP4_SSL.read() API. If read returns less than n
-        bytes, things break left and right."""
-        chunks = []
-        read = 0
-        while read < n:
-            data = self._read_upto (n-read)
-            if not data:
-                break
-            read += len(data)
-            chunks.append(data)
-
-        return ''.join(chunks)
-
-    def readline(self):
-        """Get the next line. This implementation is more efficient
-        than IMAP4_SSL.readline() which reads one char at a time and
-        reassembles the string by appending those chars. Uggh."""
-        retval = ''
-        while 1:
-            linebuf = self._read_upto(1024)
-            if not linebuf:
-                return retval
-            nlindex = linebuf.find("\n")
-            if nlindex != -1:
-                retval += linebuf[:nlindex + 1]
-                self._readbuf = linebuf[nlindex + 1:] + self._readbuf
-                return retval
-            else:
-                retval += linebuf
-
-
 class WrappedIMAP4(IMAP4):
     """Improved version of imaplib.IMAP4 that can also connect to IPv6"""
 
@@ -262,6 +224,9 @@ class WrappedIMAP4(IMAP4):
             raise socket.error(last_error)
         self.file = self.sock.makefile('rb')
 
+        # imaplib2 uses this to poll()
+        self.read_fd = self.sock.fileno()
+
 mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]")
 
 def Internaldate2epoch(resp):

offlineimap/imapserver.py

@@ -16,11 +16,11 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-import imaplib
+from offlineimap import imaplib2 as imaplib
 from offlineimap import imaplibutil, imaputil, threadutil
 from offlineimap.ui import getglobalui
 from threading import *
-import thread, hmac, os, time
+import thread, hmac, os, time, socket
 import base64
 
 from StringIO import StringIO
@@ -48,6 +48,8 @@ class UsefulIMAPMixIn:
            and self.is_readonly == readonly:
             # No change; return.
             return
+        # Wipe out all old responses, to maintain semantics with old imaplib2
+        del self.untagged_responses[:]
         result = self.__class__.__bases__[1].select(self, mailbox, readonly)
         if result[0] != 'OK':
             raise ValueError, "Error from select: %s" % str(result)
@@ -55,9 +57,10 @@ class UsefulIMAPMixIn:
             self.selectedfolder = mailbox
         else:
             self.selectedfolder = None
+        return result
 
-    def _mesg(self, s, secs=None):
+    def _mesg(self, s, tn=None, secs=None):
-        imaplibutil.new_mesg(self, s, secs)
+        imaplibutil.new_mesg(self, s, tn, secs)
 
 class UsefulIMAP4(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4):
     # This is a hack around Darwin's implementation of realloc() (which
@@ -151,7 +154,11 @@ class IMAPServer:
         """Releases a connection, returning it to the pool."""
         self.connectionlock.acquire()
         self.assignedconnections.remove(connection)
-        self.availableconnections.append(connection)
+        # Don't reuse broken connections
+        if connection.Terminate:
+            connection.logout()
+        else:
+            self.availableconnections.append(connection)
         self.connectionlock.release()
         self.semaphore.release()
 
@@ -236,16 +243,18 @@ class IMAPServer:
                 # Generate a new connection.
                 if self.tunnel:
                     self.ui.connecting('tunnel', self.tunnel)
-                    imapobj = UsefulIMAP4_Tunnel(self.tunnel)
+                    imapobj = UsefulIMAP4_Tunnel(self.tunnel, timeout=socket.getdefaulttimeout())
                     success = 1
                 elif self.usessl:
                     self.ui.connecting(self.hostname, self.port)
                     imapobj = UsefulIMAP4_SSL(self.hostname, self.port,
-                                              self.sslclientkey, self.sslclientcert, 
+                                              self.sslclientkey, self.sslclientcert,
+                                              timeout=socket.getdefaulttimeout(),
                                               cacertfile = self.sslcacertfile)
                 else:
                     self.ui.connecting(self.hostname, self.port)
-                    imapobj = UsefulIMAP4(self.hostname, self.port)
+                    imapobj = UsefulIMAP4(self.hostname, self.port,
+                                          timeout=socket.getdefaulttimeout())
 
                 imapobj.mustquote = imaplibutil.mustquote
 

offlineimap/init.py

@@ -19,7 +19,7 @@
 import os
 import sys
 import threading
-import imaplib
+import offlineimap.imaplib2 as imaplib
 import signal
 import socket
 import logging