tobias/docker-offlineimap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'rmimaplib2', removing imaplib2.py and IDLE support

Browse Source 
The IDLE support appears to be fine, but it required imaplib2.py,
which was a significant source of stability issues for users.  Some
were highlighted at:

http://lists.complete.org/offlineimap@complete.org/2009/08/msg00008.html.gz

Fixes #177, #176, #174
Closes: #536156, #536329.
This commit is contained in:
John Goerzen 2009-08-12 00:30:03 -05:00
commit 762b589842
9 changed files with 113 additions and 2291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
offlineimap.conf
View File

@ -307,19 +307,6 @@ remoteuser = username
# #
# reference = Mail # reference = Mail
# In between synchronisations, OfflineIMAP can monitor mailboxes for new
# messages using the IDLE command. If you want to enable this, specify here
# the folders you wish to monitor. Note that the IMAP protocol requires a
# separate connection for each folder monitored in this way, so setting
# this option will force settings for:
# maxconnections - to be at least the number of folders you give
# holdconnectionopen - to be true
# keepalive - to be 29 minutes unless you specify otherwise
# This option should return a Python list. For example
#
# idlefolders = ['INBOX', 'INBOX.Alerts']
#
# OfflineIMAP can use multiple connections to the server in order # OfflineIMAP can use multiple connections to the server in order
# to perform multiple synchronization actions simultaneously. # to perform multiple synchronization actions simultaneously.
# This may place a higher burden on the server. In most cases, # This may place a higher burden on the server. In most cases,

3
offlineimap/folder/Gmail.py
View File

@ -20,7 +20,8 @@
""" """
from IMAP import IMAPFolder from IMAP import IMAPFolder
from offlineimap import imaplib2, imaputil, imaplibutil import imaplib
from offlineimap import imaputil, imaplibutil
from offlineimap.ui import UIBase from offlineimap.ui import UIBase
from copy import copy from copy import copy

7
offlineimap/folder/IMAP.py
View File

@ -17,7 +17,8 @@
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
from Base import BaseFolder from Base import BaseFolder
from offlineimap import imaplib2, imaputil, imaplibutil import imaplib
from offlineimap import imaputil, imaplibutil
from offlineimap.ui import UIBase from offlineimap.ui import UIBase
from offlineimap.version import versionstr from offlineimap.version import versionstr
import rfc822, time, string, random, binascii, re import rfc822, time, string, random, binascii, re
@ -270,13 +271,13 @@ class IMAPFolder(BaseFolder):
raise ValueError raise ValueError
# This could raise a value error if it's not a valid format. # This could raise a value error if it's not a valid format.
date = imaplib2.Time2Internaldate(datetuple) date = imaplib.Time2Internaldate(datetuple)
except (ValueError, OverflowError): except (ValueError, OverflowError):
# Argh, sometimes it's a valid format but year is 0102 # Argh, sometimes it's a valid format but year is 0102
# or something. Argh. It seems that Time2Internaldate # or something. Argh. It seems that Time2Internaldate
# will rause a ValueError if the year is 0102 but not 1902, # will rause a ValueError if the year is 0102 but not 1902,
# but some IMAP servers nonetheless choke on 1902. # but some IMAP servers nonetheless choke on 1902.
date = imaplib2.Time2Internaldate(time.localtime()) date = imaplib.Time2Internaldate(time.localtime())
ui.debug('imap', 'savemessage: using date ' + str(date)) ui.debug('imap', 'savemessage: using date ' + str(date))
content = re.sub("(?<!\r)\n", "\r\n", content) content = re.sub("(?<!\r)\n", "\r\n", content)

2153
offlineimap/imaplib2.py
View File

File diff suppressed because it is too large Load Diff

78
offlineimap/imaplibutil.py
View File

@ -16,12 +16,12 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import os, re, socket, time, subprocess, sys, threading import re, string, types, binascii, socket, time, random, subprocess, sys, os
from offlineimap.ui import UIBase from offlineimap.ui import UIBase
from offlineimap.imaplib2 import * from imaplib import *
# Import the symbols we need that aren't exported by default # Import the symbols we need that aren't exported by default
from offlineimap.imaplib2 import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num
# ssl is new in python 2.6 # ssl is new in python 2.6
if (sys.version_info[0] == 2 and sys.version_info[1] >= 6) or sys.version_info[0] >= 3: if (sys.version_info[0] == 2 and sys.version_info[1] >= 6) or sys.version_info[0] >= 3:
@ -43,10 +43,12 @@ class IMAP4_Tunnel(IMAP4):
self.process = subprocess.Popen(host, shell=True, close_fds=True, self.process = subprocess.Popen(host, shell=True, close_fds=True,
stdin=subprocess.PIPE, stdout=subprocess.PIPE) stdin=subprocess.PIPE, stdout=subprocess.PIPE)
(self.outfd, self.infd) = (self.process.stdin, self.process.stdout) (self.outfd, self.infd) = (self.process.stdin, self.process.stdout)
self.read_fd = self.infd.fileno()
def read(self, size): def read(self, size):
return os.read(self.read_fd, size) retval = ''
while len(retval) < size:
retval += self.infd.read(size - len(retval))
return retval
def readline(self): def readline(self):
return self.infd.readline() return self.infd.readline()
@ -98,22 +100,78 @@ class sslwrapper:
else: else:
retval += linebuf retval += linebuf
def new_mesg(self, s, tn=None, secs=None): def new_mesg(self, s, secs=None):
if secs is None: if secs is None:
secs = time.time() secs = time.time()
if tn is None:
tn = threading.currentThread().getName()
tm = time.strftime('%M:%S', time.localtime(secs)) tm = time.strftime('%M:%S', time.localtime(secs))
UIBase.getglobalui().debug('imap', ' %s.%02d %s %s' % (tm, (secs*100)%100, tn, s)) UIBase.getglobalui().debug('imap', ' %s.%02d %s' % (tm, (secs*100)%100, s))
class WrappedIMAP4_SSL(IMAP4_SSL): class WrappedIMAP4_SSL(IMAP4_SSL):
def open(self, host=None, port=None): def open(self, host = '', port = IMAP4_SSL_PORT):
IMAP4_SSL.open(self, host, port) IMAP4_SSL.open(self, host, port)
self.sslobj = sslwrapper(self.sslobj) self.sslobj = sslwrapper(self.sslobj)
def readline(self): def readline(self):
return self.sslobj.readline() return self.sslobj.readline()
def new_open(self, host = '', port = IMAP4_PORT):
"""Setup connection to remote server on "host:port"
(default: localhost:standard IMAP4 port).
This connection will be used by the routines:
read, readline, send, shutdown.
"""
self.host = host
self.port = port
res = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM)
# Try each address returned by getaddrinfo in turn until we
# manage to connect to one.
# Try all the addresses in turn until we connect()
last_error = 0
for remote in res:
af, socktype, proto, canonname, sa = remote
self.sock = socket.socket(af, socktype, proto)
last_error = self.sock.connect_ex(sa)
if last_error == 0:
break
else:
self.sock.close()
if last_error != 0:
# FIXME
raise socket.error(last_error)
self.file = self.sock.makefile('rb')
def new_open_ssl(self, host = '', port = IMAP4_SSL_PORT):
"""Setup connection to remote server on "host:port".
(default: localhost:standard IMAP4 SSL port).
This connection will be used by the routines:
read, readline, send, shutdown.
"""
self.host = host
self.port = port
#This connects to the first ip found ipv4/ipv6
#Added by Adriaan Peeters <apeeters@lashout.net> based on a socket
#example from the python documentation:
#http://www.python.org/doc/lib/socket-example.html
res = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM)
# Try all the addresses in turn until we connect()
last_error = 0
for remote in res:
af, socktype, proto, canonname, sa = remote
self.sock = socket.socket(af, socktype, proto)
last_error = self.sock.connect_ex(sa)
if last_error == 0:
break
else:
self.sock.close()
if last_error != 0:
# FIXME
raise socket.error(last_error)
self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile)
self.sslobj = sslwrapper(self.sslobj)
mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]") mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]")
def Internaldate2epoch(resp): def Internaldate2epoch(resp):

126
offlineimap/imapserver.py
View File

@ -16,9 +16,9 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
from offlineimap import imaplib2, imaplibutil, imaputil, threadutil import imaplib
from offlineimap import imaplibutil, imaputil, threadutil
from offlineimap.ui import UIBase from offlineimap.ui import UIBase
from offlineimap.accounts import syncfolder
from threading import * from threading import *
import thread, hmac, os, time import thread, hmac, os, time
import base64 import base64
@ -56,10 +56,13 @@ class UsefulIMAPMixIn:
else: else:
self.selectedfolder = None self.selectedfolder = None
def _mesg(self, s, tn=None, secs=None): def _mesg(self, s, secs=None):
imaplibutil.new_mesg(self, s, tn, secs) imaplibutil.new_mesg(self, s, secs)
class UsefulIMAP4(UsefulIMAPMixIn, imaplib.IMAP4):
def open(self, host = '', port = imaplib.IMAP4_PORT):
imaplibutil.new_open(self, host, port)
class UsefulIMAP4(UsefulIMAPMixIn, imaplib2.IMAP4):
# This is a hack around Darwin's implementation of realloc() (which # This is a hack around Darwin's implementation of realloc() (which
# Python uses inside the socket code). On Darwin, we split the # Python uses inside the socket code). On Darwin, we split the
# message into 100k chunks, which should be small enough - smaller # message into 100k chunks, which should be small enough - smaller
@ -70,17 +73,17 @@ class UsefulIMAP4(UsefulIMAPMixIn, imaplib2.IMAP4):
read = 0 read = 0
io = StringIO() io = StringIO()
while read < size: while read < size:
sz = min(size-read, 8192) data = imaplib.IMAP4.read (self, min(size-read,8192))
data = imaplib2.IMAP4.read (self, sz)
read += len(data) read += len(data)
io.write(data) io.write(data)
if len(data) < sz:
break
return io.getvalue() return io.getvalue()
else: else:
return imaplib2.IMAP4.read (self, size) return imaplib.IMAP4.read (self, size)
class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL): class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
def open(self, host = '', port = imaplib.IMAP4_SSL_PORT):
imaplibutil.new_open_ssl(self, host, port)
# This is the same hack as above, to be used in the case of an SSL # This is the same hack as above, to be used in the case of an SSL
# connexion. # connexion.
@ -89,12 +92,9 @@ class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
read = 0 read = 0
io = StringIO() io = StringIO()
while read < size: while read < size:
sz = min(size-read,8192) data = imaplibutil.WrappedIMAP4_SSL.read (self, min(size-read,8192))
data = imaplibutil.WrappedIMAP4_SSL.read (self, sz)
read += len(data) read += len(data)
io.write(data) io.write(data)
if len(data) < sz:
break
return io.getvalue() return io.getvalue()
else: else:
return imaplibutil.WrappedIMAP4_SSL.read (self,size) return imaplibutil.WrappedIMAP4_SSL.read (self,size)
@ -107,8 +107,7 @@ class IMAPServer:
def __init__(self, config, reposname, def __init__(self, config, reposname,
username = None, password = None, hostname = None, username = None, password = None, hostname = None,
port = None, ssl = 1, maxconnections = 1, tunnel = None, port = None, ssl = 1, maxconnections = 1, tunnel = None,
reference = '""', sslclientcert = None, sslclientkey = None, reference = '""', sslclientcert = None, sslclientkey = None):
idlefolders = []):
self.reposname = reposname self.reposname = reposname
self.config = config self.config = config
self.username = username self.username = username
@ -135,7 +134,6 @@ class IMAPServer:
self.semaphore = BoundedSemaphore(self.maxconnections) self.semaphore = BoundedSemaphore(self.maxconnections)
self.connectionlock = Lock() self.connectionlock = Lock()
self.reference = reference self.reference = reference
self.idlefolders = idlefolders
self.gss_step = self.GSS_STATE_STEP self.gss_step = self.GSS_STATE_STEP
self.gss_vc = None self.gss_vc = None
self.gssapi = False self.gssapi = False
@ -351,6 +349,8 @@ class IMAPServer:
ui.debug('imap', 'keepalive thread started') ui.debug('imap', 'keepalive thread started')
while 1: while 1:
ui.debug('imap', 'keepalive: top of loop') ui.debug('imap', 'keepalive: top of loop')
time.sleep(timeout)
ui.debug('imap', 'keepalive: after wait')
if event.isSet(): if event.isSet():
ui.debug('imap', 'keepalive: event is set; exiting') ui.debug('imap', 'keepalive: event is set; exiting')
return return
@ -361,91 +361,32 @@ class IMAPServer:
self.connectionlock.release() self.connectionlock.release()
ui.debug('imap', 'keepalive: connectionlock released') ui.debug('imap', 'keepalive: connectionlock released')
threads = [] threads = []
imapobjs = []
for i in range(numconnections): for i in range(numconnections):
ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections)) ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
if len(self.idlefolders) > i: imapobj = self.acquireconnection()
idler = IdleThread(self, self.idlefolders[i]) ui.debug('imap', 'keepalive: connection %d acquired' % i)
else: imapobjs.append(imapobj)
idler = IdleThread(self) thr = threadutil.ExitNotifyThread(target = imapobj.noop)
idler.start() thr.setDaemon(1)
threads.append(idler) thr.start()
threads.append(thr)
ui.debug('imap', 'keepalive: thread started') ui.debug('imap', 'keepalive: thread started')
ui.debug('imap', 'keepalive: waiting for timeout')
event.wait(timeout)
ui.debug('imap', 'keepalive: joining threads') ui.debug('imap', 'keepalive: joining threads')
for idler in threads: for thr in threads:
# Make sure all the commands have completed. # Make sure all the commands have completed.
idler.stop() thr.join()
idler.join()
ui.debug('imap', 'keepalive: releasing connections')
for imapobj in imapobjs:
self.releaseconnection(imapobj)
ui.debug('imap', 'keepalive: bottom of loop') ui.debug('imap', 'keepalive: bottom of loop')
class IdleThread(object):
def __init__(self, parent, folder=None):
self.parent = parent
self.folder = folder
self.event = Event()
if folder is None:
self.thread = Thread(target=self.noop)
else:
self.thread = Thread(target=self.idle)
self.thread.setDaemon(1)
def start(self):
self.thread.start()
def stop(self):
self.event.set()
def join(self):
self.thread.join()
def noop(self):
imapobj = self.parent.acquireconnection()
imapobj.noop()
self.event.wait()
self.parent.releaseconnection(imapobj)
def dosync(self):
remoterepos = self.parent.repos
account = remoterepos.account
localrepos = account.localrepos
remoterepos = account.remoterepos
statusrepos = account.statusrepos
remotefolder = remoterepos.getfolder(self.folder)
syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False)
ui = UIBase.getglobalui()
ui.unregisterthread(currentThread())
def idle(self):
imapobj = self.parent.acquireconnection()
imapobj.select(self.folder)
self.parent.releaseconnection(imapobj)
while True:
if self.event.isSet():
return
self.needsync = False
def callback(args):
if not self.event.isSet():
self.needsync = True
self.event.set()
imapobj = self.parent.acquireconnection()
if "IDLE" in imapobj.capabilities:
imapobj.idle(callback=callback)
else:
imapobj.noop()
self.event.wait()
if self.event.isSet():
imapobj.noop()
self.parent.releaseconnection(imapobj)
if self.needsync:
self.event.clear()
self.dosync()
class ConfigedIMAPServer(IMAPServer): class ConfigedIMAPServer(IMAPServer):
"""This class is designed for easier initialization given a ConfigParser """This class is designed for easier initialization given a ConfigParser
object and an account name. The passwordhash is used if object and an account name. The passwordhash is used if
@ -465,7 +406,6 @@ class ConfigedIMAPServer(IMAPServer):
sslclientcert = self.repos.getsslclientcert() sslclientcert = self.repos.getsslclientcert()
sslclientkey = self.repos.getsslclientkey() sslclientkey = self.repos.getsslclientkey()
reference = self.repos.getreference() reference = self.repos.getreference()
idlefolders = self.repos.getidlefolders()
server = None server = None
password = None password = None
@ -477,7 +417,6 @@ class ConfigedIMAPServer(IMAPServer):
IMAPServer.__init__(self, self.config, self.repos.getname(), IMAPServer.__init__(self, self.config, self.repos.getname(),
tunnel = usetunnel, tunnel = usetunnel,
reference = reference, reference = reference,
idlefolders = idlefolders,
maxconnections = self.repos.getmaxconnections()) maxconnections = self.repos.getmaxconnections())
else: else:
if not password: if not password:
@ -486,6 +425,5 @@ class ConfigedIMAPServer(IMAPServer):
user, password, host, port, ssl, user, password, host, port, ssl,
self.repos.getmaxconnections(), self.repos.getmaxconnections(),
reference = reference, reference = reference,
idlefolders = idlefolders,
sslclientcert = sslclientcert, sslclientcert = sslclientcert,
sslclientkey = sslclientkey) sslclientkey = sslclientkey)

5
offlineimap/init.py
View File

@ -16,7 +16,8 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
from offlineimap import imaplib2, imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts import imaplib
from offlineimap import imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
from offlineimap.localeval import LocalEval from offlineimap.localeval import LocalEval
from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
from offlineimap.ui import UIBase from offlineimap.ui import UIBase
@ -102,7 +103,7 @@ def startup(versionno):
for debugtype in options['-d'].split(','): for debugtype in options['-d'].split(','):
ui.add_debug(debugtype.strip()) ui.add_debug(debugtype.strip())
if debugtype == 'imap': if debugtype == 'imap':
imaplib2.Debug = 5 imaplib.Debug = 5
if debugtype == 'thread': if debugtype == 'thread':
threading._VERBOSE = 1 threading._VERBOSE = 1

16
offlineimap/repository/IMAP.py
View File

@ -74,16 +74,10 @@ class IMAPRepository(BaseRepository):
self.imapserver.close() self.imapserver.close()
def getholdconnectionopen(self): def getholdconnectionopen(self):
if self.getidlefolders():
return 1
return self.getconfboolean("holdconnectionopen", 0) return self.getconfboolean("holdconnectionopen", 0)
def getkeepalive(self): def getkeepalive(self):
num = self.getconfint("keepalive", 0) return self.getconfint("keepalive", 0)
if num == 0 and self.getidlefolders():
return 29*60
else:
return num
def getsep(self): def getsep(self):
return self.imapserver.delim return self.imapserver.delim
@ -151,14 +145,8 @@ class IMAPRepository(BaseRepository):
def getreference(self): def getreference(self):
return self.getconf('reference', '""') return self.getconf('reference', '""')
def getidlefolders(self):
localeval = self.localeval
return localeval.eval(self.getconf('idlefolders', '[]'))
def getmaxconnections(self): def getmaxconnections(self):
num1 = len(self.getidlefolders()) return self.getconfint('maxconnections', 1)
num2 = self.getconfint('maxconnections', 1)
return max(num1, num2)
def getexpunge(self): def getexpunge(self):
return self.getconfboolean('expunge', 1) return self.getconfboolean('expunge', 1)

3
offlineimap/syncmaster.py
View File

@ -16,7 +16,8 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
from offlineimap import imaplib2, imapserver, repository, folder, mbnames, threadutil, version import imaplib
from offlineimap import imapserver, repository, folder, mbnames, threadutil, version
from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
import offlineimap.accounts import offlineimap.accounts
from offlineimap.accounts import SyncableAccount, SigListener from offlineimap.accounts import SyncableAccount, SigListener