tobias/docker-offlineimap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implementation of IMAP IDLE

Browse Source 
- Use a newer version of imaplib
 - Hijack the keepalive process to send IDLE instead of NOOP
This commit is contained in:
James Bunton 2009-02-10 11:27:48 +11:00
parent 8db951ed61
commit 3847d0ba9d
9 changed files with 2204 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
offlineimap.conf
View File

@ -307,6 +307,19 @@ remoteuser = username
#
# reference = Mail
# In between synchronisations, OfflineIMAP can monitor mailboxes for new
# messages using the IDLE command. If you want to enable this, specify here
# the folders you wish to monitor. Note that the IMAP protocol requires a
# separate connection for each folder monitored in this way, so setting
# this option will force settings for:
# maxconnections - to be at least the number of folders you give
# holdconnectionopen - to be true
# keepalive - to be 29 minutes unless you specify otherwise
# This option should return a Python list. For example
#
# idlefolders = ['INBOX', 'INBOX.Alerts']
#
# OfflineIMAP can use multiple connections to the server in order
# to perform multiple synchronization actions simultaneously.
# This may place a higher burden on the server. In most cases,

3
offlineimap/folder/Gmail.py
View File

@ -20,8 +20,7 @@
"""
from IMAP import IMAPFolder
import imaplib
from offlineimap import imaputil, imaplibutil
from offlineimap import imaplib2, imaputil, imaplibutil
from offlineimap.ui import UIBase
from copy import copy

7
offlineimap/folder/IMAP.py
View File

@ -17,8 +17,7 @@
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
from Base import BaseFolder
import imaplib
from offlineimap import imaputil, imaplibutil
from offlineimap import imaplib2, imaputil, imaplibutil
from offlineimap.ui import UIBase
from offlineimap.version import versionstr
import rfc822, time, string, random, binascii, re
@ -271,13 +270,13 @@ class IMAPFolder(BaseFolder):
raise ValueError
# This could raise a value error if it's not a valid format.
date = imaplib.Time2Internaldate(datetuple)
date = imaplib2.Time2Internaldate(datetuple)
except (ValueError, OverflowError):
# Argh, sometimes it's a valid format but year is 0102
# or something. Argh. It seems that Time2Internaldate
# will rause a ValueError if the year is 0102 but not 1902,
# but some IMAP servers nonetheless choke on 1902.
date = imaplib.Time2Internaldate(time.localtime())
date = imaplib2.Time2Internaldate(time.localtime())
ui.debug('imap', 'savemessage: using date ' + str(date))
content = re.sub("(?<!\r)\n", "\r\n", content)

2072
offlineimap/imaplib2.py Normal file
View File

File diff suppressed because it is too large Load Diff

78
offlineimap/imaplibutil.py
View File

@ -16,12 +16,12 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import re, string, types, binascii, socket, time, random, subprocess, sys, os
import os, re, socket, time, subprocess, sys, threading
from offlineimap.ui import UIBase
from imaplib import *
from offlineimap.imaplib2 import *
# Import the symbols we need that aren't exported by default
from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num
from offlineimap.imaplib2 import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num
class IMAP4_Tunnel(IMAP4):
@ -40,12 +40,10 @@ class IMAP4_Tunnel(IMAP4):
self.process = subprocess.Popen(host, shell=True, close_fds=True,
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
(self.outfd, self.infd) = (self.process.stdin, self.process.stdout)
self.read_fd = self.infd.fileno()
def read(self, size):
retval = ''
while len(retval) < size:
retval += self.infd.read(size - len(retval))
return retval
return os.read(self.read_fd, size)
def readline(self):
return self.infd.readline()
@ -97,78 +95,22 @@ class sslwrapper:
else:
retval += linebuf
def new_mesg(self, s, secs=None):
def new_mesg(self, s, tn=None, secs=None):
if secs is None:
secs = time.time()
if tn is None:
tn = threading.currentThread().getName()
tm = time.strftime('%M:%S', time.localtime(secs))
UIBase.getglobalui().debug('imap', ' %s.%02d %s' % (tm, (secs*100)%100, s))
UIBase.getglobalui().debug('imap', ' %s.%02d %s %s' % (tm, (secs*100)%100, tn, s))
class WrappedIMAP4_SSL(IMAP4_SSL):
def open(self, host = '', port = IMAP4_SSL_PORT):
def open(self, host=None, port=None):
IMAP4_SSL.open(self, host, port)
self.sslobj = sslwrapper(self.sslobj)
def readline(self):
return self.sslobj.readline()
def new_open(self, host = '', port = IMAP4_PORT):
"""Setup connection to remote server on "host:port"
(default: localhost:standard IMAP4 port).
This connection will be used by the routines:
read, readline, send, shutdown.
"""
self.host = host
self.port = port
res = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM)
# Try each address returned by getaddrinfo in turn until we
# manage to connect to one.
# Try all the addresses in turn until we connect()
last_error = 0
for remote in res:
af, socktype, proto, canonname, sa = remote
self.sock = socket.socket(af, socktype, proto)
last_error = self.sock.connect_ex(sa)
if last_error == 0:
break
else:
self.sock.close()
if last_error != 0:
# FIXME
raise socket.error(last_error)
self.file = self.sock.makefile('rb')
def new_open_ssl(self, host = '', port = IMAP4_SSL_PORT):
"""Setup connection to remote server on "host:port".
(default: localhost:standard IMAP4 SSL port).
This connection will be used by the routines:
read, readline, send, shutdown.
"""
self.host = host
self.port = port
#This connects to the first ip found ipv4/ipv6
#Added by Adriaan Peeters <apeeters@lashout.net> based on a socket
#example from the python documentation:
#http://www.python.org/doc/lib/socket-example.html
res = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM)
# Try all the addresses in turn until we connect()
last_error = 0
for remote in res:
af, socktype, proto, canonname, sa = remote
self.sock = socket.socket(af, socktype, proto)
last_error = self.sock.connect_ex(sa)
if last_error == 0:
break
else:
self.sock.close()
if last_error != 0:
# FIXME
raise socket.error(last_error)
self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile)
self.sslobj = sslwrapper(self.sslobj)
mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]")
def Internaldate2epoch(resp):

123
offlineimap/imapserver.py
View File

@ -16,9 +16,9 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import imaplib
from offlineimap import imaplibutil, imaputil, threadutil
from offlineimap import imaplib2, imaplibutil, imaputil, threadutil
from offlineimap.ui import UIBase
from offlineimap.accounts import syncfolder
from threading import *
import thread, hmac, os, time
import base64
@ -56,13 +56,10 @@ class UsefulIMAPMixIn:
else:
self.selectedfolder = None
def _mesg(self, s, secs=None):
imaplibutil.new_mesg(self, s, secs)
class UsefulIMAP4(UsefulIMAPMixIn, imaplib.IMAP4):
def open(self, host = '', port = imaplib.IMAP4_PORT):
imaplibutil.new_open(self, host, port)
def _mesg(self, s, tn=None, secs=None):
imaplibutil.new_mesg(self, s, tn, secs)
class UsefulIMAP4(UsefulIMAPMixIn, imaplib2.IMAP4):
# This is a hack around Darwin's implementation of realloc() (which
# Python uses inside the socket code). On Darwin, we split the
# message into 100k chunks, which should be small enough - smaller
@ -73,17 +70,17 @@ class UsefulIMAP4(UsefulIMAPMixIn, imaplib.IMAP4):
read = 0
io = StringIO()
while read < size:
data = imaplib.IMAP4.read (self, min(size-read,8192))
sz = min(size-read, 8192)
data = imaplib2.IMAP4.read (self, sz)
read += len(data)
io.write(data)
if len(data) < sz:
break
return io.getvalue()
else:
return imaplib.IMAP4.read (self, size)
return imaplib2.IMAP4.read (self, size)
class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
def open(self, host = '', port = imaplib.IMAP4_SSL_PORT):
imaplibutil.new_open_ssl(self, host, port)
# This is the same hack as above, to be used in the case of an SSL
# connexion.
@ -92,9 +89,12 @@ class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
read = 0
io = StringIO()
while read < size:
data = imaplibutil.WrappedIMAP4_SSL.read (self, min(size-read,8192))
sz = min(size-read,8192)
data = imaplibutil.WrappedIMAP4_SSL.read (self, sz)
read += len(data)
io.write(data)
if len(data) < sz:
break
return io.getvalue()
else:
return imaplibutil.WrappedIMAP4_SSL.read (self,size)
@ -107,7 +107,8 @@ class IMAPServer:
def __init__(self, config, reposname,
username = None, password = None, hostname = None,
port = None, ssl = 1, maxconnections = 1, tunnel = None,
reference = '""', sslclientcert = None, sslclientkey = None):
reference = '""', sslclientcert = None, sslclientkey = None,
idlefolders = []):
self.reposname = reposname
self.config = config
self.username = username
@ -134,6 +135,7 @@ class IMAPServer:
self.semaphore = BoundedSemaphore(self.maxconnections)
self.connectionlock = Lock()
self.reference = reference
self.idlefolders = idlefolders
self.gss_step = self.GSS_STATE_STEP
self.gss_vc = None
self.gssapi = False
@ -344,8 +346,6 @@ class IMAPServer:
ui.debug('imap', 'keepalive thread started')
while 1:
ui.debug('imap', 'keepalive: top of loop')
time.sleep(timeout)
ui.debug('imap', 'keepalive: after wait')
if event.isSet():
ui.debug('imap', 'keepalive: event is set; exiting')
return
@ -356,32 +356,88 @@ class IMAPServer:
self.connectionlock.release()
ui.debug('imap', 'keepalive: connectionlock released')
threads = []
imapobjs = []
for i in range(numconnections):
ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
imapobj = self.acquireconnection()
ui.debug('imap', 'keepalive: connection %d acquired' % i)
imapobjs.append(imapobj)
thr = threadutil.ExitNotifyThread(target = imapobj.noop)
thr.setDaemon(1)
thr.start()
threads.append(thr)
if len(self.idlefolders) > i:
idler = IdleThread(self, self.idlefolders[i])
else:
idler = IdleThread(self)
idler.start()
threads.append(idler)
ui.debug('imap', 'keepalive: thread started')
ui.debug('imap', 'keepalive: waiting for timeout')
event.wait(timeout)
ui.debug('imap', 'keepalive: joining threads')
for thr in threads:
for idler in threads:
# Make sure all the commands have completed.
thr.join()
ui.debug('imap', 'keepalive: releasing connections')
for imapobj in imapobjs:
self.releaseconnection(imapobj)
idler.stop()
idler.join()
ui.debug('imap', 'keepalive: bottom of loop')
class IdleThread(object):
def __init__(self, parent, folder=None):
self.parent = parent
self.folder = folder
self.event = Event()
if folder is None:
self.thread = Thread(target=self.noop)
else:
self.thread = Thread(target=self.idle)
self.thread.setDaemon(1)
def start(self):
self.thread.start()
def stop(self):
self.event.set()
def join(self):
self.thread.join()
def noop(self):
imapobj = self.parent.acquireconnection()
self.event.wait()
imapobj.noop()
self.parent.releaseconnection(imapobj)
def dosync(self):
remoterepos = self.parent.repos
account = remoterepos.account
localrepos = account.localrepos
remoterepos = account.remoterepos
statusrepos = account.statusrepos
remotefolder = remoterepos.getfolder(self.folder)
syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False)
ui = UIBase.getglobalui()
ui.unregisterthread(currentThread())
def idle(self):
imapobj = self.parent.acquireconnection()
imapobj.select(self.folder)
self.parent.releaseconnection(imapobj)
while True:
if self.event.isSet():
return
self.needsync = False
def callback(args):
if not self.event.isSet():
self.needsync = True
self.event.set()
imapobj = self.parent.acquireconnection()
imapobj.idle(callback=callback)
self.event.wait()
if self.event.isSet():
imapobj.noop()
self.parent.releaseconnection(imapobj)
if self.needsync:
self.event.clear()
self.dosync()
class ConfigedIMAPServer(IMAPServer):
"""This class is designed for easier initialization given a ConfigParser
object and an account name. The passwordhash is used if
@ -401,6 +457,7 @@ class ConfigedIMAPServer(IMAPServer):
sslclientcert = self.repos.getsslclientcert()
sslclientkey = self.repos.getsslclientkey()
reference = self.repos.getreference()
idlefolders = self.repos.getidlefolders()
server = None
password = None
@ -412,6 +469,7 @@ class ConfigedIMAPServer(IMAPServer):
IMAPServer.__init__(self, self.config, self.repos.getname(),
tunnel = usetunnel,
reference = reference,
idlefolders = idlefolders,
maxconnections = self.repos.getmaxconnections())
else:
if not password:
@ -420,5 +478,6 @@ class ConfigedIMAPServer(IMAPServer):
user, password, host, port, ssl,
self.repos.getmaxconnections(),
reference = reference,
idlefolders = idlefolders,
sslclientcert = sslclientcert,
sslclientkey = sslclientkey)

5
offlineimap/init.py
View File

@ -16,8 +16,7 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import imaplib
from offlineimap import imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
from offlineimap import imaplib2, imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
from offlineimap.localeval import LocalEval
from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
from offlineimap.ui import UIBase
@ -103,7 +102,7 @@ def startup(versionno):
for debugtype in options['-d'].split(','):
ui.add_debug(debugtype.strip())
if debugtype == 'imap':
imaplib.Debug = 5
imaplib2.Debug = 5
if debugtype == 'thread':
threading._VERBOSE = 1

12
offlineimap/repository/IMAP.py
View File

@ -74,9 +74,13 @@ class IMAPRepository(BaseRepository):
self.imapserver.close()
def getholdconnectionopen(self):
if self.getidlefolders():
return 1
return self.getconfboolean("holdconnectionopen", 0)
def getkeepalive(self):
if self.getidlefolders():
return 29*60
return self.getconfint("keepalive", 0)
def getsep(self):
@ -145,8 +149,14 @@ class IMAPRepository(BaseRepository):
def getreference(self):
return self.getconf('reference', '""')
def getidlefolders(self):
localeval = self.localeval
return localeval.eval(self.getconf('idlefolders', '[]'))
def getmaxconnections(self):
return self.getconfint('maxconnections', 1)
num1 = len(self.getidlefolders())
num2 = self.getconfint('maxconnections', 1)
return max(num1, num2)
def getexpunge(self):
return self.getconfboolean('expunge', 1)

3
offlineimap/syncmaster.py
View File

@ -16,8 +16,7 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import imaplib
from offlineimap import imapserver, repository, folder, mbnames, threadutil, version
from offlineimap import imaplib2, imapserver, repository, folder, mbnames, threadutil, version
from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
import offlineimap.accounts
from offlineimap.accounts import SyncableAccount, SigListener