""" IMAP repository support """ # Copyright (C) 2002-2017 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import netrc import errno import codecs from sys import exc_info from threading import Event import six from offlineimap import folder, imaputil, imapserver, OfflineImapError from offlineimap.repository.Base import BaseRepository from offlineimap.threadutil import ExitNotifyThread from offlineimap.utils.distro import get_os_sslcertfile, get_os_sslcertfile_searchpath class IMAPRepository(BaseRepository): def __init__(self, reposname, account): self.idlefolders = None BaseRepository.__init__(self, reposname, account) # self.ui is being set by the BaseRepository self._host = None # Must be set before calling imapserver.IMAPServer(self) self.oauth2_request_url = None self.imapserver = imapserver.IMAPServer(self) self.folders = None self.copy_ignore_eval = None # Keep alive. self.kaevent = None self.kathread = None # Only set the newmail_hook in an IMAP repository. if self.config.has_option(self.getsection(), 'newmail_hook'): self.newmail_hook = self.localeval.eval( self.getconf('newmail_hook')) if self.getconf('sep', None): self.ui.info("The 'sep' setting is being ignored for IMAP " "repository '%s' (it's autodetected)"% self) def startkeepalive(self): keepalivetime = self.getkeepalive() if not keepalivetime: return self.kaevent = Event() self.kathread = ExitNotifyThread(target=self.imapserver.keepalive, name="Keep alive " + self.getname(), args=(keepalivetime, self.kaevent)) self.kathread.setDaemon(1) self.kathread.start() def stopkeepalive(self): if self.kaevent is None: return # Keepalive is not active. self.kaevent.set() self.kathread = None self.kaevent = None def holdordropconnections(self): if not self.getholdconnectionopen(): self.dropconnections() def dropconnections(self): self.imapserver.close() def get_copy_ignore_UIDs(self, foldername): """Return a list of UIDs to not copy for this foldername.""" if self.copy_ignore_eval is None: if self.config.has_option(self.getsection(), 'copy_ignore_eval'): self.copy_ignore_eval = self.localeval.eval( self.getconf('copy_ignore_eval')) else: self.copy_ignore_eval = lambda x: None return self.copy_ignore_eval(foldername) def getholdconnectionopen(self): if self.getidlefolders(): return True return self.getconfboolean("holdconnectionopen", False) def getkeepalive(self): num = self.getconfint("keepalive", 0) if num == 0 and self.getidlefolders(): return 29*60 return num def getsep(self): """Return the folder separator for the IMAP repository This requires that self.imapserver has been initialized with an acquireconnection() or it will still be `None`""" assert self.imapserver.delim != None, "'%s' " \ "repository called getsep() before the folder separator was " \ "queried from the server"% self return self.imapserver.delim def gethost(self): """Return the configured hostname to connect to :returns: hostname as string or throws Exception""" if self._host: # Use cached value if possible. return self._host # 1) Check for remotehosteval setting. if self.config.has_option(self.getsection(), 'remotehosteval'): host = self.getconf('remotehosteval') try: host = self.localeval.eval(host) except Exception as e: six.reraise(OfflineImapError, OfflineImapError( "remotehosteval option for repository " "'%s' failed:\n%s"% (self, e), OfflineImapError.ERROR.REPO), exc_info()[2]) if host: self._host = host return self._host # 2) Check for plain remotehost setting. host = self.getconf('remotehost', None) if host != None: self._host = host return self._host # No success. raise OfflineImapError("No remote host for repository " "'%s' specified."% self, OfflineImapError.ERROR.REPO) def get_remote_identity(self): """Remote identity is used for certain SASL mechanisms (currently -- PLAIN) to inform server about the ID we want to authorize as instead of our login name.""" identity = self.getconf('remote_identity', default=None) if identity != None: identity = identity.encode('UTF-8') return identity def get_auth_mechanisms(self): supported = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"] # Mechanisms are ranged from the strongest to the # weakest ones. # TODO: we need DIGEST-MD5, it must come before CRAM-MD5 # due to the chosen-plaintext resistance. default = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"] mechs = self.getconflist('auth_mechanisms', r',\s*', default) for m in mechs: if m not in supported: raise OfflineImapError("Repository %s: "% self + \ "unknown authentication mechanism '%s'"% m, OfflineImapError.ERROR.REPO) self.ui.debug('imap', "Using authentication mechanisms %s" % mechs) return mechs def getuser(self): user = None localeval = self.localeval if self.config.has_option(self.getsection(), 'remoteusereval'): user = self.getconf('remoteusereval') if user != None: return localeval.eval(user).encode('UTF-8') if self.config.has_option(self.getsection(), 'remoteuser'): # Assume the configuration file to be UTF-8 encoded so we must not # encode this string again. user = self.getconf('remoteuser') if user != None: return user try: netrcentry = netrc.netrc().authenticators(self.gethost()) except IOError as inst: if inst.errno != errno.ENOENT: raise else: if netrcentry: return netrcentry[0] try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) except IOError as inst: if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: return netrcentry[0] def getport(self): port = None if self.config.has_option(self.getsection(), 'remoteporteval'): port = self.getconf('remoteporteval') if port != None: return self.localeval.eval(port) return self.getconfint('remoteport', None) def getipv6(self): return self.getconfboolean('ipv6', None) def getssl(self): return self.getconfboolean('ssl', True) def getsslclientcert(self): xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] return self.getconf_xform('sslclientcert', xforms, None) def getsslclientkey(self): xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] return self.getconf_xform('sslclientkey', xforms, None) def getsslcacertfile(self): """Determines CA bundle. Returns path to the CA bundle. It is either explicitely specified or requested via "OS-DEFAULT" value (and we will search known locations for the current OS and distribution). If search via "OS-DEFAULT" route yields nothing, we will throw an exception to make our callers distinguish between not specified value and non-existent default CA bundle. It is also an error to specify non-existent file via configuration: it will error out later, but, perhaps, with less verbose explanation, so we will also throw an exception. It is consistent with the above behaviour, so any explicitely-requested configuration that doesn't result in an existing file will give an exception. """ xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] cacertfile = self.getconf_xform('sslcacertfile', xforms, None) # Can't use above cacertfile because of abspath. if self.getconf('sslcacertfile', None) == "OS-DEFAULT": cacertfile = get_os_sslcertfile() if cacertfile == None: searchpath = get_os_sslcertfile_searchpath() if searchpath: reason = "Default CA bundle was requested, "\ "but no existing locations available. "\ "Tried %s." % (", ".join(searchpath)) else: reason = "Default CA bundle was requested, "\ "but OfflineIMAP doesn't know any for your "\ "current operating system." raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) if cacertfile is None: return None if not os.path.isfile(cacertfile): reason = "CA certfile for repository '%s' couldn't be found. "\ "No such file: '%s'" % (self.name, cacertfile) raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) return cacertfile def gettlslevel(self): return self.getconf('tls_level', 'tls_compat') def getsslversion(self): return self.getconf('ssl_version', None) def getstarttls(self): return self.getconfboolean('starttls', True) def get_ssl_fingerprint(self): """Return array of possible certificate fingerprints. Configuration item cert_fingerprint can contain multiple comma-separated fingerprints in hex form.""" value = self.getconf('cert_fingerprint', "") return [f.strip().lower().replace(":", "") for f in value.split(',') if f] def setoauth2_request_url(self, url): self.oauth2_request_url = url def getoauth2_request_url(self): if self.oauth2_request_url is not None: # Use cached value if possible. return self.oauth2_request_url self.setoauth2_request_url(self.getconf('oauth2_request_url', None)) return self.oauth2_request_url def getoauth2_refresh_token(self): refresh_token = self.getconf('oauth2_refresh_token', None) if refresh_token is None: refresh_token = self.localeval.eval( self.getconf('oauth2_refresh_token_eval', "None") ) if refresh_token is not None: refresh_token = refresh_token.strip("\n") return refresh_token def getoauth2_access_token(self): access_token = self.getconf('oauth2_access_token', None) if access_token is None: access_token = self.localeval.eval( self.getconf('oauth2_access_token_eval', "None") ) if access_token is not None: access_token = access_token.strip("\n") return access_token def getoauth2_client_id(self): client_id = self.getconf('oauth2_client_id', None) if client_id is None: client_id = self.localeval.eval( self.getconf('oauth2_client_id_eval', "None") ) if client_id is not None: client_id = client_id.strip("\n") return client_id def getoauth2_client_secret(self): client_secret = self.getconf('oauth2_client_secret', None) if client_secret is None: client_secret = self.localeval.eval( self.getconf('oauth2_client_secret_eval', "None") ) if client_secret is not None: client_secret = client_secret.strip("\n") return client_secret def getpreauthtunnel(self): return self.getconf('preauthtunnel', None) def gettransporttunnel(self): return self.getconf('transporttunnel', None) def getreference(self): return self.getconf('reference', '') def getdecodefoldernames(self): return self.getconfboolean('decodefoldernames', False) def getidlefolders(self): if self.idlefolders is None: self.idlefolders = self.localeval.eval( self.getconf('idlefolders', '[]') ) return self.idlefolders def getmaxconnections(self): num1 = len(self.getidlefolders()) num2 = self.getconfint('maxconnections', 1) return max(num1, num2) def getexpunge(self): return self.getconfboolean('expunge', True) def getpassword(self): """Return the IMAP password for this repository. It tries to get passwords in the following order: 1. evaluate Repository 'remotepasseval' 2. read password from Repository 'remotepass' 3. read password from file specified in Repository 'remotepassfile' 4. read password from ~/.netrc 5. read password from /etc/netrc On success we return the password. If all strategies fail we return None.""" # 1. Evaluate Repository 'remotepasseval'. passwd = self.getconf('remotepasseval', None) if passwd is not None: return self.localeval.eval(passwd).encode('UTF-8') # 2. Read password from Repository 'remotepass'. password = self.getconf('remotepass', None) if password is not None: # Assume the configuration file to be UTF-8 encoded so we must not # encode this string again. return password # 3. Read password from file specified in Repository 'remotepassfile'. passfile = self.getconf('remotepassfile', None) if passfile is not None: fd = codecs.open(os.path.expanduser(passfile), 'r', 'UTF-8') password = fd.readline().strip() fd.close() return password.encode('UTF-8') # 4. Read password from ~/.netrc. try: netrcentry = netrc.netrc().authenticators(self.gethost()) except IOError as inst: if inst.errno != errno.ENOENT: raise else: if netrcentry: user = self.getuser() if user is None or user == netrcentry[0]: return netrcentry[2] # 5. Read password from /etc/netrc. try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) except IOError as inst: if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: user = self.getuser() if user is None or user == netrcentry[0]: return netrcentry[2] # No strategy yielded a password! return None def getfolder(self, foldername, decode=True): """Return instance of OfflineIMAP representative folder.""" return self.getfoldertype()(self.imapserver, foldername, self, decode) def getfoldertype(self): return folder.IMAP.IMAPFolder def connect(self): imapobj = self.imapserver.acquireconnection() self.imapserver.releaseconnection(imapobj) def forgetfolders(self): self.folders = None def getfolders(self): """Return a list of instances of OfflineIMAP representative folder.""" if self.folders is not None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.getconfboolean('subscribedonly', False): listfunction = imapobj.lsub try: result, listresult = listfunction(directory=self.imapserver.reference) if result != 'OK': raise OfflineImapError("Could not list the folders for" " repository %s. Server responded: %s"% (self.name, self, str(listresult)), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) for s in listresult: if s == None or \ (isinstance(s, str) and s == ''): # Bug in imaplib: empty strings in results from # literals. TODO: still relevant? continue try: flags, delim, name = imaputil.imapsplit(s) except ValueError: self.ui.error( "could not correctly parse server response; got: %s"% s) raise flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue retval.append(self.getfoldertype()(self.imapserver, name, self)) # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(imaputil.utf8_IMAP(foldername), readonly=True) except OfflineImapError as e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2], 'Invalid folderinclude:') continue retval.append(self.getfoldertype()( self.imapserver, foldername, self, decode=False)) finally: self.imapserver.releaseconnection(imapobj) if self.foldersort is None: # default sorting by case insensitive transposed name retval.sort(key=lambda x: str.lower(x.getvisiblename())) else: # do foldersort in a python3-compatible way # http://bytes.com/topic/python/answers/844614-python-3-sorting-comparison-function def cmp2key(mycmp): """Converts a cmp= function into a key= function We need to keep cmp functions for backward compatibility""" class K(object): def __init__(self, obj, *args): self.obj = obj def __cmp__(self, other): return mycmp(self.obj.getvisiblename(), other.obj.getvisiblename()) return K retval.sort(key=cmp2key(self.foldersort)) self.folders = retval return self.folders def deletefolder(self, foldername): """Delete a folder on the IMAP server.""" if self.account.utf_8_support: foldername = imaputil.utf8_IMAP(foldername) imapobj = self.imapserver.acquireconnection() try: result = imapobj.delete(foldername) if result[0] != 'OK': raise OfflineImapError("Folder '%s'[%s] could not be deleted. " "Server responded: %s"% (foldername, self, str(result)), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) def makefolder(self, foldername): """Create a folder on the IMAP server This will not update the list cached in :meth:`getfolders`. You will need to invoke :meth:`forgetfolders` to force new caching when you are done creating folders yourself. :param foldername: Full path of the folder to be created.""" if foldername is '': return if self.getreference(): foldername = self.getreference() + self.getsep() + foldername if not foldername: # Create top level folder as folder separator. foldername = self.getsep() self.ui.makefolder(self, foldername) if self.account.dryrun: return imapobj = self.imapserver.acquireconnection() try: if self.account.utf_8_support: foldername = imaputil.utf8_IMAP(foldername) result = imapobj.create(foldername) if result[0] != 'OK': raise OfflineImapError("Folder '%s'[%s] could not be created. " "Server responded: %s"% (foldername, self, str(result)), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) class MappedIMAPRepository(IMAPRepository): def getfoldertype(self): return folder.UIDMaps.MappedIMAPFolder