diff --git a/offlineimap.conf b/offlineimap.conf index b7c9fbd..f6b5438 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -432,6 +432,55 @@ remoterepository = RemoteExample #proxy = SOCKS5:IP:9999 +# EXPERIMENTAL: This option stands in the [Account Test] section. +# +# IMAP defines an encoding for non-ASCII ("international") characters, and most +# IMAP servers store folder names in this encoding. Note that the IMAP 4rev1 +# specification (RFC 3501) allows both UTF-8 and modified UTF-7 folder names +# so it is *possible* that an IMAP server already uses UTF-8 encoded folder +# names. But usually Folders that are shown as, say, "Gäste" will be represented +# as "G&AOQ-ste", and by default will be synchronized like this by offlineIMAP. +# +# This option converts IMAP folder names from IMAP4-UTF-7 to UTF-8 and back +# in order to have nicely readable UTF-8 folder names in the local copy. +# +# WARNING: with this option enabled: +# - compatibility with any other version is NOT GUARANTED (including newer); +# - existing set-ups will probably break. +# - no support is provided. +# +# IMPORTANT: READ THIS SECTION if you intend to enable this feature for an +# EXISTING ACCOUNT that has already been synchronized! +# Enabling UTF-8 encoded folder names will change many things on the local +# repository of an account, so you really have to create a new local repository +# and review the configuration. The least that would happen otherwise is a +# duplication of all folders containing non-ASCII characters. +# But also the following functionality may change, so the configuration in the +# remote repository configuration has to be reviewed/updated: +# - decodefoldernames +# This option is replaced by utf8foldernames and must be removed +# If both utf8foldernames and decodefoldernames are enabled the synchronization +# for the given account is aborted before doing any changes. +# - nametrans +# With utf8foldernames enabled any nametrans function will operate on the +# UTF-8 encoded folder names, while even with decodefoldernames enabled they +# operate on the original IMAP4-UTF-7 encoded names. +# - folderfilter +# Folder filters still work on the untranslated names before applying a +# nametrans function, but still this operates on the UTF-8 encoded names. +# - folderincludes +# With utf8foldernames enabled this function expects UTF-8 encoded folder +# names. +# - foldersort +# With utf8foldernames enabled the folder names passed to the sorting routine +# will be the UTF encoded names. +# - idlefolders +# With utf8foldernames enabled folders passed to this function are expected to +# be UTF-8 encoded. +# +#utf8foldernames = no + + # TESTING: This option stands in the [Account Test] section. # # Use authproxy connection for this account. Useful to bypass the GFW in China. @@ -1005,7 +1054,7 @@ remoteuser = username #reference = Mail -# This option stands in the [Repository RemoteExample] section. +# DEPRECATED: This option stands in the [Repository RemoteExample] section. # # IMAP defines an encoding for non-ASCII ("international") characters. Enable # this option if you want to decode them to the nowadays ubiquitous UTF-8. @@ -1013,10 +1062,26 @@ remoteuser = username # Note that the IMAP 4rev1 specification (RFC 3501) allows both UTF-8 and # modified UTF-7 folder names. # +# This option converts IMAP folder names from IMAP4-UTF-7 to UTF-8. +# +# NOTE/LIMITATION: +# - The reencoding is applied *after* a nametrans function that may be given, +# so it is important to note that nametrans will work on the undecoded +# UTF-7 names. +# - This option only works from a remote IMAP to a local Maildir repository +# - It only works *once*, so it can only be used for one-off backups +# (see https://github.com/OfflineIMAP/offlineimap/issues/299 and especially +# https://github.com/OfflineIMAP/offlineimap/issues/299#issuecomment-331243827) +# # WARNING: with this option enabled: # - compatibility with any other version is NOT GUARANTED (including newer); # - no support is provided. # +# DEPRECATION: +# This option is only there for backward compatibility with existing set-ups. +# For newly created accounts please use the utf8foldernames option on account +# level. +# # This feature was merged because it's small changes in the code. However, this # might seriously decrease the stability of the program. That's why it will # likely never be marked stable. The approach is: if it works for you, you're diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 8298655..71b5135 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -69,6 +69,8 @@ class Account(CustomConfig.ConfigHelperMixin): self.name = name self.metadatadir = config.getmetadatadir() self.localeval = config.getlocaleval() + # Store utf-8 support as a property of Account object + self.utf_8_support = self.getconfboolean('utf8foldernames', False) # Current :mod:`offlineimap.ui`, can be used for logging: self.ui = getglobalui() self.refreshperiod = self.getconffloat('autorefresh', 0.0) @@ -272,6 +274,21 @@ class SyncableAccount(Account): raise return + if self.utf_8_support and self.remoterepos.getdecodefoldernames(): + e = OfflineImapError("Configuration mismatch in account " + + "'%s'. "% self.getname() + + "\nAccount setting 'utf8foldernames' and repository " + + "setting 'decodefoldernames'\nmay not be used at the " + + "same time. This account has not been synchronized.\n" + + "Please check the configuration and documentation.", + OfflineImapError.ERROR.REPO) + self.ui.error(e, exc_info()[2], + msg="Configuration mismatch in account " + + "'%s'. "% self.getname()) + # Abort *this* account without doing any changes. + # Other accounts are not affected. + return + # Loop account sync if needed (bail out after 3 failures). looping = 3 while looping: @@ -361,7 +378,7 @@ class SyncableAccount(Account): if not remotefolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" - "[%s]"% (remotefolder, remoterepos)) + "[%s]"% (remotefolder.getname(), remoterepos)) continue # Ignore filtered folder. # The remote folder names must not have the local sep char in @@ -379,7 +396,7 @@ class SyncableAccount(Account): localfolder = self.get_local_folder(remotefolder) if not localfolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" - "[%s]"% (localfolder, localfolder.repository)) + "[%s]"% (localfolder.getname(), localfolder.repository)) continue # Ignore filtered folder. if not globals.options.singlethreading: diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index a46ef31..49104c8 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -41,12 +41,21 @@ MSGCOPY_NAMESPACE = 'MSGCOPY_' class IMAPFolder(BaseFolder): - def __init__(self, imapserver, name, repository): - # FIXME: decide if unquoted name is from the responsability of the - # caller or not, but not both. + def __init__(self, imapserver, name, repository, decode=True): + # decode the folder name from IMAP4_utf_7 to utf_8 if + # - utf8foldernames is enabled for the *account* + # - the decode argument is given + # (default True is used when the folder name is the result of + # querying the IMAP server, while False is used when creating + # a folder object from a locally available utf_8 name) + # In any case the given name is first dequoted. name = imaputil.dequote(name) + if decode and repository.account.utf_8_support: + name = imaputil.IMAP_utf8(name) self.sep = imapserver.delim super(IMAPFolder, self).__init__(name, repository) + if repository.getdecodefoldernames(): + self.visiblename = imaputil.decode_mailbox_name(self.visiblename) self.idle_mode = False self.expunge = repository.getexpunge() self.root = None # imapserver.root @@ -67,7 +76,6 @@ class IMAPFolder(BaseFolder): if self.repository.getidlefolders(): self.idle_mode = True - def __selectro(self, imapobj, force=False): """Select this folder when we do not need write access. @@ -78,9 +86,15 @@ class IMAPFolder(BaseFolder): :param: Enforce new SELECT even if we are on that folder already. :returns: raises :exc:`OfflineImapError` severity FOLDER on error""" try: - imapobj.select(self.getfullname(), force = force) + imapobj.select(self.getfullIMAPname(), force=force) except imapobj.readonly: - imapobj.select(self.getfullname(), readonly = True, force = force) + imapobj.select(self.getfullIMAPname(), readonly=True, force=force) + + def getfullIMAPname(self): + name = self.getfullname() + if self.repository.account.utf_8_support: + name = imaputil.utf8_IMAP(name) + return name # Interface from BaseFolder def suggeststhreads(self): @@ -145,7 +159,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: # Select folder and get number of messages. - restype, imapdata = imapobj.select(self.getfullname(), True, + restype, imapdata = imapobj.select(self.getfullIMAPname(), True, True) self.imapserver.releaseconnection(imapobj) except OfflineImapError as e: @@ -211,7 +225,7 @@ class IMAPFolder(BaseFolder): res_data.remove(0) return res_data - res_type, imapdata = imapobj.select(self.getfullname(), True, True) + res_type, imapdata = imapobj.select(self.getfullIMAPname(), True, True) if imapdata == [None] or imapdata[0] == '0': # Empty folder, no need to populate message list. return None @@ -290,13 +304,6 @@ class IMAPFolder(BaseFolder): 'keywords': keywords} self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) - # Interface from BaseFolder - def getvisiblename(self): - vname = super(IMAPFolder, self).getvisiblename() - if self.repository.getdecodefoldernames(): - return imaputil.decode_mailbox_name(vname) - return vname - # Interface from BaseFolder def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body). @@ -635,7 +642,7 @@ class IMAPFolder(BaseFolder): try: # Select folder for append and make the box READ-WRITE. - imapobj.select(self.getfullname()) + imapobj.select(self.getfullIMAPname()) except imapobj.readonly: # readonly exception. Return original uid to notify that # we did not save the message. (see savemessage in Base.py) @@ -644,7 +651,7 @@ class IMAPFolder(BaseFolder): # Do the APPEND. try: - (typ, dat) = imapobj.append(self.getfullname(), + (typ, dat) = imapobj.append(self.getfullIMAPname(), imaputil.flagsmaildir2imap(flags), date, content) # This should only catch 'NO' responses since append() # will raise an exception for 'BAD' responses: @@ -758,7 +765,7 @@ class IMAPFolder(BaseFolder): fails_left = retry_num # Retry on dropped connection. while fails_left: try: - imapobj.select(self.getfullname(), readonly=True) + imapobj.select(self.getfullIMAPname(), readonly=True) res_type, data = imapobj.uid('fetch', uids, query) break except imapobj.abort as e: @@ -818,7 +825,7 @@ class IMAPFolder(BaseFolder): - field: field name to be stored/updated - data: field contents """ - imapobj.select(self.getfullname()) + imapobj.select(self.getfullIMAPname()) res_type, retdata = imapobj.uid('store', uid, field, data) if res_type != 'OK': severity = OfflineImapError.ERROR.MESSAGE @@ -879,7 +886,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: try: - imapobj.select(self.getfullname()) + imapobj.select(self.getfullIMAPname()) except imapobj.readonly: self.ui.flagstoreadonly(self, uidlist, flags) return @@ -954,7 +961,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: try: - imapobj.select(self.getfullname()) + imapobj.select(self.getfullIMAPname()) except imapobj.readonly: self.ui.deletereadonly(self, uidlist) return diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 48d1e96..e89a121 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -40,8 +40,8 @@ class MappedIMAPFolder(IMAPFolder): diskr2l: dict mapping message uids: self.r2l[remoteuid]=localuid diskl2r: dict mapping message uids: self.r2l[localuid]=remoteuid""" - def __init__(self, *args, **kwargs): - IMAPFolder.__init__(self, *args, **kwargs) + def __init__(self, imapserver, name, repository, decode=True): + IMAPFolder.__init__(self, imapserver, name, repository, decode=False) self.dryrun = self.config.getdefaultboolean("general", "dry-run", True) self.maplock = Lock() self.diskr2l, self.diskl2r = self._loadmaps() @@ -49,7 +49,7 @@ class MappedIMAPFolder(IMAPFolder): # Representing the local IMAP Folder using local UIDs. # XXX: This should be removed since we inherit from IMAPFolder. # See commit 3ce514e92ba7 to know more. - self._mb = IMAPFolder(*args, **kwargs) + self._mb = IMAPFolder(imapserver, name, repository, decode=False) def _getmapfilename(self): return os.path.join(self.repository.getmapdir(), diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 71c325f..8971298 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -797,7 +797,7 @@ class IdleThread(object): localrepos = account.localrepos remoterepos = account.remoterepos statusrepos = account.statusrepos - remotefolder = remoterepos.getfolder(self.folder) + remotefolder = remoterepos.getfolder(self.folder, decode=False) hook = account.getconf('presynchook', '') account.callhook(hook) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index cc51531..39bd8d4 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -17,6 +17,8 @@ import re import string +import binascii +import codecs from offlineimap.ui import getglobalui @@ -370,3 +372,85 @@ def decode_mailbox_name(name): return ret.decode('utf-7').encode('utf-8') except (UnicodeDecodeError, UnicodeEncodeError): return name + +# Functionality to convert folder names encoded in IMAP_utf_7 to utf_8. +# This is achieved by defining 'imap4_utf_7' as a proper encoding scheme. + +# Public API, to be used in repository definitions + +def IMAP_utf8(foldername): + """Convert IMAP4_utf_7 encoded string to utf-8""" + return foldername.decode('imap4-utf-7').encode('utf-8') + +def utf8_IMAP(foldername): + """Convert utf-8 encoded string to IMAP4_utf_7""" + return foldername.decode('utf-8').encode('imap4-utf-7') + +# Codec definition + +def modified_base64(s): + s = s.encode('utf-16be') + return binascii.b2a_base64(s).rstrip('\n=').replace('/', ',') + +def doB64(_in, r): + if _in: + r.append('&%s-' % modified_base64(''.join(_in))) + del _in[:] + +def encoder(s): + r = [] + _in = [] + for c in s: + ordC = ord(c) + if 0x20 <= ordC <= 0x25 or 0x27 <= ordC <= 0x7e: + doB64(_in, r) + r.append(c) + elif c == '&': + doB64(_in, r) + r.append('&-') + else: + _in.append(c) + doB64(_in, r) + return (str(''.join(r)), len(s)) + +# decoding +def modified_unbase64(s): + b = binascii.a2b_base64(s.replace(',', '/') + '===') + return unicode(b, 'utf-16be') + +def decoder(s): + r = [] + decode = [] + for c in s: + if c == '&' and not decode: + decode.append('&') + elif c == '-' and decode: + if len(decode) == 1: + r.append('&') + else: + r.append(modified_unbase64(''.join(decode[1:]))) + decode = [] + elif decode: + decode.append(c) + else: + r.append(c) + + if decode: + r.append(modified_unbase64(''.join(decode[1:]))) + bin_str = ''.join(r) + return (bin_str, len(s)) + +class StreamReader(codecs.StreamReader): + def decode(self, s, errors='strict'): + return decoder(s) + +class StreamWriter(codecs.StreamWriter): + def decode(self, s, errors='strict'): + return encoder(s) + +def imap4_utf_7(name): + if name == 'imap4-utf-7': + return (encoder, decoder, StreamReader, StreamWriter) + + +codecs.register(imap4_utf_7) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 0c3d16f..127ea79 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -242,7 +242,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin): # Get IMAPFolder and see if the reverse nametrans works fine. # TODO: getfolder() works only because we succeed in getting # inexisting folders which I would like to change. Take care! - tmp_remotefolder = remote_repo.getfolder(remote_name) + tmp_remotefolder = remote_repo.getfolder(remote_name, decode=False) loop_name = tmp_remotefolder.getvisiblename().replace( remote_repo.getsep(), local_repo.getsep()) if local_name != loop_name: diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 7581858..02c8368 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -428,10 +428,10 @@ class IMAPRepository(BaseRepository): # No strategy yielded a password! return None - def getfolder(self, foldername): + def getfolder(self, foldername, decode=True): """Return instance of OfflineIMAP representative folder.""" - return self.getfoldertype()(self.imapserver, foldername, self) + return self.getfoldertype()(self.imapserver, foldername, self, decode) def getfoldertype(self): return folder.IMAP.IMAPFolder @@ -480,8 +480,7 @@ class IMAPRepository(BaseRepository): flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue - foldername = imaputil.dequote(name) - retval.append(self.getfoldertype()(self.imapserver, foldername, + retval.append(self.getfoldertype()(self.imapserver, name, self)) # Add all folderincludes if len(self.folderincludes): @@ -489,7 +488,7 @@ class IMAPRepository(BaseRepository): try: for foldername in self.folderincludes: try: - imapobj.select(foldername, readonly=True) + imapobj.select(imaputil.utf8_IMAP(foldername), readonly=True) except OfflineImapError as e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: @@ -498,7 +497,7 @@ class IMAPRepository(BaseRepository): 'Invalid folderinclude:') continue retval.append(self.getfoldertype()( - self.imapserver, foldername, self)) + self.imapserver, foldername, self, decode=False)) finally: self.imapserver.releaseconnection(imapobj) @@ -556,6 +555,9 @@ class IMAPRepository(BaseRepository): return imapobj = self.imapserver.acquireconnection() try: + if self.account.utf_8_support: + foldername = imaputil.utf8_IMAP(foldername) + result = imapobj.create(foldername) if result[0] != 'OK': raise OfflineImapError("Folder '%s'[%s] could not be created. "