Merge branch 'uliska/utf8foldernames' into next

offlineimap.conf

@@ -432,6 +432,55 @@ remoterepository = RemoteExample
 #proxy = SOCKS5:IP:9999
 
 
+# EXPERIMENTAL: This option stands in the [Account Test] section.
+#
+# IMAP defines an encoding for non-ASCII ("international") characters, and most
+# IMAP servers store folder names in this encoding. Note that the IMAP 4rev1
+# specification (RFC 3501) allows both UTF-8 and modified UTF-7 folder names
+# so it is *possible* that an IMAP server already uses UTF-8 encoded folder
+# names. But usually Folders that are shown as, say, "Gäste" will be represented
+# as "G&AOQ-ste", and by default will be synchronized like this by offlineIMAP.
+#
+# This option converts IMAP folder names from IMAP4-UTF-7 to UTF-8 and back
+# in order to have nicely readable UTF-8 folder names in the local copy.
+#
+# WARNING: with this option enabled:
+# - compatibility with any other version is NOT GUARANTED (including newer);
+# - existing set-ups will probably break.
+# - no support is provided.
+#
+# IMPORTANT: READ THIS SECTION if you intend to enable this feature for an
+# EXISTING ACCOUNT that has already been synchronized!
+# Enabling UTF-8 encoded folder names will change many things on the local
+# repository of an account, so you really have to create a new local repository
+# and review the configuration. The least that would happen otherwise is a
+# duplication of all folders containing non-ASCII characters.
+# But also the following functionality may change, so the configuration in the
+# remote repository configuration has to be reviewed/updated:
+# - decodefoldernames
+#   This option is replaced by utf8foldernames and must be removed
+#   If both utf8foldernames and decodefoldernames are enabled the synchronization
+#   for the given account is aborted before doing any changes.
+# - nametrans
+#   With utf8foldernames enabled any nametrans function will operate on the
+#   UTF-8 encoded folder names, while even with decodefoldernames enabled they
+#   operate on the original IMAP4-UTF-7 encoded names.
+# - folderfilter
+#   Folder filters still work on the untranslated names before applying a
+#   nametrans function, but still this operates on the UTF-8 encoded names.
+# - folderincludes
+#   With utf8foldernames enabled this function expects UTF-8 encoded folder
+#   names.
+# - foldersort
+#   With utf8foldernames enabled the folder names passed to the sorting routine
+#   will be the UTF encoded names.
+# - idlefolders
+#   With utf8foldernames enabled folders passed to this function are expected to
+#   be UTF-8 encoded.
+#
+#utf8foldernames = no
+
+
 # TESTING: This option stands in the [Account Test] section.
 #
 # Use authproxy connection for this account. Useful to bypass the GFW in China.
@@ -1005,7 +1054,7 @@ remoteuser = username
 #reference = Mail
 
 
-# This option stands in the [Repository RemoteExample] section.
+# DEPRECATED: This option stands in the [Repository RemoteExample] section.
 #
 # IMAP defines an encoding for non-ASCII ("international") characters. Enable
 # this option if you want to decode them to the nowadays ubiquitous UTF-8.
@@ -1013,10 +1062,26 @@ remoteuser = username
 # Note that the IMAP 4rev1 specification (RFC 3501) allows both UTF-8 and
 # modified UTF-7 folder names.
 #
+# This option converts IMAP folder names from IMAP4-UTF-7 to UTF-8.
+#
+# NOTE/LIMITATION:
+# - The reencoding is applied *after* a nametrans function that may be given,
+#   so it is important to note that nametrans will work on the undecoded
+#   UTF-7 names.
+# - This option only works from a remote IMAP to a local Maildir repository
+# - It only works *once*, so it can only be used for one-off backups
+#   (see https://github.com/OfflineIMAP/offlineimap/issues/299 and especially
+#   https://github.com/OfflineIMAP/offlineimap/issues/299#issuecomment-331243827)
+#
 # WARNING: with this option enabled:
 # - compatibility with any other version is NOT GUARANTED (including newer);
 # - no support is provided.
 #
+# DEPRECATION:
+# This option is only there for backward compatibility with existing set-ups.
+# For newly created accounts please use the utf8foldernames option on account
+# level.
+#
 # This feature was merged because it's small changes in the code.  However, this
 # might seriously decrease the stability of the program. That's why it will
 # likely never be marked stable. The approach is: if it works for you, you're

offlineimap/accounts.py

@@ -69,6 +69,8 @@ class Account(CustomConfig.ConfigHelperMixin):
         self.name = name
         self.metadatadir = config.getmetadatadir()
         self.localeval = config.getlocaleval()
+        # Store utf-8 support as a property of Account object
+        self.utf_8_support = self.getconfboolean('utf8foldernames', False)
         # Current :mod:`offlineimap.ui`, can be used for logging:
         self.ui = getglobalui()
         self.refreshperiod = self.getconffloat('autorefresh', 0.0)
@@ -272,6 +274,21 @@ class SyncableAccount(Account):
                 raise
             return
 
+        if self.utf_8_support and self.remoterepos.getdecodefoldernames():
+            e = OfflineImapError("Configuration mismatch in account " +
+                        "'%s'. "% self.getname() +
+                        "\nAccount setting 'utf8foldernames' and repository " +
+                        "setting 'decodefoldernames'\nmay not be used at the " +
+                        "same time. This account has not been synchronized.\n" +
+                        "Please check the configuration and documentation.",
+                    OfflineImapError.ERROR.REPO)
+            self.ui.error(e, exc_info()[2],
+                          msg="Configuration mismatch in account " +
+                        "'%s'. "% self.getname())
+            # Abort *this* account without doing any changes.
+            # Other accounts are not affected.
+            return
+
         # Loop account sync if needed (bail out after 3 failures).
         looping = 3
         while looping:
@@ -361,7 +378,7 @@ class SyncableAccount(Account):
 
                 if not remotefolder.sync_this:
                     self.ui.debug('', "Not syncing filtered folder '%s'"
-                                  "[%s]"% (remotefolder, remoterepos))
+                                  "[%s]"% (remotefolder.getname(), remoterepos))
                     continue # Ignore filtered folder.
 
                 # The remote folder names must not have the local sep char in
@@ -379,7 +396,7 @@ class SyncableAccount(Account):
                 localfolder = self.get_local_folder(remotefolder)
                 if not localfolder.sync_this:
                     self.ui.debug('', "Not syncing filtered folder '%s'"
-                                 "[%s]"% (localfolder, localfolder.repository))
+                                 "[%s]"% (localfolder.getname(), localfolder.repository))
                     continue # Ignore filtered folder.
 
                 if not globals.options.singlethreading:

offlineimap/folder/IMAP.py

@@ -41,12 +41,21 @@ MSGCOPY_NAMESPACE = 'MSGCOPY_'
 
 
 class IMAPFolder(BaseFolder):
-    def __init__(self, imapserver, name, repository):
+    def __init__(self, imapserver, name, repository, decode=True):
-        # FIXME: decide if unquoted name is from the responsability of the
+        # decode the folder name from IMAP4_utf_7 to utf_8 if
-        # caller or not, but not both.
+        # - utf8foldernames is enabled for the *account*
+        # - the decode argument is given
+        #   (default True is used when the folder name is the result of
+        #    querying the IMAP server, while False is used when creating
+        #    a folder object from a locally available utf_8 name)
+        # In any case the given name is first dequoted.
         name = imaputil.dequote(name)
+        if decode and repository.account.utf_8_support:
+            name = imaputil.IMAP_utf8(name)
         self.sep = imapserver.delim
         super(IMAPFolder, self).__init__(name, repository)
+        if repository.getdecodefoldernames():
+            self.visiblename = imaputil.decode_mailbox_name(self.visiblename)
         self.idle_mode = False
         self.expunge = repository.getexpunge()
         self.root = None # imapserver.root
@@ -67,7 +76,6 @@ class IMAPFolder(BaseFolder):
         if self.repository.getidlefolders():
             self.idle_mode = True
 
-
     def __selectro(self, imapobj, force=False):
         """Select this folder when we do not need write access.
 
@@ -78,9 +86,15 @@ class IMAPFolder(BaseFolder):
         :param: Enforce new SELECT even if we are on that folder already.
         :returns: raises :exc:`OfflineImapError` severity FOLDER on error"""
         try:
-            imapobj.select(self.getfullname(), force = force)
+            imapobj.select(self.getfullIMAPname(), force=force)
         except imapobj.readonly:
-            imapobj.select(self.getfullname(), readonly = True, force = force)
+            imapobj.select(self.getfullIMAPname(), readonly=True, force=force)
+
+    def getfullIMAPname(self):
+        name = self.getfullname()
+        if self.repository.account.utf_8_support:
+            name = imaputil.utf8_IMAP(name)
+        return name
 
     # Interface from BaseFolder
     def suggeststhreads(self):
@@ -145,7 +159,7 @@ class IMAPFolder(BaseFolder):
             imapobj = self.imapserver.acquireconnection()
             try:
                 # Select folder and get number of messages.
-                restype, imapdata = imapobj.select(self.getfullname(), True,
+                restype, imapdata = imapobj.select(self.getfullIMAPname(), True,
                                                    True)
                 self.imapserver.releaseconnection(imapobj)
             except OfflineImapError as e:
@@ -211,7 +225,7 @@ class IMAPFolder(BaseFolder):
                 res_data.remove(0)
             return res_data
 
-        res_type, imapdata = imapobj.select(self.getfullname(), True, True)
+        res_type, imapdata = imapobj.select(self.getfullIMAPname(), True, True)
         if imapdata == [None] or imapdata[0] == '0':
             # Empty folder, no need to populate message list.
             return None
@@ -290,13 +304,6 @@ class IMAPFolder(BaseFolder):
                     'keywords': keywords}
         self.ui.messagelistloaded(self.repository, self, self.getmessagecount())
 
-    # Interface from BaseFolder
-    def getvisiblename(self):
-        vname = super(IMAPFolder, self).getvisiblename()
-        if self.repository.getdecodefoldernames():
-            return imaputil.decode_mailbox_name(vname)
-        return vname
-
     # Interface from BaseFolder
     def getmessage(self, uid):
         """Retrieve message with UID from the IMAP server (incl body).
@@ -635,7 +642,7 @@ class IMAPFolder(BaseFolder):
 
                 try:
                     # Select folder for append and make the box READ-WRITE.
-                    imapobj.select(self.getfullname())
+                    imapobj.select(self.getfullIMAPname())
                 except imapobj.readonly:
                     # readonly exception. Return original uid to notify that
                     # we did not save the message. (see savemessage in Base.py)
@@ -644,7 +651,7 @@ class IMAPFolder(BaseFolder):
 
                 # Do the APPEND.
                 try:
-                    (typ, dat) = imapobj.append(self.getfullname(),
+                    (typ, dat) = imapobj.append(self.getfullIMAPname(),
                         imaputil.flagsmaildir2imap(flags), date, content)
                     # This should only catch 'NO' responses since append()
                     # will raise an exception for 'BAD' responses:
@@ -758,7 +765,7 @@ class IMAPFolder(BaseFolder):
             fails_left = retry_num  # Retry on dropped connection.
             while fails_left:
                 try:
-                    imapobj.select(self.getfullname(), readonly=True)
+                    imapobj.select(self.getfullIMAPname(), readonly=True)
                     res_type, data = imapobj.uid('fetch', uids, query)
                     break
                 except imapobj.abort as e:
@@ -818,7 +825,7 @@ class IMAPFolder(BaseFolder):
         - field: field name to be stored/updated
         - data: field contents
         """
-        imapobj.select(self.getfullname())
+        imapobj.select(self.getfullIMAPname())
         res_type, retdata = imapobj.uid('store', uid, field, data)
         if res_type != 'OK':
             severity = OfflineImapError.ERROR.MESSAGE
@@ -879,7 +886,7 @@ class IMAPFolder(BaseFolder):
         imapobj = self.imapserver.acquireconnection()
         try:
             try:
-                imapobj.select(self.getfullname())
+                imapobj.select(self.getfullIMAPname())
             except imapobj.readonly:
                 self.ui.flagstoreadonly(self, uidlist, flags)
                 return
@@ -954,7 +961,7 @@ class IMAPFolder(BaseFolder):
         imapobj = self.imapserver.acquireconnection()
         try:
             try:
-                imapobj.select(self.getfullname())
+                imapobj.select(self.getfullIMAPname())
             except imapobj.readonly:
                 self.ui.deletereadonly(self, uidlist)
                 return

offlineimap/folder/UIDMaps.py

@@ -40,8 +40,8 @@ class MappedIMAPFolder(IMAPFolder):
       diskr2l: dict mapping message uids: self.r2l[remoteuid]=localuid
       diskl2r: dict mapping message uids: self.r2l[localuid]=remoteuid"""
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, imapserver, name, repository, decode=True):
-        IMAPFolder.__init__(self, *args, **kwargs)
+        IMAPFolder.__init__(self, imapserver, name, repository, decode=False)
         self.dryrun = self.config.getdefaultboolean("general", "dry-run", True)
         self.maplock = Lock()
         self.diskr2l, self.diskl2r = self._loadmaps()
@@ -49,7 +49,7 @@ class MappedIMAPFolder(IMAPFolder):
         # Representing the local IMAP Folder using local UIDs.
         # XXX: This should be removed since we inherit from IMAPFolder.
         # See commit 3ce514e92ba7 to know more.
-        self._mb = IMAPFolder(*args, **kwargs)
+        self._mb = IMAPFolder(imapserver, name, repository, decode=False)
 
     def _getmapfilename(self):
         return os.path.join(self.repository.getmapdir(),

offlineimap/imapserver.py

@@ -797,7 +797,7 @@ class IdleThread(object):
         localrepos = account.localrepos
         remoterepos = account.remoterepos
         statusrepos = account.statusrepos
-        remotefolder = remoterepos.getfolder(self.folder)
+        remotefolder = remoterepos.getfolder(self.folder, decode=False)
 
         hook = account.getconf('presynchook', '')
         account.callhook(hook)

offlineimap/imaputil.py

@@ -17,6 +17,8 @@
 
 import re
 import string
+import binascii
+import codecs
 from offlineimap.ui import getglobalui
 
 
@@ -370,3 +372,85 @@ def decode_mailbox_name(name):
         return ret.decode('utf-7').encode('utf-8')
     except (UnicodeDecodeError, UnicodeEncodeError):
         return name
+
+# Functionality to convert folder names encoded in IMAP_utf_7 to utf_8.
+# This is achieved by defining 'imap4_utf_7' as a proper encoding scheme.
+
+# Public API, to be used in repository definitions
+
+def IMAP_utf8(foldername):
+    """Convert IMAP4_utf_7 encoded string to utf-8"""
+    return foldername.decode('imap4-utf-7').encode('utf-8')
+
+def utf8_IMAP(foldername):
+    """Convert utf-8 encoded string to IMAP4_utf_7"""
+    return foldername.decode('utf-8').encode('imap4-utf-7')
+
+# Codec definition
+
+def modified_base64(s):
+    s = s.encode('utf-16be')
+    return binascii.b2a_base64(s).rstrip('\n=').replace('/', ',')
+
+def doB64(_in, r):
+    if _in:
+        r.append('&%s-' % modified_base64(''.join(_in)))
+        del _in[:]
+
+def encoder(s):
+    r = []
+    _in = []
+    for c in s:
+        ordC = ord(c)
+        if 0x20 <= ordC <= 0x25 or 0x27 <= ordC <= 0x7e:
+            doB64(_in, r)
+            r.append(c)
+        elif c == '&':
+            doB64(_in, r)
+            r.append('&-')
+        else:
+            _in.append(c)
+    doB64(_in, r)
+    return (str(''.join(r)), len(s))
+
+# decoding
+def modified_unbase64(s):
+    b = binascii.a2b_base64(s.replace(',', '/') + '===')
+    return unicode(b, 'utf-16be')
+
+def decoder(s):
+    r = []
+    decode = []
+    for c in s:
+        if c == '&' and not decode:
+            decode.append('&')
+        elif c == '-' and decode:
+            if len(decode) == 1:
+                r.append('&')
+            else:
+                r.append(modified_unbase64(''.join(decode[1:])))
+            decode = []
+        elif decode:
+            decode.append(c)
+        else:
+            r.append(c)
+
+    if decode:
+        r.append(modified_unbase64(''.join(decode[1:])))
+    bin_str = ''.join(r)
+    return (bin_str, len(s))
+
+class StreamReader(codecs.StreamReader):
+    def decode(self, s, errors='strict'):
+        return decoder(s)
+
+class StreamWriter(codecs.StreamWriter):
+    def decode(self, s, errors='strict'):
+        return encoder(s)
+
+def imap4_utf_7(name):
+    if name == 'imap4-utf-7':
+        return (encoder, decoder, StreamReader, StreamWriter)
+
+
+codecs.register(imap4_utf_7)

offlineimap/repository/Base.py

@@ -242,7 +242,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin):
                 # Get IMAPFolder and see if the reverse nametrans works fine.
                 # TODO: getfolder() works only because we succeed in getting
                 # inexisting folders which I would like to change. Take care!
-                tmp_remotefolder = remote_repo.getfolder(remote_name)
+                tmp_remotefolder = remote_repo.getfolder(remote_name, decode=False)
                 loop_name = tmp_remotefolder.getvisiblename().replace(
                     remote_repo.getsep(), local_repo.getsep())
                 if local_name != loop_name:

offlineimap/repository/IMAP.py

@@ -428,10 +428,10 @@ class IMAPRepository(BaseRepository):
         # No strategy yielded a password!
         return None
 
-    def getfolder(self, foldername):
+    def getfolder(self, foldername, decode=True):
         """Return instance of OfflineIMAP representative folder."""
 
-        return self.getfoldertype()(self.imapserver, foldername, self)
+        return self.getfoldertype()(self.imapserver, foldername, self, decode)
 
     def getfoldertype(self):
         return folder.IMAP.IMAPFolder
@@ -480,8 +480,7 @@ class IMAPRepository(BaseRepository):
             flaglist = [x.lower() for x in imaputil.flagsplit(flags)]
             if '\\noselect' in flaglist:
                 continue
-            foldername = imaputil.dequote(name)
+            retval.append(self.getfoldertype()(self.imapserver, name,
-            retval.append(self.getfoldertype()(self.imapserver, foldername,
                                                self))
         # Add all folderincludes
         if len(self.folderincludes):
@@ -489,7 +488,7 @@ class IMAPRepository(BaseRepository):
             try:
                 for foldername in self.folderincludes:
                     try:
-                        imapobj.select(foldername, readonly=True)
+                        imapobj.select(imaputil.utf8_IMAP(foldername), readonly=True)
                     except OfflineImapError as e:
                         # couldn't select this folderinclude, so ignore folder.
                         if e.severity > OfflineImapError.ERROR.FOLDER:
@@ -498,7 +497,7 @@ class IMAPRepository(BaseRepository):
                                       'Invalid folderinclude:')
                         continue
                     retval.append(self.getfoldertype()(
-                        self.imapserver, foldername, self))
+                        self.imapserver, foldername, self, decode=False))
             finally:
                 self.imapserver.releaseconnection(imapobj)
 
@@ -556,6 +555,9 @@ class IMAPRepository(BaseRepository):
             return
         imapobj = self.imapserver.acquireconnection()
         try:
+            if self.account.utf_8_support:
+                foldername = imaputil.utf8_IMAP(foldername)
+
             result = imapobj.create(foldername)
             if result[0] != 'OK':
                 raise OfflineImapError("Folder '%s'[%s] could not be created. "