require usernames and passwords to be UTF-8 encoded

- Learn to support UTF-8 characters where it was not supported for usernames and
  passwords (but for netrc).
- Fix the types in the code for both py2 and py3: we now expect unicode for
  usernames and passwords.

Unicode (UTF-8) is required only for variables with non-ASCII characters.

Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>

offlineimap.conf

@@ -590,9 +590,16 @@ type = IMAP
 # "getcredentials" that parses a file "filename" and returns the account
 # details for "hostname".
 #
+#
 #remotehosteval = getcredentials("filename", "hostname", "hostname")
+#
+# The returned value must be int type.
 #remoteporteval = getcredentials("filename", "hostname", "port")
+#
+# The returned value must be unicode type.
 #remoteusereval = getcredentials("filename", "hostname", "user")
+#
+# The returned value must be unicode type.
 #remotepasseval = getcredentials("filename", "hostname", "passwd")
 
 
@@ -742,9 +749,9 @@ remotehost = examplehost
 
 # This option stands in the [Repository RemoteExample] section.
 #
-# Specify the remote user name.
+# Specify the remote user name. Must be unicode.
 #
-remoteuser = username
+remoteuser = u"username"
 
 
 # This option stands in the [Repository RemoteExample] section.
@@ -760,7 +767,9 @@ remoteuser = username
 # mechanism, so consider using auth_mechanisms to prioritize PLAIN
 # or even make it the only mechanism to be tried.
 #
-#remote_identity = authzuser
+# Must be unicode type.
+#
+#remote_identity = u"authzuser"
 
 
 # This option stands in the [Repository RemoteExample] section.
@@ -842,11 +851,11 @@ remoteuser = username
 #
 # 2. The remote password stored in this file with the remotepass
 #    option. Any '%' needs to be encoded as '%%'. Example:
-#    remotepass = mypassword
+#remotepass = u"mypassword"
 #
 # 3. The remote password stored as a single line in an external
 #    file, which is referenced by the remotefile option.  Example:
-#    remotepassfile = ~/Password.IMAP.Account1
+#remotepassfile = ~/Password.IMAP.Account1
 #
 # 4. With a preauth tunnel.  With this method, you invoke an external
 #    program that is guaranteed *NOT* to ask for a password, but rather
@@ -855,7 +864,7 @@ remoteuser = username
 #    NOT specify a user or password (if you do, they'll be ignored.)
 #    Instead, you specify a preauthtunnel, as this example illustrates
 #    for Courier IMAP on Debian:
-#    preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir'
+#preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir'
 #
 # 5. If you are using Kerberos and have the Python Kerberos package
 #    installed, you should not specify a remotepass.  If the user has a
@@ -865,9 +874,9 @@ remoteuser = username
 # 6. Using arbitrary python code.  With this method, you invoke a
 #    function from your pythonfile.  To use this method assign the name
 #    of the function to the variable 'remotepasseval'.  Example:
-#    remotepasseval = get_password("imap.example.net")
+#remotepasseval = get_password("imap.example.net")
 #    You can also query for the username:
-#    remoteusereval = get_username("imap.example.net")
+#remoteusereval = get_username("imap.example.net")
 #    This method can be used to design more elaborate setups, e.g. by
 #    querying the gnome-keyring via its python bindings.
 
@@ -1186,7 +1195,9 @@ type = Gmail
 #
 # Specify the Gmail user name. This is the only mandatory parameter.
 #
-remoteuser = username@gmail.com
+# Must be unicode type.
+#
+remoteuser = u"username@gmail.com"
 
 
 # This option stands in the [Repository GmailExample] section.

offlineimap/imapserver.py

@@ -154,6 +154,7 @@ class IMAPServer(object):
 
     def __getpassword(self):
         """Returns the server password or None"""
+
         if self.goodpassword != None: # use cached good one first
             return self.goodpassword
 
@@ -216,13 +217,15 @@ class IMAPServer(object):
 
         authc = self.username
         passwd = self.__getpassword()
-        authz = ''
+        authz = b''
         if self.user_identity != None:
             authz = self.user_identity
-        NULL = u'\x00'
-        retval = NULL.join((authz, authc, passwd)).encode('utf-8')
-        logsafe_retval = NULL.join((authz, authc, "(passwd hidden for log)")).encode('utf-8')
-        self.ui.debug('imap', '__plainhandler: returning %s' % logsafe_retval)
+        # At this point all authz, authc and passwd are expected bytes encoded
+        # in UTF-8.
+        NULL = b'\x00'
+        retval = NULL.join((authz, authc, passwd))
+        logsafe_retval = NULL.join((authz, authc, "(passwd hidden for log)"))
+        self.ui.debug('imap', '__plainhandler: returning %s'% logsafe_retval)
         return retval
 
 

offlineimap/repository/IMAP.py

@@ -1,5 +1,5 @@
 # IMAP repository support
-# Copyright (C) 2002-2015 John Goerzen & contributors
+# Copyright (C) 2002-2016 John Goerzen & contributors
 #
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@ from sys import exc_info
 import netrc
 import errno
 import six
+import codecs
 
 from offlineimap.repository.Base import BaseRepository
 from offlineimap import folder, imaputil, imapserver, OfflineImapError
@@ -129,7 +130,10 @@ class IMAPRepository(BaseRepository):
         (currently -- PLAIN) to inform server about the ID
         we want to authorize as instead of our login name."""
 
-        return self.getconf('remote_identity', default=None)
+        identity = self.getconf('remote_identity', default=None)
+        if identity != None:
+            identity = identity.encode('UTF-8')
+        return identity
 
     def get_auth_mechanisms(self):
         supported = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"]
@@ -159,12 +163,12 @@ class IMAPRepository(BaseRepository):
         if self.config.has_option(self.getsection(), 'remoteusereval'):
             user = self.getconf('remoteusereval')
         if user != None:
-            return localeval.eval(user)
+            return localeval.eval(user).encode('UTF-8')
 
         if self.config.has_option(self.getsection(), 'remoteuser'):
             user = self.getconf('remoteuser')
         if user != None:
-            return user
+            return user.encode('UTF-8')
 
         try:
             netrcentry = netrc.netrc().authenticators(self.gethost())
@@ -326,18 +330,18 @@ class IMAPRepository(BaseRepository):
         # 1. evaluate Repository 'remotepasseval'
         passwd = self.getconf('remotepasseval', None)
         if passwd != None:
-            return self.localeval.eval(passwd)
+            return self.localeval.eval(passwd).encode('UTF-8')
         # 2. read password from Repository 'remotepass'
         password = self.getconf('remotepass', None)
         if password != None:
-            return password
+            return password.encode('UTF-8')
         # 3. read password from file specified in Repository 'remotepassfile'
         passfile = self.getconf('remotepassfile', None)
         if passfile != None:
-            fd = open(os.path.expanduser(passfile))
+            fd = codecs.open(os.path.expanduser(passfile), 'r', 'UTF-8')
             password = fd.readline().strip()
             fd.close()
-            return password
+            return password.encode('UTF-8')
         # 4. read password from ~/.netrc
         try:
             netrcentry = netrc.netrc().authenticators(self.gethost())