From d19024ef2041393dcd9300e6fa0c4eeaf26f572e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20G=C3=BCnther?= Date: Wed, 3 Feb 2021 19:19:15 +0100 Subject: [PATCH 1/6] imapserver: GSSAPI: make sure reply is all bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current code mixed string and bytes leading to: ERROR: Exceptions occurred during the run! ERROR: While attempting to sync account 'honk.sigxcpu.org' sequence item 1: expected str instance, int found Traceback: File "/usr/share/offlineimap3/offlineimap/accounts.py", line 298, in syncrunner self.__sync() File "/usr/share/offlineimap3/offlineimap/accounts.py", line 374, in __sync remoterepos.getfolders() File "/usr/share/offlineimap3/offlineimap/repository/IMAP.py", line 648, in getfolders imapobj = self.imapserver.acquireconnection() File "/usr/share/offlineimap3/offlineimap/imapserver.py", line 592, in acquireconnection self.__authn_helper(imapobj) File "/usr/share/offlineimap3/offlineimap/imapserver.py", line 449, in __authn_helper if func(imapobj): File "/usr/share/offlineimap3/offlineimap/imapserver.py", line 362, in __authn_gssapi imapobj.authenticate('GSSAPI', self.__gsshandler) File "/usr/lib/python3/dist-packages/imaplib2.py", line 691, in authenticate typ, dat = self._simple_command('AUTHENTICATE', mechanism.upper()) File "/usr/lib/python3/dist-packages/imaplib2.py", line 1684, in _simple_command return self._command_complete(self._command(name, *args), kw) File "/usr/lib/python3/dist-packages/imaplib2.py", line 1404, in _command literal = literator(data, rqb) File "/usr/lib/python3/dist-packages/imaplib2.py", line 2247, in process ret = self.mech(self.decode(data)) File "/usr/share/offlineimap3/offlineimap/imapserver.py", line 318, in __gsshandler reply = ''.join(reply) Closes: #46 Signed-off-by: Guido Günther --- offlineimap/imapserver.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 3970541..bd377f7 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -310,12 +310,8 @@ class IMAPServer: # This is a behavior we got from pykerberos. First byte is one, # first four bytes are preserved (pykerberos calls this a length). # Any additional bytes are username. - reply = [] - reply[0:4] = response.message[0:4] - reply[0] = '\x01' - if self.username: - reply[5:] = self.username - reply = ''.join(reply) + reply = b'\x01' + response.message[1:4] + reply += bytes(self.username, 'utf-8') response = self.gss_vc.wrap(reply, response.encrypted) return response.message if response.message else "" From 910e194a999430b78d481bcca6e9ac9991056284 Mon Sep 17 00:00:00 2001 From: Amit Ramon Date: Fri, 5 Feb 2021 16:57:51 +0200 Subject: [PATCH 2/6] Fix utf8foldernames configuration option to work with Python3 The current code for supporting the utf8foldernames does not work with Python3 due to changes in the 'codecs' module and related functions. This fix adapts the existing code to work with Python3. Signed-off-by: Amit Ramon --- offlineimap/imaputil.py | 59 ++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 4083d34..758e78c 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -18,6 +18,7 @@ import re import binascii import codecs +from typing import Tuple from offlineimap.ui import getglobalui # Globals @@ -352,42 +353,45 @@ def decode_mailbox_name(name): def IMAP_utf8(foldername): """Convert IMAP4_utf_7 encoded string to utf-8""" - return foldername.decode('imap4-utf-7').encode('utf-8') + return codecs.decode( + foldername.encode(), + 'imap4-utf-7' + ).encode('utf-8').decode() def utf8_IMAP(foldername): """Convert utf-8 encoded string to IMAP4_utf_7""" - return foldername.decode('utf-8').encode('imap4-utf-7') + return codecs.decode( + foldername.encode(), + 'utf-8' + ).encode('imap4-utf-7').decode() # Codec definition - def modified_base64(s): s = s.encode('utf-16be') - return binascii.b2a_base64(s).rstrip('\n=').replace('/', ',') + return binascii.b2a_base64(s).rstrip(b'\n=').replace(b'/', b',') def doB64(_in, r): if _in: - r.append('&%s-' % modified_base64(''.join(_in))) + r.append(b'&%s-' % modified_base64(''.join(_in))) del _in[:] -def encoder(s): +def utf7m_encode(text: str) -> Tuple[bytes, int]: r = [] _in = [] - for c in s: - ordC = ord(c) - if 0x20 <= ordC <= 0x25 or 0x27 <= ordC <= 0x7e: + + for c in text: + if 0x20 <= ord(c) <= 0x7e: doB64(_in, r) - r.append(c) - elif c == '&': - doB64(_in, r) - r.append('&-') + r.append(b'&-' if c == '&' else c.encode()) else: _in.append(c) + doB64(_in, r) - return str(''.join(r)), len(s) + return b''.join(r), len(text) # decoding @@ -396,10 +400,10 @@ def modified_unbase64(s): return str(b, 'utf-16be') -def decoder(s): +def utf7m_decode(binary: bytes) -> Tuple[str, int]: r = [] decode = [] - for c in s: + for c in binary.decode(): if c == '&' and not decode: decode.append('&') elif c == '-' and decode: @@ -415,26 +419,31 @@ def decoder(s): if decode: r.append(modified_unbase64(''.join(decode[1:]))) - bin_str = ''.join(r) - return bin_str, len(s) + + return ''.join(r), len(binary) class StreamReader(codecs.StreamReader): def decode(self, s, errors='strict'): - return decoder(s) + return utf7m_decode(s) class StreamWriter(codecs.StreamWriter): def decode(self, s, errors='strict'): - return encoder(s) + return utf7m_encode(s) -def imap4_utf_7(name): - if name == 'imap4-utf-7': - return encoder, decoder, StreamReader, StreamWriter +def utf7m_search_function(name): + return codecs.CodecInfo( + utf7m_encode, + utf7m_decode, + StreamReader, + StreamWriter, + name='imap4-utf-7' + ) -codecs.register(imap4_utf_7) +codecs.register(utf7m_search_function) def foldername_to_imapname(folder_name): @@ -454,4 +463,4 @@ def foldername_to_imapname(folder_name): if any((c in atom_specials) for c in folder_name): folder_name = '"' + folder_name + '"' - return folder_name \ No newline at end of file + return folder_name From fa080b8d92944a1692a44e982292ffde8066950a Mon Sep 17 00:00:00 2001 From: Konstantinos Natsakis <5933427+knatsakis@users.noreply.github.com> Date: Mon, 8 Feb 2021 23:40:32 +0200 Subject: [PATCH 3/6] BUG: Right format for username using remoteusereval Similarly to 7a4285370f338a6653e8bb1a8fb99e3703683b6f, reading the username using remoteusereval returns a bytes objects instead an utf-8 string. This patch includes support for both string and bytes objects. Signed-off-by: Konstantinos Natsakis <5933427+knatsakis@users.noreply.github.com> --- offlineimap/repository/IMAP.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index b732243..e90d30c 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -206,12 +206,23 @@ class IMAPRepository(BaseRepository): Returns: Returns the remoteusereval or remoteuser or netrc user value. """ - localeval = self.localeval - if self.config.has_option(self.getsection(), 'remoteusereval'): user = self.getconf('remoteusereval') if user is not None: - return localeval.eval(user).encode('UTF-8') + l_user = self.localeval.eval(user) + + # We need a str username + if isinstance(l_user, bytes): + return l_user.decode(encoding='utf-8') + elif isinstance(l_user, str): + return l_user + + # If is not bytes or str, we have a problem + raise OfflineImapError("Could not get a right username format for" + " repository %s. Type found: %s. " + "Please, open a bug." % + (self.name, type(l_user)), + OfflineImapError.ERROR.FOLDER) if self.config.has_option(self.getsection(), 'remoteuser'): # Assume the configuration file to be UTF-8 encoded so we must not From 96793820dec2d405b6c16efaa233f3ddf3da5e19 Mon Sep 17 00:00:00 2001 From: Sudip Mukherjee Date: Wed, 10 Feb 2021 21:56:07 +0000 Subject: [PATCH 4/6] BUG: Right format for password from remotepassfile Reading the password using remotepassfile returns a bytes objects instead an utf-8 string. This patch includes support strings and bytes objects. Closes: #40 Signed-off-by: Sudip Mukherjee --- offlineimap/repository/IMAP.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index b732243..56ebe74 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -590,7 +590,20 @@ class IMAPRepository(BaseRepository): encoding='utf-8') password = file_desc.readline().strip() file_desc.close() - return password.encode('UTF-8') + + # We need a str password + if isinstance(password, bytes): + return password.decode(encoding='utf-8') + elif isinstance(password, str): + return password + + # If is not bytes or str, we have a problem + raise OfflineImapError("Could not get a right password format for" + " repository %s. Type found: %s. " + "Please, open a bug." % + (self.name, type(password)), + OfflineImapError.ERROR.FOLDER) + # 4. Read password from ~/.netrc. try: netrcentry = netrc.netrc().authenticators(self.gethost()) From 62490ff1835b22941d1775d1d4bb98d042df026b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodolfo=20Garc=C3=ADa=20Pe=C3=B1as=20=28kix=29?= Date: Fri, 19 Feb 2021 16:39:17 +0100 Subject: [PATCH 5/6] Included charset detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch includes charset detection to read the message. This patch is related to issue #43 Signed-off-by: Rodolfo García Peñas (kix) --- offlineimap/folder/Maildir.py | 15 ++++++++++++--- requirements.txt | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 205feea..0c44b60 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -20,9 +20,11 @@ import socket import time import re import os +from pathlib import Path from sys import exc_info from threading import Lock from hashlib import md5 +import chardet from offlineimap import OfflineImapError, emailutil from .Base import BaseFolder @@ -256,11 +258,18 @@ class MaildirFolder(BaseFolder): def getmessage(self, uid): """Return the content of the message.""" + # TODO: Perhaps, force the encoding using config file filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) - file = open(filepath, 'rt') - retval = file.read() - file.close() + # Open the file as binary and read it + file = Path(filepath) + blob = file.read_bytes() + # Detect the encoding + detection = chardet.detect(blob) + encoding = detection["encoding"] + # Read the file as text + retval = blob.decode(encoding) + # TODO: WHY are we replacing \r\n with \n here? And why do we # read it as text? return retval.replace("\r\n", "\n") diff --git a/requirements.txt b/requirements.txt index f543c51..19d3fff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,8 @@ gssapi[kerberos] portalocker[cygwin] rfc6555 distro + +imaplib2~=3.5 +urllib3~=1.25.9 +certifi~=2020.6.20 +chardet~=3.0.4 \ No newline at end of file From c8b275cdc3914ffd85859199fab7ba345613ee81 Mon Sep 17 00:00:00 2001 From: Joseph Ishac Date: Sun, 21 Feb 2021 22:31:49 -0500 Subject: [PATCH 6/6] Revert "Included charset detection" This reverts commit 62490ff1835b22941d1775d1d4bb98d042df026b. --- offlineimap/folder/Maildir.py | 15 +++------------ requirements.txt | 5 ----- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 0c44b60..205feea 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -20,11 +20,9 @@ import socket import time import re import os -from pathlib import Path from sys import exc_info from threading import Lock from hashlib import md5 -import chardet from offlineimap import OfflineImapError, emailutil from .Base import BaseFolder @@ -258,18 +256,11 @@ class MaildirFolder(BaseFolder): def getmessage(self, uid): """Return the content of the message.""" - # TODO: Perhaps, force the encoding using config file filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) - # Open the file as binary and read it - file = Path(filepath) - blob = file.read_bytes() - # Detect the encoding - detection = chardet.detect(blob) - encoding = detection["encoding"] - # Read the file as text - retval = blob.decode(encoding) - + file = open(filepath, 'rt') + retval = file.read() + file.close() # TODO: WHY are we replacing \r\n with \n here? And why do we # read it as text? return retval.replace("\r\n", "\n") diff --git a/requirements.txt b/requirements.txt index 19d3fff..f543c51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,3 @@ gssapi[kerberos] portalocker[cygwin] rfc6555 distro - -imaplib2~=3.5 -urllib3~=1.25.9 -certifi~=2020.6.20 -chardet~=3.0.4 \ No newline at end of file