From 36d726763d0097a53cedfff35b01424c0dcef33b Mon Sep 17 00:00:00 2001 From: Urs Liska Date: Mon, 2 Oct 2017 01:26:29 +0200 Subject: [PATCH] utf8: implement utf8foldernames option If utf8foldernames is enabled on account level all folder names read from the IMAP server will immediately be reencoded to UTF-8. Names will be treated as UTF-8 as long as the IMAP server isn't contacted again, for which they are reencoded to IMAP4-UTF-7. This means that any further processing such as nametrans, folderfilter etc. will act upon the UTF-8 names, which will have to be documented carefully. NOTE 1: GMail repositories and folders inherit from the IMAP... classes, so I don't know yet if these changes have ugly side-effects. But web research suggests that GMail IMAP folders are equally encoded in UTF-7 so that should work identically here and incorporate the same improvements. NOTE 2: I could not test the behaviour with idlefolders as I didn't get this option to work at all, not even with the latest stable version. NOTE 3: I *did* test to sync an IMAP repository against another IMAP repository. Signed-off-by: Urs Liska Signed-off-by: Nicolas Sebrecht --- offlineimap/accounts.py | 6 +++-- offlineimap/folder/IMAP.py | 40 ++++++++++++++++++++++------------ offlineimap/folder/UIDMaps.py | 6 ++--- offlineimap/imapserver.py | 2 +- offlineimap/repository/Base.py | 2 +- offlineimap/repository/IMAP.py | 11 ++++++---- 6 files changed, 42 insertions(+), 25 deletions(-) diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index f50aa78..7f10aed 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -69,6 +69,8 @@ class Account(CustomConfig.ConfigHelperMixin): self.name = name self.metadatadir = config.getmetadatadir() self.localeval = config.getlocaleval() + # Store utf-8 support as a property of Account object + self.utf_8_support = self.getconfboolean('utf8foldernames', False) # Current :mod:`offlineimap.ui`, can be used for logging: self.ui = getglobalui() self.refreshperiod = self.getconffloat('autorefresh', 0.0) @@ -360,7 +362,7 @@ class SyncableAccount(Account): if not remotefolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" - "[%s]"% (remotefolder, remoterepos)) + "[%s]"% (remotefolder.getname(), remoterepos)) continue # Ignore filtered folder. # The remote folder names must not have the local sep char in @@ -378,7 +380,7 @@ class SyncableAccount(Account): localfolder = self.get_local_folder(remotefolder) if not localfolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" - "[%s]"% (localfolder, localfolder.repository)) + "[%s]"% (localfolder.getname(), localfolder.repository)) continue # Ignore filtered folder. if not globals.options.singlethreading: diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index a23708c..b9d4be8 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -41,10 +41,17 @@ MSGCOPY_NAMESPACE = 'MSGCOPY_' class IMAPFolder(BaseFolder): - def __init__(self, imapserver, name, repository): - # FIXME: decide if unquoted name is from the responsability of the - # caller or not, but not both. + def __init__(self, imapserver, name, repository, decode=True): + # decode the folder name from IMAP4_utf_7 to utf_8 if + # - utf8foldernames is enabled for the *account* + # - the decode argument is given + # (default True is used when the folder name is the result of + # querying the IMAP server, while False is used when creating + # a folder object from a locally available utf_8 name) + # In any case the given name is first dequoted. name = imaputil.dequote(name) + if decode and repository.account.utf_8_support: + name = imaputil.IMAP_utf8(name) self.sep = imapserver.delim super(IMAPFolder, self).__init__(name, repository) if repository.getdecodefoldernames(): @@ -69,7 +76,6 @@ class IMAPFolder(BaseFolder): if self.repository.getidlefolders(): self.idle_mode = True - def __selectro(self, imapobj, force=False): """Select this folder when we do not need write access. @@ -80,9 +86,15 @@ class IMAPFolder(BaseFolder): :param: Enforce new SELECT even if we are on that folder already. :returns: raises :exc:`OfflineImapError` severity FOLDER on error""" try: - imapobj.select(self.getfullname(), force = force) + imapobj.select(self.getfullIMAPname(), force = force) except imapobj.readonly: - imapobj.select(self.getfullname(), readonly = True, force = force) + imapobj.select(self.getfullIMAPname(), readonly = True, force = force) + + def getfullIMAPname(self): + name = self.getfullname() + if self.repository.account.utf_8_support: + name = imaputil.utf8_IMAP(name) + return name # Interface from BaseFolder def suggeststhreads(self): @@ -147,7 +159,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: # Select folder and get number of messages. - restype, imapdata = imapobj.select(self.getfullname(), True, + restype, imapdata = imapobj.select(self.getfullIMAPname(), True, True) self.imapserver.releaseconnection(imapobj) except OfflineImapError as e: @@ -213,7 +225,7 @@ class IMAPFolder(BaseFolder): res_data.remove(0) return res_data - res_type, imapdata = imapobj.select(self.getfullname(), True, True) + res_type, imapdata = imapobj.select(self.getfullIMAPname(), True, True) if imapdata == [None] or imapdata[0] == '0': # Empty folder, no need to populate message list. return None @@ -630,7 +642,7 @@ class IMAPFolder(BaseFolder): try: # Select folder for append and make the box READ-WRITE. - imapobj.select(self.getfullname()) + imapobj.select(self.getfullIMAPname()) except imapobj.readonly: # readonly exception. Return original uid to notify that # we did not save the message. (see savemessage in Base.py) @@ -639,7 +651,7 @@ class IMAPFolder(BaseFolder): # Do the APPEND. try: - (typ, dat) = imapobj.append(self.getfullname(), + (typ, dat) = imapobj.append(self.getfullIMAPname(), imaputil.flagsmaildir2imap(flags), date, content) # This should only catch 'NO' responses since append() # will raise an exception for 'BAD' responses: @@ -753,7 +765,7 @@ class IMAPFolder(BaseFolder): fails_left = retry_num # Retry on dropped connection. while fails_left: try: - imapobj.select(self.getfullname(), readonly=True) + imapobj.select(self.getfullIMAPname(), readonly=True) res_type, data = imapobj.uid('fetch', uids, query) break except imapobj.abort as e: @@ -813,7 +825,7 @@ class IMAPFolder(BaseFolder): - field: field name to be stored/updated - data: field contents """ - imapobj.select(self.getfullname()) + imapobj.select(self.getfullIMAPname()) res_type, retdata = imapobj.uid('store', uid, field, data) if res_type != 'OK': severity = OfflineImapError.ERROR.MESSAGE @@ -874,7 +886,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: try: - imapobj.select(self.getfullname()) + imapobj.select(self.getfullIMAPname()) except imapobj.readonly: self.ui.flagstoreadonly(self, uidlist, flags) return @@ -949,7 +961,7 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: try: - imapobj.select(self.getfullname()) + imapobj.select(self.getfullIMAPname()) except imapobj.readonly: self.ui.deletereadonly(self, uidlist) return diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 48d1e96..e89a121 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -40,8 +40,8 @@ class MappedIMAPFolder(IMAPFolder): diskr2l: dict mapping message uids: self.r2l[remoteuid]=localuid diskl2r: dict mapping message uids: self.r2l[localuid]=remoteuid""" - def __init__(self, *args, **kwargs): - IMAPFolder.__init__(self, *args, **kwargs) + def __init__(self, imapserver, name, repository, decode=True): + IMAPFolder.__init__(self, imapserver, name, repository, decode=False) self.dryrun = self.config.getdefaultboolean("general", "dry-run", True) self.maplock = Lock() self.diskr2l, self.diskl2r = self._loadmaps() @@ -49,7 +49,7 @@ class MappedIMAPFolder(IMAPFolder): # Representing the local IMAP Folder using local UIDs. # XXX: This should be removed since we inherit from IMAPFolder. # See commit 3ce514e92ba7 to know more. - self._mb = IMAPFolder(*args, **kwargs) + self._mb = IMAPFolder(imapserver, name, repository, decode=False) def _getmapfilename(self): return os.path.join(self.repository.getmapdir(), diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 71c325f..8971298 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -797,7 +797,7 @@ class IdleThread(object): localrepos = account.localrepos remoterepos = account.remoterepos statusrepos = account.statusrepos - remotefolder = remoterepos.getfolder(self.folder) + remotefolder = remoterepos.getfolder(self.folder, decode=False) hook = account.getconf('presynchook', '') account.callhook(hook) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 0c3d16f..127ea79 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -242,7 +242,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin): # Get IMAPFolder and see if the reverse nametrans works fine. # TODO: getfolder() works only because we succeed in getting # inexisting folders which I would like to change. Take care! - tmp_remotefolder = remote_repo.getfolder(remote_name) + tmp_remotefolder = remote_repo.getfolder(remote_name, decode=False) loop_name = tmp_remotefolder.getvisiblename().replace( remote_repo.getsep(), local_repo.getsep()) if local_name != loop_name: diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 010d6e8..02c8368 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -428,10 +428,10 @@ class IMAPRepository(BaseRepository): # No strategy yielded a password! return None - def getfolder(self, foldername): + def getfolder(self, foldername, decode=True): """Return instance of OfflineIMAP representative folder.""" - return self.getfoldertype()(self.imapserver, foldername, self) + return self.getfoldertype()(self.imapserver, foldername, self, decode) def getfoldertype(self): return folder.IMAP.IMAPFolder @@ -488,7 +488,7 @@ class IMAPRepository(BaseRepository): try: for foldername in self.folderincludes: try: - imapobj.select(foldername, readonly=True) + imapobj.select(imaputil.utf8_IMAP(foldername), readonly=True) except OfflineImapError as e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: @@ -497,7 +497,7 @@ class IMAPRepository(BaseRepository): 'Invalid folderinclude:') continue retval.append(self.getfoldertype()( - self.imapserver, foldername, self)) + self.imapserver, foldername, self, decode=False)) finally: self.imapserver.releaseconnection(imapobj) @@ -555,6 +555,9 @@ class IMAPRepository(BaseRepository): return imapobj = self.imapserver.acquireconnection() try: + if self.account.utf_8_support: + foldername = imaputil.utf8_IMAP(foldername) + result = imapobj.create(foldername) if result[0] != 'OK': raise OfflineImapError("Folder '%s'[%s] could not be created. "