tobias/docker-offlineimap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #24 from thekix/master

Browse Source 
BUGs related with issue #16
This commit is contained in:
Rodolfo García Peñas (kix) 2020-11-07 22:54:44 +01:00 committed by GitHub
commit c5ca7dd1a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 116 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
offlineimap/folder/IMAP.py
View File

@ -150,6 +150,7 @@ class IMAPFolder(BaseFolder):
# messages or the UID of the last message have changed. Otherwise # messages or the UID of the last message have changed. Otherwise
# only flag changes could have occurred. # only flag changes could have occurred.
retry = True # Should we attempt another round or exit? retry = True # Should we attempt another round or exit?
imapdata = None
while retry: while retry:
retry = False retry = False
imapobj = self.imapserver.acquireconnection() imapobj = self.imapserver.acquireconnection()
@ -191,7 +192,8 @@ class IMAPFolder(BaseFolder):
Arguments: Arguments:
- imapobj: instance of IMAPlib - imapobj: instance of IMAPlib
- min_date (optional): a time_struct; only fetch messages newer than this - min_date (optional): a time_struct; only fetch messages newer
than this
- min_uid (optional): only fetch messages with UID >= min_uid - min_uid (optional): only fetch messages with UID >= min_uid
This function should be called with at MOST one of min_date OR This function should be called with at MOST one of min_date OR
@ -208,15 +210,17 @@ class IMAPFolder(BaseFolder):
try: try:
res_type, res_data = imapobj.search(None, search_conditions) res_type, res_data = imapobj.search(None, search_conditions)
if res_type != 'OK': if res_type != 'OK':
raise OfflineImapError("SEARCH in folder [%s]%s failed. " msg = "SEARCH in folder [%s]%s failed. " \
"Search string was '%s'. Server responded '[%s] %s'" % ( "Search string was '%s'. " \
self.getrepository(), self, search_cond, res_type, res_data), "Server responded '[%s] %s'" % \
OfflineImapError.ERROR.FOLDER) (self.getrepository(), self, search_cond,
res_type, res_data)
raise OfflineImapError(msg, OfflineImapError.ERROR.FOLDER)
except Exception as e: except Exception as e:
raise OfflineImapError("SEARCH in folder [%s]%s failed. " msg = "SEARCH in folder [%s]%s failed. "\
"Search string was '%s'. Error: %s" % ( "Search string was '%s'. Error: %s" % \
self.getrepository(), self, search_cond, str(e)), (self.getrepository(), self, search_cond, str(e))
OfflineImapError.ERROR.FOLDER) raise OfflineImapError(msg, OfflineImapError.ERROR.FOLDER)
# Davmail returns list instead of list of one element string. # Davmail returns list instead of list of one element string.
# On first run the first element is empty. # On first run the first element is empty.
if b' ' in res_data[0] or res_data[0] == b'': if b' ' in res_data[0] or res_data[0] == b'':
@ -286,10 +290,10 @@ class IMAPFolder(BaseFolder):
res_type, response = imapobj.fetch( res_type, response = imapobj.fetch(
fetch_msg, '(FLAGS UID INTERNALDATE)') fetch_msg, '(FLAGS UID INTERNALDATE)')
if res_type != 'OK': if res_type != 'OK':
raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " msg = "FETCHING UIDs in folder [%s]%s failed. "\
"Server responded '[%s] %s'" % (self.getrepository(), self, "Server responded '[%s] %s'" % \
res_type, response), (self.getrepository(), self, res_type, response)
OfflineImapError.ERROR.FOLDER) raise OfflineImapError(msg, OfflineImapError.ERROR.FOLDER)
finally: finally:
self.imapserver.releaseconnection(imapobj) self.imapserver.releaseconnection(imapobj)
@ -308,8 +312,11 @@ class IMAPFolder(BaseFolder):
self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid] = self.msglist_item_initializer(uid)
flags = imaputil.flagsimap2maildir(options['FLAGS']) flags = imaputil.flagsimap2maildir(options['FLAGS'])
keywords = imaputil.flagsimap2keywords(options['FLAGS']) keywords = imaputil.flagsimap2keywords(options['FLAGS'])
rtime = imaplibutil.Internaldate2epoch(messagestr.encode('utf-8')) rtime = imaplibutil.Internaldate2epoch(
self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, messagestr.encode('utf-8'))
self.messagelist[uid] = {'uid': uid,
'flags': flags,
'time': rtime,
'keywords': keywords} 'keywords': keywords}
self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) self.ui.messagelistloaded(self.repository, self, self.getmessagecount())
@ -376,12 +383,14 @@ class IMAPFolder(BaseFolder):
# Compute unsigned crc32 of 'content' as unique hash. # Compute unsigned crc32 of 'content' as unique hash.
# NB: crc32 returns unsigned only starting with python 3.0. # NB: crc32 returns unsigned only starting with python 3.0.
headervalue = str(binascii.crc32(str.encode(content)) & 0xffffffff) + '-' headervalue = str(binascii.crc32(str.encode(content))
& 0xffffffff) + '-'
headervalue += str(self.randomgenerator.randint(0, 9999999999)) headervalue += str(self.randomgenerator.randint(0, 9999999999))
return headername, headervalue return headername, headervalue
def __savemessage_searchforheader(self, imapobj, headername, headervalue): def __savemessage_searchforheader(self, imapobj, headername, headervalue):
self.ui.debug('imap', '__savemessage_searchforheader called for %s: %s' % self.ui.debug('imap',
'__savemessage_searchforheader called for %s: %s' %
(headername, headervalue)) (headername, headervalue))
# Now find the UID it got. # Now find the UID it got.
headervalue = imapobj._quote(headervalue) headervalue = imapobj._quote(headervalue)
@ -394,16 +403,20 @@ class IMAPFolder(BaseFolder):
except imapobj.error as err: except imapobj.error as err:
# IMAP server doesn't implement search or had a problem. # IMAP server doesn't implement search or had a problem.
self.ui.debug('imap', "__savemessage_searchforheader: got IMAP " self.ui.debug('imap',
"error '%s' while attempting to UID SEARCH for message with " "__savemessage_searchforheader: got IMAP error '%s' "
"header %s" % (err, headername)) "while attempting to UID SEARCH for message with "
"header %s" % (err, headername))
return 0 return 0
self.ui.debug('imap', "__savemessage_searchforheader got initial " self.ui.debug('imap',
"matchinguids: " + repr(matchinguids)) "__savemessage_searchforheader got initial "
"matchinguids: " + repr(matchinguids))
if matchinguids == '': if matchinguids == '':
self.ui.debug('imap', "__savemessage_searchforheader: UID SEARCH " self.ui.debug('imap',
"for message with header %s yielded no results" % headername) "__savemessage_searchforheader: UID SEARCH "
"for message with header %s yielded no results" %
headername)
return 0 return 0
matchinguids = matchinguids.split(' ') matchinguids = matchinguids.split(' ')
@ -459,13 +472,15 @@ class IMAPFolder(BaseFolder):
result = imapobj.uid('FETCH', '%d:*' % start, 'rfc822.header') result = imapobj.uid('FETCH', '%d:*' % start, 'rfc822.header')
if result[0] != 'OK': if result[0] != 'OK':
raise OfflineImapError('Error fetching mail headers: %s' % msg = 'Error fetching mail headers: %s' % '. '.join(result[1])
'. '.join(result[1]), OfflineImapError.ERROR.MESSAGE) raise OfflineImapError(msg, OfflineImapError.ERROR.MESSAGE)
# result is like: # result is like:
# [ # [
# ('185 (RFC822.HEADER {1789}', '... mail headers ...'), ' UID 2444)', # ('185 (RFC822.HEADER {1789}', '... mail headers ...'),
# ('186 (RFC822.HEADER {1789}', '... 2nd mail headers ...'), ' UID 2445)' # ' UID 2444)',
# ('186 (RFC822.HEADER {1789}', '... 2nd mail headers ...'),
# ' UID 2445)'
# ] # ]
result = result[1] result = result[1]
@ -478,7 +493,8 @@ class IMAPFolder(BaseFolder):
item = [x.decode('utf-8') for x in item] item = [x.decode('utf-8') for x in item]
# Walk just tuples. # Walk just tuples.
if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)" % (headername, headervalue), if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)" %
(headername, headervalue),
item[1], flags=re.IGNORECASE): item[1], flags=re.IGNORECASE):
found = item[0] found = item[0]
elif found is not None: elif found is not None:
@ -494,18 +510,17 @@ class IMAPFolder(BaseFolder):
# ')' # ')'
# and item[0] stored in "found" is like: # and item[0] stored in "found" is like:
# '1694 (UID 1694 RFC822.HEADER {1294}' # '1694 (UID 1694 RFC822.HEADER {1294}'
uid = re.search("\d+\s+\(UID\s+(\d+)", found, flags=re.IGNORECASE) uid = re.search("\d+\s+\(UID\s+(\d+)", found,
flags=re.IGNORECASE)
if uid: if uid:
return int(uid.group(1)) return int(uid.group(1))
self.ui.warn("Can't parse FETCH response, can't find UID in %s" % self.ui.warn("Can't parse FETCH response, "
item "can't find UID in %s" % item)
)
self.ui.debug('imap', "Got: %s" % repr(result)) self.ui.debug('imap', "Got: %s" % repr(result))
else: else:
self.ui.warn("Can't parse FETCH response, we awaited string: %s" % self.ui.warn("Can't parse FETCH response, "
repr(item) "we awaited string: %s" % repr(item))
)
return 0 return 0
@ -567,8 +582,10 @@ class IMAPFolder(BaseFolder):
# Produce a string representation of datetuple that works as # Produce a string representation of datetuple that works as
# INTERNALDATE. # INTERNALDATE.
num2mon = {1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', num2mon = {1: 'Jan', 2: 'Feb', 3: 'Mar',
7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec'} 4: 'Apr', 5: 'May', 6: 'Jun',
7: 'Jul', 8: 'Aug', 9: 'Sep',
10: 'Oct', 11: 'Nov', 12: 'Dec'}
# tm_isdst coming from email.parsedate is not usable, we still use it # tm_isdst coming from email.parsedate is not usable, we still use it
# here, mhh. # here, mhh.
@ -579,8 +596,10 @@ class IMAPFolder(BaseFolder):
offset_h, offset_m = divmod(zone // 60, 60) offset_h, offset_m = divmod(zone // 60, 60)
internaldate = '"%02d-%s-%04d %02d:%02d:%02d %+03d%02d"' % \ internaldate = '"%02d-%s-%04d %02d:%02d:%02d %+03d%02d"' % \
(datetuple.tm_mday, num2mon[datetuple.tm_mon], datetuple.tm_year, (datetuple.tm_mday, num2mon[datetuple.tm_mon],
datetuple.tm_hour, datetuple.tm_min, datetuple.tm_sec, offset_h, offset_m) datetuple.tm_year, datetuple.tm_hour,
datetuple.tm_min, datetuple.tm_sec,
offset_h, offset_m)
return internaldate return internaldate
@ -645,7 +664,8 @@ class IMAPFolder(BaseFolder):
content) content)
self.ui.debug('imap', 'savemessage: header is: %s: %s' % self.ui.debug('imap', 'savemessage: header is: %s: %s' %
(headername, headervalue)) (headername, headervalue))
content = self.addmessageheader(content, CRLF, headername, headervalue) content = self.addmessageheader(content, CRLF,
headername, headervalue)
if len(content) > 200: if len(content) > 200:
dbg_output = "%s...%s" % (content[:150], content[-50:]) dbg_output = "%s...%s" % (content[:150], content[-50:])
@ -665,19 +685,25 @@ class IMAPFolder(BaseFolder):
# Do the APPEND. # Do the APPEND.
try: try:
(typ, dat) = imapobj.append(self.getfullIMAPname(), (typ, dat) = imapobj.append(
imaputil.flagsmaildir2imap(flags), date, bytes(content, 'utf-8')) self.getfullIMAPname(),
imaputil.flagsmaildir2imap(flags),
date, bytes(content, 'utf-8'))
# This should only catch 'NO' responses since append() # This should only catch 'NO' responses since append()
# will raise an exception for 'BAD' responses: # will raise an exception for 'BAD' responses:
if typ != 'OK': if typ != 'OK':
# For example, Groupwise IMAP server can return something like: # For example, Groupwise IMAP server
# can return something like:
# #
# NO APPEND The 1500 MB storage limit has been exceeded. # NO APPEND The 1500 MB storage limit \
# has been exceeded.
# #
# In this case, we should immediately abort the repository sync # In this case, we should immediately abort
# and continue with the next account. # the repository sync and continue
# with the next account.
msg = \ msg = \
"Saving msg (%s) in folder '%s', repository '%s' failed (abort). " \ "Saving msg (%s) in folder '%s', " \
"repository '%s' failed (abort). " \
"Server responded: %s %s\n" % \ "Server responded: %s %s\n" % \
(msg_id, self, self.getrepository(), typ, dat) (msg_id, self, self.getrepository(), typ, dat)
raise OfflineImapError(msg, OfflineImapError.ERROR.REPO) raise OfflineImapError(msg, OfflineImapError.ERROR.REPO)
@ -690,9 +716,11 @@ class IMAPFolder(BaseFolder):
if not retry_left: if not retry_left:
raise OfflineImapError( raise OfflineImapError(
"Saving msg (%s) in folder '%s', " "Saving msg (%s) in folder '%s', "
"repository '%s' failed (abort). Server responded: %s\n" "repository '%s' failed (abort). "
"Server responded: %s\n"
"Message content was: %s" % "Message content was: %s" %
(msg_id, self, self.getrepository(), str(e), dbg_output), (msg_id, self, self.getrepository(),
str(e), dbg_output),
OfflineImapError.ERROR.MESSAGE, OfflineImapError.ERROR.MESSAGE,
exc_info()[2]) exc_info()[2])
@ -706,8 +734,10 @@ class IMAPFolder(BaseFolder):
imapobj = None imapobj = None
raise OfflineImapError( raise OfflineImapError(
"Saving msg (%s) folder '%s', repo '%s'" "Saving msg (%s) folder '%s', repo '%s'"
"failed (error). Server responded: %s\nMessage content was: " "failed (error). Server responded: %s\n"
"%s" % (msg_id, self, self.getrepository(), str(e), dbg_output), "Message content was: %s" %
(msg_id, self, self.getrepository(),
str(e), dbg_output),
OfflineImapError.ERROR.MESSAGE, OfflineImapError.ERROR.MESSAGE,
exc_info()[2]) exc_info()[2])
@ -730,39 +760,47 @@ class IMAPFolder(BaseFolder):
"appending a message. Got: %s." % str(resp)) "appending a message. Got: %s." % str(resp))
return 0 return 0
try: try:
uid = int(resp[-1].split(' ')[1]) # Convert the UID from [b'4 1532'] to ['4 1532']
s_uid = [x.decode('utf-8') for x in resp]
# Now, read the UID field
uid = int(s_uid[-1].split(' ')[1])
except ValueError: except ValueError:
uid = 0 # Definetly not what we should have. uid = 0 # Definetly not what we should have.
except Exception: except Exception:
raise OfflineImapError("Unexpected response: %s" % str(resp), raise OfflineImapError("Unexpected response: %s" %
str(resp),
OfflineImapError.ERROR.MESSAGE) OfflineImapError.ERROR.MESSAGE)
if uid == 0: if uid == 0:
self.ui.warn("savemessage: Server supports UIDPLUS, but" self.ui.warn("savemessage: Server supports UIDPLUS, but"
" we got no usable UID back. APPENDUID reponse was " " we got no usable UID back. APPENDUID "
"'%s'" % str(resp)) "reponse was '%s'" % str(resp))
else: else:
try: try:
# We don't use UIDPLUS. # We don't use UIDPLUS.
uid = self.__savemessage_searchforheader(imapobj, headername, uid = self.__savemessage_searchforheader(imapobj,
headername,
headervalue) headervalue)
# See docs for savemessage in Base.py for explanation # See docs for savemessage in Base.py for explanation
# of this and other return values. # of this and other return values.
if uid == 0: if uid == 0:
self.ui.debug('imap', 'savemessage: attempt to get new UID ' self.ui.debug('imap',
'UID failed. Search headers manually.') 'savemessage: attempt to get new UID '
uid = self.__savemessage_fetchheaders(imapobj, headername, 'UID failed. Search headers manually.')
uid = self.__savemessage_fetchheaders(imapobj,
headername,
headervalue) headervalue)
self.ui.warn("savemessage: Searching mails for new " self.ui.warn("savemessage: Searching mails for new "
"Message-ID failed. Could not determine new UID " "Message-ID failed. "
"on %s." % self.getname()) "Could not determine new UID on %s." %
self.getname())
# Something wrong happened while trying to get the UID. Explain # Something wrong happened while trying to get the UID. Explain
# the error might be about the 'get UID' process not necesseraly # the error might be about the 'get UID' process not necesseraly
# the APPEND. # the APPEND.
except Exception: except Exception:
self.ui.warn("%s: could not determine the UID while we got " self.ui.warn("%s: could not determine the UID while we got "
"no error while appending the email with '%s: %s'" % "no error while appending the "
(self.getname(), headername, headervalue) "email with '%s: %s'" %
) (self.getname(), headername, headervalue))
raise raise
finally: finally:
if imapobj: if imapobj:
@ -805,9 +843,7 @@ class IMAPFolder(BaseFolder):
self.ui.error("%s. While fetching msg %r in folder %r." self.ui.error("%s. While fetching msg %r in folder %r."
" Query: %s Retrying (%d/%d)" % ( " Query: %s Retrying (%d/%d)" % (
e, uids, self.name, query, e, uids, self.name, query,
retry_num - fails_left, retry_num retry_num - fails_left, retry_num))
)
)
# Release dropped connection, and get a new one. # Release dropped connection, and get a new one.
self.imapserver.releaseconnection(imapobj, True) self.imapserver.releaseconnection(imapobj, True)
imapobj = self.imapserver.acquireconnection() imapobj = self.imapserver.acquireconnection()
@ -828,9 +864,9 @@ class IMAPFolder(BaseFolder):
# data for the UID FETCH command. # data for the UID FETCH command.
if data == [None] or res_type != 'OK' or len(data) != 1: if data == [None] or res_type != 'OK' or len(data) != 1:
severity = OfflineImapError.ERROR.MESSAGE severity = OfflineImapError.ERROR.MESSAGE
reason = "IMAP server '%s' failed to fetch messages UID '%s'." \ reason = "IMAP server '%s' failed to fetch messages UID '%s'. " \
" Server responded: %s %s" % (self.getrepository(), uids, "Server responded: %s %s" % (self.getrepository(), uids,
res_type, data) res_type, data)
if data == [None] or len(data) < 1: if data == [None] or len(data) < 1:
# IMAP server did not find a message with this UID. # IMAP server did not find a message with this UID.
reason = "IMAP server '%s' does not have a message " \ reason = "IMAP server '%s' does not have a message " \
@ -857,7 +893,8 @@ class IMAPFolder(BaseFolder):
res_type, retdata = imapobj.uid('store', uid, field, data) res_type, retdata = imapobj.uid('store', uid, field, data)
if res_type != 'OK': if res_type != 'OK':
severity = OfflineImapError.ERROR.MESSAGE severity = OfflineImapError.ERROR.MESSAGE
reason = "IMAP server '%s' failed to store %s for message UID '%d'." \ reason = "IMAP server '%s' failed to store %s " \
"for message UID '%d'." \
"Server responded: %s %s" % ( "Server responded: %s %s" % (
self.getrepository(), field, uid, res_type, retdata) self.getrepository(), field, uid, res_type, retdata)
raise OfflineImapError(reason, severity) raise OfflineImapError(reason, severity)
@ -919,7 +956,8 @@ class IMAPFolder(BaseFolder):
self.ui.flagstoreadonly(self, uidlist, flags) self.ui.flagstoreadonly(self, uidlist, flags)
return return
response = imapobj.uid('store', response = imapobj.uid('store',
imaputil.uid_sequence(uidlist), operation + 'FLAGS', imaputil.uid_sequence(uidlist),
operation + 'FLAGS',
imaputil.flagsmaildir2imap(flags)) imaputil.flagsmaildir2imap(flags))
if response[0] != 'OK': if response[0] != 'OK':
raise OfflineImapError( raise OfflineImapError(
@ -969,7 +1007,8 @@ class IMAPFolder(BaseFolder):
If the backend supports it. IMAP does not and will throw errors.""" If the backend supports it. IMAP does not and will throw errors."""
raise OfflineImapError('IMAP backend cannot change a messages UID from ' raise OfflineImapError('IMAP backend cannot change a messages UID from '
'%d to %d' % (uid, new_uid), OfflineImapError.ERROR.MESSAGE) '%d to %d' %
(uid, new_uid), OfflineImapError.ERROR.MESSAGE)
# Interface from BaseFolder # Interface from BaseFolder
def deletemessage(self, uid): def deletemessage(self, uid):

4
offlineimap/imapserver.py
View File

@ -595,7 +595,9 @@ class IMAPServer:
# update capabilities after login, e.g. gmail serves different ones # update capabilities after login, e.g. gmail serves different ones
typ, dat = imapobj.capability() typ, dat = imapobj.capability()
if dat != [None]: if dat != [None]:
imapobj.capabilities = tuple(dat[-1].upper().split()) # Get the capabilities and convert them to string from bytes
s_dat = [x.decode('utf-8') for x in dat[-1].upper().split()]
imapobj.capabilities = tuple(s_dat)
if self.delim is None: if self.delim is None:
listres = imapobj.list(self.reference, '""')[1] listres = imapobj.list(self.reference, '""')[1]