tobias/docker-offlineimap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixing up the rest of the parsing and IMAP functions, and GMAIL classes as well. Also adding is_debugging() to the UI to allow us to quickly determine if we should build some of the expensive debug objects

Browse Source
This commit is contained in:
Joseph Ishac 2021-02-23 16:17:54 -05:00
parent 5fc08e529b
commit 259bf83607
6 changed files with 80 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
offlineimap/folder/Base.py
View File

@ -60,9 +60,9 @@ class BaseFolder:
policy.default.clone(cte_type='8bit', utf8=True, refold_source='none', linesep='\r\n'),
}
# Parsers
self.parse = {
'8bit': BytesParser(policy=p1),
}
self.parser = {}
for key in self.policy:
self.parser[key] = BytesParser(policy=self.policy[key])
# Save original name for folderfilter operations.
self.ffilter_name = name
# Top level dir name is always ''.

39
offlineimap/folder/Gmail.py
View File

@ -69,10 +69,9 @@ class GmailFolder(IMAPFolder):
data = self._fetch_from_imap(str(uid), self.retrycount)
# data looks now e.g.
# ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....']
# ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}',<email.message.EmailMessage object>]
# we only asked for one message, and that msg is in data[1].
# msbody is in [1].
body = data[1].replace("\r\n", "\n")
msg = data[1]
# Embed the labels into the message headers
if self.synclabels:
@ -84,19 +83,23 @@ class GmailFolder(IMAPFolder):
labels = labels - self.ignorelabels
labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels))
# First remove old label headers that may be in the message content retrieved
# First remove old label headers that may be in the message body retrieved
# from gmail Then add a labels header with current gmail labels.
body = self.deletemessageheaders(body, self.labelsheader)
body = self.addmessageheader(body, '\n', self.labelsheader, labels_str)
self.deletemessageheaders(msg, self.labelsheader)
self.addmessageheader(msg, self.labelsheader, labels_str)
if len(body) > 200:
dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:])
else:
dbg_output = body
if self.ui.is_debugging('imap'):
# Optimization: don't create the debugging objects unless needed
msg_s = msg.as_string(policy=self.policy['8bit-RFC'])
if len(msg_s) > 200:
dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
else:
dbg_output = msg_s
self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
(uid, dbg_output))
return body
self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
(uid, dbg_output))
return msg
def getmessagelabels(self, uid):
if 'labels' in self.messagelist[uid]:
@ -167,7 +170,7 @@ class GmailFolder(IMAPFolder):
rtime = imaplibutil.Internaldate2epoch(messagestr)
self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime}
def savemessage(self, uid, content, flags, rtime):
def savemessage(self, uid, msg, flags, rtime):
"""Save the message on the Server
This backend always assigns a new uid, so the uid arg is ignored.
@ -180,7 +183,7 @@ class GmailFolder(IMAPFolder):
savemessage is never called in a dryrun mode.
:param uid: Message UID
:param content: Message content
:param msg: Message object
:param flags: Message flags
:param rtime: A timestamp to be used as the mail date
:returns: the UID of the new message as assigned by the server. If the
@ -189,13 +192,13 @@ class GmailFolder(IMAPFolder):
read-only for example) it will return -1."""
if not self.synclabels:
return super(GmailFolder, self).savemessage(uid, content, flags, rtime)
return super(GmailFolder, self).savemessage(uid, msg, flags, rtime)
labels = set()
for hstr in self.getmessageheaderlist(content, self.labelsheader):
for hstr in self.getmessageheaderlist(msg, self.labelsheader):
labels.update(imaputil.labels_from_header(self.labelsheader, hstr))
ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime)
ret = super(GmailFolder, self).savemessage(uid, msg, flags, rtime)
self.savemessagelabels(ret, labels)
return ret

32
offlineimap/folder/GmailMaildir.py
View File

@ -90,12 +90,12 @@ class GmailMaildirFolder(MaildirFolder):
if not os.path.exists(filepath):
return set()
file = open(filepath, 'rt')
content = file.read()
file.close()
fd = open(filepath, 'rb')
msg = self.parser['8bit'].parse(fd)
fd.close()
self.messagelist[uid]['labels'] = set()
for hstr in self.getmessageheaderlist(content, self.labelsheader):
for hstr in self.getmessageheaderlist(msg, self.labelsheader):
self.messagelist[uid]['labels'].update(
imaputil.labels_from_header(self.labelsheader, hstr))
self.messagelist[uid]['labels_cached'] = True
@ -108,7 +108,7 @@ class GmailMaildirFolder(MaildirFolder):
else:
return self.messagelist[uid]['mtime']
def savemessage(self, uid, content, flags, rtime):
def savemessage(self, uid, msg, flags, rtime):
"""Writes a new message, with the specified uid.
See folder/Base for detail. Note that savemessage() does not
@ -116,14 +116,15 @@ class GmailMaildirFolder(MaildirFolder):
savemessage is never called in a dryrun mode."""
if not self.synclabels:
return super(GmailMaildirFolder, self).savemessage(uid, content,
return super(GmailMaildirFolder, self).savemessage(uid, msg,
flags, rtime)
labels = set()
for hstr in self.getmessageheaderlist(content, self.labelsheader):
for hstr in self.getmessageheaderlist(msg, self.labelsheader):
labels.update(imaputil.labels_from_header(self.labelsheader, hstr))
ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags,
# TODO - Not sure why the returned uid is stored early as ret here?
ret = super(GmailMaildirFolder, self).savemessage(uid, msg, flags,
rtime)
# Update the mtime and labels.
@ -145,12 +146,12 @@ class GmailMaildirFolder(MaildirFolder):
filename = self.messagelist[uid]['filename']
filepath = os.path.join(self.getfullname(), filename)
file = open(filepath, 'rt')
content = file.read()
file.close()
fd = open(filepath, 'rb')
msg = self.parser['8bit'].parse(fd)
fd.close()
oldlabels = set()
for hstr in self.getmessageheaderlist(content, self.labelsheader):
for hstr in self.getmessageheaderlist(msg, self.labelsheader):
oldlabels.update(imaputil.labels_from_header(self.labelsheader,
hstr))
@ -167,15 +168,14 @@ class GmailMaildirFolder(MaildirFolder):
sorted(labels | ignoredlabels))
# First remove old labels header, and then add the new one.
content = self.deletemessageheaders(content, self.labelsheader)
content = self.addmessageheader(content, '\n', self.labelsheader,
labels_str)
self.deletemessageheaders(msg, self.labelsheader)
self.addmessageheader(msg, self.labelsheader, labels_str)
mtime = int(os.stat(filepath).st_mtime)
# Write file with new labels to a unique file in tmp.
messagename = self.new_message_filename(uid, set())
tmpname = self.save_to_tmp_file(messagename, content)
tmpname = self.save_to_tmp_file(messagename, msg)
tmppath = os.path.join(self.getfullname(), tmpname)
# Move to actual location.

55
offlineimap/folder/IMAP.py
View File

@ -342,19 +342,22 @@ class IMAPFolder(BaseFolder):
data = self._fetch_from_imap(str(uid), self.retrycount)
# Data looks now e.g.
# ['320 (17061 BODY[] {2565}','msgbody....']
# ['320 (17061 BODY[] {2565}',<email.message.EmailMessage object>]
# Is a list of two elements. Message is at [1]
data = data[1].replace(CRLF, "\n")
msg = data[1]
if len(data) > 200:
dbg_output = "%s...%s" % (str(data)[:150], str(data)[-50:])
else:
dbg_output = data
if self.ui.is_debugging('imap'):
# Optimization: don't create the debugging objects unless needed
msg_s = msg.as_string(policy=self.policy['8bit-RFC'])
if len(msg_s) > 200:
dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
else:
dbg_output = msg_s
self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
(uid, dbg_output))
self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
(uid, dbg_output))
return data
return msg
# Interface from BaseFolder
def getmessagetime(self, uid):
@ -679,13 +682,15 @@ class IMAPFolder(BaseFolder):
(headername, headervalue))
self.addmessageheader(msg, headername, headervalue)
msg_s = msg.as_string(policy=output_policy)
if len(msg_s) > 200:
dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
else:
dbg_output = msg_s
self.ui.debug('imap', "savemessage: date: %s, content: '%s'" %
(date, dbg_output))
if self.ui.is_debugging('imap'):
# Optimization: don't create the debugging objects unless needed
msg_s = msg.as_string(policy=output_policy)
if len(msg_s) > 200:
dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
else:
dbg_output = msg_s
self.ui.debug('imap', "savemessage: date: %s, content: '%s'" %
(date, dbg_output))
try:
# Select folder for append and make the box READ-WRITE.
@ -830,7 +835,7 @@ class IMAPFolder(BaseFolder):
"""Fetches data from IMAP server.
Arguments:
- uids: message UIDS
- uids: message UIDS (OfflineIMAP3: First UID returned only)
- retry_num: number of retries to make
Returns: data obtained by this query."""
@ -886,9 +891,21 @@ class IMAPFolder(BaseFolder):
"with UID '%s'" % (self.getrepository(), uids)
raise OfflineImapError(reason, severity)
# Convert bytes to str
# JI: In offlineimap, this function returned a tuple of strings for each
# fetched UID, offlineimap3 calls to the imap object return bytes and so
# originally a fixed, utf-8 conversion was done and *only* the first
# response (d[0]) was returned. Note that this alters the behavior
# between code bases. However, it seems like a single UID is the intent
# of this function so retaining the modfication here for now.
#
# TODO: Can we assume the server response containing the meta data is
# always 'utf-8' encoded? Assuming yes for now.
#
# Convert responses, d[0][0], into a 'utf-8' string (from bytes) and
# Convert email, d[0][1], into a message object (from bytes)
ndata0 = data[0][0].decode('utf-8')
ndata1 = data[0][1].decode('utf-8', errors='replace')
ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1])
ndata = [ndata0, ndata1]
return ndata

2
offlineimap/folder/Maildir.py
View File

@ -259,7 +259,7 @@ class MaildirFolder(BaseFolder):
filename = self.messagelist[uid]['filename']
filepath = os.path.join(self.getfullname(), filename)
fd = open(filepath, 'rb')
retval = self.parse['8bit'](fd)
retval = self.parser['8bit'].parse(fd)
fd.close()
return retval

3
offlineimap/ui/UIBase.py
View File

@ -231,6 +231,9 @@ class UIBase:
else:
self.invaliddebug(debugtype)
def is_debugging(self, debugtype):
return (debugtype in self.debuglist)
def debugging(self, debugtype):
global debugtypes
self.logger.debug("Now debugging for %s: %s" % (debugtype,