Merge pull request #56 from jishac/multiple_encoding_support
Multiple encoding support
This commit is contained in:
Rodolfo García Peñas (kix)
GitHub
commit
1e7ef9e7e6
@ -1,38 +0,0 @@
|
||||
# Some useful functions to extract data out of emails
|
||||
# Copyright (C) 2002-2015 John Goerzen & contributors
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
import email
|
||||
from email.parser import Parser as MailParser
|
||||
|
||||
|
||||
def get_message_date(content, header='Date'):
|
||||
"""Parses mail and returns resulting timestamp.
|
||||
|
||||
:param content: Mail content
|
||||
:param header: the header to extract date from;
|
||||
:returns: timestamp or `None` in the case of failure.
|
||||
"""
|
||||
|
||||
message = MailParser().parsestr(content, True)
|
||||
dateheader = message.get(header)
|
||||
# parsedate_tz returns a 10-tuple that can be passed to mktime_tz
|
||||
# Will be None if missing or not in a valid format. Note that
|
||||
# indexes 6, 7, and 8 of the result tuple are not usable.
|
||||
datetuple = email.utils.parsedate_tz(dateheader)
|
||||
if datetuple is None:
|
||||
return None
|
||||
return email.utils.mktime_tz(datetuple)
|
||||
@ -22,6 +22,11 @@ import re
|
||||
import time
|
||||
from sys import exc_info
|
||||
|
||||
from email import policy
|
||||
from email.parser import BytesParser
|
||||
from email.generator import BytesGenerator
|
||||
from email.utils import parsedate_tz, mktime_tz
|
||||
|
||||
from offlineimap import threadutil
|
||||
from offlineimap.ui import getglobalui
|
||||
from offlineimap.error import OfflineImapError
|
||||
@ -42,6 +47,22 @@ class BaseFolder:
|
||||
|
||||
self.ui = getglobalui()
|
||||
self.messagelist = {}
|
||||
# Use the built-in email libraries
|
||||
# Establish some policies
|
||||
self.policy = {
|
||||
'7bit':
|
||||
policy.default.clone(cte_type='7bit', utf8=False, refold_source='none'),
|
||||
'7bit-RFC':
|
||||
policy.default.clone(cte_type='7bit', utf8=False, refold_source='none', linesep='\r\n'),
|
||||
'8bit':
|
||||
policy.default.clone(cte_type='8bit', utf8=True, refold_source='none'),
|
||||
'8bit-RFC':
|
||||
policy.default.clone(cte_type='8bit', utf8=True, refold_source='none', linesep='\r\n'),
|
||||
}
|
||||
# Parsers
|
||||
self.parser = {}
|
||||
for key in self.policy:
|
||||
self.parser[key] = BytesParser(policy=self.policy[key])
|
||||
# Save original name for folderfilter operations.
|
||||
self.ffilter_name = name
|
||||
# Top level dir name is always ''.
|
||||
@ -349,7 +370,7 @@ class BaseFolder:
|
||||
return len(self.getmessagelist())
|
||||
|
||||
def getmessage(self, uid):
|
||||
"""Returns the content of the specified message."""
|
||||
"""Returns an email message object."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@ -466,7 +487,7 @@ class BaseFolder:
|
||||
except:
|
||||
raise IOError("Can't read %s" % uidfile)
|
||||
|
||||
def savemessage(self, uid, content, flags, rtime):
|
||||
def savemessage(self, uid, msg, flags, rtime):
|
||||
"""Writes a new message, with the specified uid.
|
||||
|
||||
If the uid is < 0: The backend should assign a new uid and
|
||||
@ -637,211 +658,90 @@ class BaseFolder:
|
||||
for uid in uidlist:
|
||||
self.deletemessagelabels(uid, labels)
|
||||
|
||||
def addmessageheader(self, content, linebreak, headername, headervalue):
|
||||
def addmessageheader(self, msg, headername, headervalue):
|
||||
"""Adds new header to the provided message.
|
||||
|
||||
WARNING: This function is a bit tricky, and modifying it in the wrong
|
||||
way, may easily lead to data-loss.
|
||||
|
||||
Arguments:
|
||||
- content: message content, headers and body as a single string
|
||||
- linebreak: string that carries line ending
|
||||
- msg: message object
|
||||
- headername: name of the header to add
|
||||
- headervalue: value of the header to add
|
||||
|
||||
.. note::
|
||||
Returns: None
|
||||
|
||||
The following documentation will not get displayed correctly after
|
||||
being processed by Sphinx. View the source of this method to read it.
|
||||
|
||||
This has to deal with strange corner cases where the header is
|
||||
missing or empty. Here are illustrations for all the cases,
|
||||
showing where the header gets inserted and what the end result
|
||||
is. In each illustration, '+' means the added contents. Note
|
||||
that these examples assume LF for linebreak, not CRLF, so '\n'
|
||||
denotes a linebreak and '\n\n' corresponds to the transition
|
||||
between header and body. However if the linebreak parameter
|
||||
is set to '\r\n' then you would have to substitute '\r\n' for
|
||||
'\n' in the below examples.
|
||||
|
||||
* Case 1: No '\n\n', leading '\n'
|
||||
|
||||
+X-Flying-Pig-Header: i am here\n
|
||||
\n
|
||||
This is the body\n
|
||||
next line\n
|
||||
|
||||
* Case 2: '\n\n' at position 0
|
||||
|
||||
+X-Flying-Pig-Header: i am here
|
||||
\n
|
||||
\n
|
||||
This is the body\n
|
||||
next line\n
|
||||
|
||||
* Case 3: No '\n\n', no leading '\n'
|
||||
|
||||
+X-Flying-Pig-Header: i am here\n
|
||||
+\n
|
||||
This is the body\n
|
||||
next line\n
|
||||
|
||||
* Case 4: '\n\n' at non-zero position
|
||||
|
||||
Subject: Something wrong with OI\n
|
||||
From: some@person.at
|
||||
+\nX-Flying-Pig-Header: i am here
|
||||
\n
|
||||
\n
|
||||
This is the body\n
|
||||
next line\n
|
||||
"""
|
||||
|
||||
self.ui.debug('', 'addmessageheader: called to add %s: %s' %
|
||||
(headername, headervalue))
|
||||
|
||||
insertionpoint = content.find(linebreak * 2)
|
||||
if insertionpoint == -1:
|
||||
self.ui.debug('', 'addmessageheader: headers were missing')
|
||||
else:
|
||||
self.ui.debug('',
|
||||
'addmessageheader: headers end at position %d' %
|
||||
insertionpoint)
|
||||
mark = '==>EOH<=='
|
||||
contextstart = max(0, insertionpoint - 100)
|
||||
contextend = min(len(content), insertionpoint + 100)
|
||||
self.ui.debug('', 'addmessageheader: header/body transition " \
|
||||
"context (marked by %s): %s%s%s' % (
|
||||
mark, repr(content[contextstart:insertionpoint]),
|
||||
mark, repr(content[insertionpoint:contextend])
|
||||
)
|
||||
)
|
||||
msg.add_header(headername, headervalue)
|
||||
return
|
||||
|
||||
# Hoping for case #4.
|
||||
prefix = linebreak
|
||||
suffix = ''
|
||||
# Case #2.
|
||||
if insertionpoint == 0:
|
||||
prefix = ''
|
||||
suffix = ''
|
||||
# Either case #1 or #3.
|
||||
elif insertionpoint == -1:
|
||||
prefix = ''
|
||||
suffix = linebreak
|
||||
insertionpoint = 0
|
||||
# Case #3: when body starts immediately, without preceding '\n'
|
||||
# (this shouldn't happen with proper mail messages, but
|
||||
# we seen many broken ones), we should add '\n' to make
|
||||
# new (and the only header, in this case) to be properly
|
||||
# separated from the message body.
|
||||
if content[0:len(linebreak)] != linebreak:
|
||||
suffix = suffix + linebreak
|
||||
|
||||
self.ui.debug('',
|
||||
'addmessageheader: insertionpoint = %d' % insertionpoint)
|
||||
headers = content[0:insertionpoint]
|
||||
self.ui.debug('',
|
||||
'addmessageheader: headers = %s' % repr(headers))
|
||||
new_header = prefix + ("%s: %s" % (headername, headervalue)) + suffix
|
||||
self.ui.debug('',
|
||||
'addmessageheader: new_header = %s' % repr(new_header))
|
||||
return headers + new_header + content[insertionpoint:]
|
||||
|
||||
def __find_eoh(self, content):
|
||||
"""Searches for the point where mail headers end.
|
||||
|
||||
Either double '\n', or end of string.
|
||||
|
||||
Arguments:
|
||||
- content: contents of the message to search in
|
||||
Returns: position of the first non-header byte.
|
||||
"""
|
||||
|
||||
eoh_cr = content.find('\n\n')
|
||||
if eoh_cr == -1:
|
||||
eoh_cr = len(content)
|
||||
|
||||
return eoh_cr
|
||||
|
||||
def getmessageheader(self, content, name):
|
||||
"""Return the value of the first occurence of the given header.
|
||||
def getmessageheader(self, msg, headername):
|
||||
"""Return the value of an undefined occurence of the given header.
|
||||
|
||||
Header name is case-insensitive.
|
||||
|
||||
Arguments:
|
||||
- contents: message itself
|
||||
- name: name of the header to be searched
|
||||
- msg: message object
|
||||
- headername: name of the header to be searched
|
||||
|
||||
Returns: header value or None if no such header was found.
|
||||
"""
|
||||
|
||||
self.ui.debug('', 'getmessageheader: called to get %s' % name)
|
||||
eoh = self.__find_eoh(content)
|
||||
self.ui.debug('', 'getmessageheader: eoh = %d' % eoh)
|
||||
headers = content[0:eoh]
|
||||
self.ui.debug('', 'getmessageheader: headers = %s' % repr(headers))
|
||||
self.ui.debug('', 'getmessageheader: called to get %s' % headername)
|
||||
return msg.get(headername)
|
||||
|
||||
m = re.search('^%s:(.*)$' % name, headers,
|
||||
flags=re.MULTILINE | re.IGNORECASE)
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
else:
|
||||
return None
|
||||
|
||||
def getmessageheaderlist(self, content, name):
|
||||
def getmessageheaderlist(self, msg, headername):
|
||||
"""Return a list of values for the given header.
|
||||
|
||||
Header name is case-insensitive.
|
||||
|
||||
Arguments:
|
||||
- contents: message itself
|
||||
- name: name of the header to be searched
|
||||
- msg: message object
|
||||
- headername: name of the header to be searched
|
||||
|
||||
Returns: list of header values or empty list if no such header was
|
||||
found.
|
||||
"""
|
||||
|
||||
self.ui.debug('', 'getmessageheaderlist: called to get %s' % name)
|
||||
eoh = self.__find_eoh(content)
|
||||
self.ui.debug('', 'getmessageheaderlist: eoh = %d' % eoh)
|
||||
headers = content[0:eoh]
|
||||
self.ui.debug('', 'getmessageheaderlist: headers = %s' % repr(headers))
|
||||
self.ui.debug('', 'getmessageheaderlist: called to get %s' % headername)
|
||||
return msg.get_all(headername, [])
|
||||
|
||||
return re.findall('^%s:(.*)$' %
|
||||
name, headers, flags=re.MULTILINE | re.IGNORECASE)
|
||||
|
||||
def deletemessageheaders(self, content, header_list):
|
||||
"""Deletes headers in the given list from the message content.
|
||||
def deletemessageheaders(self, msg, header_list):
|
||||
"""Deletes headers in the given list from the message.
|
||||
|
||||
Arguments:
|
||||
- content: message itself
|
||||
- msg: message object
|
||||
- header_list: list of headers to be deleted or just the header name
|
||||
|
||||
We expect our message to have '\n' as line endings."""
|
||||
"""
|
||||
|
||||
if type(header_list) != type([]):
|
||||
header_list = [header_list]
|
||||
self.ui.debug('',
|
||||
'deletemessageheaders: called to delete %s' % header_list)
|
||||
|
||||
if not len(header_list):
|
||||
return content
|
||||
for h in header_list:
|
||||
del msg[h]
|
||||
|
||||
eoh = self.__find_eoh(content)
|
||||
self.ui.debug('', 'deletemessageheaders: end of headers = %d' % eoh)
|
||||
headers = content[0:eoh]
|
||||
rest = content[eoh:]
|
||||
self.ui.debug('', 'deletemessageheaders: headers = %s' % repr(headers))
|
||||
new_headers = []
|
||||
for h in headers.split('\n'):
|
||||
keep_it = True
|
||||
for trim_h in header_list:
|
||||
if len(h) > len(trim_h) \
|
||||
and h[0:len(trim_h) + 1] == (trim_h + ":"):
|
||||
keep_it = False
|
||||
break
|
||||
if keep_it:
|
||||
new_headers.append(h)
|
||||
return
|
||||
|
||||
return '\n'.join(new_headers) + rest
|
||||
def get_message_date(self, msg, header="Date"):
|
||||
"""Returns the Unix timestamp of the email message, derived from the
|
||||
Date field header by default.
|
||||
|
||||
Arguments:
|
||||
- msg: message object
|
||||
- header: header to extract the date from
|
||||
|
||||
Returns: timestamp or `None` in the case of failure.
|
||||
"""
|
||||
|
||||
datetuple = parsedate_tz(msg.get(header))
|
||||
if datetuple is None:
|
||||
return None
|
||||
|
||||
return mktime_tz(datetuple)
|
||||
|
||||
def change_message_uid(self, uid, new_uid):
|
||||
"""Change the message from existing uid to new_uid.
|
||||
|
||||
@ -69,14 +69,13 @@ class GmailFolder(IMAPFolder):
|
||||
data = self._fetch_from_imap(str(uid), self.retrycount)
|
||||
|
||||
# data looks now e.g.
|
||||
# ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....']
|
||||
# ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}',<email.message.EmailMessage object>]
|
||||
# we only asked for one message, and that msg is in data[1].
|
||||
# msbody is in [1].
|
||||
body = data[1].replace("\r\n", "\n")
|
||||
msg = data[1]
|
||||
|
||||
# Embed the labels into the message headers
|
||||
if self.synclabels:
|
||||
m = re.search('X-GM-LABELS\s*[(](.*)[)]', data[0][0])
|
||||
m = re.search('X-GM-LABELS\s*[(](.*)[)]', data[0])
|
||||
if m:
|
||||
labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))])
|
||||
else:
|
||||
@ -84,19 +83,23 @@ class GmailFolder(IMAPFolder):
|
||||
labels = labels - self.ignorelabels
|
||||
labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels))
|
||||
|
||||
# First remove old label headers that may be in the message content retrieved
|
||||
# First remove old label headers that may be in the message body retrieved
|
||||
# from gmail Then add a labels header with current gmail labels.
|
||||
body = self.deletemessageheaders(body, self.labelsheader)
|
||||
body = self.addmessageheader(body, '\n', self.labelsheader, labels_str)
|
||||
self.deletemessageheaders(msg, self.labelsheader)
|
||||
self.addmessageheader(msg, self.labelsheader, labels_str)
|
||||
|
||||
if len(body) > 200:
|
||||
dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:])
|
||||
if self.ui.is_debugging('imap'):
|
||||
# Optimization: don't create the debugging objects unless needed
|
||||
msg_s = msg.as_string(policy=self.policy['8bit-RFC'])
|
||||
if len(msg_s) > 200:
|
||||
dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
|
||||
else:
|
||||
dbg_output = body
|
||||
dbg_output = msg_s
|
||||
|
||||
self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
|
||||
(uid, dbg_output))
|
||||
return body
|
||||
|
||||
return msg
|
||||
|
||||
def getmessagelabels(self, uid):
|
||||
if 'labels' in self.messagelist[uid]:
|
||||
@ -167,7 +170,7 @@ class GmailFolder(IMAPFolder):
|
||||
rtime = imaplibutil.Internaldate2epoch(messagestr)
|
||||
self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime}
|
||||
|
||||
def savemessage(self, uid, content, flags, rtime):
|
||||
def savemessage(self, uid, msg, flags, rtime):
|
||||
"""Save the message on the Server
|
||||
|
||||
This backend always assigns a new uid, so the uid arg is ignored.
|
||||
@ -180,7 +183,7 @@ class GmailFolder(IMAPFolder):
|
||||
savemessage is never called in a dryrun mode.
|
||||
|
||||
:param uid: Message UID
|
||||
:param content: Message content
|
||||
:param msg: Message object
|
||||
:param flags: Message flags
|
||||
:param rtime: A timestamp to be used as the mail date
|
||||
:returns: the UID of the new message as assigned by the server. If the
|
||||
@ -189,13 +192,13 @@ class GmailFolder(IMAPFolder):
|
||||
read-only for example) it will return -1."""
|
||||
|
||||
if not self.synclabels:
|
||||
return super(GmailFolder, self).savemessage(uid, content, flags, rtime)
|
||||
return super(GmailFolder, self).savemessage(uid, msg, flags, rtime)
|
||||
|
||||
labels = set()
|
||||
for hstr in self.getmessageheaderlist(content, self.labelsheader):
|
||||
for hstr in self.getmessageheaderlist(msg, self.labelsheader):
|
||||
labels.update(imaputil.labels_from_header(self.labelsheader, hstr))
|
||||
|
||||
ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime)
|
||||
ret = super(GmailFolder, self).savemessage(uid, msg, flags, rtime)
|
||||
self.savemessagelabels(ret, labels)
|
||||
return ret
|
||||
|
||||
|
||||
@ -90,12 +90,12 @@ class GmailMaildirFolder(MaildirFolder):
|
||||
if not os.path.exists(filepath):
|
||||
return set()
|
||||
|
||||
file = open(filepath, 'rt')
|
||||
content = file.read()
|
||||
file.close()
|
||||
fd = open(filepath, 'rb')
|
||||
msg = self.parser['8bit'].parse(fd)
|
||||
fd.close()
|
||||
|
||||
self.messagelist[uid]['labels'] = set()
|
||||
for hstr in self.getmessageheaderlist(content, self.labelsheader):
|
||||
for hstr in self.getmessageheaderlist(msg, self.labelsheader):
|
||||
self.messagelist[uid]['labels'].update(
|
||||
imaputil.labels_from_header(self.labelsheader, hstr))
|
||||
self.messagelist[uid]['labels_cached'] = True
|
||||
@ -108,7 +108,7 @@ class GmailMaildirFolder(MaildirFolder):
|
||||
else:
|
||||
return self.messagelist[uid]['mtime']
|
||||
|
||||
def savemessage(self, uid, content, flags, rtime):
|
||||
def savemessage(self, uid, msg, flags, rtime):
|
||||
"""Writes a new message, with the specified uid.
|
||||
|
||||
See folder/Base for detail. Note that savemessage() does not
|
||||
@ -116,14 +116,15 @@ class GmailMaildirFolder(MaildirFolder):
|
||||
savemessage is never called in a dryrun mode."""
|
||||
|
||||
if not self.synclabels:
|
||||
return super(GmailMaildirFolder, self).savemessage(uid, content,
|
||||
return super(GmailMaildirFolder, self).savemessage(uid, msg,
|
||||
flags, rtime)
|
||||
|
||||
labels = set()
|
||||
for hstr in self.getmessageheaderlist(content, self.labelsheader):
|
||||
for hstr in self.getmessageheaderlist(msg, self.labelsheader):
|
||||
labels.update(imaputil.labels_from_header(self.labelsheader, hstr))
|
||||
|
||||
ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags,
|
||||
# TODO - Not sure why the returned uid is stored early as ret here?
|
||||
ret = super(GmailMaildirFolder, self).savemessage(uid, msg, flags,
|
||||
rtime)
|
||||
|
||||
# Update the mtime and labels.
|
||||
@ -145,12 +146,12 @@ class GmailMaildirFolder(MaildirFolder):
|
||||
filename = self.messagelist[uid]['filename']
|
||||
filepath = os.path.join(self.getfullname(), filename)
|
||||
|
||||
file = open(filepath, 'rt')
|
||||
content = file.read()
|
||||
file.close()
|
||||
fd = open(filepath, 'rb')
|
||||
msg = self.parser['8bit'].parse(fd)
|
||||
fd.close()
|
||||
|
||||
oldlabels = set()
|
||||
for hstr in self.getmessageheaderlist(content, self.labelsheader):
|
||||
for hstr in self.getmessageheaderlist(msg, self.labelsheader):
|
||||
oldlabels.update(imaputil.labels_from_header(self.labelsheader,
|
||||
hstr))
|
||||
|
||||
@ -167,15 +168,14 @@ class GmailMaildirFolder(MaildirFolder):
|
||||
sorted(labels | ignoredlabels))
|
||||
|
||||
# First remove old labels header, and then add the new one.
|
||||
content = self.deletemessageheaders(content, self.labelsheader)
|
||||
content = self.addmessageheader(content, '\n', self.labelsheader,
|
||||
labels_str)
|
||||
self.deletemessageheaders(msg, self.labelsheader)
|
||||
self.addmessageheader(msg, self.labelsheader, labels_str)
|
||||
|
||||
mtime = int(os.stat(filepath).st_mtime)
|
||||
|
||||
# Write file with new labels to a unique file in tmp.
|
||||
messagename = self.new_message_filename(uid, set())
|
||||
tmpname = self.save_to_tmp_file(messagename, content)
|
||||
tmpname = self.save_to_tmp_file(messagename, msg)
|
||||
tmppath = os.path.join(self.getfullname(), tmpname)
|
||||
|
||||
# Move to actual location.
|
||||
|
||||
@ -20,7 +20,7 @@ import binascii
|
||||
import re
|
||||
import time
|
||||
from sys import exc_info
|
||||
from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError
|
||||
from offlineimap import imaputil, imaplibutil, OfflineImapError
|
||||
from offlineimap import globals
|
||||
from imaplib2 import MonthNames
|
||||
from .Base import BaseFolder
|
||||
@ -30,13 +30,6 @@ CRLF = '\r\n'
|
||||
MSGCOPY_NAMESPACE = 'MSGCOPY_'
|
||||
|
||||
|
||||
# NB: message returned from getmessage() will have '\n' all over the place,
|
||||
# NB: there will be no CRLFs. Just before the sending stage of savemessage()
|
||||
# NB: '\n' will be transformed back to CRLF. So, for the most parts of the
|
||||
# NB: code the stored content will be clean of CRLF and one can rely that
|
||||
# NB: line endings will be pure '\n'.
|
||||
|
||||
|
||||
class IMAPFolder(BaseFolder):
|
||||
def __init__(self, imapserver, name, repository, decode=True):
|
||||
# decode the folder name from IMAP4_utf_7 to utf_8 if
|
||||
@ -349,19 +342,22 @@ class IMAPFolder(BaseFolder):
|
||||
data = self._fetch_from_imap(str(uid), self.retrycount)
|
||||
|
||||
# Data looks now e.g.
|
||||
# ['320 (17061 BODY[] {2565}','msgbody....']
|
||||
# ['320 (17061 BODY[] {2565}',<email.message.EmailMessage object>]
|
||||
# Is a list of two elements. Message is at [1]
|
||||
data = data[1].replace(CRLF, "\n")
|
||||
msg = data[1]
|
||||
|
||||
if len(data) > 200:
|
||||
dbg_output = "%s...%s" % (str(data)[:150], str(data)[-50:])
|
||||
if self.ui.is_debugging('imap'):
|
||||
# Optimization: don't create the debugging objects unless needed
|
||||
msg_s = msg.as_string(policy=self.policy['8bit-RFC'])
|
||||
if len(msg_s) > 200:
|
||||
dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
|
||||
else:
|
||||
dbg_output = data
|
||||
dbg_output = msg_s
|
||||
|
||||
self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
|
||||
(uid, dbg_output))
|
||||
|
||||
return data
|
||||
return msg
|
||||
|
||||
# Interface from BaseFolder
|
||||
def getmessagetime(self, uid):
|
||||
@ -375,7 +371,7 @@ class IMAPFolder(BaseFolder):
|
||||
def getmessagekeywords(self, uid):
|
||||
return self.messagelist[uid]['keywords']
|
||||
|
||||
def __generate_randomheader(self, content):
|
||||
def __generate_randomheader(self, msg, policy=None):
|
||||
"""Returns a unique X-OfflineIMAP header
|
||||
|
||||
Generate an 'X-OfflineIMAP' mail header which contains a random
|
||||
@ -390,17 +386,21 @@ class IMAPFolder(BaseFolder):
|
||||
"""
|
||||
|
||||
headername = 'X-OfflineIMAP'
|
||||
if policy is None:
|
||||
output_policy = self.policy['8bit-RFC']
|
||||
else:
|
||||
output_policy = policy
|
||||
# We need a random component too. If we ever upload the same
|
||||
# mail twice (e.g. in different folders), we would still need to
|
||||
# get the UID for the correct one. As we won't have too many
|
||||
# mails with identical content, the randomness requirements are
|
||||
# not extremly critial though.
|
||||
|
||||
# Compute unsigned crc32 of 'content' as unique hash.
|
||||
# Compute unsigned crc32 of 'msg' (as bytes) into a unique hash.
|
||||
# NB: crc32 returns unsigned only starting with python 3.0.
|
||||
headervalue = str(binascii.crc32(str.encode(content))
|
||||
& 0xffffffff) + '-'
|
||||
headervalue += str(self.randomgenerator.randint(0, 9999999999))
|
||||
headervalue = '{}-{}'.format(
|
||||
(binascii.crc32(msg.as_bytes(policy=output_policy)) & 0xffffffff),
|
||||
self.randomgenerator.randint(0, 9999999999))
|
||||
return headername, headervalue
|
||||
|
||||
def __savemessage_searchforheader(self, imapobj, headername, headervalue):
|
||||
@ -539,7 +539,7 @@ class IMAPFolder(BaseFolder):
|
||||
|
||||
return 0
|
||||
|
||||
def __getmessageinternaldate(self, content, rtime=None):
|
||||
def __getmessageinternaldate(self, msg, rtime=None):
|
||||
"""Parses mail and returns an INTERNALDATE string
|
||||
|
||||
It will use information in the following order, falling back as an
|
||||
@ -571,7 +571,7 @@ class IMAPFolder(BaseFolder):
|
||||
(which is fine as value for append)."""
|
||||
|
||||
if rtime is None:
|
||||
rtime = emailutil.get_message_date(content)
|
||||
rtime = self.get_message_date(msg)
|
||||
if rtime is None:
|
||||
return None
|
||||
datetuple = time.localtime(rtime)
|
||||
@ -619,7 +619,7 @@ class IMAPFolder(BaseFolder):
|
||||
return internaldate
|
||||
|
||||
# Interface from BaseFolder
|
||||
def savemessage(self, uid, content, flags, rtime):
|
||||
def savemessage(self, uid, msg, flags, rtime):
|
||||
"""Save the message on the Server
|
||||
|
||||
This backend always assigns a new uid, so the uid arg is ignored.
|
||||
@ -632,7 +632,7 @@ class IMAPFolder(BaseFolder):
|
||||
savemessage is never called in a dryrun mode.
|
||||
|
||||
:param uid: Message UID
|
||||
:param content: Message content
|
||||
:param msg: Message Object
|
||||
:param flags: Message flags
|
||||
:param rtime: A timestamp to be used as the mail date
|
||||
:returns: the UID of the new message as assigned by the server. If the
|
||||
@ -647,16 +647,17 @@ class IMAPFolder(BaseFolder):
|
||||
self.savemessageflags(uid, flags)
|
||||
return uid
|
||||
|
||||
content = self.deletemessageheaders(content, self.filterheaders)
|
||||
# Filter user requested headers before uploading to the IMAP server
|
||||
self.deletemessageheaders(msg, self.filterheaders)
|
||||
|
||||
# Use proper CRLF all over the message.
|
||||
content = re.sub("(?<!\r)\n", CRLF, content)
|
||||
# Should just be able to set the policy, to use CRLF in msg output
|
||||
output_policy = self.policy['8bit-RFC']
|
||||
|
||||
# Get the date of the message, so we can pass it to the server.
|
||||
date = self.__getmessageinternaldate(content, rtime)
|
||||
date = self.__getmessageinternaldate(msg, rtime)
|
||||
|
||||
# Message-ID is handy for debugging messages.
|
||||
msg_id = self.getmessageheader(content, "message-id")
|
||||
msg_id = self.getmessageheader(msg, "message-id")
|
||||
if not msg_id:
|
||||
msg_id = '[unknown message-id]'
|
||||
|
||||
@ -676,16 +677,18 @@ class IMAPFolder(BaseFolder):
|
||||
if not use_uidplus:
|
||||
# Insert a random unique header that we can fetch later.
|
||||
(headername, headervalue) = self.__generate_randomheader(
|
||||
content)
|
||||
msg)
|
||||
self.ui.debug('imap', 'savemessage: header is: %s: %s' %
|
||||
(headername, headervalue))
|
||||
content = self.addmessageheader(content, CRLF,
|
||||
headername, headervalue)
|
||||
self.addmessageheader(msg, headername, headervalue)
|
||||
|
||||
if len(content) > 200:
|
||||
dbg_output = "%s...%s" % (content[:150], content[-50:])
|
||||
if self.ui.is_debugging('imap'):
|
||||
# Optimization: don't create the debugging objects unless needed
|
||||
msg_s = msg.as_string(policy=output_policy)
|
||||
if len(msg_s) > 200:
|
||||
dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
|
||||
else:
|
||||
dbg_output = content
|
||||
dbg_output = msg_s
|
||||
self.ui.debug('imap', "savemessage: date: %s, content: '%s'" %
|
||||
(date, dbg_output))
|
||||
|
||||
@ -695,7 +698,7 @@ class IMAPFolder(BaseFolder):
|
||||
except imapobj.readonly:
|
||||
# readonly exception. Return original uid to notify that
|
||||
# we did not save the message. (see savemessage in Base.py)
|
||||
self.ui.msgtoreadonly(self, uid, content, flags)
|
||||
self.ui.msgtoreadonly(self, uid)
|
||||
return uid
|
||||
|
||||
# Do the APPEND.
|
||||
@ -703,7 +706,7 @@ class IMAPFolder(BaseFolder):
|
||||
(typ, dat) = imapobj.append(
|
||||
self.getfullIMAPname(),
|
||||
imaputil.flagsmaildir2imap(flags),
|
||||
date, bytes(content, 'utf-8'))
|
||||
date, msg.as_bytes(policy=output_policy))
|
||||
# This should only catch 'NO' responses since append()
|
||||
# will raise an exception for 'BAD' responses:
|
||||
if typ != 'OK':
|
||||
@ -716,12 +719,12 @@ class IMAPFolder(BaseFolder):
|
||||
# In this case, we should immediately abort
|
||||
# the repository sync and continue
|
||||
# with the next account.
|
||||
msg = \
|
||||
err_msg = \
|
||||
"Saving msg (%s) in folder '%s', " \
|
||||
"repository '%s' failed (abort). " \
|
||||
"Server responded: %s %s\n" % \
|
||||
(msg_id, self, self.getrepository(), typ, dat)
|
||||
raise OfflineImapError(msg, OfflineImapError.ERROR.REPO)
|
||||
raise OfflineImapError(err_msg, OfflineImapError.ERROR.REPO)
|
||||
retry_left = 0 # Mark as success.
|
||||
except imapobj.abort as e:
|
||||
# Connection has been reset, release connection and retry.
|
||||
@ -832,7 +835,7 @@ class IMAPFolder(BaseFolder):
|
||||
"""Fetches data from IMAP server.
|
||||
|
||||
Arguments:
|
||||
- uids: message UIDS
|
||||
- uids: message UIDS (OfflineIMAP3: First UID returned only)
|
||||
- retry_num: number of retries to make
|
||||
|
||||
Returns: data obtained by this query."""
|
||||
@ -888,9 +891,21 @@ class IMAPFolder(BaseFolder):
|
||||
"with UID '%s'" % (self.getrepository(), uids)
|
||||
raise OfflineImapError(reason, severity)
|
||||
|
||||
# Convert bytes to str
|
||||
# JI: In offlineimap, this function returned a tuple of strings for each
|
||||
# fetched UID, offlineimap3 calls to the imap object return bytes and so
|
||||
# originally a fixed, utf-8 conversion was done and *only* the first
|
||||
# response (d[0]) was returned. Note that this alters the behavior
|
||||
# between code bases. However, it seems like a single UID is the intent
|
||||
# of this function so retaining the modfication here for now.
|
||||
#
|
||||
# TODO: Can we assume the server response containing the meta data is
|
||||
# always 'utf-8' encoded? Assuming yes for now.
|
||||
#
|
||||
# Convert responses, d[0][0], into a 'utf-8' string (from bytes) and
|
||||
# Convert email, d[0][1], into a message object (from bytes)
|
||||
|
||||
ndata0 = data[0][0].decode('utf-8')
|
||||
ndata1 = data[0][1].decode('utf-8', errors='replace')
|
||||
ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1])
|
||||
ndata = [ndata0, ndata1]
|
||||
|
||||
return ndata
|
||||
|
||||
@ -190,7 +190,7 @@ class LocalStatusFolder(BaseFolder):
|
||||
os.close(fd)
|
||||
|
||||
# Interface from BaseFolder
|
||||
def savemessage(self, uid, content, flags, rtime, mtime=0, labels=None):
|
||||
def savemessage(self, uid, msg, flags, rtime, mtime=0, labels=None):
|
||||
"""Writes a new message, with the specified uid.
|
||||
|
||||
See folder/Base for detail. Note that savemessage() does not
|
||||
|
||||
@ -323,7 +323,7 @@ class LocalStatusSQLiteFolder(BaseFolder):
|
||||
# assert False,"getmessageflags() called on non-existing message"
|
||||
|
||||
# Interface from BaseFolder
|
||||
def savemessage(self, uid, content, flags, rtime, mtime=0, labels=None):
|
||||
def savemessage(self, uid, msg, flags, rtime, mtime=0, labels=None):
|
||||
"""Writes a new message, with the specified uid.
|
||||
|
||||
See folder/Base for detail. Note that savemessage() does not
|
||||
|
||||
@ -20,12 +20,10 @@ import socket
|
||||
import time
|
||||
import re
|
||||
import os
|
||||
from pathlib import Path
|
||||
from sys import exc_info
|
||||
from threading import Lock
|
||||
from hashlib import md5
|
||||
import chardet
|
||||
from offlineimap import OfflineImapError, emailutil
|
||||
from offlineimap import OfflineImapError
|
||||
from .Base import BaseFolder
|
||||
|
||||
# Find the UID in a message filename
|
||||
@ -256,23 +254,14 @@ class MaildirFolder(BaseFolder):
|
||||
|
||||
# Interface from BaseFolder
|
||||
def getmessage(self, uid):
|
||||
"""Return the content of the message."""
|
||||
"""Returns an email message object."""
|
||||
|
||||
# TODO: Perhaps, force the encoding using config file
|
||||
filename = self.messagelist[uid]['filename']
|
||||
filepath = os.path.join(self.getfullname(), filename)
|
||||
# Open the file as binary and read it
|
||||
file = Path(filepath)
|
||||
blob = file.read_bytes()
|
||||
# Detect the encoding
|
||||
detection = chardet.detect(blob)
|
||||
encoding = detection["encoding"]
|
||||
# Read the file as text
|
||||
retval = blob.decode(encoding)
|
||||
|
||||
# TODO: WHY are we replacing \r\n with \n here? And why do we
|
||||
# read it as text?
|
||||
return retval.replace("\r\n", "\n")
|
||||
fd = open(filepath, 'rb')
|
||||
retval = self.parser['8bit'].parse(fd)
|
||||
fd.close()
|
||||
return retval
|
||||
|
||||
# Interface from BaseFolder
|
||||
def getmessagetime(self, uid):
|
||||
@ -297,17 +286,21 @@ class MaildirFolder(BaseFolder):
|
||||
uid, self._foldermd5, self.infosep, ''.join(sorted(flags)))
|
||||
return uniq_name.replace(os.path.sep, self.sep_subst)
|
||||
|
||||
def save_to_tmp_file(self, filename, content):
|
||||
"""Saves given content to the named temporary file in the
|
||||
def save_to_tmp_file(self, filename, msg, policy=None):
|
||||
"""Saves given message to the named temporary file in the
|
||||
'tmp' subdirectory of $CWD.
|
||||
|
||||
Arguments:
|
||||
- filename: name of the temporary file;
|
||||
- content: data to be saved.
|
||||
- msg: Email message object
|
||||
|
||||
Returns: relative path to the temporary file
|
||||
that was created."""
|
||||
|
||||
if policy is None:
|
||||
output_policy = self.policy['8bit']
|
||||
else:
|
||||
output_policy = policy
|
||||
tmpname = os.path.join('tmp', filename)
|
||||
# Open file and write it out.
|
||||
# XXX: why do we need to loop 7 times?
|
||||
@ -333,8 +326,8 @@ class MaildirFolder(BaseFolder):
|
||||
else:
|
||||
raise
|
||||
|
||||
fd = os.fdopen(fd, 'wt')
|
||||
fd.write(content)
|
||||
fd = os.fdopen(fd, 'wb')
|
||||
fd.write(msg.as_bytes(policy=output_policy))
|
||||
# Make sure the data hits the disk.
|
||||
fd.flush()
|
||||
if self.dofsync():
|
||||
@ -344,7 +337,7 @@ class MaildirFolder(BaseFolder):
|
||||
return tmpname
|
||||
|
||||
# Interface from BaseFolder
|
||||
def savemessage(self, uid, content, flags, rtime):
|
||||
def savemessage(self, uid, msg, flags, rtime):
|
||||
"""Writes a new message, with the specified uid.
|
||||
|
||||
See folder/Base for detail. Note that savemessage() does not
|
||||
@ -368,15 +361,15 @@ class MaildirFolder(BaseFolder):
|
||||
message_timestamp = None
|
||||
if self._filename_use_mail_timestamp is not False:
|
||||
try:
|
||||
message_timestamp = emailutil.get_message_date(content, 'Date')
|
||||
message_timestamp = self.get_message_date(msg, 'Date')
|
||||
if message_timestamp is None:
|
||||
# Give a try with Delivery-date
|
||||
message_timestamp = emailutil.get_message_date(
|
||||
content, 'Delivery-date')
|
||||
message_timestamp = self.get_message_date(
|
||||
msg, 'Delivery-date')
|
||||
except Exception as e:
|
||||
# This should never happen.
|
||||
from offlineimap.ui import getglobalui
|
||||
datestr = emailutil.get_message_date(content)
|
||||
datestr = self.get_message_date(msg)
|
||||
ui = getglobalui()
|
||||
ui.warn("UID %d has invalid date %s: %s\n"
|
||||
"Not using message timestamp as file prefix" %
|
||||
@ -384,11 +377,11 @@ class MaildirFolder(BaseFolder):
|
||||
# No need to check if message_timestamp is None here since it
|
||||
# would be overridden by _gettimeseq.
|
||||
messagename = self.new_message_filename(uid, flags, date=message_timestamp)
|
||||
tmpname = self.save_to_tmp_file(messagename, content)
|
||||
tmpname = self.save_to_tmp_file(messagename, msg)
|
||||
|
||||
if self._utime_from_header is True:
|
||||
try:
|
||||
date = emailutil.get_message_date(content, 'Date')
|
||||
date = self.get_message_date(msg, 'Date')
|
||||
if date is not None:
|
||||
os.utime(os.path.join(self.getfullname(), tmpname),
|
||||
(date, date))
|
||||
@ -396,7 +389,7 @@ class MaildirFolder(BaseFolder):
|
||||
# int32.
|
||||
except Exception as e:
|
||||
from offlineimap.ui import getglobalui
|
||||
datestr = emailutil.get_message_date(content)
|
||||
datestr = self.get_message_date(msg)
|
||||
ui = getglobalui()
|
||||
ui.warn("UID %d has invalid date %s: %s\n"
|
||||
"Not changing file modification time" % (uid, datestr, e))
|
||||
|
||||
@ -236,11 +236,11 @@ class MappedIMAPFolder(IMAPFolder):
|
||||
|
||||
# Interface from BaseFolder
|
||||
def getmessage(self, uid):
|
||||
"""Returns the content of the specified message."""
|
||||
"""Returns the specified message."""
|
||||
return self._mb.getmessage(self.r2l[uid])
|
||||
|
||||
# Interface from BaseFolder
|
||||
def savemessage(self, uid, content, flags, rtime):
|
||||
def savemessage(self, uid, msg, flags, rtime):
|
||||
"""Writes a new message, with the specified uid.
|
||||
|
||||
The UIDMaps class will not return a newly assigned uid, as it
|
||||
@ -271,7 +271,7 @@ class MappedIMAPFolder(IMAPFolder):
|
||||
self.savemessageflags(uid, flags)
|
||||
return uid
|
||||
|
||||
newluid = self._mb.savemessage(-1, content, flags, rtime)
|
||||
newluid = self._mb.savemessage(-1, msg, flags, rtime)
|
||||
if newluid < 1:
|
||||
raise OfflineImapError("server of repository '%s' did not return "
|
||||
"a valid UID (got '%s') for UID '%s' from '%s'" % (
|
||||
|
||||
@ -231,6 +231,9 @@ class UIBase:
|
||||
else:
|
||||
self.invaliddebug(debugtype)
|
||||
|
||||
def is_debugging(self, debugtype):
|
||||
return (debugtype in self.debuglist)
|
||||
|
||||
def debugging(self, debugtype):
|
||||
global debugtypes
|
||||
self.logger.debug("Now debugging for %s: %s" % (debugtype,
|
||||
@ -266,7 +269,7 @@ class UIBase:
|
||||
(self.getnicename(x), x.getname()) for x in folder_list])
|
||||
|
||||
# WARNINGS
|
||||
def msgtoreadonly(self, destfolder, uid, content, flags):
|
||||
def msgtoreadonly(self, destfolder, uid):
|
||||
if self.config.has_option('general', 'ignore-readonly') and \
|
||||
self.config.getboolean('general', 'ignore-readonly'):
|
||||
return
|
||||
|
||||
@ -7,4 +7,4 @@ distro
|
||||
imaplib2~=3.5
|
||||
urllib3~=1.25.9
|
||||
certifi~=2020.6.20
|
||||
chardet~=3.0.4
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user