From 2382b15e3ed152edeb882ff784300223242b6f68 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 21 May 2015 11:16:25 +0200 Subject: [PATCH 001/100] CONTRIBUTING: fix links to offlineimap.org Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9bc6f7f..c2bd120 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -7,9 +7,9 @@ .. _maintainers: https://github.com/OfflineIMAP/offlineimap/blob/next/MAINTAINERS.rst .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst -.. _Community's website: https://offlineimap.org +.. _Community's website: http://offlineimap.org .. _APIs in OfflineIMAP: http://offlineimap.org/documentation.html#available-apis -.. _documentation: https://offlineimap.org/documentation.html +.. _documentation: http://offlineimap.org/documentation.html .. _Coding Guidelines: http://offlineimap.org/doc/CodingGuidelines.html .. _Know the status of your patches: http://offlineimap.org/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission From eeb7e89ca60b04f3c0a3ee4313e8d6db2d04f583 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 8 Jun 2015 13:09:28 +0200 Subject: [PATCH 002/100] README: small improvements Based-on-patch-by: Luke Skibinski Signed-off-by: Nicolas Sebrecht --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ae9c6b6..3b3323f 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,13 @@ OfflineIMAP is a software to dispose your e-mail mailbox(es) as a **local Maildir**. OfflineIMAP will synchronize both sides via *IMAP*. -The main downside about IMAP is that you have to **trust** your MAIL provider to -not loose your mails. This is not something impossible while not very common. -With OfflineIMAP, you can download your Mailboxes and make you own backups of -the Maildir. +The main downside to IMAP is that you must **trust** your MAIL provider to not +lose your mail. This is not common but certainly possible. -This allows reading your mails while offline without the need for the mail +With OfflineIMAP you can download your mail boxes and create your own backup +[Maildir](https://en.wikipedia.org/wiki/Maildir). + +This allows reading your mail while offline without the need for the mail reader (MUA) to support IMAP disconnected operations. Need an attachement from a message without internet? It's fine, the message is still there. From a6e7b6627b51cd33d2447016adbe01f22a12cb4a Mon Sep 17 00:00:00 2001 From: Tommie Gannert Date: Sat, 29 Aug 2015 13:53:20 +0100 Subject: [PATCH 003/100] Add decodefoldernames option to decode IMAP folder names using UTF-7. Signed-off-by: Tommie Gannert --- offlineimap.conf | 14 ++++++++++++++ offlineimap/folder/IMAP.py | 7 +++++++ offlineimap/imaputil.py | 28 ++++++++++++++++++++++++++++ offlineimap/repository/IMAP.py | 3 +++ 4 files changed, 52 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 69ede92..b0ef554 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -762,6 +762,20 @@ remoteuser = username #reference = Mail +# This option stands in the [Repository RemoteExample] section. +# +# IMAP defines an encoding for non-ASCII ("international") characters. Enable +# this option if you want to decode them to the nowadays ubiquitous UTF-8. +# +# Note that the IMAP 4rev1 specification (RFC 3501) allows both UTF-8 and +# modified UTF-7 folder names. +# +# This option is disabled by default to retain compatibility with older versions +# of offlineimap. +# +#decodefoldernames = no + + # This option stands in the [Repository RemoteExample] section. # # In between synchronisations, OfflineIMAP can monitor mailboxes for new diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 9ed7fec..659ed5f 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -258,6 +258,13 @@ class IMAPFolder(BaseFolder): def dropmessagelistcache(self): self.messagelist = {} + # Interface from BaseFolder + def getvisiblename(self): + vname = super(IMAPFolder, self).getvisiblename() + if self.repository.getdecodefoldernames(): + return imaputil.decode_mailbox_name(vname) + return vname + # Interface from BaseFolder def getmessagelist(self): return self.messagelist diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index f1f287b..6d8b444 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -25,6 +25,9 @@ from offlineimap.ui import getglobalui # Message headers that use space as the separator (for label storage) SPACE_SEPARATED_LABEL_HEADERS = ('X-Label', 'Keywords') +# Find the modified UTF-7 shifts of an international mailbox name. +MUTF7_SHIFT_RE = re.compile(r'&[^-]*-|\+') + def __debug(*args): msg = [] @@ -328,3 +331,28 @@ def labels_from_header(header_name, header_value): return labels + +def decode_mailbox_name(name): + """Decodes a modified UTF-7 mailbox name. + + If the string cannot be decoded, it is returned unmodified. + + See RFC 3501, sec. 5.1.3. + + Arguments: + - name: string, possibly encoded with modified UTF-7 + + Returns: decoded UTF-8 string. + """ + def demodify(m): + s = m.group() + if s == '+': + return '+-' + return '+' + s[1:-1].replace(',', '/') + '-' + + ret = MUTF7_SHIFT_RE.sub(demodify, name) + + try: + return ret.decode('utf-7').encode('utf-8') + except UnicodeEncodeError: + return name diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 68c8e33..b1ee473 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -261,6 +261,9 @@ class IMAPRepository(BaseRepository): def getreference(self): return self.getconf('reference', '') + def getdecodefoldernames(self): + return self.getconfboolean('decodefoldernames', 0) + def getidlefolders(self): localeval = self.localeval return localeval.eval(self.getconf('idlefolders', '[]')) From 14a0433d5c38b44780a5e52e852694b7e03a301f Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 4 Sep 2015 00:31:15 +0200 Subject: [PATCH 004/100] utf-7 feature is set experimental Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index b0ef554..27962d3 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -773,6 +773,8 @@ remoteuser = username # This option is disabled by default to retain compatibility with older versions # of offlineimap. # +# This option is EXPERIMENTAL. +# #decodefoldernames = no From 41692d03282bae408eaf1ec73c867c1086d0a678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Deparis?= Date: Fri, 4 Sep 2015 12:12:28 +0200 Subject: [PATCH 005/100] =?UTF-8?q?Try=20to=20fix=20#225=20=C2=AB=20Runonc?= =?UTF-8?q?e=20(offlineimap=20-o)=20does=20not=20stop=20if=20autorefresh?= =?UTF-8?q?=20is=20declared=20in=20DEFAULT=20section=20=C2=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Étienne Deparis --- offlineimap/init.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/init.py b/offlineimap/init.py index a48c152..c6b8a64 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -217,6 +217,9 @@ class OfflineImap: imaplib.Debug = 5 if options.runonce: + # Must kill the possible default option + if config.has_option('DEFAULT', 'autorefresh'): + config.remove_option('DEFAULT', 'autorefresh') # FIXME: spaghetti code alert! for section in accounts.getaccountlist(config): config.remove_option('Account ' + section, "autorefresh") From 48bb2f4113a6a304da995175fa4ebd2c46adfa20 Mon Sep 17 00:00:00 2001 From: Matthew Krafczyk Date: Tue, 27 Jan 2015 18:56:54 +0100 Subject: [PATCH 006/100] Added the newmail_hook When new mail arrives, this hook is triggered, allowing the user to play a sound, or launch a popup. Signed-off-by: Matthew Krafczyk --- offlineimap.conf | 13 +++++++++++++ offlineimap/folder/Base.py | 16 ++++++++++++++++ offlineimap/repository/Base.py | 1 + offlineimap/repository/IMAP.py | 5 +++++ 4 files changed, 35 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 69ede92..6b61e64 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -295,6 +295,19 @@ remoterepository = RemoteExample #postsynchook = notifysync.sh +# This option stands in the [Account Test] section. +# +# You can specify a newmail hook to execute an external command upon receipt +# of new mail in the INBOX. +# +# This example plays a sound file of your chosing when new mail arrives. +# +# This feature is experimental. +# +#newmail_hook = lambda: os.sytem("cvlc --play-and-exit /path/to/sound/file.mp3" + +# " > /dev/null 2>&1") + + # This option stands in the [Account Test] section. # # OfflineImap caches the state of the synchronisation to e.g. be able to diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 7957b83..d81f3ef 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -40,6 +40,11 @@ class BaseFolder(object): # Top level dir name is always '' self.root = None self.name = name if not name == self.getsep() else '' + self.newmail_hook = None + # Only set the newmail_hook if the IMAP folder is named 'INBOX' + if self.name == 'INBOX': + self.newmail_hook = repository.newmail_hook + self.have_newmail = False self.repository = repository self.visiblename = repository.nametrans(name) # In case the visiblename becomes '.' or '/' (top-level) we use @@ -781,6 +786,9 @@ class BaseFolder(object): # Got new UID, change the local uid. # Save uploaded status in the statusfolder statusfolder.savemessage(new_uid, message, flags, rtime) + # Check whether the mail has been seen + if 'S' not in flags: + self.have_newmail = True elif new_uid == 0: # Message was stored to dstfolder, but we can't find it's UID # This means we can't link current message to the one created @@ -817,6 +825,9 @@ class BaseFolder(object): This function checks and protects us from action in dryrun mode.""" + # We have no new mail yet + self.have_newmail = False + threads = [] copylist = filter(lambda uid: not statusfolder.uidexists(uid), @@ -854,6 +865,11 @@ class BaseFolder(object): for thread in threads: thread.join() + # Execute new mail hook if we have new mail + if self.have_newmail: + if self.newmail_hook != None: + self.newmail_hook(); + def __syncmessagesto_delete(self, dstfolder, statusfolder): """Pass 2: Remove locally deleted messages on dst. diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index 0cf44f8..adae16d 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -48,6 +48,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): self.folderfilter = lambda foldername: 1 self.folderincludes = [] self.foldersort = None + self.newmail_hook = None if self.config.has_option(self.getsection(), 'nametrans'): self.nametrans = self.localeval.eval( self.getconf('nametrans'), {'re': re}) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 68c8e33..6f8d829 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -36,6 +36,11 @@ class IMAPRepository(BaseRepository): self._host = None self.imapserver = imapserver.IMAPServer(self) self.folders = None + # Only set the newmail_hook in an IMAP repository. + if self.config.has_option(self.getsection(), 'newmail_hook'): + self.newmail_hook = self.localeval.eval( + self.getconf('newmail_hook')) + if self.getconf('sep', None): self.ui.info("The 'sep' setting is being ignored for IMAP " "repository '%s' (it's autodetected)"% self) From 71dd03e88c3feed6d1ff685a9246cfe2750330f5 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 14 Sep 2015 14:56:17 +0200 Subject: [PATCH 007/100] Bump imaplib2 from 2.43 to 2.48 Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 136 +++++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 58 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index eeb4792..aa08a13 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.43" +__version__ = "2.48" __release__ = "2" -__revision__ = "43" +__revision__ = "48" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -46,18 +46,23 @@ Fix for offlineimap "indexerror: string index out of range" bug provided by Eyge Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014. Conversion to Python3 provided by F. Malina February 2015. Fix for READ-ONLY error from multiple EXAMINE/SELECT calls by Pierre-Louis Bonicoli March 2015. -Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli March 2015.""" +Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli March 2015. +Fix for correct byte encoding for _CRAM_MD5_AUTH taken from python3.5 imaplib.py June 2015. +Fix for correct Python 3 exception handling by Tobias Brink August 2015. +Fix to allow interruptible IDLE command by Tim Peoples September 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" import binascii, errno, os, random, re, select, socket, sys, time, threading, zlib -try: - import queue # py3 +if bytes != str: + # Python 3, but NB assumes strings in all I/O + # for backwards compatibility with python 2 usage. + import queue string_types = str -except ImportError: - import Queue as queue # py2 +else: + import Queue as queue string_types = basestring @@ -179,7 +184,7 @@ class Request(object): def get_response(self, exc_fmt=None): self.callback = None if __debug__: self.parent._log(3, '%s:%s.ready.wait' % (self.name, self.tag)) - self.ready.wait() + self.ready.wait(sys.float_info.max) if self.aborted is not None: typ, val = self.aborted @@ -441,19 +446,22 @@ class IMAP4(object): af, socktype, proto, canonname, sa = res try: s = socket.socket(af, socktype, proto) - except socket.error as msg: + except socket.error as m: + msg = m continue try: for i in (0, 1): try: s.connect(sa) break - except socket.error as msg: + except socket.error as m: + msg = m if len(msg.args) < 2 or msg.args[0] != errno.EINTR: raise else: raise socket.error(msg) - except socket.error as msg: + except socket.error as m: + msg = m s.close() continue break @@ -534,9 +542,9 @@ class IMAP4(object): data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: - self.sock.sendall(bytes(data, 'utf8')) - else: - self.sock.sendall(data) + data = bytes(data, 'utf8') + + self.sock.sendall(data) def shutdown(self): @@ -881,7 +889,9 @@ class IMAP4(object): def _CRAM_MD5_AUTH(self, challenge): """Authobject to use with CRAM-MD5 authentication.""" import hmac - return self.user + " " + hmac.HMAC(self.password, challenge).hexdigest() + pwd = (self.password.encode('utf-8') if isinstance(self.password, str) + else self.password) + return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest() def logout(self, **kw): @@ -1229,7 +1239,7 @@ class IMAP4(object): self.commands_lock.release() - if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%s"]' % (typ, len(urd)-1, dat)) + if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%.80s"]' % (typ, len(urd)-1, dat)) def _check_bye(self): @@ -1297,7 +1307,7 @@ class IMAP4(object): self.commands_lock.release() if need_event: if __debug__: self._log(3, 'sync command %s waiting for empty commands Q' % name) - self.state_change_free.wait() + self.state_change_free.wait(sys.float_info.max) if __debug__: self._log(3, 'sync command %s proceeding' % name) if self.state not in Commands[name][CMD_VAL_STATES]: @@ -1316,7 +1326,7 @@ class IMAP4(object): while self._get_untagged_response(typ): continue - if self._get_untagged_response('READ-ONLY', leave=True) and not self.is_readonly: + if not self.is_readonly and self._get_untagged_response('READ-ONLY', leave=True): self.literal = None raise self.readonly('mailbox status changed to READ-ONLY') @@ -1402,7 +1412,7 @@ class IMAP4(object): def _command_completer(self, cb_arg_list): # Called for callback commands - (response, cb_arg, error) = cb_arg_list + response, cb_arg, error = cb_arg_list rqb, kw = cb_arg rqb.callback = kw['callback'] rqb.callback_arg = kw.get('cb_arg') @@ -1420,6 +1430,7 @@ class IMAP4(object): if __debug__: self._print_log() rqb.abort(self.error, '%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) return + if __debug__: self._log(4, '_command_completer(%s, %s, None) = %s' % (response, cb_arg, rqb.tag)) if 'untagged_response' in kw: response = self._untagged_response(typ, dat, kw['untagged_response']) rqb.deliver(response) @@ -1463,7 +1474,7 @@ class IMAP4(object): if not leave: del self.untagged_responses[i] self.commands_lock.release() - if __debug__: self._log(5, '_get_untagged_response(%s) => %s' % (name, dat)) + if __debug__: self._log(5, '_get_untagged_response(%s) => %.80s' % (name, dat)) return dat self.commands_lock.release() @@ -1605,11 +1616,17 @@ class IMAP4(object): self.commands_lock.acquire() rqb = self.tagged_commands.pop(name) if not self.tagged_commands: + need_event = True + else: + need_event = False + self.commands_lock.release() + + if __debug__: self._log(4, '_request_pop(%s, %s) [%d] = %s' % (name, data, len(self.tagged_commands), rqb.tag)) + rqb.deliver(data) + + if need_event: if __debug__: self._log(3, 'state_change_free.set') self.state_change_free.set() - self.commands_lock.release() - if __debug__: self._log(4, '_request_pop(%s, %s) = %s' % (name, data, rqb.tag)) - rqb.deliver(data) def _request_push(self, tag=None, name=None, **kw): @@ -1645,7 +1662,7 @@ class IMAP4(object): if not dat: break data += dat - if __debug__: self._log(4, '_untagged_response(%s, ?, %s) => %s' % (typ, name, data)) + if __debug__: self._log(4, '_untagged_response(%s, ?, %s) => %.80s' % (typ, name, data)) return typ, data @@ -1762,7 +1779,10 @@ class IMAP4(object): } return ' '.join([PollErrors[s] for s in PollErrors.keys() if (s & state)]) - line_part = '' + if bytes != str: + line_part = b'' + else: + line_part = '' poll = select.poll() @@ -1774,7 +1794,7 @@ class IMAP4(object): while not (terminate or self.Terminate): if self.state == LOGOUT: - timeout = 1 + timeout = 10 else: timeout = read_poll_timeout try: @@ -1802,11 +1822,11 @@ class IMAP4(object): if bytes != str: stop = data.find(b'\n', start) if stop < 0: - line_part += data[start:].decode() + line_part += data[start:] break stop += 1 line_part, start, line = \ - '', stop, line_part + data[start:stop].decode() + b'', stop, (line_part + data[start:stop]).decode(errors='ignore') else: stop = data.find('\n', start) if stop < 0: @@ -1846,7 +1866,10 @@ class IMAP4(object): if __debug__: self._log(1, 'starting using select') - line_part = '' + if bytes != str: + line_part = b'' + else: + line_part = '' rxzero = 0 terminate = False @@ -1878,11 +1901,11 @@ class IMAP4(object): if bytes != str: stop = data.find(b'\n', start) if stop < 0: - line_part += data[start:].decode() + line_part += data[start:] break stop += 1 line_part, start, line = \ - '', stop, line_part + data[start:stop].decode() + b'', stop, (line_part + data[start:stop]).decode(errors='ignore') else: stop = data.find('\n', start) if stop < 0: @@ -2100,27 +2123,18 @@ class IMAP4_SSL(IMAP4): data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: - if hasattr(self.sock, "sendall"): - self.sock.sendall(bytes(data, 'utf8')) - else: - dlen = len(data) - while dlen > 0: - sent = self.sock.write(bytes(data, 'utf8')) - if sent == dlen: - break # avoid copy - data = data[sent:] - dlen = dlen - sent + data = bytes(data, 'utf8') + + if hasattr(self.sock, "sendall"): + self.sock.sendall(data) else: - if hasattr(self.sock, "sendall"): - self.sock.sendall(data) - else: - dlen = len(data) - while dlen > 0: - sent = self.sock.write(data) - if sent == dlen: - break # avoid copy - data = data[sent:] - dlen = dlen - sent + dlen = len(data) + while dlen > 0: + sent = self.sock.write(data) + if sent == dlen: + break # avoid copy + data = data[sent:] + dlen = dlen - sent def ssl(self): @@ -2195,9 +2209,9 @@ class IMAP4_stream(IMAP4): data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: - self.writefile.write(bytes(data, 'utf8')) - else: - self.writefile.write(data) + data = bytes(data, 'utf8') + + self.writefile.write(data) self.writefile.flush() @@ -2372,8 +2386,14 @@ if __name__ == '__main__': # To test: invoke either as 'python imaplib2.py [IMAP4_server_hostname]', # or as 'python imaplib2.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' - # or as 'python imaplib2.py -l "keyfile[:certfile]" [IMAP4_SSL_server_hostname]' + # or as 'python imaplib2.py -l keyfile[:certfile]|: [IMAP4_SSL_server_hostname]' + # + # Option "-d " turns on debugging (use "-d 5" for everything) # Option "-i" tests that IDLE is interruptible + # Option "-p " allows alternate ports + + if not __debug__: + raise ValueError('Please run without -O') import getopt, getpass @@ -2446,10 +2466,10 @@ if __name__ == '__main__': ) - AsyncError = None + AsyncError, M = None, None def responder(cb_arg_list): - (response, cb_arg, error) = cb_arg_list + response, cb_arg, error = cb_arg_list global AsyncError cmd, args = cb_arg if error is not None: @@ -2569,7 +2589,7 @@ if __name__ == '__main__': print('All tests OK.') except: - if not idle_intr or not 'IDLE' in M.capabilities: + if not idle_intr or M is None or not 'IDLE' in M.capabilities: print('Tests failed.') if not debug: From 9143ea5b93fc5781b54c06c329fa90345a8ac9a8 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 22 Sep 2015 09:07:21 +0200 Subject: [PATCH 008/100] man page: fingerprint can be used with SSL Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 618f2ab..e547f6d 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -259,8 +259,8 @@ out the connection that is used by default. + Unfortunately, by default we will not verify the certificate of an IMAP TLS/SSL server we connect to, so connecting by SSL is no guarantee against -man-in-the-middle attacks. While verifying a server certificate fingerprint is -being planned, it is not implemented yet. There is currently only one safe way +man-in-the-middle attacks. While verifying a server certificate checking the +fingerprint is recommended. There is currently only one safe way to ensure that you connect to the correct server in an encrypted manner: you can specify a 'sslcacertfile' setting in your repository section of offlineimap.conf pointing to a file that contains (among others) a CA From d53e5fc8bde03271f031d6be2855c30cc4499e60 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 22 Sep 2015 09:14:03 +0200 Subject: [PATCH 009/100] offlineimap.conf: don't use quotes for sep option Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap.conf b/offlineimap.conf index 68bbb86..7ced4b8 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -465,7 +465,9 @@ localfolders = ~/Test # ignored for IMAP repositories, as it is queried automatically. # Otherwise, default value is ".". # -#sep = "." +# Don't use quotes. +# +#sep = . # This option stands in the [Repository LocalExample] section. From a3986b20349715f5c2c74b1e6a8af67216e54524 Mon Sep 17 00:00:00 2001 From: Psy-Q Date: Thu, 24 Sep 2015 17:45:30 +0200 Subject: [PATCH 010/100] Fix spelling inconsistency. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae9c6b6..f47b9d2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [website]: http://offlineimap.org [wiki]: http://github.com/OfflineIMAP/offlineimap/wiki -# OfflineImap +# OfflineIMAP ## Description From a76f01c9ff32c398f895371e18383b01a882caa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ram=C3=B3n=20Cahenzli?= Date: Thu, 24 Sep 2015 17:51:01 +0200 Subject: [PATCH 011/100] Fix language. --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ae9c6b6..0c29d6a 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ OfflineIMAP is a software to dispose your e-mail mailbox(es) as a **local Maildir**. OfflineIMAP will synchronize both sides via *IMAP*. -The main downside about IMAP is that you have to **trust** your MAIL provider to -not loose your mails. This is not something impossible while not very common. +The main downside about IMAP is that you have to **trust** your email provider to +not lose your mails. This is not something impossible while not very common. With OfflineIMAP, you can download your Mailboxes and make you own backups of the Maildir. -This allows reading your mails while offline without the need for the mail -reader (MUA) to support IMAP disconnected operations. Need an attachement from a -message without internet? It's fine, the message is still there. +This allows reading your email while offline without the need for the mail +reader (MUA) to support IMAP disconnected operations. Need an attachment from a +message without internet connection? It's fine, the message is still there. ## License @@ -34,14 +34,14 @@ GNU General Public License v2. ## Downloads -You should first check if your distribution already package OfflineIMAP for you. +You should first check if your distribution already packages OfflineIMAP for you. Downloads releases as [tarball or zipball](https://github.com/OfflineIMAP/offlineimap/tags). ## Feedbacks and contributions -**The user discussions, development, announces and all the exciting stuff take -place in the mailing list.** While not mandatory to send emails, you can +**The user discussions, development, announcements and all the exciting stuff take +place on the mailing list.** While not mandatory to send emails, you can [subscribe here](http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project). Bugs, issues and contributions can be requested to both the mailing list or the @@ -65,14 +65,14 @@ Bugs, issues and contributions can be requested to both the mailing list or the All the current and updated documentation is at the [community's website][website]. -### Dispose locally +### Read documentation locally -You might want to dispose the documentation locally. Get the sources of the website. -For the other documentations, run the approppriate make target: +You might want to read the documentation locally. Get the sources of the website. +For the other documentation, run the appropriate make target: ``` $ ./scripts/get-repository.sh website $ cd docs -$ make html # Require rst2html -$ make man # Require a2x -$ make api # Require sphinx +$ make html # Requires rst2html +$ make man # Requires a2x +$ make api # Requires sphinx ``` From 839d0201be7bf093a3e86d25fb505222d64c622f Mon Sep 17 00:00:00 2001 From: Matthew Krafczyk Date: Sat, 26 Sep 2015 10:48:51 +0200 Subject: [PATCH 012/100] offlineimap.conf: fix a typo in the new mail hook example Also add --play-and-stop since if vlc is being used with the loop option, at the same time that cvlc is executed, the sound will play endlessly. Signed-off-by: Matthew Krafczyk Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap.conf b/offlineimap.conf index 7ced4b8..1db5b52 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -304,7 +304,7 @@ remoterepository = RemoteExample # # This feature is experimental. # -#newmail_hook = lambda: os.sytem("cvlc --play-and-exit /path/to/sound/file.mp3" + +#newmail_hook = lambda: os.system("cvlc --play-and-stop --play-and-exit /path/to/sound/file.mp3" + # " > /dev/null 2>&1") From 585e5d5e2ef377f72110b9849adf59d1aee2c2fa Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Sat, 26 Sep 2015 20:43:09 -0400 Subject: [PATCH 013/100] logging: add a switch to log to syslog Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 4 ++++ offlineimap/init.py | 9 +++++++++ offlineimap/ui/UIBase.py | 12 ++++++++++++ 3 files changed, 25 insertions(+) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 618f2ab..7a6f1c7 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -105,6 +105,10 @@ included), implies the single-thread option -1. Send logs to . +-s:: + + Send logs to syslog. + -f :: diff --git a/offlineimap/init.py b/offlineimap/init.py index a48c152..631cd13 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -89,6 +89,11 @@ class OfflineImap: parser.add_option("-l", dest="logfile", metavar="FILE", help="log to FILE") + parser.add_option("-s", + action="store_true", dest="syslog", + default=False, + help="log to syslog") + parser.add_option("-f", dest="folders", metavar="folder1[,folder2[,...]]", help="only sync the specified folders") @@ -196,6 +201,10 @@ class OfflineImap: if options.logfile: self.ui.setlogfile(options.logfile) + #set up syslog + if options.syslog: + self.ui.setup_sysloghandler() + #welcome blurb self.ui.init_banner() diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 3e698ab..f5ba5ba 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -91,6 +91,18 @@ class UIBase(object): self.logger.info(offlineimap.banner) return ch + def setup_sysloghandler(self): + """Backend specific syslog handler.""" + + # create console handler with a higher log level + ch = logging.SysLogHandler(sys.stdout) + #ch.setLevel(logging.DEBUG) + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + def setlogfile(self, logfile): """Create file handler which logs to file.""" From fb12fc19759e800af445f953e0befae355d21b7c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 28 Sep 2015 03:09:43 +0200 Subject: [PATCH 014/100] v6.6.0-rc1 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 32 ++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 4 ++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index bda0b3d..ef8c8ab 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,38 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} +### OfflineIMAP v6.6.0-rc1 (2015-09-28) + +#### Notes + +Let's go with a new release. + +Basic UTF support was implemented while it is still exeprimental. Use this with +care. OfflineIMAP can now send the logs to syslog and notify on new mail. + + +#### Features + +- 585e5d5 logging: add a switch to log to syslog +- 48bb2f4 Added the newmail_hook +- 14a0433 utf-7 feature is set experimental + +#### Fixes + +- 839d020 offlineimap.conf: fix a typo in the new mail hook example +- a76f01c Fix language. +- a3986b2 Fix spelling inconsistency. +- d53e5fc offlineimap.conf: don't use quotes for sep option +- 9143ea5 man page: fingerprint can be used with SSL +- 41692d0 fix #225 « Runonce (offlineimap -o) does not stop if autorefresh is declared in DEFAULT section » +- 2382b15 CONTRIBUTING: fix links to offlineimap.org + +#### Changes + +- 71dd03e Bump imaplib2 from 2.43 to 2.48 +- eeb7e89 README: small improvements + + ### OfflineIMAP v6.5.7 (2015-05-15) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 8b93c96..e76c1f9 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,8 +1,8 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.7" -__revision__ = "" +__version__ = "6.6.0" +__revision__ = "-rc1" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" From d6c48d3faea0e6ec4276fbdb40cc3d1ee8e3941f Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 28 Sep 2015 03:30:22 +0200 Subject: [PATCH 015/100] contrib/release.sh: fix changelog edition Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index 72e9936..6978a23 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -231,12 +231,13 @@ function update_changelog () { # Check and edit Changelog. ask "Next step: you'll be asked to review the diff of $CHANGELOG" - action=$No - while test ! $action -eq $Yes + while true do git diff -- "$CHANGELOG" | less ask 'edit Changelog?' $CHANGELOG - action=$? + test ! $? -eq $Yes && break + # Asked to edit the Changelog; will loop again. + $EDITOR "$CHANGELOG" done } From 54ef0748f2faf811e6153ef07410582f239a9939 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 28 Sep 2015 03:38:45 +0200 Subject: [PATCH 016/100] contrib/release.sh: add CLI samples Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/contrib/release.sh b/contrib/release.sh index 6978a23..a9d75d7 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -16,7 +16,7 @@ # TODO: move configuration out and source it. # TODO: implement rollback. -__VERSION__='v0.2' +__VERSION__='v0.3' SPHINXBUILD=sphinx-build @@ -430,6 +430,16 @@ cat < master:master +- git push next:next +- git push $new_version +- cd website +- git checkout master +- git merge $branch_name +- git push master:master +- cd .. +- git send-email $TMP_ANNOUNCE Have fun! ,-) EOF From 2f1c856c0460a504216d717fc169ee43795a1bdc Mon Sep 17 00:00:00 2001 From: Prashant Sachdeva Date: Sat, 26 Sep 2015 22:28:06 -0700 Subject: [PATCH 017/100] fix status code to reflect success or failure to sync Return value set to 1 if sync failed and error is thrown. Otherwise set to 0 if successful. Signed-off-by: Prashant Sachdeva Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 5c90377..761ff61 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -48,7 +48,7 @@ class OfflineImap: if options.diagnostics: self.__serverdiagnostics(options) else: - self.__sync(options) + return self.__sync(options) def __parse_cmd_options(self): parser = OptionParser(version=offlineimap.__bigversion__, @@ -351,11 +351,13 @@ class OfflineImap: offlineimap.mbnames.write(True) self.ui.terminate() + return 0 except (SystemExit): raise except Exception as e: self.ui.error(e) self.ui.terminate() + return 1 def __sync_singlethreaded(self, accs): """Executed if we do not want a separate syncmaster thread From cc3cbaea05af8ae6eed788a5fc81c561ee038324 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 30 Sep 2015 13:29:59 +0200 Subject: [PATCH 018/100] bump imaplib2 from v2.48 to v2.49 Includes changes to new TLS option. Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 68 +++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index aa08a13..d20d614 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.48" +__version__ = "2.49" __release__ = "2" -__revision__ = "48" +__revision__ = "49" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -49,7 +49,8 @@ Fix for READ-ONLY error from multiple EXAMINE/SELECT calls by Pierre-Louis Bonic Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli March 2015. Fix for correct byte encoding for _CRAM_MD5_AUTH taken from python3.5 imaplib.py June 2015. Fix for correct Python 3 exception handling by Tobias Brink August 2015. -Fix to allow interruptible IDLE command by Tim Peoples September 2015.""" +Fix to allow interruptible IDLE command by Tim Peoples September 2015. +Add support for TLS levels by Ben Boeckel September 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -82,6 +83,10 @@ READ_SIZE = 32768 # Consume all available in socke DFLT_DEBUG_BUF_LVL = 3 # Level above which the logging output goes directly to stderr +TLS_SECURE = "tls_secure" # Recognised TLS levels +TLS_NO_SSL = "tls_no_ssl" +TLS_COMPAT = "tls_compat" + AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first # Commands @@ -479,21 +484,38 @@ class IMAP4(object): try: import ssl + + TLS_MAP = {} + if hasattr(ssl, "PROTOCOL_TLSv1_2"): # py3 + TLS_MAP[TLS_SECURE] = { + "tls1_2": ssl.PROTOCOL_TLSv1_2, + "tls1_1": ssl.PROTOCOL_TLSv1_1, + } + else: + TLS_MAP[TLS_SECURE] = {} + TLS_MAP[TLS_NO_SSL] = TLS_MAP[TLS_SECURE].copy() + TLS_MAP[TLS_NO_SSL].update({ + "tls1": ssl.PROTOCOL_TLSv1, + }) + TLS_MAP[TLS_COMPAT] = TLS_MAP[TLS_NO_SSL].copy() + TLS_MAP[TLS_COMPAT].update({ + "ssl3": ssl.PROTOCOL_SSLv3, + "ssl23": ssl.PROTOCOL_SSLv23, + None: ssl.PROTOCOL_SSLv23, + }) + if self.ca_certs is not None: cert_reqs = ssl.CERT_REQUIRED else: cert_reqs = ssl.CERT_NONE - if self.ssl_version == "tls1": - ssl_version = ssl.PROTOCOL_TLSv1 - elif self.ssl_version == "ssl2": - ssl_version = ssl.PROTOCOL_SSLv2 - elif self.ssl_version == "ssl3": - ssl_version = ssl.PROTOCOL_SSLv3 - elif self.ssl_version == "ssl23" or self.ssl_version is None: - ssl_version = ssl.PROTOCOL_SSLv23 - else: - raise socket.sslerror("Invalid SSL version requested: %s", self.ssl_version) + if self.tls_level not in TLS_MAP: + raise RuntimeError("unknown tls_level: %s" % self.tls_level) + + if self.ssl_version not in TLS_MAP[self.tls_level]: + raise socket.sslerror("Invalid SSL version '%s' requested for tls_version '%s'" % (self.ssl_version, self.tls_level)) + + ssl_version = TLS_MAP[self.tls_level][self.ssl_version] self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version) ssl_exc = ssl.SSLError @@ -1075,8 +1097,8 @@ class IMAP4(object): return self._simple_command(name, sort_criteria, charset, *search_criteria, **kw) - def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", **kw): - """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23") + def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", tls_level=TLS_COMPAT, **kw): + """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", tls_level="tls_compat") Start TLS negotiation as per RFC 2595.""" name = 'STARTTLS' @@ -1112,6 +1134,7 @@ class IMAP4(object): self.ca_certs = ca_certs self.cert_verify_cb = cert_verify_cb self.ssl_version = ssl_version + self.tls_level = tls_level try: self.ssl_wrap_socket() @@ -2058,7 +2081,7 @@ class IMAP4_SSL(IMAP4): """IMAP4 client class over SSL connection Instantiate with: - IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None) + IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, tls_level="tls_compat") host - host's name (default: localhost); port - port number (default: standard IMAP4 SSL port); @@ -2066,23 +2089,30 @@ class IMAP4_SSL(IMAP4): certfile - PEM formatted certificate chain file (default: None); ca_certs - PEM formatted certificate chain file used to validate server certificates (default: None); cert_verify_cb - function to verify authenticity of server certificates (default: None); - ssl_version - SSL version to use (default: "ssl23", choose from: "tls1","ssl2","ssl3","ssl23"); + ssl_version - SSL version to use (default: "ssl23", choose from: "tls1","ssl3","ssl23"); debug - debug level (default: 0 - no debug); debug_file - debug stream (default: sys.stderr); identifier - thread identifier prefix (default: host); timeout - timeout in seconds when expecting a command response. debug_buf_lvl - debug level at which buffering is turned off. + tls_level - TLS security level (default: "tls_compat"). + + The recognized values for tls_level are: + tls_secure: accept only TLS protocols recognized as "secure" + tls_no_ssl: disable SSLv2 and SSLv3 support + tls_compat: accept all SSL/TLS versions For more documentation see the docstring of the parent class IMAP4. """ - def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): + def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None, tls_level=TLS_COMPAT): self.keyfile = keyfile self.certfile = certfile self.ca_certs = ca_certs self.cert_verify_cb = cert_verify_cb self.ssl_version = ssl_version + self.tls_level = tls_level IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout, debug_buf_lvl) @@ -2511,7 +2541,7 @@ if __name__ == '__main__': if keyfile is not None: if not keyfile: keyfile = None if not certfile: certfile = None - M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) + M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, ssl_version="tls1", debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl, tls_level="tls_no_ssl") elif stream_command: M = IMAP4_stream(stream_command, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) else: From f7efaa2093d73281c1d43302439dc19ab5364fa0 Mon Sep 17 00:00:00 2001 From: Christopher Corley Date: Thu, 9 Jan 2014 14:30:41 -0600 Subject: [PATCH 019/100] learn XOAUTH2 authentication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: François Lamboley Signed-off-by: Nicolas Sebrecht --- README.md | 1 + offlineimap.conf | 35 ++++++++++++++++++++++++++- offlineimap/imapserver.py | 43 ++++++++++++++++++++++++++++++++- offlineimap/repository/Gmail.py | 14 +++++++++++ offlineimap/repository/IMAP.py | 29 ++++++++++++++++++++-- 5 files changed, 118 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ae9c6b6..79f11c1 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Bugs, issues and contributions can be requested to both the mailing list or the * Python v2.7 * Python SQlite (optional while recommended) +* Python json and urllib (used for XOAuth2 authentication) ## Documentation diff --git a/offlineimap.conf b/offlineimap.conf index 69ede92..01c54be 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -673,9 +673,42 @@ remoteuser = username # limitations, if GSSAPI is set, it will be tried first, no matter where it was # specified in the list. # -#auth_mechanisms = GSSAPI, CRAM-MD5, PLAIN, LOGIN +#auth_mechanisms = GSSAPI, CRAM-MD5, XOAUTH2, PLAIN, LOGIN +# This option stands in the [Repository RemoteExample] section. +# +# XOAuth2 authentication (for instance, to use with Gmail). +# +# This feature is currently EXPERIMENTAL (tested on Gmail only, but should work +# with type = IMAP for compatible servers). +# +# Mandatory parameters are "oauth2_client_id", "oauth2_client_secret" and +# "oauth2_refresh_token". See below to learn how to get those. +# +# Specify the OAuth2 client id and secret to use for the connection.. +# Here's how to register an OAuth2 client for Gmail, as of 10-2-2016: +# - Go to the Google developer console +# https://console.developers.google.com/project +# - Create a new project +# - In API & Auth, select Credentials +# - Setup the OAuth Consent Screen +# - Then add Credentials of type OAuth 2.0 Client ID +# - Choose application type Other; type in a name for your client +# - You now have a client ID and client secret +# +#oauth2_client_id = YOUR_CLIENT_ID +#oauth2_client_secret = YOUR_CLIENT_SECRET + +# Specify the refresh token to use for the connection to the mail server. +# Here's an example of a way to get a refresh token: +# - Clone this project: https://github.com/google/gmail-oauth2-tools +# - Type the following command-line in a terminal and follow the instructions +# python python/oauth2.py --generate_oauth2_token \ +# --client_id=YOUR_CLIENT_ID --client_secret=YOUR_CLIENT_SECRET +# +#oauth2_refresh_token = REFRESH_TOKEN + ########## Passwords # There are six ways to specify the password for the IMAP server: diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index f0b2248..72da4cc 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -19,6 +19,10 @@ from threading import Lock, BoundedSemaphore, Thread, Event, currentThread import hmac import socket import base64 + +import json +import urllib + import time import errno from sys import exc_info @@ -89,6 +93,12 @@ class IMAPServer: self.fingerprint = repos.get_ssl_fingerprint() self.sslversion = repos.getsslversion() + self.oauth2_refresh_token = repos.getoauth2_refresh_token() + self.oauth2_client_id = repos.getoauth2_client_id() + self.oauth2_client_secret = repos.getoauth2_client_secret() + self.oauth2_request_url = repos.getoauth2_request_url() + self.oauth2_access_token = None + self.delim = None self.root = None self.maxconnections = repos.getmaxconnections() @@ -199,7 +209,33 @@ class IMAPServer: return retval - # XXX: describe function + def __xoauth2handler(self, response): + if self.oauth2_refresh_token is None: + return None + + if self.oauth2_access_token is None: + # need to move these to config + # generate new access token + params = {} + params['client_id'] = self.oauth2_client_id + params['client_secret'] = self.oauth2_client_secret + params['refresh_token'] = self.oauth2_refresh_token + params['grant_type'] = 'refresh_token' + + self.ui.debug('imap', 'xoauth2handler: url "%s"' % self.oauth2_request_url) + self.ui.debug('imap', 'xoauth2handler: params "%s"' % params) + + response = urllib.urlopen(self.oauth2_request_url, urllib.urlencode(params)).read() + resp = json.loads(response) + self.ui.debug('imap', 'xoauth2handler: response "%s"' % resp) + self.oauth2_access_token = resp['access_token'] + + self.ui.debug('imap', 'xoauth2handler: access_token "%s"' % self.oauth2_access_token) + auth_string = 'user=%s\1auth=Bearer %s\1\1' % (self.username, self.oauth2_access_token) + #auth_string = base64.b64encode(auth_string) + self.ui.debug('imap', 'xoauth2handler: returning "%s"' % auth_string) + return auth_string + def __gssauth(self, response): data = base64.b64encode(response) try: @@ -283,6 +319,10 @@ class IMAPServer: imapobj.authenticate('PLAIN', self.__plainhandler) return True + def __authn_xoauth2(self, imapobj): + imapobj.authenticate('XOAUTH2', self.__xoauth2handler) + return True + def __authn_login(self, imapobj): # Use LOGIN command, unless LOGINDISABLED is advertized # (per RFC 2595) @@ -314,6 +354,7 @@ class IMAPServer: auth_methods = { "GSSAPI": (self.__authn_gssapi, False, True), "CRAM-MD5": (self.__authn_cram_md5, True, True), + "XOAUTH2": (self.__authn_xoauth2, True, True), "PLAIN": (self.__authn_plain, True, True), "LOGIN": (self.__authn_login, True, False), } diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 2e23e62..1e2069b 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -29,6 +29,8 @@ class GmailRepository(IMAPRepository): # Gmail IMAP server port PORT = 993 + OAUTH2_URL = 'https://accounts.google.com/o/oauth2/token' + def __init__(self, reposname, account): """Initialize a GmailRepository object.""" # Enforce SSL usage @@ -49,6 +51,18 @@ class GmailRepository(IMAPRepository): self._host = GmailRepository.HOSTNAME return self._host + def getoauth2_request_url(self): + """Return the server name to connect to. + + Gmail implementation first checks for the usual IMAP settings + and falls back to imap.gmail.com if not specified.""" + try: + return super(GmailRepository, self).getoauth2_request_url() + except OfflineImapError: + # nothing was configured, cache and return hardcoded one + self._oauth2_request_url = GmailRepository.OAUTH2_URL + return self._oauth2_request_url + def getport(self): return GmailRepository.PORT diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 68c8e33..e3e12c8 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -34,6 +34,7 @@ class IMAPRepository(BaseRepository): BaseRepository.__init__(self, reposname, account) # self.ui is being set by the BaseRepository self._host = None + self._oauth2_request_url = None self.imapserver = imapserver.IMAPServer(self) self.folders = None if self.getconf('sep', None): @@ -125,12 +126,12 @@ class IMAPRepository(BaseRepository): return self.getconf('remote_identity', default=None) def get_auth_mechanisms(self): - supported = ["GSSAPI", "CRAM-MD5", "PLAIN", "LOGIN"] + supported = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"] # Mechanisms are ranged from the strongest to the # weakest ones. # TODO: we need DIGEST-MD5, it must come before CRAM-MD5 # TODO: due to the chosen-plaintext resistance. - default = ["GSSAPI", "CRAM-MD5", "PLAIN", "LOGIN"] + default = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"] mechs = self.getconflist('auth_mechanisms', r',\s*', default) @@ -252,6 +253,30 @@ class IMAPRepository(BaseRepository): value = self.getconf('cert_fingerprint', "") return [f.strip().lower() for f in value.split(',') if f] + def getoauth2_request_url(self): + if self._oauth2_request_url: # use cached value if possible + return self._oauth2_request_url + + oauth2_request_url = self.getconf('oauth2_request_url', None) + if oauth2_request_url != None: + self._oauth2_request_url = oauth2_request_url + return self._oauth2_request_url + + # no success + raise OfflineImapError("No remote oauth2_request_url for repository "\ + "'%s' specified." % self, + OfflineImapError.ERROR.REPO) + return self.getconf('oauth2_request_url', None) + + def getoauth2_refresh_token(self): + return self.getconf('oauth2_refresh_token', None) + + def getoauth2_client_id(self): + return self.getconf('oauth2_client_id', None) + + def getoauth2_client_secret(self): + return self.getconf('oauth2_client_secret', None) + def getpreauthtunnel(self): return self.getconf('preauthtunnel', None) From 626441544dbb3407717c7d7cf9ef570c9d8f811e Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 6 Oct 2015 15:01:08 +0200 Subject: [PATCH 020/100] manual: add IDLE known issue about suspend/resume Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index f4a2dd7..cfebe36 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -344,6 +344,8 @@ Email will show up, but may not be processed until the next refresh cycle. - IMAP IDLE <-> IMAP IDLE doesn't work yet. + - IDLE might stop syncing on a system suspend/resume. + - IDLE may only work "once" per refresh. + If you encounter this bug, please send a report to the list! From af640208e18e9f7997ba37587302605af0c95022 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 12 Oct 2015 07:37:39 +0200 Subject: [PATCH 021/100] make XOAUTH2 configuration option optional Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Gmail.py | 10 +++++----- offlineimap/repository/IMAP.py | 9 +++------ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 1e2069b..47ad744 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -56,12 +56,12 @@ class GmailRepository(IMAPRepository): Gmail implementation first checks for the usual IMAP settings and falls back to imap.gmail.com if not specified.""" - try: - return super(GmailRepository, self).getoauth2_request_url() - except OfflineImapError: - # nothing was configured, cache and return hardcoded one + + url = super(GmailRepository, self).getoauth2_request_url() + if url is None: + # Nothing was configured, cache and return hardcoded one. self._oauth2_request_url = GmailRepository.OAUTH2_URL - return self._oauth2_request_url + return self._oauth2_request_url def getport(self): return GmailRepository.PORT diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 414b918..fd2464f 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -259,7 +259,7 @@ class IMAPRepository(BaseRepository): return [f.strip().lower() for f in value.split(',') if f] def getoauth2_request_url(self): - if self._oauth2_request_url: # use cached value if possible + if self._oauth2_request_url: # Use cached value if possible. return self._oauth2_request_url oauth2_request_url = self.getconf('oauth2_request_url', None) @@ -267,11 +267,8 @@ class IMAPRepository(BaseRepository): self._oauth2_request_url = oauth2_request_url return self._oauth2_request_url - # no success - raise OfflineImapError("No remote oauth2_request_url for repository "\ - "'%s' specified." % self, - OfflineImapError.ERROR.REPO) - return self.getconf('oauth2_request_url', None) + #raise OfflineImapError("No remote oauth2_request_url for repository " + #"'%s' specified."% self, OfflineImapError.ERROR.REPO) def getoauth2_refresh_token(self): return self.getconf('oauth2_refresh_token', None) From 9bb27de3753433fa340a99e22c8d574b60b44024 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 12 Oct 2015 07:51:05 +0200 Subject: [PATCH 022/100] do use the XOAUTH2 url if configured fix regression introduced by af640208e18e9f7. Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Gmail.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 47ad744..a45d274 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -61,6 +61,8 @@ class GmailRepository(IMAPRepository): if url is None: # Nothing was configured, cache and return hardcoded one. self._oauth2_request_url = GmailRepository.OAUTH2_URL + else: + self._oauth2_request_url = url return self._oauth2_request_url def getport(self): From f03afcd224c38abf6bc6dd9eb0437a4ae8dec6ba Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 24 Aug 2015 23:32:00 -0400 Subject: [PATCH 023/100] imapserver: add a tls_level option Allow the user to block usage of known-bad versions of SSL and TLS. Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 11 +++++++++++ offlineimap/imapserver.py | 2 ++ offlineimap/repository/IMAP.py | 3 +++ 3 files changed, 16 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 60285a8..90fe771 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -646,6 +646,17 @@ remotehost = examplehost #ssl_version = ssl23 +# This option stands in the [Repository RemoteExample] section. +# +# TLS support level (optional). +# +# Specify the level of support that should be allowed for this repository. +# Can be used to disallow insecure SSL versions. Supported values are: +# tls_secure, tls_no_ssl, tls_compat (the default). +# +#tls_level = tls_compat + + # This option stands in the [Repository RemoteExample] section. # # Specify the port. If not specified, use a default port. diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 72da4cc..6cdbf26 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -92,6 +92,7 @@ class IMAPServer: self.__verifycert = None # disable cert verification self.fingerprint = repos.get_ssl_fingerprint() self.sslversion = repos.getsslversion() + self.tlslevel = repos.gettlslevel() self.oauth2_refresh_token = repos.getoauth2_refresh_token() self.oauth2_client_id = repos.getoauth2_client_id() @@ -478,6 +479,7 @@ class IMAPServer: timeout=socket.getdefaulttimeout(), fingerprint=self.fingerprint, use_socket=self.proxied_socket, + tls_level=self.tlslevel, ) else: self.ui.connecting(self.hostname, self.port) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index fd2464f..167295a 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -246,6 +246,9 @@ class IMAPRepository(BaseRepository): raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) return cacertfile + def gettlslevel(self): + return self.getconf('tls_level', 'tls_compat') + def getsslversion(self): return self.getconf('ssl_version', None) From e2f2ec16a5166f6a3e927df068642f24238c8a3b Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 12 Oct 2015 06:42:29 +0200 Subject: [PATCH 024/100] offlineimap.conf: improve documentation on SSL, mark tls_level experimental Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 90fe771..e1731a2 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -637,11 +637,14 @@ remotehost = examplehost # This option stands in the [Repository RemoteExample] section. # -# SSL version (optional). +# Set SSL version tu use (optional). # # It is best to leave this unset, in which case the correct version will be # automatically detected. In rare cases, it may be necessary to specify a -# particular version from: tls1, ssl2, ssl3, ssl23 (SSLv2 or SSLv3) +# particular version from: tls1, ssl2, ssl3, ssl23 (SSLv2 or SSLv3). +# +# See the configuration option tls_level to automatically disable insecure +# protocols. # #ssl_version = ssl23 @@ -651,9 +654,14 @@ remotehost = examplehost # TLS support level (optional). # # Specify the level of support that should be allowed for this repository. -# Can be used to disallow insecure SSL versions. Supported values are: +# Can be used to disallow insecure SSL versions as defined by IETF +# (see https://tools.ietf.org/html/rfc6176). +# +# Supported values are: # tls_secure, tls_no_ssl, tls_compat (the default). # +# This option is EXPERIMENTAL. +# #tls_level = tls_compat From ee0de28cc457b6983f1e75cf28748f5a16320ed9 Mon Sep 17 00:00:00 2001 From: Janna Martl Date: Sat, 29 Aug 2015 16:02:33 -0400 Subject: [PATCH 025/100] utime_from_header: handle out-of-bounds dates Handle case where email's internal time is erroneously so large as to cause overflow errors when setting file modification time with utime_from_header = true. Reported-by: Cameron Simpson Signed-off-by: Janna Martl Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 3f5c071..07d1b73 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -350,9 +350,19 @@ class MaildirFolder(BaseFolder): tmpname = self.save_to_tmp_file(messagename, content) if self.utime_from_header: - date = emailutil.get_message_date(content, 'Date') - if date != None: - os.utime(os.path.join(self.getfullname(), tmpname), (date, date)) + try: + date = emailutil.get_message_date(content, 'Date') + if date is not None: + os.utime(os.path.join(self.getfullname(), tmpname), + (date, date)) + # In case date is wrongly so far into the future as to be > max int32 + except Exception as e: + from email.Parser import Parser + from offlineimap.ui import getglobalui + datestr = Parser().parsestr(content, True).get("Date") + ui = getglobalui() + ui.warn("UID %d has invalid date %s: %s\n" + "Not changing file modification time" % (uid, datestr, e)) self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags From 7a2306be16773323051a9309ecd0eaae397f3533 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 12 Oct 2015 13:10:14 -0400 Subject: [PATCH 026/100] systemd: log to syslog rather than stderr This allows the journal to capture output with the appropriate level. Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- contrib/systemd/offlineimap.service | 2 +- contrib/systemd/offlineimap@.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/systemd/offlineimap.service b/contrib/systemd/offlineimap.service index f29f93c..a587761 100644 --- a/contrib/systemd/offlineimap.service +++ b/contrib/systemd/offlineimap.service @@ -3,7 +3,7 @@ Description=Offlineimap Service [Service] Type=oneshot -ExecStart=/usr/bin/offlineimap -o +ExecStart=/usr/bin/offlineimap -o -s -u quiet [Install] WantedBy=mail.target diff --git a/contrib/systemd/offlineimap@.service b/contrib/systemd/offlineimap@.service index 7be965a..5439817 100644 --- a/contrib/systemd/offlineimap@.service +++ b/contrib/systemd/offlineimap@.service @@ -3,7 +3,7 @@ Description=Offlineimap Service for account %i [Service] Type=oneshot -ExecStart=/usr/bin/offlineimap -o -a %i +ExecStart=/usr/bin/offlineimap -o -a %i -s -u quiet [Install] WantedBy=mail.target From acd368479cc262fdd6adcd3a200c917f904f5a7a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 12 Oct 2015 23:40:12 +0200 Subject: [PATCH 027/100] offlineimap.conf: fix erroneous assumption about ssl23 Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index e1731a2..07f9901 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -637,11 +637,14 @@ remotehost = examplehost # This option stands in the [Repository RemoteExample] section. # -# Set SSL version tu use (optional). +# Set SSL version to use (optional). # # It is best to leave this unset, in which case the correct version will be # automatically detected. In rare cases, it may be necessary to specify a -# particular version from: tls1, ssl2, ssl3, ssl23 (SSLv2 or SSLv3). +# particular version from: tls1, ssl2, ssl3, ssl23. +# +# ssl23 is the highest protocol version that both the client and server support. +# Despite the name, this option can select “TLS” protocols as well as “SSL”. # # See the configuration option tls_level to automatically disable insecure # protocols. From 456d1724ca76853d60fa15dbe777f51373e7e057 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 13 Oct 2015 00:30:03 +0200 Subject: [PATCH 028/100] make: improve making targz Signed-off-by: Nicolas Sebrecht --- Makefile | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index da5c660..5de3e00 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -VERSION=`./offlineimap.py --version` -TARGZ=offlineimap_$(VERSION).tar.gz +VERSION=$(shell ./offlineimap.py --version) +ABBREV=$(shell git log --format='%h' HEAD~1..) +TARGZ=offlineimap-$(VERSION)-$(ABBREV) SHELL=/bin/bash RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` @@ -30,12 +31,12 @@ build: clean: -python setup.py clean --all - -rm -f bin/offlineimapc + -rm -f bin/offlineimapc 2>/dev/null -find . -name '*.pyc' -exec rm -f {} \; -find . -name '*.pygc' -exec rm -f {} \; -find . -name '*.class' -exec rm -f {} \; -find . -name '.cache*' -exec rm -f {} \; - -rm -f manpage.links manpage.refs + -rm -f manpage.links manpage.refs 2>/dev/null -find . -name auth -exec rm -vf {}/password {}/username \; @$(MAKE) -C clean @@ -47,11 +48,7 @@ websitedoc: targz: ../$(TARGZ) ../$(TARGZ): - if ! pwd | grep -q "/offlineimap-$(VERSION)$$"; then \ - echo "Containing directory must be called offlineimap-$(VERSION)"; \ - exit 1; \ - fi; \ - pwd && cd .. && pwd && tar -zhcv --exclude '.git' --exclude 'website' --exclude 'wiki' -f $(TARGZ) offlineimap-$(VERSION) + cd .. && tar -zhcv --transform s,^offlineimap,$(TARGZ), -f $(TARGZ).tar.gz --exclude '*.pyc' offlineimap/{bin,Changelog.md,contrib,CONTRIBUTING.rst,COPYING,docs,MAINTAINERS.rst,MANIFEST.in,offlineimap,offlineimap.conf,offlineimap.conf.minimal,offlineimap.py,README.md,scripts,setup.py,test,TODO.rst} rpm: targz cd .. && sudo rpmbuild -ta $(TARGZ) From 428bb7798497d946431f3f7787c9cd82a2db0d3a Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 12 Oct 2015 14:11:19 -0400 Subject: [PATCH 029/100] Noninteractive: fix docstring for Basic Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Noninteractive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index de1e8df..4e87d50 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -19,7 +19,7 @@ import logging from offlineimap.ui.UIBase import UIBase class Basic(UIBase): - """'Quiet' simply sets log level to INFO""" + """'Basic' simply sets log level to INFO""" def __init__(self, config, loglevel = logging.INFO): return super(Basic, self).__init__(config, loglevel) From 3daddb9b33bba9178c7b6d1b5721f3d89db01527 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 27 Mar 2015 12:38:16 +0100 Subject: [PATCH 030/100] folder/IMAP: minor indent fix Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 659ed5f..60b5301 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -273,7 +273,7 @@ class IMAPFolder(BaseFolder): def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body). - After this function all CRLFs will be transformed to '\n'. + After this function all CRLFs will be transformed to '\n'. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with From e18428b25badc9f7a58b23e7bdf90923d5350b91 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 12 Oct 2015 13:35:57 -0400 Subject: [PATCH 031/100] UIBase: add a syslog ui Rather than having an option for syslog output, make a separate UI option. Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- contrib/systemd/offlineimap.service | 2 +- contrib/systemd/offlineimap@.service | 2 +- docs/offlineimap.txt | 2 +- docs/offlineimapui.txt | 11 +++++++++++ offlineimap/init.py | 2 +- offlineimap/ui/Noninteractive.py | 20 ++++++++++++++++++++ offlineimap/ui/UIBase.py | 6 +++--- offlineimap/ui/__init__.py | 1 + 8 files changed, 39 insertions(+), 7 deletions(-) diff --git a/contrib/systemd/offlineimap.service b/contrib/systemd/offlineimap.service index a587761..8b77bc4 100644 --- a/contrib/systemd/offlineimap.service +++ b/contrib/systemd/offlineimap.service @@ -3,7 +3,7 @@ Description=Offlineimap Service [Service] Type=oneshot -ExecStart=/usr/bin/offlineimap -o -s -u quiet +ExecStart=/usr/bin/offlineimap -o -u syslog [Install] WantedBy=mail.target diff --git a/contrib/systemd/offlineimap@.service b/contrib/systemd/offlineimap@.service index 5439817..6730652 100644 --- a/contrib/systemd/offlineimap@.service +++ b/contrib/systemd/offlineimap@.service @@ -3,7 +3,7 @@ Description=Offlineimap Service for account %i [Service] Type=oneshot -ExecStart=/usr/bin/offlineimap -o -a %i -s -u quiet +ExecStart=/usr/bin/offlineimap -o -a %i -u syslog [Install] WantedBy=mail.target diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index cfebe36..a2da7f5 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -149,7 +149,7 @@ option is ignored if maxage is set. + This overrides the default specified in the configuration file. The UI specified with -u will be forced to be used, even if checks determine that it -is not usable. Possible interface choices are: quiet, basic, ttyui, +is not usable. Possible interface choices are: quiet, basic, syslog, ttyui, blinkenlights, machineui. diff --git a/docs/offlineimapui.txt b/docs/offlineimapui.txt index c087ece..2ea661b 100644 --- a/docs/offlineimapui.txt +++ b/docs/offlineimapui.txt @@ -127,6 +127,17 @@ It will output nothing except errors and serious warnings. Like Basic, this user interface is not capable of reading a password from the keyboard; account passwords must be specified using one of the configuration file options. + +Syslog +------ + +Syslog is designed for situations where OfflineIMAP is run as a daemon (e.g., +as a systemd --user service), but errors should be forwarded to the system log. +Like Basic, this user interface is not capable of reading a password from the +keyboard; account passwords must be specified using one of the configuration +file options. + + MachineUI --------- diff --git a/offlineimap/init.py b/offlineimap/init.py index 761ff61..d94dcb7 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -115,7 +115,7 @@ class OfflineImap: parser.add_option("-u", dest="interface", help="specifies an alternative user interface" - " (quiet, basic, ttyui, blinkenlights, machineui)") + " (quiet, basic, syslog, ttyui, blinkenlights, machineui)") (options, args) = parser.parse_args() globals.set_options (options) diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index 4e87d50..11b7247 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -17,6 +17,7 @@ import logging from offlineimap.ui.UIBase import UIBase +import offlineimap class Basic(UIBase): """'Basic' simply sets log level to INFO""" @@ -27,3 +28,22 @@ class Quiet(UIBase): """'Quiet' simply sets log level to WARNING""" def __init__(self, config, loglevel = logging.WARNING): return super(Quiet, self).__init__(config, loglevel) + +class Syslog(UIBase): + """'Syslog' sets log level to INFO and outputs to syslog instead of stdout""" + def __init__(self, config, loglevel = logging.INFO): + return super(Syslog, self).__init__(config, loglevel) + + def setup_consolehandler(self): + # create syslog handler + ch = logging.handlers.SysLogHandler('/dev/log') + # create formatter and add it to the handlers + self.formatter = logging.Formatter("%(message)s") + ch.setFormatter(self.formatter) + # add the handlers to the logger + self.logger.addHandler(ch) + self.logger.info(offlineimap.banner) + return ch + + def setup_sysloghandler(self): + pass diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index f5ba5ba..bc53ae6 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import logging +import logging.handlers import re import time import sys @@ -94,9 +95,8 @@ class UIBase(object): def setup_sysloghandler(self): """Backend specific syslog handler.""" - # create console handler with a higher log level - ch = logging.SysLogHandler(sys.stdout) - #ch.setLevel(logging.DEBUG) + # create syslog handler + ch = logging.handlers.SysLogHandler('/dev/log') # create formatter and add it to the handlers self.formatter = logging.Formatter("%(message)s") ch.setFormatter(self.formatter) diff --git a/offlineimap/ui/__init__.py b/offlineimap/ui/__init__.py index 3da42b9..258b5bb 100644 --- a/offlineimap/ui/__init__.py +++ b/offlineimap/ui/__init__.py @@ -21,6 +21,7 @@ from offlineimap.ui import TTY, Noninteractive, Machine UI_LIST = {'ttyui': TTY.TTYUI, 'basic': Noninteractive.Basic, 'quiet': Noninteractive.Quiet, + 'syslog': Noninteractive.Syslog, 'machineui': Machine.MachineUI} #add Blinkenlights UI if it imports correctly (curses installed) From 63db8776e0bf65ea98ade9ffa80398a2d2e8ac0e Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 13 Oct 2015 00:52:43 +0200 Subject: [PATCH 032/100] Noninteractive: add minor comment Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Noninteractive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/ui/Noninteractive.py b/offlineimap/ui/Noninteractive.py index 11b7247..0e23a93 100644 --- a/offlineimap/ui/Noninteractive.py +++ b/offlineimap/ui/Noninteractive.py @@ -46,4 +46,4 @@ class Syslog(UIBase): return ch def setup_sysloghandler(self): - pass + pass # Do not honor -s (log to syslog) CLI option. From 2a637bd457b3006f536cfdf01157039b4e262a0e Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 13 Oct 2015 14:37:43 +0200 Subject: [PATCH 033/100] bump impalib2 from v2.49 to v2.50 Allows ssl3 protocol to not be available. Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index d20d614..f448404 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.49" +__version__ = "2.50" __release__ = "2" -__revision__ = "49" +__revision__ = "50" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -83,7 +83,7 @@ READ_SIZE = 32768 # Consume all available in socke DFLT_DEBUG_BUF_LVL = 3 # Level above which the logging output goes directly to stderr -TLS_SECURE = "tls_secure" # Recognised TLS levels +TLS_SECURE = "tls_secure" # Recognised TLS levels TLS_NO_SSL = "tls_no_ssl" TLS_COMPAT = "tls_compat" @@ -486,7 +486,7 @@ class IMAP4(object): import ssl TLS_MAP = {} - if hasattr(ssl, "PROTOCOL_TLSv1_2"): # py3 + if hasattr(ssl, "PROTOCOL_TLSv1_2"): # py3 TLS_MAP[TLS_SECURE] = { "tls1_2": ssl.PROTOCOL_TLSv1_2, "tls1_1": ssl.PROTOCOL_TLSv1_1, @@ -499,10 +499,13 @@ class IMAP4(object): }) TLS_MAP[TLS_COMPAT] = TLS_MAP[TLS_NO_SSL].copy() TLS_MAP[TLS_COMPAT].update({ - "ssl3": ssl.PROTOCOL_SSLv3, "ssl23": ssl.PROTOCOL_SSLv23, None: ssl.PROTOCOL_SSLv23, }) + if hasattr(ssl, "PROTOCOL_SSLv3"): # Might not be available. + TLS_MAP[TLS_COMPAT].update({ + "ssl3": ssl.PROTOCOL_SSLv3 + }) if self.ca_certs is not None: cert_reqs = ssl.CERT_REQUIRED From 8787b377f3b4f75e7cac33d02d4456898e72e497 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 14 Oct 2015 00:09:40 +0200 Subject: [PATCH 034/100] README: new section status and future Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 16 +++++++++------- README.md | 23 ++++++++++++++++++++++- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c2bd120..66a911c 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -27,6 +27,15 @@ contributions. .. contents:: :depth: 3 +Submit issues +============= + +Issues are welcome to both Github_ and the `mailing list`_, at your own +convenience. + +You might help closing some issues, too. :-) + + For the imaptients ================== @@ -36,13 +45,6 @@ For the imaptients - All the `documentation`_ -Submit issues -============= - -Issues are welcome to both Github_ and the `mailing list`_, at your own -convenience. - - Community ========= diff --git a/README.md b/README.md index 56141ec..72b1094 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ -[offlineimap]: https://github.com/OfflineIMAP/offlineimap +[offlineimap]: http://github.com/OfflineIMAP/offlineimap [website]: http://offlineimap.org [wiki]: http://github.com/OfflineIMAP/offlineimap/wiki +[blog]: http://offlineimap.org/posts.html # OfflineIMAP @@ -32,6 +33,26 @@ GNU General Public License v2. * It is **safe**. +## Project status and future + +> As one of the maintainer of OfflineIMAP, I'd like to put my efforts into +> [imapfw](http://github.com/OfflineIMAP/imapfw). **imapfw** is a software in +> development that I intend to replace OfflineIMAP in the long term. +> +> That's why I'm not going to do development in OfflineIMAP. I continue to do +> the minimal maintenance job in OfflineIMAP: fixing small bugs, (quick) +> reviewing/merging patches and rolling out new releases, but that's all. +> +> While I keep tracking issues for OfflineIMAP, you should not expect support +> from me anymore. +> +> You won't be left at the side. OfflineIMAP's community is large enough so that +> you'll find people for most of your issues. +> +> Get news from the blog. +> +> Nicolas Sebrecht. ,-) + ## Downloads You should first check if your distribution already packages OfflineIMAP for you. From aa30dc41a695ff56d87446d173a66daba48779b0 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 14 Oct 2015 00:17:22 +0200 Subject: [PATCH 035/100] README add link to the blog Signed-off-by: Nicolas Sebrecht --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 72b1094..58aa3e3 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ GNU General Public License v2. > You won't be left at the side. OfflineIMAP's community is large enough so that > you'll find people for most of your issues. > -> Get news from the blog. +> Get news from the [blog][blog]. > > Nicolas Sebrecht. ,-) From 91d089641bd1a18cd7938125a88c9dbc6d20b308 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 15 Oct 2015 04:15:37 +0200 Subject: [PATCH 036/100] bump imaplib2 from v2.50 to v2.51 Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 54 ++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index f448404..be3a7b4 100644 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.50" +__version__ = "2.51" __release__ = "2" -__revision__ = "50" +__revision__ = "51" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -65,7 +65,7 @@ if bytes != str: else: import Queue as queue string_types = basestring - + threading.TIMEOUT_MAX = 9223372036854.0 select_module = select @@ -189,7 +189,7 @@ class Request(object): def get_response(self, exc_fmt=None): self.callback = None if __debug__: self.parent._log(3, '%s:%s.ready.wait' % (self.name, self.tag)) - self.ready.wait(sys.float_info.max) + self.ready.wait(threading.TIMEOUT_MAX) if self.aborted is not None: typ, val = self.aborted @@ -329,6 +329,7 @@ class IMAP4(object): self.compressor = None # COMPRESS/DEFLATE if not None self.decompressor = None + self._tls_established = False # Create unique tag for this session, # and compile tagged response matcher. @@ -390,7 +391,7 @@ class IMAP4(object): # request and store CAPABILITY response. try: - self.welcome = self._request_push(tag='continuation').get_response('IMAP4 protocol error: %s')[1] + self.welcome = self._request_push(name='welcome', tag='continuation').get_response('IMAP4 protocol error: %s')[1] if self._get_untagged_response('PREAUTH'): self.state = AUTH @@ -478,10 +479,6 @@ class IMAP4(object): def ssl_wrap_socket(self): - # Allow sending of keep-alive messages - seems to prevent some servers - # from closing SSL, leading to deadlocks. - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - try: import ssl @@ -525,13 +522,17 @@ class IMAP4(object): self.read_fd = self.sock.fileno() except ImportError: # No ssl module, and socket.ssl has no fileno(), and does not allow certificate verification - raise socket.sslerror("imaplib2 SSL mode does not work without ssl module") + raise socket.sslerror("imaplib SSL mode does not work without ssl module") if self.cert_verify_cb is not None: cert_err = self.cert_verify_cb(self.sock.getpeercert(), self.host) if cert_err: raise ssl_exc(cert_err) + # Allow sending of keep-alive messages - seems to prevent some servers + # from closing SSL, leading to deadlocks. + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + def start_compressing(self): @@ -567,7 +568,7 @@ class IMAP4(object): data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: - data = bytes(data, 'utf8') + data = bytes(data, 'ASCII') self.sock.sendall(data) @@ -576,7 +577,14 @@ class IMAP4(object): """shutdown() Close I/O established in "open".""" - self.sock.close() + try: + self.sock.shutdown(socket.SHUT_RDWR) + except OSError as e: + # The server might already have closed the connection + if e.errno != errno.ENOTCONN: + raise + finally: + self.sock.close() def socket(self): @@ -914,7 +922,7 @@ class IMAP4(object): def _CRAM_MD5_AUTH(self, challenge): """Authobject to use with CRAM-MD5 authentication.""" import hmac - pwd = (self.password.encode('utf-8') if isinstance(self.password, str) + pwd = (self.password.encode('ASCII') if isinstance(self.password, str) else self.password) return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest() @@ -1109,7 +1117,7 @@ class IMAP4(object): if name not in self.capabilities: raise self.abort('TLS not supported by server') - if hasattr(self, '_tls_established') and self._tls_established: + if self._tls_established: raise self.abort('TLS session already established') # Must now shutdown reader thread after next response, and restart after changing read_fd @@ -1272,7 +1280,10 @@ class IMAP4(object): bye = self._get_untagged_response('BYE', leave=True) if bye: - raise self.abort(bye[-1]) + if str != bytes: + raise self.abort(bye[-1].decode('ASCII', 'replace')) + else: + raise self.abort(bye[-1]) def _checkquote(self, arg): @@ -1333,7 +1344,7 @@ class IMAP4(object): self.commands_lock.release() if need_event: if __debug__: self._log(3, 'sync command %s waiting for empty commands Q' % name) - self.state_change_free.wait(sys.float_info.max) + self.state_change_free.wait(threading.TIMEOUT_MAX) if __debug__: self._log(3, 'sync command %s proceeding' % name) if self.state not in Commands[name][CMD_VAL_STATES]: @@ -1384,7 +1395,7 @@ class IMAP4(object): return rqb # Must setup continuation expectancy *before* ouq.put - crqb = self._request_push(tag='continuation') + crqb = self._request_push(name=name, tag='continuation') self.ouq.put(rqb) @@ -1409,7 +1420,7 @@ class IMAP4(object): if literator is not None: # Need new request for next continuation response - crqb = self._request_push(tag='continuation') + crqb = self._request_push(name=name, tag='continuation') if __debug__: self._log(4, 'write literal size %s' % len(literal)) crqb.data = '%s%s' % (literal, CRLF) @@ -1449,7 +1460,10 @@ class IMAP4(object): return bye = self._get_untagged_response('BYE', leave=True) if bye: - rqb.abort(self.abort, bye[-1]) + if str != bytes: + rqb.abort(self.abort, bye[-1].decode('ASCII', 'replace')) + else: + rqb.abort(self.abort, bye[-1]) return typ, dat = response if typ == 'BAD': @@ -2084,7 +2098,7 @@ class IMAP4_SSL(IMAP4): """IMAP4 client class over SSL connection Instantiate with: - IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, tls_level="tls_compat") + IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None, tls_level="tls_compat") host - host's name (default: localhost); port - port number (default: standard IMAP4 SSL port); From b7fec936615fc28eef6f308c472d1cb7019f1b03 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 15 Oct 2015 04:55:34 +0200 Subject: [PATCH 037/100] v6.6.0-rc2 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 37 +++++++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index ef8c8ab..0bf5b5f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,43 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} +### OfflineIMAP v6.6.0-rc2 (2015-10-15) + +#### Notes + +Interesting job was done in this release with 3 new features: + +- Support for XOAUTH2; +- New 'tls_level' configuration option to automatically discard insecure SSL protocols; +- New interface 'syslog' comes in, next to the -s CLI option. This allows better + integration with systemd. + +I won't merge big changes until the stable is out. IOW, you can seriously start +testing this rc2. + +#### Features + +- Add a new syslog ui. +- Introduce the 'tls_level' configuration option. +- Learn XOAUTH2 authentication (used by Gmail servers). +- Manual IDLE section improved (minor). + +#### Fixes + +- Configuration option utime_from_header handles out-of-bounds dates. +- offlineimap.conf: fix erroneous assumption about ssl23. +- Fix status code to reflect success or failure of a sync. +- contrib/release.sh: fix changelog edition. + +#### Changes + +- Bump imaplib2 from v2.48 to v2.51. +- README: new section status and future. +- Minor code cleanups. +- Makefile: improve building of targz. +- systemd: log to syslog rather than stderr for better integration. + + ### OfflineIMAP v6.6.0-rc1 (2015-09-28) #### Notes diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index e76c1f9..e785c98 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,7 +2,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.6.0" -__revision__ = "-rc1" +__revision__ = "-rc2" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" From 5ea2106da8b9a519f445a2018e59b3c3aa54641c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 15 Oct 2015 07:30:19 +0200 Subject: [PATCH 038/100] minor fixes Signed-off-by: Nicolas Sebrecht --- Changelog.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Changelog.md b/Changelog.md index 0bf5b5f..959075c 100644 --- a/Changelog.md +++ b/Changelog.md @@ -64,24 +64,24 @@ care. OfflineIMAP can now send the logs to syslog and notify on new mail. #### Features -- 585e5d5 logging: add a switch to log to syslog -- 48bb2f4 Added the newmail_hook -- 14a0433 utf-7 feature is set experimental +- logging: add a switch to log to syslog. +- Added the newmail_hook. +- utf-7 feature is set experimental. #### Fixes -- 839d020 offlineimap.conf: fix a typo in the new mail hook example -- a76f01c Fix language. -- a3986b2 Fix spelling inconsistency. -- d53e5fc offlineimap.conf: don't use quotes for sep option -- 9143ea5 man page: fingerprint can be used with SSL -- 41692d0 fix #225 « Runonce (offlineimap -o) does not stop if autorefresh is declared in DEFAULT section » -- 2382b15 CONTRIBUTING: fix links to offlineimap.org +- offlineimap.conf: fix a typo in the new mail hook example. +- Fix language. +- Fix spelling inconsistency. +- offlineimap.conf: don't use quotes for sep option. +- man page: fingerprint can be used with SSL. +- fix #225 « Runonce (offlineimap -o) does not stop if autorefresh is declared in DEFAULT section ». +- CONTRIBUTING: fix links to offlineimap.org. #### Changes -- 71dd03e Bump imaplib2 from 2.43 to 2.48 -- eeb7e89 README: small improvements +- Bump imaplib2 from 2.43 to 2.48 +- README: small improvements From 8cbe5640b27e3e9f83d1390f952e20e750653603 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 15 Oct 2015 13:42:17 +0200 Subject: [PATCH 039/100] offlineimap.conf: improve namtrans doc a bit Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 07f9901..b8f2469 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -941,16 +941,17 @@ remoteuser = username # folders, UNLESS the second values are filtered out by folderfilter below. # Failure to follow this rule will result in undefined behavior. # -# See the user documentation for details and use cases. They are also online at: -# http://docs.offlineimap.org/en/latest/nametrans.html +# If you enable nametrans, you will likely need to set the reversed nametrans on +# the other side. See the user documentation for details and use cases. They +# are also online at: http://offlineimap.org/doc/nametrans.html # # This example below will remove "INBOX." from the leading edge of folders # (great for Courier IMAP users). # #nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) # -# Using Courier remotely and want to duplicate its mailbox naming -# locally? Try this: +# Using Courier remotely and want to duplicate its mailbox naming locally? Try +# this: # #nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) From 1becbff786abc73dc6f7cc21bd16e579f74eef1e Mon Sep 17 00:00:00 2001 From: Max Vilimpoc Date: Wed, 28 Oct 2015 02:22:14 +0100 Subject: [PATCH 040/100] enable SSL by default Fix-github-issue: #263 Signed-off-by: Max Vilimpoc Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 167295a..4ecfb38 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -195,7 +195,7 @@ class IMAPRepository(BaseRepository): return self.getconfint('remoteport', None) def getssl(self): - return self.getconfboolean('ssl', 0) + return self.getconfboolean('ssl', 1) def getsslclientcert(self): xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] From 3b30c4aa93737c7ac3fd8be304b9515988657366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Gross?= Date: Wed, 28 Oct 2015 10:56:16 +0100 Subject: [PATCH 041/100] add new config option filename_use_mail_timestamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If this value is true, use (if possible) a timestamp based on message Date or Delivery-date headers. The current system time is used otherwise. filename_use_mail_timestamp and utime_from_header are now completely separated option that do not interfere one with other. To handle this feature in a multithread context we use a hash to count the number of mail with the same timestamp. This method is more accurate than using the old lasttime and timeseq variables. Signed-off-by: Sébastien Gross Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 7 ++++++ offlineimap/folder/Maildir.py | 46 ++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index d81f3ef..d2a0706 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -60,6 +60,13 @@ class BaseFolder(object): self._utime_from_header = self.config.getdefaultboolean(repo, "utime_from_header", utime_from_header_global) + # Do we need to use mail timestamp for filename prefix? + filename_use_mail_timestamp_global = self.config.getdefaultboolean( + "general", "filename_use_mail_timestamp", False) + repo = "Repository " + repository.name + self._filename_use_mail_timestamp = self.config.getdefaultboolean(repo, + "filename_use_mail_timestamp", filename_use_mail_timestamp_global) + # Determine if we're running static or dynamic folder filtering # and check filtering status self._dynamic_folderfilter = self.config.getdefaultboolean( diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 07d1b73..a7dbf26 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -38,22 +38,20 @@ re_uidmatch = re.compile(',U=(\d+)') # Find a numeric timestamp in a string (filename prefix) re_timestampmatch = re.compile('(\d+)'); -timeseq = 0 -lasttime = 0 +timehash = {} timelock = Lock() -def _gettimeseq(): - global lasttime, timeseq, timelock +def _gettimeseq(date=None): + global timehash, timelock timelock.acquire() try: - thistime = long(time.time()) - if thistime == lasttime: - timeseq += 1 - return (thistime, timeseq) + if date is None: + date = long(time.time()) + if timehash.has_key(date): + timehash[date] += 1 else: - lasttime = thistime - timeseq = 0 - return (thistime, timeseq) + timehash[date] = 0 + return (date, timehash[date]) finally: timelock.release() @@ -269,14 +267,14 @@ class MaildirFolder(BaseFolder): filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) - def new_message_filename(self, uid, flags=set()): + def new_message_filename(self, uid, flags=set(), date=None): """Creates a new unique Maildir filename :param uid: The UID`None`, or a set of maildir flags :param flags: A set of maildir flags :returns: String containing unique message filename""" - timeval, timeseq = _gettimeseq() + timeval, timeseq = _gettimeseq(date) return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \ (timeval, timeseq, os.getpid(), socket.gethostname(), uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) @@ -346,7 +344,27 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - messagename = self.new_message_filename(uid, flags) + + # use the mail timestamp given by either Date or Delivery-date mail + # headers. + message_timestamp = None + if self._filename_use_mail_timestamp: + try: + message_timestamp = emailutil.get_message_date(content, 'Date') + if message_timestamp is None: + # Give a try with Delivery-date + date = emailutil.get_message_date(content, 'Delivery-date') + except: + # This should never happen + from email.Parser import Parser + from offlineimap.ui import getglobalui + datestr = Parser().parsestr(content, True).get("Date") + ui = getglobalui() + ui.warn("UID %d has invalid date %s: %s\n" + "Not using message timestamp as file prefix" % (uid, datestr, e)) + # No need to check if date is None here since it would + # be overridden by _gettimeseq. + messagename = self.new_message_filename(uid, flags, date=message_timestamp) tmpname = self.save_to_tmp_file(messagename, content) if self.utime_from_header: From b2adf2d2583cbbe0ab8a048806557bb125075b58 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Tue, 3 Nov 2015 13:49:20 +0800 Subject: [PATCH 042/100] fix: avoid writing password to log Github-fix: #266 Signed-off-by: Valentin Lab Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 6cdbf26..5cb1be4 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -206,7 +206,8 @@ class IMAPServer: authz = self.user_identity NULL = u'\x00' retval = NULL.join((authz, authc, passwd)).encode('utf-8') - self.ui.debug('imap', '__plainhandler: returning %s' % retval) + logsafe_retval = NULL.join((authz, authc, "(passwd hidden for log)")).encode('utf-8') + self.ui.debug('imap', '__plainhandler: returning %s' % logsafe_retval) return retval From 3d125602f64bfb55164956a22925451ba4a2e27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Gross?= Date: Wed, 4 Nov 2015 10:32:22 +0100 Subject: [PATCH 043/100] Add documentation for filename_use_mail_timestamp. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sébastien Gross Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 07f9901..6c5ac24 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -514,6 +514,29 @@ localfolders = ~/Test #utime_from_header = no +# This option stands in the [Repository LocalExample] section. +# +# This option is similar to "utime_from_header" and could be use as a +# complementary feature to keep track of a message date. This option only +# makes sense for the Maildir type. +# +# By default each message is stored in a file which prefix is the fetch +# timestamp and an order rank such as "1446590057_0". In a multithreading +# environment message are fetched in a random order, then you can't trust +# the file name to sort your boxes. +# +# If set to "yes" the file name prefix if build on the message "Date" header +# (which should be present) or the "Received-date" if "Date" is not +# found. If neither "Received-date" nor "Date" is found, the current system +# date is used. Now you can quickly sort your messages using their file +# names. +# +# Used in combination with "utime_from_header" all your message would be in +# order with the correct mtime attribute. +# +#filename_use_mail_timestamp = no + + [Repository GmailLocalExample] # This type of repository enables syncing of Gmail. All Maildir From 04babea60735c489a88548a4d3625e2a96fa88fb Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 5 Nov 2015 09:07:31 +0100 Subject: [PATCH 044/100] bump from imaplib2 v2.51 to v2.52 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks-to: Sébastien Gross Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) mode change 100644 => 100755 offlineimap/imaplib2.py diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py old mode 100644 new mode 100755 index be3a7b4..61878f6 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.51" +__version__ = "2.52" __release__ = "2" -__revision__ = "51" +__revision__ = "52" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -50,7 +50,8 @@ Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli August 2015. Fix to allow interruptible IDLE command by Tim Peoples September 2015. -Add support for TLS levels by Ben Boeckel September 2015.""" +Add support for TLS levels by Ben Boeckel September 2015. +Fix for shutown exception by Sebastien Gross November 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -579,7 +580,7 @@ class IMAP4(object): try: self.sock.shutdown(socket.SHUT_RDWR) - except OSError as e: + except Exception as e: # The server might already have closed the connection if e.errno != errno.ENOTCONN: raise @@ -1349,8 +1350,8 @@ class IMAP4(object): if self.state not in Commands[name][CMD_VAL_STATES]: self.literal = None - raise self.error('command %s illegal in state %s' - % (name, self.state)) + raise self.error('command %s illegal in state %s, only allowed in states %s' + % (name, self.state, ', '.join(Commands[name][CMD_VAL_STATES]))) self._check_bye() From 86a91f28c0cc3489f591570a52575f1d64f79fe3 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 5 Nov 2015 09:26:01 +0100 Subject: [PATCH 045/100] v6.6.0-rc3 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 28 ++++++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 959075c..9310c1b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,34 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} + +### OfflineIMAP v6.6.0-rc3 (2015-11-05) + +#### Notes + +Changes are slowing down and the code is under serious testing by some new +contributors. Everything expected at this time in the release cycle. Thanks to +them. + +SSL is now enabled by default to prevent from sending private data in clear +stream to the wild. + +#### Features + +- Add new config option `filename_use_mail_timestamp`. + +#### Fixes + +- Bump from imaplib2 v2.51 to v2.52. +- Minor fixes. + +#### Changes + +- Enable SSL by default. +- Fix: avoid writing password to log. +- offlineimap.conf: improve namtrans doc a bit. + + ### OfflineIMAP v6.6.0-rc2 (2015-10-15) #### Notes diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index e785c98..aa19add 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,7 +2,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.6.0" -__revision__ = "-rc2" +__revision__ = "-rc3" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" From 4087f3a4e75cec917a734e57196ef4fb77b2f589 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Mon, 9 Nov 2015 10:35:03 +0800 Subject: [PATCH 046/100] add a full stack of all thread dump upon EXIT or KILL signal in thread debug mode Note that the stacks are grouped if similar, and the current process (the one handling the signal) is identified and reports where it was before the signal. This can be quite handy when wanting to debug thread locks for instance. Signed-off-by: Valentin Lab Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/offlineimap/init.py b/offlineimap/init.py index d94dcb7..aeb089e 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -31,6 +31,9 @@ from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser from offlineimap.utils import stacktrace +import traceback +import collections + class OfflineImap: """The main class that encapsulates the high level use of OfflineImap. @@ -272,6 +275,42 @@ class OfflineImap: self.config = config return (options, args) + def __dumpstacks(self, context=1, sighandler_deep=2): + """ Signal handler: dump a stack trace for each existing thread.""" + + currentThreadId = threading.currentThread().ident + + def unique_count(l): + d = collections.defaultdict(lambda: 0) + for v in l: + d[tuple(v)] += 1 + return list((k, v) for k, v in d.iteritems()) + + stack_displays = [] + for threadId, stack in sys._current_frames().items(): + stack_display = [] + for filename, lineno, name, line in traceback.extract_stack(stack): + stack_display.append(' File: "%s", line %d, in %s' + % (filename, lineno, name)) + if line: + stack_display.append(" %s" % (line.strip())) + if currentThreadId == threadId: + stack_display = stack_display[:- (sighandler_deep * 2)] + stack_display.append(' => Stopped to handle current signal. ') + stack_displays.append(stack_display) + stacks = unique_count(stack_displays) + print "** Thread List:\n" + for stack, times in stacks: + if times == 1: + msg = "%s Thread is at:\n%s\n" + else: + msg = "%s Threads are at:\n%s\n" + self.ui.debug('thread', msg % (times, '\n'.join(stack[- (context * 2):]))) + + self.ui.debug('thread', "Dumped a total of %d Threads." % + len(sys._current_frames().keys())) + + def __sync(self, options): """Invoke the correct single/multithread syncing @@ -321,6 +360,8 @@ class OfflineImap: getglobalui().warn("Terminating NOW (this may "\ "take a few seconds)...") accounts.Account.set_abort_event(self.config, 3) + if 'thread' in self.ui.debuglist: + self.__dumpstacks(5) elif sig == signal.SIGQUIT: stacktrace.dump(sys.stderr) os.abort() From c5c45cc687c85ed1ef8c642f74951012dd2f90c3 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 9 Nov 2015 04:37:51 +0100 Subject: [PATCH 047/100] minor: add parenthesis to print in init Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index aeb089e..91e0f55 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -299,7 +299,7 @@ class OfflineImap: stack_display.append(' => Stopped to handle current signal. ') stack_displays.append(stack_display) stacks = unique_count(stack_displays) - print "** Thread List:\n" + print("** Thread List:\n") for stack, times in stacks: if times == 1: msg = "%s Thread is at:\n%s\n" From 6a6b01f1a175b14a64730a4112ca95dce8f51fb2 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Mon, 9 Nov 2015 14:31:06 +0800 Subject: [PATCH 048/100] fix: replace rogue `print` statement by `self.ui.debug` Signed-off-by: Valentin Lab Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/init.py b/offlineimap/init.py index 91e0f55..beb90de 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -299,7 +299,7 @@ class OfflineImap: stack_display.append(' => Stopped to handle current signal. ') stack_displays.append(stack_display) stacks = unique_count(stack_displays) - print("** Thread List:\n") + self.ui.debug('thread', "** Thread List:\n") for stack, times in stacks: if times == 1: msg = "%s Thread is at:\n%s\n" From c52ca6687401639b716155cc90097f190b2739fa Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:14 -0300 Subject: [PATCH 049/100] Maildir folder: extract lower-case letters (custom flags) from filename Remove filtering that was previously done to avoid errors in flag handling. Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index a7dbf26..19a9ecf 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -135,9 +135,7 @@ class MaildirFolder(BaseFolder): uid = long(uidmatch.group(1)) flagmatch = self.re_flagmatch.search(filename) if flagmatch: - # Filter out all lowercase (custom maildir) flags. We don't - # handle them yet. - flags = set((c for c in flagmatch.group(1) if not c.islower())) + flags = set((c for c in flagmatch.group(1))) return prefix, uid, fmd5, flags def _scanfolder(self, min_date=None, min_uid=None): @@ -149,7 +147,7 @@ class MaildirFolder(BaseFolder): with similar UID's (e.g. the UID was reassigned much later). Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F - (flagged). + (flagged), plus lower-case letters for custom flags. :returns: dict that can be used as self.messagelist. """ @@ -414,8 +412,7 @@ class MaildirFolder(BaseFolder): if flags != self.messagelist[uid]['flags']: # Flags have actually changed, construct new filename Strip - # off existing infostring (possibly discarding small letter - # flags that dovecot uses TODO) + # off existing infostring infomatch = self.re_flagmatch.search(filename) if infomatch: filename = filename[:-len(infomatch.group())] #strip off From 4e2de8f58a45181e7073990e62ff7c60286203e5 Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:09 -0300 Subject: [PATCH 050/100] Maildir repository: add config keys for IMAP keywords This commit assembles a dictionary mapping user-specified IMAP keywords to Maildir lower-case flags, similar to Dovecot's format http://wiki2.dovecot.org/MailboxFormat/Maildir Configuration example: [Repository Local] type = Maildir localfolders = ~/Maildir/ customflag_a = $label1 customflag_b = $Forwarded customflag_c = Junk Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 3 +++ offlineimap/repository/Maildir.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index adae16d..8634628 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -133,6 +133,9 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getsep(self): raise NotImplementedError + def getkeywordmap(self): + raise NotImplementedError + def should_sync_folder(self, fname): """Should this folder be synced?""" diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 0262ba2..fef57f3 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -39,6 +39,14 @@ class MaildirRepository(BaseRepository): if not os.path.isdir(self.root): os.mkdir(self.root, 0o700) + # Create the keyword->char mapping + self.keyword2char = dict() + for c in 'abcdefghijklmnopqrstuvwxyz': + confkey = 'customflag_' + c + keyword = self.getconf(confkey, None) + if keyword is not None: + self.keyword2char[keyword] = c + def _append_folder_atimes(self, foldername): """Store the atimes of a folder's new|cur in self.folder_atimes""" @@ -72,6 +80,9 @@ class MaildirRepository(BaseRepository): def getsep(self): return self.getconf('sep', '.').strip() + def getkeywordmap(self): + return self.keyword2char + def makefolder(self, foldername): """Create new Maildir folder if necessary From 73a3767d11614d95e9468af64d7c5af4d3349c6b Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:10 -0300 Subject: [PATCH 051/100] IMAP folder: expose the message keywords The keywords are in the flag string, so imaputil can just strip the usual \Flags. Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 5 +++++ offlineimap/folder/IMAP.py | 8 +++++++- offlineimap/imaputil.py | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index d2a0706..5031a40 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -420,6 +420,11 @@ class BaseFolder(object): raise NotImplementedError + def getmessagekeywords(self, uid): + """Returns the keywords for the specified message.""" + + raise NotImplementedError + def savemessageflags(self, uid, flags): """Sets the specified message's flags to the given set. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 60b5301..7f0fd6d 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -251,8 +251,10 @@ class IMAPFolder(BaseFolder): uid = long(options['UID']) self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) + keywords = imaputil.flagsimap2keywords(options['FLAGS']) rtime = imaplibutil.Internaldate2epoch(messagestr) - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, + 'keywords': keywords} self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) def dropmessagelistcache(self): @@ -309,6 +311,10 @@ class IMAPFolder(BaseFolder): def getmessageflags(self, uid): return self.messagelist[uid]['flags'] + # Interface from BaseFolder + def getmessagekeywords(self, uid): + return self.messagelist[uid]['keywords'] + def __generate_randomheader(self, content): """Returns a unique X-OfflineIMAP header diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 6d8b444..6a18732 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -195,6 +195,14 @@ def flagsimap2maildir(flagstring): retval.add(maildirflag) return retval +def flagsimap2keywords(flagstring): + """Convert string '(\\Draft \\Deleted somekeyword otherkeyword)' into a + keyword set (somekeyword otherkeyword).""" + + imapflagset = set(flagstring[1:-1].split()) + serverflagset = set([flag for (flag, c) in flagmap]) + return imapflagset - serverflagset + def flagsmaildir2imap(maildirflaglist): """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'.""" From 61ee6e783e6b6faf3258f7b847fb8c870ab4395e Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:11 -0300 Subject: [PATCH 052/100] __syncmessagesto_flags: store keywords This uses the destination folder's keyword mapping to translate the message's keywords into some appropriate format. Tested only with local Maildir. Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 5031a40..b48afbe 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -937,6 +937,32 @@ class BaseFolder(object): else: statusflags = set() + #keywords: if there is a keyword map, use it to figure out what + #other 'flags' we should add + try: + keywordmap = dstfolder.getrepository().getkeywordmap() + knownkeywords = set(keywordmap.keys()) + + selfkeywords = self.getmessagekeywords(uid) + + if not knownkeywords >= selfkeywords: + #some of the message's keywords are not in the mapping, so + #skip them + + skipped_keywords = list(selfkeywords - knownkeywords) + selfkeywords &= knownkeywords + + self.ui.warn("Unknown keywords skipped: %s\n" + "You may want to change your configuration to include " + "those\n" % (skipped_keywords)) + + keywordletterset = set([keywordmap[keyw] for keyw in selfkeywords]) + + #add the lower-case letters to the list of message flags + selfflags |= keywordletterset + except NotImplementedError: + pass + addflags = selfflags - statusflags delflags = statusflags - selfflags From 58a6f8b401cf8608f020711be6b4f63b0083fcda Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:12 -0300 Subject: [PATCH 053/100] __syncmessagesto_flags: refactor for readability Extract the flag/keyword translation and combination logic to a function. Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 64 ++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index b48afbe..d77a16c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -908,6 +908,42 @@ class BaseFolder(object): return #don't delete messages in dry-run mode dstfolder.deletemessages(deletelist) + def combine_flags_and_keywords(self, uid, dstfolder): + """Combine the message's flags and keywords using the mapping for the + destination folder.""" + + # Take a copy of the message flag set, otherwise + # __syncmessagesto_flags() will fail because statusflags is actually a + # reference to selfflags (which it should not, but I don't have time to + # debug THAT). + selfflags = set(self.getmessageflags(uid)) + + try: + keywordmap = dstfolder.getrepository().getkeywordmap() + knownkeywords = set(keywordmap.keys()) + + selfkeywords = self.getmessagekeywords(uid) + + if not knownkeywords >= selfkeywords: + #some of the message's keywords are not in the mapping, so + #skip them + + skipped_keywords = list(selfkeywords - knownkeywords) + selfkeywords &= knownkeywords + + self.ui.warn("Unknown keywords skipped: %s\n" + "You may want to change your configuration to include " + "those\n" % (skipped_keywords)) + + keywordletterset = set([keywordmap[keyw] for keyw in selfkeywords]) + + #add the mapped keywords to the list of message flags + selfflags |= keywordletterset + except NotImplementedError: + pass + + return selfflags + def __syncmessagesto_flags(self, dstfolder, statusfolder): """Pass 3: Flag synchronization. @@ -930,38 +966,12 @@ class BaseFolder(object): if uid < 0 or not dstfolder.uidexists(uid): continue - selfflags = self.getmessageflags(uid) - if statusfolder.uidexists(uid): statusflags = statusfolder.getmessageflags(uid) else: statusflags = set() - #keywords: if there is a keyword map, use it to figure out what - #other 'flags' we should add - try: - keywordmap = dstfolder.getrepository().getkeywordmap() - knownkeywords = set(keywordmap.keys()) - - selfkeywords = self.getmessagekeywords(uid) - - if not knownkeywords >= selfkeywords: - #some of the message's keywords are not in the mapping, so - #skip them - - skipped_keywords = list(selfkeywords - knownkeywords) - selfkeywords &= knownkeywords - - self.ui.warn("Unknown keywords skipped: %s\n" - "You may want to change your configuration to include " - "those\n" % (skipped_keywords)) - - keywordletterset = set([keywordmap[keyw] for keyw in selfkeywords]) - - #add the lower-case letters to the list of message flags - selfflags |= keywordletterset - except NotImplementedError: - pass + selfflags = self.combine_flags_and_keywords(uid, dstfolder) addflags = selfflags - statusflags delflags = statusflags - selfflags From def087eeeab072a7cb0925edbe90a3827310bf3a Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:13 -0300 Subject: [PATCH 054/100] IMAP keyword sync: add Maildir configuration example Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 2d01221..4be3ce5 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -536,6 +536,31 @@ localfolders = ~/Test # #filename_use_mail_timestamp = no +# This option stands in the [Repository LocalExample] section. +# +# Map IMAP [user-defined] keywords to lowercase letters, similar to Dovecot's +# format described in http://wiki2.dovecot.org/MailboxFormat/Maildir . This +# option makes sense for the Maildir type, only. +# +# Configuration example: +# customflag_x = some_keyword +# +# With the configuration example above enabled, all IMAP messages that have +# 'some_keyword' in their FLAGS field will have an 'x' in the flags part of the +# maildir filename: +# 1234567890.M20046P2137.mailserver,S=4542,W=4642:2,Sx +# +# Valid fields are customflag_[a-z], valid values are whatever the IMAP server +# allows. +# +# Comparison in offlineimap is case-sensitive. +# +# This option is EXPERIMENTAL. +# +#customflag_a = some_keyword +#customflag_b = $OtherKeyword +#customflag_c = NonJunk +#customflag_d = ToDo [Repository GmailLocalExample] From d6077a09cf8a7e4c30635a8cec2ac2e278c8da5a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 22 Nov 2015 19:52:32 +0100 Subject: [PATCH 055/100] Keywords: avoid warning at each message when no keywords are used This fix does not apply when any keyword in configured which is already harmless. Written-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 3 +++ offlineimap/repository/Maildir.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index d77a16c..14b0867 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -920,6 +920,9 @@ class BaseFolder(object): try: keywordmap = dstfolder.getrepository().getkeywordmap() + if keywordmap is None: + return selfflags + knownkeywords = set(keywordmap.keys()) selfkeywords = self.getmessagekeywords(uid) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index fef57f3..10085e7 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -81,7 +81,7 @@ class MaildirRepository(BaseRepository): return self.getconf('sep', '.').strip() def getkeywordmap(self): - return self.keyword2char + return self.keyword2char if len(self.keyword2char) > 0 else None def makefolder(self, foldername): """Create new Maildir folder if necessary From 573d55827fabe5e70c6c38bd2e2d655d503ab1ae Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 23 Nov 2015 02:24:33 +0100 Subject: [PATCH 056/100] update TODO.rst Signed-off-by: Nicolas Sebrecht --- TODO.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/TODO.rst b/TODO.rst index 73f80da..cb10f2b 100644 --- a/TODO.rst +++ b/TODO.rst @@ -120,8 +120,4 @@ TODO list so don't matter much about that if you don't get the point or what could be done. - -* Support Python 3. - - * Support Unicode. From f2ca4217c622f7d47ccdd8e95cd82d979784a62a Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Wed, 25 Nov 2015 14:39:56 +0800 Subject: [PATCH 057/100] fix: broken retry loop would break connection management The retry loop would release connection that would get re-released upon ``finally`` clause. In consequence, an exception would be cast. Signed-off-by: Valentin Lab Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Gmail.py | 6 +--- offlineimap/folder/IMAP.py | 55 ++++++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 3d83b91..ee52aed 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -72,11 +72,7 @@ class GmailFolder(IMAPFolder): (probably severity MESSAGE) if e.g. no message with this UID could be found. """ - imapobj = self.imapserver.acquireconnection() - try: - data = self._fetch_from_imap(imapobj, str(uid), 2) - finally: - self.imapserver.releaseconnection(imapobj) + data = self._fetch_from_imap(str(uid), 2) # data looks now e.g. #[('320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....')] diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 7f0fd6d..5a26051 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -282,11 +282,7 @@ class IMAPFolder(BaseFolder): this UID could be found. """ - imapobj = self.imapserver.acquireconnection() - try: - data = self._fetch_from_imap(imapobj, str(uid), 2) - finally: - self.imapserver.releaseconnection(imapobj) + data = self._fetch_from_imap(str(uid), 2) # data looks now e.g. [('320 (UID 17061 BODY[] # {2565}','msgbody....')] we only asked for one message, @@ -680,7 +676,7 @@ class IMAPFolder(BaseFolder): return uid - def _fetch_from_imap(self, imapobj, uids, retry_num=1): + def _fetch_from_imap(self, uids, retry_num=1): """Fetches data from IMAP server. Arguments: @@ -690,22 +686,37 @@ class IMAPFolder(BaseFolder): Returns: data obtained by this query.""" - query = "(%s)"% (" ".join(self.imap_query)) - fails_left = retry_num # retry on dropped connection - while fails_left: - try: - imapobj.select(self.getfullname(), readonly = True) - res_type, data = imapobj.uid('fetch', uids, query) - fails_left = 0 - except imapobj.abort as e: - # Release dropped connection, and get a new one - self.imapserver.releaseconnection(imapobj, True) - imapobj = self.imapserver.acquireconnection() - self.ui.error(e, exc_info()[2]) - fails_left -= 1 - # self.ui.error() will show the original traceback - if not fails_left: - raise e + imapobj = self.imapserver.acquireconnection() + try: + query = "(%s)"% (" ".join(self.imap_query)) + fails_left = retry_num ## retry on dropped connection + while fails_left: + try: + imapobj.select(self.getfullname(), readonly = True) + res_type, data = imapobj.uid('fetch', uids, query) + break + except imapobj.abort as e: + fails_left -= 1 + # self.ui.error() will show the original traceback + if fails_left <= 0: + message = ("%s, while fetching msg %r in folder %r." + " Max retry reached (%d)"% + (e, uids, self.name, retry_num)) + severity = OfflineImapError.ERROR.MESSAGE + raise OfflineImapError(message, + OfflineImapError.ERROR.MESSAGE) + # Release dropped connection, and get a new one + self.imapserver.releaseconnection(imapobj, True) + imapobj = self.imapserver.acquireconnection() + self.ui.error("%s. While fetching msg %r in folder %r." + " Retrying (%d/%d)"% + (e, uids, self.name, retry_num - fails_left, retry_num)) + finally: + # The imapobj here might be different than the one created before + # the ``try`` clause. So please avoid transforming this to a nice + # ``with`` without taking this into account. + self.imapserver.releaseconnection(imapobj) + if data == [None] or res_type != 'OK': #IMAP server says bad request or UID does not exist severity = OfflineImapError.ERROR.MESSAGE From 02a5f777801d887f0df49aa0e7e46ccf874a6160 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 28 Nov 2015 23:38:22 +0100 Subject: [PATCH 058/100] bump imaplib2 from v2.52 to v2.53 Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index 61878f6..a6b13ad 100755 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.52" +__version__ = "2.53" __release__ = "2" -__revision__ = "52" +__revision__ = "53" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -51,7 +51,8 @@ Fix for correct byte encoding for _CRAM_MD5_AUTH taken from python3.5 imaplib.py Fix for correct Python 3 exception handling by Tobias Brink August 2015. Fix to allow interruptible IDLE command by Tim Peoples September 2015. Add support for TLS levels by Ben Boeckel September 2015. -Fix for shutown exception by Sebastien Gross November 2015.""" +Fix for shutown exception by Sebastien Gross November 2015. +Add support for server BINARY mode (supplied by offlineimap-project) November 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -1383,7 +1384,10 @@ class IMAP4(object): self.literal = None if isinstance(literal, string_types): literator = None - data = '%s {%s}' % (data, len(literal)) + if 'BINARY' in self.capabilities: + data = '%s ~{%s}' % (data, len(literal)) + else: + data = '%s {%s}' % (data, len(literal)) else: literator = literal From 4a06c2af24442622b788aa047174250cd7e56266 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 5 Dec 2015 00:39:43 +0100 Subject: [PATCH 059/100] v6.6.0 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 19 +++++++++++++++++++ offlineimap/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 9310c1b..3b7d45e 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,25 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} +### OfflineIMAP v6.6.0 (2015-12-05) + +#### Features + +- Maildir learns to mimic Dovecot's format of lower-case letters (a,b,c..) for + "custom flags" or user keywords. + +#### Fixes + +- Broken retry loop would break connection management. +- Replace rogue `print` statement by `self.ui.debug`. + +#### Changes + +- Bump imaplib2 from v2.52 to v2.53. +- Code cleanups. +- Add a full stack of all thread dump upon EXIT or KILL signal in thread debug + mode. + ### OfflineIMAP v6.6.0-rc3 (2015-11-05) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index aa19add..1ee59d0 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,7 +2,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.6.0" -__revision__ = "-rc3" +__revision__ = "" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" From 452ada330a73c04d51a3c45fd601c80a424c37e8 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 5 Dec 2015 12:49:41 +0100 Subject: [PATCH 060/100] folder/Maildir: ignore dot files Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 19a9ecf..64570a1 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -163,6 +163,8 @@ class MaildirFolder(BaseFolder): date_excludees = {} for dirannex, filename in files: + if filename.startswith('.'): + continue # Ignore dot files. # We store just dirannex and filename, ie 'cur/123...' filepath = os.path.join(dirannex, filename) # Check maxsize if this message should be considered. From c8a511bb7e48a22f2f1bd92242ea0a7326b23601 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 28 Dec 2015 01:17:42 +0100 Subject: [PATCH 061/100] bump imaplib2 from 2.52 to 2.53 Remove support for binary send. Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplib2.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/offlineimap/imaplib2.py b/offlineimap/imaplib2.py index a6b13ad..61878f6 100755 --- a/offlineimap/imaplib2.py +++ b/offlineimap/imaplib2.py @@ -17,9 +17,9 @@ Public functions: Internaldate2Time __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate") -__version__ = "2.53" +__version__ = "2.52" __release__ = "2" -__revision__ = "53" +__revision__ = "52" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. @@ -51,8 +51,7 @@ Fix for correct byte encoding for _CRAM_MD5_AUTH taken from python3.5 imaplib.py Fix for correct Python 3 exception handling by Tobias Brink August 2015. Fix to allow interruptible IDLE command by Tim Peoples September 2015. Add support for TLS levels by Ben Boeckel September 2015. -Fix for shutown exception by Sebastien Gross November 2015. -Add support for server BINARY mode (supplied by offlineimap-project) November 2015.""" +Fix for shutown exception by Sebastien Gross November 2015.""" __author__ = "Piers Lauder " __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" @@ -1384,10 +1383,7 @@ class IMAP4(object): self.literal = None if isinstance(literal, string_types): literator = None - if 'BINARY' in self.capabilities: - data = '%s ~{%s}' % (data, len(literal)) - else: - data = '%s {%s}' % (data, len(literal)) + data = '%s {%s}' % (data, len(literal)) else: literator = literal From 7ed71fa742a865c4936092046f939f149502424d Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 28 Dec 2015 01:32:37 +0100 Subject: [PATCH 062/100] v6.6.1 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 15 +++++++++++++++ offlineimap/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 3b7d45e..1f84c42 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,21 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} +### OfflineIMAP v6.6.1 (2015-12-28) + +#### Notes + +This is a very small new stable release for two fixes. + +Amending support for BINARY APPEND which is not correctly implemented. Also, +remove potential harms from dot files in a local maildir. + +#### Fixes + +- Bump imaplib2 from 2.52 to 2.53. Remove support for binary send. +- Ignore aloo dot files in the Maildir while scanning for mails. + + ### OfflineIMAP v6.6.0 (2015-12-05) #### Features diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 1ee59d0..758c9cb 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.6.0" +__version__ = "6.6.1" __revision__ = "" __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" From 72850e3c232ccb671686ac7ea5790bfc89202382 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 28 Dec 2015 01:41:45 +0100 Subject: [PATCH 063/100] fix Changelog Signed-off-by: Nicolas Sebrecht --- Changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 1f84c42..a2cad23 100644 --- a/Changelog.md +++ b/Changelog.md @@ -26,7 +26,7 @@ remove potential harms from dot files in a local maildir. #### Fixes -- Bump imaplib2 from 2.52 to 2.53. Remove support for binary send. +- Bump imaplib2 from 2.53 to 2.52. Remove support for binary send. - Ignore aloo dot files in the Maildir while scanning for mails. From cfa704bbf2a083542d8d9b9ce49d02dc98eca095 Mon Sep 17 00:00:00 2001 From: ojab Date: Mon, 28 Dec 2015 13:53:53 +0000 Subject: [PATCH 064/100] Allow authorization via XOAUTH2 using access token Signed-off-by: Slava Kardakov Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 7 ++++++- offlineimap/imapserver.py | 4 ++-- offlineimap/repository/IMAP.py | 3 +++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 4be3ce5..5970bdf 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -769,7 +769,8 @@ remoteuser = username # with type = IMAP for compatible servers). # # Mandatory parameters are "oauth2_client_id", "oauth2_client_secret" and -# "oauth2_refresh_token". See below to learn how to get those. +# either "oauth2_refresh_token" or "oauth2_access_token". +# See below to learn how to get those. # # Specify the OAuth2 client id and secret to use for the connection.. # Here's how to register an OAuth2 client for Gmail, as of 10-2-2016: @@ -791,8 +792,12 @@ remoteuser = username # - Type the following command-line in a terminal and follow the instructions # python python/oauth2.py --generate_oauth2_token \ # --client_id=YOUR_CLIENT_ID --client_secret=YOUR_CLIENT_SECRET +# - Access token can be obtained using refresh token with command +# python python/oauth2.py --user=YOUR_EMAIL --client_id=YOUR_CLIENT_ID +# --client_secret=YOUR_CLIENT_SECRET --refresh_token=REFRESH_TOKEN # #oauth2_refresh_token = REFRESH_TOKEN +#oauth2_access_token = ACCESS_TOKEN ########## Passwords diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 5cb1be4..018ba7b 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -95,10 +95,10 @@ class IMAPServer: self.tlslevel = repos.gettlslevel() self.oauth2_refresh_token = repos.getoauth2_refresh_token() + self.oauth2_access_token = repos.getoauth2_access_token() self.oauth2_client_id = repos.getoauth2_client_id() self.oauth2_client_secret = repos.getoauth2_client_secret() self.oauth2_request_url = repos.getoauth2_request_url() - self.oauth2_access_token = None self.delim = None self.root = None @@ -212,7 +212,7 @@ class IMAPServer: def __xoauth2handler(self, response): - if self.oauth2_refresh_token is None: + if self.oauth2_refresh_token is None and self.oauth2_access_token is None: return None if self.oauth2_access_token is None: diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 4ecfb38..7a2a98c 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -276,6 +276,9 @@ class IMAPRepository(BaseRepository): def getoauth2_refresh_token(self): return self.getconf('oauth2_refresh_token', None) + def getoauth2_access_token(self): + return self.getconf('oauth2_access_token', None) + def getoauth2_client_id(self): return self.getconf('oauth2_client_id', None) From 1553e843b34afaf6ee0b402d751d799a16cd13fb Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 8 Jan 2016 17:30:54 +0100 Subject: [PATCH 065/100] Revert "Don't output initial blurb in "quiet" mode" This reverts commit a1dc76ae917d93d0c387bbd506faea3487211a6c. Causes a crash when using Blinkenlights UI with -l CLI option. $ ./offlineimap.py -c offlineimap.conf.minimal -u blinkenlights -l foo Traceback (most recent call last): File "./offlineimap.py", line 36, in oi.run() File "/tmp/offlineimap/offlineimap/init.py", line 50, in run options, args = self.__parse_cmd_options() File "/tmp/offlineimap/offlineimap/init.py", line 205, in __parse_cmd_options self.ui.setlogfile(options.logfile) File "/tmp/offlineimap/offlineimap/ui/UIBase.py", line 119, in setlogfile self.logger.info(msg) File "/usr/lib/python2.7/logging/__init__.py", line 1159, in info self._log(INFO, msg, args, **kwargs) File "/usr/lib/python2.7/logging/__init__.py", line 1278, in _log self.handle(record) File "/usr/lib/python2.7/logging/__init__.py", line 1288, in handle self.callHandlers(record) File "/usr/lib/python2.7/logging/__init__.py", line 1328, in callHandlers hdlr.handle(record) File "/usr/lib/python2.7/logging/__init__.py", line 751, in handle self.emit(record) File "/tmp/offlineimap/offlineimap/ui/Curses.py", line 305, in emit color = self.ui.gettf().curses_color AttributeError: 'CursesLogHandler' object has no attribute 'ui' Reported-by: iliastsi Github-issue: #293 Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/UIBase.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index bc53ae6..427c8c7 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -107,6 +107,7 @@ class UIBase(object): """Create file handler which logs to file.""" fh = logging.FileHandler(logfile, 'at') + #fh.setLevel(logging.DEBUG) file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " "%(message)s", '%Y-%m-%d %H:%M:%S') fh.setFormatter(file_formatter) @@ -116,7 +117,9 @@ class UIBase(object): msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ "Args: %s"% (offlineimap.__bigversion__, p_ver, sys.platform, " ".join(sys.argv)) - self.logger.info(msg) + record = logging.LogRecord('OfflineImap', logging.INFO, __file__, + None, msg, None, None) + fh.emit(record) def _msg(self, msg): """Display a message.""" From 94fbe8773d703233ef6db02c7d890e43303a5750 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 24 Jan 2016 19:14:12 +0100 Subject: [PATCH 066/100] declare IMAP Keywords option stable Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 5970bdf..4875a56 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -555,8 +555,6 @@ localfolders = ~/Test # # Comparison in offlineimap is case-sensitive. # -# This option is EXPERIMENTAL. -# #customflag_a = some_keyword #customflag_b = $OtherKeyword #customflag_c = NonJunk From 64763b0645c5a71343075afe581da83db153a992 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 24 Jan 2016 19:15:23 +0100 Subject: [PATCH 067/100] declare tls_level option stable Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 4875a56..d7ed8f6 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -709,8 +709,6 @@ remotehost = examplehost # Supported values are: # tls_secure, tls_no_ssl, tls_compat (the default). # -# This option is EXPERIMENTAL. -# #tls_level = tls_compat From 65d2c6938abde4a056014bacd6d1a302143973ed Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 24 Jan 2016 19:16:25 +0100 Subject: [PATCH 068/100] declare XOAUTH2 stable Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index d7ed8f6..56d393c 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -761,8 +761,8 @@ remoteuser = username # # XOAuth2 authentication (for instance, to use with Gmail). # -# This feature is currently EXPERIMENTAL (tested on Gmail only, but should work -# with type = IMAP for compatible servers). +# This option was tested on Gmail only, but should work +# with type = IMAP for compatible servers. # # Mandatory parameters are "oauth2_client_id", "oauth2_client_secret" and # either "oauth2_refresh_token" or "oauth2_access_token". From e5580ddba9e67a90ded3346761f74507d7c14ab0 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 24 Jan 2016 19:21:33 +0100 Subject: [PATCH 069/100] decode foldernames is removed EXPERIMENTAL flag Introduce warnings because I'm not using this feature and encoding issues might be a nightmare to fix. I'm not ready to support this. Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 56d393c..91dfab7 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -889,10 +889,9 @@ remoteuser = username # Note that the IMAP 4rev1 specification (RFC 3501) allows both UTF-8 and # modified UTF-7 folder names. # -# This option is disabled by default to retain compatibility with older versions -# of offlineimap. -# -# This option is EXPERIMENTAL. +# WARNING: with this option enabled: +# - compatibility with any other version is NOT GUARANTED (including newer); +# - no support is provided. # #decodefoldernames = no From 70708e2b6f2705f8e25e4390ee085f2c52137e6c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 24 Jan 2016 19:24:07 +0100 Subject: [PATCH 070/100] declare utime_from_header option stable Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap.conf b/offlineimap.conf index 91dfab7..fdd8cb1 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -507,7 +507,6 @@ localfolders = ~/Test # file/message content. # # If enabled, this forbid the -q (quick mode) CLI option to work correctly. -# This option is still "TESTING" feature. # # Default: no. # From d5bd6dd5dd0fcf9e18a138580e70d678615ffa66 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 24 Jan 2016 19:31:26 +0100 Subject: [PATCH 071/100] declare newmail_hook option stable Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index fdd8cb1..b9c0991 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -302,8 +302,6 @@ remoterepository = RemoteExample # # This example plays a sound file of your chosing when new mail arrives. # -# This feature is experimental. -# #newmail_hook = lambda: os.system("cvlc --play-and-stop --play-and-exit /path/to/sound/file.mp3" + # " > /dev/null 2>&1") From 426905ba8517b9ce7e03a0b9d7c36bc9baedeafb Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 24 Jan 2016 19:47:14 +0100 Subject: [PATCH 072/100] v6.7.0-rc1 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 32 ++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 4 ++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index a2cad23..2e32c00 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,38 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} + +### OfflineIMAP v6.7.0-rc1 (2016-01-24) + +#### Notes + +Starting a new cycle with all EXPERIMENTAL and TESTING stuff marked stable. +Otherwise, not much exciting yet. There's pending work that would need some +love by contributors: + +- https://github.com/OfflineIMAP/offlineimap/issues/211 +- https://github.com/OfflineIMAP/offlineimap/pull/111 +- https://github.com/OfflineIMAP/offlineimap/issues/184 + +#### Features + +- Allow authorization via XOAUTH2 using access token. + +#### Fixes + +- Revert "Don't output initial blurb in "quiet" mode". +- Fix Changelog. + +#### Changes + +- Declare newmail_hook option stable. +- Declare utime_from_header option stable. +- Decode foldernames is removed EXPERIMENTAL flag. +- Declare XOAUTH2 stable. +- Declare tls_level option stable. +- Declare IMAP Keywords option stable. + + ### OfflineIMAP v6.6.1 (2015-12-28) #### Notes diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 758c9cb..a3ec482 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,8 +1,8 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.6.1" -__revision__ = "" +__version__ = "6.7.0" +__revision__ = "-rc1" # Expecting "-rcN" or "" for stable releases. __bigversion__ = __version__ + __revision__ __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" From 3174ea0bd30c7dcd4915cce52a966f4bc56af275 Mon Sep 17 00:00:00 2001 From: Ilias Tsitsimpis Date: Thu, 4 Feb 2016 12:29:49 +0200 Subject: [PATCH 073/100] Fix typos in offlineimap(1) manpage Signed-off-by: Ilias Tsitsimpis Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index a2da7f5..16f8eba 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -77,7 +77,7 @@ amounts of data. This option implies the -1 option. Overrides the accounts section in the config file. + -Allows to specify a particular account or set of accounts to sync without +Allows one to specify a particular account or set of accounts to sync without having to edit the config file. @@ -211,7 +211,7 @@ in between. 5. Turn off fsync. + In the [general] section you can set fsync to True or False. If you want to -play 110% safe and wait for all operations to hit the disk before continueing, +play 110% safe and wait for all operations to hit the disk before continuing, you can set this to True. If you set it to False, you lose some of that safety, trading it for speed. @@ -382,7 +382,7 @@ You should enable this option with a value like 10. * OfflineIMAP confused when mails change while in a sync. + -When OfflineIMAP is syncing, some events happening since the invokation on +When OfflineIMAP is syncing, some events happening since the invocation on remote or local side are badly handled. OfflineIMAP won't track for changes during the sync. From 37bcd45e4ef53804b11a9a30e48135d5bdeffd1f Mon Sep 17 00:00:00 2001 From: Ilias Tsitsimpis Date: Thu, 4 Feb 2016 12:31:29 +0200 Subject: [PATCH 074/100] Abort after three Ctrl-C keystrokes By default, OfflineIMAP catches SIGTERM/SIGHUP/SIGINT and attempts to gracefully terminate as soon as possible. Allow the user to abort immediately, by hitting Ctrl-C several times. Bug-Debian: https://bugs.debian.org/679975 Signed-off-by: Ilias Tsitsimpis Signed-off-by: Nicolas Sebrecht --- offlineimap/init.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/offlineimap/init.py b/offlineimap/init.py index beb90de..58b6880 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -362,10 +362,17 @@ class OfflineImap: accounts.Account.set_abort_event(self.config, 3) if 'thread' in self.ui.debuglist: self.__dumpstacks(5) + + # Abort after three Ctrl-C keystrokes + self.num_sigterm += 1 + if self.num_sigterm >= 3: + getglobalui().warn("Signaled thrice. Aborting!") + sys.exit(1) elif sig == signal.SIGQUIT: stacktrace.dump(sys.stderr) os.abort() + self.num_sigterm = 0 signal.signal(signal.SIGHUP, sig_handler) signal.signal(signal.SIGUSR1, sig_handler) signal.signal(signal.SIGUSR2, sig_handler) From 015ec2ada45439eb4742d46086b06e04559734f7 Mon Sep 17 00:00:00 2001 From: "Aaron C. de Bruyn" Date: Wed, 10 Feb 2016 09:48:15 -0800 Subject: [PATCH 075/100] manual: small grammar fix Sent-by: Aaron C. de Bruyn Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 16f8eba..3ee58ca 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -219,7 +219,7 @@ safety, trading it for speed. Upgrading from plain text to SQLite cache format ------------------------------------------------ -OfflineImap uses a cache to store the last know status of mails (flags etc). +OfflineImap uses a cache to store the last known status of mails (flags etc). Historically that has meant plain text files, but recently we introduced sqlite-based cache, which helps with performance and CPU usage on large From b670bb2022b10873dffbdb7700e9fcdf8082fcdc Mon Sep 17 00:00:00 2001 From: Ray Song Date: Mon, 15 Feb 2016 16:16:26 +0800 Subject: [PATCH 076/100] XOAUTH2 handler: urlopen with proxied socket Signed-off-by: Ray Song Signed-off-by: Nicolas Sebrecht --- offlineimap/imapserver.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index 018ba7b..cbe8d23 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -227,7 +227,13 @@ class IMAPServer: self.ui.debug('imap', 'xoauth2handler: url "%s"' % self.oauth2_request_url) self.ui.debug('imap', 'xoauth2handler: params "%s"' % params) - response = urllib.urlopen(self.oauth2_request_url, urllib.urlencode(params)).read() + original_socket = socket.socket + socket.socket = self.proxied_socket + try: + response = urllib.urlopen(self.oauth2_request_url, urllib.urlencode(params)).read() + finally: + socket.socket = original_socket + resp = json.loads(response) self.ui.debug('imap', 'xoauth2handler: response "%s"' % resp) self.oauth2_access_token = resp['access_token'] From 36165e391f6f2a02d6114a3e714c7410da6fb551 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 19 Feb 2016 12:10:23 +0100 Subject: [PATCH 077/100] update links to the new URL www.offlineimap.org Signed-off-by: Nicolas Sebrecht --- CONTRIBUTING.rst | 10 +++++----- README.md | 4 ++-- docs/doc-src/index.rst | 2 +- docs/offlineimap.txt | 2 +- offlineimap.conf | 4 ++-- offlineimap/__init__.py | 2 +- test/OLItest/__init__.py | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 66a911c..ac0beff 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -7,11 +7,11 @@ .. _maintainers: https://github.com/OfflineIMAP/offlineimap/blob/next/MAINTAINERS.rst .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst -.. _Community's website: http://offlineimap.org -.. _APIs in OfflineIMAP: http://offlineimap.org/documentation.html#available-apis -.. _documentation: http://offlineimap.org/documentation.html -.. _Coding Guidelines: http://offlineimap.org/doc/CodingGuidelines.html -.. _Know the status of your patches: http://offlineimap.org/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission +.. _Community's website: http://www.offlineimap.org +.. _APIs in OfflineIMAP: http://www.offlineimap.org/documentation.html#available-apis +.. _documentation: http://www.offlineimap.org/documentation.html +.. _Coding Guidelines: http://www.offlineimap.org/doc/CodingGuidelines.html +.. _Know the status of your patches: http://www.offlineimap.org/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission ================= diff --git a/README.md b/README.md index 3f04a99..056d8b8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [offlineimap]: http://github.com/OfflineIMAP/offlineimap -[website]: http://offlineimap.org +[website]: http://www.offlineimap.org [wiki]: http://github.com/OfflineIMAP/offlineimap/wiki -[blog]: http://offlineimap.org/posts.html +[blog]: http://www.offlineimap.org/posts.html # OfflineIMAP diff --git a/docs/doc-src/index.rst b/docs/doc-src/index.rst index e923a64..5915936 100644 --- a/docs/doc-src/index.rst +++ b/docs/doc-src/index.rst @@ -1,5 +1,5 @@ .. OfflineImap documentation master file -.. _OfflineIMAP: http://offlineimap.org +.. _OfflineIMAP: http://www.offlineimap.org Welcome to OfflineIMAP's developer documentation diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 3ee58ca..33451cc 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -428,4 +428,4 @@ See Also -------- offlineimapui(7), openssl(1), signal(7), sqlite3(1). - http://offlineimap.org + http://www.offlineimap.org diff --git a/offlineimap.conf b/offlineimap.conf index b9c0991..df5ed95 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -2,7 +2,7 @@ # This file documents *all* possible options and can be quite scary. # Looking for a quick start? Take a look at offlineimap.conf.minimal. -# More details can be found at http://offlineimap.org . +# More details can be found at http://www.offlineimap.org . ################################################## # Overview @@ -988,7 +988,7 @@ remoteuser = username # # If you enable nametrans, you will likely need to set the reversed nametrans on # the other side. See the user documentation for details and use cases. They -# are also online at: http://offlineimap.org/doc/nametrans.html +# are also online at: http://www.offlineimap.org/doc/nametrans.html # # This example below will remove "INBOX." from the leading edge of folders # (great for Courier IMAP users). diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index a3ec482..3e5ed4d 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -11,7 +11,7 @@ __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Suppo __license__ = "Licensed under the GNU GPL v2 or any later version (with an OpenSSL exception)" __bigcopyright__ = """%(__productname__)s %(__bigversion__)s %(__license__)s""" % locals() -__homepage__ = "http://offlineimap.org" +__homepage__ = "http://www.offlineimap.org" banner = __bigcopyright__ diff --git a/test/OLItest/__init__.py b/test/OLItest/__init__.py index ca6ef61..e6dc341 100644 --- a/test/OLItest/__init__.py +++ b/test/OLItest/__init__.py @@ -24,7 +24,7 @@ __author__ = 'Sebastian Spaeth' __author_email__= 'Sebastian@SSpaeth.de' __description__ = 'Moo' __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" -__homepage__ = "http://offlineimap.org" +__homepage__ = "http://www.offlineimap.org" banner = """%(__productname__)s %(__version__)s %(__license__)s""" % locals() From 36375daee67e8dd1a8256abec6798618f4e05061 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 19 Feb 2016 12:33:57 +0100 Subject: [PATCH 078/100] fix: exceptions.OSError might not have attribute EEXIST defined Since this is used in an except calse, we first don't mask the real cause and raise the original error. Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 64570a1..7d188b5 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -292,7 +292,8 @@ class MaildirFolder(BaseFolder): that was created.""" tmpname = os.path.join('tmp', filename) - # open file and write it out + # Open file and write it out. + # XXX: why do we need to loop 7 times? tries = 7 while tries: tries = tries - 1 @@ -301,6 +302,8 @@ class MaildirFolder(BaseFolder): os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666) break except OSError as e: + if not hasattr(e, 'EEXIST'): + raise if e.errno == e.EEXIST: if tries: time.sleep(0.23) From 281bcefb52fde0e8a714c65bff54b1a5a839b0e9 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 22 Feb 2016 12:14:49 +0100 Subject: [PATCH 079/100] versioning: avoid confusing pip by spliting out __version__ with __revision__ Python tools are not used to a __revision__. Signed-off-by: Nicolas Sebrecht --- offlineimap/__init__.py | 6 ++---- offlineimap/init.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 3e5ed4d..af4c569 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,15 +1,13 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.7.0" -__revision__ = "-rc1" # Expecting "-rcN" or "" for stable releases. -__bigversion__ = __version__ + __revision__ +__version__ = "6.7.0-rc1" # Expecting "-rcN" or "" for stable releases. __copyright__ = "Copyright 2002-2015 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "offlineimap-project@lists.alioth.debian.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" __license__ = "Licensed under the GNU GPL v2 or any later version (with an OpenSSL exception)" -__bigcopyright__ = """%(__productname__)s %(__bigversion__)s +__bigcopyright__ = """%(__productname__)s %(__version__)s %(__license__)s""" % locals() __homepage__ = "http://www.offlineimap.org" diff --git a/offlineimap/init.py b/offlineimap/init.py index 58b6880..7ed3b6b 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -54,7 +54,7 @@ class OfflineImap: return self.__sync(options) def __parse_cmd_options(self): - parser = OptionParser(version=offlineimap.__bigversion__, + parser = OptionParser(version=offlineimap.__version__, description="%s.\n\n%s" % (offlineimap.__copyright__, offlineimap.__license__)) From 9fe337d4fb77fd5c8351c51ef166c05de4107bcd Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 22 Feb 2016 12:24:15 +0100 Subject: [PATCH 080/100] fix year of copyright Signed-off-by: Nicolas Sebrecht --- offlineimap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index af4c569..12479ec 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -2,7 +2,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' __version__ = "6.7.0-rc1" # Expecting "-rcN" or "" for stable releases. -__copyright__ = "Copyright 2002-2015 John Goerzen & contributors" +__copyright__ = "Copyright 2002-2016 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "offlineimap-project@lists.alioth.debian.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" From 893a90c291962c6a19b5214c0e6f4f7bf2b0cfd8 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 22 Feb 2016 12:47:42 +0100 Subject: [PATCH 081/100] v6.7.0-rc2 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 33 +++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 2e32c00..47329e0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -16,6 +16,39 @@ Note to mainainers: {:toc} +### OfflineIMAP v6.7.0-rc2 (2016-02-22) + +#### Notes + +Learn to abruptly abort on multiple Ctrl+C. + +Some bugs got fixed. XOAUTH2 now honors the proxy configuration option. Error +message was improved when it fails to write a new mail in a local Maildir. + +enabled the hook for integration with Github. You'll get notifications on +updates of the master branch of the repository (mostly for new releases). I may +write some tweets about OfflineIMAP sometimes. + +#### Features + +- Abort after three Ctrl-C keystrokes. + +#### Fixes + +- Fix year of copyright. +- Versioning: avoid confusing pip by spliting out __version__ with __revision__. +- Fix: exceptions.OSError might not have attribute EEXIST defined. +- XOAUTH2 handler: urlopen with proxied socket. +- Manual: small grammar fix. +- Fix typos in offlineimap(1) manpage. + +#### Changes + +- Update links to the new URL www.offlineimap.org. + + + + ### OfflineIMAP v6.7.0-rc1 (2016-01-24) #### Notes diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 12479ec..45a7677 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.7.0-rc1" # Expecting "-rcN" or "" for stable releases. +__version__ = "6.7.0-rc2" # Expecting "-rcN" or "" for stable releases. __copyright__ = "Copyright 2002-2016 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "offlineimap-project@lists.alioth.debian.org" From b0cf523ba7db1259e4d9dff7a8bc18ad3de36547 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 22 Feb 2016 12:48:18 +0100 Subject: [PATCH 082/100] Changelog: fix typo Signed-off-by: Nicolas Sebrecht --- Changelog.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Changelog.md b/Changelog.md index 47329e0..2ac31b3 100644 --- a/Changelog.md +++ b/Changelog.md @@ -25,7 +25,7 @@ Learn to abruptly abort on multiple Ctrl+C. Some bugs got fixed. XOAUTH2 now honors the proxy configuration option. Error message was improved when it fails to write a new mail in a local Maildir. -enabled the hook for integration with Github. You'll get notifications on +I've enabled the hook for integration with Github. You'll get notifications on updates of the master branch of the repository (mostly for new releases). I may write some tweets about OfflineIMAP sometimes. @@ -47,8 +47,6 @@ write some tweets about OfflineIMAP sometimes. - Update links to the new URL www.offlineimap.org. - - ### OfflineIMAP v6.7.0-rc1 (2016-01-24) #### Notes From 8f94ef1973d798247d42e4281e945fd95d8873ba Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 22 Feb 2016 13:04:16 +0100 Subject: [PATCH 083/100] release: add command to install from pip in the announces Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/release.sh b/contrib/release.sh index a9d75d7..1b2f6d0 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -353,6 +353,9 @@ OfflineIMAP $1 is out. Downloads: http://github.com/OfflineIMAP/offlineimap/archive/${1}.tar.gz http://github.com/OfflineIMAP/offlineimap/archive/${1}.zip + +Pip: + pip install --user git+https://github.com/OfflineIMAP/offlineimap.git@${1} EOF } From 9bfd610230b027c1f8700e2650dba9c9c26fa57c Mon Sep 17 00:00:00 2001 From: Ebben Aries Date: Mon, 22 Feb 2016 17:45:18 -0700 Subject: [PATCH 084/100] change hard coding of AF_UNSPEC to user-defined address-families per repository Some environments that return AAAA records for their IMAP servers can pose problems for clients that do not have end-to-end IPv6 connectivity for a number of reasons (e.g. policy, lack of full routing, security, etc..) Even with a fallback mechanism in place, you can still arrive at IMAP implementations that could prevent authentication from unknown IPv6 space. This in itself is not enough to fallback to IPv4 since there is an actual connection on that socket. This change is for introducing a user-defined value: [Repository imap-remote] ipv6 = no to create a preference per repository on which AF to connect to the remote server on ipv6 = yes (AF_INET6) ipv6 = no (AF_INET) unspecified = default Signed-off-by: Ebben Aries Signed-off-by: Nicolas Sebrecht --- offlineimap/imaplibutil.py | 8 +++++++- offlineimap/imapserver.py | 10 ++++++++++ offlineimap/repository/IMAP.py | 3 +++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index 952404a..9b1095b 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -74,7 +74,7 @@ class UsefulIMAPMixIn(object): """open_socket() Open socket choosing first address family available.""" msg = (-1, 'could not open socket') - for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): + for res in socket.getaddrinfo(self.host, self.port, self.af, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: # use socket of our own, possiblly socksified socket. @@ -175,6 +175,9 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): """Improved version of imaplib.IMAP4_SSL overriding select().""" def __init__(self, *args, **kwargs): + if "af" in kwargs: + self.af = kwargs['af'] + del kwargs['af'] if "use_socket" in kwargs: self.socket = kwargs['use_socket'] del kwargs['use_socket'] @@ -209,6 +212,9 @@ class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): """Improved version of imaplib.IMAP4 overriding select().""" def __init__(self, *args, **kwargs): + if "af" in kwargs: + self.af = kwargs['af'] + del kwargs['af'] if "use_socket" in kwargs: self.socket = kwargs['use_socket'] del kwargs['use_socket'] diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py index cbe8d23..a604993 100644 --- a/offlineimap/imapserver.py +++ b/offlineimap/imapserver.py @@ -23,6 +23,7 @@ import base64 import json import urllib +import socket import time import errno from sys import exc_info @@ -80,6 +81,13 @@ class IMAPServer: self.goodpassword = None self.usessl = repos.getssl() + self.useipv6 = repos.getipv6() + if self.useipv6 == True: + self.af = socket.AF_INET6 + elif self.useipv6 == False: + self.af = socket.AF_INET + else: + self.af = socket.AF_UNSPEC self.hostname = \ None if self.preauth_tunnel else repos.gethost() self.port = repos.getport() @@ -487,6 +495,7 @@ class IMAPServer: fingerprint=self.fingerprint, use_socket=self.proxied_socket, tls_level=self.tlslevel, + af=self.af, ) else: self.ui.connecting(self.hostname, self.port) @@ -494,6 +503,7 @@ class IMAPServer: self.hostname, self.port, timeout=socket.getdefaulttimeout(), use_socket=self.proxied_socket, + af=self.af, ) if not self.preauth_tunnel: diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 7a2a98c..60d5a08 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -194,6 +194,9 @@ class IMAPRepository(BaseRepository): return self.getconfint('remoteport', None) + def getipv6(self): + return self.getconfboolean('ipv6', None) + def getssl(self): return self.getconfboolean('ssl', 1) From e51b82801348da73bb2183690dc3f12ab4e42448 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 23 Feb 2016 07:06:47 +0100 Subject: [PATCH 085/100] add documentation for the ipv6 configuration option Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index df5ed95..6f05dcc 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -580,6 +580,18 @@ type = GmailMaildir type = IMAP +# This option stands in the [Repository RemoteExample] section. +# +# Configure which address family to use for the connection. If not specified, +# AF_UNSPEC is used as a fallback (default). +# +# AF_INET6: +#ipv6 = True +# +# AF_INET: +#ipv6 = False + + # These options stands in the [Repository RemoteExample] section. # # The following can fetch the account credentials via a python expression that From 07b6c895c59ebe7eb88589098372d510544d3ee5 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Tue, 23 Feb 2016 11:11:38 +0100 Subject: [PATCH 086/100] release.sh: improve release annoucements Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index 1b2f6d0..a0756bd 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -154,7 +154,19 @@ function update_offlineimap_version () { # function get_git_history () { debug 'in get_git_history' - git log --oneline "${1}.." | sed -r -e 's,^(.),\- \1,' + git log --format='- %h %s. [%aN]' --no-merges "${1}.." | \ + sed -r -e 's, \[Nicolas Sebrecht\]$,,' +} + + +# +# $1: previous version +# +function get_git_who () { + debug 'in get_git_who' + echo + git shortlog --no-merges -sn "${1}.." | \ + sed -r -e 's, +([0-9]+)\t(.*),- \2 (\1),' } @@ -178,8 +190,9 @@ function changelog_template () { #### Notes -// Add some notes. Good notes are about what was done in this release. -// HINT: explain big changes. +// Add some notes. Good notes are about what was done in this release from the +// bigger perspective. +// HINT: explain most important changes. #### Features @@ -193,8 +206,12 @@ function changelog_template () { // Use list syntax with '- ' -// The preformatted shortlog was added below. -// Make use of this to fill the sections 'Features' and 'Fixes' above. +#### Authors + +// Use list syntax with '- ' + +// The preformatted log was added below. Make use of this to fill the sections +// above. EOF } @@ -213,6 +230,7 @@ function update_changelog () { then changelog_template "$1" > "$TMP_CHANGELOG_EXCERPT" get_git_history "$2" >> "$TMP_CHANGELOG_EXCERPT" + get_git_who "$2" >> "$TMP_CHANGELOG_EXCERPT" edit_file "the Changelog excerpt" $TMP_CHANGELOG_EXCERPT # Remove comments. From 543890e2575f8ddd3f4a965c4951f0c2c5acfac0 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 27 Feb 2016 05:25:09 +0100 Subject: [PATCH 087/100] MANIFEST: exclude rfcs Signed-off-by: Nicolas Sebrecht --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 3a403a3..aac4358 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,9 @@ include Makefile include README.md include offlineimap.conf* include offlineimap.py +recursive-include contrib * recursive-include offlineimap *.py recursive-include bin * recursive-include docs * recursive-include test * +prune docs/rfcs From c549af78eb89468f8321efd25febe99490888c52 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 27 Feb 2016 05:34:46 +0100 Subject: [PATCH 088/100] pypi requires a setup.cfg when README is Markdown Signed-off-by: Nicolas Sebrecht --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..9e99ac2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ + +[metadata] +description-file = README.md From f0a585b02f6efd4fa912deb01075018c10aef531 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 27 Feb 2016 05:35:09 +0100 Subject: [PATCH 089/100] release.sh: add pypi instructions Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/release.sh b/contrib/release.sh index a0756bd..b378b72 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -455,6 +455,7 @@ Command samples to do manually: - git push master:master - git push next:next - git push $new_version +- python setup.py sdist && twine upload dist/* && rm -rf dist MANIFEST - cd website - git checkout master - git merge $branch_name From 8b661eb58e8ebcb54cc8e6a51522aa5833febd31 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 27 Feb 2016 20:10:34 +0100 Subject: [PATCH 090/100] MAINTAINERS: update Signed-off-by: Nicolas Sebrecht --- MAINTAINERS.rst | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/MAINTAINERS.rst b/MAINTAINERS.rst index 2cff161..3341287 100644 --- a/MAINTAINERS.rst +++ b/MAINTAINERS.rst @@ -1,7 +1,7 @@ .. -*- coding: utf-8 -*- -Official maintainers -==================== +Maintainers +=========== Eygene Ryabinkin email: rea at freebsd.org @@ -15,15 +15,31 @@ Nicolas Sebrecht email: nicolas.s-dev at laposte.net github: nicolas33 -Mailing List maintainers -======================== -Eygene Ryabinkin - email: rea at freebsd.org +Github +------ -Sebastian Spaeth - email: sebastian at sspaeth.de +- Eygene Ryabinkin +- Sebastian Spaeth +- Nicolas Sebrecht -Nicolas Sebrecht - email: nicolas.s-dev at laposte.net +Mailing List +------------ + +- Eygene Ryabinkin +- Sebastian Spaeth +- Nicolas Sebrecht + + +Twitter +------- + +- Nicolas Sebrecht + + +Pypi +---- + +- Nicolas Sebrecht +- Sebastian Spaeth From 90afb1b338e29d64cdc534d88414620b6da1634c Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 28 Feb 2016 18:59:10 +0100 Subject: [PATCH 091/100] add github templates Signed-off-by: Nicolas Sebrecht --- .github/ISSUE_TEMPLATE.md | 18 ++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 13 +++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..54779e3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,18 @@ + +### General informations + +- OfflineIMAP version: +- server name or domain: +- CLI options: +- configuration (remove private data): +``` + +``` + +### Log error + +``` + +``` + +### Step to reproduce the error diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..ed6face --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ + +> Add character x. + +- [] I've read the [DCO](http://www.offlineimap.org/doc/dco.html). +- [] I've read the [Coding Guidelines](http://www.offlineimap.org/doc/CodingGuidelines.html) +- [] The relevant informations about the changes stands in the commit message, not here. +- [] Code changes follow the style of the files they change. +- [] Code is tested (provide details). + +### References + +- Bug: #no_space + From 06881f9a717fd73d261356a4ecdf7e53098f1051 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 2 Mar 2016 15:36:22 +0100 Subject: [PATCH 092/100] README: add slogan Thanks-to: Norbert Preining Signed-off-by: Nicolas Sebrecht --- README.md | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 056d8b8..1dc84e0 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ # OfflineIMAP +***Get the emails where you need them.*** + ## Description OfflineIMAP is a software to dispose your e-mail mailbox(es) as a **local @@ -20,6 +22,27 @@ reader (MUA) to support IMAP disconnected operations. Need an attachment from a message without internet connection? It's fine, the message is still there. +## Project status and future + +> As one of the maintainer of OfflineIMAP, I'd like to put my efforts into +> [imapfw](http://github.com/OfflineIMAP/imapfw). **imapfw** is a software in +> development that I intend to replace OfflineIMAP in the long term. +> +> That's why I'm not going to do development in OfflineIMAP. I continue to do +> the maintenance job in OfflineIMAP: fixing small bugs, (quick) +> reviewing/merging patches and rolling out new releases, but that's all. +> +> While I keep tracking issues for OfflineIMAP, you should not expect support +> much from me anymore. +> +> You won't be left at the side. OfflineIMAP's community is large enough so that +> you'll find people for most of your issues. +> +> Get news from the [blog][blog]. +> +> Nicolas Sebrecht. ,-) + + ## License GNU General Public License v2. @@ -32,27 +55,6 @@ GNU General Public License v2. * It is **flexible**. * It is **safe**. - -## Project status and future - -> As one of the maintainer of OfflineIMAP, I'd like to put my efforts into -> [imapfw](http://github.com/OfflineIMAP/imapfw). **imapfw** is a software in -> development that I intend to replace OfflineIMAP in the long term. -> -> That's why I'm not going to do development in OfflineIMAP. I continue to do -> the minimal maintenance job in OfflineIMAP: fixing small bugs, (quick) -> reviewing/merging patches and rolling out new releases, but that's all. -> -> While I keep tracking issues for OfflineIMAP, you should not expect support -> from me anymore. -> -> You won't be left at the side. OfflineIMAP's community is large enough so that -> you'll find people for most of your issues. -> -> Get news from the [blog][blog]. -> -> Nicolas Sebrecht. ,-) - ## Downloads You should first check if your distribution already packages OfflineIMAP for you. From 29a7dbd51be635e93cb5af77259e0770c9e39957 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 4 Mar 2016 22:31:39 +0100 Subject: [PATCH 093/100] sphinx doc: remove usage of __bigversion__ Fix regression introduced by 281bcef. Reported-by: mathstuf Signed-off-by: Nicolas Sebrecht --- docs/doc-src/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doc-src/conf.py b/docs/doc-src/conf.py index 0886d4a..95cfb51 100644 --- a/docs/doc-src/conf.py +++ b/docs/doc-src/conf.py @@ -18,7 +18,7 @@ import sys, os # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('../..')) -from offlineimap import __version__, __bigversion__, __author__, __copyright__ +from offlineimap import __version__, __author__, __copyright__ # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions @@ -50,7 +50,7 @@ copyright = __copyright__ # The short X.Y version. version = __version__ # The full version, including alpha/beta/rc tags. -release = __bigversion__ +release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From d8398ba374e1ea7881125504500cbdeca3991963 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 4 Mar 2016 18:58:59 -0500 Subject: [PATCH 094/100] Curses, UIBase: remove references to __bigversion__ __bigversion__ was removed in 281bcefb52fde0e8a714c65bff54b1a5a839b0e9. Signed-off-by: Ben Boeckel Signed-off-by: Nicolas Sebrecht --- offlineimap/ui/Curses.py | 2 +- offlineimap/ui/UIBase.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index ddc05ea..d5b148d 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -603,7 +603,7 @@ class Blinkenlights(UIBase, CursesUtil): self.bannerwin.clear() # Delete old content (eg before resizes) self.bannerwin.bkgd(' ', color) # Fill background with that color string = "%s %s"% (offlineimap.__productname__, - offlineimap.__bigversion__) + offlineimap.__version__) self.bannerwin.addstr(0, 0, string, color) self.bannerwin.addstr(0, self.width -len(offlineimap.__copyright__) -1, offlineimap.__copyright__, color) diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 427c8c7..62e42d1 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -115,7 +115,7 @@ class UIBase(object): # write out more verbose initial info blurb on the log file p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ - "Args: %s"% (offlineimap.__bigversion__, p_ver, sys.platform, + "Args: %s"% (offlineimap.__version__, p_ver, sys.platform, " ".join(sys.argv)) record = logging.LogRecord('OfflineImap', logging.INFO, __file__, None, msg, None, None) @@ -445,7 +445,7 @@ class UIBase(object): #TODO: Debug and make below working, it hangs Gmail #res_type, response = conn.id(( # 'name', offlineimap.__productname__, - # 'version', offlineimap.__bigversion__)) + # 'version', offlineimap.__version__)) #self._msg("Server ID: %s %s" % (res_type, response[0])) self._msg("Server welcome string: %s" % str(conn.welcome)) self._msg("Server capabilities: %s\n" % str(conn.capabilities)) From 2f541e4872375ac6580fa2a23d623cf6c1f294aa Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sat, 5 Mar 2016 14:36:54 +0100 Subject: [PATCH 095/100] release.sh: move the authors section up Signed-off-by: Nicolas Sebrecht --- contrib/release.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/release.sh b/contrib/release.sh index b378b72..14d80f8 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -29,6 +29,7 @@ CHANGELOG='Changelog.md' CACHEDIR='.git/offlineimap-release' WEBSITE='website' WEBSITE_LATEST="${WEBSITE}/_data/latest.yml" +ME='Nicolas Sebrecht' TMP_CHANGELOG_EXCERPT="${CACHEDIR}/changelog.excerpt.md" TMP_CHANGELOG_EXCERPT_OLD="${TMP_CHANGELOG_EXCERPT}.old" @@ -155,7 +156,7 @@ function update_offlineimap_version () { function get_git_history () { debug 'in get_git_history' git log --format='- %h %s. [%aN]' --no-merges "${1}.." | \ - sed -r -e 's, \[Nicolas Sebrecht\]$,,' + sed -r -e "s, \[${ME}\]$,," } @@ -194,6 +195,12 @@ function changelog_template () { // bigger perspective. // HINT: explain most important changes. +#### Authors + +The authors of this release. + +// Use list syntax with '- ' + #### Features // Use list syntax with '- ' @@ -206,10 +213,6 @@ function changelog_template () { // Use list syntax with '- ' -#### Authors - -// Use list syntax with '- ' - // The preformatted log was added below. Make use of this to fill the sections // above. From 334571038e41eb78f8d07199de0eb942a4109d0e Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 6 Mar 2016 21:01:13 +0100 Subject: [PATCH 096/100] minor improvement of github issue template Signed-off-by: Nicolas Sebrecht --- .github/ISSUE_TEMPLATE.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 54779e3..6fffc8c 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,15 +4,22 @@ - OfflineIMAP version: - server name or domain: - CLI options: -- configuration (remove private data): + ``` - + ``` +``` + +``` + + ### Log error ``` ``` -### Step to reproduce the error +### Steps to reproduce the error + + From c84d23b65670f3480691259a4c686e3903d2ca92 Mon Sep 17 00:00:00 2001 From: Ilias Tsitsimpis Date: Sat, 5 Mar 2016 19:07:07 +0200 Subject: [PATCH 097/100] Identify and fix messages with FMD5 inconsistencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce the '--migrate-fmd5-using-nametrans' option which migrates the FMD5 hashes from versions prior to 6.3.5. It seems that commit 'Apply nametrans to all Foldertypes' (6b2ec956cf) introduced a regression because it changed the FMD5 part of the filename calculated by OfflineIMAP. Thus, OfflineIMAP believes that the messages has been removed and adds them back. For more information, see: http://www.offlineimap.org/configuration/2016/02/12/debian-upgrade-from-jessie-to-stretch.html Bug-Debian: https://bugs.debian.org/812108 Reported-by: François Signed-off-by: Ilias Tsitsimpis Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 14 ++++++++++++++ offlineimap/folder/Maildir.py | 34 ++++++++++++++++++++++++++++++++++ offlineimap/init.py | 27 ++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 33451cc..0c66489 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -163,6 +163,20 @@ blinkenlights, machineui. This option is only applicable in non-verbose mode. +--migrate-fmd5-using-nametrans:: + Migrate FMD5 hashes from versions prior to 6.3.5. ++ +The way that FMD5 hashes are calculated was changed in version 6.3.5 (now using +the nametrans folder name) introducing a regression which may lead to +re-uploading all messages. Try and fix the above regression by calculating the +correct FMD5 values and renaming the corresponding messages. + +CAUTION: Since the FMD5 part of the filename changes, this may lead to UID +conflicts. Ensure to dispose a proper backup of both the cache and the Maildir +before running this fix as well as verify the results using the `--dry-run' +flag first. + + Synchronization Performance --------------------------- diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 7d188b5..77a774b 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -485,3 +485,37 @@ class MaildirFolder(BaseFolder): os.unlink(filepath) # Yep -- return. del(self.messagelist[uid]) + + def migratefmd5(self, dryrun=False): + """Migrate FMD5 hashes from versions prior to 6.3.5 + + :param dryrun: Run in dry run mode + :type fix: Boolean + :return: None + """ + oldfmd5 = md5(self.name).hexdigest() + msglist = self._scanfolder() + for mkey, mvalue in msglist.iteritems(): + filename = os.path.join(self.getfullname(), mvalue['filename']) + match = re.search("FMD5=([a-fA-F0-9]+)", filename) + if match is None: + self.ui.debug("maildir", + "File `%s' doesn't have an FMD5 assigned" + % filename) + elif match.group(1) == oldfmd5: + self.ui.info("Migrating file `%s' to FMD5 `%s'" + % (filename, self._foldermd5)) + if not dryrun: + newfilename = filename.replace( + "FMD5=" + match.group(1), "FMD5=" + self._foldermd5) + try: + os.rename(filename, newfilename) + except OSError as e: + raise OfflineImapError( + "Can't rename file '%s' to '%s': %s" % ( + filename, newfilename, e[1]), + OfflineImapError.ERROR.FOLDER), None, exc_info()[2] + elif match.group(1) != self._foldermd5: + self.ui.warn(("Inconsistent FMD5 for file `%s':" + " Neither `%s' nor `%s' found") + % (filename, oldfmd5, self._foldermd5)) diff --git a/offlineimap/init.py b/offlineimap/init.py index 7ed3b6b..636db5d 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -25,11 +25,12 @@ import logging from optparse import OptionParser import offlineimap -from offlineimap import accounts, threadutil, syncmaster +from offlineimap import accounts, threadutil, syncmaster, folder from offlineimap import globals from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser from offlineimap.utils import stacktrace +from offlineimap.repository import Repository import traceback import collections @@ -50,6 +51,8 @@ class OfflineImap: options, args = self.__parse_cmd_options() if options.diagnostics: self.__serverdiagnostics(options) + elif options.migrate_fmd5: + self.__migratefmd5(options) else: return self.__sync(options) @@ -120,6 +123,10 @@ class OfflineImap: help="specifies an alternative user interface" " (quiet, basic, syslog, ttyui, blinkenlights, machineui)") + parser.add_option("--migrate-fmd5-using-nametrans", + action="store_true", dest="migrate_fmd5", default=False, + help="migrate FMD5 hashes from versions prior to 6.3.5") + (options, args) = parser.parse_args() globals.set_options (options) @@ -427,3 +434,21 @@ class OfflineImap: for account in allaccounts: if account.name not in activeaccounts: continue account.serverdiagnostics() + + def __migratefmd5(self, options): + activeaccounts = self.config.get("general", "accounts") + if options.accounts: + activeaccounts = options.accounts + activeaccounts = activeaccounts.replace(" ", "") + activeaccounts = activeaccounts.split(",") + allaccounts = accounts.AccountListGenerator(self.config) + + for account in allaccounts: + if account.name not in activeaccounts: + continue + localrepo = Repository(account, 'local') + if localrepo.getfoldertype() != folder.Maildir.MaildirFolder: + continue + folders = localrepo.getfolders() + for f in folders: + f.migratefmd5(options.dryrun) From 8ed4f35fc864cbf15d9647ec63e45d9177589de1 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Mon, 7 Mar 2016 13:36:33 +0100 Subject: [PATCH 098/100] introduce a code of conduct Signed-off-by: Nicolas Sebrecht --- CODE_OF_CONDUCT.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..995a1b9 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,18 @@ + +# Realistic Code of Conduct + +1. We mostly care about making our softwares better. + +2. Everybody is free to decide how to contribute. + +3. Free speech owns to anyone of us. + +4. Feel offended? This might be very well-deserved. + +5. We don't need a code of conduct imposed on us, thanks. + +6. Ignoring this Realistic Code of Conduct is welcome. + + From 15efea24b90355f62065165189fa33a85a128ec1 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Wed, 9 Mar 2016 09:35:59 +0100 Subject: [PATCH 099/100] .github/: improve templates Signed-off-by: Nicolas Sebrecht --- .github/ISSUE_TEMPLATE.md | 9 ++++++--- .github/PULL_REQUEST_TEMPLATE.md | 22 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 6fffc8c..b819e0a 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,4 @@ +> This v1.0 template stands in `.github/`. ### General informations @@ -6,20 +7,22 @@ - CLI options: ``` - +Configuration file offlineimaprc goes here. REMOVE PRIVATE DATA. ``` ``` - +The pythonfile file goes here (if any). REMOVE PRIVATE DATA. ``` ### Log error ``` - +Logs go here. REMOVE PRIVATE DATA. ``` ### Steps to reproduce the error +- +- diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ed6face..f41dae2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,13 +1,29 @@ +> This v1.0 template stands in `.github/`. -> Add character x. +### Peer reviews + +Trick to [fetch the pull +request](https://help.github.com/articles/checking-out-pull-requests-locally): +there is a (read-only) `refs/pull/` namespace. + +``` bash +git fetch OFFICIAL_REPOSITORY_NAME pull/PULL_ID/head:LOCAL_BRANCH_NAME +``` + +### This PR + +> Add character x `[x]`. - [] I've read the [DCO](http://www.offlineimap.org/doc/dco.html). - [] I've read the [Coding Guidelines](http://www.offlineimap.org/doc/CodingGuidelines.html) -- [] The relevant informations about the changes stands in the commit message, not here. +- [] The relevant informations about the changes stands in the commit message, not here in the message of the pull request. - [] Code changes follow the style of the files they change. - [] Code is tested (provide details). ### References -- Bug: #no_space +- Issue #no_space + +### Additional information + From 8c7a7355a31ce16b6d05eba6a2d0b416ff224345 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Thu, 10 Mar 2016 17:08:18 +0100 Subject: [PATCH 100/100] v6.7.0 Signed-off-by: Nicolas Sebrecht --- Changelog.md | 72 +++++++++++++++++++++++++++++++++++++++++ offlineimap/__init__.py | 3 +- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 2ac31b3..8d49115 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,78 @@ Note to mainainers: * The following excerpt is only usefull when rendered in the website. {:toc} +### OfflineIMAP v6.7.0 (2016-03-10) + +#### Notes + +New stable release out! + +With the work of Ilias, maintainer at Debian, OfflineIMAP is learning a new CLI +option to help fixing filenames for the users using nametrans and updating from +versions prior to v6.3.5. Distribution maintainers might want to backport this +feature for their packaged versions out after v6.3.5. Have a look at commit +c84d23b65670f to know more. + +OfflineIMAP earns the slogan "Get the emails where you need them", authored by +Norbert Preining. + +Julien Danjou, the author of the book _The Hacker’s Guide To Python_, shared us +his screenshot of a running session of OfflineIMAP. + +I recently created rooms for chat sessions at Gitter. It appears to be really +cool, supports seamless authentication with a github account, persistent logs, +desktop/mobile clients and many more usefull features. Join us at Gitter! + +- https://gitter.im/OfflineIMAP/offlineimap [NEW] +- https://gitter.im/OfflineIMAP/imapfw [NEW] + +Now, the OfflineIMAP community has 2 official websites: + +- http://www.offlineimap.org (for offlineimap) +- http://imapfw.offlineimap.org (for imapfw) [NEW] + +The Twitter account was resurrected, too. Feel free to join us: + + https://twitter.com/OfflineIMAP + +Finally, the teams of the OfflineIMAP organization at Github were renewed to +facilitate the integration of new contributors and directly improve both the +documentation and the websites. + +As a side note, the [imapfw repository](https://github.com/OfflineIMAP/imapfw) +has now more than 50 stargazers. This is very encouraging. + +Thank you much everybody for your various contributions into OfflineIMAP! + +#### Authors + +- Ben Boeckel (1) +- Ebben Aries (1) +- Ilias Tsitsimpis (1) + +#### Features + +- Introduce a code of conduct. +- Add github templates. +- Change hard coding of AF_UNSPEC to user-defined address-families per repository. [Ebben Aries] +- Add documentation for the ipv6 configuration option. + +#### Fixes + +- Identify and fix messages with FMD5 inconsistencies. [Ilias Tsitsimpis] +- Curses, UIBase: remove references to __bigversion__. [Ben Boeckel] +- Sphinx doc: remove usage of __bigversion__. +- MANIFEST: exclude rfcs (used for Pypi packages). +- Changelog: fix typo. + +#### Changes + +- release.sh: move the authors section up. +- release.sh: add pypi instructions. +- MAINTAINERS: update. + + + ### OfflineIMAP v6.7.0-rc2 (2016-02-22) diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 45a7677..369a477 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,8 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.7.0-rc2" # Expecting "-rcN" or "" for stable releases. +# Expecting trailing "-rcN" or "" for stable releases. +__version__ = "6.7.0" __copyright__ = "Copyright 2002-2016 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "offlineimap-project@lists.alioth.debian.org"