2015-01-01 21:41:11 +01:00
|
|
|
# Copyright (C) 2003-2015 John Goerzen & contributors
|
2003-01-04 05:57:46 +01:00
|
|
|
#
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
2003-04-16 21:23:45 +02:00
|
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
|
|
# (at your option) any later version.
|
2003-01-04 05:57:46 +01:00
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with this program; if not, write to the Free Software
|
2006-08-12 06:15:55 +02:00
|
|
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
2003-01-04 05:57:46 +01:00
|
|
|
|
2009-01-14 07:05:00 +01:00
|
|
|
from subprocess import Popen, PIPE
|
2011-05-07 17:40:32 +02:00
|
|
|
from threading import Event
|
2003-01-04 05:57:46 +01:00
|
|
|
import os
|
maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option
1. When using maxage, local and remote messagelists are supposed to only
contain messages from at most maxage days ago. But local and remote used
different timezones to calculate what "maxage days ago" means, resulting
in removals on one side. Now, we ask the local folder for maxage days'
worth of mail, find the lowest UID, and then ask the remote folder for
all UID's starting with that lowest one.
2. maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that
remote messages have UIDs in the same order as their local counterparts,
which could be false, e.g. when messages are copied in quick succession.
So, remove support for maxage in the IMAP-IMAP case.
3. Add startdate option for IMAP-IMAP syncs: use messages from the given
repository starting at startdate, and all messages from the other
repository. In the first sync, the other repository must be empty.
4. Allow maxage to be specified either as number of days to sync (as
previously) or as a fixed date.
Signed-off-by: Janna Martl <janna.martl109@gmail.com>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2015-04-07 10:14:11 +02:00
|
|
|
import time
|
2011-08-11 12:22:35 +02:00
|
|
|
from sys import exc_info
|
2011-01-28 01:46:29 +01:00
|
|
|
import traceback
|
2008-12-02 20:12:36 +01:00
|
|
|
|
2015-04-07 16:34:35 +02:00
|
|
|
from offlineimap import mbnames, CustomConfig, OfflineImapError
|
2015-01-01 21:41:11 +01:00
|
|
|
from offlineimap import globals
|
|
|
|
from offlineimap.repository import Repository
|
|
|
|
from offlineimap.ui import getglobalui
|
|
|
|
from offlineimap.threadutil import InstanceLimitedThread
|
|
|
|
|
2011-07-08 12:23:16 +02:00
|
|
|
try:
|
|
|
|
import fcntl
|
|
|
|
except:
|
|
|
|
pass # ok if this fails, we can do without
|
|
|
|
|
2015-01-01 21:41:11 +01:00
|
|
|
# FIXME: spaghetti code alert!
|
2003-04-18 04:18:34 +02:00
|
|
|
def getaccountlist(customconfig):
|
2015-01-11 13:54:53 +01:00
|
|
|
# Account names in a list.
|
2003-04-18 04:18:34 +02:00
|
|
|
return customconfig.getsectionlist('Account')
|
|
|
|
|
2015-01-01 21:41:11 +01:00
|
|
|
# FIXME: spaghetti code alert!
|
2003-04-18 04:18:34 +02:00
|
|
|
def AccountListGenerator(customconfig):
|
2015-01-11 13:54:53 +01:00
|
|
|
"""Returns a list of instanciated Account class, one per account name."""
|
|
|
|
|
2003-04-18 04:18:34 +02:00
|
|
|
return [Account(customconfig, accountname)
|
|
|
|
for accountname in getaccountlist(customconfig)]
|
|
|
|
|
2015-01-01 21:41:11 +01:00
|
|
|
# FIXME: spaghetti code alert!
|
2003-04-18 04:18:34 +02:00
|
|
|
def AccountHashGenerator(customconfig):
|
2015-01-11 13:54:53 +01:00
|
|
|
"""Returns a dict of instanciated Account class with the account name as
|
|
|
|
key."""
|
|
|
|
|
2003-04-18 04:18:34 +02:00
|
|
|
retval = {}
|
|
|
|
for item in AccountListGenerator(customconfig):
|
|
|
|
retval[item.getname()] = item
|
|
|
|
return retval
|
|
|
|
|
2003-01-04 05:57:46 +01:00
|
|
|
|
2003-04-18 04:18:34 +02:00
|
|
|
class Account(CustomConfig.ConfigHelperMixin):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""Represents an account (ie. 2 repositories) to sync.
|
2011-05-02 17:11:40 +02:00
|
|
|
|
|
|
|
Most of the time you will actually want to use the derived
|
|
|
|
:class:`accounts.SyncableAccount` which contains all functions used
|
|
|
|
for syncing an account."""
|
2015-01-11 10:54:19 +01:00
|
|
|
|
|
|
|
# Signal gets set when we should stop looping.
|
2012-01-04 19:24:19 +01:00
|
|
|
abort_soon_signal = Event()
|
2015-01-11 10:54:19 +01:00
|
|
|
# Signal gets set on CTRL-C/SIGTERM.
|
2012-01-04 19:24:19 +01:00
|
|
|
abort_NOW_signal = Event()
|
2011-05-02 17:11:40 +02:00
|
|
|
|
2003-01-04 05:57:46 +01:00
|
|
|
def __init__(self, config, name):
|
2011-05-02 17:11:40 +02:00
|
|
|
"""
|
|
|
|
:param config: Representing the offlineimap configuration file.
|
|
|
|
:type config: :class:`offlineimap.CustomConfig.CustomConfigParser`
|
|
|
|
|
2015-01-14 22:58:25 +01:00
|
|
|
:param name: A (str) string denoting the name of the Account
|
|
|
|
as configured.
|
|
|
|
"""
|
2015-01-11 10:54:19 +01:00
|
|
|
|
2003-01-04 05:57:46 +01:00
|
|
|
self.config = config
|
|
|
|
self.name = name
|
|
|
|
self.metadatadir = config.getmetadatadir()
|
|
|
|
self.localeval = config.getlocaleval()
|
2011-09-15 13:56:04 +02:00
|
|
|
# current :mod:`offlineimap.ui`, can be used for logging:
|
2011-01-05 17:00:55 +01:00
|
|
|
self.ui = getglobalui()
|
2003-04-29 04:25:42 +02:00
|
|
|
self.refreshperiod = self.getconffloat('autorefresh', 0.0)
|
2011-09-15 13:56:04 +02:00
|
|
|
# should we run in "dry-run" mode?
|
|
|
|
self.dryrun = self.config.getboolean('general', 'dry-run')
|
Daniel Jacobowitz patches
fixes deb#433732
Date: Sun, 30 Sep 2007 13:54:56 -0400
From: Daniel Jacobowitz <drow@false.org>
To: offlineimap@complete.org
Subject: Assorted patches
Here's the result of a lazy Sunday hacking on offlineimap. Sorry for
not breaking this into multiple patches. They're mostly logically
independent so just ask if that would make a difference.
First, a new -q (quick) option. The quick option means to only update
folders that seem to have had significant changes. For Maildir, any
change to any message UID or flags is significant, because checking
the flags doesn't add a significant cost. For IMAP, only a change to
the total number of messages or a change in the UID of the most recent
message is significant. This should catch everything except for
flags changes.
The difference in bandwidth is astonishing: a quick sync takes 80K
instead of 5.3MB, and 28 seconds instead of 90.
There's a configuration variable that lets you say every tenth sync
should update flags, but let all the intervening ones be lighter.
Second, a fix to the UID validity problems many people have been
reporting with Courier. As discussed in Debian bug #433732, I changed
the UID validity check to use SELECT unless the server complains that
the folder is read-only. This avoids the Courier bug (see the Debian
log for more details). This won't fix existing validity errors, you
need to remove the local status and validity files by hand and resync.
Third, some speedups in Maildir checking. It's still pretty slow
due to a combination of poor performance in os.listdir (never reads
more than 4K of directory entries at a time) and some semaphore that
leads to lots of futex wake operations, but at least this saves
20% or so of the CPU time running offlineimap on a single folder:
Time with quick refresh and md5 in loop: 4.75s user 0.46s system 12%
cpu 41.751 total
Time with quick refresh and md5 out of loop: 4.38s user 0.50s system
14% cpu 34.799 total
Time using string compare to check folder: 4.11s user 0.47s system 13%
cpu 34.788 total
And fourth, some display fixes for Curses.Blinkenlights. I made
warnings more visible, made the new quick sync message cyan, and
made all not explicitly colored messages grey. That last one was
really bugging me. Any time OfflineIMAP printed a warning in
this UI, it had even odds of coming out black on black!
Anyway, I hope these are useful. I'm happy to revise them if you see
a problem.
--
Daniel Jacobowitz
CodeSourcery
2007-10-01 23:20:37 +02:00
|
|
|
self.quicknum = 0
|
2003-04-29 04:25:42 +02:00
|
|
|
if self.refreshperiod == 0.0:
|
2003-01-04 05:57:46 +01:00
|
|
|
self.refreshperiod = None
|
|
|
|
|
2003-04-18 04:18:34 +02:00
|
|
|
def getlocaleval(self):
|
|
|
|
return self.localeval
|
|
|
|
|
2014-10-05 09:49:08 +02:00
|
|
|
# Interface from CustomConfig.ConfigHelperMixin
|
2003-04-18 04:18:34 +02:00
|
|
|
def getconfig(self):
|
|
|
|
return self.config
|
|
|
|
|
|
|
|
def getname(self):
|
|
|
|
return self.name
|
|
|
|
|
2011-04-27 12:15:50 +02:00
|
|
|
def __str__(self):
|
|
|
|
return self.name
|
|
|
|
|
2011-06-30 15:18:03 +02:00
|
|
|
def getaccountmeta(self):
|
|
|
|
return os.path.join(self.metadatadir, 'Account-' + self.name)
|
|
|
|
|
2014-10-05 09:49:08 +02:00
|
|
|
# Interface from CustomConfig.ConfigHelperMixin
|
2003-04-18 04:18:34 +02:00
|
|
|
def getsection(self):
|
|
|
|
return 'Account ' + self.getname()
|
2003-01-04 05:57:46 +01:00
|
|
|
|
2011-05-07 17:40:32 +02:00
|
|
|
@classmethod
|
|
|
|
def set_abort_event(cls, config, signum):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""Set skip sleep/abort event for all accounts.
|
2003-01-04 05:57:46 +01:00
|
|
|
|
2011-05-07 17:40:32 +02:00
|
|
|
If we want to skip a current (or the next) sleep, or if we want
|
|
|
|
to abort an autorefresh loop, the main thread can use
|
|
|
|
set_abort_event() to send the corresponding signal. Signum = 1
|
|
|
|
implies that we want all accounts to abort or skip the current
|
|
|
|
or next sleep phase. Signum = 2 will end the autorefresh loop,
|
2012-01-04 19:24:19 +01:00
|
|
|
ie all accounts will return after they finished a sync. signum=3
|
|
|
|
means, abort NOW, e.g. on SIGINT or SIGTERM.
|
2011-05-07 17:40:32 +02:00
|
|
|
|
|
|
|
This is a class method, it will send the signal to all accounts.
|
|
|
|
"""
|
2015-01-14 22:58:25 +01:00
|
|
|
|
2011-05-07 17:40:32 +02:00
|
|
|
if signum == 1:
|
|
|
|
# resync signal, set config option for all accounts
|
|
|
|
for acctsection in getaccountlist(config):
|
|
|
|
config.set('Account ' + acctsection, "skipsleep", '1')
|
|
|
|
elif signum == 2:
|
|
|
|
# don't autorefresh anymore
|
2012-01-04 19:24:19 +01:00
|
|
|
cls.abort_soon_signal.set()
|
|
|
|
elif signum == 3:
|
|
|
|
# abort ASAP
|
|
|
|
cls.abort_NOW_signal.set()
|
2011-05-07 17:40:32 +02:00
|
|
|
|
|
|
|
def get_abort_event(self):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""Checks if an abort signal had been sent.
|
2011-05-07 17:40:32 +02:00
|
|
|
|
|
|
|
If the 'skipsleep' config option for this account had been set,
|
|
|
|
with `set_abort_event(config, 1)` it will get cleared in this
|
|
|
|
function. Ie, we will only skip one sleep and not all.
|
|
|
|
|
|
|
|
:returns: True, if the main thread had called
|
|
|
|
:meth:`set_abort_event` earlier, otherwise 'False'.
|
|
|
|
"""
|
2015-01-14 22:58:25 +01:00
|
|
|
|
2011-05-07 17:40:32 +02:00
|
|
|
skipsleep = self.getconfboolean("skipsleep", 0)
|
|
|
|
if skipsleep:
|
|
|
|
self.config.set(self.getsection(), "skipsleep", '0')
|
2012-01-04 19:24:19 +01:00
|
|
|
return skipsleep or Account.abort_soon_signal.is_set() or \
|
|
|
|
Account.abort_NOW_signal.is_set()
|
2011-05-07 17:40:32 +02:00
|
|
|
|
2014-03-16 13:27:35 +01:00
|
|
|
def _sleeper(self):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""Sleep if the account is set to autorefresh.
|
2011-05-07 17:40:32 +02:00
|
|
|
|
|
|
|
:returns: 0:timeout expired, 1: canceled the timer,
|
|
|
|
2:request to abort the program,
|
|
|
|
100: if configured to not sleep at all.
|
|
|
|
"""
|
2015-01-14 22:58:25 +01:00
|
|
|
|
2003-01-04 05:57:46 +01:00
|
|
|
if not self.refreshperiod:
|
|
|
|
return 100
|
2003-04-18 04:18:34 +02:00
|
|
|
|
|
|
|
kaobjs = []
|
|
|
|
|
|
|
|
if hasattr(self, 'localrepos'):
|
|
|
|
kaobjs.append(self.localrepos)
|
|
|
|
if hasattr(self, 'remoterepos'):
|
|
|
|
kaobjs.append(self.remoterepos)
|
|
|
|
|
|
|
|
for item in kaobjs:
|
|
|
|
item.startkeepalive()
|
2011-10-26 16:47:21 +02:00
|
|
|
|
2009-07-17 07:03:29 +02:00
|
|
|
refreshperiod = int(self.refreshperiod * 60)
|
2011-05-07 17:40:32 +02:00
|
|
|
sleepresult = self.ui.sleep(refreshperiod, self)
|
Patch for signal handling to start a sync by Jim Pryor
Here's the way I'd like to use offlineimap on my laptop:
1. Have a regular cron job running infrequently. The cron job
checks to see
if I'm online, plugged in, and that no other copy of offlineimap is
running. If
all of these conditions are satisfied, it runs offlineimap just once:
"offlineimap -o -u Noninteractive.Quiet"
2. When I start up mutt, I do it by calling a wrapper script that
delays
until cron-started copies of offlineimap have finished, then starts
offlineimap
on its regular, stay-alive and keep checking schedule. When I quit
mutt, the
wrapper script tells offlineimap to stop.
This way I get frequent regular checks while I have mutt running, but
I don't
waste my battery/cpu checking frequently for mail when I'm not
interested in
it.
To make this work, though, it'd be nicer if it were easier to tell
offlineimap,
from the outside, things like "terminate cleanly now" and "when you've
finished
synching, then terminate instead of sleeping and synching again."
OK, to put my money where my mouth is, I attach two patches against
offlineimap
6.0.3.
The first, "cleanup.patch", cleans up a few spots that tend to throw
exceptions
for me as offlineimap is exiting from a KeyboardInterrupt.
The second adds signaling capabilities to offlineimap.
* sending a SIGTERM tells offlineimap to terminate immediately but
cleanly,
just as if "q" had been pressed in the GUI interface
* sending a SIGUSR1 tells every account to do a full sync asap: if
it's
sleeping, then wake up and do the sync now. If it's mid-sync, then
re-synch
any folders whose syncing has already been started or completed, and
continue
to synch the other, queued but not-yet-synched folders.
* sending a SIGHUP tells every account to die as soon as it can (but
not
immediately: only after finishing any synch it's now engaged in)
* sending a SIGUSR2 tells every account to do a full sync asap (as
with
SIGUSR1), then die
It's tricky to mix signals with threads, but I think I've done this
correctly.
I've been using it now for a few weeks without any obvious
problems. But I'm passing it
on so that others can review the code and test it out on their
systems. I developed the
patch when I was running Python 2.5.2, but to my knowledge I don't use
any Python 2.5-specific
code. Now I'm using the patch with Python 2.6.
Although I said "without any obvious problems," let me confess that
I'm
seeing offlineimap regularly choke when I do things like this: start
up
my offlineimap-wrapped copy of mutt, wait a while, put the machine to
sleep (not sure if offlineimap is active in the background or idling),
move to a different spot, wake the machine up again and it acquires a
new network, sometimes a wired network instead of wifi. Offlineimap
doesn't like that so much. I don't yet have any reason to think the
problems here come from my patches. But I'm just acknowledging them,
so
that if others are able to use offlineimap without any difficulty in
situations like I described, then maybe the fault is with my patches.
2008-12-01 23:13:16 +01:00
|
|
|
|
2008-08-03 00:04:32 +02:00
|
|
|
# Cancel keepalive
|
|
|
|
for item in kaobjs:
|
|
|
|
item.stopkeepalive()
|
2011-05-07 17:40:32 +02:00
|
|
|
|
|
|
|
if sleepresult:
|
2012-01-04 19:24:19 +01:00
|
|
|
if Account.abort_soon_signal.is_set() or \
|
|
|
|
Account.abort_NOW_signal.is_set():
|
2011-05-07 17:40:32 +02:00
|
|
|
return 2
|
|
|
|
self.quicknum = 0
|
|
|
|
return 1
|
|
|
|
return 0
|
2011-06-30 15:18:03 +02:00
|
|
|
|
|
|
|
def serverdiagnostics(self):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""Output diagnostics for all involved repositories."""
|
|
|
|
|
2011-06-30 15:18:03 +02:00
|
|
|
remote_repo = Repository(self, 'remote')
|
|
|
|
local_repo = Repository(self, 'local')
|
|
|
|
#status_repo = Repository(self, 'status')
|
|
|
|
self.ui.serverdiagnostics(remote_repo, 'Remote')
|
|
|
|
self.ui.serverdiagnostics(local_repo, 'Local')
|
|
|
|
#self.ui.serverdiagnostics(statusrepos, 'Status')
|
2011-10-26 16:47:21 +02:00
|
|
|
|
|
|
|
|
2011-01-31 15:53:35 +01:00
|
|
|
class SyncableAccount(Account):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""A syncable email account connecting 2 repositories.
|
2011-01-31 15:53:35 +01:00
|
|
|
|
2011-05-02 17:11:40 +02:00
|
|
|
Derives from :class:`accounts.Account` but contains the additional
|
|
|
|
functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`,
|
|
|
|
used for syncing."""
|
2011-01-31 15:53:35 +01:00
|
|
|
|
2011-07-08 12:23:16 +02:00
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
Account.__init__(self, *args, **kwargs)
|
|
|
|
self._lockfd = None
|
2015-01-14 22:58:25 +01:00
|
|
|
self._lockfilepath = os.path.join(
|
|
|
|
self.config.getmetadatadir(), "%s.lock"% self)
|
2011-07-08 12:23:16 +02:00
|
|
|
|
2014-03-16 13:27:35 +01:00
|
|
|
def __lock(self):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""Lock the account, throwing an exception if it is locked already."""
|
|
|
|
|
2011-07-08 12:23:16 +02:00
|
|
|
self._lockfd = open(self._lockfilepath, 'w')
|
|
|
|
try:
|
|
|
|
fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
|
|
except NameError:
|
|
|
|
#fcntl not available (Windows), disable file locking... :(
|
|
|
|
pass
|
|
|
|
except IOError:
|
|
|
|
self._lockfd.close()
|
2012-02-05 11:49:28 +01:00
|
|
|
raise OfflineImapError("Could not lock account %s. Is another "
|
2015-01-14 22:58:25 +01:00
|
|
|
"instance using this account?"% self,
|
|
|
|
OfflineImapError.ERROR.REPO), None, exc_info()[2]
|
2011-07-08 12:23:16 +02:00
|
|
|
|
2014-03-16 13:27:35 +01:00
|
|
|
def _unlock(self):
|
2011-07-08 12:23:16 +02:00
|
|
|
"""Unlock the account, deleting the lock file"""
|
2015-01-14 22:58:25 +01:00
|
|
|
|
2011-07-08 12:23:16 +02:00
|
|
|
#If we own the lock file, delete it
|
|
|
|
if self._lockfd and not self._lockfd.closed:
|
|
|
|
self._lockfd.close()
|
|
|
|
try:
|
|
|
|
os.unlink(self._lockfilepath)
|
|
|
|
except OSError:
|
2015-01-14 22:58:25 +01:00
|
|
|
pass # Failed to delete for some reason.
|
2011-07-08 12:23:16 +02:00
|
|
|
|
2011-05-07 17:40:32 +02:00
|
|
|
def syncrunner(self):
|
2011-11-03 13:45:44 +01:00
|
|
|
self.ui.registerthread(self)
|
2012-08-31 20:34:54 +02:00
|
|
|
try:
|
|
|
|
accountmetadata = self.getaccountmeta()
|
|
|
|
if not os.path.exists(accountmetadata):
|
|
|
|
os.mkdir(accountmetadata, 0o700)
|
|
|
|
|
|
|
|
self.remoterepos = Repository(self, 'remote')
|
|
|
|
self.localrepos = Repository(self, 'local')
|
|
|
|
self.statusrepos = Repository(self, 'status')
|
|
|
|
except OfflineImapError as e:
|
|
|
|
self.ui.error(e, exc_info()[2])
|
|
|
|
if e.severity >= OfflineImapError.ERROR.CRITICAL:
|
|
|
|
raise
|
|
|
|
return
|
2009-08-16 14:42:39 +02:00
|
|
|
|
2011-05-04 16:45:28 +02:00
|
|
|
# Loop account sync if needed (bail out after 3 failures)
|
|
|
|
looping = 3
|
2003-01-04 05:57:46 +01:00
|
|
|
while looping:
|
2011-09-29 17:22:02 +02:00
|
|
|
self.ui.acct(self)
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
try:
|
2014-03-16 13:27:35 +01:00
|
|
|
self.__lock()
|
|
|
|
self.__sync()
|
2011-09-29 17:22:02 +02:00
|
|
|
except (KeyboardInterrupt, SystemExit):
|
|
|
|
raise
|
2012-02-05 10:14:23 +01:00
|
|
|
except OfflineImapError as e:
|
2011-09-29 17:22:02 +02:00
|
|
|
# Stop looping and bubble up Exception if needed.
|
|
|
|
if e.severity >= OfflineImapError.ERROR.REPO:
|
|
|
|
if looping:
|
|
|
|
looping -= 1
|
|
|
|
if e.severity >= OfflineImapError.ERROR.CRITICAL:
|
|
|
|
raise
|
|
|
|
self.ui.error(e, exc_info()[2])
|
2012-02-05 10:14:23 +01:00
|
|
|
except Exception as e:
|
2015-01-14 22:58:25 +01:00
|
|
|
self.ui.error(e, exc_info()[2], msg=
|
|
|
|
"While attempting to sync account '%s'"% self)
|
2011-09-29 17:22:02 +02:00
|
|
|
else:
|
|
|
|
# after success sync, reset the looping counter to 3
|
|
|
|
if self.refreshperiod:
|
|
|
|
looping = 3
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
finally:
|
2011-09-29 17:22:02 +02:00
|
|
|
self.ui.acctdone(self)
|
2014-03-16 13:27:35 +01:00
|
|
|
self._unlock()
|
|
|
|
if looping and self._sleeper() >= 2:
|
2011-10-26 16:47:21 +02:00
|
|
|
looping = 0
|
2009-08-16 14:42:39 +02:00
|
|
|
|
2012-09-01 02:16:06 +02:00
|
|
|
def get_local_folder(self, remotefolder):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""Return the corresponding local folder for a given remotefolder."""
|
|
|
|
|
2012-09-01 02:16:06 +02:00
|
|
|
return self.localrepos.getfolder(
|
|
|
|
remotefolder.getvisiblename().
|
|
|
|
replace(self.remoterepos.getsep(), self.localrepos.getsep()))
|
|
|
|
|
2014-03-16 13:27:35 +01:00
|
|
|
def __sync(self):
|
2015-01-14 22:58:25 +01:00
|
|
|
"""Synchronize the account once, then return.
|
2011-05-07 17:40:32 +02:00
|
|
|
|
|
|
|
Assumes that `self.remoterepos`, `self.localrepos`, and
|
|
|
|
`self.statusrepos` has already been populated, so it should only
|
2015-01-14 22:58:25 +01:00
|
|
|
be called from the :meth:`syncrunner` function."""
|
|
|
|
|
2011-05-07 17:40:32 +02:00
|
|
|
folderthreads = []
|
Daniel Jacobowitz patches
fixes deb#433732
Date: Sun, 30 Sep 2007 13:54:56 -0400
From: Daniel Jacobowitz <drow@false.org>
To: offlineimap@complete.org
Subject: Assorted patches
Here's the result of a lazy Sunday hacking on offlineimap. Sorry for
not breaking this into multiple patches. They're mostly logically
independent so just ask if that would make a difference.
First, a new -q (quick) option. The quick option means to only update
folders that seem to have had significant changes. For Maildir, any
change to any message UID or flags is significant, because checking
the flags doesn't add a significant cost. For IMAP, only a change to
the total number of messages or a change in the UID of the most recent
message is significant. This should catch everything except for
flags changes.
The difference in bandwidth is astonishing: a quick sync takes 80K
instead of 5.3MB, and 28 seconds instead of 90.
There's a configuration variable that lets you say every tenth sync
should update flags, but let all the intervening ones be lighter.
Second, a fix to the UID validity problems many people have been
reporting with Courier. As discussed in Debian bug #433732, I changed
the UID validity check to use SELECT unless the server complains that
the folder is read-only. This avoids the Courier bug (see the Debian
log for more details). This won't fix existing validity errors, you
need to remove the local status and validity files by hand and resync.
Third, some speedups in Maildir checking. It's still pretty slow
due to a combination of poor performance in os.listdir (never reads
more than 4K of directory entries at a time) and some semaphore that
leads to lots of futex wake operations, but at least this saves
20% or so of the CPU time running offlineimap on a single folder:
Time with quick refresh and md5 in loop: 4.75s user 0.46s system 12%
cpu 41.751 total
Time with quick refresh and md5 out of loop: 4.38s user 0.50s system
14% cpu 34.799 total
Time using string compare to check folder: 4.11s user 0.47s system 13%
cpu 34.788 total
And fourth, some display fixes for Curses.Blinkenlights. I made
warnings more visible, made the new quick sync message cyan, and
made all not explicitly colored messages grey. That last one was
really bugging me. Any time OfflineIMAP printed a warning in
this UI, it had even odds of coming out black on black!
Anyway, I hope these are useful. I'm happy to revise them if you see
a problem.
--
Daniel Jacobowitz
CodeSourcery
2007-10-01 23:20:37 +02:00
|
|
|
|
2008-10-01 07:03:04 +02:00
|
|
|
hook = self.getconf('presynchook', '')
|
|
|
|
self.callhook(hook)
|
|
|
|
|
Daniel Jacobowitz patches
fixes deb#433732
Date: Sun, 30 Sep 2007 13:54:56 -0400
From: Daniel Jacobowitz <drow@false.org>
To: offlineimap@complete.org
Subject: Assorted patches
Here's the result of a lazy Sunday hacking on offlineimap. Sorry for
not breaking this into multiple patches. They're mostly logically
independent so just ask if that would make a difference.
First, a new -q (quick) option. The quick option means to only update
folders that seem to have had significant changes. For Maildir, any
change to any message UID or flags is significant, because checking
the flags doesn't add a significant cost. For IMAP, only a change to
the total number of messages or a change in the UID of the most recent
message is significant. This should catch everything except for
flags changes.
The difference in bandwidth is astonishing: a quick sync takes 80K
instead of 5.3MB, and 28 seconds instead of 90.
There's a configuration variable that lets you say every tenth sync
should update flags, but let all the intervening ones be lighter.
Second, a fix to the UID validity problems many people have been
reporting with Courier. As discussed in Debian bug #433732, I changed
the UID validity check to use SELECT unless the server complains that
the folder is read-only. This avoids the Courier bug (see the Debian
log for more details). This won't fix existing validity errors, you
need to remove the local status and validity files by hand and resync.
Third, some speedups in Maildir checking. It's still pretty slow
due to a combination of poor performance in os.listdir (never reads
more than 4K of directory entries at a time) and some semaphore that
leads to lots of futex wake operations, but at least this saves
20% or so of the CPU time running offlineimap on a single folder:
Time with quick refresh and md5 in loop: 4.75s user 0.46s system 12%
cpu 41.751 total
Time with quick refresh and md5 out of loop: 4.38s user 0.50s system
14% cpu 34.799 total
Time using string compare to check folder: 4.11s user 0.47s system 13%
cpu 34.788 total
And fourth, some display fixes for Curses.Blinkenlights. I made
warnings more visible, made the new quick sync message cyan, and
made all not explicitly colored messages grey. That last one was
really bugging me. Any time OfflineIMAP printed a warning in
this UI, it had even odds of coming out black on black!
Anyway, I hope these are useful. I'm happy to revise them if you see
a problem.
--
Daniel Jacobowitz
CodeSourcery
2007-10-01 23:20:37 +02:00
|
|
|
quickconfig = self.getconfint('quick', 0)
|
|
|
|
if quickconfig < 0:
|
|
|
|
quick = True
|
|
|
|
elif quickconfig > 0:
|
|
|
|
if self.quicknum == 0 or self.quicknum > quickconfig:
|
|
|
|
self.quicknum = 1
|
|
|
|
quick = False
|
|
|
|
else:
|
|
|
|
self.quicknum = self.quicknum + 1
|
|
|
|
quick = True
|
|
|
|
else:
|
|
|
|
quick = False
|
|
|
|
|
2003-01-04 05:57:46 +01:00
|
|
|
try:
|
2003-04-18 04:18:34 +02:00
|
|
|
remoterepos = self.remoterepos
|
|
|
|
localrepos = self.localrepos
|
|
|
|
statusrepos = self.statusrepos
|
2011-12-01 10:27:12 +01:00
|
|
|
|
|
|
|
# init repos with list of folders, so we have them (and the
|
|
|
|
# folder delimiter etc)
|
|
|
|
remoterepos.getfolders()
|
|
|
|
localrepos.getfolders()
|
|
|
|
|
2012-01-05 14:05:51 +01:00
|
|
|
remoterepos.sync_folder_structure(localrepos, statusrepos)
|
2011-09-30 08:58:08 +02:00
|
|
|
# replicate the folderstructure between REMOTE to LOCAL
|
2011-09-15 12:06:06 +02:00
|
|
|
if not localrepos.getconfboolean('readonly', False):
|
2011-05-01 20:18:29 +02:00
|
|
|
self.ui.syncfolders(remoterepos, localrepos)
|
2003-01-04 05:57:46 +01:00
|
|
|
|
2011-05-07 17:40:32 +02:00
|
|
|
# iterate through all folders on the remote repo and sync
|
|
|
|
for remotefolder in remoterepos.getfolders():
|
2012-01-04 19:24:19 +01:00
|
|
|
# check for CTRL-C or SIGTERM
|
|
|
|
if Account.abort_NOW_signal.is_set(): break
|
2012-09-01 02:16:06 +02:00
|
|
|
|
2012-09-01 03:12:16 +02:00
|
|
|
if not remotefolder.sync_this:
|
2012-09-01 02:16:06 +02:00
|
|
|
self.ui.debug('', "Not syncing filtered folder '%s'"
|
2015-01-08 17:13:33 +01:00
|
|
|
"[%s]"% (remotefolder, remoterepos))
|
2012-09-01 02:16:06 +02:00
|
|
|
continue # Ignore filtered folder
|
2012-09-01 03:12:16 +02:00
|
|
|
localfolder = self.get_local_folder(remotefolder)
|
|
|
|
if not localfolder.sync_this:
|
|
|
|
self.ui.debug('', "Not syncing filtered folder '%s'"
|
2015-01-08 17:13:33 +01:00
|
|
|
"[%s]"% (localfolder, localfolder.repository))
|
2012-09-01 03:12:16 +02:00
|
|
|
continue # Ignore filtered folder
|
2013-02-05 04:49:56 +01:00
|
|
|
if not globals.options.singlethreading:
|
2013-01-17 12:02:41 +01:00
|
|
|
thread = InstanceLimitedThread(\
|
|
|
|
instancename = 'FOLDER_' + self.remoterepos.getname(),
|
|
|
|
target = syncfolder,
|
2015-01-08 17:13:33 +01:00
|
|
|
name = "Folder %s [acc: %s]"% (remotefolder.getexplainedname(), self),
|
2013-01-17 12:02:41 +01:00
|
|
|
args = (self, remotefolder, quick))
|
|
|
|
thread.start()
|
|
|
|
folderthreads.append(thread)
|
|
|
|
else:
|
|
|
|
syncfolder(self, remotefolder, quick)
|
2011-05-07 17:40:32 +02:00
|
|
|
# wait for all threads to finish
|
|
|
|
for thr in folderthreads:
|
|
|
|
thr.join()
|
2011-09-15 13:56:04 +02:00
|
|
|
# Write out mailbox names if required and not in dry-run mode
|
|
|
|
if not self.dryrun:
|
2015-03-22 19:34:24 +01:00
|
|
|
mbnames.write(False)
|
2007-07-06 18:46:29 +02:00
|
|
|
localrepos.forgetfolders()
|
|
|
|
remoterepos.forgetfolders()
|
2011-04-27 11:44:24 +02:00
|
|
|
except:
|
|
|
|
#error while syncing. Drop all connections that we have, they
|
|
|
|
#might be bogus by now (e.g. after suspend)
|
|
|
|
localrepos.dropconnections()
|
|
|
|
remoterepos.dropconnections()
|
|
|
|
raise
|
|
|
|
else:
|
|
|
|
# sync went fine. Hold or drop depending on config
|
2003-04-18 04:18:34 +02:00
|
|
|
localrepos.holdordropconnections()
|
|
|
|
remoterepos.holdordropconnections()
|
2008-10-01 07:03:04 +02:00
|
|
|
|
|
|
|
hook = self.getconf('postsynchook', '')
|
|
|
|
self.callhook(hook)
|
|
|
|
|
|
|
|
def callhook(self, cmd):
|
2012-01-04 19:24:19 +01:00
|
|
|
# check for CTRL-C or SIGTERM and run postsynchook
|
|
|
|
if Account.abort_NOW_signal.is_set():
|
|
|
|
return
|
2008-10-01 07:03:04 +02:00
|
|
|
if not cmd:
|
|
|
|
return
|
|
|
|
try:
|
|
|
|
self.ui.callhook("Calling hook: " + cmd)
|
2011-09-15 13:56:04 +02:00
|
|
|
if self.dryrun: # don't if we are in dry-run mode
|
|
|
|
return
|
2008-10-01 07:03:04 +02:00
|
|
|
p = Popen(cmd, shell=True,
|
|
|
|
stdin=PIPE, stdout=PIPE, stderr=PIPE,
|
|
|
|
close_fds=True)
|
|
|
|
r = p.communicate()
|
2015-01-14 22:58:25 +01:00
|
|
|
self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n"% r)
|
|
|
|
self.ui.callhook("Hook return code: %d"% p.returncode)
|
2011-08-11 12:22:35 +02:00
|
|
|
except (KeyboardInterrupt, SystemExit):
|
|
|
|
raise
|
2012-02-05 10:14:23 +01:00
|
|
|
except Exception as e:
|
2015-01-14 22:58:25 +01:00
|
|
|
self.ui.error(e, exc_info()[2], msg="Calling hook")
|
2011-01-31 15:53:35 +01:00
|
|
|
|
2011-11-03 13:27:35 +01:00
|
|
|
def syncfolder(account, remotefolder, quick):
|
2013-01-17 12:02:41 +01:00
|
|
|
"""Synchronizes given remote folder for the specified account.
|
2011-08-14 13:38:13 +02:00
|
|
|
|
2016-07-25 14:39:11 +02:00
|
|
|
Filtered folders on the remote side will not invoke this function. However,
|
|
|
|
this might be called in a concurrently."""
|
2015-01-08 17:13:33 +01:00
|
|
|
|
maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option
1. When using maxage, local and remote messagelists are supposed to only
contain messages from at most maxage days ago. But local and remote used
different timezones to calculate what "maxage days ago" means, resulting
in removals on one side. Now, we ask the local folder for maxage days'
worth of mail, find the lowest UID, and then ask the remote folder for
all UID's starting with that lowest one.
2. maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that
remote messages have UIDs in the same order as their local counterparts,
which could be false, e.g. when messages are copied in quick succession.
So, remove support for maxage in the IMAP-IMAP case.
3. Add startdate option for IMAP-IMAP syncs: use messages from the given
repository starting at startdate, and all messages from the other
repository. In the first sync, the other repository must be empty.
4. Allow maxage to be specified either as number of days to sync (as
previously) or as a fixed date.
Signed-off-by: Janna Martl <janna.martl109@gmail.com>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2015-04-07 10:14:11 +02:00
|
|
|
def check_uid_validity(localfolder, remotefolder, statusfolder):
|
|
|
|
# If either the local or the status folder has messages and
|
|
|
|
# there is a UID validity problem, warn and abort. If there are
|
|
|
|
# no messages, UW IMAPd loses UIDVALIDITY. But we don't really
|
|
|
|
# need it if both local folders are empty. So, in that case,
|
|
|
|
# just save it off.
|
|
|
|
if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount() > 0:
|
|
|
|
if not localfolder.check_uidvalidity():
|
|
|
|
ui.validityproblem(localfolder)
|
|
|
|
localfolder.repository.restore_atime()
|
|
|
|
return
|
|
|
|
if not remotefolder.check_uidvalidity():
|
|
|
|
ui.validityproblem(remotefolder)
|
|
|
|
localrepos.restore_atime()
|
|
|
|
return
|
|
|
|
else:
|
|
|
|
# Both folders empty, just save new UIDVALIDITY
|
|
|
|
localfolder.save_uidvalidity()
|
|
|
|
remotefolder.save_uidvalidity()
|
|
|
|
|
|
|
|
def save_min_uid(folder, min_uid):
|
|
|
|
uidfile = folder.get_min_uid_file()
|
|
|
|
fd = open(uidfile, 'wt')
|
|
|
|
fd.write(str(min_uid) + "\n")
|
|
|
|
fd.close()
|
|
|
|
|
|
|
|
def cachemessagelists_upto_date(localfolder, remotefolder, date):
|
2016-07-25 14:39:11 +02:00
|
|
|
"""Returns messages with uid > min(uids of messages newer than date)."""
|
maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option
1. When using maxage, local and remote messagelists are supposed to only
contain messages from at most maxage days ago. But local and remote used
different timezones to calculate what "maxage days ago" means, resulting
in removals on one side. Now, we ask the local folder for maxage days'
worth of mail, find the lowest UID, and then ask the remote folder for
all UID's starting with that lowest one.
2. maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that
remote messages have UIDs in the same order as their local counterparts,
which could be false, e.g. when messages are copied in quick succession.
So, remove support for maxage in the IMAP-IMAP case.
3. Add startdate option for IMAP-IMAP syncs: use messages from the given
repository starting at startdate, and all messages from the other
repository. In the first sync, the other repository must be empty.
4. Allow maxage to be specified either as number of days to sync (as
previously) or as a fixed date.
Signed-off-by: Janna Martl <janna.martl109@gmail.com>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2015-04-07 10:14:11 +02:00
|
|
|
|
|
|
|
localfolder.cachemessagelist(min_date=date)
|
|
|
|
check_uid_validity(localfolder, remotefolder, statusfolder)
|
|
|
|
# local messagelist had date restriction applied already. Restrict
|
|
|
|
# sync to messages with UIDs >= min_uid from this list.
|
|
|
|
#
|
|
|
|
# local messagelist might contain new messages (with uid's < 0).
|
|
|
|
positive_uids = filter(
|
|
|
|
lambda uid: uid > 0, localfolder.getmessageuidlist())
|
|
|
|
if len(positive_uids) > 0:
|
|
|
|
remotefolder.cachemessagelist(min_uid=min(positive_uids))
|
|
|
|
else:
|
|
|
|
# No messages with UID > 0 in range in localfolder.
|
|
|
|
# date restriction was applied with respect to local dates but
|
|
|
|
# remote folder timezone might be different from local, so be
|
|
|
|
# safe and make sure the range isn't bigger than in local.
|
|
|
|
remotefolder.cachemessagelist(
|
|
|
|
min_date=time.gmtime(time.mktime(date) + 24*60*60))
|
|
|
|
|
|
|
|
def cachemessagelists_startdate(new, partial, date):
|
2016-07-25 14:39:11 +02:00
|
|
|
"""Retrieve messagelists when startdate has been set for
|
maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option
1. When using maxage, local and remote messagelists are supposed to only
contain messages from at most maxage days ago. But local and remote used
different timezones to calculate what "maxage days ago" means, resulting
in removals on one side. Now, we ask the local folder for maxage days'
worth of mail, find the lowest UID, and then ask the remote folder for
all UID's starting with that lowest one.
2. maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that
remote messages have UIDs in the same order as their local counterparts,
which could be false, e.g. when messages are copied in quick succession.
So, remove support for maxage in the IMAP-IMAP case.
3. Add startdate option for IMAP-IMAP syncs: use messages from the given
repository starting at startdate, and all messages from the other
repository. In the first sync, the other repository must be empty.
4. Allow maxage to be specified either as number of days to sync (as
previously) or as a fixed date.
Signed-off-by: Janna Martl <janna.martl109@gmail.com>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2015-04-07 10:14:11 +02:00
|
|
|
the folder 'partial'.
|
|
|
|
|
|
|
|
Idea: suppose you want to clone the messages after date in one
|
|
|
|
account (partial) to a new one (new). If new is empty, then copy
|
|
|
|
messages in partial newer than date to new, and keep track of the
|
|
|
|
min uid. On subsequent syncs, sync all the messages in new against
|
|
|
|
those after that min uid in partial. This is a partial replacement
|
|
|
|
for maxage in the IMAP-IMAP sync case, where maxage doesn't work:
|
|
|
|
the UIDs of the messages in localfolder might not be in the same
|
|
|
|
order as those of corresponding messages in remotefolder, so if L in
|
|
|
|
local corresponds to R in remote, the ranges [L, ...] and [R, ...]
|
|
|
|
might not correspond. But, if we're cloning a folder into a new one,
|
|
|
|
[min_uid, ...] does correspond to [1, ...].
|
|
|
|
|
2016-07-25 14:39:11 +02:00
|
|
|
This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead."""
|
maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option
1. When using maxage, local and remote messagelists are supposed to only
contain messages from at most maxage days ago. But local and remote used
different timezones to calculate what "maxage days ago" means, resulting
in removals on one side. Now, we ask the local folder for maxage days'
worth of mail, find the lowest UID, and then ask the remote folder for
all UID's starting with that lowest one.
2. maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that
remote messages have UIDs in the same order as their local counterparts,
which could be false, e.g. when messages are copied in quick succession.
So, remove support for maxage in the IMAP-IMAP case.
3. Add startdate option for IMAP-IMAP syncs: use messages from the given
repository starting at startdate, and all messages from the other
repository. In the first sync, the other repository must be empty.
4. Allow maxage to be specified either as number of days to sync (as
previously) or as a fixed date.
Signed-off-by: Janna Martl <janna.martl109@gmail.com>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2015-04-07 10:14:11 +02:00
|
|
|
|
|
|
|
new.cachemessagelist()
|
|
|
|
min_uid = partial.retrieve_min_uid()
|
|
|
|
if min_uid == None: # min_uid file didn't exist
|
|
|
|
if len(new.getmessageuidlist()) > 0:
|
|
|
|
raise OfflineImapError("To use startdate on Repository %s, "
|
|
|
|
"Repository %s must be empty"%
|
|
|
|
(partial.repository.name, new.repository.name),
|
|
|
|
OfflineImapError.ERROR.MESSAGE)
|
|
|
|
else:
|
|
|
|
partial.cachemessagelist(min_date=date)
|
|
|
|
# messagelist.keys() instead of getuidmessagelist() because in
|
|
|
|
# the UID mapped case we want the actual local UIDs, not their
|
|
|
|
# remote counterparts
|
|
|
|
positive_uids = filter(
|
|
|
|
lambda uid: uid > 0, partial.messagelist.keys())
|
|
|
|
if len(positive_uids) > 0:
|
|
|
|
min_uid = min(positive_uids)
|
|
|
|
else:
|
|
|
|
min_uid = 1
|
|
|
|
save_min_uid(partial, min_uid)
|
|
|
|
else:
|
|
|
|
partial.cachemessagelist(min_uid=min_uid)
|
|
|
|
|
|
|
|
|
2011-11-03 13:27:35 +01:00
|
|
|
remoterepos = account.remoterepos
|
|
|
|
localrepos = account.localrepos
|
|
|
|
statusrepos = account.statusrepos
|
|
|
|
|
2011-01-05 17:00:55 +01:00
|
|
|
ui = getglobalui()
|
2011-11-03 13:45:44 +01:00
|
|
|
ui.registerthread(account)
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
try:
|
|
|
|
# Load local folder.
|
2012-09-01 02:16:06 +02:00
|
|
|
localfolder = account.get_local_folder(remotefolder)
|
|
|
|
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
# Write the mailboxes
|
2013-08-27 16:11:57 +02:00
|
|
|
mbnames.add(account.name, localfolder.getname(),
|
2015-01-14 22:58:25 +01:00
|
|
|
localrepos.getlocalroot())
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
|
|
|
|
# Load status folder.
|
2015-01-14 22:58:25 +01:00
|
|
|
statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().
|
|
|
|
replace(remoterepos.getsep(), statusrepos.getsep()))
|
2016-05-12 18:27:14 +02:00
|
|
|
statusfolder.openfiles()
|
2015-01-14 22:58:25 +01:00
|
|
|
|
2012-01-19 11:24:16 +01:00
|
|
|
if localfolder.get_uidvalidity() == None:
|
2012-01-19 11:54:16 +01:00
|
|
|
# This is a new folder, so delete the status cache to be
|
|
|
|
# sure we don't have a conflict.
|
|
|
|
# TODO: This does not work. We always return a value, need
|
|
|
|
# to rework this...
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
statusfolder.deletemessagelist()
|
|
|
|
|
|
|
|
statusfolder.cachemessagelist()
|
|
|
|
|
|
|
|
|
2015-01-14 22:58:25 +01:00
|
|
|
# Load local folder.
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)
|
|
|
|
|
maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option
1. When using maxage, local and remote messagelists are supposed to only
contain messages from at most maxage days ago. But local and remote used
different timezones to calculate what "maxage days ago" means, resulting
in removals on one side. Now, we ask the local folder for maxage days'
worth of mail, find the lowest UID, and then ask the remote folder for
all UID's starting with that lowest one.
2. maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that
remote messages have UIDs in the same order as their local counterparts,
which could be false, e.g. when messages are copied in quick succession.
So, remove support for maxage in the IMAP-IMAP case.
3. Add startdate option for IMAP-IMAP syncs: use messages from the given
repository starting at startdate, and all messages from the other
repository. In the first sync, the other repository must be empty.
4. Allow maxage to be specified either as number of days to sync (as
previously) or as a fixed date.
Signed-off-by: Janna Martl <janna.martl109@gmail.com>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2015-04-07 10:14:11 +02:00
|
|
|
# Retrieve messagelists, taking into account age-restriction
|
|
|
|
# options
|
|
|
|
maxage = localfolder.getmaxage()
|
|
|
|
localstart = localfolder.getstartdate()
|
|
|
|
remotestart = remotefolder.getstartdate()
|
|
|
|
if (maxage != None) + (localstart != None) + (remotestart != None) > 1:
|
|
|
|
raise OfflineImapError("You can set at most one of the "
|
|
|
|
"following: maxage, startdate (for the local folder), "
|
|
|
|
"startdate (for the remote folder)",
|
|
|
|
OfflineImapError.ERROR.REPO), None, exc_info()[2]
|
|
|
|
if (maxage != None or localstart or remotestart) and quick:
|
|
|
|
# IMAP quickchanged isn't compatible with options that
|
|
|
|
# involve restricting the messagelist, since the "quick"
|
|
|
|
# check can only retrieve a full list of UIDs in the folder.
|
|
|
|
ui.warn("Quick syncs (-q) not supported in conjunction "
|
|
|
|
"with maxage or startdate; ignoring -q.")
|
|
|
|
if maxage != None:
|
|
|
|
cachemessagelists_upto_date(localfolder, remotefolder, maxage)
|
|
|
|
elif localstart != None:
|
|
|
|
cachemessagelists_startdate(remotefolder, localfolder,
|
|
|
|
localstart)
|
|
|
|
check_uid_validity(localfolder, remotefolder, statusfolder)
|
|
|
|
elif remotestart != None:
|
|
|
|
cachemessagelists_startdate(localfolder, remotefolder,
|
|
|
|
remotestart)
|
|
|
|
check_uid_validity(localfolder, remotefolder, statusfolder)
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
else:
|
maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option
1. When using maxage, local and remote messagelists are supposed to only
contain messages from at most maxage days ago. But local and remote used
different timezones to calculate what "maxage days ago" means, resulting
in removals on one side. Now, we ask the local folder for maxage days'
worth of mail, find the lowest UID, and then ask the remote folder for
all UID's starting with that lowest one.
2. maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that
remote messages have UIDs in the same order as their local counterparts,
which could be false, e.g. when messages are copied in quick succession.
So, remove support for maxage in the IMAP-IMAP case.
3. Add startdate option for IMAP-IMAP syncs: use messages from the given
repository starting at startdate, and all messages from the other
repository. In the first sync, the other repository must be empty.
4. Allow maxage to be specified either as number of days to sync (as
previously) or as a fixed date.
Signed-off-by: Janna Martl <janna.martl109@gmail.com>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2015-04-07 10:14:11 +02:00
|
|
|
localfolder.cachemessagelist()
|
|
|
|
if quick:
|
|
|
|
if (not localfolder.quickchanged(statusfolder) and
|
|
|
|
not remotefolder.quickchanged(statusfolder)):
|
|
|
|
ui.skippingfolder(remotefolder)
|
|
|
|
localrepos.restore_atime()
|
|
|
|
return
|
|
|
|
check_uid_validity(localfolder, remotefolder, statusfolder)
|
|
|
|
remotefolder.cachemessagelist()
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
|
|
|
|
# Synchronize remote changes.
|
2011-09-15 12:06:06 +02:00
|
|
|
if not localrepos.getconfboolean('readonly', False):
|
2011-05-01 20:18:29 +02:00
|
|
|
ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder)
|
|
|
|
remotefolder.syncmessagesto(localfolder, statusfolder)
|
|
|
|
else:
|
|
|
|
ui.debug('imap', "Not syncing to read-only repository '%s'" \
|
|
|
|
% localrepos.getname())
|
2011-10-26 16:47:21 +02:00
|
|
|
|
2011-03-06 20:03:04 +01:00
|
|
|
# Synchronize local changes
|
2011-09-15 12:06:06 +02:00
|
|
|
if not remoterepos.getconfboolean('readonly', False):
|
2011-05-01 20:18:29 +02:00
|
|
|
ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder)
|
|
|
|
localfolder.syncmessagesto(remotefolder, statusfolder)
|
|
|
|
else:
|
|
|
|
ui.debug('', "Not syncing to read-only repository '%s'" \
|
|
|
|
% remoterepos.getname())
|
Patch for error handling / separation of accounts etc.
Dear All,
I have made the attached patch to try and make offlineimap a bit more
stable in challenging situations. It's extremely useful in slow
connection environments - but sometimes if one account had the wrong
password or the connection went down then unfortunately the whole
program would crash.
I have tested this on our connection and tried throwing at it just about
every situation - connection, up down, up, down again, change password,
error whilst copying one message, etc. I have been running this patch
for the last 5 days or so syncing 6 accounts at the moment... It seems
to work and stay alive nicely (even if your connection does not)...
Hope that this can go in for the next release... Please let me know if
anyone notices any problems with this...
Regards,
-Mike
-- Attached file included as plaintext by Ecartis --
-- File: submit
From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001
From: mike <mike@mikelaptop.(none)>
Date: Mon, 24 Aug 2009 19:37:59 +0430
Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g.
if one account has an error because of a changed password or something
that should not affect the other accounts.
Specifically:
If one sync run has an issue this is in a try-except clause - if it
has an auto refresh period the thread will sleep and try again - this
could be quite useful in the event of the connection going down for a
little while, changed password etc.
If one folder cannot be created an error message will be displayed through
the UI and the program will continue (e.g. permission denied to create a folder)
If one message does not want to copy for whatever resaon an error message
will be displayed through the UI and at least the other messages will
be copied
If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
|
|
|
|
|
|
|
statusfolder.save()
|
|
|
|
localrepos.restore_atime()
|
2011-01-12 11:15:08 +01:00
|
|
|
except (KeyboardInterrupt, SystemExit):
|
|
|
|
raise
|
2012-02-05 10:14:23 +01:00
|
|
|
except OfflineImapError as e:
|
2011-05-07 18:39:13 +02:00
|
|
|
# bubble up severe Errors, skip folder otherwise
|
|
|
|
if e.severity > OfflineImapError.ERROR.FOLDER:
|
|
|
|
raise
|
|
|
|
else:
|
2016-07-25 14:39:11 +02:00
|
|
|
ui.error(e, exc_info()[2], msg="Aborting sync, folder '%s' "
|
|
|
|
"[acc: '%s']"% (localfolder, account))
|
2012-02-05 10:14:23 +01:00
|
|
|
except Exception as e:
|
2015-01-14 22:58:25 +01:00
|
|
|
ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s"%
|
|
|
|
(account, remotefolder.getvisiblename(), traceback.format_exc()))
|
2014-01-05 16:07:03 +01:00
|
|
|
finally:
|
|
|
|
for folder in ["statusfolder", "localfolder", "remotefolder"]:
|
|
|
|
if folder in locals():
|
|
|
|
locals()[folder].dropmessagelistcache()
|
2016-04-09 19:52:33 +02:00
|
|
|
statusfolder.closefiles()
|