docker-offlineimap/offlineimap/accounts.py

600 lines
23 KiB
Python
Raw Normal View History

# Copyright (C) 2003-2015 John Goerzen & contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
2006-08-12 06:15:55 +02:00
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
from subprocess import Popen, PIPE
from threading import Event
import os
import time
from sys import exc_info
import traceback
2008-12-02 20:12:36 +01:00
from offlineimap import mbnames, CustomConfig, OfflineImapError
from offlineimap import globals
from offlineimap.repository import Repository
from offlineimap.ui import getglobalui
from offlineimap.threadutil import InstanceLimitedThread
try:
import fcntl
except:
pass # ok if this fails, we can do without
# FIXME: spaghetti code alert!
def getaccountlist(customconfig):
# Account names in a list.
return customconfig.getsectionlist('Account')
# FIXME: spaghetti code alert!
def AccountListGenerator(customconfig):
"""Returns a list of instanciated Account class, one per account name."""
return [Account(customconfig, accountname)
for accountname in getaccountlist(customconfig)]
# FIXME: spaghetti code alert!
def AccountHashGenerator(customconfig):
"""Returns a dict of instanciated Account class with the account name as
key."""
retval = {}
for item in AccountListGenerator(customconfig):
retval[item.getname()] = item
return retval
class Account(CustomConfig.ConfigHelperMixin):
"""Represents an account (ie. 2 repositories) to sync.
Most of the time you will actually want to use the derived
:class:`accounts.SyncableAccount` which contains all functions used
for syncing an account."""
# Signal gets set when we should stop looping.
abort_soon_signal = Event()
# Signal gets set on CTRL-C/SIGTERM.
abort_NOW_signal = Event()
def __init__(self, config, name):
"""
:param config: Representing the offlineimap configuration file.
:type config: :class:`offlineimap.CustomConfig.CustomConfigParser`
:param name: A (str) string denoting the name of the Account
as configured.
"""
self.config = config
self.name = name
self.metadatadir = config.getmetadatadir()
self.localeval = config.getlocaleval()
# current :mod:`offlineimap.ui`, can be used for logging:
self.ui = getglobalui()
self.refreshperiod = self.getconffloat('autorefresh', 0.0)
# should we run in "dry-run" mode?
self.dryrun = self.config.getboolean('general', 'dry-run')
Daniel Jacobowitz patches fixes deb#433732 Date: Sun, 30 Sep 2007 13:54:56 -0400 From: Daniel Jacobowitz <drow@false.org> To: offlineimap@complete.org Subject: Assorted patches Here's the result of a lazy Sunday hacking on offlineimap. Sorry for not breaking this into multiple patches. They're mostly logically independent so just ask if that would make a difference. First, a new -q (quick) option. The quick option means to only update folders that seem to have had significant changes. For Maildir, any change to any message UID or flags is significant, because checking the flags doesn't add a significant cost. For IMAP, only a change to the total number of messages or a change in the UID of the most recent message is significant. This should catch everything except for flags changes. The difference in bandwidth is astonishing: a quick sync takes 80K instead of 5.3MB, and 28 seconds instead of 90. There's a configuration variable that lets you say every tenth sync should update flags, but let all the intervening ones be lighter. Second, a fix to the UID validity problems many people have been reporting with Courier. As discussed in Debian bug #433732, I changed the UID validity check to use SELECT unless the server complains that the folder is read-only. This avoids the Courier bug (see the Debian log for more details). This won't fix existing validity errors, you need to remove the local status and validity files by hand and resync. Third, some speedups in Maildir checking. It's still pretty slow due to a combination of poor performance in os.listdir (never reads more than 4K of directory entries at a time) and some semaphore that leads to lots of futex wake operations, but at least this saves 20% or so of the CPU time running offlineimap on a single folder: Time with quick refresh and md5 in loop: 4.75s user 0.46s system 12% cpu 41.751 total Time with quick refresh and md5 out of loop: 4.38s user 0.50s system 14% cpu 34.799 total Time using string compare to check folder: 4.11s user 0.47s system 13% cpu 34.788 total And fourth, some display fixes for Curses.Blinkenlights. I made warnings more visible, made the new quick sync message cyan, and made all not explicitly colored messages grey. That last one was really bugging me. Any time OfflineIMAP printed a warning in this UI, it had even odds of coming out black on black! Anyway, I hope these are useful. I'm happy to revise them if you see a problem. -- Daniel Jacobowitz CodeSourcery
2007-10-01 23:20:37 +02:00
self.quicknum = 0
if self.refreshperiod == 0.0:
self.refreshperiod = None
def getlocaleval(self):
return self.localeval
# Interface from CustomConfig.ConfigHelperMixin
def getconfig(self):
return self.config
def getname(self):
return self.name
def __str__(self):
return self.name
def getaccountmeta(self):
return os.path.join(self.metadatadir, 'Account-' + self.name)
# Interface from CustomConfig.ConfigHelperMixin
def getsection(self):
return 'Account ' + self.getname()
@classmethod
def set_abort_event(cls, config, signum):
"""Set skip sleep/abort event for all accounts.
If we want to skip a current (or the next) sleep, or if we want
to abort an autorefresh loop, the main thread can use
set_abort_event() to send the corresponding signal. Signum = 1
implies that we want all accounts to abort or skip the current
or next sleep phase. Signum = 2 will end the autorefresh loop,
ie all accounts will return after they finished a sync. signum=3
means, abort NOW, e.g. on SIGINT or SIGTERM.
This is a class method, it will send the signal to all accounts.
"""
if signum == 1:
# resync signal, set config option for all accounts
for acctsection in getaccountlist(config):
config.set('Account ' + acctsection, "skipsleep", '1')
elif signum == 2:
# don't autorefresh anymore
cls.abort_soon_signal.set()
elif signum == 3:
# abort ASAP
cls.abort_NOW_signal.set()
def get_abort_event(self):
"""Checks if an abort signal had been sent.
If the 'skipsleep' config option for this account had been set,
with `set_abort_event(config, 1)` it will get cleared in this
function. Ie, we will only skip one sleep and not all.
:returns: True, if the main thread had called
:meth:`set_abort_event` earlier, otherwise 'False'.
"""
skipsleep = self.getconfboolean("skipsleep", 0)
if skipsleep:
self.config.set(self.getsection(), "skipsleep", '0')
return skipsleep or Account.abort_soon_signal.is_set() or \
Account.abort_NOW_signal.is_set()
def _sleeper(self):
"""Sleep if the account is set to autorefresh.
:returns: 0:timeout expired, 1: canceled the timer,
2:request to abort the program,
100: if configured to not sleep at all.
"""
if not self.refreshperiod:
return 100
kaobjs = []
if hasattr(self, 'localrepos'):
kaobjs.append(self.localrepos)
if hasattr(self, 'remoterepos'):
kaobjs.append(self.remoterepos)
for item in kaobjs:
item.startkeepalive()
refreshperiod = int(self.refreshperiod * 60)
sleepresult = self.ui.sleep(refreshperiod, self)
Patch for signal handling to start a sync by Jim Pryor Here's the way I'd like to use offlineimap on my laptop: 1. Have a regular cron job running infrequently. The cron job checks to see if I'm online, plugged in, and that no other copy of offlineimap is running. If all of these conditions are satisfied, it runs offlineimap just once: "offlineimap -o -u Noninteractive.Quiet" 2. When I start up mutt, I do it by calling a wrapper script that delays until cron-started copies of offlineimap have finished, then starts offlineimap on its regular, stay-alive and keep checking schedule. When I quit mutt, the wrapper script tells offlineimap to stop. This way I get frequent regular checks while I have mutt running, but I don't waste my battery/cpu checking frequently for mail when I'm not interested in it. To make this work, though, it'd be nicer if it were easier to tell offlineimap, from the outside, things like "terminate cleanly now" and "when you've finished synching, then terminate instead of sleeping and synching again." OK, to put my money where my mouth is, I attach two patches against offlineimap 6.0.3. The first, "cleanup.patch", cleans up a few spots that tend to throw exceptions for me as offlineimap is exiting from a KeyboardInterrupt. The second adds signaling capabilities to offlineimap. * sending a SIGTERM tells offlineimap to terminate immediately but cleanly, just as if "q" had been pressed in the GUI interface * sending a SIGUSR1 tells every account to do a full sync asap: if it's sleeping, then wake up and do the sync now. If it's mid-sync, then re-synch any folders whose syncing has already been started or completed, and continue to synch the other, queued but not-yet-synched folders. * sending a SIGHUP tells every account to die as soon as it can (but not immediately: only after finishing any synch it's now engaged in) * sending a SIGUSR2 tells every account to do a full sync asap (as with SIGUSR1), then die It's tricky to mix signals with threads, but I think I've done this correctly. I've been using it now for a few weeks without any obvious problems. But I'm passing it on so that others can review the code and test it out on their systems. I developed the patch when I was running Python 2.5.2, but to my knowledge I don't use any Python 2.5-specific code. Now I'm using the patch with Python 2.6. Although I said "without any obvious problems," let me confess that I'm seeing offlineimap regularly choke when I do things like this: start up my offlineimap-wrapped copy of mutt, wait a while, put the machine to sleep (not sure if offlineimap is active in the background or idling), move to a different spot, wake the machine up again and it acquires a new network, sometimes a wired network instead of wifi. Offlineimap doesn't like that so much. I don't yet have any reason to think the problems here come from my patches. But I'm just acknowledging them, so that if others are able to use offlineimap without any difficulty in situations like I described, then maybe the fault is with my patches.
2008-12-01 23:13:16 +01:00
# Cancel keepalive
for item in kaobjs:
item.stopkeepalive()
if sleepresult:
if Account.abort_soon_signal.is_set() or \
Account.abort_NOW_signal.is_set():
return 2
self.quicknum = 0
return 1
return 0
def serverdiagnostics(self):
"""Output diagnostics for all involved repositories."""
remote_repo = Repository(self, 'remote')
local_repo = Repository(self, 'local')
#status_repo = Repository(self, 'status')
self.ui.serverdiagnostics(remote_repo, 'Remote')
self.ui.serverdiagnostics(local_repo, 'Local')
#self.ui.serverdiagnostics(statusrepos, 'Status')
class SyncableAccount(Account):
"""A syncable email account connecting 2 repositories.
Derives from :class:`accounts.Account` but contains the additional
functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`,
used for syncing."""
def __init__(self, *args, **kwargs):
Account.__init__(self, *args, **kwargs)
self._lockfd = None
self._lockfilepath = os.path.join(
self.config.getmetadatadir(), "%s.lock"% self)
def __lock(self):
"""Lock the account, throwing an exception if it is locked already."""
self._lockfd = open(self._lockfilepath, 'w')
try:
fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
except NameError:
#fcntl not available (Windows), disable file locking... :(
pass
except IOError:
self._lockfd.close()
raise OfflineImapError("Could not lock account %s. Is another "
"instance using this account?"% self,
OfflineImapError.ERROR.REPO), None, exc_info()[2]
def _unlock(self):
"""Unlock the account, deleting the lock file"""
#If we own the lock file, delete it
if self._lockfd and not self._lockfd.closed:
self._lockfd.close()
try:
os.unlink(self._lockfilepath)
except OSError:
pass # Failed to delete for some reason.
def syncrunner(self):
self.ui.registerthread(self)
try:
accountmetadata = self.getaccountmeta()
if not os.path.exists(accountmetadata):
os.mkdir(accountmetadata, 0o700)
self.remoterepos = Repository(self, 'remote')
self.localrepos = Repository(self, 'local')
self.statusrepos = Repository(self, 'status')
except OfflineImapError as e:
self.ui.error(e, exc_info()[2])
if e.severity >= OfflineImapError.ERROR.CRITICAL:
raise
return
Patch for maxage and maxsize options Dear All, Attached is the patch that I have developed to provide maxage and maxsize options. You can thus sync only the last x days of messages and exclude large messages. All details in the attached git file. Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 04fead2b46a79675a5b29de6f2b4088b9c9448e5 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Sun, 16 Aug 2009 17:00:49 +0430 Subject: [PATCH] Patch to provide maxage and maxsize account options to exclude old/large messages This is designed to make offlineimap even better for low bandwidth connections. maxage allows you to specify a number of days and only messages within that range will be considered by offlineimap for the sync. This can be useful if you would like to start using offlineimap with a large existing account and do not want to import large archives of mail. maxsize allows you to specify the maximum size of a message to consider so that you can exclude messages with large attachments etc. In both cases the cachemessagelist function of the folder was modified to ignore messages that do not meet the criteria. If the criteria are not specified then the existing code will be executed the same as before. If a message does not meet the criteria it will be as though this message does not exist - offlineimap will completely ignore it. It will not have flags updated, it will not be deleted, it will not be considered at all. When operating against an IMAP repository a server side search function is used. This of course requires support for server side search. I have tested this with either option, no options etc. against IMAP, Maildir and Gmail. I have run variations of this patch here for the last 3 weeks or so syncing about 4 accounts normally.
2009-08-16 14:42:39 +02:00
# Loop account sync if needed (bail out after 3 failures)
looping = 3
while looping:
self.ui.acct(self)
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
try:
self.__lock()
self.__sync()
except (KeyboardInterrupt, SystemExit):
raise
except OfflineImapError as e:
# Stop looping and bubble up Exception if needed.
if e.severity >= OfflineImapError.ERROR.REPO:
if looping:
looping -= 1
if e.severity >= OfflineImapError.ERROR.CRITICAL:
raise
self.ui.error(e, exc_info()[2])
except Exception as e:
self.ui.error(e, exc_info()[2], msg=
"While attempting to sync account '%s'"% self)
else:
# after success sync, reset the looping counter to 3
if self.refreshperiod:
looping = 3
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
finally:
self.ui.acctdone(self)
self._unlock()
if looping and self._sleeper() >= 2:
looping = 0
Patch for maxage and maxsize options Dear All, Attached is the patch that I have developed to provide maxage and maxsize options. You can thus sync only the last x days of messages and exclude large messages. All details in the attached git file. Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 04fead2b46a79675a5b29de6f2b4088b9c9448e5 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Sun, 16 Aug 2009 17:00:49 +0430 Subject: [PATCH] Patch to provide maxage and maxsize account options to exclude old/large messages This is designed to make offlineimap even better for low bandwidth connections. maxage allows you to specify a number of days and only messages within that range will be considered by offlineimap for the sync. This can be useful if you would like to start using offlineimap with a large existing account and do not want to import large archives of mail. maxsize allows you to specify the maximum size of a message to consider so that you can exclude messages with large attachments etc. In both cases the cachemessagelist function of the folder was modified to ignore messages that do not meet the criteria. If the criteria are not specified then the existing code will be executed the same as before. If a message does not meet the criteria it will be as though this message does not exist - offlineimap will completely ignore it. It will not have flags updated, it will not be deleted, it will not be considered at all. When operating against an IMAP repository a server side search function is used. This of course requires support for server side search. I have tested this with either option, no options etc. against IMAP, Maildir and Gmail. I have run variations of this patch here for the last 3 weeks or so syncing about 4 accounts normally.
2009-08-16 14:42:39 +02:00
def get_local_folder(self, remotefolder):
"""Return the corresponding local folder for a given remotefolder."""
return self.localrepos.getfolder(
remotefolder.getvisiblename().
replace(self.remoterepos.getsep(), self.localrepos.getsep()))
def __sync(self):
"""Synchronize the account once, then return.
Assumes that `self.remoterepos`, `self.localrepos`, and
`self.statusrepos` has already been populated, so it should only
be called from the :meth:`syncrunner` function."""
folderthreads = []
Daniel Jacobowitz patches fixes deb#433732 Date: Sun, 30 Sep 2007 13:54:56 -0400 From: Daniel Jacobowitz <drow@false.org> To: offlineimap@complete.org Subject: Assorted patches Here's the result of a lazy Sunday hacking on offlineimap. Sorry for not breaking this into multiple patches. They're mostly logically independent so just ask if that would make a difference. First, a new -q (quick) option. The quick option means to only update folders that seem to have had significant changes. For Maildir, any change to any message UID or flags is significant, because checking the flags doesn't add a significant cost. For IMAP, only a change to the total number of messages or a change in the UID of the most recent message is significant. This should catch everything except for flags changes. The difference in bandwidth is astonishing: a quick sync takes 80K instead of 5.3MB, and 28 seconds instead of 90. There's a configuration variable that lets you say every tenth sync should update flags, but let all the intervening ones be lighter. Second, a fix to the UID validity problems many people have been reporting with Courier. As discussed in Debian bug #433732, I changed the UID validity check to use SELECT unless the server complains that the folder is read-only. This avoids the Courier bug (see the Debian log for more details). This won't fix existing validity errors, you need to remove the local status and validity files by hand and resync. Third, some speedups in Maildir checking. It's still pretty slow due to a combination of poor performance in os.listdir (never reads more than 4K of directory entries at a time) and some semaphore that leads to lots of futex wake operations, but at least this saves 20% or so of the CPU time running offlineimap on a single folder: Time with quick refresh and md5 in loop: 4.75s user 0.46s system 12% cpu 41.751 total Time with quick refresh and md5 out of loop: 4.38s user 0.50s system 14% cpu 34.799 total Time using string compare to check folder: 4.11s user 0.47s system 13% cpu 34.788 total And fourth, some display fixes for Curses.Blinkenlights. I made warnings more visible, made the new quick sync message cyan, and made all not explicitly colored messages grey. That last one was really bugging me. Any time OfflineIMAP printed a warning in this UI, it had even odds of coming out black on black! Anyway, I hope these are useful. I'm happy to revise them if you see a problem. -- Daniel Jacobowitz CodeSourcery
2007-10-01 23:20:37 +02:00
hook = self.getconf('presynchook', '')
self.callhook(hook)
Daniel Jacobowitz patches fixes deb#433732 Date: Sun, 30 Sep 2007 13:54:56 -0400 From: Daniel Jacobowitz <drow@false.org> To: offlineimap@complete.org Subject: Assorted patches Here's the result of a lazy Sunday hacking on offlineimap. Sorry for not breaking this into multiple patches. They're mostly logically independent so just ask if that would make a difference. First, a new -q (quick) option. The quick option means to only update folders that seem to have had significant changes. For Maildir, any change to any message UID or flags is significant, because checking the flags doesn't add a significant cost. For IMAP, only a change to the total number of messages or a change in the UID of the most recent message is significant. This should catch everything except for flags changes. The difference in bandwidth is astonishing: a quick sync takes 80K instead of 5.3MB, and 28 seconds instead of 90. There's a configuration variable that lets you say every tenth sync should update flags, but let all the intervening ones be lighter. Second, a fix to the UID validity problems many people have been reporting with Courier. As discussed in Debian bug #433732, I changed the UID validity check to use SELECT unless the server complains that the folder is read-only. This avoids the Courier bug (see the Debian log for more details). This won't fix existing validity errors, you need to remove the local status and validity files by hand and resync. Third, some speedups in Maildir checking. It's still pretty slow due to a combination of poor performance in os.listdir (never reads more than 4K of directory entries at a time) and some semaphore that leads to lots of futex wake operations, but at least this saves 20% or so of the CPU time running offlineimap on a single folder: Time with quick refresh and md5 in loop: 4.75s user 0.46s system 12% cpu 41.751 total Time with quick refresh and md5 out of loop: 4.38s user 0.50s system 14% cpu 34.799 total Time using string compare to check folder: 4.11s user 0.47s system 13% cpu 34.788 total And fourth, some display fixes for Curses.Blinkenlights. I made warnings more visible, made the new quick sync message cyan, and made all not explicitly colored messages grey. That last one was really bugging me. Any time OfflineIMAP printed a warning in this UI, it had even odds of coming out black on black! Anyway, I hope these are useful. I'm happy to revise them if you see a problem. -- Daniel Jacobowitz CodeSourcery
2007-10-01 23:20:37 +02:00
quickconfig = self.getconfint('quick', 0)
if quickconfig < 0:
quick = True
elif quickconfig > 0:
if self.quicknum == 0 or self.quicknum > quickconfig:
self.quicknum = 1
quick = False
else:
self.quicknum = self.quicknum + 1
quick = True
else:
quick = False
try:
remoterepos = self.remoterepos
localrepos = self.localrepos
statusrepos = self.statusrepos
# init repos with list of folders, so we have them (and the
# folder delimiter etc)
remoterepos.getfolders()
localrepos.getfolders()
remoterepos.sync_folder_structure(localrepos, statusrepos)
# replicate the folderstructure between REMOTE to LOCAL
if not localrepos.getconfboolean('readonly', False):
self.ui.syncfolders(remoterepos, localrepos)
# iterate through all folders on the remote repo and sync
for remotefolder in remoterepos.getfolders():
# check for CTRL-C or SIGTERM
if Account.abort_NOW_signal.is_set(): break
if not remotefolder.sync_this:
self.ui.debug('', "Not syncing filtered folder '%s'"
"[%s]"% (remotefolder, remoterepos))
continue # Ignore filtered folder
localfolder = self.get_local_folder(remotefolder)
if not localfolder.sync_this:
self.ui.debug('', "Not syncing filtered folder '%s'"
"[%s]"% (localfolder, localfolder.repository))
continue # Ignore filtered folder
if not globals.options.singlethreading:
thread = InstanceLimitedThread(\
instancename = 'FOLDER_' + self.remoterepos.getname(),
target = syncfolder,
name = "Folder %s [acc: %s]"% (remotefolder.getexplainedname(), self),
args = (self, remotefolder, quick))
thread.start()
folderthreads.append(thread)
else:
syncfolder(self, remotefolder, quick)
# wait for all threads to finish
for thr in folderthreads:
thr.join()
# Write out mailbox names if required and not in dry-run mode
if not self.dryrun:
mbnames.write(False)
localrepos.forgetfolders()
remoterepos.forgetfolders()
except:
#error while syncing. Drop all connections that we have, they
#might be bogus by now (e.g. after suspend)
localrepos.dropconnections()
remoterepos.dropconnections()
raise
else:
# sync went fine. Hold or drop depending on config
localrepos.holdordropconnections()
remoterepos.holdordropconnections()
hook = self.getconf('postsynchook', '')
self.callhook(hook)
def callhook(self, cmd):
# check for CTRL-C or SIGTERM and run postsynchook
if Account.abort_NOW_signal.is_set():
return
if not cmd:
return
try:
self.ui.callhook("Calling hook: " + cmd)
if self.dryrun: # don't if we are in dry-run mode
return
p = Popen(cmd, shell=True,
stdin=PIPE, stdout=PIPE, stderr=PIPE,
close_fds=True)
r = p.communicate()
self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n"% r)
self.ui.callhook("Hook return code: %d"% p.returncode)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
self.ui.error(e, exc_info()[2], msg="Calling hook")
def syncfolder(account, remotefolder, quick):
"""Synchronizes given remote folder for the specified account.
Filtered folders on the remote side will not invoke this function. However,
this might be called in a concurrently."""
def check_uid_validity(localfolder, remotefolder, statusfolder):
# If either the local or the status folder has messages and
# there is a UID validity problem, warn and abort. If there are
# no messages, UW IMAPd loses UIDVALIDITY. But we don't really
# need it if both local folders are empty. So, in that case,
# just save it off.
if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount() > 0:
if not localfolder.check_uidvalidity():
ui.validityproblem(localfolder)
localfolder.repository.restore_atime()
return
if not remotefolder.check_uidvalidity():
ui.validityproblem(remotefolder)
localrepos.restore_atime()
return
else:
# Both folders empty, just save new UIDVALIDITY
localfolder.save_uidvalidity()
remotefolder.save_uidvalidity()
def save_min_uid(folder, min_uid):
uidfile = folder.get_min_uid_file()
fd = open(uidfile, 'wt')
fd.write(str(min_uid) + "\n")
fd.close()
def cachemessagelists_upto_date(localfolder, remotefolder, date):
"""Returns messages with uid > min(uids of messages newer than date)."""
localfolder.cachemessagelist(min_date=date)
check_uid_validity(localfolder, remotefolder, statusfolder)
# local messagelist had date restriction applied already. Restrict
# sync to messages with UIDs >= min_uid from this list.
#
# local messagelist might contain new messages (with uid's < 0).
positive_uids = filter(
lambda uid: uid > 0, localfolder.getmessageuidlist())
if len(positive_uids) > 0:
remotefolder.cachemessagelist(min_uid=min(positive_uids))
else:
# No messages with UID > 0 in range in localfolder.
# date restriction was applied with respect to local dates but
# remote folder timezone might be different from local, so be
# safe and make sure the range isn't bigger than in local.
remotefolder.cachemessagelist(
min_date=time.gmtime(time.mktime(date) + 24*60*60))
def cachemessagelists_startdate(new, partial, date):
"""Retrieve messagelists when startdate has been set for
the folder 'partial'.
Idea: suppose you want to clone the messages after date in one
account (partial) to a new one (new). If new is empty, then copy
messages in partial newer than date to new, and keep track of the
min uid. On subsequent syncs, sync all the messages in new against
those after that min uid in partial. This is a partial replacement
for maxage in the IMAP-IMAP sync case, where maxage doesn't work:
the UIDs of the messages in localfolder might not be in the same
order as those of corresponding messages in remotefolder, so if L in
local corresponds to R in remote, the ranges [L, ...] and [R, ...]
might not correspond. But, if we're cloning a folder into a new one,
[min_uid, ...] does correspond to [1, ...].
This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead."""
new.cachemessagelist()
min_uid = partial.retrieve_min_uid()
if min_uid == None: # min_uid file didn't exist
if len(new.getmessageuidlist()) > 0:
raise OfflineImapError("To use startdate on Repository %s, "
"Repository %s must be empty"%
(partial.repository.name, new.repository.name),
OfflineImapError.ERROR.MESSAGE)
else:
partial.cachemessagelist(min_date=date)
# messagelist.keys() instead of getuidmessagelist() because in
# the UID mapped case we want the actual local UIDs, not their
# remote counterparts
positive_uids = filter(
lambda uid: uid > 0, partial.messagelist.keys())
if len(positive_uids) > 0:
min_uid = min(positive_uids)
else:
min_uid = 1
save_min_uid(partial, min_uid)
else:
partial.cachemessagelist(min_uid=min_uid)
remoterepos = account.remoterepos
localrepos = account.localrepos
statusrepos = account.statusrepos
ui = getglobalui()
ui.registerthread(account)
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
try:
# Load local folder.
localfolder = account.get_local_folder(remotefolder)
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
# Write the mailboxes
mbnames.add(account.name, localfolder.getname(),
localrepos.getlocalroot())
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
# Load status folder.
statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().
replace(remoterepos.getsep(), statusrepos.getsep()))
statusfolder.openfiles()
if localfolder.get_uidvalidity() == None:
# This is a new folder, so delete the status cache to be
# sure we don't have a conflict.
# TODO: This does not work. We always return a value, need
# to rework this...
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
statusfolder.deletemessagelist()
statusfolder.cachemessagelist()
# Load local folder.
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)
# Retrieve messagelists, taking into account age-restriction
# options
maxage = localfolder.getmaxage()
localstart = localfolder.getstartdate()
remotestart = remotefolder.getstartdate()
if (maxage != None) + (localstart != None) + (remotestart != None) > 1:
raise OfflineImapError("You can set at most one of the "
"following: maxage, startdate (for the local folder), "
"startdate (for the remote folder)",
OfflineImapError.ERROR.REPO), None, exc_info()[2]
if (maxage != None or localstart or remotestart) and quick:
# IMAP quickchanged isn't compatible with options that
# involve restricting the messagelist, since the "quick"
# check can only retrieve a full list of UIDs in the folder.
ui.warn("Quick syncs (-q) not supported in conjunction "
"with maxage or startdate; ignoring -q.")
if maxage != None:
cachemessagelists_upto_date(localfolder, remotefolder, maxage)
elif localstart != None:
cachemessagelists_startdate(remotefolder, localfolder,
localstart)
check_uid_validity(localfolder, remotefolder, statusfolder)
elif remotestart != None:
cachemessagelists_startdate(localfolder, remotefolder,
remotestart)
check_uid_validity(localfolder, remotefolder, statusfolder)
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
else:
localfolder.cachemessagelist()
if quick:
if (not localfolder.quickchanged(statusfolder) and
not remotefolder.quickchanged(statusfolder)):
ui.skippingfolder(remotefolder)
localrepos.restore_atime()
return
check_uid_validity(localfolder, remotefolder, statusfolder)
remotefolder.cachemessagelist()
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
# Synchronize remote changes.
if not localrepos.getconfboolean('readonly', False):
ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder)
remotefolder.syncmessagesto(localfolder, statusfolder)
else:
ui.debug('imap', "Not syncing to read-only repository '%s'" \
% localrepos.getname())
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
# Synchronize local changes
if not remoterepos.getconfboolean('readonly', False):
ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder)
localfolder.syncmessagesto(remotefolder, statusfolder)
else:
ui.debug('', "Not syncing to read-only repository '%s'" \
% remoterepos.getname())
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
statusfolder.save()
localrepos.restore_atime()
except (KeyboardInterrupt, SystemExit):
raise
except OfflineImapError as e:
# bubble up severe Errors, skip folder otherwise
if e.severity > OfflineImapError.ERROR.FOLDER:
raise
else:
ui.error(e, exc_info()[2], msg="Aborting sync, folder '%s' "
"[acc: '%s']"% (localfolder, account))
except Exception as e:
ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s"%
(account, remotefolder.getvisiblename(), traceback.format_exc()))
finally:
for folder in ["statusfolder", "localfolder", "remotefolder"]:
if folder in locals():
locals()[folder].dropmessagelistcache()
statusfolder.closefiles()