docker-offlineimap/offlineimap/folder/Base.py

440 lines
18 KiB
Python
Raw Normal View History

# Base folder support
# Copyright (C) 2002-2011 John Goerzen & contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
2006-08-12 06:15:55 +02:00
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
2002-07-04 05:59:19 +02:00
from offlineimap import threadutil
from offlineimap.ui import getglobalui
from offlineimap.error import OfflineImapError
import os.path
import re
from sys import exc_info
import traceback
2002-06-21 08:51:21 +02:00
class BaseFolder(object):
def __init__(self):
self.ui = getglobalui()
def getname(self):
"""Returns name"""
return self.name
def __str__(self):
return self.name
def suggeststhreads(self):
"""Returns true if this folder suggests using threads for actions;
false otherwise. Probably only IMAP will return true."""
return 0
2002-07-04 05:59:19 +02:00
def getcopyinstancelimit(self):
"""For threading folders, returns the instancelimitname for
InstanceLimitedThreads."""
raise NotImplementedException
def storesmessages(self):
"""Should be true for any backend that actually saves message bodies.
(Almost all of them). False for the LocalStatus backend. Saves
us from having to slurp up messages just for localstatus purposes."""
return 1
2002-06-21 08:51:21 +02:00
def getvisiblename(self):
return self.name
def getrepository(self):
"""Returns the repository object that this folder is within."""
return self.repository
def getroot(self):
"""Returns the root of the folder, in a folder-specific fashion."""
return self.root
def getsep(self):
"""Returns the separator for this folder type."""
return self.sep
def getfullname(self):
if self.getroot():
return self.getroot() + self.getsep() + self.getname()
else:
return self.getname()
def getfolderbasename(self):
foldername = self.getname()
foldername = foldername.replace(self.repository.getsep(), '.')
foldername = re.sub('/\.$', '/dot', foldername)
foldername = re.sub('^\.$', 'dot', foldername)
return foldername
def isuidvalidityok(self):
"""Does the cached UID match the real UID
If required it caches the UID. In this case the function is not
threadsafe. So don't attempt to call it from concurrent threads."""
if self.getsaveduidvalidity() != None:
return self.getsaveduidvalidity() == self.getuidvalidity()
else:
self.saveuidvalidity()
return 1
def _getuidfilename(self):
return os.path.join(self.repository.getuiddir(),
self.getfolderbasename())
def getsaveduidvalidity(self):
if hasattr(self, '_base_saved_uidvalidity'):
return self._base_saved_uidvalidity
uidfilename = self._getuidfilename()
if not os.path.exists(uidfilename):
self._base_saved_uidvalidity = None
else:
file = open(uidfilename, "rt")
self._base_saved_uidvalidity = long(file.readline().strip())
file.close()
return self._base_saved_uidvalidity
def saveuidvalidity(self):
"""Save the UID value of the folder to the status
This function is not threadsafe, so don't attempt to call it
from concurrent threads."""
newval = self.getuidvalidity()
uidfilename = self._getuidfilename()
file = open(uidfilename + ".tmp", "wt")
file.write("%d\n" % newval)
file.close()
os.rename(uidfilename + ".tmp", uidfilename)
self._base_saved_uidvalidity = newval
def getuidvalidity(self):
raise NotImplementedException
def cachemessagelist(self):
"""Reads the message list from disk or network and stores it in
memory for later use. This list will not be re-read from disk or
memory unless this function is called again."""
raise NotImplementedException
def getmessagelist(self):
2002-06-20 05:50:58 +02:00
"""Gets the current message list.
You must call cachemessagelist() before calling this function!"""
raise NotImplementedException
def uidexists(self, uid):
"""Returns True if uid exists"""
return uid in self.getmessagelist()
def getmessageuidlist(self):
"""Gets a list of UIDs.
You may have to call cachemessagelist() before calling this function!"""
return self.getmessagelist().keys()
def getmessagecount(self):
"""Gets the number of messages."""
return len(self.getmessagelist())
2002-06-20 05:50:58 +02:00
def getmessage(self, uid):
"""Returns the content of the specified message."""
raise NotImplementedException
def savemessage(self, uid, content, flags, rtime):
"""Writes a new message, with the specified uid.
If the uid is < 0: The backend should assign a new uid and
return it. In case it cannot assign a new uid, it returns
the negative uid passed in WITHOUT saving the message.
If the backend CAN assign a new uid, but cannot find out what
this UID is (as is the case with some IMAP servers), it
returns 0 but DOES save the message.
IMAP backend should be the only one that can assign a new
uid.
If the uid is > 0, the backend should set the uid to this, if it can.
If it cannot set the uid to that, it will save it anyway.
It will return the uid assigned in any case.
"""
2002-06-20 05:50:58 +02:00
raise NotImplementedException
def getmessagetime(self, uid):
"""Return the received time for the specified message."""
raise NotImplementedException
2002-06-20 05:50:58 +02:00
def getmessageflags(self, uid):
"""Returns the flags for the specified message."""
raise NotImplementedException
def savemessageflags(self, uid, flags):
"""Sets the specified message's flags to the given set."""
raise NotImplementedException
def addmessageflags(self, uid, flags):
"""Adds the specified flags to the message's flag set. If a given
flag is already present, it will not be duplicated."""
newflags = self.getmessageflags(uid)
for flag in flags:
if not flag in newflags:
newflags.append(flag)
newflags.sort()
self.savemessageflags(uid, newflags)
def addmessagesflags(self, uidlist, flags):
for uid in uidlist:
self.addmessageflags(uid, flags)
def deletemessageflags(self, uid, flags):
"""Removes each flag given from the message's flag set. If a given
flag is already removed, no action will be taken for that flag."""
newflags = self.getmessageflags(uid)
for flag in flags:
if flag in newflags:
newflags.remove(flag)
newflags.sort()
self.savemessageflags(uid, newflags)
def deletemessagesflags(self, uidlist, flags):
for uid in uidlist:
self.deletemessageflags(uid, flags)
2002-06-20 05:50:58 +02:00
def deletemessage(self, uid):
raise NotImplementedException
2002-06-21 03:55:06 +02:00
def deletemessages(self, uidlist):
for uid in uidlist:
self.deletemessage(uid)
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
def copymessageto(self, uid, dstfolder, statusfolder, register = 1):
"""Copies a message from self to dst if needed, updating the status
:param uid: uid of the message to be copied.
:param dstfolder: A BaseFolder-derived instance
:param statusfolder: A LocalStatusFolder instance
:param register: whether we should register a new thread."
:returns: Nothing on success, or raises an Exception."""
# Sometimes, it could be the case that if a sync takes awhile,
# a message might be deleted from the maildir before it can be
# synced to the status cache. This is only a problem with
# self.getmessage(). So, don't call self.getmessage unless
# really needed.
if register: # output that we start a new thread
self.ui.registerthread(self.getaccountname())
try:
message = None
flags = self.getmessageflags(uid)
rtime = self.getmessagetime(uid)
if uid > 0 and dstfolder.uidexists(uid):
# dst has message with that UID already, only update status
statusfolder.savemessage(uid, None, flags, rtime)
return
self.ui.copyingmessage(uid, self, [dstfolder])
# If any of the destinations actually stores the message body,
# load it up.
if dstfolder.storesmessages():
message = self.getmessage(uid)
#Succeeded? -> IMAP actually assigned a UID. If newid
#remained negative, no server was willing to assign us an
#UID. If newid is 0, saving succeeded, but we could not
#retrieve the new UID. Ignore message in this case.
newuid = dstfolder.savemessage(uid, message, flags, rtime)
if newuid > 0:
if newuid != uid:
# Got new UID, change the local uid.
#TODO: Maildir could do this with a rename rather than
#load/save/del operation, IMPLEMENT a changeuid()
#function or so.
self.savemessage(newuid, message, flags, rtime)
self.deletemessage(uid)
uid = newuid
# Save uploaded status in the statusfolder
statusfolder.savemessage(uid, message, flags, rtime)
elif newuid == 0:
# Message was stored to dstfolder, but we can't find it's UID
# This means we can't link current message to the one created
# in IMAP. So we just delete local message and on next run
# we'll sync it back
# XXX This could cause infinite loop on syncing between two
# IMAP servers ...
self.deletemessage(uid)
else:
raise OfflineImapError("Trying to save msg (uid %d) on folder "
"%s returned invalid uid %d" % \
(uid,
dstfolder.getvisiblename(),
newuid),
OfflineImapError.ERROR.MESSAGE)
except OfflineImapError, e:
if e.severity > OfflineImapError.ERROR.MESSAGE:
raise # buble severe errors up
self.ui.error(e, exc_info()[2])
except Exception, e:
self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\
(uid, self.getaccountname(),
traceback.format_exc()))
raise #raise on unknown errors, so we can fix those
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
def syncmessagesto_copy(self, dstfolder, statusfolder):
"""Pass1: Copy locally existing messages not on the other side
This will copy messages to dstfolder that exist locally but are
not in the statusfolder yet. The strategy is:
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
1) Look for messages present in self but not in statusfolder.
2) invoke copymessageto() on those which:
- If dstfolder doesn't have it yet, add them to dstfolder.
- Update statusfolder
"""
threads = []
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
copylist = filter(lambda uid: not \
statusfolder.uidexists(uid),
self.getmessageuidlist())
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
for uid in copylist:
# exceptions are caught in copymessageto()
if self.suggeststhreads():
self.waitforthread()
thread = threadutil.InstanceLimitedThread(\
self.getcopyinstancelimit(),
target = self.copymessageto,
name = "Copy message %d from %s" % (uid,
self.getvisiblename()),
args = (uid, dstfolder, statusfolder))
thread.setDaemon(1)
thread.start()
threads.append(thread)
else:
self.copymessageto(uid, dstfolder, statusfolder,
register = 0)
for thread in threads:
thread.join()
2002-06-20 05:50:58 +02:00
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
def syncmessagesto_delete(self, dstfolder, statusfolder):
"""Pass 2: Remove locally deleted messages on dst
2002-06-20 05:50:58 +02:00
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
Get all UIDS in statusfolder but not self. These are messages
that were deleted in 'self'. Delete those from dstfolder and
statusfolder."""
deletelist = filter(lambda uid: uid>=0 \
and not self.uidexists(uid),
statusfolder.getmessageuidlist())
if len(deletelist):
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
self.ui.deletingmessages(deletelist, [dstfolder])
# delete in statusfolder first to play safe. In case of abort, we
# won't lose message, we will just retransmit some unneccessary.
for folder in [statusfolder, dstfolder]:
folder.deletemessages(deletelist)
def syncmessagesto_flags(self, dstfolder, statusfolder):
"""Pass 3: Flag synchronization
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
Compare flag mismatches in self with those in statusfolder. If
msg has a valid UID and exists on dstfolder (has not e.g. been
deleted there), sync the flag change to both dstfolder and
statusfolder.
"""
# For each flag, we store a list of uids to which it should be
# added. Then, we can call addmessagesflags() to apply them in
# bulk, rather than one call per message.
addflaglist = {}
delflaglist = {}
for uid in self.getmessageuidlist():
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
# Ignore messages with negative UIDs missed by pass 1
# also don't do anything if the message has been deleted remotely
if uid < 0 or not dstfolder.uidexists(uid):
continue
2002-06-20 05:50:58 +02:00
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
selfflags = self.getmessageflags(uid)
statusflags = statusfolder.getmessageflags(uid)
#if we could not get message flags from LocalStatus, assume empty.
if statusflags is None:
statusflags = []
addflags = [x for x in selfflags if x not in statusflags]
for flag in addflags:
if not flag in addflaglist:
addflaglist[flag] = []
addflaglist[flag].append(uid)
2002-06-20 06:14:54 +02:00
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
delflags = [x for x in statusflags if x not in selfflags]
for flag in delflags:
if not flag in delflaglist:
delflaglist[flag] = []
delflaglist[flag].append(uid)
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
for flag in addflaglist.keys():
self.ui.addingflags(addflaglist[flag], flag, dstfolder)
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
dstfolder.addmessagesflags(addflaglist[flag], [flag])
statusfolder.addmessagesflags(addflaglist[flag], [flag])
for flag in delflaglist.keys():
self.ui.deletingflags(delflaglist[flag], flag, dstfolder)
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
dstfolder.deletemessagesflags(delflaglist[flag], [flag])
statusfolder.deletemessagesflags(delflaglist[flag], [flag])
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
def syncmessagesto(self, dstfolder, statusfolder):
"""Syncs messages in this folder to the destination dstfolder.
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
This is the high level entry for syncing messages in one direction.
Syncsteps are:
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
Pass1: Copy locally existing messages
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
Copy messages in self, but not statusfolder to dstfolder if not
already in dstfolder. dstfolder might assign a new UID (e.g. if
uploading to IMAP). Update statusfolder.
Patch for error handling / separation of accounts etc. Dear All, I have made the attached patch to try and make offlineimap a bit more stable in challenging situations. It's extremely useful in slow connection environments - but sometimes if one account had the wrong password or the connection went down then unfortunately the whole program would crash. I have tested this on our connection and tried throwing at it just about every situation - connection, up down, up, down again, change password, error whilst copying one message, etc. I have been running this patch for the last 5 days or so syncing 6 accounts at the moment... It seems to work and stay alive nicely (even if your connection does not)... Hope that this can go in for the next release... Please let me know if anyone notices any problems with this... Regards, -Mike -- Attached file included as plaintext by Ecartis -- -- File: submit From 1d6777cab23637eb830031c7cab0ae9b8589afd6 Mon Sep 17 00:00:00 2001 From: mike <mike@mikelaptop.(none)> Date: Mon, 24 Aug 2009 19:37:59 +0430 Subject: [PATCH] This patch attempts to introduce a little more error handling - e.g. if one account has an error because of a changed password or something that should not affect the other accounts. Specifically: If one sync run has an issue this is in a try-except clause - if it has an auto refresh period the thread will sleep and try again - this could be quite useful in the event of the connection going down for a little while, changed password etc. If one folder cannot be created an error message will be displayed through the UI and the program will continue (e.g. permission denied to create a folder) If one message does not want to copy for whatever resaon an error message will be displayed through the UI and at least the other messages will be copied If one folder run has an exception then the others will still run
2009-08-24 17:17:57 +02:00
Pass2: Remove locally deleted messages
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
Get all UIDS in statusfolder but not self. These are messages
that were deleted in 'self'. Delete those from dstfolder and
statusfolder.
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
After this pass, the message lists should be identical wrt the
uids present (except for potential negative uids that couldn't
be placed anywhere).
Pass3: Synchronize flag changes
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
Compare flag mismatches in self with those in statusfolder. If
msg has a valid UID and exists on dstfolder (has not e.g. been
deleted there), sync the flag change to both dstfolder and
statusfolder.
:param dstfolder: Folderinstance to sync the msgs to.
:param statusfolder: LocalStatus instance to sync against.
"""
passes = [('copying messages' , self.syncmessagesto_copy),
Simplify the syncing strategy a bit The previous syncing strategy was doing more than we needed to and was a bit underdocumented. This is an attempt to clean it up. 1) Do away with the previous different code paths depending on whether there is a LocalStatus file or not (the isnewfolder() test). We always use the same strategy now, which makes the strategy easier to understand. This strategy is simply: a) Sync remote to local folder first b) Sync local to remote Where each sync implies a 4 pass strategy which does basically the same as before (explained below). 2) Don't delete messages on LOCAL which don't exist on REMOTE right at the beginning anymore. This prevented us e.g. from keeping local messages rather than redownloading everything once LocalStatus got corrupted or deleted. This surprised many who put in an existing local maildir and expected it to be synced to the remote place. Instead, the local maildir was deleted. This is a data loss that actually occured to people! 3) No need to separately sync the statusfolder, we update that one simultanously with the destfolders... 3) Simplified the sync function API by only taking one destdir rather than a list of destdirs, we never used more anyway. This makes the code easier to read. 4) Added plenty of code comments while I was going through to make sure the strategy is easy to understand. ----------------------------------------- Pass1: Transfer new local messages Upload msg with negative/no UIDs to dstfolder. dstfolder should assign that message a new UID. Update statusfolder. Pass2: Copy existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. Update statusfolder. Pass3: Remove deleted messages Get all UIDS in statusfolder but not self. These are messages that we have locally deleted. Delete those from dstfolder and statusfolder. Pass4: Synchronize flag changes Compare flags in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to dstfolder and statusfolder. The user visible implications of this change should be unnoticable except in one situation: Blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de> Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2011-03-06 20:03:04 +01:00
('deleting messages' , self.syncmessagesto_delete),
('syncing flags' , self.syncmessagesto_flags)]
for (passdesc, action) in passes:
try:
action(dstfolder, statusfolder)
except (KeyboardInterrupt):
raise
except OfflineImap, e:
if e.severity > OfflineImapError.ERROR.FOLDER:
raise
self.ui.error(e, exc_info()[2])
except Exception, e:
self.ui.error(e, msg = "ERROR attempting to sync folder %s "
"[acc: %s]:\n %s" (self, self.getaccountname(),
traceback.format_exc()))
raise # raise unknown Exceptions so we can fix them