tobias/docker-offlineimap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

avoid removing of data when user removed a maildir

Browse Source 
When a maildir is removed it must be considered new for the sync. However, the
local cache of the folder remains. This means the sync of the folder removes all
the missing emails.

Avoid loosing of data for users not aware of the local cache by removing any
pre-existing status cache of a folder when we actually want to create the
database.

Improve style.

Github-fix: https://github.com/OfflineIMAP/offlineimap/issues/333
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
This commit is contained in:
Nicolas Sebrecht 2016-06-17 19:47:37 +02:00
parent 08e17de7e2
commit 1410a391bc
5 changed files with 63 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
offlineimap/accounts.py
View File

@ -83,10 +83,9 @@ class Account(CustomConfig.ConfigHelperMixin):
self.name = name self.name = name
self.metadatadir = config.getmetadatadir() self.metadatadir = config.getmetadatadir()
self.localeval = config.getlocaleval() self.localeval = config.getlocaleval()
# current :mod:`offlineimap.ui`, can be used for logging: # Current :mod:`offlineimap.ui`, can be used for logging:
self.ui = getglobalui() self.ui = getglobalui()
self.refreshperiod = self.getconffloat('autorefresh', 0.0) self.refreshperiod = self.getconffloat('autorefresh', 0.0)
# should we run in "dry-run" mode?
self.dryrun = self.config.getboolean('general', 'dry-run') self.dryrun = self.config.getboolean('general', 'dry-run')
self.quicknum = 0 self.quicknum = 0
if self.refreshperiod == 0.0: if self.refreshperiod == 0.0:
@ -262,7 +261,7 @@ class SyncableAccount(Account):
raise raise
return return
# Loop account sync if needed (bail out after 3 failures) # Loop account sync if needed (bail out after 3 failures).
looping = 3 looping = 3
while looping: while looping:
self.ui.acct(self) self.ui.acct(self)
@ -329,30 +328,30 @@ class SyncableAccount(Account):
localrepos = self.localrepos localrepos = self.localrepos
statusrepos = self.statusrepos statusrepos = self.statusrepos
# init repos with list of folders, so we have them (and the # Init repos with list of folders, so we have them (and the
# folder delimiter etc) # folder delimiter etc).
remoterepos.getfolders() remoterepos.getfolders()
localrepos.getfolders() localrepos.getfolders()
remoterepos.sync_folder_structure(localrepos, statusrepos) remoterepos.sync_folder_structure(localrepos, statusrepos)
# replicate the folderstructure between REMOTE to LOCAL # Replicate the folderstructure between REMOTE to LOCAL.
if not localrepos.getconfboolean('readonly', False): if not localrepos.getconfboolean('readonly', False):
self.ui.syncfolders(remoterepos, localrepos) self.ui.syncfolders(remoterepos, localrepos)
# iterate through all folders on the remote repo and sync # Iterate through all folders on the remote repo and sync.
for remotefolder in remoterepos.getfolders(): for remotefolder in remoterepos.getfolders():
# check for CTRL-C or SIGTERM # Check for CTRL-C or SIGTERM.
if Account.abort_NOW_signal.is_set(): break if Account.abort_NOW_signal.is_set(): break
if not remotefolder.sync_this: if not remotefolder.sync_this:
self.ui.debug('', "Not syncing filtered folder '%s'" self.ui.debug('', "Not syncing filtered folder '%s'"
"[%s]"% (remotefolder, remoterepos)) "[%s]"% (remotefolder, remoterepos))
continue # Ignore filtered folder continue # Ignore filtered folder.
localfolder = self.get_local_folder(remotefolder) localfolder = self.get_local_folder(remotefolder)
if not localfolder.sync_this: if not localfolder.sync_this:
self.ui.debug('', "Not syncing filtered folder '%s'" self.ui.debug('', "Not syncing filtered folder '%s'"
"[%s]"% (localfolder, localfolder.repository)) "[%s]"% (localfolder, localfolder.repository))
continue # Ignore filtered folder continue # Ignore filtered folder.
if not globals.options.singlethreading: if not globals.options.singlethreading:
thread = InstanceLimitedThread( thread = InstanceLimitedThread(
limitNamespace = "%s%s"% ( limitNamespace = "%s%s"% (
@ -366,20 +365,20 @@ class SyncableAccount(Account):
folderthreads.append(thread) folderthreads.append(thread)
else: else:
syncfolder(self, remotefolder, quick) syncfolder(self, remotefolder, quick)
# wait for all threads to finish # Wait for all threads to finish.
for thr in folderthreads: for thr in folderthreads:
thr.join() thr.join()
mbnames.writeIntermediateFile(self.name) # Write out mailbox names. mbnames.writeIntermediateFile(self.name) # Write out mailbox names.
localrepos.forgetfolders() localrepos.forgetfolders()
remoterepos.forgetfolders() remoterepos.forgetfolders()
except: except:
#error while syncing. Drop all connections that we have, they # Error while syncing. Drop all connections that we have, they
#might be bogus by now (e.g. after suspend) # might be bogus by now (e.g. after suspend).
localrepos.dropconnections() localrepos.dropconnections()
remoterepos.dropconnections() remoterepos.dropconnections()
raise raise
else: else:
# sync went fine. Hold or drop depending on config # Sync went fine. Hold or drop depending on config.
localrepos.holdordropconnections() localrepos.holdordropconnections()
remoterepos.holdordropconnections() remoterepos.holdordropconnections()
@ -387,14 +386,14 @@ class SyncableAccount(Account):
self.callhook(hook) self.callhook(hook)
def callhook(self, cmd): def callhook(self, cmd):
# check for CTRL-C or SIGTERM and run postsynchook # Check for CTRL-C or SIGTERM and run postsynchook.
if Account.abort_NOW_signal.is_set(): if Account.abort_NOW_signal.is_set():
return return
if not cmd: if not cmd:
return return
try: try:
self.ui.callhook("Calling hook: " + cmd) self.ui.callhook("Calling hook: " + cmd)
if self.dryrun: # don't if we are in dry-run mode if self.dryrun:
return return
p = Popen(cmd, shell=True, p = Popen(cmd, shell=True,
stdin=PIPE, stdout=PIPE, stderr=PIPE, stdin=PIPE, stdout=PIPE, stderr=PIPE,
@ -428,7 +427,7 @@ def syncfolder(account, remotefolder, quick):
localrepos.restore_atime() localrepos.restore_atime()
return return
else: else:
# Both folders empty, just save new UIDVALIDITY # Both folders empty, just save new UIDVALIDITY.
localfolder.save_uidvalidity() localfolder.save_uidvalidity()
remotefolder.save_uidvalidity() remotefolder.save_uidvalidity()
@ -443,10 +442,10 @@ def syncfolder(account, remotefolder, quick):
localfolder.cachemessagelist(min_date=date) localfolder.cachemessagelist(min_date=date)
check_uid_validity(localfolder, remotefolder, statusfolder) check_uid_validity(localfolder, remotefolder, statusfolder)
# local messagelist had date restriction applied already. Restrict # Local messagelist had date restriction applied already. Restrict
# sync to messages with UIDs >= min_uid from this list. # sync to messages with UIDs >= min_uid from this list.
# #
# local messagelist might contain new messages (with uid's < 0). # Local messagelist might contain new messages (with uid's < 0).
positive_uids = [uid for uid in localfolder.getmessageuidlist() if uid > 0] positive_uids = [uid for uid in localfolder.getmessageuidlist() if uid > 0]
if len(positive_uids) > 0: if len(positive_uids) > 0:
remotefolder.cachemessagelist(min_uid=min(positive_uids)) remotefolder.cachemessagelist(min_uid=min(positive_uids))
@ -489,7 +488,7 @@ def syncfolder(account, remotefolder, quick):
partial.cachemessagelist(min_date=date) partial.cachemessagelist(min_date=date)
# messagelist.keys() instead of getuidmessagelist() because in # messagelist.keys() instead of getuidmessagelist() because in
# the UID mapped case we want the actual local UIDs, not their # the UID mapped case we want the actual local UIDs, not their
# remote counterparts # remote counterparts.
positive_uids = [uid for uid in list(partial.messagelist.keys()) if uid > 0] positive_uids = [uid for uid in list(partial.messagelist.keys()) if uid > 0]
if len(positive_uids) > 0: if len(positive_uids) > 0:
min_uid = min(positive_uids) min_uid = min(positive_uids)
@ -533,7 +532,7 @@ def syncfolder(account, remotefolder, quick):
ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)
# Retrieve messagelists, taking into account age-restriction # Retrieve messagelists, taking into account age-restriction
# options # options.
maxage = localfolder.getmaxage() maxage = localfolder.getmaxage()
localstart = localfolder.getstartdate() localstart = localfolder.getstartdate()
remotestart = remotefolder.getstartdate() remotestart = remotefolder.getstartdate()
@ -590,7 +589,7 @@ def syncfolder(account, remotefolder, quick):
except (KeyboardInterrupt, SystemExit): except (KeyboardInterrupt, SystemExit):
raise raise
except OfflineImapError as e: except OfflineImapError as e:
# bubble up severe Errors, skip folder otherwise # Bubble up severe Errors, skip folder otherwise.
if e.severity > OfflineImapError.ERROR.FOLDER: if e.severity > OfflineImapError.ERROR.FOLDER:
raise raise
else: else:

18
offlineimap/folder/LocalStatus.py
View File

@ -1,5 +1,5 @@
# Local status cache virtual folder # Local status cache virtual folder
# Copyright (C) 2002-2015 John Goerzen & contributors # Copyright (C) 2002-2016 John Goerzen & contributors
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@ -18,11 +18,10 @@
from sys import exc_info from sys import exc_info
import os import os
import threading import threading
import six
from .Base import BaseFolder from .Base import BaseFolder
import six
class LocalStatusFolder(BaseFolder): class LocalStatusFolder(BaseFolder):
"""LocalStatus backend implemented as a plain text file.""" """LocalStatus backend implemented as a plain text file."""
@ -73,8 +72,8 @@ class LocalStatusFolder(BaseFolder):
uid = int(uid) uid = int(uid)
flags = set(flags) flags = set(flags)
except ValueError as e: except ValueError as e:
errstr = "Corrupt line '%s' in cache file '%s'" % \ errstr = ("Corrupt line '%s' in cache file '%s'"%
(line, self.filename) (line, self.filename))
self.ui.warn(errstr) self.ui.warn(errstr)
six.reraise(ValueError(errstr), None, exc_info()[2]) six.reraise(ValueError(errstr), None, exc_info()[2])
self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid] = self.msglist_item_initializer(uid)
@ -162,6 +161,15 @@ class LocalStatusFolder(BaseFolder):
def closefiles(self): def closefiles(self):
pass # Closing files is done on a per-transaction basis. pass # Closing files is done on a per-transaction basis.
def purge(self):
"""Remove any pre-existing database."""
try:
os.unlink(self.filename)
except OSError as e:
self.ui.debug('', "could not remove file %s: %s"%
(self.filename, e))
def save(self): def save(self):
"""Save changed data to disk. For this backend it is the same as saveall.""" """Save changed data to disk. For this backend it is the same as saveall."""

8
offlineimap/folder/LocalStatusSQLite.py
View File

@ -88,6 +88,14 @@ class LocalStatusSQLiteFolder(BaseFolder):
if version < LocalStatusSQLiteFolder.cur_version: if version < LocalStatusSQLiteFolder.cur_version:
self.__upgrade_db(version) self.__upgrade_db(version)
def purge(self):
"""Remove any pre-existing database."""
try:
os.unlink(self.filename)
except OSError as e:
self.ui.debug('', "could not remove file %s: %s"%
(self.filename, e))
def storesmessages(self): def storesmessages(self):
return False return False

20
offlineimap/repository/Base.py
View File

@ -1,5 +1,5 @@
# Base repository support # Base repository support
# Copyright (C) 2002-2015 John Goerzen & contributors # Copyright (C) 2002-2016 John Goerzen & contributors
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@ -222,12 +222,13 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
"') as it would be filtered out on that repository."% "') as it would be filtered out on that repository."%
(dst_name_t, self)) (dst_name_t, self))
continue continue
# get IMAPFolder and see if the reverse nametrans # Get IMAPFolder and see if the reverse nametrans
# works fine TODO: getfolder() works only because we # works fine TODO: getfolder() works only because we
# succeed in getting inexisting folders which I would # succeed in getting inexisting folders which I would
# like to change. Take care! # like to change. Take care!
folder = self.getfolder(dst_name_t) folder = self.getfolder(dst_name_t)
# apply reverse nametrans to see if we end up with the same name # Apply reverse nametrans to see if we end up with the same
# name.
newdst_name = folder.getvisiblename().replace( newdst_name = folder.getvisiblename().replace(
src_repo.getsep(), dst_repo.getsep()) src_repo.getsep(), dst_repo.getsep())
if dst_folder.name != newdst_name: if dst_folder.name != newdst_name:
@ -239,13 +240,14 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
"itories so they lead to identical names if applied bac" "itories so they lead to identical names if applied bac"
"k and forth. 2) Use folderfilter settings on a reposit" "k and forth. 2) Use folderfilter settings on a reposit"
"ory to prevent some folders from being created on the " "ory to prevent some folders from being created on the "
"other side." % (dst_folder.name, dst_repo, dst_name_t, "other side."%
src_repo, newdst_name), (dst_folder.name, dst_repo, dst_name_t,
OfflineImapError.ERROR.REPO) src_repo, newdst_name),
# end sanity check, actually create the folder OfflineImapError.ERROR.REPO)
# End sanity check, actually create the folder.
try: try:
src_repo.makefolder(dst_name_t) src_repo.makefolder(dst_name_t)
src_haschanged = True # Need to refresh list src_haschanged = True # Need to refresh list.
except OfflineImapError as e: except OfflineImapError as e:
self.ui.error(e, exc_info()[2], "Creating folder %s on " self.ui.error(e, exc_info()[2], "Creating folder %s on "
"repository %s" % (dst_name_t, src_repo)) "repository %s" % (dst_name_t, src_repo))
@ -255,7 +257,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
# Find deleted folders. # Find deleted folders.
# TODO: We don't delete folders right now. # TODO: We don't delete folders right now.
#Forget old list of cached folders so we get new ones if needed # Forget old list of cached folders so we get new ones if needed.
if src_haschanged: if src_haschanged:
self.forgetfolders() self.forgetfolders()
if dst_haschanged: if dst_haschanged:

16
offlineimap/repository/LocalStatus.py
View File

@ -1,5 +1,5 @@
# Local status cache repository support # Local status cache repository support
# Copyright (C) 2002-2015 John Goerzen & contributors # Copyright (C) 2002-2016 John Goerzen & contributors
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@ -61,15 +61,15 @@ class LocalStatusRepository(BaseRepository):
def import_other_backend(self, folder): def import_other_backend(self, folder):
for bk, dic in self.backends.items(): for bk, dic in self.backends.items():
# skip folder's own type # Skip folder's own type.
if dic['class'] == type(folder): if dic['class'] == type(folder):
continue continue
repobk = LocalStatusRepository(self.name, self.account) repobk = LocalStatusRepository(self.name, self.account)
repobk.setup_backend(bk) # fake the backend repobk.setup_backend(bk) # Fake the backend.
folderbk = dic['class'](folder.name, repobk) folderbk = dic['class'](folder.name, repobk)
# if backend contains data, import it to folder. # If backend contains data, import it to folder.
if not folderbk.isnewfolder(): if not folderbk.isnewfolder():
self.ui._msg("Migrating LocalStatus cache from %s to %s " self.ui._msg("Migrating LocalStatus cache from %s to %s "
"status folder for %s:%s"% "status folder for %s:%s"%
@ -87,10 +87,14 @@ class LocalStatusRepository(BaseRepository):
"""Create a LocalStatus Folder.""" """Create a LocalStatus Folder."""
if self.account.dryrun: if self.account.dryrun:
return # bail out in dry-run mode return # Bail out in dry-run mode.
# Create an empty StatusFolder # Create an empty StatusFolder.
folder = self._instanciatefolder(foldername) folder = self._instanciatefolder(foldername)
# First delete any existing data to make sure we won't consider obsolete
# data. This might happen if the user removed the folder (maildir) and
# it is re-created afterwards.
folder.purge()
folder.save() folder.save()
folder.closefiles() folder.closefiles()