docker-offlineimap/offlineimap/mbnames.py
Nicolas Sebrecht 5d5ad62fa7 mbnames: don't duplicate entries in autorefresh mode
mbnames is initialized and written once in the run from OfflineImap.__sync().

However, the in-memory instance is fed with data at sync time for each folder
and the intermediate files are written as soon as all the folders are synced for
the account. All of this is done inside the SyncableAccount.__sync() method
while the syncrunner is looping on this.

This means that we duplicate entries for mbnames in each loop (most likely when
autorefresh is enabled).

It is wrong to have duplicates in mbnames for each account. Ignore duplicates
when adding data in the mbnames intermediate files.

Github-ref: https://github.com/OfflineIMAP/offlineimap/issues/467
Reported-and-tested-by: Shin Kojima <shin@kojima.org>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
2017-05-19 08:32:22 +02:00

281 lines
8.9 KiB
Python

# Mailbox name generator
# Copyright (C) 2002-2016 John Goerzen & contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import codecs
import re # For folderfilter.
import json
from threading import Lock
from os import listdir, makedirs, path, unlink
from sys import exc_info
try:
from ConfigParser import NoSectionError
except ImportError: # Py3.
from configparser import NoSectionError
_mbLock = Lock()
_mbnames = None
# Called at sync time for each folder.
def add(accountname, folder_root, foldername):
global _mbnames
if _mbnames.is_enabled() is not True:
return
with _mbLock:
_mbnames.addAccountFolder(accountname, folder_root, foldername)
# Called once.
def init(conf, ui, dry_run):
global _mbnames
if _mbnames is None:
_mbnames = _Mbnames(conf, ui, dry_run)
# Called once.
def prune(accounts):
global _mbnames
if _mbnames.is_enabled() is True:
_mbnames.prune(accounts)
else:
_mbnames.pruneAll()
# Called once.
def write():
"""Write the mbnames file."""
global _mbnames
if _mbnames.is_enabled() is not True:
return
if _mbnames.get_incremental() is not True:
_mbnames.write()
# Called as soon as all the folders are synced for the account.
def writeIntermediateFile(accountname):
"""Write intermediate mbnames file."""
global _mbnames
if _mbnames.is_enabled() is not True:
return
_mbnames.writeIntermediateFile(accountname)
if _mbnames.get_incremental() is True:
_mbnames.write()
class _IntermediateMbnames(object):
"""mbnames data for one account."""
def __init__(self, accountname, folder_root, mbnamesdir, folderfilter,
dry_run, ui):
self.ui = ui
self._foldernames = []
self._accountname = accountname
self._folder_root = folder_root
self._folderfilter = folderfilter
self._path = path.join(mbnamesdir, "%s.json"% accountname)
self._dryrun = dry_run
def add(self, foldername):
if foldername not in self._foldernames:
self._foldernames.append(foldername)
def get_folder_root(self):
return self._folder_root
def write(self):
"""Write intermediate mbnames file in JSON format."""
itemlist = []
for foldername in self._foldernames:
if self._folderfilter(self._accountname, foldername):
itemlist.append({
'accountname': self._accountname,
'foldername': foldername.decode('utf-8'),
'localfolders': self._folder_root,
})
if self._dryrun:
self.ui.info("mbnames would write %s"% self._path)
else:
with codecs.open(
self._path, "wt", encoding='UTF-8') as intermediateFD:
json.dump(itemlist, intermediateFD)
class _Mbnames(object):
def __init__(self, config, ui, dry_run):
self._config = config
self.ui = ui
self._dryrun = dry_run
self._enabled = None
# Keys: accountname, values: _IntermediateMbnames instance.
self._intermediates = {}
self._incremental = None
self._mbnamesdir = None
self._path = None
self._folderfilter = lambda accountname, foldername: True
self._func_sortkey = lambda d: (d['accountname'], d['foldername'])
localeval = config.getlocaleval()
mbnamesdir = path.join(config.getmetadatadir(), "mbnames")
self._peritem = None
self._header = None
self._sep = None
self._footer = None
try:
if not self._dryrun:
makedirs(mbnamesdir)
except OSError:
pass
self._mbnamesdir = mbnamesdir
try:
self._enabled = self._config.getdefaultboolean(
"mbnames", "enabled", False)
self._peritem = self._config.get("mbnames", "peritem", raw=1)
self._header = localeval.eval(config.get("mbnames", "header"))
self._sep = localeval.eval(config.get("mbnames", "sep"))
self._footer = localeval.eval(config.get("mbnames", "footer"))
xforms = [path.expanduser, path.expandvars]
self._path = config.apply_xforms(
config.get("mbnames", "filename"), xforms)
if self._config.has_option("mbnames", "sort_keyfunc"):
self._func_sortkey = localeval.eval(
self._config.get("mbnames", "sort_keyfunc"), {'re': re})
if self._config.has_option("mbnames", "folderfilter"):
self._folderfilter = localeval.eval(
self._config.get("mbnames", "folderfilter"), {'re': re})
except NoSectionError:
pass
def _iterIntermediateFiles(self):
for foo in listdir(self._mbnamesdir):
foo = path.join(self._mbnamesdir, foo)
if path.isfile(foo) and foo[-5:] == '.json':
yield foo
def _removeIntermediateFile(self, path):
if self._dryrun:
self.ui.info("mbnames would remove %s"% path)
else:
unlink(path)
self.ui.info("removed %s"% path)
def addAccountFolder(self, accountname, folder_root, foldername):
"""Add foldername entry for an account."""
if accountname not in self._intermediates:
self._intermediates[accountname] = _IntermediateMbnames(
accountname,
folder_root,
self._mbnamesdir,
self._folderfilter,
self._dryrun,
self.ui,
)
self._intermediates[accountname].add(foldername)
def get_incremental(self):
if self._incremental is None:
self._incremental = self._config.getdefaultboolean(
"mbnames", "incremental", False)
return self._incremental
def is_enabled(self):
return self._enabled
def prune(self, accounts):
removals = False
for intermediateFile in self._iterIntermediateFiles():
filename = path.basename(intermediateFile)
accountname = filename[:-5]
if accountname not in accounts:
removals = True
self._removeIntermediateFile(intermediateFile)
if removals is False:
self.ui.info("no cache file to remove")
def pruneAll(self):
for intermediateFile in self._iterIntermediateFiles():
self._removeIntermediateFile(intermediateFile)
def write(self):
itemlist = []
for intermediateFile in self._iterIntermediateFiles():
try:
with codecs.open(
intermediateFile, 'rt', encoding="UTF-8") as intermediateFD:
for item in json.load(intermediateFD):
itemlist.append(item)
except (OSError, IOError) as e:
self.ui.error("could not read intermediate mbnames file '%s':"
"%s"% (intermediateFile, str(e)))
except Exception as e:
self.ui.error(
e,
exc_info()[2],
("intermediate mbnames file %s not properly read"%
intermediateFile)
)
itemlist.sort(key=self._func_sortkey)
itemlist = [self._peritem % d for d in itemlist]
if self._dryrun:
self.ui.info("mbnames would write %s"% self._path)
else:
try:
with codecs.open(
self._path, 'wt', encoding='UTF-8') as mbnamesFile:
mbnamesFile.write(self._header)
mbnamesFile.write(self._sep.join(itemlist))
mbnamesFile.write(self._footer)
except (OSError, IOError) as e:
self.ui.error(
e,
exc_info()[2],
"mbnames file %s not properly written"% self._path
)
def writeIntermediateFile(self, accountname):
try:
self._intermediates[accountname].write()
except (OSError, IOError) as e:
self.ui.error(
e,
exc_info()[2],
"intermediate mbnames file %s not properly written"% self._path
)