learn --mbnames-prune CLI option

This is usefull to remove dangling entries for removed accounts or if mbnames is
not enabled anymore.

Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
This commit is contained in:
Nicolas Sebrecht 2016-06-26 18:12:35 +02:00
parent 4ef05fe6e1
commit 69c0080323
4 changed files with 121 additions and 42 deletions

View File

@ -177,6 +177,22 @@ before running this fix as well as verify the results using the `--dry-run'
flag first. flag first.
--mbnames-prune::
Remove dangling entries for removed accounts or if mbnames is not enabled/used
anymore.
+
Internally, offlineimap build intermediate mbnames files. They are added
automatically when mbnames is enabled. However, disabling accounts so they are
not synced anymore does not necessarily means they should be removed from the file
built by mbnames. It is required to start offlineimap with this CLI option each
time accounts are removed. When run, any account not in the 'accounts'
configuration option are removed in the mbnames file.
+
It is possible to manually remove intermediate files in '<metadata>/mbnames/'.
+
Notice this option honors --dry-run.
Synchronization Performance Synchronization Performance
--------------------------- ---------------------------

View File

@ -1,4 +1,4 @@
# Copyright (C) 2003-2015 John Goerzen & contributors # Copyright (C) 2003-2016 John Goerzen & contributors
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@ -22,7 +22,7 @@ import six
try: try:
from ConfigParser import SafeConfigParser, Error from ConfigParser import SafeConfigParser, Error
except ImportError: #python3 except ImportError: # Python3.
from configparser import SafeConfigParser, Error from configparser import SafeConfigParser, Error
from offlineimap.localeval import LocalEval from offlineimap.localeval import LocalEval

View File

@ -148,6 +148,10 @@ class OfflineImap(object):
action="store_true", dest="migrate_fmd5", default=False, action="store_true", dest="migrate_fmd5", default=False,
help="migrate FMD5 hashes from versions prior to 6.3.5") help="migrate FMD5 hashes from versions prior to 6.3.5")
parser.add_option("--mbnames-prune",
action="store_true", dest="mbnames_prune", default=False,
help="remove mbnames entries for accounts not in accounts")
parser.add_option("-V", parser.add_option("-V",
action="store_true", dest="version", action="store_true", dest="version",
default=False, default=False,
@ -266,8 +270,14 @@ class OfflineImap(object):
if dtype.lower() == u'imap': if dtype.lower() == u'imap':
imaplib.Debug = 5 imaplib.Debug = 5
if options.mbnames_prune:
mbnames.init(config, self.ui, options.dryrun)
mbnames.prune(config.get("general", "accounts"))
mbnames.write()
sys.exit(0)
if options.runonce: if options.runonce:
# Must kill the possible default option # Must kill the possible default option.
if config.has_option('DEFAULT', 'autorefresh'): if config.has_option('DEFAULT', 'autorefresh'):
config.remove_option('DEFAULT', 'autorefresh') config.remove_option('DEFAULT', 'autorefresh')
# FIXME: spaghetti code alert! # FIXME: spaghetti code alert!

View File

@ -19,22 +19,28 @@
import re # For folderfilter. import re # For folderfilter.
import json import json
from threading import Lock from threading import Lock
from os import listdir, makedirs, path from os import listdir, makedirs, path, unlink
from sys import exc_info from sys import exc_info
try: try:
import UserDict import UserDict
except ImportError: except ImportError: # Py3.
# Py3
from collections import UserDict from collections import UserDict
try:
from ConfigParser import NoSectionError
except ImportError: # Py3.
from configparser import NoSectionError
_mbLock = Lock() _mbLock = Lock()
_mbnames = None _mbnames = None
def _is_enabled(conf):
return False
def add(accountname, folder_root, foldername): def add(accountname, folder_root, foldername):
global _mbnames global _mbnames
if _mbnames is None: if _mbnames.is_enabled() is not True:
return return
with _mbLock: with _mbLock:
@ -42,15 +48,21 @@ def add(accountname, folder_root, foldername):
def init(conf, ui, dry_run): def init(conf, ui, dry_run):
global _mbnames global _mbnames
enabled = conf.getdefaultboolean("mbnames", "enabled", False) if _mbnames is None:
if enabled is True and _mbnames is None:
_mbnames = _Mbnames(conf, ui, dry_run) _mbnames = _Mbnames(conf, ui, dry_run)
def prune(accounts):
global _mbnames
if _mbnames.is_enabled() is True:
_mbnames.prune(accounts)
else:
_mbnames.pruneAll(accounts)
def write(): def write():
"""Write the mbnames file.""" """Write the mbnames file."""
global _mbnames global _mbnames
if _mbnames is None: if _mbnames.is_enabled() is not True:
return return
if _mbnames.get_incremental() is not True: if _mbnames.get_incremental() is not True:
@ -60,7 +72,7 @@ def writeIntermediateFile(accountname):
"""Write intermediate mbnames file.""" """Write intermediate mbnames file."""
global _mbnames global _mbnames
if _mbnames is None: if _mbnames.is_enabled() is not True:
return return
_mbnames.writeIntermediateFile(accountname) _mbnames.writeIntermediateFile(accountname)
@ -101,17 +113,18 @@ class _IntermediateMbnames(object):
}) })
if not self._dryrun: if not self._dryrun:
with open(self._path, "wt") as intermediateFile: with open(self._path, "wt") as intermediateFD:
json.dump(itemlist, intermediateFile) json.dump(itemlist, intermediateFD)
class _Mbnames(object): class _Mbnames(object):
def __init__(self, config, ui, dry_run): def __init__(self, config, ui, dry_run):
self._config = config self._config = config
self._dryrun = dry_run
self.ui = ui self.ui = ui
self._dryrun = dry_run
self._enabled = None
# Keys: accountname, values: _IntermediateMbnames instance # Keys: accountname, values: _IntermediateMbnames instance
self._intermediates = {} self._intermediates = {}
self._incremental = None self._incremental = None
@ -119,14 +132,13 @@ class _Mbnames(object):
self._path = None self._path = None
self._folderfilter = lambda accountname, foldername: True self._folderfilter = lambda accountname, foldername: True
self._func_sortkey = lambda d: (d['accountname'], d['foldername']) self._func_sortkey = lambda d: (d['accountname'], d['foldername'])
self._peritem = self._config.get("mbnames", "peritem", raw=1)
localeval = config.getlocaleval() localeval = config.getlocaleval()
self._header = localeval.eval(config.get("mbnames", "header"))
self._sep = localeval.eval(config.get("mbnames", "sep"))
self._footer = localeval.eval(config.get("mbnames", "footer"))
mbnamesdir = path.join(config.getmetadatadir(), "mbnames") mbnamesdir = path.join(config.getmetadatadir(), "mbnames")
self._peritem = None
self._header = None
self._sep = None
self._footer = None
try: try:
if not self._dryrun: if not self._dryrun:
makedirs(mbnamesdir) makedirs(mbnamesdir)
@ -134,6 +146,14 @@ class _Mbnames(object):
pass pass
self._mbnamesdir = mbnamesdir self._mbnamesdir = mbnamesdir
try:
self._enabled = self._config.getdefaultboolean(
"mbnames", "enabled", False)
self._peritem = self._config.get("mbnames", "peritem", raw=1)
self._header = localeval.eval(config.get("mbnames", "header"))
self._sep = localeval.eval(config.get("mbnames", "sep"))
self._footer = localeval.eval(config.get("mbnames", "footer"))
xforms = [path.expanduser, path.expandvars] xforms = [path.expanduser, path.expandvars]
self._path = config.apply_xforms( self._path = config.apply_xforms(
config.get("mbnames", "filename"), xforms) config.get("mbnames", "filename"), xforms)
@ -145,6 +165,21 @@ class _Mbnames(object):
if self._config.has_option("mbnames", "folderfilter"): if self._config.has_option("mbnames", "folderfilter"):
self._folderfilter = localeval.eval( self._folderfilter = localeval.eval(
self._config.get("mbnames", "folderfilter"), {'re': re}) self._config.get("mbnames", "folderfilter"), {'re': re})
except NoSectionError:
pass
def _iterIntermediateFiles(self):
for foo in listdir(self._mbnamesdir):
foo = path.join(self._mbnamesdir, foo)
if path.isfile(foo) and foo[-5:] == '.json':
yield foo
def _removeIntermediateFile(self, path):
if self._dryrun:
self.ui.info("would remove %s"% path)
else:
unlink(path)
self.ui.info("removed %s"% path)
def addAccountFolder(self, accountname, folder_root, foldername): def addAccountFolder(self, accountname, folder_root, foldername):
"""Add foldername entry for an account.""" """Add foldername entry for an account."""
@ -167,22 +202,40 @@ class _Mbnames(object):
return self._incremental return self._incremental
def is_enabled(self):
return self._enabled
def prune(self, accounts):
removals = False
for intermediateFile in self._iterIntermediateFiles():
filename = path.basename(intermediateFile)
accountname = filename[:-5]
if accountname not in accounts:
removals = True
self._removeIntermediateFile(intermediateFile)
if removals is False:
self.ui.info("no cache file to remove")
def pruneAll(self, accounts):
for intermediateFile in self._iterIntermediateFiles():
self._removeIntermediateFile(intermediateFile)
def write(self): def write(self):
itemlist = [] itemlist = []
try: try:
for foo in listdir(self._mbnamesdir): for intermediateFile in self._iterIntermediateFiles():
foo = path.join(self._mbnamesdir, foo)
if path.isfile(foo) and foo[-5:] == '.json':
try: try:
with open(foo, 'rt') as intermediateFile: with open(intermediateFile, 'rt') as intermediateFD:
for item in json.load(intermediateFile): for item in json.load(intermediateFD):
itemlist.append(item) itemlist.append(item)
except Exception as e: except Exception as e:
self.ui.error( self.ui.error(
e, e,
exc_info()[2], exc_info()[2],
"intermediate mbnames file %s not properly read"% foo ("intermediate mbnames file %s not properly read"%
intermediateFile)
) )
except OSError: except OSError:
pass pass