tobias/docker-offlineimap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

learn --mbnames-prune CLI option

Browse Source 
This is usefull to remove dangling entries for removed accounts or if mbnames is
not enabled anymore.

Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
This commit is contained in:
Nicolas Sebrecht 2016-06-26 18:12:35 +02:00
parent 4ef05fe6e1
commit 69c0080323
4 changed files with 121 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs/offlineimap.txt
View File

@ -177,6 +177,22 @@ before running this fix as well as verify the results using the `--dry-run'
flag first.
--mbnames-prune::
Remove dangling entries for removed accounts or if mbnames is not enabled/used
anymore.
+
Internally, offlineimap build intermediate mbnames files. They are added
automatically when mbnames is enabled. However, disabling accounts so they are
not synced anymore does not necessarily means they should be removed from the file
built by mbnames. It is required to start offlineimap with this CLI option each
time accounts are removed. When run, any account not in the 'accounts'
configuration option are removed in the mbnames file.
+
It is possible to manually remove intermediate files in '<metadata>/mbnames/'.
+
Notice this option honors --dry-run.
Synchronization Performance
---------------------------

4
offlineimap/CustomConfig.py
View File

@ -1,4 +1,4 @@
# Copyright (C) 2003-2015 John Goerzen & contributors
# Copyright (C) 2003-2016 John Goerzen & contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -22,7 +22,7 @@ import six
try:
from ConfigParser import SafeConfigParser, Error
except ImportError: #python3
except ImportError: # Python3.
from configparser import SafeConfigParser, Error
from offlineimap.localeval import LocalEval

12
offlineimap/init.py
View File

@ -148,6 +148,10 @@ class OfflineImap(object):
action="store_true", dest="migrate_fmd5", default=False,
help="migrate FMD5 hashes from versions prior to 6.3.5")
parser.add_option("--mbnames-prune",
action="store_true", dest="mbnames_prune", default=False,
help="remove mbnames entries for accounts not in accounts")
parser.add_option("-V",
action="store_true", dest="version",
default=False,
@ -266,8 +270,14 @@ class OfflineImap(object):
if dtype.lower() == u'imap':
imaplib.Debug = 5
if options.mbnames_prune:
mbnames.init(config, self.ui, options.dryrun)
mbnames.prune(config.get("general", "accounts"))
mbnames.write()
sys.exit(0)
if options.runonce:
# Must kill the possible default option
# Must kill the possible default option.
if config.has_option('DEFAULT', 'autorefresh'):
config.remove_option('DEFAULT', 'autorefresh')
# FIXME: spaghetti code alert!

131
offlineimap/mbnames.py
View File

@ -19,22 +19,28 @@
import re # For folderfilter.
import json
from threading import Lock
from os import listdir, makedirs, path
from os import listdir, makedirs, path, unlink
from sys import exc_info
try:
import UserDict
except ImportError:
# Py3
except ImportError: # Py3.
from collections import UserDict
try:
from ConfigParser import NoSectionError
except ImportError: # Py3.
from configparser import NoSectionError
_mbLock = Lock()
_mbnames = None
def _is_enabled(conf):
return False
def add(accountname, folder_root, foldername):
global _mbnames
if _mbnames is None:
if _mbnames.is_enabled() is not True:
return
with _mbLock:
@ -42,15 +48,21 @@ def add(accountname, folder_root, foldername):
def init(conf, ui, dry_run):
global _mbnames
enabled = conf.getdefaultboolean("mbnames", "enabled", False)
if enabled is True and _mbnames is None:
if _mbnames is None:
_mbnames = _Mbnames(conf, ui, dry_run)
def prune(accounts):
global _mbnames
if _mbnames.is_enabled() is True:
_mbnames.prune(accounts)
else:
_mbnames.pruneAll(accounts)
def write():
"""Write the mbnames file."""
global _mbnames
if _mbnames is None:
if _mbnames.is_enabled() is not True:
return
if _mbnames.get_incremental() is not True:
@ -60,7 +72,7 @@ def writeIntermediateFile(accountname):
"""Write intermediate mbnames file."""
global _mbnames
if _mbnames is None:
if _mbnames.is_enabled() is not True:
return
_mbnames.writeIntermediateFile(accountname)
@ -101,17 +113,18 @@ class _IntermediateMbnames(object):
})
if not self._dryrun:
with open(self._path, "wt") as intermediateFile:
json.dump(itemlist, intermediateFile)
with open(self._path, "wt") as intermediateFD:
json.dump(itemlist, intermediateFD)
class _Mbnames(object):
def __init__(self, config, ui, dry_run):
self._config = config
self._dryrun = dry_run
self.ui = ui
self._dryrun = dry_run
self._enabled = None
# Keys: accountname, values: _IntermediateMbnames instance
self._intermediates = {}
self._incremental = None
@ -119,14 +132,13 @@ class _Mbnames(object):
self._path = None
self._folderfilter = lambda accountname, foldername: True
self._func_sortkey = lambda d: (d['accountname'], d['foldername'])
self._peritem = self._config.get("mbnames", "peritem", raw=1)
localeval = config.getlocaleval()
self._header = localeval.eval(config.get("mbnames", "header"))
self._sep = localeval.eval(config.get("mbnames", "sep"))
self._footer = localeval.eval(config.get("mbnames", "footer"))
mbnamesdir = path.join(config.getmetadatadir(), "mbnames")
self._peritem = None
self._header = None
self._sep = None
self._footer = None
try:
if not self._dryrun:
makedirs(mbnamesdir)
@ -134,17 +146,40 @@ class _Mbnames(object):
pass
self._mbnamesdir = mbnamesdir
xforms = [path.expanduser, path.expandvars]
self._path = config.apply_xforms(
config.get("mbnames", "filename"), xforms)
try:
self._enabled = self._config.getdefaultboolean(
"mbnames", "enabled", False)
self._peritem = self._config.get("mbnames", "peritem", raw=1)
self._header = localeval.eval(config.get("mbnames", "header"))
self._sep = localeval.eval(config.get("mbnames", "sep"))
self._footer = localeval.eval(config.get("mbnames", "footer"))
if self._config.has_option("mbnames", "sort_keyfunc"):
self._func_sortkey = localeval.eval(
self._config.get("mbnames", "sort_keyfunc"), {'re': re})
xforms = [path.expanduser, path.expandvars]
self._path = config.apply_xforms(
config.get("mbnames", "filename"), xforms)
if self._config.has_option("mbnames", "folderfilter"):
self._folderfilter = localeval.eval(
self._config.get("mbnames", "folderfilter"), {'re': re})
if self._config.has_option("mbnames", "sort_keyfunc"):
self._func_sortkey = localeval.eval(
self._config.get("mbnames", "sort_keyfunc"), {'re': re})
if self._config.has_option("mbnames", "folderfilter"):
self._folderfilter = localeval.eval(
self._config.get("mbnames", "folderfilter"), {'re': re})
except NoSectionError:
pass
def _iterIntermediateFiles(self):
for foo in listdir(self._mbnamesdir):
foo = path.join(self._mbnamesdir, foo)
if path.isfile(foo) and foo[-5:] == '.json':
yield foo
def _removeIntermediateFile(self, path):
if self._dryrun:
self.ui.info("would remove %s"% path)
else:
unlink(path)
self.ui.info("removed %s"% path)
def addAccountFolder(self, accountname, folder_root, foldername):
"""Add foldername entry for an account."""
@ -167,23 +202,41 @@ class _Mbnames(object):
return self._incremental
def is_enabled(self):
return self._enabled
def prune(self, accounts):
removals = False
for intermediateFile in self._iterIntermediateFiles():
filename = path.basename(intermediateFile)
accountname = filename[:-5]
if accountname not in accounts:
removals = True
self._removeIntermediateFile(intermediateFile)
if removals is False:
self.ui.info("no cache file to remove")
def pruneAll(self, accounts):
for intermediateFile in self._iterIntermediateFiles():
self._removeIntermediateFile(intermediateFile)
def write(self):
itemlist = []
try:
for foo in listdir(self._mbnamesdir):
foo = path.join(self._mbnamesdir, foo)
if path.isfile(foo) and foo[-5:] == '.json':
try:
with open(foo, 'rt') as intermediateFile:
for item in json.load(intermediateFile):
itemlist.append(item)
except Exception as e:
self.ui.error(
e,
exc_info()[2],
"intermediate mbnames file %s not properly read"% foo
)
for intermediateFile in self._iterIntermediateFiles():
try:
with open(intermediateFile, 'rt') as intermediateFD:
for item in json.load(intermediateFD):
itemlist.append(item)
except Exception as e:
self.ui.error(
e,
exc_info()[2],
("intermediate mbnames file %s not properly read"%
intermediateFile)
)
except OSError:
pass