Identify and fix messages with FMD5 inconsistencies

Introduce the '--migrate-fmd5-using-nametrans' option which migrates the
FMD5 hashes from versions prior to 6.3.5.

It seems that commit 'Apply nametrans to all Foldertypes' (6b2ec956cf)
introduced a regression because it changed the FMD5 part of the filename
calculated by OfflineIMAP. Thus, OfflineIMAP believes that the messages
has been removed and adds them back.

For more information, see:
http://www.offlineimap.org/configuration/2016/02/12/debian-upgrade-from-jessie-to-stretch.html

Bug-Debian: https://bugs.debian.org/812108
Reported-by: François <francois@avalenn.eu>
Signed-off-by: Ilias Tsitsimpis <i.tsitsimpis@gmail.com>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>

docs/offlineimap.txt

@@ -163,6 +163,20 @@ blinkenlights, machineui.
 This option is only applicable in non-verbose mode.
 
 
+--migrate-fmd5-using-nametrans::
+  Migrate FMD5 hashes from versions prior to 6.3.5.
++
+The way that FMD5 hashes are calculated was changed in version 6.3.5 (now using
+the nametrans folder name) introducing a regression which may lead to
+re-uploading all messages. Try and fix the above regression by calculating the
+correct FMD5 values and renaming the corresponding messages.
+
+CAUTION: Since the FMD5 part of the filename changes, this may lead to UID
+conflicts. Ensure to dispose a proper backup of both the cache and the Maildir
+before running this fix as well as verify the results using the `--dry-run'
+flag first.
+
+
 Synchronization Performance
 ---------------------------
 

offlineimap/folder/Maildir.py

@@ -485,3 +485,37 @@ class MaildirFolder(BaseFolder):
                 os.unlink(filepath)
             # Yep -- return.
         del(self.messagelist[uid])
+
+    def migratefmd5(self, dryrun=False):
+        """Migrate FMD5 hashes from versions prior to 6.3.5
+
+        :param dryrun: Run in dry run mode
+        :type fix: Boolean
+        :return: None
+        """
+        oldfmd5 = md5(self.name).hexdigest()
+        msglist = self._scanfolder()
+        for mkey, mvalue in msglist.iteritems():
+            filename = os.path.join(self.getfullname(), mvalue['filename'])
+            match = re.search("FMD5=([a-fA-F0-9]+)", filename)
+            if match is None:
+                self.ui.debug("maildir",
+                              "File `%s' doesn't have an FMD5 assigned"
+                              % filename)
+            elif match.group(1) == oldfmd5:
+                self.ui.info("Migrating file `%s' to FMD5 `%s'"
+                             % (filename, self._foldermd5))
+                if not dryrun:
+                    newfilename = filename.replace(
+                        "FMD5=" + match.group(1), "FMD5=" + self._foldermd5)
+                    try:
+                        os.rename(filename, newfilename)
+                    except OSError as e:
+                        raise OfflineImapError(
+                            "Can't rename file '%s' to '%s': %s" % (
+                                filename, newfilename, e[1]),
+                            OfflineImapError.ERROR.FOLDER), None, exc_info()[2]
+            elif match.group(1) != self._foldermd5:
+                self.ui.warn(("Inconsistent FMD5 for file `%s':"
+                              " Neither `%s' nor `%s' found")
+                             % (filename, oldfmd5, self._foldermd5))

offlineimap/init.py

@@ -25,11 +25,12 @@ import logging
 from optparse import OptionParser
 
 import offlineimap
-from offlineimap import accounts, threadutil, syncmaster
+from offlineimap import accounts, threadutil, syncmaster, folder
 from offlineimap import globals
 from offlineimap.ui import UI_LIST, setglobalui, getglobalui
 from offlineimap.CustomConfig import CustomConfigParser
 from offlineimap.utils import stacktrace
+from offlineimap.repository import Repository
 
 import traceback
 import collections
@@ -50,6 +51,8 @@ class OfflineImap:
         options, args = self.__parse_cmd_options()
         if options.diagnostics:
             self.__serverdiagnostics(options)
+        elif options.migrate_fmd5:
+            self.__migratefmd5(options)
         else:
             return self.__sync(options)
 
@@ -120,6 +123,10 @@ class OfflineImap:
                   help="specifies an alternative user interface"
                   " (quiet, basic, syslog, ttyui, blinkenlights, machineui)")
 
+        parser.add_option("--migrate-fmd5-using-nametrans",
+                  action="store_true", dest="migrate_fmd5", default=False,
+                  help="migrate FMD5 hashes from versions prior to 6.3.5")
+
         (options, args) = parser.parse_args()
         globals.set_options (options)
 
@@ -427,3 +434,21 @@ class OfflineImap:
         for account in allaccounts:
             if account.name not in activeaccounts: continue
             account.serverdiagnostics()
+
+    def __migratefmd5(self, options):
+        activeaccounts = self.config.get("general", "accounts")
+        if options.accounts:
+            activeaccounts = options.accounts
+        activeaccounts = activeaccounts.replace(" ", "")
+        activeaccounts = activeaccounts.split(",")
+        allaccounts = accounts.AccountListGenerator(self.config)
+
+        for account in allaccounts:
+            if account.name not in activeaccounts:
+                continue
+            localrepo = Repository(account, 'local')
+            if localrepo.getfoldertype() != folder.Maildir.MaildirFolder:
+                continue
+            folders = localrepo.getfolders()
+            for f in folders:
+                f.migratefmd5(options.dryrun)