Add --verify-storage argument

radicale/__main__.py

@@ -33,7 +33,7 @@ import sys
 from wsgiref.simple_server import make_server
 
 from . import (VERSION, Application, RequestHandler, ThreadedHTTPServer,
-               ThreadedHTTPSServer, config, log)
+               ThreadedHTTPSServer, config, log, storage)
 
 
 def run():
@@ -42,6 +42,8 @@ def run():
     parser = argparse.ArgumentParser(usage="radicale [OPTIONS]")
 
     parser.add_argument("--version", action="version", version=VERSION)
+    parser.add_argument("--verify-storage", action="store_true",
+                        help="check the storage for errors and exit")
     parser.add_argument(
         "-C", "--config", help="use a specific configuration file")
 
@@ -103,6 +105,10 @@ def run():
             if value is not None:
                 configuration.set(section, action.split('_', 1)[1], value)
 
+    if args.verify_storage:
+        # Write to stderr when storage verification is requested
+        configuration["logging"]["config"] = ""
+
     # Start logging
     filename = os.path.expanduser(configuration.get("logging", "config"))
     debug = configuration.getboolean("logging", "debug")
@@ -114,6 +120,20 @@ def run():
             raise
         exit(1)
 
+    if args.verify_storage:
+        logger.info("Verifying storage")
+        try:
+            Collection = storage.load(configuration, logger)
+            with Collection.acquire_lock("r"):
+                if not Collection.verify():
+                    logger.error("Storage verifcation failed")
+                    exit(1)
+        except Exception as e:
+            logger.error("An exception occurred during storage verification: "
+                         "%s", e, exc_info=True)
+            exit(1)
+        return
+
     try:
         serve(configuration, logger)
     except Exception as e:

radicale/storage.py

@@ -699,6 +699,11 @@ class BaseCollection:
         """
         raise NotImplementedError
 
+    @classmethod
+    def verify(cls):
+        """Check the storage for errors."""
+        return True
+
 
 class Collection(BaseCollection):
     """Collection stored in several files per calendar."""
@@ -807,7 +812,8 @@ class Collection(BaseCollection):
         cls._sync_directory(parent_filesystem_path)
 
     @classmethod
-    def discover(cls, path, depth="0"):
+    def discover(cls, path, depth="0", child_context_manager=(
+                 lambda path, href=None: contextlib.ExitStack())):
         # Path should already be sanitized
         sane_path = sanitize_path(path).strip("/")
         attributes = sane_path.split("/") if sane_path else []
@@ -844,8 +850,9 @@ class Collection(BaseCollection):
         if depth == "0":
             return
 
-        for item in collection.list():
-            yield collection.get(item)
+        for href in collection.list():
+            with child_context_manager(sane_path, href):
+                yield collection.get(href)
 
         for href in scandir(filesystem_path, only_dirs=True):
             if not is_safe_filesystem_path_component(href):
@@ -854,8 +861,48 @@ class Collection(BaseCollection):
                                      sane_path)
                 continue
             child_path = posixpath.join(sane_path, href)
+            with child_context_manager(child_path):
                 yield cls(child_path)
 
+    @classmethod
+    def verify(cls):
+        item_errors = collection_errors = 0
+
+        @contextlib.contextmanager
+        def exception_cm(path, href=None):
+            nonlocal item_errors, collection_errors
+            try:
+                yield
+            except Exception as e:
+                if href:
+                    item_errors += 1
+                    name = "item %r in %r" % (href, path.strip("/"))
+                else:
+                    collection_errors += 1
+                    name = "collection %r" % path.strip("/")
+                cls.logger.error("Invalid %s: %s", name, e, exc_info=True)
+
+        remaining_paths = [""]
+        while remaining_paths:
+            path = remaining_paths.pop(0)
+            cls.logger.debug("Verifying collection %r", path)
+            with exception_cm(path):
+                saved_item_errors = item_errors
+                collection = None
+                for item in cls.discover(path, "1", exception_cm):
+                    if not collection:
+                        collection = item
+                        collection.get_meta()
+                        continue
+                    if isinstance(item, BaseCollection):
+                        remaining_paths.append(item.path)
+                    else:
+                        cls.logger.debug("Verified item %r in %r",
+                                         item.href, path)
+                if item_errors == saved_item_errors:
+                    collection.sync()
+        return item_errors == 0 and collection_errors == 0
+
     @classmethod
     def create_collection(cls, href, collection=None, props=None):
         folder = cls._get_collection_root_folder()