diff --git a/radicale/config.py b/radicale/config.py
index 64dc5cc..a9b7d7f 100644
--- a/radicale/config.py
+++ b/radicale/config.py
@@ -283,7 +283,8 @@ def load(paths: Optional[Iterable[Tuple[str, bool]]] = None
                 config = {s: {o: parser[s][o] for o in parser.options(s)}
                           for s in parser.sections()}
         except Exception as e:
-            if not ignore_if_missing or not isinstance(e, FileNotFoundError):
+            if not (ignore_if_missing and
+                    isinstance(e, (FileNotFoundError, PermissionError))):
                 raise RuntimeError("Failed to load %s: %s" % (config_source, e)
                                    ) from e
             config = Configuration.SOURCE_MISSING