diff --git a/CONFIG.md b/CONFIG.md index 0775eb1..d4a6634 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -86,9 +86,9 @@ common to all crawlers: load for the crawl target. (Default: `0.0`) - `windows_paths`: Whether PFERD should find alternative names for paths that are invalid on Windows. (Default: `yes` on Windows, `no` otherwise) -- `alias`: List of strings that are considered as an alias when invoking with - the `--crawler` or `-C` flag. If there is more then one crawl section with - the same alias all are selected. Thereby you can group different crawlers. +- `aliases`: List of strings that are considered as an alias when invoking with + the `--crawler` or `-C` flag. If there is more than one crawl section with + the same aliases all are selected. Thereby, you can group different crawlers. Some crawlers may also require credentials for authentication. To configure how the crawler obtains its credentials, the `auth` option is used. It is set to the @@ -103,7 +103,7 @@ username = foo password = bar [crawl:something] -alias = [sth, some] +aliases = [sth, some] type = some-complex-crawler auth = auth:example on_conflict = no-delete diff --git a/PFERD/pferd.py b/PFERD/pferd.py index 6ed31a0..2d199e5 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -51,12 +51,14 @@ class Pferd: if section_name in crawl_sections: log.explain(f"Crawler section named {section_name!r} exists") crawlers_to_run.add(section_name) + # interprete name as alias of a crawler alias_names = self._find_crawlers_by_alias(name, config) if alias_names: crawlers_to_run.update(alias_names) log.explain_topic(f"Crawler alias {name!r} found corresponding crawler sections:") for alias_name in alias_names: log.explain(f"Crawler section named {alias_name!r} with alias {name!r} exists") + if not section_name in crawl_sections and not alias_names: log.explain(f"There's neither a crawler section named {section_name!r} nor does a crawler with alias {name!r} exist.") unknown_names.append(name) @@ -74,8 +76,8 @@ class Pferd: def _find_crawlers_by_alias(self, alias: str, config: Config) -> Set[str]: alias_names = set() for (section_name, section) in config.crawl_sections(): - section_alias = section.get("alias", []) - if alias in section_alias: + section_aliases = section.get("aliases", []) + if alias in section_aliases: alias_names.add(section_name) return alias_names