mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Used proper plural

Browse Source
This commit is contained in:
Julius Rüberg 2021-11-02 12:41:40 +01:00
parent 9e12e96d90
commit 77c1f1516c
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CONFIG.md
View File

@ -86,9 +86,9 @@ common to all crawlers:
load for the crawl target. (Default: `0.0`) load for the crawl target. (Default: `0.0`)
- `windows_paths`: Whether PFERD should find alternative names for paths that - `windows_paths`: Whether PFERD should find alternative names for paths that
are invalid on Windows. (Default: `yes` on Windows, `no` otherwise) are invalid on Windows. (Default: `yes` on Windows, `no` otherwise)
- `alias`: List of strings that are considered as an alias when invoking with - `aliases`: List of strings that are considered as an alias when invoking with
the `--crawler` or `-C` flag. If there is more then one crawl section with the `--crawler` or `-C` flag. If there is more than one crawl section with
the same alias all are selected. Thereby you can group different crawlers. the same aliases all are selected. Thereby, you can group different crawlers.
Some crawlers may also require credentials for authentication. To configure how Some crawlers may also require credentials for authentication. To configure how
the crawler obtains its credentials, the `auth` option is used. It is set to the the crawler obtains its credentials, the `auth` option is used. It is set to the
@ -103,7 +103,7 @@ username = foo
password = bar password = bar
[crawl:something] [crawl:something]
alias = [sth, some] aliases = [sth, some]
type = some-complex-crawler type = some-complex-crawler
auth = auth:example auth = auth:example
on_conflict = no-delete on_conflict = no-delete

6
PFERD/pferd.py
View File

@ -51,12 +51,14 @@ class Pferd:
if section_name in crawl_sections: if section_name in crawl_sections:
log.explain(f"Crawler section named {section_name!r} exists") log.explain(f"Crawler section named {section_name!r} exists")
crawlers_to_run.add(section_name) crawlers_to_run.add(section_name)
# interprete name as alias of a crawler
alias_names = self._find_crawlers_by_alias(name, config) alias_names = self._find_crawlers_by_alias(name, config)
if alias_names: if alias_names:
crawlers_to_run.update(alias_names) crawlers_to_run.update(alias_names)
log.explain_topic(f"Crawler alias {name!r} found corresponding crawler sections:") log.explain_topic(f"Crawler alias {name!r} found corresponding crawler sections:")
for alias_name in alias_names: for alias_name in alias_names:
log.explain(f"Crawler section named {alias_name!r} with alias {name!r} exists") log.explain(f"Crawler section named {alias_name!r} with alias {name!r} exists")
if not section_name in crawl_sections and not alias_names: if not section_name in crawl_sections and not alias_names:
log.explain(f"There's neither a crawler section named {section_name!r} nor does a crawler with alias {name!r} exist.") log.explain(f"There's neither a crawler section named {section_name!r} nor does a crawler with alias {name!r} exist.")
unknown_names.append(name) unknown_names.append(name)
@ -74,8 +76,8 @@ class Pferd:
def _find_crawlers_by_alias(self, alias: str, config: Config) -> Set[str]: def _find_crawlers_by_alias(self, alias: str, config: Config) -> Set[str]:
alias_names = set() alias_names = set()
for (section_name, section) in config.crawl_sections(): for (section_name, section) in config.crawl_sections():
section_alias = section.get("alias", []) section_aliases = section.get("aliases", [])
if alias in section_alias: if alias in section_aliases:
alias_names.add(section_name) alias_names.add(section_name)
return alias_names return alias_names