Make crawler sections start with "crawl:"

Also, use only the part of the section name after the "crawl:" as the crawler's
output directory. Now, the implementation matches the documentation again
This commit is contained in:
Joscha 2021-05-15 17:23:33 +02:00
parent 868f486922
commit b70b62cef5
3 changed files with 5 additions and 2 deletions

View File

@ -40,7 +40,7 @@ crawlers:
- `type`: The types are specified in [this section](#crawler-types).
- `output_dir`: The directory the crawler synchronizes files to. A crawler will
never place any files outside of this directory. (Default: crawler's name)
never place any files outside of this directory. (Default: the crawler's name)
- `redownload`: When to download again a file that is already present locally.
(Default: `never-smart`)
- `never`: If a file is present locally, it is not downloaded again.

View File

@ -140,7 +140,7 @@ class Config:
def crawler_sections(self) -> List[Tuple[str, SectionProxy]]:
result = []
for name, proxy in self._parser.items():
if name.startswith("crawler:"):
if name.startswith("crawl:"):
result.append((name, proxy))
return result

View File

@ -110,6 +110,9 @@ def arepeat(attempts: int) -> Callable[[AWrapped], AWrapped]:
class CrawlerSection(Section):
def output_dir(self, name: str) -> Path:
# TODO Use removeprefix() after switching to 3.9
if name.startswith("crawl:"):
name = name[len("crawl:"):]
return Path(self.s.get("output_dir", name)).expanduser()
def redownload(self) -> Redownload: