mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Make crawler sections start with "crawl:"
Also, use only the part of the section name after the "crawl:" as the crawler's output directory. Now, the implementation matches the documentation again
This commit is contained in:
parent
868f486922
commit
b70b62cef5
@ -40,7 +40,7 @@ crawlers:
|
||||
|
||||
- `type`: The types are specified in [this section](#crawler-types).
|
||||
- `output_dir`: The directory the crawler synchronizes files to. A crawler will
|
||||
never place any files outside of this directory. (Default: crawler's name)
|
||||
never place any files outside of this directory. (Default: the crawler's name)
|
||||
- `redownload`: When to download again a file that is already present locally.
|
||||
(Default: `never-smart`)
|
||||
- `never`: If a file is present locally, it is not downloaded again.
|
||||
|
@ -140,7 +140,7 @@ class Config:
|
||||
def crawler_sections(self) -> List[Tuple[str, SectionProxy]]:
|
||||
result = []
|
||||
for name, proxy in self._parser.items():
|
||||
if name.startswith("crawler:"):
|
||||
if name.startswith("crawl:"):
|
||||
result.append((name, proxy))
|
||||
|
||||
return result
|
||||
|
@ -110,6 +110,9 @@ def arepeat(attempts: int) -> Callable[[AWrapped], AWrapped]:
|
||||
|
||||
class CrawlerSection(Section):
|
||||
def output_dir(self, name: str) -> Path:
|
||||
# TODO Use removeprefix() after switching to 3.9
|
||||
if name.startswith("crawl:"):
|
||||
name = name[len("crawl:"):]
|
||||
return Path(self.s.get("output_dir", name)).expanduser()
|
||||
|
||||
def redownload(self) -> Redownload:
|
||||
|
Loading…
Reference in New Issue
Block a user