mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Make crawler sections start with "crawl:"
Also, use only the part of the section name after the "crawl:" as the crawler's output directory. Now, the implementation matches the documentation again
This commit is contained in:
@ -140,7 +140,7 @@ class Config:
|
||||
def crawler_sections(self) -> List[Tuple[str, SectionProxy]]:
|
||||
result = []
|
||||
for name, proxy in self._parser.items():
|
||||
if name.startswith("crawler:"):
|
||||
if name.startswith("crawl:"):
|
||||
result.append((name, proxy))
|
||||
|
||||
return result
|
||||
|
@ -110,6 +110,9 @@ def arepeat(attempts: int) -> Callable[[AWrapped], AWrapped]:
|
||||
|
||||
class CrawlerSection(Section):
|
||||
def output_dir(self, name: str) -> Path:
|
||||
# TODO Use removeprefix() after switching to 3.9
|
||||
if name.startswith("crawl:"):
|
||||
name = name[len("crawl:"):]
|
||||
return Path(self.s.get("output_dir", name)).expanduser()
|
||||
|
||||
def redownload(self) -> Redownload:
|
||||
|
Reference in New Issue
Block a user