mirror of
				https://github.com/Garmelon/PFERD.git
				synced 2025-10-30 12:22:43 +01:00 
			
		
		
		
	Make crawler sections start with "crawl:"
Also, use only the part of the section name after the "crawl:" as the crawler's output directory. Now, the implementation matches the documentation again
This commit is contained in:
		| @@ -40,7 +40,7 @@ crawlers: | ||||
|  | ||||
| - `type`: The types are specified in [this section](#crawler-types). | ||||
| - `output_dir`: The directory the crawler synchronizes files to. A crawler will | ||||
|   never place any files outside of this directory. (Default: crawler's name) | ||||
|   never place any files outside of this directory. (Default: the crawler's name) | ||||
| - `redownload`: When to download again a file that is already present locally. | ||||
|   (Default: `never-smart`) | ||||
|     - `never`: If a file is present locally, it is not downloaded again. | ||||
|   | ||||
| @@ -140,7 +140,7 @@ class Config: | ||||
|     def crawler_sections(self) -> List[Tuple[str, SectionProxy]]: | ||||
|         result = [] | ||||
|         for name, proxy in self._parser.items(): | ||||
|             if name.startswith("crawler:"): | ||||
|             if name.startswith("crawl:"): | ||||
|                 result.append((name, proxy)) | ||||
|  | ||||
|         return result | ||||
|   | ||||
| @@ -110,6 +110,9 @@ def arepeat(attempts: int) -> Callable[[AWrapped], AWrapped]: | ||||
|  | ||||
| class CrawlerSection(Section): | ||||
|     def output_dir(self, name: str) -> Path: | ||||
|         # TODO Use removeprefix() after switching to 3.9 | ||||
|         if name.startswith("crawl:"): | ||||
|             name = name[len("crawl:"):] | ||||
|         return Path(self.s.get("output_dir", name)).expanduser() | ||||
|  | ||||
|     def redownload(self) -> Redownload: | ||||
|   | ||||
		Reference in New Issue
	
	Block a user
	 Joscha
					Joscha