mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2025-07-12 06:02:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Make crawlers use transformers

Browse Source
This commit is contained in:
Joscha
2021-05-15 14:03:15 +02:00
parent 302b8c0c34
commit b0f731bf84
3 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
PFERD/crawler.py
View File

@ -273,6 +273,9 @@ class Crawler(ABC):
with self._conductor.progress_bar(desc, total=total) as bar:
yield bar
def should_crawl(self, path: PurePath) -> bool:
return self._transformer.transform(path) is not None
async def download(
self,
path: PurePath,
@ -280,8 +283,12 @@ class Crawler(ABC):
redownload: Optional[Redownload] = None,
on_conflict: Optional[OnConflict] = None,
) -> Optional[AsyncContextManager[FileSink]]:
transformed_path = self._transformer.transform(path)
if transformed_path is None:
return None
return await self._output_dir.download(
path, mtime, redownload, on_conflict)
transformed_path, mtime, redownload, on_conflict)
async def cleanup(self) -> None:
await self._output_dir.cleanup()