mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make crawlers use transformers

Browse Source
This commit is contained in:
Joscha
2021-05-15 14:03:15 +02:00
parent 302b8c0c34
commit b0f731bf84
3 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
PFERD/crawlers/local.py
View File

@ -81,7 +81,8 @@ class LocalCrawler(Crawler):
for child in path.iterdir():
pure_child = pure / child.name
tasks.append(self._crawl_path(child, pure_child))
if self.should_crawl(child):
tasks.append(self._crawl_path(child, pure_child))
await asyncio.gather(*tasks)