Make local crawler synchronous

This commit is contained in:
Joscha 2022-10-26 19:41:34 +02:00
parent 0294ceb7d5
commit a2ffce4702

View File

@ -71,8 +71,6 @@ class LocalCrawler(Crawler):
if not cl: if not cl:
return return
tasks = []
async with cl: async with cl:
await asyncio.sleep(random.uniform( await asyncio.sleep(random.uniform(
0.5 * self._crawl_delay, 0.5 * self._crawl_delay,
@ -81,9 +79,7 @@ class LocalCrawler(Crawler):
for child in path.iterdir(): for child in path.iterdir():
pure_child = cl.path / child.name pure_child = cl.path / child.name
tasks.append(self._crawl_path(child, pure_child)) await self._crawl_path(child, pure_child)
await self.gather(tasks)
async def _crawl_file(self, path: Path, pure: PurePath) -> None: async def _crawl_file(self, path: Path, pure: PurePath) -> None:
stat = path.stat() stat = path.stat()