From a2ffce47029d5bb1b5e78186a6fded9aa89ca101 Mon Sep 17 00:00:00 2001 From: Joscha Date: Wed, 26 Oct 2022 19:41:34 +0200 Subject: [PATCH] Make local crawler synchronous --- PFERD/crawl/local_crawler.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/PFERD/crawl/local_crawler.py b/PFERD/crawl/local_crawler.py index f102bc9..e274634 100644 --- a/PFERD/crawl/local_crawler.py +++ b/PFERD/crawl/local_crawler.py @@ -71,8 +71,6 @@ class LocalCrawler(Crawler): if not cl: return - tasks = [] - async with cl: await asyncio.sleep(random.uniform( 0.5 * self._crawl_delay, @@ -81,9 +79,7 @@ class LocalCrawler(Crawler): for child in path.iterdir(): pure_child = cl.path / child.name - tasks.append(self._crawl_path(child, pure_child)) - - await self.gather(tasks) + await self._crawl_path(child, pure_child) async def _crawl_file(self, path: Path, pure: PurePath) -> None: stat = path.stat()