mirror of
https://github.com/Garmelon/PFERD.git
synced 2025-09-07 13:22:26 +02:00
Replace asyncio.gather with custom Crawler function
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
import asyncio
|
||||
import re
|
||||
from pathlib import PurePath
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional, Set, TypeVar, Union
|
||||
@@ -215,7 +214,7 @@ class KitIliasWebCrawler(HttpCrawler):
|
||||
# this method without having spawned a single task. Due to this we do
|
||||
# not need to cancel anything or worry about this gather call or the forks
|
||||
# further up.
|
||||
await asyncio.gather(*tasks)
|
||||
await self.gather(tasks)
|
||||
|
||||
await impl()
|
||||
|
||||
@@ -240,7 +239,7 @@ class KitIliasWebCrawler(HttpCrawler):
|
||||
# this method without having spawned a single task. Due to this we do
|
||||
# not need to cancel anything or worry about this gather call or the forks
|
||||
# further up.
|
||||
await asyncio.gather(*tasks)
|
||||
await self.gather(tasks)
|
||||
|
||||
await impl()
|
||||
|
||||
|
@@ -83,7 +83,7 @@ class LocalCrawler(Crawler):
|
||||
pure_child = pure / child.name
|
||||
tasks.append(self._crawl_path(child, pure_child))
|
||||
|
||||
await asyncio.gather(*tasks)
|
||||
await self.gather(tasks)
|
||||
|
||||
async def _crawl_file(self, path: Path, pure: PurePath) -> None:
|
||||
stat = path.stat()
|
||||
|
Reference in New Issue
Block a user