mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Rename local crawler path to target
This commit is contained in:
parent
b2a2b5999b
commit
868f486922
@ -113,7 +113,7 @@ This crawler crawls a local directory. It is really simple and mostly useful for
|
||||
testing different setups. The various delay options are meant to make the
|
||||
crawler simulate a slower, network-based crawler.
|
||||
|
||||
- `path`: Path to the local directory to crawl. (Required)
|
||||
- `target`: Path to the local directory to crawl. (Required)
|
||||
- `crawl_delay`: Maximum artificial delay (in seconds) to simulate for crawl
|
||||
requests. (Default: 0.0)
|
||||
- `download_delay`: Maximum artificial delay (in seconds) to simulate for
|
||||
|
@ -10,10 +10,10 @@ from ..crawler import Crawler, CrawlerSection, anoncritical
|
||||
|
||||
|
||||
class LocalCrawlerSection(CrawlerSection):
|
||||
def path(self) -> Path:
|
||||
value = self.s.get("path")
|
||||
def target(self) -> Path:
|
||||
value = self.s.get("target")
|
||||
if value is None:
|
||||
self.missing_value("path")
|
||||
self.missing_value("target")
|
||||
return Path(value).expanduser()
|
||||
|
||||
def crawl_delay(self) -> float:
|
||||
@ -48,7 +48,7 @@ class LocalCrawler(Crawler):
|
||||
):
|
||||
super().__init__(name, section, config, conductor)
|
||||
|
||||
self._path = config.working_dir / section.path()
|
||||
self._target = config.working_dir / section.target()
|
||||
self._crawl_delay = section.crawl_delay()
|
||||
self._download_delay = section.download_delay()
|
||||
self._download_speed = section.download_speed()
|
||||
@ -59,7 +59,7 @@ class LocalCrawler(Crawler):
|
||||
self._block_size = 1024**2 # 1 MiB
|
||||
|
||||
async def crawl(self) -> None:
|
||||
await self._crawl_path(self._path, PurePath())
|
||||
await self._crawl_path(self._target, PurePath())
|
||||
if self.error_free:
|
||||
await self.cleanup()
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user