mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix errors loading local crawler config

Browse Source 
Apparently getint and getfloat may return a None even though this is not
mentioned in their type annotations.
This commit is contained in:
Joscha 2021-05-15 13:32:13 +02:00
parent acd674f0a0
commit 302b8c0c34
2 changed files with 19 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CONFIG.md
View File

@ -115,9 +115,9 @@ crawler simulate a slower, network-based crawler.
- `path`: Path to the local directory to crawl. (Required)
- `crawl_delay`: Maximum artificial delay (in seconds) to simulate for crawl
requests. (Optional)
requests. (Default: 0.0)
- `download_delay`: Maximum artificial delay (in seconds) to simulate for
download requests. (Optional)
download requests. (Default: 0.0)
- `download_speed`: Download speed (in bytes per second) to simulate. (Optional)
## Authenticator types

36
PFERD/crawlers/local.py
View File

@ -16,23 +16,23 @@ class LocalCrawlerSection(CrawlerSection):
self.missing_value("path")
return Path(value).expanduser()
def crawl_delay(self) -> Optional[float]:
value = self.s.getfloat("crawl_delay")
if value <= 0:
def crawl_delay(self) -> float:
value = self.s.getfloat("crawl_delay", fallback=0.0)
if value < 0:
self.invalid_value("crawl_delay", value,
"Must be greater than 0")
"Must not be negative")
return value
def download_delay(self) -> Optional[float]:
value = self.s.getfloat("download_delay")
if value <= 0:
def download_delay(self) -> float:
value = self.s.getfloat("download_delay", fallback=0.0)
if value < 0:
self.invalid_value("download_delay", value,
"Must be greater than 0")
"Must not be negative")
return value
def download_speed(self) -> Optional[int]:
value = self.s.getint("download_speed")
if value <= 0:
if value is not None and value <= 0:
self.invalid_value("download_speed", value,
"Must be greater than 0")
return value
@ -74,11 +74,10 @@ class LocalCrawler(Crawler):
tasks = []
async with self.crawl_bar(pure):
if self._crawl_delay:
await asyncio.sleep(random.uniform(
0.5 * self._crawl_delay,
self._crawl_delay,
))
await asyncio.sleep(random.uniform(
0.5 * self._crawl_delay,
self._crawl_delay,
))
for child in path.iterdir():
pure_child = pure / child.name
@ -94,11 +93,10 @@ class LocalCrawler(Crawler):
return
async with self.download_bar(path) as bar:
if self._download_delay:
await asyncio.sleep(random.uniform(
0.5 * self._download_delay,
self._download_delay,
))
await asyncio.sleep(random.uniform(
0.5 * self._download_delay,
self._download_delay,
))
bar.set_total(stat.st_size)