mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2025-07-12 06:02:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Let crawlers obtain authenticators

Browse Source
This commit is contained in:
Joscha
2021-05-13 18:57:20 +02:00
parent c3ce6bb31c
commit 0acdee15a0
3 changed files with 19 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
PFERD/crawler.py
View File

@ -4,10 +4,11 @@ from datetime import datetime
from pathlib import Path, PurePath
# TODO In Python 3.9 and above, AsyncContextManager is deprecated
from typing import (Any, AsyncContextManager, AsyncIterator, Awaitable,
Callable, Optional, TypeVar)
Callable, Dict, Optional, TypeVar)
from rich.markup import escape
from .authenticator import Authenticator
from .conductor import ProgressBar, TerminalConductor
from .config import Config, Section
from .limiter import Limiter
@ -136,6 +137,15 @@ class CrawlerSection(Section):
def transform(self) -> str:
return self.s.get("transform", "")
def auth(self, authenticators: Dict[str, Authenticator]) -> Authenticator:
value = self.s.get("auth")
if value is None:
self.missing_value("auth")
auth = authenticators.get(f"auth:{value}")
if auth is None:
self.invalid_value("auth", value)
return auth
class Crawler(ABC):
def __init__(