from typing import Dict from rich import print from rich.markup import escape from .config import Config from .crawler import Crawler from .crawlers import CRAWLERS class PferdLoadException(Exception): pass class Pferd: def __init__(self, config: Config): self._config = config self._crawlers: Dict[str, Crawler] = {} def _load_crawlers(self) -> None: abort = False for name, section in self._config.crawler_sections(): print(f"[bold bright_cyan]Loading[/] crawler:{escape(name)}") crawler_type = section.get("type") crawler_constructor = CRAWLERS.get(crawler_type) if crawler_constructor is None: abort = True t = escape(repr(crawler_type)) print(f"[red]Error: Unknown crawler type {t}") continue crawler = crawler_constructor(name, self._config, section) self._crawlers[name] = crawler if abort: raise PferdLoadException() async def run(self) -> None: try: self._load_crawlers() except PferdLoadException: print("[bold red]Could not initialize PFERD properly") exit(1) for name, crawler in self._crawlers.items(): print() print(f"[bold bright_cyan]Running[/] crawler:{escape(name)}") await crawler.run()