pferd/PFERD/pferd.py

50 lines
1.4 KiB
Python
Raw Normal View History

2021-04-30 16:22:14 +02:00
from typing import Dict
from rich import print
from rich.markup import escape
2021-04-29 13:45:04 +02:00
from .config import Config
2021-04-30 16:22:14 +02:00
from .crawler import Crawler
2021-04-29 13:45:04 +02:00
from .crawlers import CRAWLERS
2020-04-23 11:44:13 +02:00
2021-04-30 16:22:14 +02:00
class PferdLoadException(Exception):
pass
2021-04-29 13:45:04 +02:00
class Pferd:
def __init__(self, config: Config):
self._config = config
2021-04-30 16:22:14 +02:00
self._crawlers: Dict[str, Crawler] = {}
def _load_crawlers(self) -> None:
abort = False
for name, section in self._config.crawler_sections():
print(f"[bold bright_cyan]Loading[/] crawler:{escape(name)}")
crawler_type = section.get("type")
crawler_constructor = CRAWLERS.get(crawler_type)
if crawler_constructor is None:
abort = True
2021-05-05 18:08:34 +02:00
t = escape(repr(crawler_type))
2021-05-06 01:02:40 +02:00
print(f"[red]Error: Unknown crawler type {t}")
2021-04-30 16:22:14 +02:00
continue
crawler = crawler_constructor(name, self._config, section)
self._crawlers[name] = crawler
if abort:
raise PferdLoadException()
2020-04-23 11:44:13 +02:00
2021-04-29 13:45:04 +02:00
async def run(self) -> None:
2021-04-30 16:22:14 +02:00
try:
self._load_crawlers()
except PferdLoadException:
print("[bold red]Could not initialize PFERD properly")
exit(1)
for name, crawler in self._crawlers.items():
print()
print(f"[bold bright_cyan]Running[/] crawler:{escape(name)}")
await crawler.run()