pferd/PFERD/pferd.py

53 lines
1.5 KiB
Python
Raw Normal View History

2021-04-30 16:22:14 +02:00
from typing import Dict
from rich import print
from rich.markup import escape
2021-04-29 13:45:04 +02:00
from .config import Config
2021-04-30 16:22:14 +02:00
from .crawler import Crawler
2021-04-29 13:45:04 +02:00
from .crawlers import CRAWLERS
2020-04-23 11:44:13 +02:00
2021-04-30 16:22:14 +02:00
class PferdLoadException(Exception):
pass
2021-04-29 13:45:04 +02:00
class Pferd:
def __init__(self, config: Config):
self._config = config
2021-04-30 16:22:14 +02:00
self._crawlers: Dict[str, Crawler] = {}
def _load_crawlers(self) -> None:
abort = False
for name, section in self._config.crawler_sections():
print(f"[bold bright_cyan]Loading[/] crawler:{escape(name)}")
crawler_type = section.get("type")
crawler_constructor = CRAWLERS.get(crawler_type)
if crawler_constructor is None:
abort = True
if crawler_type is None:
print("[red]Error: No type")
else:
t = escape(repr(crawler_type))
print(f"[red]Error: Unknown type {t}")
continue
crawler = crawler_constructor(name, self._config, section)
self._crawlers[name] = crawler
if abort:
raise PferdLoadException()
2020-04-23 11:44:13 +02:00
2021-04-29 13:45:04 +02:00
async def run(self) -> None:
2021-04-30 16:22:14 +02:00
try:
self._load_crawlers()
except PferdLoadException:
print("[bold red]Could not initialize PFERD properly")
exit(1)
for name, crawler in self._crawlers.items():
print()
print(f"[bold bright_cyan]Running[/] crawler:{escape(name)}")
await crawler.run()