Use global conductor instance

The switch from crawler-local conductors to a single pferd-global conductor was
made to prepare for auth section credential providers.
This commit is contained in:
Joscha
2021-05-10 23:50:16 +02:00
parent 595ba8b7ab
commit d5f29f01c5
6 changed files with 34 additions and 13 deletions

View File

@ -1,10 +1,19 @@
from configparser import SectionProxy
from typing import Callable, Dict
from ..conductor import TerminalConductor
from ..config import Config
from ..crawler import Crawler, CrawlerSection
from ..crawler import Crawler
from .local import LocalCrawler, LocalCrawlerSection
CRAWLERS: Dict[str, Callable[[str, Config, SectionProxy], Crawler]] = {
"local": lambda n, c, s: LocalCrawler(n, c, LocalCrawlerSection(s)),
CrawlerConstructor = Callable[[
str, # Name (without the "crawl:" prefix)
SectionProxy, # Crawler's section of global config
Config, # Global config
TerminalConductor, # Global conductor instance
], Crawler]
CRAWLERS: Dict[str, CrawlerConstructor] = {
"local": lambda n, s, c, t:
LocalCrawler(n, LocalCrawlerSection(s), c, t),
}

View File

@ -1,6 +1,7 @@
import asyncio
from pathlib import Path, PurePath
from ..conductor import TerminalConductor
from ..config import Config
from ..crawler import Crawler, CrawlerSection, anoncritical
@ -17,10 +18,11 @@ class LocalCrawler(Crawler):
def __init__(
self,
name: str,
config: Config,
section: LocalCrawlerSection,
config: Config,
conductor: TerminalConductor,
):
super().__init__(name, config, section)
super().__init__(name, section, config, conductor)
self._path = config.working_dir / section.path()