mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Use global conductor instance
The switch from crawler-local conductors to a single pferd-global conductor was made to prepare for auth section credential providers.
This commit is contained in:
@ -1,10 +1,19 @@
|
||||
from configparser import SectionProxy
|
||||
from typing import Callable, Dict
|
||||
|
||||
from ..conductor import TerminalConductor
|
||||
from ..config import Config
|
||||
from ..crawler import Crawler, CrawlerSection
|
||||
from ..crawler import Crawler
|
||||
from .local import LocalCrawler, LocalCrawlerSection
|
||||
|
||||
CRAWLERS: Dict[str, Callable[[str, Config, SectionProxy], Crawler]] = {
|
||||
"local": lambda n, c, s: LocalCrawler(n, c, LocalCrawlerSection(s)),
|
||||
CrawlerConstructor = Callable[[
|
||||
str, # Name (without the "crawl:" prefix)
|
||||
SectionProxy, # Crawler's section of global config
|
||||
Config, # Global config
|
||||
TerminalConductor, # Global conductor instance
|
||||
], Crawler]
|
||||
|
||||
CRAWLERS: Dict[str, CrawlerConstructor] = {
|
||||
"local": lambda n, s, c, t:
|
||||
LocalCrawler(n, LocalCrawlerSection(s), c, t),
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
from pathlib import Path, PurePath
|
||||
|
||||
from ..conductor import TerminalConductor
|
||||
from ..config import Config
|
||||
from ..crawler import Crawler, CrawlerSection, anoncritical
|
||||
|
||||
@ -17,10 +18,11 @@ class LocalCrawler(Crawler):
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
config: Config,
|
||||
section: LocalCrawlerSection,
|
||||
config: Config,
|
||||
conductor: TerminalConductor,
|
||||
):
|
||||
super().__init__(name, config, section)
|
||||
super().__init__(name, section, config, conductor)
|
||||
|
||||
self._path = config.working_dir / section.path()
|
||||
|
||||
|
Reference in New Issue
Block a user