Move logging logic to singleton

- Renamed module and class because "conductor" didn't make a lot of sense
- Used singleton approach (there's only one stdout after all)
- Redesigned progress bars (now with download speed!)
This commit is contained in:
Joscha 2021-05-18 22:43:46 +02:00
parent 1525aa15a6
commit 4b68fa771f
12 changed files with 195 additions and 193 deletions

View File

@ -1,7 +1,6 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Tuple from typing import Tuple
from .conductor import TerminalConductor
from .config import Config, Section from .config import Config, Section
@ -23,7 +22,6 @@ class Authenticator(ABC):
name: str, name: str,
section: AuthSection, section: AuthSection,
config: Config, config: Config,
conductor: TerminalConductor,
) -> None: ) -> None:
""" """
Initialize an authenticator from its name and its section in the config Initialize an authenticator from its name and its section in the config
@ -36,7 +34,6 @@ class Authenticator(ABC):
""" """
self.name = name self.name = name
self.conductor = conductor
@abstractmethod @abstractmethod
async def credentials(self) -> Tuple[str, str]: async def credentials(self) -> Tuple[str, str]:

View File

@ -2,7 +2,6 @@ from configparser import SectionProxy
from typing import Callable, Dict from typing import Callable, Dict
from ..authenticator import Authenticator, AuthSection from ..authenticator import Authenticator, AuthSection
from ..conductor import TerminalConductor
from ..config import Config from ..config import Config
from .simple import SimpleAuthenticator, SimpleAuthSection from .simple import SimpleAuthenticator, SimpleAuthSection
from .tfa import TfaAuthenticator from .tfa import TfaAuthenticator
@ -11,12 +10,11 @@ AuthConstructor = Callable[[
str, # Name (without the "auth:" prefix) str, # Name (without the "auth:" prefix)
SectionProxy, # Authenticator's section of global config SectionProxy, # Authenticator's section of global config
Config, # Global config Config, # Global config
TerminalConductor, # Global conductor instance
], Authenticator] ], Authenticator]
AUTHENTICATORS: Dict[str, AuthConstructor] = { AUTHENTICATORS: Dict[str, AuthConstructor] = {
"simple": lambda n, s, c, t: "simple": lambda n, s, c:
SimpleAuthenticator(n, SimpleAuthSection(s), c, t), SimpleAuthenticator(n, SimpleAuthSection(s), c),
"tfa": lambda n, s, c, t: "tfa": lambda n, s, c:
TfaAuthenticator(n, AuthSection(s), c, t), TfaAuthenticator(n, AuthSection(s), c),
} }

View File

@ -1,8 +1,8 @@
from typing import Optional, Tuple from typing import Optional, Tuple
from ..authenticator import Authenticator, AuthException, AuthSection from ..authenticator import Authenticator, AuthException, AuthSection
from ..conductor import TerminalConductor
from ..config import Config from ..config import Config
from ..logging import log
from ..utils import agetpass, ainput from ..utils import agetpass, ainput
@ -20,9 +20,8 @@ class SimpleAuthenticator(Authenticator):
name: str, name: str,
section: SimpleAuthSection, section: SimpleAuthSection,
config: Config, config: Config,
conductor: TerminalConductor,
) -> None: ) -> None:
super().__init__(name, section, config, conductor) super().__init__(name, section, config)
self._username = section.username() self._username = section.username()
self._password = section.password() self._password = section.password()
@ -34,7 +33,7 @@ class SimpleAuthenticator(Authenticator):
if self._username is not None and self._password is not None: if self._username is not None and self._password is not None:
return self._username, self._password return self._username, self._password
async with self.conductor.exclusive_output(): async with log.exclusive_output():
if self._username is None: if self._username is None:
self._username = await ainput("Username: ") self._username = await ainput("Username: ")
else: else:

View File

@ -1,8 +1,8 @@
from typing import Tuple from typing import Tuple
from ..authenticator import Authenticator, AuthException, AuthSection from ..authenticator import Authenticator, AuthException, AuthSection
from ..conductor import TerminalConductor
from ..config import Config from ..config import Config
from ..logging import log
from ..utils import ainput from ..utils import ainput
@ -12,15 +12,14 @@ class TfaAuthenticator(Authenticator):
name: str, name: str,
section: AuthSection, section: AuthSection,
config: Config, config: Config,
conductor: TerminalConductor,
) -> None: ) -> None:
super().__init__(name, section, config, conductor) super().__init__(name, section, config)
async def username(self) -> str: async def username(self) -> str:
raise AuthException("TFA authenticator does not support usernames") raise AuthException("TFA authenticator does not support usernames")
async def password(self) -> str: async def password(self) -> str:
async with self.conductor.exclusive_output(): async with log.exclusive_output():
code = await ainput("TFA code: ") code = await ainput("TFA code: ")
return code return code

View File

@ -1,95 +0,0 @@
import asyncio
from contextlib import asynccontextmanager, contextmanager
from types import TracebackType
from typing import AsyncIterator, Iterator, List, Optional, Type
from rich.console import Console
from rich.progress import Progress, TaskID
class ProgressBar:
def __init__(self, progress: Progress, taskid: TaskID):
self._progress = progress
self._taskid = taskid
def advance(self, amount: float = 1) -> None:
self._progress.advance(self._taskid, advance=amount)
def set_total(self, total: float) -> None:
self._progress.update(self._taskid, total=total)
self._progress.start_task(self._taskid)
class TerminalConductor:
def __init__(self) -> None:
self._stopped = False
self._lock = asyncio.Lock()
self._lines: List[str] = []
self._console = Console(highlight=False)
self._progress = Progress(console=self._console)
async def _start(self) -> None:
for task in self._progress.tasks:
task.visible = True
self._progress.start()
self._stopped = False
for line in self._lines:
self.print(line)
self._lines = []
async def _stop(self) -> None:
self._stopped = True
for task in self._progress.tasks:
task.visible = False
self._progress.stop()
async def __aenter__(self) -> None:
async with self._lock:
await self._start()
async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc_value: Optional[BaseException],
traceback: Optional[TracebackType],
) -> Optional[bool]:
async with self._lock:
await self._stop()
return None
def print(self, line: str) -> None:
if self._stopped:
self._lines.append(line)
else:
self._console.print(line)
@asynccontextmanager
async def exclusive_output(self) -> AsyncIterator[None]:
async with self._lock:
await self._stop()
try:
yield
finally:
await self._start()
@contextmanager
def progress_bar(
self,
description: str,
total: Optional[float] = None,
) -> Iterator[ProgressBar]:
if total is None:
# Indeterminate progress bar
taskid = self._progress.add_task(description, start=False)
else:
taskid = self._progress.add_task(description, total=total)
bar = ProgressBar(self._progress, taskid)
try:
yield bar
finally:
self._progress.remove_task(taskid)

View File

@ -9,9 +9,9 @@ import aiohttp
from rich.markup import escape from rich.markup import escape
from .authenticator import Authenticator from .authenticator import Authenticator
from .conductor import ProgressBar, TerminalConductor
from .config import Config, Section from .config import Config, Section
from .limiter import Limiter from .limiter import Limiter
from .logging import ProgressBar, log
from .output_dir import FileSink, OnConflict, OutputDirectory, Redownload from .output_dir import FileSink, OnConflict, OutputDirectory, Redownload
from .transformer import RuleParseException, Transformer from .transformer import RuleParseException, Transformer
from .version import __version__ from .version import __version__
@ -36,7 +36,7 @@ def noncritical(f: Wrapped) -> Wrapped:
try: try:
f(self, *args, **kwargs) f(self, *args, **kwargs)
except Exception as e: except Exception as e:
self.print(f"[red]Something went wrong: {escape(str(e))}") log.print(f"[red]Something went wrong: {escape(str(e))}")
self.error_free = False self.error_free = False
return wrapper # type: ignore return wrapper # type: ignore
@ -79,7 +79,7 @@ def anoncritical(f: AWrapped) -> AWrapped:
try: try:
await f(self, *args, **kwargs) await f(self, *args, **kwargs)
except Exception as e: except Exception as e:
self.print(f"[red]Something went wrong: {escape(str(e))}") log.print(f"[red]Something went wrong: {escape(str(e))}")
self.error_free = False self.error_free = False
return wrapper # type: ignore return wrapper # type: ignore
@ -182,7 +182,6 @@ class Crawler(ABC):
name: str, name: str,
section: CrawlerSection, section: CrawlerSection,
config: Config, config: Config,
conductor: TerminalConductor,
) -> None: ) -> None:
""" """
Initialize a crawler from its name and its section in the config file. Initialize a crawler from its name and its section in the config file.
@ -194,7 +193,6 @@ class Crawler(ABC):
""" """
self.name = name self.name = name
self._conductor = conductor
self.error_free = True self.error_free = True
self._limiter = Limiter( self._limiter = Limiter(
@ -213,34 +211,8 @@ class Crawler(ABC):
config.working_dir / section.output_dir(name), config.working_dir / section.output_dir(name),
section.redownload(), section.redownload(),
section.on_conflict(), section.on_conflict(),
self._conductor,
) )
def print(self, text: str) -> None:
"""
Print rich markup to the terminal. Crawlers *must* use this function to
print things unless they are holding an exclusive output context
manager! Be careful to escape all user-supplied strings.
"""
self._conductor.print(text)
def exclusive_output(self) -> AsyncContextManager[None]:
"""
Acquire exclusive rights to the terminal output. While this context
manager is held, output such as printing and progress bars from other
threads is suspended and the current thread may do whatever it wants
with the terminal. However, it must return the terminal to its original
state before exiting the context manager.
No two threads can hold this context manager at the same time.
Useful for password or confirmation prompts as well as running other
programs while crawling (e. g. to get certain credentials).
"""
return self._conductor.exclusive_output()
@asynccontextmanager @asynccontextmanager
async def crawl_bar( async def crawl_bar(
self, self,
@ -249,7 +221,7 @@ class Crawler(ABC):
) -> AsyncIterator[ProgressBar]: ) -> AsyncIterator[ProgressBar]:
desc = f"[bold bright_cyan]Crawling[/] {escape(str(path))}" desc = f"[bold bright_cyan]Crawling[/] {escape(str(path))}"
async with self._limiter.limit_crawl(): async with self._limiter.limit_crawl():
with self._conductor.progress_bar(desc, total=total) as bar: with log.crawl_bar(desc, total=total) as bar:
yield bar yield bar
@asynccontextmanager @asynccontextmanager
@ -260,7 +232,7 @@ class Crawler(ABC):
) -> AsyncIterator[ProgressBar]: ) -> AsyncIterator[ProgressBar]:
desc = f"[bold bright_cyan]Downloading[/] {escape(str(path))}" desc = f"[bold bright_cyan]Downloading[/] {escape(str(path))}"
async with self._limiter.limit_download(): async with self._limiter.limit_download():
with self._conductor.progress_bar(desc, total=total) as bar: with log.download_bar(desc, total=total) as bar:
yield bar yield bar
def should_crawl(self, path: PurePath) -> bool: def should_crawl(self, path: PurePath) -> bool:
@ -289,7 +261,7 @@ class Crawler(ABC):
crawler. crawler.
""" """
async with self._conductor: with log.show_progress():
await self.crawl() await self.crawl()
@abstractmethod @abstractmethod
@ -312,9 +284,8 @@ class HttpCrawler(Crawler):
name: str, name: str,
section: CrawlerSection, section: CrawlerSection,
config: Config, config: Config,
conductor: TerminalConductor,
) -> None: ) -> None:
super().__init__(name, section, config, conductor) super().__init__(name, section, config)
self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE) self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE)
self._output_dir.register_reserved(self.COOKIE_FILE) self._output_dir.register_reserved(self.COOKIE_FILE)
@ -340,7 +311,4 @@ class HttpCrawler(Crawler):
try: try:
cookie_jar.save(self._cookie_jar_path) cookie_jar.save(self._cookie_jar_path)
except Exception: except Exception:
self.print( log.print(f"[bold red]Warning:[/] Failed to save cookies to {escape(str(self.COOKIE_FILE))}")
"[bold red]Warning:[/] Failed to save cookies to "
+ escape(str(self.COOKIE_FILE))
)

View File

@ -2,7 +2,6 @@ from configparser import SectionProxy
from typing import Callable, Dict from typing import Callable, Dict
from ..authenticator import Authenticator from ..authenticator import Authenticator
from ..conductor import TerminalConductor
from ..config import Config from ..config import Config
from ..crawler import Crawler from ..crawler import Crawler
from .ilias import KitIliasCrawler, KitIliasCrawlerSection from .ilias import KitIliasCrawler, KitIliasCrawlerSection
@ -12,13 +11,12 @@ CrawlerConstructor = Callable[[
str, # Name (without the "crawl:" prefix) str, # Name (without the "crawl:" prefix)
SectionProxy, # Crawler's section of global config SectionProxy, # Crawler's section of global config
Config, # Global config Config, # Global config
TerminalConductor, # Global conductor instance
Dict[str, Authenticator], # Loaded authenticators by name Dict[str, Authenticator], # Loaded authenticators by name
], Crawler] ], Crawler]
CRAWLERS: Dict[str, CrawlerConstructor] = { CRAWLERS: Dict[str, CrawlerConstructor] = {
"local": lambda n, s, c, t, a: "local": lambda n, s, c, a:
LocalCrawler(n, LocalCrawlerSection(s), c, t), LocalCrawler(n, LocalCrawlerSection(s), c),
"kit-ilias": lambda n, s, c, t, a: "kit-ilias": lambda n, s, c, a:
KitIliasCrawler(n, KitIliasCrawlerSection(s), c, t, a), KitIliasCrawler(n, KitIliasCrawlerSection(s), c, a),
} }

View File

@ -16,7 +16,6 @@ from PFERD.output_dir import Redownload
from PFERD.utils import soupify from PFERD.utils import soupify
from ..authenticators import Authenticator from ..authenticators import Authenticator
from ..conductor import TerminalConductor
from ..config import Config from ..config import Config
from ..crawler import CrawlerSection, HttpCrawler, anoncritical, arepeat from ..crawler import CrawlerSection, HttpCrawler, anoncritical, arepeat
@ -533,10 +532,9 @@ class KitIliasCrawler(HttpCrawler):
name: str, name: str,
section: KitIliasCrawlerSection, section: KitIliasCrawlerSection,
config: Config, config: Config,
conductor: TerminalConductor,
authenticators: Dict[str, Authenticator] authenticators: Dict[str, Authenticator]
): ):
super().__init__(name, section, config, conductor) super().__init__(name, section, config)
self._shibboleth_login = KitShibbolethLogin( self._shibboleth_login = KitShibbolethLogin(
section.auth(authenticators), section.auth(authenticators),
@ -615,7 +613,7 @@ class KitIliasCrawler(HttpCrawler):
await self._download_file(element, element_path) await self._download_file(element, element_path)
elif element.type == IliasElementType.FORUM: elif element.type == IliasElementType.FORUM:
# TODO: Delete # TODO: Delete
self.print(f"Skipping forum [green]{element_path}[/]") print(f"Skipping forum [green]{element_path}[/]")
elif element.type == IliasElementType.LINK: elif element.type == IliasElementType.LINK:
await self._download_link(element, element_path) await self._download_link(element, element_path)
elif element.type == IliasElementType.VIDEO: elif element.type == IliasElementType.VIDEO:

View File

@ -4,7 +4,6 @@ import random
from pathlib import Path, PurePath from pathlib import Path, PurePath
from typing import Optional from typing import Optional
from ..conductor import TerminalConductor
from ..config import Config from ..config import Config
from ..crawler import Crawler, CrawlerSection, anoncritical from ..crawler import Crawler, CrawlerSection, anoncritical
@ -44,9 +43,8 @@ class LocalCrawler(Crawler):
name: str, name: str,
section: LocalCrawlerSection, section: LocalCrawlerSection,
config: Config, config: Config,
conductor: TerminalConductor,
): ):
super().__init__(name, section, config, conductor) super().__init__(name, section, config)
self._target = config.working_dir / section.target() self._target = config.working_dir / section.target()
self._crawl_delay = section.crawl_delay() self._crawl_delay = section.crawl_delay()

160
PFERD/logging.py Normal file
View File

@ -0,0 +1,160 @@
import asyncio
from contextlib import asynccontextmanager, contextmanager
# TODO In Python 3.9 and above, ContextManager and AsyncContextManager are deprecated
from typing import AsyncIterator, ContextManager, Iterator, List, Optional
from rich.console import Console, RenderGroup
from rich.live import Live
from rich.markup import escape
from rich.progress import (BarColumn, DownloadColumn, Progress, TaskID, TextColumn, TimeRemainingColumn,
TransferSpeedColumn)
from rich.table import Column
class ProgressBar:
def __init__(self, progress: Progress, taskid: TaskID):
self._progress = progress
self._taskid = taskid
def advance(self, amount: float = 1) -> None:
self._progress.advance(self._taskid, advance=amount)
def set_total(self, total: float) -> None:
self._progress.update(self._taskid, total=total)
self._progress.start_task(self._taskid)
class Log:
def __init__(self) -> None:
self.console = Console(highlight=False)
self._crawl_progress = Progress(
TextColumn("{task.description}", table_column=Column(ratio=1)),
BarColumn(),
TimeRemainingColumn(),
expand=True,
)
self._download_progress = Progress(
TextColumn("{task.description}", table_column=Column(ratio=1)),
TransferSpeedColumn(),
DownloadColumn(),
BarColumn(),
TimeRemainingColumn(),
expand=True,
)
self._live = Live(console=self.console, transient=True)
self._update_live()
self._showing_progress = False
self._progress_suspended = False
self._lock = asyncio.Lock()
self._lines: List[str] = []
# Whether different parts of the output are enabled or disabled
self._enabled_explain = False
self._enabled_action = True
self._enabled_report = True
def _update_live(self) -> None:
elements = []
if self._crawl_progress.task_ids:
elements.append(self._crawl_progress)
if self._download_progress.task_ids:
elements.append(self._download_progress)
group = RenderGroup(*elements) # type: ignore
self._live.update(group)
def configure(self, explain: bool, action: bool, report: bool) -> None:
self._enabled_explain = explain
self._enabled_action = action
self._enabled_report = report
@contextmanager
def show_progress(self) -> Iterator[None]:
if self._showing_progress:
raise RuntimeError("Calling 'show_progress' while already showing progress")
self._showing_progress = True
try:
with self._live:
yield
finally:
self._showing_progress = False
@asynccontextmanager
async def exclusive_output(self) -> AsyncIterator[None]:
if not self._showing_progress:
raise RuntimeError("Calling 'exclusive_output' while not showing progress")
async with self._lock:
self._progress_suspended = True
self._live.stop()
try:
yield
finally:
self._live.start()
self._progress_suspended = False
for line in self._lines:
self.print(line)
self._lines = []
def print(self, text: str) -> None:
if self._progress_suspended:
self._lines.append(text)
else:
self.console.print(text)
def explain_topic(self, text: str) -> None:
if self._enabled_explain:
self.print(f"[cyan]{escape(text)}")
def explain(self, text: str) -> None:
if self._enabled_explain:
self.print(f" {escape(text)}")
def action(self, text: str) -> None:
if self._enabled_action:
self.print(text)
def report(self, text: str) -> None:
if self._enabled_report:
self.print(text)
@contextmanager
def _bar(
self,
progress: Progress,
description: str,
total: Optional[float],
) -> Iterator[ProgressBar]:
if total is None:
# Indeterminate progress bar
taskid = progress.add_task(description, start=False)
else:
taskid = progress.add_task(description, total=total)
self._update_live()
try:
yield ProgressBar(progress, taskid)
finally:
progress.remove_task(taskid)
self._update_live()
def crawl_bar(
self,
description: str,
total: Optional[float] = None,
) -> ContextManager[ProgressBar]:
return self._bar(self._crawl_progress, description, total)
def download_bar(
self,
description: str,
total: Optional[float] = None,
) -> ContextManager[ProgressBar]:
return self._bar(self._download_progress, description, total)
log = Log()

View File

@ -13,7 +13,7 @@ from typing import AsyncContextManager, AsyncIterator, BinaryIO, Iterator, Optio
from rich.markup import escape from rich.markup import escape
from .conductor import TerminalConductor from .logging import log
from .report import MarkConflictException, MarkDuplicateException, Report from .report import MarkConflictException, MarkDuplicateException, Report
from .utils import prompt_yes_no from .utils import prompt_yes_no
@ -93,12 +93,10 @@ class OutputDirectory:
root: Path, root: Path,
redownload: Redownload, redownload: Redownload,
on_conflict: OnConflict, on_conflict: OnConflict,
conductor: TerminalConductor,
): ):
self._root = root self._root = root
self._redownload = redownload self._redownload = redownload
self._on_conflict = on_conflict self._on_conflict = on_conflict
self._conductor = conductor
self._report = Report() self._report = Report()
@ -176,7 +174,7 @@ class OutputDirectory:
path: PurePath, path: PurePath,
) -> bool: ) -> bool:
if on_conflict == OnConflict.PROMPT: if on_conflict == OnConflict.PROMPT:
async with self._conductor.exclusive_output(): async with log.exclusive_output():
prompt = f"Replace {path} with remote file?" prompt = f"Replace {path} with remote file?"
return await prompt_yes_no(prompt, default=False) return await prompt_yes_no(prompt, default=False)
elif on_conflict == OnConflict.LOCAL_FIRST: elif on_conflict == OnConflict.LOCAL_FIRST:
@ -195,7 +193,7 @@ class OutputDirectory:
path: PurePath, path: PurePath,
) -> bool: ) -> bool:
if on_conflict == OnConflict.PROMPT: if on_conflict == OnConflict.PROMPT:
async with self._conductor.exclusive_output(): async with log.exclusive_output():
prompt = f"Recursively delete {path} and replace with remote file?" prompt = f"Recursively delete {path} and replace with remote file?"
return await prompt_yes_no(prompt, default=False) return await prompt_yes_no(prompt, default=False)
elif on_conflict == OnConflict.LOCAL_FIRST: elif on_conflict == OnConflict.LOCAL_FIRST:
@ -215,7 +213,7 @@ class OutputDirectory:
parent: PurePath, parent: PurePath,
) -> bool: ) -> bool:
if on_conflict == OnConflict.PROMPT: if on_conflict == OnConflict.PROMPT:
async with self._conductor.exclusive_output(): async with log.exclusive_output():
prompt = f"Delete {parent} so remote file {path} can be downloaded?" prompt = f"Delete {parent} so remote file {path} can be downloaded?"
return await prompt_yes_no(prompt, default=False) return await prompt_yes_no(prompt, default=False)
elif on_conflict == OnConflict.LOCAL_FIRST: elif on_conflict == OnConflict.LOCAL_FIRST:
@ -234,7 +232,7 @@ class OutputDirectory:
path: PurePath, path: PurePath,
) -> bool: ) -> bool:
if on_conflict == OnConflict.PROMPT: if on_conflict == OnConflict.PROMPT:
async with self._conductor.exclusive_output(): async with log.exclusive_output():
prompt = f"Delete {path}?" prompt = f"Delete {path}?"
return await prompt_yes_no(prompt, default=False) return await prompt_yes_no(prompt, default=False)
elif on_conflict == OnConflict.LOCAL_FIRST: elif on_conflict == OnConflict.LOCAL_FIRST:
@ -356,12 +354,10 @@ class OutputDirectory:
self._update_metadata(info) self._update_metadata(info)
if changed: if changed:
self._conductor.print( log.action(f"[bold bright_yellow]Changed[/] {escape(str(info.path))}")
f"[bold bright_yellow]Changed[/] {escape(str(info.path))}")
self._report.change_file(info.path) self._report.change_file(info.path)
else: else:
self._conductor.print( log.action(f"[bold bright_green]Added[/] {escape(str(info.path))}")
f"[bold bright_green]Added[/] {escape(str(info.path))}")
self._report.add_file(info.path) self._report.add_file(info.path)
async def cleanup(self) -> None: async def cleanup(self) -> None:
@ -390,8 +386,7 @@ class OutputDirectory:
if await self._conflict_delete_lf(self._on_conflict, pure): if await self._conflict_delete_lf(self._on_conflict, pure):
try: try:
path.unlink() path.unlink()
self._conductor.print( log.action(f"[bold bright_magenta]Deleted[/] {escape(str(path))}")
f"[bold bright_magenta]Deleted[/] {escape(str(path))}")
self._report.delete_file(pure) self._report.delete_file(pure)
except OSError: except OSError:
pass pass

View File

@ -5,7 +5,6 @@ from rich.markup import escape
from .authenticator import Authenticator from .authenticator import Authenticator
from .authenticators import AUTHENTICATORS from .authenticators import AUTHENTICATORS
from .conductor import TerminalConductor
from .config import Config from .config import Config
from .crawler import Crawler from .crawler import Crawler
from .crawlers import CRAWLERS from .crawlers import CRAWLERS
@ -18,7 +17,6 @@ class PferdLoadException(Exception):
class Pferd: class Pferd:
def __init__(self, config: Config): def __init__(self, config: Config):
self._config = config self._config = config
self._conductor = TerminalConductor()
self._authenticators: Dict[str, Authenticator] = {} self._authenticators: Dict[str, Authenticator] = {}
self._crawlers: Dict[str, Crawler] = {} self._crawlers: Dict[str, Crawler] = {}
@ -34,12 +32,7 @@ class Pferd:
print(f"[red]Error: Unknown authenticator type {t}") print(f"[red]Error: Unknown authenticator type {t}")
continue continue
authenticator = authenticator_constructor( authenticator = authenticator_constructor(name, section, self._config)
name,
section,
self._config,
self._conductor,
)
self._authenticators[name] = authenticator self._authenticators[name] = authenticator
if abort: if abort:
@ -57,13 +50,7 @@ class Pferd:
print(f"[red]Error: Unknown crawler type {t}") print(f"[red]Error: Unknown crawler type {t}")
continue continue
crawler = crawler_constructor( crawler = crawler_constructor(name, section, self._config, self._authenticators)
name,
section,
self._config,
self._conductor,
self._authenticators,
)
self._crawlers[name] = crawler self._crawlers[name] = crawler
if abort: if abort: