diff --git a/PFERD/authenticator.py b/PFERD/authenticator.py index 7475e2a..d67b263 100644 --- a/PFERD/authenticator.py +++ b/PFERD/authenticator.py @@ -1,7 +1,6 @@ from abc import ABC, abstractmethod from typing import Tuple -from .conductor import TerminalConductor from .config import Config, Section @@ -23,7 +22,6 @@ class Authenticator(ABC): name: str, section: AuthSection, config: Config, - conductor: TerminalConductor, ) -> None: """ Initialize an authenticator from its name and its section in the config @@ -36,7 +34,6 @@ class Authenticator(ABC): """ self.name = name - self.conductor = conductor @abstractmethod async def credentials(self) -> Tuple[str, str]: diff --git a/PFERD/authenticators/__init__.py b/PFERD/authenticators/__init__.py index 97ff03a..35096cf 100644 --- a/PFERD/authenticators/__init__.py +++ b/PFERD/authenticators/__init__.py @@ -2,7 +2,6 @@ from configparser import SectionProxy from typing import Callable, Dict from ..authenticator import Authenticator, AuthSection -from ..conductor import TerminalConductor from ..config import Config from .simple import SimpleAuthenticator, SimpleAuthSection from .tfa import TfaAuthenticator @@ -11,12 +10,11 @@ AuthConstructor = Callable[[ str, # Name (without the "auth:" prefix) SectionProxy, # Authenticator's section of global config Config, # Global config - TerminalConductor, # Global conductor instance ], Authenticator] AUTHENTICATORS: Dict[str, AuthConstructor] = { - "simple": lambda n, s, c, t: - SimpleAuthenticator(n, SimpleAuthSection(s), c, t), - "tfa": lambda n, s, c, t: - TfaAuthenticator(n, AuthSection(s), c, t), + "simple": lambda n, s, c: + SimpleAuthenticator(n, SimpleAuthSection(s), c), + "tfa": lambda n, s, c: + TfaAuthenticator(n, AuthSection(s), c), } diff --git a/PFERD/authenticators/simple.py b/PFERD/authenticators/simple.py index f21661c..caa0002 100644 --- a/PFERD/authenticators/simple.py +++ b/PFERD/authenticators/simple.py @@ -1,8 +1,8 @@ from typing import Optional, Tuple from ..authenticator import Authenticator, AuthException, AuthSection -from ..conductor import TerminalConductor from ..config import Config +from ..logging import log from ..utils import agetpass, ainput @@ -20,9 +20,8 @@ class SimpleAuthenticator(Authenticator): name: str, section: SimpleAuthSection, config: Config, - conductor: TerminalConductor, ) -> None: - super().__init__(name, section, config, conductor) + super().__init__(name, section, config) self._username = section.username() self._password = section.password() @@ -34,7 +33,7 @@ class SimpleAuthenticator(Authenticator): if self._username is not None and self._password is not None: return self._username, self._password - async with self.conductor.exclusive_output(): + async with log.exclusive_output(): if self._username is None: self._username = await ainput("Username: ") else: diff --git a/PFERD/authenticators/tfa.py b/PFERD/authenticators/tfa.py index 3513d09..b0eef18 100644 --- a/PFERD/authenticators/tfa.py +++ b/PFERD/authenticators/tfa.py @@ -1,8 +1,8 @@ from typing import Tuple from ..authenticator import Authenticator, AuthException, AuthSection -from ..conductor import TerminalConductor from ..config import Config +from ..logging import log from ..utils import ainput @@ -12,15 +12,14 @@ class TfaAuthenticator(Authenticator): name: str, section: AuthSection, config: Config, - conductor: TerminalConductor, ) -> None: - super().__init__(name, section, config, conductor) + super().__init__(name, section, config) async def username(self) -> str: raise AuthException("TFA authenticator does not support usernames") async def password(self) -> str: - async with self.conductor.exclusive_output(): + async with log.exclusive_output(): code = await ainput("TFA code: ") return code diff --git a/PFERD/conductor.py b/PFERD/conductor.py deleted file mode 100644 index d50574e..0000000 --- a/PFERD/conductor.py +++ /dev/null @@ -1,95 +0,0 @@ -import asyncio -from contextlib import asynccontextmanager, contextmanager -from types import TracebackType -from typing import AsyncIterator, Iterator, List, Optional, Type - -from rich.console import Console -from rich.progress import Progress, TaskID - - -class ProgressBar: - def __init__(self, progress: Progress, taskid: TaskID): - self._progress = progress - self._taskid = taskid - - def advance(self, amount: float = 1) -> None: - self._progress.advance(self._taskid, advance=amount) - - def set_total(self, total: float) -> None: - self._progress.update(self._taskid, total=total) - self._progress.start_task(self._taskid) - - -class TerminalConductor: - def __init__(self) -> None: - self._stopped = False - self._lock = asyncio.Lock() - self._lines: List[str] = [] - - self._console = Console(highlight=False) - self._progress = Progress(console=self._console) - - async def _start(self) -> None: - for task in self._progress.tasks: - task.visible = True - self._progress.start() - - self._stopped = False - - for line in self._lines: - self.print(line) - self._lines = [] - - async def _stop(self) -> None: - self._stopped = True - - for task in self._progress.tasks: - task.visible = False - self._progress.stop() - - async def __aenter__(self) -> None: - async with self._lock: - await self._start() - - async def __aexit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> Optional[bool]: - async with self._lock: - await self._stop() - return None - - def print(self, line: str) -> None: - if self._stopped: - self._lines.append(line) - else: - self._console.print(line) - - @asynccontextmanager - async def exclusive_output(self) -> AsyncIterator[None]: - async with self._lock: - await self._stop() - try: - yield - finally: - await self._start() - - @contextmanager - def progress_bar( - self, - description: str, - total: Optional[float] = None, - ) -> Iterator[ProgressBar]: - if total is None: - # Indeterminate progress bar - taskid = self._progress.add_task(description, start=False) - else: - taskid = self._progress.add_task(description, total=total) - - bar = ProgressBar(self._progress, taskid) - try: - yield bar - finally: - self._progress.remove_task(taskid) diff --git a/PFERD/crawler.py b/PFERD/crawler.py index cb31223..677baa2 100644 --- a/PFERD/crawler.py +++ b/PFERD/crawler.py @@ -9,9 +9,9 @@ import aiohttp from rich.markup import escape from .authenticator import Authenticator -from .conductor import ProgressBar, TerminalConductor from .config import Config, Section from .limiter import Limiter +from .logging import ProgressBar, log from .output_dir import FileSink, OnConflict, OutputDirectory, Redownload from .transformer import RuleParseException, Transformer from .version import __version__ @@ -36,7 +36,7 @@ def noncritical(f: Wrapped) -> Wrapped: try: f(self, *args, **kwargs) except Exception as e: - self.print(f"[red]Something went wrong: {escape(str(e))}") + log.print(f"[red]Something went wrong: {escape(str(e))}") self.error_free = False return wrapper # type: ignore @@ -79,7 +79,7 @@ def anoncritical(f: AWrapped) -> AWrapped: try: await f(self, *args, **kwargs) except Exception as e: - self.print(f"[red]Something went wrong: {escape(str(e))}") + log.print(f"[red]Something went wrong: {escape(str(e))}") self.error_free = False return wrapper # type: ignore @@ -182,7 +182,6 @@ class Crawler(ABC): name: str, section: CrawlerSection, config: Config, - conductor: TerminalConductor, ) -> None: """ Initialize a crawler from its name and its section in the config file. @@ -194,7 +193,6 @@ class Crawler(ABC): """ self.name = name - self._conductor = conductor self.error_free = True self._limiter = Limiter( @@ -213,34 +211,8 @@ class Crawler(ABC): config.working_dir / section.output_dir(name), section.redownload(), section.on_conflict(), - self._conductor, ) - def print(self, text: str) -> None: - """ - Print rich markup to the terminal. Crawlers *must* use this function to - print things unless they are holding an exclusive output context - manager! Be careful to escape all user-supplied strings. - """ - - self._conductor.print(text) - - def exclusive_output(self) -> AsyncContextManager[None]: - """ - Acquire exclusive rights™ to the terminal output. While this context - manager is held, output such as printing and progress bars from other - threads is suspended and the current thread may do whatever it wants - with the terminal. However, it must return the terminal to its original - state before exiting the context manager. - - No two threads can hold this context manager at the same time. - - Useful for password or confirmation prompts as well as running other - programs while crawling (e. g. to get certain credentials). - """ - - return self._conductor.exclusive_output() - @asynccontextmanager async def crawl_bar( self, @@ -249,7 +221,7 @@ class Crawler(ABC): ) -> AsyncIterator[ProgressBar]: desc = f"[bold bright_cyan]Crawling[/] {escape(str(path))}" async with self._limiter.limit_crawl(): - with self._conductor.progress_bar(desc, total=total) as bar: + with log.crawl_bar(desc, total=total) as bar: yield bar @asynccontextmanager @@ -260,7 +232,7 @@ class Crawler(ABC): ) -> AsyncIterator[ProgressBar]: desc = f"[bold bright_cyan]Downloading[/] {escape(str(path))}" async with self._limiter.limit_download(): - with self._conductor.progress_bar(desc, total=total) as bar: + with log.download_bar(desc, total=total) as bar: yield bar def should_crawl(self, path: PurePath) -> bool: @@ -289,7 +261,7 @@ class Crawler(ABC): crawler. """ - async with self._conductor: + with log.show_progress(): await self.crawl() @abstractmethod @@ -312,9 +284,8 @@ class HttpCrawler(Crawler): name: str, section: CrawlerSection, config: Config, - conductor: TerminalConductor, ) -> None: - super().__init__(name, section, config, conductor) + super().__init__(name, section, config) self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE) self._output_dir.register_reserved(self.COOKIE_FILE) @@ -340,7 +311,4 @@ class HttpCrawler(Crawler): try: cookie_jar.save(self._cookie_jar_path) except Exception: - self.print( - "[bold red]Warning:[/] Failed to save cookies to " - + escape(str(self.COOKIE_FILE)) - ) + log.print(f"[bold red]Warning:[/] Failed to save cookies to {escape(str(self.COOKIE_FILE))}") diff --git a/PFERD/crawlers/__init__.py b/PFERD/crawlers/__init__.py index 41733cb..72d6798 100644 --- a/PFERD/crawlers/__init__.py +++ b/PFERD/crawlers/__init__.py @@ -2,7 +2,6 @@ from configparser import SectionProxy from typing import Callable, Dict from ..authenticator import Authenticator -from ..conductor import TerminalConductor from ..config import Config from ..crawler import Crawler from .ilias import KitIliasCrawler, KitIliasCrawlerSection @@ -12,13 +11,12 @@ CrawlerConstructor = Callable[[ str, # Name (without the "crawl:" prefix) SectionProxy, # Crawler's section of global config Config, # Global config - TerminalConductor, # Global conductor instance Dict[str, Authenticator], # Loaded authenticators by name ], Crawler] CRAWLERS: Dict[str, CrawlerConstructor] = { - "local": lambda n, s, c, t, a: - LocalCrawler(n, LocalCrawlerSection(s), c, t), - "kit-ilias": lambda n, s, c, t, a: - KitIliasCrawler(n, KitIliasCrawlerSection(s), c, t, a), + "local": lambda n, s, c, a: + LocalCrawler(n, LocalCrawlerSection(s), c), + "kit-ilias": lambda n, s, c, a: + KitIliasCrawler(n, KitIliasCrawlerSection(s), c, a), } diff --git a/PFERD/crawlers/ilias.py b/PFERD/crawlers/ilias.py index 014f231..beac208 100644 --- a/PFERD/crawlers/ilias.py +++ b/PFERD/crawlers/ilias.py @@ -16,7 +16,6 @@ from PFERD.output_dir import Redownload from PFERD.utils import soupify from ..authenticators import Authenticator -from ..conductor import TerminalConductor from ..config import Config from ..crawler import CrawlerSection, HttpCrawler, anoncritical, arepeat @@ -533,10 +532,9 @@ class KitIliasCrawler(HttpCrawler): name: str, section: KitIliasCrawlerSection, config: Config, - conductor: TerminalConductor, authenticators: Dict[str, Authenticator] ): - super().__init__(name, section, config, conductor) + super().__init__(name, section, config) self._shibboleth_login = KitShibbolethLogin( section.auth(authenticators), @@ -615,7 +613,7 @@ class KitIliasCrawler(HttpCrawler): await self._download_file(element, element_path) elif element.type == IliasElementType.FORUM: # TODO: Delete - self.print(f"Skipping forum [green]{element_path}[/]") + print(f"Skipping forum [green]{element_path}[/]") elif element.type == IliasElementType.LINK: await self._download_link(element, element_path) elif element.type == IliasElementType.VIDEO: diff --git a/PFERD/crawlers/local.py b/PFERD/crawlers/local.py index 2dde0d4..363107f 100644 --- a/PFERD/crawlers/local.py +++ b/PFERD/crawlers/local.py @@ -4,7 +4,6 @@ import random from pathlib import Path, PurePath from typing import Optional -from ..conductor import TerminalConductor from ..config import Config from ..crawler import Crawler, CrawlerSection, anoncritical @@ -44,9 +43,8 @@ class LocalCrawler(Crawler): name: str, section: LocalCrawlerSection, config: Config, - conductor: TerminalConductor, ): - super().__init__(name, section, config, conductor) + super().__init__(name, section, config) self._target = config.working_dir / section.target() self._crawl_delay = section.crawl_delay() diff --git a/PFERD/logging.py b/PFERD/logging.py new file mode 100644 index 0000000..b075d35 --- /dev/null +++ b/PFERD/logging.py @@ -0,0 +1,160 @@ +import asyncio +from contextlib import asynccontextmanager, contextmanager +# TODO In Python 3.9 and above, ContextManager and AsyncContextManager are deprecated +from typing import AsyncIterator, ContextManager, Iterator, List, Optional + +from rich.console import Console, RenderGroup +from rich.live import Live +from rich.markup import escape +from rich.progress import (BarColumn, DownloadColumn, Progress, TaskID, TextColumn, TimeRemainingColumn, + TransferSpeedColumn) +from rich.table import Column + + +class ProgressBar: + def __init__(self, progress: Progress, taskid: TaskID): + self._progress = progress + self._taskid = taskid + + def advance(self, amount: float = 1) -> None: + self._progress.advance(self._taskid, advance=amount) + + def set_total(self, total: float) -> None: + self._progress.update(self._taskid, total=total) + self._progress.start_task(self._taskid) + + +class Log: + def __init__(self) -> None: + self.console = Console(highlight=False) + + self._crawl_progress = Progress( + TextColumn("{task.description}", table_column=Column(ratio=1)), + BarColumn(), + TimeRemainingColumn(), + expand=True, + ) + self._download_progress = Progress( + TextColumn("{task.description}", table_column=Column(ratio=1)), + TransferSpeedColumn(), + DownloadColumn(), + BarColumn(), + TimeRemainingColumn(), + expand=True, + ) + + self._live = Live(console=self.console, transient=True) + self._update_live() + + self._showing_progress = False + self._progress_suspended = False + self._lock = asyncio.Lock() + self._lines: List[str] = [] + + # Whether different parts of the output are enabled or disabled + self._enabled_explain = False + self._enabled_action = True + self._enabled_report = True + + def _update_live(self) -> None: + elements = [] + if self._crawl_progress.task_ids: + elements.append(self._crawl_progress) + if self._download_progress.task_ids: + elements.append(self._download_progress) + + group = RenderGroup(*elements) # type: ignore + self._live.update(group) + + def configure(self, explain: bool, action: bool, report: bool) -> None: + self._enabled_explain = explain + self._enabled_action = action + self._enabled_report = report + + @contextmanager + def show_progress(self) -> Iterator[None]: + if self._showing_progress: + raise RuntimeError("Calling 'show_progress' while already showing progress") + + self._showing_progress = True + try: + with self._live: + yield + finally: + self._showing_progress = False + + @asynccontextmanager + async def exclusive_output(self) -> AsyncIterator[None]: + if not self._showing_progress: + raise RuntimeError("Calling 'exclusive_output' while not showing progress") + + async with self._lock: + self._progress_suspended = True + self._live.stop() + try: + yield + finally: + self._live.start() + self._progress_suspended = False + for line in self._lines: + self.print(line) + self._lines = [] + + def print(self, text: str) -> None: + if self._progress_suspended: + self._lines.append(text) + else: + self.console.print(text) + + def explain_topic(self, text: str) -> None: + if self._enabled_explain: + self.print(f"[cyan]{escape(text)}") + + def explain(self, text: str) -> None: + if self._enabled_explain: + self.print(f" {escape(text)}") + + def action(self, text: str) -> None: + if self._enabled_action: + self.print(text) + + def report(self, text: str) -> None: + if self._enabled_report: + self.print(text) + + @contextmanager + def _bar( + self, + progress: Progress, + description: str, + total: Optional[float], + ) -> Iterator[ProgressBar]: + if total is None: + # Indeterminate progress bar + taskid = progress.add_task(description, start=False) + else: + taskid = progress.add_task(description, total=total) + self._update_live() + + try: + yield ProgressBar(progress, taskid) + finally: + progress.remove_task(taskid) + self._update_live() + + def crawl_bar( + self, + description: str, + total: Optional[float] = None, + ) -> ContextManager[ProgressBar]: + return self._bar(self._crawl_progress, description, total) + + def download_bar( + self, + description: str, + total: Optional[float] = None, + ) -> ContextManager[ProgressBar]: + return self._bar(self._download_progress, description, total) + + +log = Log() diff --git a/PFERD/output_dir.py b/PFERD/output_dir.py index ae69d10..417fa52 100644 --- a/PFERD/output_dir.py +++ b/PFERD/output_dir.py @@ -13,7 +13,7 @@ from typing import AsyncContextManager, AsyncIterator, BinaryIO, Iterator, Optio from rich.markup import escape -from .conductor import TerminalConductor +from .logging import log from .report import MarkConflictException, MarkDuplicateException, Report from .utils import prompt_yes_no @@ -93,12 +93,10 @@ class OutputDirectory: root: Path, redownload: Redownload, on_conflict: OnConflict, - conductor: TerminalConductor, ): self._root = root self._redownload = redownload self._on_conflict = on_conflict - self._conductor = conductor self._report = Report() @@ -176,7 +174,7 @@ class OutputDirectory: path: PurePath, ) -> bool: if on_conflict == OnConflict.PROMPT: - async with self._conductor.exclusive_output(): + async with log.exclusive_output(): prompt = f"Replace {path} with remote file?" return await prompt_yes_no(prompt, default=False) elif on_conflict == OnConflict.LOCAL_FIRST: @@ -195,7 +193,7 @@ class OutputDirectory: path: PurePath, ) -> bool: if on_conflict == OnConflict.PROMPT: - async with self._conductor.exclusive_output(): + async with log.exclusive_output(): prompt = f"Recursively delete {path} and replace with remote file?" return await prompt_yes_no(prompt, default=False) elif on_conflict == OnConflict.LOCAL_FIRST: @@ -215,7 +213,7 @@ class OutputDirectory: parent: PurePath, ) -> bool: if on_conflict == OnConflict.PROMPT: - async with self._conductor.exclusive_output(): + async with log.exclusive_output(): prompt = f"Delete {parent} so remote file {path} can be downloaded?" return await prompt_yes_no(prompt, default=False) elif on_conflict == OnConflict.LOCAL_FIRST: @@ -234,7 +232,7 @@ class OutputDirectory: path: PurePath, ) -> bool: if on_conflict == OnConflict.PROMPT: - async with self._conductor.exclusive_output(): + async with log.exclusive_output(): prompt = f"Delete {path}?" return await prompt_yes_no(prompt, default=False) elif on_conflict == OnConflict.LOCAL_FIRST: @@ -356,12 +354,10 @@ class OutputDirectory: self._update_metadata(info) if changed: - self._conductor.print( - f"[bold bright_yellow]Changed[/] {escape(str(info.path))}") + log.action(f"[bold bright_yellow]Changed[/] {escape(str(info.path))}") self._report.change_file(info.path) else: - self._conductor.print( - f"[bold bright_green]Added[/] {escape(str(info.path))}") + log.action(f"[bold bright_green]Added[/] {escape(str(info.path))}") self._report.add_file(info.path) async def cleanup(self) -> None: @@ -390,8 +386,7 @@ class OutputDirectory: if await self._conflict_delete_lf(self._on_conflict, pure): try: path.unlink() - self._conductor.print( - f"[bold bright_magenta]Deleted[/] {escape(str(path))}") + log.action(f"[bold bright_magenta]Deleted[/] {escape(str(path))}") self._report.delete_file(pure) except OSError: pass diff --git a/PFERD/pferd.py b/PFERD/pferd.py index 9154a80..10cd1c2 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -5,7 +5,6 @@ from rich.markup import escape from .authenticator import Authenticator from .authenticators import AUTHENTICATORS -from .conductor import TerminalConductor from .config import Config from .crawler import Crawler from .crawlers import CRAWLERS @@ -18,7 +17,6 @@ class PferdLoadException(Exception): class Pferd: def __init__(self, config: Config): self._config = config - self._conductor = TerminalConductor() self._authenticators: Dict[str, Authenticator] = {} self._crawlers: Dict[str, Crawler] = {} @@ -34,12 +32,7 @@ class Pferd: print(f"[red]Error: Unknown authenticator type {t}") continue - authenticator = authenticator_constructor( - name, - section, - self._config, - self._conductor, - ) + authenticator = authenticator_constructor(name, section, self._config) self._authenticators[name] = authenticator if abort: @@ -57,13 +50,7 @@ class Pferd: print(f"[red]Error: Unknown crawler type {t}") continue - crawler = crawler_constructor( - name, - section, - self._config, - self._conductor, - self._authenticators, - ) + crawler = crawler_constructor(name, section, self._config, self._authenticators) self._crawlers[name] = crawler if abort: