Move logging logic to singleton

- Renamed module and class because "conductor" didn't make a lot of sense
- Used singleton approach (there's only one stdout after all)
- Redesigned progress bars (now with download speed!)

PFERD/authenticator.py

@@ -1,7 +1,6 @@
 from abc import ABC, abstractmethod
 from typing import Tuple
 
-from .conductor import TerminalConductor
 from .config import Config, Section
 
 
@@ -23,7 +22,6 @@ class Authenticator(ABC):
             name: str,
             section: AuthSection,
             config: Config,
-            conductor: TerminalConductor,
     ) -> None:
         """
         Initialize an authenticator from its name and its section in the config
@@ -36,7 +34,6 @@ class Authenticator(ABC):
         """
 
         self.name = name
-        self.conductor = conductor
 
     @abstractmethod
     async def credentials(self) -> Tuple[str, str]:

PFERD/authenticators/__init__.py

@@ -2,7 +2,6 @@ from configparser import SectionProxy
 from typing import Callable, Dict
 
 from ..authenticator import Authenticator, AuthSection
-from ..conductor import TerminalConductor
 from ..config import Config
 from .simple import SimpleAuthenticator, SimpleAuthSection
 from .tfa import TfaAuthenticator
@@ -11,12 +10,11 @@ AuthConstructor = Callable[[
     str,                # Name (without the "auth:" prefix)
     SectionProxy,       # Authenticator's section of global config
     Config,             # Global config
-    TerminalConductor,  # Global conductor instance
 ], Authenticator]
 
 AUTHENTICATORS: Dict[str, AuthConstructor] = {
-    "simple": lambda n, s, c, t:
+    "simple": lambda n, s, c:
-        SimpleAuthenticator(n, SimpleAuthSection(s), c, t),
+        SimpleAuthenticator(n, SimpleAuthSection(s), c),
-    "tfa": lambda n, s, c, t:
+    "tfa": lambda n, s, c:
-        TfaAuthenticator(n, AuthSection(s), c, t),
+        TfaAuthenticator(n, AuthSection(s), c),
 }

PFERD/authenticators/simple.py

@@ -1,8 +1,8 @@
 from typing import Optional, Tuple
 
 from ..authenticator import Authenticator, AuthException, AuthSection
-from ..conductor import TerminalConductor
 from ..config import Config
+from ..logging import log
 from ..utils import agetpass, ainput
 
 
@@ -20,9 +20,8 @@ class SimpleAuthenticator(Authenticator):
             name: str,
             section: SimpleAuthSection,
             config: Config,
-            conductor: TerminalConductor,
     ) -> None:
-        super().__init__(name, section, config, conductor)
+        super().__init__(name, section, config)
 
         self._username = section.username()
         self._password = section.password()
@@ -34,7 +33,7 @@ class SimpleAuthenticator(Authenticator):
         if self._username is not None and self._password is not None:
             return self._username, self._password
 
-        async with self.conductor.exclusive_output():
+        async with log.exclusive_output():
             if self._username is None:
                 self._username = await ainput("Username: ")
             else:

PFERD/authenticators/tfa.py

@@ -1,8 +1,8 @@
 from typing import Tuple
 
 from ..authenticator import Authenticator, AuthException, AuthSection
-from ..conductor import TerminalConductor
 from ..config import Config
+from ..logging import log
 from ..utils import ainput
 
 
@@ -12,15 +12,14 @@ class TfaAuthenticator(Authenticator):
             name: str,
             section: AuthSection,
             config: Config,
-            conductor: TerminalConductor,
     ) -> None:
-        super().__init__(name, section, config, conductor)
+        super().__init__(name, section, config)
 
     async def username(self) -> str:
         raise AuthException("TFA authenticator does not support usernames")
 
     async def password(self) -> str:
-        async with self.conductor.exclusive_output():
+        async with log.exclusive_output():
             code = await ainput("TFA code: ")
             return code
 

PFERD/conductor.py

@@ -1,95 +0,0 @@
-import asyncio
-from contextlib import asynccontextmanager, contextmanager
-from types import TracebackType
-from typing import AsyncIterator, Iterator, List, Optional, Type
-
-from rich.console import Console
-from rich.progress import Progress, TaskID
-
-
-class ProgressBar:
-    def __init__(self, progress: Progress, taskid: TaskID):
-        self._progress = progress
-        self._taskid = taskid
-
-    def advance(self, amount: float = 1) -> None:
-        self._progress.advance(self._taskid, advance=amount)
-
-    def set_total(self, total: float) -> None:
-        self._progress.update(self._taskid, total=total)
-        self._progress.start_task(self._taskid)
-
-
-class TerminalConductor:
-    def __init__(self) -> None:
-        self._stopped = False
-        self._lock = asyncio.Lock()
-        self._lines: List[str] = []
-
-        self._console = Console(highlight=False)
-        self._progress = Progress(console=self._console)
-
-    async def _start(self) -> None:
-        for task in self._progress.tasks:
-            task.visible = True
-        self._progress.start()
-
-        self._stopped = False
-
-        for line in self._lines:
-            self.print(line)
-        self._lines = []
-
-    async def _stop(self) -> None:
-        self._stopped = True
-
-        for task in self._progress.tasks:
-            task.visible = False
-        self._progress.stop()
-
-    async def __aenter__(self) -> None:
-        async with self._lock:
-            await self._start()
-
-    async def __aexit__(
-            self,
-            exc_type: Optional[Type[BaseException]],
-            exc_value: Optional[BaseException],
-            traceback: Optional[TracebackType],
-    ) -> Optional[bool]:
-        async with self._lock:
-            await self._stop()
-        return None
-
-    def print(self, line: str) -> None:
-        if self._stopped:
-            self._lines.append(line)
-        else:
-            self._console.print(line)
-
-    @asynccontextmanager
-    async def exclusive_output(self) -> AsyncIterator[None]:
-        async with self._lock:
-            await self._stop()
-            try:
-                yield
-            finally:
-                await self._start()
-
-    @contextmanager
-    def progress_bar(
-            self,
-            description: str,
-            total: Optional[float] = None,
-    ) -> Iterator[ProgressBar]:
-        if total is None:
-            # Indeterminate progress bar
-            taskid = self._progress.add_task(description, start=False)
-        else:
-            taskid = self._progress.add_task(description, total=total)
-
-        bar = ProgressBar(self._progress, taskid)
-        try:
-            yield bar
-        finally:
-            self._progress.remove_task(taskid)

PFERD/crawler.py

@@ -9,9 +9,9 @@ import aiohttp
 from rich.markup import escape
 
 from .authenticator import Authenticator
-from .conductor import ProgressBar, TerminalConductor
 from .config import Config, Section
 from .limiter import Limiter
+from .logging import ProgressBar, log
 from .output_dir import FileSink, OnConflict, OutputDirectory, Redownload
 from .transformer import RuleParseException, Transformer
 from .version import __version__
@@ -36,7 +36,7 @@ def noncritical(f: Wrapped) -> Wrapped:
         try:
             f(self, *args, **kwargs)
         except Exception as e:
-            self.print(f"[red]Something went wrong: {escape(str(e))}")
+            log.print(f"[red]Something went wrong: {escape(str(e))}")
             self.error_free = False
     return wrapper  # type: ignore
 
@@ -79,7 +79,7 @@ def anoncritical(f: AWrapped) -> AWrapped:
         try:
             await f(self, *args, **kwargs)
         except Exception as e:
-            self.print(f"[red]Something went wrong: {escape(str(e))}")
+            log.print(f"[red]Something went wrong: {escape(str(e))}")
             self.error_free = False
     return wrapper  # type: ignore
 
@@ -182,7 +182,6 @@ class Crawler(ABC):
             name: str,
             section: CrawlerSection,
             config: Config,
-            conductor: TerminalConductor,
     ) -> None:
         """
         Initialize a crawler from its name and its section in the config file.
@@ -194,7 +193,6 @@ class Crawler(ABC):
         """
 
         self.name = name
-        self._conductor = conductor
         self.error_free = True
 
         self._limiter = Limiter(
@@ -213,34 +211,8 @@ class Crawler(ABC):
             config.working_dir / section.output_dir(name),
             section.redownload(),
             section.on_conflict(),
-            self._conductor,
         )
 
-    def print(self, text: str) -> None:
-        """
-        Print rich markup to the terminal. Crawlers *must* use this function to
-        print things unless they are holding an exclusive output context
-        manager! Be careful to escape all user-supplied strings.
-        """
-
-        self._conductor.print(text)
-
-    def exclusive_output(self) -> AsyncContextManager[None]:
-        """
-        Acquire exclusive rights™ to the terminal output. While this context
-        manager is held, output such as printing and progress bars from other
-        threads is suspended and the current thread may do whatever it wants
-        with the terminal. However, it must return the terminal to its original
-        state before exiting the context manager.
-
-        No two threads can hold this context manager at the same time.
-
-        Useful for password or confirmation prompts as well as running other
-        programs while crawling (e. g. to get certain credentials).
-        """
-
-        return self._conductor.exclusive_output()
-
     @asynccontextmanager
     async def crawl_bar(
             self,
@@ -249,7 +221,7 @@ class Crawler(ABC):
     ) -> AsyncIterator[ProgressBar]:
         desc = f"[bold bright_cyan]Crawling[/] {escape(str(path))}"
         async with self._limiter.limit_crawl():
-            with self._conductor.progress_bar(desc, total=total) as bar:
+            with log.crawl_bar(desc, total=total) as bar:
                 yield bar
 
     @asynccontextmanager
@@ -260,7 +232,7 @@ class Crawler(ABC):
     ) -> AsyncIterator[ProgressBar]:
         desc = f"[bold bright_cyan]Downloading[/] {escape(str(path))}"
         async with self._limiter.limit_download():
-            with self._conductor.progress_bar(desc, total=total) as bar:
+            with log.download_bar(desc, total=total) as bar:
                 yield bar
 
     def should_crawl(self, path: PurePath) -> bool:
@@ -289,7 +261,7 @@ class Crawler(ABC):
         crawler.
         """
 
-        async with self._conductor:
+        with log.show_progress():
             await self.crawl()
 
     @abstractmethod
@@ -312,9 +284,8 @@ class HttpCrawler(Crawler):
             name: str,
             section: CrawlerSection,
             config: Config,
-            conductor: TerminalConductor,
     ) -> None:
-        super().__init__(name, section, config, conductor)
+        super().__init__(name, section, config)
 
         self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE)
         self._output_dir.register_reserved(self.COOKIE_FILE)
@@ -340,7 +311,4 @@ class HttpCrawler(Crawler):
         try:
             cookie_jar.save(self._cookie_jar_path)
         except Exception:
-            self.print(
+            log.print(f"[bold red]Warning:[/] Failed to save cookies to {escape(str(self.COOKIE_FILE))}")
-                "[bold red]Warning:[/] Failed to save cookies to "
-                + escape(str(self.COOKIE_FILE))
-            )

PFERD/crawlers/__init__.py

@@ -2,7 +2,6 @@ from configparser import SectionProxy
 from typing import Callable, Dict
 
 from ..authenticator import Authenticator
-from ..conductor import TerminalConductor
 from ..config import Config
 from ..crawler import Crawler
 from .ilias import KitIliasCrawler, KitIliasCrawlerSection
@@ -12,13 +11,12 @@ CrawlerConstructor = Callable[[
     str,                       # Name (without the "crawl:" prefix)
     SectionProxy,              # Crawler's section of global config
     Config,                    # Global config
-    TerminalConductor,         # Global conductor instance
     Dict[str, Authenticator],  # Loaded authenticators by name
 ], Crawler]
 
 CRAWLERS: Dict[str, CrawlerConstructor] = {
-    "local": lambda n, s, c, t, a:
+    "local": lambda n, s, c, a:
-        LocalCrawler(n, LocalCrawlerSection(s), c, t),
+        LocalCrawler(n, LocalCrawlerSection(s), c),
-    "kit-ilias": lambda n, s, c, t, a:
+    "kit-ilias": lambda n, s, c, a:
-        KitIliasCrawler(n, KitIliasCrawlerSection(s), c, t, a),
+        KitIliasCrawler(n, KitIliasCrawlerSection(s), c, a),
 }

PFERD/crawlers/ilias.py

@@ -16,7 +16,6 @@ from PFERD.output_dir import Redownload
 from PFERD.utils import soupify
 
 from ..authenticators import Authenticator
-from ..conductor import TerminalConductor
 from ..config import Config
 from ..crawler import CrawlerSection, HttpCrawler, anoncritical, arepeat
 
@@ -533,10 +532,9 @@ class KitIliasCrawler(HttpCrawler):
             name: str,
             section: KitIliasCrawlerSection,
             config: Config,
-            conductor: TerminalConductor,
             authenticators: Dict[str, Authenticator]
     ):
-        super().__init__(name, section, config, conductor)
+        super().__init__(name, section, config)
 
         self._shibboleth_login = KitShibbolethLogin(
             section.auth(authenticators),
@@ -615,7 +613,7 @@ class KitIliasCrawler(HttpCrawler):
             await self._download_file(element, element_path)
         elif element.type == IliasElementType.FORUM:
             # TODO: Delete
-            self.print(f"Skipping forum [green]{element_path}[/]")
+            print(f"Skipping forum [green]{element_path}[/]")
         elif element.type == IliasElementType.LINK:
             await self._download_link(element, element_path)
         elif element.type == IliasElementType.VIDEO:

PFERD/crawlers/local.py

@@ -4,7 +4,6 @@ import random
 from pathlib import Path, PurePath
 from typing import Optional
 
-from ..conductor import TerminalConductor
 from ..config import Config
 from ..crawler import Crawler, CrawlerSection, anoncritical
 
@@ -44,9 +43,8 @@ class LocalCrawler(Crawler):
             name: str,
             section: LocalCrawlerSection,
             config: Config,
-            conductor: TerminalConductor,
     ):
-        super().__init__(name, section, config, conductor)
+        super().__init__(name, section, config)
 
         self._target = config.working_dir / section.target()
         self._crawl_delay = section.crawl_delay()

PFERD/logging.py

@@ -0,0 +1,160 @@
+import asyncio
+from contextlib import asynccontextmanager, contextmanager
+# TODO In Python 3.9 and above, ContextManager and AsyncContextManager are deprecated
+from typing import AsyncIterator, ContextManager, Iterator, List, Optional
+
+from rich.console import Console, RenderGroup
+from rich.live import Live
+from rich.markup import escape
+from rich.progress import (BarColumn, DownloadColumn, Progress, TaskID, TextColumn, TimeRemainingColumn,
+                           TransferSpeedColumn)
+from rich.table import Column
+
+
+class ProgressBar:
+    def __init__(self, progress: Progress, taskid: TaskID):
+        self._progress = progress
+        self._taskid = taskid
+
+    def advance(self, amount: float = 1) -> None:
+        self._progress.advance(self._taskid, advance=amount)
+
+    def set_total(self, total: float) -> None:
+        self._progress.update(self._taskid, total=total)
+        self._progress.start_task(self._taskid)
+
+
+class Log:
+    def __init__(self) -> None:
+        self.console = Console(highlight=False)
+
+        self._crawl_progress = Progress(
+            TextColumn("{task.description}", table_column=Column(ratio=1)),
+            BarColumn(),
+            TimeRemainingColumn(),
+            expand=True,
+        )
+        self._download_progress = Progress(
+            TextColumn("{task.description}", table_column=Column(ratio=1)),
+            TransferSpeedColumn(),
+            DownloadColumn(),
+            BarColumn(),
+            TimeRemainingColumn(),
+            expand=True,
+        )
+
+        self._live = Live(console=self.console, transient=True)
+        self._update_live()
+
+        self._showing_progress = False
+        self._progress_suspended = False
+        self._lock = asyncio.Lock()
+        self._lines: List[str] = []
+
+        # Whether different parts of the output are enabled or disabled
+        self._enabled_explain = False
+        self._enabled_action = True
+        self._enabled_report = True
+
+    def _update_live(self) -> None:
+        elements = []
+        if self._crawl_progress.task_ids:
+            elements.append(self._crawl_progress)
+        if self._download_progress.task_ids:
+            elements.append(self._download_progress)
+
+        group = RenderGroup(*elements)  # type: ignore
+        self._live.update(group)
+
+    def configure(self, explain: bool, action: bool, report: bool) -> None:
+        self._enabled_explain = explain
+        self._enabled_action = action
+        self._enabled_report = report
+
+    @contextmanager
+    def show_progress(self) -> Iterator[None]:
+        if self._showing_progress:
+            raise RuntimeError("Calling 'show_progress' while already showing progress")
+
+        self._showing_progress = True
+        try:
+            with self._live:
+                yield
+        finally:
+            self._showing_progress = False
+
+    @asynccontextmanager
+    async def exclusive_output(self) -> AsyncIterator[None]:
+        if not self._showing_progress:
+            raise RuntimeError("Calling 'exclusive_output' while not showing progress")
+
+        async with self._lock:
+            self._progress_suspended = True
+            self._live.stop()
+            try:
+                yield
+            finally:
+                self._live.start()
+                self._progress_suspended = False
+                for line in self._lines:
+                    self.print(line)
+                self._lines = []
+
+    def print(self, text: str) -> None:
+        if self._progress_suspended:
+            self._lines.append(text)
+        else:
+            self.console.print(text)
+
+    def explain_topic(self, text: str) -> None:
+        if self._enabled_explain:
+            self.print(f"[cyan]{escape(text)}")
+
+    def explain(self, text: str) -> None:
+        if self._enabled_explain:
+            self.print(f"  {escape(text)}")
+
+    def action(self, text: str) -> None:
+        if self._enabled_action:
+            self.print(text)
+
+    def report(self, text: str) -> None:
+        if self._enabled_report:
+            self.print(text)
+
+    @contextmanager
+    def _bar(
+            self,
+            progress: Progress,
+            description: str,
+            total: Optional[float],
+    ) -> Iterator[ProgressBar]:
+        if total is None:
+            # Indeterminate progress bar
+            taskid = progress.add_task(description, start=False)
+        else:
+            taskid = progress.add_task(description, total=total)
+        self._update_live()
+
+        try:
+            yield ProgressBar(progress, taskid)
+        finally:
+            progress.remove_task(taskid)
+            self._update_live()
+
+    def crawl_bar(
+            self,
+            description: str,
+            total: Optional[float] = None,
+    ) -> ContextManager[ProgressBar]:
+        return self._bar(self._crawl_progress, description, total)
+
+    def download_bar(
+            self,
+            description: str,
+            total: Optional[float] = None,
+    ) -> ContextManager[ProgressBar]:
+        return self._bar(self._download_progress, description, total)
+
+
+log = Log()

PFERD/output_dir.py

@@ -13,7 +13,7 @@ from typing import AsyncContextManager, AsyncIterator, BinaryIO, Iterator, Optio
 
 from rich.markup import escape
 
-from .conductor import TerminalConductor
+from .logging import log
 from .report import MarkConflictException, MarkDuplicateException, Report
 from .utils import prompt_yes_no
 
@@ -93,12 +93,10 @@ class OutputDirectory:
             root: Path,
             redownload: Redownload,
             on_conflict: OnConflict,
-            conductor: TerminalConductor,
     ):
         self._root = root
         self._redownload = redownload
         self._on_conflict = on_conflict
-        self._conductor = conductor
 
         self._report = Report()
 
@@ -176,7 +174,7 @@ class OutputDirectory:
             path: PurePath,
     ) -> bool:
         if on_conflict == OnConflict.PROMPT:
-            async with self._conductor.exclusive_output():
+            async with log.exclusive_output():
                 prompt = f"Replace {path} with remote file?"
                 return await prompt_yes_no(prompt, default=False)
         elif on_conflict == OnConflict.LOCAL_FIRST:
@@ -195,7 +193,7 @@ class OutputDirectory:
             path: PurePath,
     ) -> bool:
         if on_conflict == OnConflict.PROMPT:
-            async with self._conductor.exclusive_output():
+            async with log.exclusive_output():
                 prompt = f"Recursively delete {path} and replace with remote file?"
                 return await prompt_yes_no(prompt, default=False)
         elif on_conflict == OnConflict.LOCAL_FIRST:
@@ -215,7 +213,7 @@ class OutputDirectory:
             parent: PurePath,
     ) -> bool:
         if on_conflict == OnConflict.PROMPT:
-            async with self._conductor.exclusive_output():
+            async with log.exclusive_output():
                 prompt = f"Delete {parent} so remote file {path} can be downloaded?"
                 return await prompt_yes_no(prompt, default=False)
         elif on_conflict == OnConflict.LOCAL_FIRST:
@@ -234,7 +232,7 @@ class OutputDirectory:
             path: PurePath,
     ) -> bool:
         if on_conflict == OnConflict.PROMPT:
-            async with self._conductor.exclusive_output():
+            async with log.exclusive_output():
                 prompt = f"Delete {path}?"
                 return await prompt_yes_no(prompt, default=False)
         elif on_conflict == OnConflict.LOCAL_FIRST:
@@ -356,12 +354,10 @@ class OutputDirectory:
             self._update_metadata(info)
 
             if changed:
-                self._conductor.print(
+                log.action(f"[bold bright_yellow]Changed[/] {escape(str(info.path))}")
-                    f"[bold bright_yellow]Changed[/] {escape(str(info.path))}")
                 self._report.change_file(info.path)
             else:
-                self._conductor.print(
+                log.action(f"[bold bright_green]Added[/] {escape(str(info.path))}")
-                    f"[bold bright_green]Added[/] {escape(str(info.path))}")
                 self._report.add_file(info.path)
 
     async def cleanup(self) -> None:
@@ -390,8 +386,7 @@ class OutputDirectory:
         if await self._conflict_delete_lf(self._on_conflict, pure):
             try:
                 path.unlink()
-                self._conductor.print(
+                log.action(f"[bold bright_magenta]Deleted[/] {escape(str(path))}")
-                    f"[bold bright_magenta]Deleted[/] {escape(str(path))}")
                 self._report.delete_file(pure)
             except OSError:
                 pass

PFERD/pferd.py

@@ -5,7 +5,6 @@ from rich.markup import escape
 
 from .authenticator import Authenticator
 from .authenticators import AUTHENTICATORS
-from .conductor import TerminalConductor
 from .config import Config
 from .crawler import Crawler
 from .crawlers import CRAWLERS
@@ -18,7 +17,6 @@ class PferdLoadException(Exception):
 class Pferd:
     def __init__(self, config: Config):
         self._config = config
-        self._conductor = TerminalConductor()
         self._authenticators: Dict[str, Authenticator] = {}
         self._crawlers: Dict[str, Crawler] = {}
 
@@ -34,12 +32,7 @@ class Pferd:
                 print(f"[red]Error: Unknown authenticator type {t}")
                 continue
 
-            authenticator = authenticator_constructor(
+            authenticator = authenticator_constructor(name, section, self._config)
-                name,
-                section,
-                self._config,
-                self._conductor,
-            )
             self._authenticators[name] = authenticator
 
         if abort:
@@ -57,13 +50,7 @@ class Pferd:
                 print(f"[red]Error: Unknown crawler type {t}")
                 continue
 
-            crawler = crawler_constructor(
+            crawler = crawler_constructor(name, section, self._config, self._authenticators)
-                name,
-                section,
-                self._config,
-                self._conductor,
-                self._authenticators,
-            )
             self._crawlers[name] = crawler
 
         if abort: