mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Move logging logic to singleton

Browse Source 
- Renamed module and class because "conductor" didn't make a lot of sense
- Used singleton approach (there's only one stdout after all)
- Redesigned progress bars (now with download speed!)
This commit is contained in:
Joscha 2021-05-18 22:43:46 +02:00
parent 1525aa15a6
commit 4b68fa771f
12 changed files with 195 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
PFERD/authenticator.py
View File

@ -1,7 +1,6 @@
from abc import ABC, abstractmethod
from typing import Tuple
from .conductor import TerminalConductor
from .config import Config, Section
@ -23,7 +22,6 @@ class Authenticator(ABC):
name: str,
section: AuthSection,
config: Config,
conductor: TerminalConductor,
) -> None:
"""
Initialize an authenticator from its name and its section in the config
@ -36,7 +34,6 @@ class Authenticator(ABC):
"""
self.name = name
self.conductor = conductor
@abstractmethod
async def credentials(self) -> Tuple[str, str]:

10
PFERD/authenticators/__init__.py
View File

@ -2,7 +2,6 @@ from configparser import SectionProxy
from typing import Callable, Dict
from ..authenticator import Authenticator, AuthSection
from ..conductor import TerminalConductor
from ..config import Config
from .simple import SimpleAuthenticator, SimpleAuthSection
from .tfa import TfaAuthenticator
@ -11,12 +10,11 @@ AuthConstructor = Callable[[
str, # Name (without the "auth:" prefix)
SectionProxy, # Authenticator's section of global config
Config, # Global config
TerminalConductor, # Global conductor instance
], Authenticator]
AUTHENTICATORS: Dict[str, AuthConstructor] = {
"simple": lambda n, s, c, t:
SimpleAuthenticator(n, SimpleAuthSection(s), c, t),
"tfa": lambda n, s, c, t:
TfaAuthenticator(n, AuthSection(s), c, t),
"simple": lambda n, s, c:
SimpleAuthenticator(n, SimpleAuthSection(s), c),
"tfa": lambda n, s, c:
TfaAuthenticator(n, AuthSection(s), c),
}

7
PFERD/authenticators/simple.py
View File

@ -1,8 +1,8 @@
from typing import Optional, Tuple
from ..authenticator import Authenticator, AuthException, AuthSection
from ..conductor import TerminalConductor
from ..config import Config
from ..logging import log
from ..utils import agetpass, ainput
@ -20,9 +20,8 @@ class SimpleAuthenticator(Authenticator):
name: str,
section: SimpleAuthSection,
config: Config,
conductor: TerminalConductor,
) -> None:
super().__init__(name, section, config, conductor)
super().__init__(name, section, config)
self._username = section.username()
self._password = section.password()
@ -34,7 +33,7 @@ class SimpleAuthenticator(Authenticator):
if self._username is not None and self._password is not None:
return self._username, self._password
async with self.conductor.exclusive_output():
async with log.exclusive_output():
if self._username is None:
self._username = await ainput("Username: ")
else:

7
PFERD/authenticators/tfa.py
View File

@ -1,8 +1,8 @@
from typing import Tuple
from ..authenticator import Authenticator, AuthException, AuthSection
from ..conductor import TerminalConductor
from ..config import Config
from ..logging import log
from ..utils import ainput
@ -12,15 +12,14 @@ class TfaAuthenticator(Authenticator):
name: str,
section: AuthSection,
config: Config,
conductor: TerminalConductor,
) -> None:
super().__init__(name, section, config, conductor)
super().__init__(name, section, config)
async def username(self) -> str:
raise AuthException("TFA authenticator does not support usernames")
async def password(self) -> str:
async with self.conductor.exclusive_output():
async with log.exclusive_output():
code = await ainput("TFA code: ")
return code

95
PFERD/conductor.py
View File

@ -1,95 +0,0 @@
import asyncio
from contextlib import asynccontextmanager, contextmanager
from types import TracebackType
from typing import AsyncIterator, Iterator, List, Optional, Type
from rich.console import Console
from rich.progress import Progress, TaskID
class ProgressBar:
def __init__(self, progress: Progress, taskid: TaskID):
self._progress = progress
self._taskid = taskid
def advance(self, amount: float = 1) -> None:
self._progress.advance(self._taskid, advance=amount)
def set_total(self, total: float) -> None:
self._progress.update(self._taskid, total=total)
self._progress.start_task(self._taskid)
class TerminalConductor:
def __init__(self) -> None:
self._stopped = False
self._lock = asyncio.Lock()
self._lines: List[str] = []
self._console = Console(highlight=False)
self._progress = Progress(console=self._console)
async def _start(self) -> None:
for task in self._progress.tasks:
task.visible = True
self._progress.start()
self._stopped = False
for line in self._lines:
self.print(line)
self._lines = []
async def _stop(self) -> None:
self._stopped = True
for task in self._progress.tasks:
task.visible = False
self._progress.stop()
async def __aenter__(self) -> None:
async with self._lock:
await self._start()
async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc_value: Optional[BaseException],
traceback: Optional[TracebackType],
) -> Optional[bool]:
async with self._lock:
await self._stop()
return None
def print(self, line: str) -> None:
if self._stopped:
self._lines.append(line)
else:
self._console.print(line)
@asynccontextmanager
async def exclusive_output(self) -> AsyncIterator[None]:
async with self._lock:
await self._stop()
try:
yield
finally:
await self._start()
@contextmanager
def progress_bar(
self,
description: str,
total: Optional[float] = None,
) -> Iterator[ProgressBar]:
if total is None:
# Indeterminate progress bar
taskid = self._progress.add_task(description, start=False)
else:
taskid = self._progress.add_task(description, total=total)
bar = ProgressBar(self._progress, taskid)
try:
yield bar
finally:
self._progress.remove_task(taskid)

48
PFERD/crawler.py
View File

@ -9,9 +9,9 @@ import aiohttp
from rich.markup import escape
from .authenticator import Authenticator
from .conductor import ProgressBar, TerminalConductor
from .config import Config, Section
from .limiter import Limiter
from .logging import ProgressBar, log
from .output_dir import FileSink, OnConflict, OutputDirectory, Redownload
from .transformer import RuleParseException, Transformer
from .version import __version__
@ -36,7 +36,7 @@ def noncritical(f: Wrapped) -> Wrapped:
try:
f(self, *args, **kwargs)
except Exception as e:
self.print(f"[red]Something went wrong: {escape(str(e))}")
log.print(f"[red]Something went wrong: {escape(str(e))}")
self.error_free = False
return wrapper # type: ignore
@ -79,7 +79,7 @@ def anoncritical(f: AWrapped) -> AWrapped:
try:
await f(self, *args, **kwargs)
except Exception as e:
self.print(f"[red]Something went wrong: {escape(str(e))}")
log.print(f"[red]Something went wrong: {escape(str(e))}")
self.error_free = False
return wrapper # type: ignore
@ -182,7 +182,6 @@ class Crawler(ABC):
name: str,
section: CrawlerSection,
config: Config,
conductor: TerminalConductor,
) -> None:
"""
Initialize a crawler from its name and its section in the config file.
@ -194,7 +193,6 @@ class Crawler(ABC):
"""
self.name = name
self._conductor = conductor
self.error_free = True
self._limiter = Limiter(
@ -213,34 +211,8 @@ class Crawler(ABC):
config.working_dir / section.output_dir(name),
section.redownload(),
section.on_conflict(),
self._conductor,
)
def print(self, text: str) -> None:
"""
Print rich markup to the terminal. Crawlers *must* use this function to
print things unless they are holding an exclusive output context
manager! Be careful to escape all user-supplied strings.
"""
self._conductor.print(text)
def exclusive_output(self) -> AsyncContextManager[None]:
"""
Acquire exclusive rights™ to the terminal output. While this context
manager is held, output such as printing and progress bars from other
threads is suspended and the current thread may do whatever it wants
with the terminal. However, it must return the terminal to its original
state before exiting the context manager.
No two threads can hold this context manager at the same time.
Useful for password or confirmation prompts as well as running other
programs while crawling (e. g. to get certain credentials).
"""
return self._conductor.exclusive_output()
@asynccontextmanager
async def crawl_bar(
self,
@ -249,7 +221,7 @@ class Crawler(ABC):
) -> AsyncIterator[ProgressBar]:
desc = f"[bold bright_cyan]Crawling[/] {escape(str(path))}"
async with self._limiter.limit_crawl():
with self._conductor.progress_bar(desc, total=total) as bar:
with log.crawl_bar(desc, total=total) as bar:
yield bar
@asynccontextmanager
@ -260,7 +232,7 @@ class Crawler(ABC):
) -> AsyncIterator[ProgressBar]:
desc = f"[bold bright_cyan]Downloading[/] {escape(str(path))}"
async with self._limiter.limit_download():
with self._conductor.progress_bar(desc, total=total) as bar:
with log.download_bar(desc, total=total) as bar:
yield bar
def should_crawl(self, path: PurePath) -> bool:
@ -289,7 +261,7 @@ class Crawler(ABC):
crawler.
"""
async with self._conductor:
with log.show_progress():
await self.crawl()
@abstractmethod
@ -312,9 +284,8 @@ class HttpCrawler(Crawler):
name: str,
section: CrawlerSection,
config: Config,
conductor: TerminalConductor,
) -> None:
super().__init__(name, section, config, conductor)
super().__init__(name, section, config)
self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE)
self._output_dir.register_reserved(self.COOKIE_FILE)
@ -340,7 +311,4 @@ class HttpCrawler(Crawler):
try:
cookie_jar.save(self._cookie_jar_path)
except Exception:
self.print(
"[bold red]Warning:[/] Failed to save cookies to "
+ escape(str(self.COOKIE_FILE))
)
log.print(f"[bold red]Warning:[/] Failed to save cookies to {escape(str(self.COOKIE_FILE))}")

10
PFERD/crawlers/__init__.py
View File

@ -2,7 +2,6 @@ from configparser import SectionProxy
from typing import Callable, Dict
from ..authenticator import Authenticator
from ..conductor import TerminalConductor
from ..config import Config
from ..crawler import Crawler
from .ilias import KitIliasCrawler, KitIliasCrawlerSection
@ -12,13 +11,12 @@ CrawlerConstructor = Callable[[
str, # Name (without the "crawl:" prefix)
SectionProxy, # Crawler's section of global config
Config, # Global config
TerminalConductor, # Global conductor instance
Dict[str, Authenticator], # Loaded authenticators by name
], Crawler]
CRAWLERS: Dict[str, CrawlerConstructor] = {
"local": lambda n, s, c, t, a:
LocalCrawler(n, LocalCrawlerSection(s), c, t),
"kit-ilias": lambda n, s, c, t, a:
KitIliasCrawler(n, KitIliasCrawlerSection(s), c, t, a),
"local": lambda n, s, c, a:
LocalCrawler(n, LocalCrawlerSection(s), c),
"kit-ilias": lambda n, s, c, a:
KitIliasCrawler(n, KitIliasCrawlerSection(s), c, a),
}

6
PFERD/crawlers/ilias.py
View File

@ -16,7 +16,6 @@ from PFERD.output_dir import Redownload
from PFERD.utils import soupify
from ..authenticators import Authenticator
from ..conductor import TerminalConductor
from ..config import Config
from ..crawler import CrawlerSection, HttpCrawler, anoncritical, arepeat
@ -533,10 +532,9 @@ class KitIliasCrawler(HttpCrawler):
name: str,
section: KitIliasCrawlerSection,
config: Config,
conductor: TerminalConductor,
authenticators: Dict[str, Authenticator]
):
super().__init__(name, section, config, conductor)
super().__init__(name, section, config)
self._shibboleth_login = KitShibbolethLogin(
section.auth(authenticators),
@ -615,7 +613,7 @@ class KitIliasCrawler(HttpCrawler):
await self._download_file(element, element_path)
elif element.type == IliasElementType.FORUM:
# TODO: Delete
self.print(f"Skipping forum [green]{element_path}[/]")
print(f"Skipping forum [green]{element_path}[/]")
elif element.type == IliasElementType.LINK:
await self._download_link(element, element_path)
elif element.type == IliasElementType.VIDEO:

4
PFERD/crawlers/local.py
View File

@ -4,7 +4,6 @@ import random
from pathlib import Path, PurePath
from typing import Optional
from ..conductor import TerminalConductor
from ..config import Config
from ..crawler import Crawler, CrawlerSection, anoncritical
@ -44,9 +43,8 @@ class LocalCrawler(Crawler):
name: str,
section: LocalCrawlerSection,
config: Config,
conductor: TerminalConductor,
):
super().__init__(name, section, config, conductor)
super().__init__(name, section, config)
self._target = config.working_dir / section.target()
self._crawl_delay = section.crawl_delay()

160
PFERD/logging.py Normal file
View File

@ -0,0 +1,160 @@
import asyncio
from contextlib import asynccontextmanager, contextmanager
# TODO In Python 3.9 and above, ContextManager and AsyncContextManager are deprecated
from typing import AsyncIterator, ContextManager, Iterator, List, Optional
from rich.console import Console, RenderGroup
from rich.live import Live
from rich.markup import escape
from rich.progress import (BarColumn, DownloadColumn, Progress, TaskID, TextColumn, TimeRemainingColumn,
TransferSpeedColumn)
from rich.table import Column
class ProgressBar:
def __init__(self, progress: Progress, taskid: TaskID):
self._progress = progress
self._taskid = taskid
def advance(self, amount: float = 1) -> None:
self._progress.advance(self._taskid, advance=amount)
def set_total(self, total: float) -> None:
self._progress.update(self._taskid, total=total)
self._progress.start_task(self._taskid)
class Log:
def __init__(self) -> None:
self.console = Console(highlight=False)
self._crawl_progress = Progress(
TextColumn("{task.description}", table_column=Column(ratio=1)),
BarColumn(),
TimeRemainingColumn(),
expand=True,
)
self._download_progress = Progress(
TextColumn("{task.description}", table_column=Column(ratio=1)),
TransferSpeedColumn(),
DownloadColumn(),
BarColumn(),
TimeRemainingColumn(),
expand=True,
)
self._live = Live(console=self.console, transient=True)
self._update_live()
self._showing_progress = False
self._progress_suspended = False
self._lock = asyncio.Lock()
self._lines: List[str] = []
# Whether different parts of the output are enabled or disabled
self._enabled_explain = False
self._enabled_action = True
self._enabled_report = True
def _update_live(self) -> None:
elements = []
if self._crawl_progress.task_ids:
elements.append(self._crawl_progress)
if self._download_progress.task_ids:
elements.append(self._download_progress)
group = RenderGroup(*elements) # type: ignore
self._live.update(group)
def configure(self, explain: bool, action: bool, report: bool) -> None:
self._enabled_explain = explain
self._enabled_action = action
self._enabled_report = report
@contextmanager
def show_progress(self) -> Iterator[None]:
if self._showing_progress:
raise RuntimeError("Calling 'show_progress' while already showing progress")
self._showing_progress = True
try:
with self._live:
yield
finally:
self._showing_progress = False
@asynccontextmanager
async def exclusive_output(self) -> AsyncIterator[None]:
if not self._showing_progress:
raise RuntimeError("Calling 'exclusive_output' while not showing progress")
async with self._lock:
self._progress_suspended = True
self._live.stop()
try:
yield
finally:
self._live.start()
self._progress_suspended = False
for line in self._lines:
self.print(line)
self._lines = []
def print(self, text: str) -> None:
if self._progress_suspended:
self._lines.append(text)
else:
self.console.print(text)
def explain_topic(self, text: str) -> None:
if self._enabled_explain:
self.print(f"[cyan]{escape(text)}")
def explain(self, text: str) -> None:
if self._enabled_explain:
self.print(f" {escape(text)}")
def action(self, text: str) -> None:
if self._enabled_action:
self.print(text)
def report(self, text: str) -> None:
if self._enabled_report:
self.print(text)
@contextmanager
def _bar(
self,
progress: Progress,
description: str,
total: Optional[float],
) -> Iterator[ProgressBar]:
if total is None:
# Indeterminate progress bar
taskid = progress.add_task(description, start=False)
else:
taskid = progress.add_task(description, total=total)
self._update_live()
try:
yield ProgressBar(progress, taskid)
finally:
progress.remove_task(taskid)
self._update_live()
def crawl_bar(
self,
description: str,
total: Optional[float] = None,
) -> ContextManager[ProgressBar]:
return self._bar(self._crawl_progress, description, total)
def download_bar(
self,
description: str,
total: Optional[float] = None,
) -> ContextManager[ProgressBar]:
return self._bar(self._download_progress, description, total)
log = Log()

21
PFERD/output_dir.py
View File

@ -13,7 +13,7 @@ from typing import AsyncContextManager, AsyncIterator, BinaryIO, Iterator, Optio
from rich.markup import escape
from .conductor import TerminalConductor
from .logging import log
from .report import MarkConflictException, MarkDuplicateException, Report
from .utils import prompt_yes_no
@ -93,12 +93,10 @@ class OutputDirectory:
root: Path,
redownload: Redownload,
on_conflict: OnConflict,
conductor: TerminalConductor,
):
self._root = root
self._redownload = redownload
self._on_conflict = on_conflict
self._conductor = conductor
self._report = Report()
@ -176,7 +174,7 @@ class OutputDirectory:
path: PurePath,
) -> bool:
if on_conflict == OnConflict.PROMPT:
async with self._conductor.exclusive_output():
async with log.exclusive_output():
prompt = f"Replace {path} with remote file?"
return await prompt_yes_no(prompt, default=False)
elif on_conflict == OnConflict.LOCAL_FIRST:
@ -195,7 +193,7 @@ class OutputDirectory:
path: PurePath,
) -> bool:
if on_conflict == OnConflict.PROMPT:
async with self._conductor.exclusive_output():
async with log.exclusive_output():
prompt = f"Recursively delete {path} and replace with remote file?"
return await prompt_yes_no(prompt, default=False)
elif on_conflict == OnConflict.LOCAL_FIRST:
@ -215,7 +213,7 @@ class OutputDirectory:
parent: PurePath,
) -> bool:
if on_conflict == OnConflict.PROMPT:
async with self._conductor.exclusive_output():
async with log.exclusive_output():
prompt = f"Delete {parent} so remote file {path} can be downloaded?"
return await prompt_yes_no(prompt, default=False)
elif on_conflict == OnConflict.LOCAL_FIRST:
@ -234,7 +232,7 @@ class OutputDirectory:
path: PurePath,
) -> bool:
if on_conflict == OnConflict.PROMPT:
async with self._conductor.exclusive_output():
async with log.exclusive_output():
prompt = f"Delete {path}?"
return await prompt_yes_no(prompt, default=False)
elif on_conflict == OnConflict.LOCAL_FIRST:
@ -356,12 +354,10 @@ class OutputDirectory:
self._update_metadata(info)
if changed:
self._conductor.print(
f"[bold bright_yellow]Changed[/] {escape(str(info.path))}")
log.action(f"[bold bright_yellow]Changed[/] {escape(str(info.path))}")
self._report.change_file(info.path)
else:
self._conductor.print(
f"[bold bright_green]Added[/] {escape(str(info.path))}")
log.action(f"[bold bright_green]Added[/] {escape(str(info.path))}")
self._report.add_file(info.path)
async def cleanup(self) -> None:
@ -390,8 +386,7 @@ class OutputDirectory:
if await self._conflict_delete_lf(self._on_conflict, pure):
try:
path.unlink()
self._conductor.print(
f"[bold bright_magenta]Deleted[/] {escape(str(path))}")
log.action(f"[bold bright_magenta]Deleted[/] {escape(str(path))}")
self._report.delete_file(pure)
except OSError:
pass

17
PFERD/pferd.py
View File

@ -5,7 +5,6 @@ from rich.markup import escape
from .authenticator import Authenticator
from .authenticators import AUTHENTICATORS
from .conductor import TerminalConductor
from .config import Config
from .crawler import Crawler
from .crawlers import CRAWLERS
@ -18,7 +17,6 @@ class PferdLoadException(Exception):
class Pferd:
def __init__(self, config: Config):
self._config = config
self._conductor = TerminalConductor()
self._authenticators: Dict[str, Authenticator] = {}
self._crawlers: Dict[str, Crawler] = {}
@ -34,12 +32,7 @@ class Pferd:
print(f"[red]Error: Unknown authenticator type {t}")
continue
authenticator = authenticator_constructor(
name,
section,
self._config,
self._conductor,
)
authenticator = authenticator_constructor(name, section, self._config)
self._authenticators[name] = authenticator
if abort:
@ -57,13 +50,7 @@ class Pferd:
print(f"[red]Error: Unknown crawler type {t}")
continue
crawler = crawler_constructor(
name,
section,
self._config,
self._conductor,
self._authenticators,
)
crawler = crawler_constructor(name, section, self._config, self._authenticators)
self._crawlers[name] = crawler
if abort: