Move logging logic to singleton

- Renamed module and class because "conductor" didn't make a lot of sense
- Used singleton approach (there's only one stdout after all)
- Redesigned progress bars (now with download speed!)
This commit is contained in:
Joscha
2021-05-18 22:43:46 +02:00
parent 1525aa15a6
commit 4b68fa771f
12 changed files with 195 additions and 193 deletions

View File

@ -2,7 +2,6 @@ from configparser import SectionProxy
from typing import Callable, Dict
from ..authenticator import Authenticator
from ..conductor import TerminalConductor
from ..config import Config
from ..crawler import Crawler
from .ilias import KitIliasCrawler, KitIliasCrawlerSection
@ -12,13 +11,12 @@ CrawlerConstructor = Callable[[
str, # Name (without the "crawl:" prefix)
SectionProxy, # Crawler's section of global config
Config, # Global config
TerminalConductor, # Global conductor instance
Dict[str, Authenticator], # Loaded authenticators by name
], Crawler]
CRAWLERS: Dict[str, CrawlerConstructor] = {
"local": lambda n, s, c, t, a:
LocalCrawler(n, LocalCrawlerSection(s), c, t),
"kit-ilias": lambda n, s, c, t, a:
KitIliasCrawler(n, KitIliasCrawlerSection(s), c, t, a),
"local": lambda n, s, c, a:
LocalCrawler(n, LocalCrawlerSection(s), c),
"kit-ilias": lambda n, s, c, a:
KitIliasCrawler(n, KitIliasCrawlerSection(s), c, a),
}

View File

@ -16,7 +16,6 @@ from PFERD.output_dir import Redownload
from PFERD.utils import soupify
from ..authenticators import Authenticator
from ..conductor import TerminalConductor
from ..config import Config
from ..crawler import CrawlerSection, HttpCrawler, anoncritical, arepeat
@ -533,10 +532,9 @@ class KitIliasCrawler(HttpCrawler):
name: str,
section: KitIliasCrawlerSection,
config: Config,
conductor: TerminalConductor,
authenticators: Dict[str, Authenticator]
):
super().__init__(name, section, config, conductor)
super().__init__(name, section, config)
self._shibboleth_login = KitShibbolethLogin(
section.auth(authenticators),
@ -615,7 +613,7 @@ class KitIliasCrawler(HttpCrawler):
await self._download_file(element, element_path)
elif element.type == IliasElementType.FORUM:
# TODO: Delete
self.print(f"Skipping forum [green]{element_path}[/]")
print(f"Skipping forum [green]{element_path}[/]")
elif element.type == IliasElementType.LINK:
await self._download_link(element, element_path)
elif element.type == IliasElementType.VIDEO:

View File

@ -4,7 +4,6 @@ import random
from pathlib import Path, PurePath
from typing import Optional
from ..conductor import TerminalConductor
from ..config import Config
from ..crawler import Crawler, CrawlerSection, anoncritical
@ -44,9 +43,8 @@ class LocalCrawler(Crawler):
name: str,
section: LocalCrawlerSection,
config: Config,
conductor: TerminalConductor,
):
super().__init__(name, section, config, conductor)
super().__init__(name, section, config)
self._target = config.working_dir / section.target()
self._crawl_delay = section.crawl_delay()