mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Move logging logic to singleton
- Renamed module and class because "conductor" didn't make a lot of sense - Used singleton approach (there's only one stdout after all) - Redesigned progress bars (now with download speed!)
This commit is contained in:
@ -2,7 +2,6 @@ from configparser import SectionProxy
|
||||
from typing import Callable, Dict
|
||||
|
||||
from ..authenticator import Authenticator
|
||||
from ..conductor import TerminalConductor
|
||||
from ..config import Config
|
||||
from ..crawler import Crawler
|
||||
from .ilias import KitIliasCrawler, KitIliasCrawlerSection
|
||||
@ -12,13 +11,12 @@ CrawlerConstructor = Callable[[
|
||||
str, # Name (without the "crawl:" prefix)
|
||||
SectionProxy, # Crawler's section of global config
|
||||
Config, # Global config
|
||||
TerminalConductor, # Global conductor instance
|
||||
Dict[str, Authenticator], # Loaded authenticators by name
|
||||
], Crawler]
|
||||
|
||||
CRAWLERS: Dict[str, CrawlerConstructor] = {
|
||||
"local": lambda n, s, c, t, a:
|
||||
LocalCrawler(n, LocalCrawlerSection(s), c, t),
|
||||
"kit-ilias": lambda n, s, c, t, a:
|
||||
KitIliasCrawler(n, KitIliasCrawlerSection(s), c, t, a),
|
||||
"local": lambda n, s, c, a:
|
||||
LocalCrawler(n, LocalCrawlerSection(s), c),
|
||||
"kit-ilias": lambda n, s, c, a:
|
||||
KitIliasCrawler(n, KitIliasCrawlerSection(s), c, a),
|
||||
}
|
||||
|
@ -16,7 +16,6 @@ from PFERD.output_dir import Redownload
|
||||
from PFERD.utils import soupify
|
||||
|
||||
from ..authenticators import Authenticator
|
||||
from ..conductor import TerminalConductor
|
||||
from ..config import Config
|
||||
from ..crawler import CrawlerSection, HttpCrawler, anoncritical, arepeat
|
||||
|
||||
@ -533,10 +532,9 @@ class KitIliasCrawler(HttpCrawler):
|
||||
name: str,
|
||||
section: KitIliasCrawlerSection,
|
||||
config: Config,
|
||||
conductor: TerminalConductor,
|
||||
authenticators: Dict[str, Authenticator]
|
||||
):
|
||||
super().__init__(name, section, config, conductor)
|
||||
super().__init__(name, section, config)
|
||||
|
||||
self._shibboleth_login = KitShibbolethLogin(
|
||||
section.auth(authenticators),
|
||||
@ -615,7 +613,7 @@ class KitIliasCrawler(HttpCrawler):
|
||||
await self._download_file(element, element_path)
|
||||
elif element.type == IliasElementType.FORUM:
|
||||
# TODO: Delete
|
||||
self.print(f"Skipping forum [green]{element_path}[/]")
|
||||
print(f"Skipping forum [green]{element_path}[/]")
|
||||
elif element.type == IliasElementType.LINK:
|
||||
await self._download_link(element, element_path)
|
||||
elif element.type == IliasElementType.VIDEO:
|
||||
|
@ -4,7 +4,6 @@ import random
|
||||
from pathlib import Path, PurePath
|
||||
from typing import Optional
|
||||
|
||||
from ..conductor import TerminalConductor
|
||||
from ..config import Config
|
||||
from ..crawler import Crawler, CrawlerSection, anoncritical
|
||||
|
||||
@ -44,9 +43,8 @@ class LocalCrawler(Crawler):
|
||||
name: str,
|
||||
section: LocalCrawlerSection,
|
||||
config: Config,
|
||||
conductor: TerminalConductor,
|
||||
):
|
||||
super().__init__(name, section, config, conductor)
|
||||
super().__init__(name, section, config)
|
||||
|
||||
self._target = config.working_dir / section.target()
|
||||
self._crawl_delay = section.crawl_delay()
|
||||
|
Reference in New Issue
Block a user