From 2fdf24495b1655feb7a2e6a2ef349d19e3442ef3 Mon Sep 17 00:00:00 2001 From: Joscha Date: Sun, 23 May 2021 19:16:42 +0200 Subject: [PATCH] Restructure crawling and auth related modules --- PFERD/{authenticators => auth}/__init__.py | 2 +- PFERD/{ => auth}/authenticator.py | 2 +- PFERD/{authenticators => auth}/simple.py | 2 +- PFERD/{authenticators => auth}/tfa.py | 2 +- PFERD/{crawlers => crawl}/__init__.py | 6 +++--- PFERD/{ => crawl}/crawler.py | 16 ++++++++-------- PFERD/{ => crawl}/http_crawler.py | 8 ++++---- PFERD/{crawlers => crawl}/ilias/__init__.py | 0 .../{crawlers => crawl}/ilias/file_templates.py | 0 .../{crawlers => crawl}/ilias/kit_ilias_html.py | 0 .../ilias/kit_ilias_web_crawler.py | 16 +++++++--------- .../local.py => crawl/local_crawler.py} | 2 +- PFERD/pferd.py | 6 ++---- 13 files changed, 29 insertions(+), 33 deletions(-) rename PFERD/{authenticators => auth}/__init__.py (91%) rename PFERD/{ => auth}/authenticator.py (98%) rename PFERD/{authenticators => auth}/simple.py (96%) rename PFERD/{authenticators => auth}/tfa.py (93%) rename PFERD/{crawlers => crawl}/__init__.py (82%) rename PFERD/{ => crawl}/crawler.py (96%) rename PFERD/{ => crawl}/http_crawler.py (97%) rename PFERD/{crawlers => crawl}/ilias/__init__.py (100%) rename PFERD/{crawlers => crawl}/ilias/file_templates.py (100%) rename PFERD/{crawlers => crawl}/ilias/kit_ilias_html.py (100%) rename PFERD/{crawlers => crawl}/ilias/kit_ilias_web_crawler.py (98%) rename PFERD/{crawlers/local.py => crawl/local_crawler.py} (98%) diff --git a/PFERD/authenticators/__init__.py b/PFERD/auth/__init__.py similarity index 91% rename from PFERD/authenticators/__init__.py rename to PFERD/auth/__init__.py index 35096cf..6247e2b 100644 --- a/PFERD/authenticators/__init__.py +++ b/PFERD/auth/__init__.py @@ -1,8 +1,8 @@ from configparser import SectionProxy from typing import Callable, Dict -from ..authenticator import Authenticator, AuthSection from ..config import Config +from .authenticator import Authenticator, AuthSection from .simple import SimpleAuthenticator, SimpleAuthSection from .tfa import TfaAuthenticator diff --git a/PFERD/authenticator.py b/PFERD/auth/authenticator.py similarity index 98% rename from PFERD/authenticator.py rename to PFERD/auth/authenticator.py index d67b263..9217dcd 100644 --- a/PFERD/authenticator.py +++ b/PFERD/auth/authenticator.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Tuple -from .config import Config, Section +from ..config import Config, Section class AuthLoadException(Exception): diff --git a/PFERD/authenticators/simple.py b/PFERD/auth/simple.py similarity index 96% rename from PFERD/authenticators/simple.py rename to PFERD/auth/simple.py index bcbe69c..a12c359 100644 --- a/PFERD/authenticators/simple.py +++ b/PFERD/auth/simple.py @@ -1,9 +1,9 @@ from typing import Optional, Tuple -from ..authenticator import Authenticator, AuthException, AuthSection from ..config import Config from ..logging import log from ..utils import agetpass, ainput +from .authenticator import Authenticator, AuthException, AuthSection class SimpleAuthSection(AuthSection): diff --git a/PFERD/authenticators/tfa.py b/PFERD/auth/tfa.py similarity index 93% rename from PFERD/authenticators/tfa.py rename to PFERD/auth/tfa.py index b0eef18..670626d 100644 --- a/PFERD/authenticators/tfa.py +++ b/PFERD/auth/tfa.py @@ -1,9 +1,9 @@ from typing import Tuple -from ..authenticator import Authenticator, AuthException, AuthSection from ..config import Config from ..logging import log from ..utils import ainput +from .authenticator import Authenticator, AuthException, AuthSection class TfaAuthenticator(Authenticator): diff --git a/PFERD/crawlers/__init__.py b/PFERD/crawl/__init__.py similarity index 82% rename from PFERD/crawlers/__init__.py rename to PFERD/crawl/__init__.py index dc7dfa0..297c490 100644 --- a/PFERD/crawlers/__init__.py +++ b/PFERD/crawl/__init__.py @@ -1,11 +1,11 @@ from configparser import SectionProxy from typing import Callable, Dict -from ..authenticator import Authenticator +from ..auth import Authenticator from ..config import Config -from ..crawler import Crawler +from .crawler import Crawler, CrawlError # noqa: F401 from .ilias import KitIliasWebCrawler, KitIliasWebCrawlerSection -from .local import LocalCrawler, LocalCrawlerSection +from .local_crawler import LocalCrawler, LocalCrawlerSection CrawlerConstructor = Callable[[ str, # Name (without the "crawl:" prefix) diff --git a/PFERD/crawler.py b/PFERD/crawl/crawler.py similarity index 96% rename from PFERD/crawler.py rename to PFERD/crawl/crawler.py index e73ce72..a79e968 100644 --- a/PFERD/crawler.py +++ b/PFERD/crawl/crawler.py @@ -6,14 +6,14 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Tup from rich.markup import escape -from .authenticator import Authenticator -from .config import Config, Section -from .limiter import Limiter -from .logging import ProgressBar, log -from .output_dir import FileSink, FileSinkToken, OnConflict, OutputDirectory, OutputDirError, Redownload -from .report import MarkConflictError, MarkDuplicateError -from .transformer import Transformer -from .utils import ReusableAsyncContextManager, fmt_path +from ..auth import Authenticator +from ..config import Config, Section +from ..limiter import Limiter +from ..logging import ProgressBar, log +from ..output_dir import FileSink, FileSinkToken, OnConflict, OutputDirectory, OutputDirError, Redownload +from ..report import MarkConflictError, MarkDuplicateError +from ..transformer import Transformer +from ..utils import ReusableAsyncContextManager, fmt_path class CrawlWarning(Exception): diff --git a/PFERD/http_crawler.py b/PFERD/crawl/http_crawler.py similarity index 97% rename from PFERD/http_crawler.py rename to PFERD/crawl/http_crawler.py index c6e679d..e82dfed 100644 --- a/PFERD/http_crawler.py +++ b/PFERD/crawl/http_crawler.py @@ -4,11 +4,11 @@ from typing import Optional import aiohttp -from .config import Config +from ..config import Config +from ..logging import log +from ..utils import fmt_real_path +from ..version import NAME, VERSION from .crawler import Crawler, CrawlerSection -from .logging import log -from .utils import fmt_real_path -from .version import NAME, VERSION class HttpCrawler(Crawler): diff --git a/PFERD/crawlers/ilias/__init__.py b/PFERD/crawl/ilias/__init__.py similarity index 100% rename from PFERD/crawlers/ilias/__init__.py rename to PFERD/crawl/ilias/__init__.py diff --git a/PFERD/crawlers/ilias/file_templates.py b/PFERD/crawl/ilias/file_templates.py similarity index 100% rename from PFERD/crawlers/ilias/file_templates.py rename to PFERD/crawl/ilias/file_templates.py diff --git a/PFERD/crawlers/ilias/kit_ilias_html.py b/PFERD/crawl/ilias/kit_ilias_html.py similarity index 100% rename from PFERD/crawlers/ilias/kit_ilias_html.py rename to PFERD/crawl/ilias/kit_ilias_html.py diff --git a/PFERD/crawlers/ilias/kit_ilias_web_crawler.py b/PFERD/crawl/ilias/kit_ilias_web_crawler.py similarity index 98% rename from PFERD/crawlers/ilias/kit_ilias_web_crawler.py rename to PFERD/crawl/ilias/kit_ilias_web_crawler.py index 7e1562c..33356ed 100644 --- a/PFERD/crawlers/ilias/kit_ilias_web_crawler.py +++ b/PFERD/crawl/ilias/kit_ilias_web_crawler.py @@ -6,15 +6,13 @@ import aiohttp from aiohttp import hdrs from bs4 import BeautifulSoup, Tag -from PFERD.authenticators import Authenticator -from PFERD.config import Config -from PFERD.crawler import CrawlError, CrawlerSection, CrawlWarning, anoncritical -from PFERD.http_crawler import HttpCrawler -from PFERD.logging import ProgressBar, log -from PFERD.output_dir import FileSink, Redownload -from PFERD.utils import soupify, url_set_query_param - -from ...utils import fmt_path +from ...auth import Authenticator +from ...config import Config +from ...logging import ProgressBar, log +from ...output_dir import FileSink, Redownload +from ...utils import fmt_path, soupify, url_set_query_param +from ..crawler import CrawlError, CrawlerSection, CrawlWarning, anoncritical +from ..http_crawler import HttpCrawler from .file_templates import link_template_plain, link_template_rich from .kit_ilias_html import IliasElementType, IliasPage, IliasPageElement diff --git a/PFERD/crawlers/local.py b/PFERD/crawl/local_crawler.py similarity index 98% rename from PFERD/crawlers/local.py rename to PFERD/crawl/local_crawler.py index 35e5829..7958169 100644 --- a/PFERD/crawlers/local.py +++ b/PFERD/crawl/local_crawler.py @@ -5,7 +5,7 @@ from pathlib import Path, PurePath from typing import Optional from ..config import Config -from ..crawler import Crawler, CrawlerSection, anoncritical +from .crawler import Crawler, CrawlerSection, anoncritical class LocalCrawlerSection(CrawlerSection): diff --git a/PFERD/pferd.py b/PFERD/pferd.py index 75b0e9d..5b5b866 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -2,11 +2,9 @@ from typing import Dict, List, Optional from rich.markup import escape -from .authenticator import Authenticator -from .authenticators import AUTHENTICATORS +from .auth import AUTHENTICATORS, Authenticator from .config import Config, ConfigOptionError -from .crawler import Crawler, CrawlError -from .crawlers import CRAWLERS +from .crawl import CRAWLERS, Crawler, CrawlError from .logging import log