From 8258fa8919d171b068d9c486621e00d3e01d39d4 Mon Sep 17 00:00:00 2001 From: Joscha Date: Fri, 24 Apr 2020 18:00:21 +0000 Subject: [PATCH] Add test run option to PFERD --- PFERD/pferd.py | 26 ++++++++++++++++++++++---- PFERD/transform.py | 3 +++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/PFERD/pferd.py b/PFERD/pferd.py index 4fee3a2..259e961 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -4,7 +4,7 @@ Convenience functions for using PFERD. import logging from pathlib import Path -from typing import Optional, Union +from typing import List, Optional, Union from .cookie_jar import CookieJar from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter, @@ -13,7 +13,7 @@ from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter, from .location import Location from .organizer import Organizer from .tmp_dir import TmpDir -from .transform import Transform, apply_transform +from .transform import Transform, Transformable, apply_transform from .utils import PrettyLogger # TODO save known-good cookies as soon as possible @@ -30,10 +30,22 @@ class Pferd(Location): useful shortcuts for running synchronizers in a single interface. """ - def __init__(self, base_dir: Path, tmp_dir: Path = Path(".tmp")): + def __init__( + self, + base_dir: Path, + tmp_dir: Path = Path(".tmp"), + test_run: bool = False + ): super().__init__(Path(base_dir)) self._tmp_dir = TmpDir(self.resolve(tmp_dir)) + self._test_run = test_run + + def _print_transformables(self, transformables: List[Transformable]) -> None: + LOGGER.info("") + LOGGER.info("Results of the test run:") + for transformable in transformables: + LOGGER.info(transformable.path) def _ilias( self, @@ -58,7 +70,13 @@ class Pferd(Location): cookie_jar.load_cookies() info = crawler.crawl() cookie_jar.save_cookies() - downloader.download_all(apply_transform(transform, info)) + + transformed = apply_transform(transform, info) + if self._test_run: + self._print_transformables(transformed) + return + + downloader.download_all(transformed) cookie_jar.save_cookies() organizer.cleanup() diff --git a/PFERD/transform.py b/PFERD/transform.py index bce7ee5..fcb4bde 100644 --- a/PFERD/transform.py +++ b/PFERD/transform.py @@ -59,6 +59,8 @@ def _pattern(regex: Regex) -> re.Pattern: # Transform combinators +keep = lambda path: path + def attempt(*args: Transform) -> Transform: def inner(path: PurePath) -> Optional[PurePath]: for transform in args: @@ -134,6 +136,7 @@ def re_rename(regex: Regex, target: str) -> Transform: return None return inner + # def match(regex: Union[str, re.Pattern]) -> Transform: # pattern: re.Pattern # if isinstance(regex, str):