mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add test run option to PFERD

Browse Source
This commit is contained in:
Joscha 2020-04-24 18:00:21 +00:00
parent 5b929f09a2
commit 8258fa8919
2 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
PFERD/pferd.py
View File

@ -4,7 +4,7 @@ Convenience functions for using PFERD.
import logging import logging
from pathlib import Path from pathlib import Path
from typing import Optional, Union from typing import List, Optional, Union
from .cookie_jar import CookieJar from .cookie_jar import CookieJar
from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter, from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
@ -13,7 +13,7 @@ from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
from .location import Location from .location import Location
from .organizer import Organizer from .organizer import Organizer
from .tmp_dir import TmpDir from .tmp_dir import TmpDir
from .transform import Transform, apply_transform from .transform import Transform, Transformable, apply_transform
from .utils import PrettyLogger from .utils import PrettyLogger
# TODO save known-good cookies as soon as possible # TODO save known-good cookies as soon as possible
@ -30,10 +30,22 @@ class Pferd(Location):
useful shortcuts for running synchronizers in a single interface. useful shortcuts for running synchronizers in a single interface.
""" """
def __init__(self, base_dir: Path, tmp_dir: Path = Path(".tmp")): def __init__(
self,
base_dir: Path,
tmp_dir: Path = Path(".tmp"),
test_run: bool = False
):
super().__init__(Path(base_dir)) super().__init__(Path(base_dir))
self._tmp_dir = TmpDir(self.resolve(tmp_dir)) self._tmp_dir = TmpDir(self.resolve(tmp_dir))
self._test_run = test_run
def _print_transformables(self, transformables: List[Transformable]) -> None:
LOGGER.info("")
LOGGER.info("Results of the test run:")
for transformable in transformables:
LOGGER.info(transformable.path)
def _ilias( def _ilias(
self, self,
@ -58,7 +70,13 @@ class Pferd(Location):
cookie_jar.load_cookies() cookie_jar.load_cookies()
info = crawler.crawl() info = crawler.crawl()
cookie_jar.save_cookies() cookie_jar.save_cookies()
downloader.download_all(apply_transform(transform, info))
transformed = apply_transform(transform, info)
if self._test_run:
self._print_transformables(transformed)
return
downloader.download_all(transformed)
cookie_jar.save_cookies() cookie_jar.save_cookies()
organizer.cleanup() organizer.cleanup()

3
PFERD/transform.py
View File

@ -59,6 +59,8 @@ def _pattern(regex: Regex) -> re.Pattern:
# Transform combinators # Transform combinators
keep = lambda path: path
def attempt(*args: Transform) -> Transform: def attempt(*args: Transform) -> Transform:
def inner(path: PurePath) -> Optional[PurePath]: def inner(path: PurePath) -> Optional[PurePath]:
for transform in args: for transform in args:
@ -134,6 +136,7 @@ def re_rename(regex: Regex, target: str) -> Transform:
return None return None
return inner return inner
# def match(regex: Union[str, re.Pattern]) -> Transform: # def match(regex: Union[str, re.Pattern]) -> Transform:
# pattern: re.Pattern # pattern: re.Pattern
# if isinstance(regex, str): # if isinstance(regex, str):