From 3c808879c9b49146da9fdc7803259423c78593c0 Mon Sep 17 00:00:00 2001 From: Joscha Date: Wed, 22 Apr 2020 18:25:09 +0000 Subject: [PATCH] Add Transforms and Transformables --- PFERD/downloaders.py | 4 ++-- PFERD/ilias/downloader.py | 4 ++-- PFERD/transform.py | 42 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 PFERD/transform.py diff --git a/PFERD/downloaders.py b/PFERD/downloaders.py index 0de6402..5d193f8 100644 --- a/PFERD/downloaders.py +++ b/PFERD/downloaders.py @@ -11,16 +11,16 @@ import requests.auth from .organizer import Organizer from .tmp_dir import TmpDir +from .transform import Transformable from .utils import stream_to_path @dataclass -class HttpDownloadInfo: +class HttpDownloadInfo(Transformable): """ This class describes a single file to be downloaded. """ - path: Path url: str parameters: Dict[str, Any] = field(default_factory=dict) diff --git a/PFERD/ilias/downloader.py b/PFERD/ilias/downloader.py index e6bbf97..66243d1 100644 --- a/PFERD/ilias/downloader.py +++ b/PFERD/ilias/downloader.py @@ -10,6 +10,7 @@ import requests from ..organizer import Organizer from ..tmp_dir import TmpDir +from ..transform import Transformable from ..utils import soupify, stream_to_path from .authenticators import IliasAuthenticator @@ -19,12 +20,11 @@ class ContentTypeException(Exception): @dataclass -class IliasDownloadInfo: +class IliasDownloadInfo(Transformable): """ This class describes a single file to be downloaded. """ - path: Path url: str modification_date: datetime.datetime # parameters: Dict[str, Any] = field(default_factory=dict) diff --git a/PFERD/transform.py b/PFERD/transform.py new file mode 100644 index 0000000..5427464 --- /dev/null +++ b/PFERD/transform.py @@ -0,0 +1,42 @@ +""" +Transforms let the user define functions to decide where the downloaded files +should be placed locally. They let the user do more advanced things like moving +only files whose names match a regex, or renaming files from one numbering +scheme to another. +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, List, Optional, TypeVar + + +Transform = Callable[[Path], Optional[Path]] + + +@dataclass +class Transformable: + """ + An object that can be transformed by a Transform. + """ + + path: Path + + +TF = TypeVar("TF", bound=Transformable) + + +def apply_transform( + transform: Callable[[Path], Optional[Path]], + transformables: List[TF], +) -> List[TF]: + """ + Apply a Transform to multiple Transformables, discarding those that were + not transformed by the Transform. + """ + + result: List[TF] = [] + for transformable in transformables: + if new_path := transform(transformable.path): + transformable.path = new_path + result.append(transformable) + return result