From f3f4be2690aea1c4cb663bd62d542eb759a46e43 Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Thu, 23 Apr 2020 19:18:21 +0200 Subject: [PATCH] More free functions --- PFERD/__init__.py | 1 - PFERD/download_strategies.py | 80 ------------------------------ PFERD/ilias/__init__.py | 1 + PFERD/ilias/download_strategies.py | 40 +++++++++++++++ PFERD/pferd.py | 6 +-- 5 files changed, 44 insertions(+), 84 deletions(-) delete mode 100644 PFERD/download_strategies.py create mode 100644 PFERD/ilias/download_strategies.py diff --git a/PFERD/__init__.py b/PFERD/__init__.py index 9759e22..87131ce 100644 --- a/PFERD/__init__.py +++ b/PFERD/__init__.py @@ -7,7 +7,6 @@ more complex configuration, you need to import the other submodules manually. import logging -from .download_strategies import * from .pferd import Pferd STYLE = "{" diff --git a/PFERD/download_strategies.py b/PFERD/download_strategies.py deleted file mode 100644 index 974fa61..0000000 --- a/PFERD/download_strategies.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Contains a few default strategies for limiting the amount of downloaded files. -""" - -import datetime -import logging -from abc import ABC, abstractmethod -from pathlib import Path -from typing import Optional - -from typing_extensions import Protocol, runtime_checkable - -from .organizer import Organizer -from .utils import PrettyLogger - -LOGGER = logging.getLogger(__name__) -PRETTY = PrettyLogger(LOGGER) - - -@runtime_checkable -class DownloadInfo(Protocol): - # pylint: disable=too-few-public-methods - """ - This class describes some minimal information about a file you can download. - """ - - @property - def path(self) -> Path: - """ - Returns the path. - """ - - @property - def modification_date(self) -> Optional[datetime.datetime]: - """ - Returns the modification date or None if not known. - """ - - -class DownloadStrategy(ABC): - # pylint: disable=too-few-public-methods - """ - A strategy deciding whether to download a given info. - """ - - @abstractmethod - def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool: - """ - Decides wether a given file should be downloaded. - """ - - -class DownloadEverythingStrategy(DownloadStrategy): - # pylint: disable=too-few-public-methods - """ - A strategy that redownloads everything. - """ - - def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool: - return True - - -class DownloadNewOrModified(DownloadStrategy): - # pylint: disable=too-few-public-methods - """ - A strategy that only downloads changed or new files. - """ - - def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool: - resolved_file = organizer.resolve(info.path) - if not resolved_file.exists() or info.modification_date is None: - return True - resolved_mod_time_seconds = resolved_file.stat().st_mtime - - # Download if the info is newer - if info.modification_date.timestamp() > resolved_mod_time_seconds: - return True - - PRETTY.filtered_path(info.path, "Local file had newer or equal modification time") - return False diff --git a/PFERD/ilias/__init__.py b/PFERD/ilias/__init__.py index 9bda6f8..b243978 100644 --- a/PFERD/ilias/__init__.py +++ b/PFERD/ilias/__init__.py @@ -4,4 +4,5 @@ Synchronizing files from ILIAS instances (https://www.ilias.de/). from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator from .crawler import IliasCrawler, IliasDirectoryFilter +from .download_strategies import * from .downloader import IliasDownloader diff --git a/PFERD/ilias/download_strategies.py b/PFERD/ilias/download_strategies.py new file mode 100644 index 0000000..c4420d6 --- /dev/null +++ b/PFERD/ilias/download_strategies.py @@ -0,0 +1,40 @@ +""" +Contains a few default strategies for limiting the amount of downloaded files. +""" + +import logging +from typing import Callable + +from ..organizer import Organizer +from ..utils import PrettyLogger +from .downloader import IliasDownloadInfo + +LOGGER = logging.getLogger(__name__) +PRETTY = PrettyLogger(LOGGER) + +DownloadStrategy = Callable[[Organizer, IliasDownloadInfo], bool] + + +def download_everything(organizer: Organizer, info: IliasDownloadInfo) -> bool: + # pylint: disable=unused-argument + """ + Accepts everything. + """ + return True + + +def download_modified_or_new(organizer: Organizer, info: IliasDownloadInfo) -> bool: + """ + Accepts new files or files with a more recent modification date. + """ + resolved_file = organizer.resolve(info.path) + if not resolved_file.exists() or info.modification_date is None: + return True + resolved_mod_time_seconds = resolved_file.stat().st_mtime + + # Download if the info is newer + if info.modification_date.timestamp() > resolved_mod_time_seconds: + return True + + PRETTY.filtered_path(info.path, "Local file had newer or equal modification time") + return False diff --git a/PFERD/pferd.py b/PFERD/pferd.py index de61da5..8b3d43e 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -2,9 +2,9 @@ from pathlib import Path from typing import Optional from .cookie_jar import CookieJar -from .download_strategies import DownloadEverythingStrategy, DownloadStrategy from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter, IliasDownloader, KitShibbolethAuthenticator) +from .ilias.download_strategies import DownloadStrategy, download_everything from .organizer import Organizer from .tmp_dir import TmpDir from .transform import Transform, apply_transform @@ -47,7 +47,7 @@ class Pferd(Location): downloader.download_all( [ info for info in apply_transform(transform, info) - if download_strategy.should_download(organizer, info) + if download_strategy(organizer, info) ] ) cookie_jar.save_cookies() @@ -61,7 +61,7 @@ class Pferd(Location): cookies: Optional[Path] = None, username: Optional[str] = None, password: Optional[str] = None, - download_strategy: DownloadStrategy = DownloadEverythingStrategy(), + download_strategy: DownloadStrategy = download_everything, ) -> None: # This authenticator only works with the KIT ilias instance. authenticator = KitShibbolethAuthenticator(username=username, password=password)