More free functions

This commit is contained in:
I-Al-Istannen 2020-04-23 19:18:21 +02:00
parent 076b8c5a1f
commit f3f4be2690
5 changed files with 44 additions and 84 deletions

View File

@ -7,7 +7,6 @@ more complex configuration, you need to import the other submodules manually.
import logging import logging
from .download_strategies import *
from .pferd import Pferd from .pferd import Pferd
STYLE = "{" STYLE = "{"

View File

@ -1,80 +0,0 @@
"""
Contains a few default strategies for limiting the amount of downloaded files.
"""
import datetime
import logging
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Optional
from typing_extensions import Protocol, runtime_checkable
from .organizer import Organizer
from .utils import PrettyLogger
LOGGER = logging.getLogger(__name__)
PRETTY = PrettyLogger(LOGGER)
@runtime_checkable
class DownloadInfo(Protocol):
# pylint: disable=too-few-public-methods
"""
This class describes some minimal information about a file you can download.
"""
@property
def path(self) -> Path:
"""
Returns the path.
"""
@property
def modification_date(self) -> Optional[datetime.datetime]:
"""
Returns the modification date or None if not known.
"""
class DownloadStrategy(ABC):
# pylint: disable=too-few-public-methods
"""
A strategy deciding whether to download a given info.
"""
@abstractmethod
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
"""
Decides wether a given file should be downloaded.
"""
class DownloadEverythingStrategy(DownloadStrategy):
# pylint: disable=too-few-public-methods
"""
A strategy that redownloads everything.
"""
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
return True
class DownloadNewOrModified(DownloadStrategy):
# pylint: disable=too-few-public-methods
"""
A strategy that only downloads changed or new files.
"""
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
resolved_file = organizer.resolve(info.path)
if not resolved_file.exists() or info.modification_date is None:
return True
resolved_mod_time_seconds = resolved_file.stat().st_mtime
# Download if the info is newer
if info.modification_date.timestamp() > resolved_mod_time_seconds:
return True
PRETTY.filtered_path(info.path, "Local file had newer or equal modification time")
return False

View File

@ -4,4 +4,5 @@ Synchronizing files from ILIAS instances (https://www.ilias.de/).
from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator
from .crawler import IliasCrawler, IliasDirectoryFilter from .crawler import IliasCrawler, IliasDirectoryFilter
from .download_strategies import *
from .downloader import IliasDownloader from .downloader import IliasDownloader

View File

@ -0,0 +1,40 @@
"""
Contains a few default strategies for limiting the amount of downloaded files.
"""
import logging
from typing import Callable
from ..organizer import Organizer
from ..utils import PrettyLogger
from .downloader import IliasDownloadInfo
LOGGER = logging.getLogger(__name__)
PRETTY = PrettyLogger(LOGGER)
DownloadStrategy = Callable[[Organizer, IliasDownloadInfo], bool]
def download_everything(organizer: Organizer, info: IliasDownloadInfo) -> bool:
# pylint: disable=unused-argument
"""
Accepts everything.
"""
return True
def download_modified_or_new(organizer: Organizer, info: IliasDownloadInfo) -> bool:
"""
Accepts new files or files with a more recent modification date.
"""
resolved_file = organizer.resolve(info.path)
if not resolved_file.exists() or info.modification_date is None:
return True
resolved_mod_time_seconds = resolved_file.stat().st_mtime
# Download if the info is newer
if info.modification_date.timestamp() > resolved_mod_time_seconds:
return True
PRETTY.filtered_path(info.path, "Local file had newer or equal modification time")
return False

View File

@ -2,9 +2,9 @@ from pathlib import Path
from typing import Optional from typing import Optional
from .cookie_jar import CookieJar from .cookie_jar import CookieJar
from .download_strategies import DownloadEverythingStrategy, DownloadStrategy
from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter, from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
IliasDownloader, KitShibbolethAuthenticator) IliasDownloader, KitShibbolethAuthenticator)
from .ilias.download_strategies import DownloadStrategy, download_everything
from .organizer import Organizer from .organizer import Organizer
from .tmp_dir import TmpDir from .tmp_dir import TmpDir
from .transform import Transform, apply_transform from .transform import Transform, apply_transform
@ -47,7 +47,7 @@ class Pferd(Location):
downloader.download_all( downloader.download_all(
[ [
info for info in apply_transform(transform, info) info for info in apply_transform(transform, info)
if download_strategy.should_download(organizer, info) if download_strategy(organizer, info)
] ]
) )
cookie_jar.save_cookies() cookie_jar.save_cookies()
@ -61,7 +61,7 @@ class Pferd(Location):
cookies: Optional[Path] = None, cookies: Optional[Path] = None,
username: Optional[str] = None, username: Optional[str] = None,
password: Optional[str] = None, password: Optional[str] = None,
download_strategy: DownloadStrategy = DownloadEverythingStrategy(), download_strategy: DownloadStrategy = download_everything,
) -> None: ) -> None:
# This authenticator only works with the KIT ilias instance. # This authenticator only works with the KIT ilias instance.
authenticator = KitShibbolethAuthenticator(username=username, password=password) authenticator = KitShibbolethAuthenticator(username=username, password=password)