mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
More free functions
This commit is contained in:
parent
076b8c5a1f
commit
f3f4be2690
@ -7,7 +7,6 @@ more complex configuration, you need to import the other submodules manually.
|
||||
|
||||
import logging
|
||||
|
||||
from .download_strategies import *
|
||||
from .pferd import Pferd
|
||||
|
||||
STYLE = "{"
|
||||
|
@ -1,80 +0,0 @@
|
||||
"""
|
||||
Contains a few default strategies for limiting the amount of downloaded files.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from typing_extensions import Protocol, runtime_checkable
|
||||
|
||||
from .organizer import Organizer
|
||||
from .utils import PrettyLogger
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
PRETTY = PrettyLogger(LOGGER)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DownloadInfo(Protocol):
|
||||
# pylint: disable=too-few-public-methods
|
||||
"""
|
||||
This class describes some minimal information about a file you can download.
|
||||
"""
|
||||
|
||||
@property
|
||||
def path(self) -> Path:
|
||||
"""
|
||||
Returns the path.
|
||||
"""
|
||||
|
||||
@property
|
||||
def modification_date(self) -> Optional[datetime.datetime]:
|
||||
"""
|
||||
Returns the modification date or None if not known.
|
||||
"""
|
||||
|
||||
|
||||
class DownloadStrategy(ABC):
|
||||
# pylint: disable=too-few-public-methods
|
||||
"""
|
||||
A strategy deciding whether to download a given info.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
|
||||
"""
|
||||
Decides wether a given file should be downloaded.
|
||||
"""
|
||||
|
||||
|
||||
class DownloadEverythingStrategy(DownloadStrategy):
|
||||
# pylint: disable=too-few-public-methods
|
||||
"""
|
||||
A strategy that redownloads everything.
|
||||
"""
|
||||
|
||||
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class DownloadNewOrModified(DownloadStrategy):
|
||||
# pylint: disable=too-few-public-methods
|
||||
"""
|
||||
A strategy that only downloads changed or new files.
|
||||
"""
|
||||
|
||||
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
|
||||
resolved_file = organizer.resolve(info.path)
|
||||
if not resolved_file.exists() or info.modification_date is None:
|
||||
return True
|
||||
resolved_mod_time_seconds = resolved_file.stat().st_mtime
|
||||
|
||||
# Download if the info is newer
|
||||
if info.modification_date.timestamp() > resolved_mod_time_seconds:
|
||||
return True
|
||||
|
||||
PRETTY.filtered_path(info.path, "Local file had newer or equal modification time")
|
||||
return False
|
@ -4,4 +4,5 @@ Synchronizing files from ILIAS instances (https://www.ilias.de/).
|
||||
|
||||
from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator
|
||||
from .crawler import IliasCrawler, IliasDirectoryFilter
|
||||
from .download_strategies import *
|
||||
from .downloader import IliasDownloader
|
||||
|
40
PFERD/ilias/download_strategies.py
Normal file
40
PFERD/ilias/download_strategies.py
Normal file
@ -0,0 +1,40 @@
|
||||
"""
|
||||
Contains a few default strategies for limiting the amount of downloaded files.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Callable
|
||||
|
||||
from ..organizer import Organizer
|
||||
from ..utils import PrettyLogger
|
||||
from .downloader import IliasDownloadInfo
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
PRETTY = PrettyLogger(LOGGER)
|
||||
|
||||
DownloadStrategy = Callable[[Organizer, IliasDownloadInfo], bool]
|
||||
|
||||
|
||||
def download_everything(organizer: Organizer, info: IliasDownloadInfo) -> bool:
|
||||
# pylint: disable=unused-argument
|
||||
"""
|
||||
Accepts everything.
|
||||
"""
|
||||
return True
|
||||
|
||||
|
||||
def download_modified_or_new(organizer: Organizer, info: IliasDownloadInfo) -> bool:
|
||||
"""
|
||||
Accepts new files or files with a more recent modification date.
|
||||
"""
|
||||
resolved_file = organizer.resolve(info.path)
|
||||
if not resolved_file.exists() or info.modification_date is None:
|
||||
return True
|
||||
resolved_mod_time_seconds = resolved_file.stat().st_mtime
|
||||
|
||||
# Download if the info is newer
|
||||
if info.modification_date.timestamp() > resolved_mod_time_seconds:
|
||||
return True
|
||||
|
||||
PRETTY.filtered_path(info.path, "Local file had newer or equal modification time")
|
||||
return False
|
@ -2,9 +2,9 @@ from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from .cookie_jar import CookieJar
|
||||
from .download_strategies import DownloadEverythingStrategy, DownloadStrategy
|
||||
from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
|
||||
IliasDownloader, KitShibbolethAuthenticator)
|
||||
from .ilias.download_strategies import DownloadStrategy, download_everything
|
||||
from .organizer import Organizer
|
||||
from .tmp_dir import TmpDir
|
||||
from .transform import Transform, apply_transform
|
||||
@ -47,7 +47,7 @@ class Pferd(Location):
|
||||
downloader.download_all(
|
||||
[
|
||||
info for info in apply_transform(transform, info)
|
||||
if download_strategy.should_download(organizer, info)
|
||||
if download_strategy(organizer, info)
|
||||
]
|
||||
)
|
||||
cookie_jar.save_cookies()
|
||||
@ -61,7 +61,7 @@ class Pferd(Location):
|
||||
cookies: Optional[Path] = None,
|
||||
username: Optional[str] = None,
|
||||
password: Optional[str] = None,
|
||||
download_strategy: DownloadStrategy = DownloadEverythingStrategy(),
|
||||
download_strategy: DownloadStrategy = download_everything,
|
||||
) -> None:
|
||||
# This authenticator only works with the KIT ilias instance.
|
||||
authenticator = KitShibbolethAuthenticator(username=username, password=password)
|
||||
|
Loading…
Reference in New Issue
Block a user