mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
More free functions
This commit is contained in:
parent
076b8c5a1f
commit
f3f4be2690
@ -7,7 +7,6 @@ more complex configuration, you need to import the other submodules manually.
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from .download_strategies import *
|
|
||||||
from .pferd import Pferd
|
from .pferd import Pferd
|
||||||
|
|
||||||
STYLE = "{"
|
STYLE = "{"
|
||||||
|
@ -1,80 +0,0 @@
|
|||||||
"""
|
|
||||||
Contains a few default strategies for limiting the amount of downloaded files.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import datetime
|
|
||||||
import logging
|
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from typing_extensions import Protocol, runtime_checkable
|
|
||||||
|
|
||||||
from .organizer import Organizer
|
|
||||||
from .utils import PrettyLogger
|
|
||||||
|
|
||||||
LOGGER = logging.getLogger(__name__)
|
|
||||||
PRETTY = PrettyLogger(LOGGER)
|
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
|
||||||
class DownloadInfo(Protocol):
|
|
||||||
# pylint: disable=too-few-public-methods
|
|
||||||
"""
|
|
||||||
This class describes some minimal information about a file you can download.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@property
|
|
||||||
def path(self) -> Path:
|
|
||||||
"""
|
|
||||||
Returns the path.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@property
|
|
||||||
def modification_date(self) -> Optional[datetime.datetime]:
|
|
||||||
"""
|
|
||||||
Returns the modification date or None if not known.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadStrategy(ABC):
|
|
||||||
# pylint: disable=too-few-public-methods
|
|
||||||
"""
|
|
||||||
A strategy deciding whether to download a given info.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
|
|
||||||
"""
|
|
||||||
Decides wether a given file should be downloaded.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadEverythingStrategy(DownloadStrategy):
|
|
||||||
# pylint: disable=too-few-public-methods
|
|
||||||
"""
|
|
||||||
A strategy that redownloads everything.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadNewOrModified(DownloadStrategy):
|
|
||||||
# pylint: disable=too-few-public-methods
|
|
||||||
"""
|
|
||||||
A strategy that only downloads changed or new files.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def should_download(self, organizer: Organizer, info: DownloadInfo) -> bool:
|
|
||||||
resolved_file = organizer.resolve(info.path)
|
|
||||||
if not resolved_file.exists() or info.modification_date is None:
|
|
||||||
return True
|
|
||||||
resolved_mod_time_seconds = resolved_file.stat().st_mtime
|
|
||||||
|
|
||||||
# Download if the info is newer
|
|
||||||
if info.modification_date.timestamp() > resolved_mod_time_seconds:
|
|
||||||
return True
|
|
||||||
|
|
||||||
PRETTY.filtered_path(info.path, "Local file had newer or equal modification time")
|
|
||||||
return False
|
|
@ -4,4 +4,5 @@ Synchronizing files from ILIAS instances (https://www.ilias.de/).
|
|||||||
|
|
||||||
from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator
|
from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator
|
||||||
from .crawler import IliasCrawler, IliasDirectoryFilter
|
from .crawler import IliasCrawler, IliasDirectoryFilter
|
||||||
|
from .download_strategies import *
|
||||||
from .downloader import IliasDownloader
|
from .downloader import IliasDownloader
|
||||||
|
40
PFERD/ilias/download_strategies.py
Normal file
40
PFERD/ilias/download_strategies.py
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
"""
|
||||||
|
Contains a few default strategies for limiting the amount of downloaded files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Callable
|
||||||
|
|
||||||
|
from ..organizer import Organizer
|
||||||
|
from ..utils import PrettyLogger
|
||||||
|
from .downloader import IliasDownloadInfo
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
PRETTY = PrettyLogger(LOGGER)
|
||||||
|
|
||||||
|
DownloadStrategy = Callable[[Organizer, IliasDownloadInfo], bool]
|
||||||
|
|
||||||
|
|
||||||
|
def download_everything(organizer: Organizer, info: IliasDownloadInfo) -> bool:
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
"""
|
||||||
|
Accepts everything.
|
||||||
|
"""
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def download_modified_or_new(organizer: Organizer, info: IliasDownloadInfo) -> bool:
|
||||||
|
"""
|
||||||
|
Accepts new files or files with a more recent modification date.
|
||||||
|
"""
|
||||||
|
resolved_file = organizer.resolve(info.path)
|
||||||
|
if not resolved_file.exists() or info.modification_date is None:
|
||||||
|
return True
|
||||||
|
resolved_mod_time_seconds = resolved_file.stat().st_mtime
|
||||||
|
|
||||||
|
# Download if the info is newer
|
||||||
|
if info.modification_date.timestamp() > resolved_mod_time_seconds:
|
||||||
|
return True
|
||||||
|
|
||||||
|
PRETTY.filtered_path(info.path, "Local file had newer or equal modification time")
|
||||||
|
return False
|
@ -2,9 +2,9 @@ from pathlib import Path
|
|||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from .cookie_jar import CookieJar
|
from .cookie_jar import CookieJar
|
||||||
from .download_strategies import DownloadEverythingStrategy, DownloadStrategy
|
|
||||||
from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
|
from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
|
||||||
IliasDownloader, KitShibbolethAuthenticator)
|
IliasDownloader, KitShibbolethAuthenticator)
|
||||||
|
from .ilias.download_strategies import DownloadStrategy, download_everything
|
||||||
from .organizer import Organizer
|
from .organizer import Organizer
|
||||||
from .tmp_dir import TmpDir
|
from .tmp_dir import TmpDir
|
||||||
from .transform import Transform, apply_transform
|
from .transform import Transform, apply_transform
|
||||||
@ -47,7 +47,7 @@ class Pferd(Location):
|
|||||||
downloader.download_all(
|
downloader.download_all(
|
||||||
[
|
[
|
||||||
info for info in apply_transform(transform, info)
|
info for info in apply_transform(transform, info)
|
||||||
if download_strategy.should_download(organizer, info)
|
if download_strategy(organizer, info)
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
cookie_jar.save_cookies()
|
cookie_jar.save_cookies()
|
||||||
@ -61,7 +61,7 @@ class Pferd(Location):
|
|||||||
cookies: Optional[Path] = None,
|
cookies: Optional[Path] = None,
|
||||||
username: Optional[str] = None,
|
username: Optional[str] = None,
|
||||||
password: Optional[str] = None,
|
password: Optional[str] = None,
|
||||||
download_strategy: DownloadStrategy = DownloadEverythingStrategy(),
|
download_strategy: DownloadStrategy = download_everything,
|
||||||
) -> None:
|
) -> None:
|
||||||
# This authenticator only works with the KIT ilias instance.
|
# This authenticator only works with the KIT ilias instance.
|
||||||
authenticator = KitShibbolethAuthenticator(username=username, password=password)
|
authenticator = KitShibbolethAuthenticator(username=username, password=password)
|
||||||
|
Loading…
Reference in New Issue
Block a user