Compare commits

...

12 Commits

Author SHA1 Message Date
I-Al-Istannen
ecaedea709 Merge pull request #8 from pavelzw/master
Fix version number
2020-06-26 17:52:05 +02:00
Pavel Zwerschke
f05d1b1261 Fix version number 2020-06-26 17:49:47 +02:00
I-Al-Istannen
6aaa3071f9 Update README with new version 2020-06-26 17:35:03 +02:00
I-Al-Istannen
c26c9352f1 Make DownloadSummary private, provide property accessors 2020-06-26 17:30:45 +02:00
I-Al-Istannen
d9ea688145 Use pretty logger for summaries 2020-06-26 17:24:36 +02:00
I-Al-Istannen
e8be6e498e Add summary to example_config_personal_desktop 2020-06-26 17:24:36 +02:00
I-Al-Istannen
e4b1fac045 Satisfy pylint 2020-06-26 15:38:22 +02:00
Joscha
402ae81335 Fix type hints 2020-06-26 13:17:44 +00:00
Daniel Augustin
52f31e2783 Add type hints to DownloadSummary 2020-06-26 13:02:37 +02:00
Daniel Augustin
739522a151 Move download summary into a separate class 2020-06-25 23:07:11 +02:00
Daniel Augustin
6c034209b6 Add deleted files to summary 2020-06-25 22:00:28 +02:00
Daniel Augustin
f6fbd5e4bb Add download summary 2020-06-25 19:19:34 +02:00
9 changed files with 133 additions and 8 deletions

3
.gitignore vendored
View File

@@ -1,5 +1,8 @@
__pycache__/ __pycache__/
.venv/ .venv/
venv/
.idea/
build/
.mypy_cache/ .mypy_cache/
.tmp/ .tmp/
.env .env

69
PFERD/download_summary.py Normal file
View File

@@ -0,0 +1,69 @@
"""
Provides a summary that keeps track of new modified or deleted files.
"""
from pathlib import Path
from typing import List
class DownloadSummary:
"""
Keeps track of all new, modified or deleted files and provides a summary.
"""
def __init__(self) -> None:
self._new_files: List[Path] = []
self._modified_files: List[Path] = []
self._deleted_files: List[Path] = []
@property
def new_files(self) -> List[Path]:
"""
Returns all new files.
"""
return self._new_files.copy()
@property
def modified_files(self) -> List[Path]:
"""
Returns all modified files.
"""
return self._modified_files.copy()
@property
def deleted_files(self) -> List[Path]:
"""
Returns all deleted files.
"""
return self._deleted_files.copy()
def merge(self, summary: 'DownloadSummary') -> None:
"""
Merges ourselves with the passed summary. Modifies this object, but not the passed one.
"""
self._new_files += summary.new_files
self._modified_files += summary.modified_files
self._deleted_files += summary.deleted_files
def add_deleted_file(self, path: Path) -> None:
"""
Registers a file as deleted.
"""
self._deleted_files.append(path)
def add_modified_file(self, path: Path) -> None:
"""
Registers a file as changed.
"""
self._modified_files.append(path)
def add_new_file(self, path: Path) -> None:
"""
Registers a file as new.
"""
self._new_files.append(path)
def has_updates(self) -> bool:
"""
Returns whether this summary has any updates.
"""
return bool(self._new_files or self._modified_files or self._deleted_files)

View File

@@ -3,14 +3,18 @@ Contains a few logger utility functions and implementations.
""" """
import logging import logging
from typing import Optional from pathlib import Path
from typing import List, Optional
from rich import print as rich_print
from rich._log_render import LogRender from rich._log_render import LogRender
from rich.console import Console from rich.console import Console
from rich.panel import Panel
from rich.style import Style from rich.style import Style
from rich.text import Text from rich.text import Text
from rich.theme import Theme from rich.theme import Theme
from .download_summary import DownloadSummary
from .utils import PathLike, to_path from .utils import PathLike, to_path
STYLE = "{" STYLE = "{"
@@ -111,6 +115,15 @@ class PrettyLogger:
f"[bold green]Created {self._format_path(path)}.[/bold green]" f"[bold green]Created {self._format_path(path)}.[/bold green]"
) )
def deleted_file(self, path: PathLike) -> None:
"""
A file has been deleted.
"""
self.logger.info(
f"[bold red]Deleted {self._format_path(path)}.[/bold red]"
)
def ignored_file(self, path: PathLike, reason: str) -> None: def ignored_file(self, path: PathLike, reason: str) -> None:
""" """
File was not downloaded or modified. File was not downloaded or modified.
@@ -138,6 +151,23 @@ class PrettyLogger:
f"([/dim]{reason}[dim]).[/dim]" f"([/dim]{reason}[dim]).[/dim]"
) )
def summary(self, download_summary: DownloadSummary) -> None:
"""
Prints a download summary.
"""
self.logger.info("")
self.logger.info("[bold cyan]Download Summary[/bold cyan]")
if not download_summary.has_updates():
self.logger.info("[bold dim]Nothing changed![/bold dim]")
return
for new_file in download_summary.new_files:
self.new_file(new_file)
for modified_file in download_summary.modified_files:
self.modified_file(modified_file)
for deleted_files in download_summary.deleted_files:
self.deleted_file(deleted_files)
def starting_synchronizer( def starting_synchronizer(
self, self,
target_directory: PathLike, target_directory: PathLike,

View File

@@ -9,6 +9,7 @@ import shutil
from pathlib import Path, PurePath from pathlib import Path, PurePath
from typing import List, Set from typing import List, Set
from .download_summary import DownloadSummary
from .location import Location from .location import Location
from .logging import PrettyLogger from .logging import PrettyLogger
from .utils import prompt_yes_no from .utils import prompt_yes_no
@@ -32,6 +33,8 @@ class Organizer(Location):
# Keep the root dir # Keep the root dir
self._known_files.add(path.resolve()) self._known_files.add(path.resolve())
self.download_summary = DownloadSummary()
def accept_file(self, src: Path, dst: PurePath) -> None: def accept_file(self, src: Path, dst: PurePath) -> None:
"""Move a file to this organizer and mark it.""" """Move a file to this organizer and mark it."""
src_absolute = src.resolve() src_absolute = src.resolve()
@@ -69,8 +72,10 @@ class Organizer(Location):
dst_absolute.touch() dst_absolute.touch()
return return
self.download_summary.add_modified_file(dst_absolute)
PRETTY.modified_file(dst_absolute) PRETTY.modified_file(dst_absolute)
else: else:
self.download_summary.add_new_file(dst_absolute)
PRETTY.new_file(dst_absolute) PRETTY.new_file(dst_absolute)
# Create parent dir if needed # Create parent dir if needed
@@ -117,9 +122,9 @@ class Organizer(Location):
if start_dir.resolve() not in self._known_files and dir_empty: if start_dir.resolve() not in self._known_files and dir_empty:
start_dir.rmdir() start_dir.rmdir()
@staticmethod def _delete_file_if_confirmed(self, path: Path) -> None:
def _delete_file_if_confirmed(path: Path) -> None:
prompt = f"Do you want to delete {path}" prompt = f"Do you want to delete {path}"
if prompt_yes_no(prompt, False): if prompt_yes_no(prompt, False):
self.download_summary.add_deleted_file(path)
path.unlink() path.unlink()

View File

@@ -9,6 +9,7 @@ from typing import Callable, List, Optional, Union
from .cookie_jar import CookieJar from .cookie_jar import CookieJar
from .diva import (DivaDownloader, DivaDownloadStrategy, DivaPlaylistCrawler, from .diva import (DivaDownloader, DivaDownloadStrategy, DivaPlaylistCrawler,
diva_download_new) diva_download_new)
from .download_summary import DownloadSummary
from .errors import FatalException, swallow_and_print_errors from .errors import FatalException, swallow_and_print_errors
from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter, from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
IliasDownloader, IliasDownloadInfo, IliasDownloadStrategy, IliasDownloader, IliasDownloadInfo, IliasDownloadStrategy,
@@ -42,6 +43,7 @@ class Pferd(Location):
): ):
super().__init__(Path(base_dir)) super().__init__(Path(base_dir))
self._download_summary = DownloadSummary()
self._tmp_dir = TmpDir(self.resolve(tmp_dir)) self._tmp_dir = TmpDir(self.resolve(tmp_dir))
self._test_run = test_run self._test_run = test_run
@@ -139,7 +141,8 @@ class Pferd(Location):
# This authenticator only works with the KIT ilias instance. # This authenticator only works with the KIT ilias instance.
authenticator = KitShibbolethAuthenticator(username=username, password=password) authenticator = KitShibbolethAuthenticator(username=username, password=password)
PRETTY.starting_synchronizer(target, "ILIAS", course_id) PRETTY.starting_synchronizer(target, "ILIAS", course_id)
return self._ilias(
organizer = self._ilias(
target=target, target=target,
base_url="https://ilias.studium.kit.edu/", base_url="https://ilias.studium.kit.edu/",
crawl_function=lambda crawler: crawler.crawl_course(course_id), crawl_function=lambda crawler: crawler.crawl_course(course_id),
@@ -151,6 +154,16 @@ class Pferd(Location):
clean=clean, clean=clean,
) )
self._download_summary.merge(organizer.download_summary)
return organizer
def print_summary(self) -> None:
"""
Prints the accumulated download summary.
"""
PRETTY.summary(self._download_summary)
@swallow_and_print_errors @swallow_and_print_errors
def ilias_kit_personal_desktop( def ilias_kit_personal_desktop(
self, self,

View File

@@ -9,7 +9,7 @@ Ensure that you have at least Python 3.8 installed.
To install PFERD or update your installation to the latest version, run this To install PFERD or update your installation to the latest version, run this
wherever you want to install/have installed PFERD: wherever you want to install/have installed PFERD:
``` ```
$ pip install git+https://github.com/Garmelon/PFERD@v2.1.1 $ pip install git+https://github.com/Garmelon/PFERD@v2.1.2
``` ```
The use of [venv](https://docs.python.org/3/library/venv.html) is recommended. The use of [venv](https://docs.python.org/3/library/venv.html) is recommended.
@@ -37,8 +37,8 @@ $ mkdir Vorlesungen
$ cd Vorlesungen $ cd Vorlesungen
$ python3 -m venv .venv $ python3 -m venv .venv
$ .venv/bin/activate $ .venv/bin/activate
$ pip install git+https://github.com/Garmelon/PFERD@v2.1.1 $ pip install git+https://github.com/Garmelon/PFERD@v2.1.2
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.1.1/example_config.py $ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.1.2/example_config.py
$ python3 example_config.py $ python3 example_config.py
$ deactivate $ deactivate
``` ```

View File

@@ -124,6 +124,8 @@ def main() -> None:
cookies="ilias_cookies.txt", cookies="ilias_cookies.txt",
) )
# Prints a summary listing all new, modified or deleted files
pferd.print_summary()
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@@ -30,6 +30,9 @@ def main() -> None:
cookies="ilias_cookies.txt", cookies="ilias_cookies.txt",
) )
# Prints a summary listing all new, modified or deleted files
pferd.print_summary()
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
setup( setup(
name="PFERD", name="PFERD",
version="2.1.1", version="2.1.2",
packages=find_packages(), packages=find_packages(),
install_requires=[ install_requires=[
"requests>=2.21.0", "requests>=2.21.0",