mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Change crawler and downloader output

Browse Source
This commit is contained in:
Joscha 2020-04-24 18:24:44 +00:00
parent 8258fa8919
commit 292e516297
4 changed files with 32 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
PFERD/ilias/crawler.py
View File

@ -162,10 +162,10 @@ class IliasCrawler:
element_path = Path(parent_path, link_element.getText().strip()) element_path = Path(parent_path, link_element.getText().strip())
if not self.dir_filter(element_path): if not self.dir_filter(element_path):
PRETTY.filtered_path(element_path, "user filter") PRETTY.not_searching(element_path, "user filter")
return [] return []
LOGGER.info("Searching %r", str(element_path)) PRETTY.searching(element_path)
found_parent: Optional[bs4.Tag] = None found_parent: Optional[bs4.Tag] = None
@ -190,7 +190,7 @@ class IliasCrawler:
# A forum # A forum
if str(img_tag["src"]).endswith("frm.svg"): if str(img_tag["src"]).endswith("frm.svg"):
LOGGER.debug("Skipping forum at %r", url) LOGGER.debug("Skipping forum at %r", url)
PRETTY.filtered_path(element_path, "forum") PRETTY.not_searching(element_path, "forum")
return [] return []
# An exercise # An exercise
@ -200,7 +200,7 @@ class IliasCrawler:
if str(img_tag["src"]).endswith("icon_webr.svg"): if str(img_tag["src"]).endswith("icon_webr.svg"):
LOGGER.debug("Skipping external link at %r", url) LOGGER.debug("Skipping external link at %r", url)
PRETTY.filtered_path(element_path, "external link") PRETTY.not_searching(element_path, "external link")
return [] return []
# Match the opencast video plugin # Match the opencast video plugin

2
PFERD/ilias/downloader.py
View File

@ -58,7 +58,7 @@ def download_modified_or_new(organizer: Organizer, info: IliasDownloadInfo) -> b
if info.modification_date.timestamp() > resolved_mod_time_seconds: if info.modification_date.timestamp() > resolved_mod_time_seconds:
return True return True
PRETTY.filtered_path(info.path, "Local file had newer or equal modification time") PRETTY.ignored_file(info.path, "local file has newer or equal modification time")
return False return False

2
PFERD/organizer.py
View File

@ -56,7 +56,7 @@ class Organizer(Location):
if dst_absolute.exists() and dst_absolute.is_file(): if dst_absolute.exists() and dst_absolute.is_file():
if filecmp.cmp(str(src_absolute), str(dst_absolute), shallow=False): if filecmp.cmp(str(src_absolute), str(dst_absolute), shallow=False):
# Bail out, nothing more to do # Bail out, nothing more to do
PRETTY.ignored_file(dst_absolute) PRETTY.ignored_file(dst_absolute, "same file contents")
self.mark(dst) self.mark(dst)
return return

42
PFERD/utils.py
View File

@ -86,39 +86,49 @@ class PrettyLogger:
def __init__(self, logger: logging.Logger) -> None: def __init__(self, logger: logging.Logger) -> None:
self.logger = logger self.logger = logger
def modified_file(self, file_name: PurePath) -> None: def modified_file(self, path: PurePath) -> None:
""" """
An existing file has changed. An existing file has changed.
""" """
self.logger.info( self.logger.info(
f"{Fore.MAGENTA}{Style.BRIGHT}Modified {str(file_name)!r}.{Style.RESET_ALL}" f"{Fore.MAGENTA}{Style.BRIGHT}Modified {str(path)!r}.{Style.RESET_ALL}"
) )
def new_file(self, file_name: PurePath) -> None: def new_file(self, path: PurePath) -> None:
""" """
A new file has been downloaded. A new file has been downloaded.
""" """
self.logger.info( self.logger.info(
f"{Fore.GREEN}{Style.BRIGHT}Created {str(file_name)!r}.{Style.RESET_ALL}") f"{Fore.GREEN}{Style.BRIGHT}Created {str(path)!r}.{Style.RESET_ALL}"
)
def ignored_file(self, file_name: PurePath) -> None: def ignored_file(self, path: PurePath, reason: str) -> None:
""" """
Nothing in particular happened to this file or directory. File was not downloaded or modified.
"""
self.logger.info(f"{Style.DIM}Ignored {str(file_name)!r}.{Style.RESET_ALL}")
def filtered_path(self, path: PurePath, reason: str) -> None:
"""
A crawler filter rejected the given path.
""" """
self.logger.info( self.logger.info(
f"{Style.DIM}Not considering {str(path)!r} due to filter rules" f"{Style.DIM}Ignored {str(path)!r} "
f" ({Style.NORMAL}{reason}{Style.DIM})." f"({Style.NORMAL}{reason}{Style.DIM}).{Style.RESET_ALL}"
f"{Style.RESET_ALL}" )
def searching(self, path: PurePath) -> None:
"""
A crawler searches a particular object.
"""
self.logger.info(f"Searching {str(path)!r}")
def not_searching(self, path: PurePath, reason: str) -> None:
"""
A crawler does not search a particular object.
"""
self.logger.info(
f"{Style.DIM}Not searching {str(path)!r} "
f"({Style.NORMAL}{reason}{Style.DIM}).{Style.RESET_ALL}"
) )
def starting_synchronizer( def starting_synchronizer(