mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Pass element type to ilias directory filter

Browse Source
This commit is contained in:
I-Al-Istannen 2020-05-12 14:38:25 +02:00
parent 4fdb67128d
commit 4f56c8f192
3 changed files with 31 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PFERD/ilias/__init__.py
View File

@ -3,7 +3,7 @@ Synchronizing files from ILIAS instances (https://www.ilias.de/).
""" """
from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator
from .crawler import IliasCrawler, IliasDirectoryFilter from .crawler import IliasCrawler, IliasDirectoryFilter, IliasDirectoryType
from .downloader import (IliasDownloader, IliasDownloadInfo, from .downloader import (IliasDownloader, IliasDownloadInfo,
IliasDownloadStrategy, download_everything, IliasDownloadStrategy, download_everything,
download_modified_or_new) download_modified_or_new)

37
PFERD/ilias/crawler.py
View File

@ -6,6 +6,7 @@ import datetime
import json import json
import logging import logging
import re import re
from enum import Enum
from pathlib import Path from pathlib import Path
from typing import Any, Callable, Dict, List, Optional from typing import Any, Callable, Dict, List, Optional
from urllib.parse import (parse_qs, urlencode, urljoin, urlparse, urlsplit, from urllib.parse import (parse_qs, urlencode, urljoin, urlparse, urlsplit,
@ -24,7 +25,17 @@ from .downloader import IliasDownloadInfo
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
PRETTY = PrettyLogger(LOGGER) PRETTY = PrettyLogger(LOGGER)
IliasDirectoryFilter = Callable[[Path], bool]
class IliasDirectoryType(Enum):
"""
The type of an ilias directory.
"""
FOLDER = "FOLDER"
VIDEO = "VIDEO"
EXERCISE = "EXERCISE"
IliasDirectoryFilter = Callable[[Path, IliasDirectoryType], bool]
class IliasCrawler: class IliasCrawler:
@ -187,12 +198,6 @@ class IliasCrawler:
element_path = Path(parent_path, link_element.getText().strip()) element_path = Path(parent_path, link_element.getText().strip())
if not self.dir_filter(element_path):
PRETTY.not_searching(element_path, "user filter")
return []
PRETTY.searching(element_path)
found_parent: Optional[bs4.Tag] = None found_parent: Optional[bs4.Tag] = None
# We look for the outer div of our inner link, to find information around it # We look for the outer div of our inner link, to find information around it
@ -213,6 +218,20 @@ class IliasCrawler:
PRETTY.warning(f"Could not find image tag for {url!r}") PRETTY.warning(f"Could not find image tag for {url!r}")
return [] return []
directory_type = IliasDirectoryType.FOLDER
if "opencast" in str(img_tag["alt"]).lower():
directory_type = IliasDirectoryType.VIDEO
if str(img_tag["src"]).endswith("icon_exc.svg"):
directory_type = IliasDirectoryType.EXERCISE
if not self.dir_filter(element_path, directory_type):
PRETTY.not_searching(element_path, "user filter")
return []
PRETTY.searching(element_path)
# A forum # A forum
if str(img_tag["src"]).endswith("frm.svg"): if str(img_tag["src"]).endswith("frm.svg"):
LOGGER.debug("Skipping forum at %r", url) LOGGER.debug("Skipping forum at %r", url)
@ -220,7 +239,7 @@ class IliasCrawler:
return [] return []
# An exercise # An exercise
if str(img_tag["src"]).endswith("icon_exc.svg"): if directory_type == IliasDirectoryType.EXERCISE:
LOGGER.debug("Crawling exercises at %r", url) LOGGER.debug("Crawling exercises at %r", url)
return self._crawl_exercises(element_path, url) return self._crawl_exercises(element_path, url)
@ -230,7 +249,7 @@ class IliasCrawler:
return [] return []
# Match the opencast video plugin # Match the opencast video plugin
if "opencast" in str(img_tag["alt"]).lower(): if directory_type == IliasDirectoryType.VIDEO:
LOGGER.debug("Found video site: %r", url) LOGGER.debug("Found video site: %r", url)
return self._crawl_video_directory(element_path, url) return self._crawl_video_directory(element_path, url)

4
PFERD/pferd.py
View File

@ -95,7 +95,7 @@ class Pferd(Location):
self, self,
target: PathLike, target: PathLike,
course_id: str, course_id: str,
dir_filter: IliasDirectoryFilter = lambda x: True, dir_filter: IliasDirectoryFilter = lambda x, y: True,
transform: Transform = lambda x: x, transform: Transform = lambda x: x,
cookies: Optional[PathLike] = None, cookies: Optional[PathLike] = None,
username: Optional[str] = None, username: Optional[str] = None,
@ -147,7 +147,7 @@ class Pferd(Location):
def ilias_kit_personal_desktop( def ilias_kit_personal_desktop(
self, self,
target: PathLike, target: PathLike,
dir_filter: IliasDirectoryFilter = lambda x: True, dir_filter: IliasDirectoryFilter = lambda x, y: True,
transform: Transform = lambda x: x, transform: Transform = lambda x: x,
cookies: Optional[PathLike] = None, cookies: Optional[PathLike] = None,
username: Optional[str] = None, username: Optional[str] = None,