mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Compare commits
41 Commits
Author | SHA1 | Date | |
---|---|---|---|
0045124a4e | |||
9618aae83b | |||
33453ede2d | |||
e467b38d73 | |||
e9d2d05030 | |||
4bf0c972e6 | |||
4ee919625d | |||
d30f25ee97 | |||
10d9d74528 | |||
43c5453e10 | |||
eb4de8ae0c | |||
e32c1f000f | |||
5f527bc697 | |||
ced8b9a2d0 | |||
6f3cfd4396 | |||
462d993fbc | |||
a99356f2a2 | |||
eac2e34161 | |||
a82a0b19c2 | |||
90cb6e989b | |||
6289938d7c | |||
13b8c3d9c6 | |||
88afe64a92 | |||
6b2a657573 | |||
d6f38a61e1 | |||
ad3f4955f7 | |||
e42ab83d32 | |||
f9a3f9b9f2 | |||
ef7d5ea2d3 | |||
55ea304ff3 | |||
fee12b3d9e | |||
6673077397 | |||
742632ed8d | |||
544d45cbc5 | |||
86f79ff1f1 | |||
ee67f9f472 | |||
8ec3f41251 | |||
89be07d4d3 | |||
91200f3684 | |||
9ffd603357 | |||
80eeb8fe97 |
33
CHANGELOG.md
33
CHANGELOG.md
@ -22,6 +22,39 @@ ambiguous situations.
|
||||
|
||||
## Unreleased
|
||||
|
||||
## 3.3.0 - 2022-01-09
|
||||
|
||||
### Added
|
||||
- A KIT IPD crawler
|
||||
- Support for ILIAS cards
|
||||
- (Rudimentary) support for content pages
|
||||
- Support for multi-stream videos
|
||||
- Support for ILIAS 7
|
||||
|
||||
### Removed
|
||||
- [Interpolation](https://docs.python.org/3/library/configparser.html#interpolation-of-values) in config file
|
||||
|
||||
### Fixed
|
||||
- Crawling of recursive courses
|
||||
- Crawling files directly placed on the personal desktop
|
||||
- Ignore timestamps at the unix epoch as they crash on windows
|
||||
|
||||
## 3.2.0 - 2021-08-04
|
||||
|
||||
### Added
|
||||
- `--skip` command line option
|
||||
- Support for ILIAS booking objects
|
||||
|
||||
### Changed
|
||||
- Using multiple path segments on left side of `-name->` now results in an
|
||||
error. This was already forbidden by the documentation but silently accepted
|
||||
by PFERD.
|
||||
- More consistent path printing in some `--explain` messages
|
||||
|
||||
### Fixed
|
||||
- Nondeterministic name deduplication due to ILIAS reordering elements
|
||||
- More exceptions are handled properly
|
||||
|
||||
## 3.1.0 - 2021-06-13
|
||||
|
||||
If your config file doesn't do weird things with transforms, it should continue
|
||||
|
30
CONFIG.md
30
CONFIG.md
@ -4,11 +4,11 @@ A config file consists of sections. A section begins with a `[section]` header,
|
||||
which is followed by a list of `key = value` pairs. Comments must be on their
|
||||
own line and start with `#`. Multiline values must be indented beyond their key.
|
||||
Boolean values can be `yes` or `no`. For more details and some examples on the
|
||||
format, see the [configparser documentation][1] ([basic interpolation][2] is
|
||||
enabled).
|
||||
format, see the [configparser documentation][1] ([interpolation][2] is
|
||||
disabled).
|
||||
|
||||
[1]: <https://docs.python.org/3/library/configparser.html#supported-ini-file-structure> "Supported INI File Structure"
|
||||
[2]: <https://docs.python.org/3/library/configparser.html#configparser.BasicInterpolation> "BasicInterpolation"
|
||||
[2]: <https://docs.python.org/3/library/configparser.html#interpolation-of-values> "Interpolation of values"
|
||||
|
||||
## The `DEFAULT` section
|
||||
|
||||
@ -36,7 +36,7 @@ Sections whose names start with `crawl:` are used to configure crawlers. The
|
||||
rest of the section name specifies the name of the crawler.
|
||||
|
||||
A crawler synchronizes a remote resource to a local directory. There are
|
||||
different types of crawlers for different kinds of resources, e. g. ILIAS
|
||||
different types of crawlers for different kinds of resources, e.g. ILIAS
|
||||
courses or lecture websites.
|
||||
|
||||
Each crawl section represents an instance of a specific type of crawler. The
|
||||
@ -53,7 +53,7 @@ common to all crawlers:
|
||||
crawler can still be executed manually using the `--crawler` or `-C` flags.
|
||||
(Default: `no`)
|
||||
- `output_dir`: The directory the crawler synchronizes files to. A crawler will
|
||||
never place any files outside of this directory. (Default: the crawler's name)
|
||||
never place any files outside this directory. (Default: the crawler's name)
|
||||
- `redownload`: When to download a file that is already present locally.
|
||||
(Default: `never-smart`)
|
||||
- `never`: If a file is present locally, it is not downloaded again.
|
||||
@ -136,6 +136,18 @@ crawler simulate a slower, network-based crawler.
|
||||
requests. (Default: `0.0`)
|
||||
- `download_speed`: Download speed (in bytes per second) to simulate. (Optional)
|
||||
|
||||
### The `kit-ipd` crawler
|
||||
|
||||
This crawler crawls a KIT-IPD page by url. The root page can be crawled from
|
||||
outside the KIT network so you will be informed about any new/deleted files,
|
||||
but downloading files requires you to be within. Adding a show delay between
|
||||
requests is likely a good idea.
|
||||
|
||||
- `target`: URL to a KIT-IPD page
|
||||
- `link_regex`: A regex that is matched against the `href` part of links. If it
|
||||
matches, the given link is downloaded as a file. This is used to extract
|
||||
files from KIT-IPD pages. (Default: `^.*/[^/]*\.(?:pdf|zip|c|java)$`)
|
||||
|
||||
### The `kit-ilias-web` crawler
|
||||
|
||||
This crawler crawls the KIT ILIAS instance.
|
||||
@ -305,11 +317,11 @@ matches `SOURCE`, the output path is created using `TARGET` as template.
|
||||
`SOURCE` is automatically anchored.
|
||||
|
||||
`TARGET` uses Python's [format string syntax][3]. The *n*-th capturing group can
|
||||
be referred to as `{g<n>}` (e. g. `{g3}`). `{g0}` refers to the original path.
|
||||
be referred to as `{g<n>}` (e.g. `{g3}`). `{g0}` refers to the original path.
|
||||
If capturing group *n*'s contents are a valid integer, the integer value is
|
||||
available as `{i<n>}` (e. g. `{i3}`). If capturing group *n*'s contents are a
|
||||
valid float, the float value is available as `{f<n>}` (e. g. `{f3}`). If a
|
||||
capturing group is not present (e. g. when matching the string `cd` with the
|
||||
available as `{i<n>}` (e.g. `{i3}`). If capturing group *n*'s contents are a
|
||||
valid float, the float value is available as `{f<n>}` (e.g. `{f3}`). If a
|
||||
capturing group is not present (e.g. when matching the string `cd` with the
|
||||
regex `(ab)?cd`), the corresponding variables are not defined.
|
||||
|
||||
Python's format string syntax has rich options for formatting its arguments. For
|
||||
|
3
LICENSE
3
LICENSE
@ -1,4 +1,5 @@
|
||||
Copyright 2019-2020 Garmelon, I-Al-Istannen, danstooamerican, pavelzw, TheChristophe, Scriptim
|
||||
Copyright 2019-2021 Garmelon, I-Al-Istannen, danstooamerican, pavelzw,
|
||||
TheChristophe, Scriptim, thelukasprobst, Toorero
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
|
@ -15,7 +15,7 @@ from .transformer import RuleParseError
|
||||
|
||||
def load_config_parser(args: argparse.Namespace) -> configparser.ConfigParser:
|
||||
log.explain_topic("Loading config")
|
||||
parser = configparser.ConfigParser()
|
||||
parser = configparser.ConfigParser(interpolation=None)
|
||||
|
||||
if args.command is None:
|
||||
log.explain("No CLI command specified, loading config from file")
|
||||
@ -116,7 +116,7 @@ def main() -> None:
|
||||
sys.exit()
|
||||
|
||||
try:
|
||||
pferd = Pferd(config, args.crawler)
|
||||
pferd = Pferd(config, args.crawler, args.skip)
|
||||
except PferdLoadError as e:
|
||||
log.unlock()
|
||||
log.error(str(e))
|
||||
|
@ -9,4 +9,5 @@
|
||||
|
||||
from . import command_local # noqa: F401 imported but unused
|
||||
from . import command_kit_ilias_web # noqa: F401 imported but unused
|
||||
from . import command_kit_ipd # noqa: F401 imported but unused
|
||||
from .parser import PARSER, ParserLoadError, load_default_section # noqa: F401 imported but unused
|
||||
|
54
PFERD/cli/command_kit_ipd.py
Normal file
54
PFERD/cli/command_kit_ipd.py
Normal file
@ -0,0 +1,54 @@
|
||||
import argparse
|
||||
import configparser
|
||||
from pathlib import Path
|
||||
|
||||
from ..logging import log
|
||||
from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
|
||||
|
||||
SUBPARSER = SUBPARSERS.add_parser(
|
||||
"kit-ipd",
|
||||
parents=[CRAWLER_PARSER],
|
||||
)
|
||||
|
||||
GROUP = SUBPARSER.add_argument_group(
|
||||
title="kit ipd crawler arguments",
|
||||
description="arguments for the 'kit-ipd' crawler",
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--link-regex",
|
||||
type=str,
|
||||
metavar="REGEX",
|
||||
help="href-matching regex to identify downloadable files"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"target",
|
||||
type=str,
|
||||
metavar="TARGET",
|
||||
help="url to crawl"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"output",
|
||||
type=Path,
|
||||
metavar="OUTPUT",
|
||||
help="output directory"
|
||||
)
|
||||
|
||||
|
||||
def load(
|
||||
args: argparse.Namespace,
|
||||
parser: configparser.ConfigParser,
|
||||
) -> None:
|
||||
log.explain("Creating config for command 'kit-ipd'")
|
||||
|
||||
parser["crawl:kit-ipd"] = {}
|
||||
section = parser["crawl:kit-ipd"]
|
||||
load_crawler(args, section)
|
||||
|
||||
section["type"] = "kit-ipd"
|
||||
section["target"] = str(args.target)
|
||||
section["output_dir"] = str(args.output)
|
||||
if args.link_regex:
|
||||
section["link_regex"] = str(args.link_regex)
|
||||
|
||||
|
||||
SUBPARSER.set_defaults(command=load)
|
@ -181,6 +181,14 @@ PARSER.add_argument(
|
||||
help="only execute a single crawler."
|
||||
" Can be specified multiple times to execute multiple crawlers"
|
||||
)
|
||||
PARSER.add_argument(
|
||||
"--skip", "-S",
|
||||
action="append",
|
||||
type=str,
|
||||
metavar="NAME",
|
||||
help="don't execute this particular crawler."
|
||||
" Can be specified multiple times to skip multiple crawlers"
|
||||
)
|
||||
PARSER.add_argument(
|
||||
"--working-dir",
|
||||
type=Path,
|
||||
|
@ -5,6 +5,7 @@ from ..auth import Authenticator
|
||||
from ..config import Config
|
||||
from .crawler import Crawler, CrawlError, CrawlerSection # noqa: F401
|
||||
from .ilias import KitIliasWebCrawler, KitIliasWebCrawlerSection
|
||||
from .kit_ipd_crawler import KitIpdCrawler, KitIpdCrawlerSection
|
||||
from .local_crawler import LocalCrawler, LocalCrawlerSection
|
||||
|
||||
CrawlerConstructor = Callable[[
|
||||
@ -19,4 +20,6 @@ CRAWLERS: Dict[str, CrawlerConstructor] = {
|
||||
LocalCrawler(n, LocalCrawlerSection(s), c),
|
||||
"kit-ilias-web": lambda n, s, c, a:
|
||||
KitIliasWebCrawler(n, KitIliasWebCrawlerSection(s), c, a),
|
||||
"kit-ipd": lambda n, s, c, a:
|
||||
KitIpdCrawler(n, KitIpdCrawlerSection(s), c),
|
||||
}
|
||||
|
@ -47,16 +47,18 @@ def noncritical(f: Wrapped) -> Wrapped:
|
||||
try:
|
||||
f(*args, **kwargs)
|
||||
except (CrawlWarning, OutputDirError, MarkDuplicateError, MarkConflictError) as e:
|
||||
crawler.report.add_warning(str(e))
|
||||
log.warn(str(e))
|
||||
crawler.error_free = False
|
||||
except: # noqa: E722 do not use bare 'except'
|
||||
except Exception as e:
|
||||
crawler.error_free = False
|
||||
crawler.report.add_error(str(e))
|
||||
raise
|
||||
|
||||
return wrapper # type: ignore
|
||||
|
||||
|
||||
AWrapped = TypeVar("AWrapped", bound=Callable[..., Awaitable[None]])
|
||||
AWrapped = TypeVar("AWrapped", bound=Callable[..., Awaitable[Optional[Any]]])
|
||||
|
||||
|
||||
def anoncritical(f: AWrapped) -> AWrapped:
|
||||
@ -72,21 +74,25 @@ def anoncritical(f: AWrapped) -> AWrapped:
|
||||
Warning: Must only be applied to member functions of the Crawler class!
|
||||
"""
|
||||
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> None:
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]:
|
||||
if not (args and isinstance(args[0], Crawler)):
|
||||
raise RuntimeError("@anoncritical must only applied to Crawler methods")
|
||||
|
||||
crawler = args[0]
|
||||
|
||||
try:
|
||||
await f(*args, **kwargs)
|
||||
return await f(*args, **kwargs)
|
||||
except (CrawlWarning, OutputDirError, MarkDuplicateError, MarkConflictError) as e:
|
||||
log.warn(str(e))
|
||||
crawler.error_free = False
|
||||
except: # noqa: E722 do not use bare 'except'
|
||||
crawler.report.add_warning(str(e))
|
||||
except Exception as e:
|
||||
crawler.error_free = False
|
||||
crawler.report.add_error(str(e))
|
||||
raise
|
||||
|
||||
return None
|
||||
|
||||
return wrapper # type: ignore
|
||||
|
||||
|
||||
@ -318,6 +324,7 @@ class Crawler(ABC):
|
||||
log.explain("Warnings or errors occurred during this run")
|
||||
log.explain("Answer: No")
|
||||
|
||||
@anoncritical
|
||||
async def run(self) -> None:
|
||||
"""
|
||||
Start the crawling process. Call this function if you want to use a
|
||||
|
@ -22,6 +22,7 @@ class IliasElementType(Enum):
|
||||
FOLDER = "folder"
|
||||
FORUM = "forum"
|
||||
LINK = "link"
|
||||
BOOKING = "booking"
|
||||
MEETING = "meeting"
|
||||
VIDEO = "video"
|
||||
VIDEO_PLAYER = "video_player"
|
||||
@ -37,6 +38,22 @@ class IliasPageElement:
|
||||
mtime: Optional[datetime] = None
|
||||
description: Optional[str] = None
|
||||
|
||||
def id(self) -> str:
|
||||
regexes = [
|
||||
r"eid=(?P<id>[0-9a-z\-]+)",
|
||||
r"file_(?P<id>\d+)",
|
||||
r"ref_id=(?P<id>\d+)",
|
||||
r"target=[a-z]+_(?P<id>\d+)"
|
||||
]
|
||||
|
||||
for regex in regexes:
|
||||
if match := re.search(regex, self.url):
|
||||
return match.groupdict()["id"]
|
||||
|
||||
# Fall back to URL
|
||||
log.warn(f"Didn't find identity for {self.name} - {self.url}. Please report this.")
|
||||
return self.url
|
||||
|
||||
|
||||
class IliasPage:
|
||||
|
||||
@ -59,6 +76,12 @@ class IliasPage:
|
||||
if self._is_exercise_file():
|
||||
log.explain("Page is an exercise, searching for elements")
|
||||
return self._find_exercise_entries()
|
||||
if self._is_personal_desktop():
|
||||
log.explain("Page is the personal desktop, searching for elements")
|
||||
return self._find_personal_desktop_entries()
|
||||
if self._is_content_page():
|
||||
log.explain("Page is a content page, searching for elements")
|
||||
return self._find_copa_entries()
|
||||
log.explain("Page is a normal folder, searching for elements")
|
||||
return self._find_normal_entries()
|
||||
|
||||
@ -103,13 +126,22 @@ class IliasPage:
|
||||
|
||||
return False
|
||||
|
||||
def _is_personal_desktop(self) -> bool:
|
||||
return self._soup.find("a", attrs={"href": lambda x: x and "block_type=pditems" in x})
|
||||
|
||||
def _is_content_page(self) -> bool:
|
||||
link = self._soup.find(id="current_perma_link")
|
||||
if not link:
|
||||
return False
|
||||
return "target=copa_" in link.get("value")
|
||||
|
||||
def _player_to_video(self) -> List[IliasPageElement]:
|
||||
# Fetch the actual video page. This is a small wrapper page initializing a javscript
|
||||
# player. Sadly we can not execute that JS. The actual video stream url is nowhere
|
||||
# on the page, but defined in a JS object inside a script tag, passed to the player
|
||||
# library.
|
||||
# We do the impossible and RegEx the stream JSON object out of the page's HTML source
|
||||
regex: re.Pattern[str] = re.compile(
|
||||
regex = re.compile(
|
||||
r"({\"streams\"[\s\S]+?),\s*{\"paella_config_file", re.IGNORECASE
|
||||
)
|
||||
json_match = regex.search(str(self._soup))
|
||||
@ -121,10 +153,64 @@ class IliasPage:
|
||||
|
||||
# parse it
|
||||
json_object = json.loads(json_str)
|
||||
# and fetch the video url!
|
||||
video_url = json_object["streams"][0]["sources"]["mp4"][0]["src"]
|
||||
streams = [stream for stream in json_object["streams"]]
|
||||
|
||||
# and just fetch the lone video url!
|
||||
if len(streams) == 1:
|
||||
video_url = streams[0]["sources"]["mp4"][0]["src"]
|
||||
return [IliasPageElement(IliasElementType.VIDEO, video_url, self._source_name)]
|
||||
|
||||
log.explain(f"Found multiple videos for stream at {self._source_name}")
|
||||
items = []
|
||||
for stream in sorted(streams, key=lambda stream: stream["content"]):
|
||||
full_name = f"{self._source_name.replace('.mp4', '')} ({stream['content']}).mp4"
|
||||
video_url = stream["sources"]["mp4"][0]["src"]
|
||||
items.append(IliasPageElement(IliasElementType.VIDEO, video_url, full_name))
|
||||
|
||||
return items
|
||||
|
||||
def _find_personal_desktop_entries(self) -> List[IliasPageElement]:
|
||||
items: List[IliasPageElement] = []
|
||||
|
||||
titles: List[Tag] = self._soup.select(".il-item-title")
|
||||
for title in titles:
|
||||
link = title.find("a")
|
||||
name = _sanitize_path_name(link.text.strip())
|
||||
url = self._abs_url_from_link(link)
|
||||
|
||||
type = self._find_type_from_link(name, link, url)
|
||||
if not type:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Could not extract type for {link}")
|
||||
continue
|
||||
|
||||
log.explain(f"Found {name!r}")
|
||||
|
||||
if type == IliasElementType.FILE and "_download" not in url:
|
||||
url = re.sub(r"(target=file_\d+)", r"\1_download", url)
|
||||
log.explain("Rewired file URL to include download part")
|
||||
|
||||
items.append(IliasPageElement(type, url, name))
|
||||
|
||||
return items
|
||||
|
||||
def _find_copa_entries(self) -> List[IliasPageElement]:
|
||||
items: List[IliasPageElement] = []
|
||||
links: List[Tag] = self._soup.findAll(class_="ilc_flist_a_FileListItemLink")
|
||||
|
||||
for link in links:
|
||||
url = self._abs_url_from_link(link)
|
||||
name = _sanitize_path_name(link.getText().strip().replace("\t", ""))
|
||||
|
||||
if "file_id" not in url:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Found unknown content page item {name!r} with url {url!r}")
|
||||
continue
|
||||
|
||||
items.append(IliasPageElement(IliasElementType.FILE, url, name))
|
||||
|
||||
return items
|
||||
|
||||
def _find_video_entries(self) -> List[IliasPageElement]:
|
||||
# ILIAS has three stages for video pages
|
||||
# 1. The initial dummy page without any videos. This page contains the link to the listing
|
||||
@ -344,6 +430,8 @@ class IliasPage:
|
||||
log.explain(f"Found {element_name!r}")
|
||||
result.append(IliasPageElement(element_type, abs_url, element_name, description=description))
|
||||
|
||||
result += self._find_cards()
|
||||
|
||||
return result
|
||||
|
||||
def _find_upwards_folder_hierarchy(self, tag: Tag) -> List[str]:
|
||||
@ -371,7 +459,10 @@ class IliasPage:
|
||||
continue
|
||||
prev: Tag = parent.findPreviousSibling("div")
|
||||
if "ilContainerBlockHeader" in prev.get("class"):
|
||||
if prev.find("h3"):
|
||||
found_titles.append(prev.find("h3").getText().strip())
|
||||
else:
|
||||
found_titles.append(prev.find("h2").getText().strip())
|
||||
|
||||
# And this for real accordions
|
||||
if "il_VAccordionContentDef" in parent.get("class"):
|
||||
@ -426,6 +517,90 @@ class IliasPage:
|
||||
log.explain(f"Found file {full_path!r}")
|
||||
return IliasPageElement(IliasElementType.FILE, url, full_path, modification_date)
|
||||
|
||||
def _find_cards(self) -> List[IliasPageElement]:
|
||||
result: List[IliasPageElement] = []
|
||||
|
||||
card_titles: List[Tag] = self._soup.select(".card-title a")
|
||||
|
||||
for title in card_titles:
|
||||
url = self._abs_url_from_link(title)
|
||||
name = _sanitize_path_name(title.getText().strip())
|
||||
type = self._find_type_from_card(title)
|
||||
|
||||
if not type:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Could not extract type for {title}")
|
||||
continue
|
||||
|
||||
result.append(IliasPageElement(type, url, name))
|
||||
|
||||
card_button_tiles: List[Tag] = self._soup.select(".card-title button")
|
||||
|
||||
for button in card_button_tiles:
|
||||
regex = re.compile(button["id"] + r".*window.open\(['\"](.+?)['\"]")
|
||||
res = regex.search(str(self._soup))
|
||||
if not res:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Could not find click handler target for {button}")
|
||||
continue
|
||||
url = self._abs_url_from_relative(res.group(1))
|
||||
name = _sanitize_path_name(button.getText().strip())
|
||||
type = self._find_type_from_card(button)
|
||||
caption_parent = button.findParent(
|
||||
"div",
|
||||
attrs={"class": lambda x: x and "caption" in x},
|
||||
)
|
||||
description = caption_parent.find_next_sibling("div").getText().strip()
|
||||
|
||||
if not type:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Could not extract type for {button}")
|
||||
continue
|
||||
|
||||
result.append(IliasPageElement(type, url, name, description=description))
|
||||
|
||||
return result
|
||||
|
||||
def _find_type_from_card(self, card_title: Tag) -> Optional[IliasElementType]:
|
||||
def is_card_root(element: Tag) -> bool:
|
||||
return "il-card" in element["class"] and "thumbnail" in element["class"]
|
||||
|
||||
card_root: Optional[Tag] = None
|
||||
|
||||
# We look for the card root
|
||||
for parent in card_title.parents:
|
||||
if is_card_root(parent):
|
||||
card_root = parent
|
||||
break
|
||||
|
||||
if card_root is None:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Tried to figure out element type, but did not find an icon for {card_title}")
|
||||
return None
|
||||
|
||||
icon: Tag = card_root.select_one(".il-card-repository-head .icon")
|
||||
|
||||
if "opencast" in icon["class"]:
|
||||
return IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED
|
||||
if "exc" in icon["class"]:
|
||||
return IliasElementType.EXERCISE
|
||||
if "webr" in icon["class"]:
|
||||
return IliasElementType.LINK
|
||||
if "book" in icon["class"]:
|
||||
return IliasElementType.BOOKING
|
||||
if "frm" in icon["class"]:
|
||||
return IliasElementType.FORUM
|
||||
if "sess" in icon["class"]:
|
||||
return IliasElementType.MEETING
|
||||
if "tst" in icon["class"]:
|
||||
return IliasElementType.TEST
|
||||
if "fold" in icon["class"]:
|
||||
return IliasElementType.FOLDER
|
||||
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Could not extract type from {icon} for card title {card_title}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _find_type_from_link(
|
||||
element_name: str,
|
||||
@ -441,9 +616,30 @@ class IliasPage:
|
||||
if "target=file_" in parsed_url.query:
|
||||
return IliasElementType.FILE
|
||||
|
||||
if "target=grp_" in parsed_url.query:
|
||||
return IliasElementType.FOLDER
|
||||
|
||||
if "target=crs_" in parsed_url.query:
|
||||
return IliasElementType.FOLDER
|
||||
|
||||
if "baseClass=ilExerciseHandlerGUI" in parsed_url.query:
|
||||
return IliasElementType.EXERCISE
|
||||
|
||||
if "baseClass=ilLinkResourceHandlerGUI" in parsed_url.query and "calldirectlink" in parsed_url.query:
|
||||
return IliasElementType.LINK
|
||||
|
||||
if "cmd=showThreads" in parsed_url.query or "target=frm_" in parsed_url.query:
|
||||
return IliasElementType.FORUM
|
||||
|
||||
if "cmdClass=ilobjtestgui" in parsed_url.query:
|
||||
return IliasElementType.TEST
|
||||
|
||||
# Booking and Meeting can not be detected based on the link. They do have a ref_id though, so
|
||||
# try to guess it from the image.
|
||||
|
||||
# Everything with a ref_id can *probably* be opened to reveal nested things
|
||||
# video groups, directories, exercises, etc
|
||||
if "ref_id=" in parsed_url.query:
|
||||
if "ref_id=" in parsed_url.query or "goto.php" in parsed_url.path:
|
||||
return IliasPage._find_type_from_folder_like(link_element, url)
|
||||
|
||||
_unexpected_html_warning()
|
||||
@ -464,7 +660,7 @@ class IliasPage:
|
||||
# We look for the outer div of our inner link, to find information around it
|
||||
# (mostly the icon)
|
||||
for parent in link_element.parents:
|
||||
if "ilContainerListItemOuter" in parent["class"]:
|
||||
if "ilContainerListItemOuter" in parent["class"] or "il-std-item" in parent["class"]:
|
||||
found_parent = parent
|
||||
break
|
||||
|
||||
@ -476,6 +672,9 @@ class IliasPage:
|
||||
# Find the small descriptive icon to figure out the type
|
||||
img_tag: Optional[Tag] = found_parent.select_one("img.ilListItemIcon")
|
||||
|
||||
if img_tag is None:
|
||||
img_tag = found_parent.select_one("img.icon")
|
||||
|
||||
if img_tag is None:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Tried to figure out element type, but did not find an image for {url}")
|
||||
@ -490,6 +689,9 @@ class IliasPage:
|
||||
if str(img_tag["src"]).endswith("icon_webr.svg"):
|
||||
return IliasElementType.LINK
|
||||
|
||||
if str(img_tag["src"]).endswith("icon_book.svg"):
|
||||
return IliasElementType.BOOKING
|
||||
|
||||
if str(img_tag["src"]).endswith("frm.svg"):
|
||||
return IliasElementType.FORUM
|
||||
|
||||
@ -523,7 +725,13 @@ class IliasPage:
|
||||
"""
|
||||
Create an absolute url from an <a> tag.
|
||||
"""
|
||||
return urljoin(self._page_url, link_tag.get("href"))
|
||||
return self._abs_url_from_relative(link_tag.get("href"))
|
||||
|
||||
def _abs_url_from_relative(self, relative_url: str) -> str:
|
||||
"""
|
||||
Create an absolute url from a relative URL.
|
||||
"""
|
||||
return urljoin(self._page_url, relative_url)
|
||||
|
||||
|
||||
def _unexpected_html_warning() -> None:
|
||||
|
@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
import re
|
||||
from pathlib import PurePath
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, TypeVar, Union
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, TypeVar, Union, cast
|
||||
|
||||
import aiohttp
|
||||
from aiohttp import hdrs
|
||||
@ -12,7 +12,7 @@ from ...config import Config
|
||||
from ...logging import ProgressBar, log
|
||||
from ...output_dir import FileSink, Redownload
|
||||
from ...utils import fmt_path, soupify, url_set_query_param
|
||||
from ..crawler import CrawlError, CrawlWarning, anoncritical
|
||||
from ..crawler import CrawlError, CrawlToken, CrawlWarning, DownloadToken, anoncritical
|
||||
from ..http_crawler import HttpCrawler, HttpCrawlerSection
|
||||
from .file_templates import Links
|
||||
from .kit_ilias_html import IliasElementType, IliasPage, IliasPageElement
|
||||
@ -81,17 +81,16 @@ _VIDEO_ELEMENTS: Set[IliasElementType] = set([
|
||||
IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED,
|
||||
])
|
||||
|
||||
AWrapped = TypeVar("AWrapped", bound=Callable[..., Awaitable[None]])
|
||||
AWrapped = TypeVar("AWrapped", bound=Callable[..., Awaitable[Optional[Any]]])
|
||||
|
||||
|
||||
def _iorepeat(attempts: int, name: str) -> Callable[[AWrapped], AWrapped]:
|
||||
def _iorepeat(attempts: int, name: str, failure_is_error: bool = False) -> Callable[[AWrapped], AWrapped]:
|
||||
def decorator(f: AWrapped) -> AWrapped:
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> None:
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]:
|
||||
last_exception: Optional[BaseException] = None
|
||||
for round in range(attempts):
|
||||
try:
|
||||
await f(*args, **kwargs)
|
||||
return
|
||||
return await f(*args, **kwargs)
|
||||
except aiohttp.ContentTypeError: # invalid content type
|
||||
raise CrawlWarning("ILIAS returned an invalid content type")
|
||||
except aiohttp.TooManyRedirects:
|
||||
@ -106,6 +105,9 @@ def _iorepeat(attempts: int, name: str) -> Callable[[AWrapped], AWrapped]:
|
||||
|
||||
if last_exception:
|
||||
message = f"Error in I/O Operation: {last_exception}"
|
||||
if failure_is_error:
|
||||
raise CrawlError(message) from last_exception
|
||||
else:
|
||||
raise CrawlWarning(message) from last_exception
|
||||
raise CrawlError("Impossible return in ilias _iorepeat")
|
||||
|
||||
@ -180,6 +182,7 @@ instance's greatest bottleneck.
|
||||
self._link_file_redirect_delay = section.link_redirect_delay()
|
||||
self._links = section.links()
|
||||
self._videos = section.videos()
|
||||
self._visited_urls: Set[str] = set()
|
||||
|
||||
async def _run(self) -> None:
|
||||
if isinstance(self._target, int):
|
||||
@ -201,7 +204,9 @@ instance's greatest bottleneck.
|
||||
await self._crawl_url(root_url, expected_id=course_id)
|
||||
|
||||
async def _crawl_desktop(self) -> None:
|
||||
await self._crawl_url(self._base_url)
|
||||
appendix = r"ILIAS\PersonalDesktop\PDMainBarProvider|mm_pd_sel_items"
|
||||
appendix = appendix.encode("ASCII").hex()
|
||||
await self._crawl_url(self._base_url + "/gs_content.php?item=" + appendix)
|
||||
|
||||
async def _crawl_url(self, url: str, expected_id: Optional[int] = None) -> None:
|
||||
maybe_cl = await self.crawl(PurePath("."))
|
||||
@ -230,17 +235,35 @@ instance's greatest bottleneck.
|
||||
|
||||
# Fill up our task list with the found elements
|
||||
await gather_elements()
|
||||
tasks = [self._handle_ilias_element(PurePath("."), element) for element in elements]
|
||||
|
||||
elements.sort(key=lambda e: e.id())
|
||||
|
||||
tasks: List[Awaitable[None]] = []
|
||||
for element in elements:
|
||||
if handle := await self._handle_ilias_element(PurePath("."), element):
|
||||
tasks.append(asyncio.create_task(handle))
|
||||
|
||||
# And execute them
|
||||
await self.gather(tasks)
|
||||
|
||||
async def _handle_ilias_page(self, url: str, parent: IliasPageElement, path: PurePath) -> None:
|
||||
async def _handle_ilias_page(
|
||||
self,
|
||||
url: str,
|
||||
parent: IliasPageElement,
|
||||
path: PurePath,
|
||||
) -> Optional[Awaitable[None]]:
|
||||
maybe_cl = await self.crawl(path)
|
||||
if not maybe_cl:
|
||||
return
|
||||
cl = maybe_cl # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
|
||||
return None
|
||||
return self._crawl_ilias_page(url, parent, maybe_cl)
|
||||
|
||||
@anoncritical
|
||||
async def _crawl_ilias_page(
|
||||
self,
|
||||
url: str,
|
||||
parent: IliasPageElement,
|
||||
cl: CrawlToken,
|
||||
) -> None:
|
||||
elements: List[IliasPageElement] = []
|
||||
|
||||
@_iorepeat(3, "crawling folder")
|
||||
@ -252,7 +275,7 @@ instance's greatest bottleneck.
|
||||
|
||||
while next_stage_url:
|
||||
soup = await self._get_page(next_stage_url)
|
||||
log.explain_topic(f"Parsing HTML page for {fmt_path(path)}")
|
||||
log.explain_topic(f"Parsing HTML page for {fmt_path(cl.path)}")
|
||||
log.explain(f"URL: {next_stage_url}")
|
||||
page = IliasPage(soup, next_stage_url, current_parent)
|
||||
if next_element := page.get_next_stage_element():
|
||||
@ -265,16 +288,34 @@ instance's greatest bottleneck.
|
||||
|
||||
# Fill up our task list with the found elements
|
||||
await gather_elements()
|
||||
tasks = [self._handle_ilias_element(cl.path, element) for element in elements]
|
||||
|
||||
elements.sort(key=lambda e: e.id())
|
||||
|
||||
tasks: List[Awaitable[None]] = []
|
||||
for element in elements:
|
||||
if handle := await self._handle_ilias_element(cl.path, element):
|
||||
tasks.append(asyncio.create_task(handle))
|
||||
|
||||
# And execute them
|
||||
await self.gather(tasks)
|
||||
|
||||
# These decorators only apply *to this method* and *NOT* to the returned
|
||||
# awaitables!
|
||||
# This method does not await the handlers but returns them instead.
|
||||
# This ensures one level is handled at a time and name deduplication
|
||||
# works correctly.
|
||||
@anoncritical
|
||||
# Shouldn't happen but we also really don't want to let I/O errors bubble up to anoncritical.
|
||||
# If that happens we will be terminated as anoncritical doesn't tream them as non-critical.
|
||||
@_wrap_io_in_warning("handling ilias element")
|
||||
async def _handle_ilias_element(self, parent_path: PurePath, element: IliasPageElement) -> None:
|
||||
async def _handle_ilias_element(
|
||||
self,
|
||||
parent_path: PurePath,
|
||||
element: IliasPageElement,
|
||||
) -> Optional[Awaitable[None]]:
|
||||
if element.url in self._visited_urls:
|
||||
raise CrawlWarning(
|
||||
f"Found second path to element {element.name!r} at {element.url!r}. Aborting subpath"
|
||||
)
|
||||
self._visited_urls.add(element.url)
|
||||
|
||||
element_path = PurePath(parent_path, element.name)
|
||||
|
||||
if element.type in _VIDEO_ELEMENTS:
|
||||
@ -282,35 +323,43 @@ instance's greatest bottleneck.
|
||||
if not self._videos:
|
||||
log.explain("Video crawling is disabled")
|
||||
log.explain("Answer: no")
|
||||
return
|
||||
return None
|
||||
else:
|
||||
log.explain("Video crawling is enabled")
|
||||
log.explain("Answer: yes")
|
||||
|
||||
if element.type == IliasElementType.FILE:
|
||||
await self._download_file(element, element_path)
|
||||
return await self._handle_file(element, element_path)
|
||||
elif element.type == IliasElementType.FORUM:
|
||||
log.explain_topic(f"Decision: Crawl {fmt_path(element_path)}")
|
||||
log.explain("Forums are not supported")
|
||||
log.explain("Answer: No")
|
||||
return None
|
||||
elif element.type == IliasElementType.TEST:
|
||||
log.explain_topic(f"Decision: Crawl {fmt_path(element_path)}")
|
||||
log.explain("Tests contain no relevant files")
|
||||
log.explain("Answer: No")
|
||||
return None
|
||||
elif element.type == IliasElementType.LINK:
|
||||
await self._download_link(element, element_path)
|
||||
return await self._handle_link(element, element_path)
|
||||
elif element.type == IliasElementType.BOOKING:
|
||||
return await self._handle_booking(element, element_path)
|
||||
elif element.type == IliasElementType.VIDEO:
|
||||
await self._download_file(element, element_path)
|
||||
return await self._handle_file(element, element_path)
|
||||
elif element.type == IliasElementType.VIDEO_PLAYER:
|
||||
await self._download_video(element, element_path)
|
||||
return await self._handle_video(element, element_path)
|
||||
elif element.type in _DIRECTORY_PAGES:
|
||||
await self._handle_ilias_page(element.url, element, element_path)
|
||||
return await self._handle_ilias_page(element.url, element, element_path)
|
||||
else:
|
||||
# This will retry it a few times, failing everytime. It doesn't make any network
|
||||
# requests, so that's fine.
|
||||
raise CrawlWarning(f"Unknown element type: {element.type!r}")
|
||||
|
||||
async def _download_link(self, element: IliasPageElement, element_path: PurePath) -> None:
|
||||
async def _handle_link(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> Optional[Awaitable[None]]:
|
||||
log.explain_topic(f"Decision: Crawl Link {fmt_path(element_path)}")
|
||||
log.explain(f"Links type is {self._links}")
|
||||
|
||||
@ -318,32 +367,74 @@ instance's greatest bottleneck.
|
||||
link_extension = self._links.extension()
|
||||
if not link_template_maybe or not link_extension:
|
||||
log.explain("Answer: No")
|
||||
return
|
||||
return None
|
||||
else:
|
||||
log.explain("Answer: Yes")
|
||||
link_template = link_template_maybe
|
||||
element_path = element_path.with_name(element_path.name + link_extension)
|
||||
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime)
|
||||
if not maybe_dl:
|
||||
return
|
||||
dl = maybe_dl # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
|
||||
return None
|
||||
|
||||
return self._download_link(element, link_template_maybe, maybe_dl)
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "resolving link")
|
||||
async def impl() -> None:
|
||||
async def _download_link(self, element: IliasPageElement, link_template: str, dl: DownloadToken) -> None:
|
||||
async with dl as (bar, sink):
|
||||
export_url = element.url.replace("cmd=calldirectlink", "cmd=exportHTML")
|
||||
real_url = await self._resolve_link_target(export_url)
|
||||
self._write_link_content(link_template, real_url, element.name, element.description, sink)
|
||||
|
||||
def _write_link_content(
|
||||
self,
|
||||
link_template: str,
|
||||
url: str,
|
||||
name: str,
|
||||
description: Optional[str],
|
||||
sink: FileSink,
|
||||
) -> None:
|
||||
content = link_template
|
||||
content = content.replace("{{link}}", real_url)
|
||||
content = content.replace("{{name}}", element.name)
|
||||
content = content.replace("{{description}}", str(element.description))
|
||||
content = content.replace("{{link}}", url)
|
||||
content = content.replace("{{name}}", name)
|
||||
content = content.replace("{{description}}", str(description))
|
||||
content = content.replace("{{redirect_delay}}", str(self._link_file_redirect_delay))
|
||||
sink.file.write(content.encode("utf-8"))
|
||||
sink.done()
|
||||
|
||||
await impl()
|
||||
async def _handle_booking(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> Optional[Awaitable[None]]:
|
||||
log.explain_topic(f"Decision: Crawl Booking Link {fmt_path(element_path)}")
|
||||
log.explain(f"Links type is {self._links}")
|
||||
|
||||
link_template_maybe = self._links.template()
|
||||
link_extension = self._links.extension()
|
||||
if not link_template_maybe or not link_extension:
|
||||
log.explain("Answer: No")
|
||||
return None
|
||||
else:
|
||||
log.explain("Answer: Yes")
|
||||
element_path = element_path.with_name(element_path.name + link_extension)
|
||||
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime)
|
||||
if not maybe_dl:
|
||||
return None
|
||||
|
||||
return self._download_booking(element, link_template_maybe, maybe_dl)
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "resolving booking")
|
||||
async def _download_booking(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
link_template: str,
|
||||
dl: DownloadToken,
|
||||
) -> None:
|
||||
async with dl as (bar, sink):
|
||||
self._write_link_content(link_template, element.url, element.name, element.description, sink)
|
||||
|
||||
async def _resolve_link_target(self, export_url: str) -> str:
|
||||
async with self.session.get(export_url, allow_redirects=False) as resp:
|
||||
@ -360,40 +451,126 @@ instance's greatest bottleneck.
|
||||
|
||||
raise CrawlError("resolve_link_target failed even after authenticating")
|
||||
|
||||
async def _download_video(self, element: IliasPageElement, element_path: PurePath) -> None:
|
||||
# Videos will NOT be redownloaded - their content doesn't really change and they are chunky
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime, redownload=Redownload.NEVER)
|
||||
if not maybe_dl:
|
||||
return
|
||||
dl = maybe_dl # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
|
||||
async def _handle_video(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> Optional[Awaitable[None]]:
|
||||
# Copy old mapping as it is likely still relevant
|
||||
if self.prev_report:
|
||||
self.report.add_custom_value(
|
||||
str(element_path),
|
||||
self.prev_report.get_custom_value(str(element_path))
|
||||
)
|
||||
|
||||
# A video might contain other videos, so let's "crawl" the video first
|
||||
# to ensure rate limits apply. This must be a download as *this token*
|
||||
# is re-used if the video consists of a single stream. In that case the
|
||||
# file name is used and *not* the stream name the ilias html parser reported
|
||||
# to ensure backwards compatibility.
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime, redownload=Redownload.ALWAYS)
|
||||
|
||||
# If we do not want to crawl it (user filter) or we have every file
|
||||
# from the cached mapping already, we can ignore this and bail
|
||||
if not maybe_dl or self._all_videos_locally_present(element_path):
|
||||
# Mark all existing cideos as known so they do not get deleted
|
||||
# during dleanup. We "downloaded" them, just without actually making
|
||||
# a network request as we assumed they did not change.
|
||||
for video in self._previous_contained_videos(element_path):
|
||||
await self.download(video)
|
||||
|
||||
return None
|
||||
|
||||
return self._download_video(element_path, element, maybe_dl)
|
||||
|
||||
def _previous_contained_videos(self, video_path: PurePath) -> List[PurePath]:
|
||||
if not self.prev_report:
|
||||
return []
|
||||
custom_value = self.prev_report.get_custom_value(str(video_path))
|
||||
if not custom_value:
|
||||
return []
|
||||
names = cast(List[str], custom_value)
|
||||
folder = video_path.parent
|
||||
return [PurePath(folder, name) for name in names]
|
||||
|
||||
def _all_videos_locally_present(self, video_path: PurePath) -> bool:
|
||||
if contained_videos := self._previous_contained_videos(video_path):
|
||||
log.explain_topic(f"Checking local cache for video {video_path.name}")
|
||||
all_found_locally = True
|
||||
for video in contained_videos:
|
||||
transformed_path = self._transformer.transform(video)
|
||||
if transformed_path:
|
||||
exists_locally = self._output_dir.resolve(transformed_path).exists()
|
||||
all_found_locally = all_found_locally and exists_locally
|
||||
if all_found_locally:
|
||||
log.explain("Found all videos locally, skipping enumeration request")
|
||||
return True
|
||||
log.explain("Missing at least one video, continuing with requests!")
|
||||
return False
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "downloading video")
|
||||
async def impl() -> None:
|
||||
assert dl # The function is only reached when dl is not None
|
||||
async def _download_video(
|
||||
self,
|
||||
original_path: PurePath,
|
||||
element: IliasPageElement,
|
||||
dl: DownloadToken
|
||||
) -> None:
|
||||
stream_elements: List[IliasPageElement] = []
|
||||
async with dl as (bar, sink):
|
||||
page = IliasPage(await self._get_page(element.url), element.url, element)
|
||||
real_element = page.get_child_elements()[0]
|
||||
stream_elements = page.get_child_elements()
|
||||
|
||||
log.explain(f"Streaming video from real url {real_element.url}")
|
||||
if len(stream_elements) > 1:
|
||||
log.explain(f"Found multiple video streams for {element.name}")
|
||||
else:
|
||||
log.explain(f"Using single video mode for {element.name}")
|
||||
stream_element = stream_elements[0]
|
||||
|
||||
await self._stream_from_url(real_element.url, sink, bar, is_video=True)
|
||||
transformed_path = self._transformer.transform(original_path)
|
||||
if not transformed_path:
|
||||
raise CrawlError(f"Download returned a path but transform did not for {original_path}")
|
||||
|
||||
await impl()
|
||||
# We do not have a local cache yet
|
||||
if self._output_dir.resolve(transformed_path).exists():
|
||||
log.explain(f"Video for {element.name} existed locally")
|
||||
else:
|
||||
await self._stream_from_url(stream_element.url, sink, bar, is_video=True)
|
||||
self.report.add_custom_value(str(original_path), [original_path.name])
|
||||
return
|
||||
|
||||
async def _download_file(self, element: IliasPageElement, element_path: PurePath) -> None:
|
||||
contained_video_paths: List[str] = []
|
||||
|
||||
for stream_element in stream_elements:
|
||||
video_path = original_path.parent / stream_element.name
|
||||
contained_video_paths.append(str(video_path))
|
||||
|
||||
maybe_dl = await self.download(video_path, mtime=element.mtime, redownload=Redownload.NEVER)
|
||||
if not maybe_dl:
|
||||
continue
|
||||
async with maybe_dl as (bar, sink):
|
||||
log.explain(f"Streaming video from real url {stream_element.url}")
|
||||
await self._stream_from_url(stream_element.url, sink, bar, is_video=True)
|
||||
|
||||
self.report.add_custom_value(str(original_path), contained_video_paths)
|
||||
|
||||
async def _handle_file(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> Optional[Awaitable[None]]:
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime)
|
||||
if not maybe_dl:
|
||||
return
|
||||
dl = maybe_dl # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
|
||||
return None
|
||||
return self._download_file(element, maybe_dl)
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "downloading file")
|
||||
async def impl() -> None:
|
||||
async def _download_file(self, element: IliasPageElement, dl: DownloadToken) -> None:
|
||||
assert dl # The function is only reached when dl is not None
|
||||
async with dl as (bar, sink):
|
||||
await self._stream_from_url(element.url, sink, bar, is_video=False)
|
||||
|
||||
await impl()
|
||||
|
||||
async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar, is_video: bool) -> None:
|
||||
async def try_stream() -> bool:
|
||||
async with self.session.get(url, allow_redirects=is_video) as resp:
|
||||
@ -443,16 +620,23 @@ instance's greatest bottleneck.
|
||||
|
||||
# We repeat this as the login method in shibboleth doesn't handle I/O errors.
|
||||
# Shibboleth is quite reliable as well, the repeat is likely not critical here.
|
||||
@_iorepeat(3, "Login")
|
||||
@_iorepeat(3, "Login", failure_is_error=True)
|
||||
async def _authenticate(self) -> None:
|
||||
await self._shibboleth_login.login(self.session)
|
||||
|
||||
@staticmethod
|
||||
def _is_logged_in(soup: BeautifulSoup) -> bool:
|
||||
# Normal ILIAS pages
|
||||
userlog = soup.find("li", {"id": "userlog"})
|
||||
if userlog is not None:
|
||||
mainbar: Optional[Tag] = soup.find(class_="il-maincontrols-metabar")
|
||||
if mainbar is not None:
|
||||
login_button = mainbar.find("button", attrs={"data-action": lambda x: x and "login.php" in x})
|
||||
shib_login = soup.find(id="button_shib_login")
|
||||
return not login_button and not shib_login
|
||||
|
||||
# Personal Desktop
|
||||
if soup.find("a", attrs={"href": lambda x: x and "block_type=pditems" in x}):
|
||||
return True
|
||||
|
||||
# Video listing embeds do not have complete ILIAS html. Try to match them by
|
||||
# their video listing table
|
||||
video_table = soup.find(
|
||||
|
164
PFERD/crawl/kit_ipd_crawler.py
Normal file
164
PFERD/crawl/kit_ipd_crawler.py
Normal file
@ -0,0 +1,164 @@
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import PurePath
|
||||
from typing import Awaitable, List, Optional, Pattern, Set, Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from ..config import Config
|
||||
from ..logging import ProgressBar, log
|
||||
from ..output_dir import FileSink
|
||||
from ..utils import soupify
|
||||
from .crawler import CrawlError
|
||||
from .http_crawler import HttpCrawler, HttpCrawlerSection
|
||||
|
||||
|
||||
class KitIpdCrawlerSection(HttpCrawlerSection):
|
||||
def target(self) -> str:
|
||||
target = self.s.get("target")
|
||||
if not target:
|
||||
self.missing_value("target")
|
||||
|
||||
if not target.startswith("https://"):
|
||||
self.invalid_value("target", target, "Should be a URL")
|
||||
|
||||
return target
|
||||
|
||||
def link_regex(self) -> Pattern[str]:
|
||||
regex = self.s.get("link_regex", r"^.*/[^/]*\.(?:pdf|zip|c|java)$")
|
||||
return re.compile(regex)
|
||||
|
||||
|
||||
@dataclass(unsafe_hash=True)
|
||||
class KitIpdFile:
|
||||
name: str
|
||||
url: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class KitIpdFolder:
|
||||
name: str
|
||||
files: List[KitIpdFile]
|
||||
|
||||
def explain(self) -> None:
|
||||
log.explain_topic(f"Folder {self.name!r}")
|
||||
for file in self.files:
|
||||
log.explain(f"File {file.name!r}")
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return self.name.__hash__()
|
||||
|
||||
|
||||
class KitIpdCrawler(HttpCrawler):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
section: KitIpdCrawlerSection,
|
||||
config: Config,
|
||||
):
|
||||
super().__init__(name, section, config)
|
||||
self._url = section.target()
|
||||
self._file_regex = section.link_regex()
|
||||
|
||||
async def _run(self) -> None:
|
||||
maybe_cl = await self.crawl(PurePath("."))
|
||||
if not maybe_cl:
|
||||
return
|
||||
|
||||
tasks: List[Awaitable[None]] = []
|
||||
|
||||
async with maybe_cl:
|
||||
for item in await self._fetch_items():
|
||||
if isinstance(item, KitIpdFolder):
|
||||
tasks.append(self._crawl_folder(item))
|
||||
else:
|
||||
# Orphan files are placed in the root folder
|
||||
tasks.append(self._download_file(PurePath("."), item))
|
||||
|
||||
await self.gather(tasks)
|
||||
|
||||
async def _crawl_folder(self, folder: KitIpdFolder) -> None:
|
||||
path = PurePath(folder.name)
|
||||
if not await self.crawl(path):
|
||||
return
|
||||
|
||||
tasks = [self._download_file(path, file) for file in folder.files]
|
||||
|
||||
await self.gather(tasks)
|
||||
|
||||
async def _download_file(self, parent: PurePath, file: KitIpdFile) -> None:
|
||||
element_path = parent / file.name
|
||||
maybe_dl = await self.download(element_path)
|
||||
if not maybe_dl:
|
||||
return
|
||||
|
||||
async with maybe_dl as (bar, sink):
|
||||
await self._stream_from_url(file.url, sink, bar)
|
||||
|
||||
async def _fetch_items(self) -> Set[Union[KitIpdFile, KitIpdFolder]]:
|
||||
page = await self.get_page()
|
||||
elements: List[Tag] = self._find_file_links(page)
|
||||
items: Set[Union[KitIpdFile, KitIpdFolder]] = set()
|
||||
|
||||
for element in elements:
|
||||
folder_label = self._find_folder_label(element)
|
||||
if folder_label:
|
||||
folder = self._extract_folder(folder_label)
|
||||
if folder not in items:
|
||||
items.add(folder)
|
||||
folder.explain()
|
||||
else:
|
||||
file = self._extract_file(element)
|
||||
items.add(file)
|
||||
log.explain_topic(f"Orphan file {file.name!r}")
|
||||
log.explain("Attributing it to root folder")
|
||||
|
||||
return items
|
||||
|
||||
def _extract_folder(self, folder_tag: Tag) -> KitIpdFolder:
|
||||
files: List[KitIpdFile] = []
|
||||
name = folder_tag.getText().strip()
|
||||
|
||||
container: Tag = folder_tag.findNextSibling(name="table")
|
||||
for link in self._find_file_links(container):
|
||||
files.append(self._extract_file(link))
|
||||
|
||||
return KitIpdFolder(name, files)
|
||||
|
||||
@staticmethod
|
||||
def _find_folder_label(file_link: Tag) -> Optional[Tag]:
|
||||
enclosing_table: Tag = file_link.findParent(name="table")
|
||||
if enclosing_table is None:
|
||||
return None
|
||||
return enclosing_table.findPreviousSibling(name=re.compile("^h[1-6]$"))
|
||||
|
||||
def _extract_file(self, link: Tag) -> KitIpdFile:
|
||||
url = self._abs_url_from_link(link)
|
||||
name = os.path.basename(url)
|
||||
return KitIpdFile(name, url)
|
||||
|
||||
def _find_file_links(self, tag: Union[Tag, BeautifulSoup]) -> List[Tag]:
|
||||
return tag.findAll(name="a", attrs={"href": self._file_regex})
|
||||
|
||||
def _abs_url_from_link(self, link_tag: Tag) -> str:
|
||||
return urljoin(self._url, link_tag.get("href"))
|
||||
|
||||
async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar) -> None:
|
||||
async with self.session.get(url, allow_redirects=False) as resp:
|
||||
if resp.status == 403:
|
||||
raise CrawlError("Received a 403. Are you within the KIT network/VPN?")
|
||||
if resp.content_length:
|
||||
bar.set_total(resp.content_length)
|
||||
|
||||
async for data in resp.content.iter_chunked(1024):
|
||||
sink.file.write(data)
|
||||
bar.advance(len(data))
|
||||
|
||||
sink.done()
|
||||
|
||||
async def get_page(self) -> BeautifulSoup:
|
||||
async with self.session.get(self._url) as request:
|
||||
return soupify(await request.read())
|
@ -5,7 +5,7 @@ from contextlib import asynccontextmanager, contextmanager
|
||||
# TODO In Python 3.9 and above, ContextManager is deprecated
|
||||
from typing import AsyncIterator, ContextManager, Iterator, List, Optional
|
||||
|
||||
from rich.console import Console, RenderGroup
|
||||
from rich.console import Console, Group
|
||||
from rich.live import Live
|
||||
from rich.markup import escape
|
||||
from rich.panel import Panel
|
||||
@ -68,7 +68,7 @@ class Log:
|
||||
if self._download_progress.task_ids:
|
||||
elements.append(self._download_progress)
|
||||
|
||||
group = RenderGroup(*elements) # type: ignore
|
||||
group = Group(*elements) # type: ignore
|
||||
self._live.update(group)
|
||||
|
||||
@contextmanager
|
||||
|
@ -231,7 +231,10 @@ class OutputDirectory:
|
||||
stat = local_path.stat()
|
||||
|
||||
remote_newer = None
|
||||
if mtime := heuristics.mtime:
|
||||
|
||||
# Python on Windows crashes when faced with timestamps around the unix epoch
|
||||
if heuristics.mtime and (os.name != "nt" or heuristics.mtime.year > 1970):
|
||||
mtime = heuristics.mtime
|
||||
remote_newer = mtime.timestamp() > stat.st_mtime
|
||||
if remote_newer:
|
||||
log.explain("Remote file seems to be newer")
|
||||
|
@ -15,13 +15,13 @@ class PferdLoadError(Exception):
|
||||
|
||||
|
||||
class Pferd:
|
||||
def __init__(self, config: Config, cli_crawlers: Optional[List[str]]):
|
||||
def __init__(self, config: Config, cli_crawlers: Optional[List[str]], cli_skips: Optional[List[str]]):
|
||||
"""
|
||||
May throw PferdLoadError.
|
||||
"""
|
||||
|
||||
self._config = config
|
||||
self._crawlers_to_run = self._find_crawlers_to_run(config, cli_crawlers)
|
||||
self._crawlers_to_run = self._find_crawlers_to_run(config, cli_crawlers, cli_skips)
|
||||
|
||||
self._authenticators: Dict[str, Authenticator] = {}
|
||||
self._crawlers: Dict[str, Crawler] = {}
|
||||
@ -65,16 +65,30 @@ class Pferd:
|
||||
|
||||
return crawlers_to_run
|
||||
|
||||
def _find_crawlers_to_run(self, config: Config, cli_crawlers: Optional[List[str]]) -> List[str]:
|
||||
def _find_crawlers_to_run(
|
||||
self,
|
||||
config: Config,
|
||||
cli_crawlers: Optional[List[str]],
|
||||
cli_skips: Optional[List[str]],
|
||||
) -> List[str]:
|
||||
log.explain_topic("Deciding which crawlers to run")
|
||||
|
||||
crawlers: List[str]
|
||||
if cli_crawlers is None:
|
||||
log.explain("No crawlers specified on CLI")
|
||||
log.explain("Running crawlers specified in config")
|
||||
return self._find_config_crawlers(config)
|
||||
crawlers = self._find_config_crawlers(config)
|
||||
else:
|
||||
log.explain("Crawlers specified on CLI")
|
||||
return self._find_cli_crawlers(config, cli_crawlers)
|
||||
crawlers = self._find_cli_crawlers(config, cli_crawlers)
|
||||
|
||||
skips = {f"crawl:{name}" for name in cli_skips} if cli_skips else set()
|
||||
for crawler in crawlers:
|
||||
if crawler in skips:
|
||||
log.explain(f"Skipping crawler {crawler!r}")
|
||||
crawlers = [crawler for crawler in crawlers if crawler not in skips]
|
||||
|
||||
return crawlers
|
||||
|
||||
def _load_authenticators(self) -> None:
|
||||
for name, section in self._config.auth_sections():
|
||||
@ -168,5 +182,13 @@ class Pferd:
|
||||
something_changed = True
|
||||
log.report(f" [bold bright_magenta]Not deleted[/] {fmt_path(path)}")
|
||||
|
||||
for warning in crawler.report.encountered_warnings:
|
||||
something_changed = True
|
||||
log.report(f" [bold bright_red]Warning[/] {warning}")
|
||||
|
||||
for error in crawler.report.encountered_errors:
|
||||
something_changed = True
|
||||
log.report(f" [bold bright_red]Error[/] {error}")
|
||||
|
||||
if not something_changed:
|
||||
log.report(" Nothing changed")
|
||||
|
@ -1,6 +1,6 @@
|
||||
import json
|
||||
from pathlib import Path, PurePath
|
||||
from typing import Any, Dict, List, Set
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
|
||||
class ReportLoadError(Exception):
|
||||
@ -68,6 +68,13 @@ class Report:
|
||||
# Files that should have been deleted by the cleanup but weren't
|
||||
self.not_deleted_files: Set[PurePath] = set()
|
||||
|
||||
# Custom crawler-specific data
|
||||
self.custom: Dict[str, Any] = dict()
|
||||
|
||||
# Encountered errors and warnings
|
||||
self.encountered_warnings: List[str] = []
|
||||
self.encountered_errors: List[str] = []
|
||||
|
||||
@staticmethod
|
||||
def _get_list_of_strs(data: Dict[str, Any], key: str) -> List[str]:
|
||||
result: Any = data.get(key, [])
|
||||
@ -81,6 +88,15 @@ class Report:
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _get_str_dictionary(data: Dict[str, Any], key: str) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = data.get(key, {})
|
||||
|
||||
if not isinstance(result, dict):
|
||||
raise ReportLoadError(f"Incorrect format: {key!r} is not a dictionary")
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "Report":
|
||||
"""
|
||||
@ -108,6 +124,9 @@ class Report:
|
||||
self.delete_file(PurePath(elem))
|
||||
for elem in self._get_list_of_strs(data, "not_deleted"):
|
||||
self.not_delete_file(PurePath(elem))
|
||||
self.custom = self._get_str_dictionary(data, "custom")
|
||||
self.encountered_errors = self._get_list_of_strs(data, "encountered_errors")
|
||||
self.encountered_warnings = self._get_list_of_strs(data, "encountered_warnings")
|
||||
|
||||
return self
|
||||
|
||||
@ -124,6 +143,9 @@ class Report:
|
||||
"changed": [str(path) for path in sorted(self.changed_files)],
|
||||
"deleted": [str(path) for path in sorted(self.deleted_files)],
|
||||
"not_deleted": [str(path) for path in sorted(self.not_deleted_files)],
|
||||
"custom": self.custom,
|
||||
"encountered_warnings": self.encountered_warnings,
|
||||
"encountered_errors": self.encountered_errors,
|
||||
}
|
||||
|
||||
with open(path, "w") as f:
|
||||
@ -190,3 +212,27 @@ class Report:
|
||||
"""
|
||||
|
||||
self.not_deleted_files.add(path)
|
||||
|
||||
def add_custom_value(self, key: str, value: Any) -> None:
|
||||
"""
|
||||
Adds a custom value under the passed key, overwriting any existing
|
||||
"""
|
||||
self.custom[key] = value
|
||||
|
||||
def get_custom_value(self, key: str) -> Optional[Any]:
|
||||
"""
|
||||
Retrieves a custom value for the given key.
|
||||
"""
|
||||
return self.custom.get(key)
|
||||
|
||||
def add_error(self, error: str) -> None:
|
||||
"""
|
||||
Adds an error to this report's error list.
|
||||
"""
|
||||
self.encountered_errors.append(error)
|
||||
|
||||
def add_warning(self, warning: str) -> None:
|
||||
"""
|
||||
Adds a warning to this report's warning list.
|
||||
"""
|
||||
self.encountered_warnings.append(warning)
|
||||
|
@ -41,9 +41,11 @@ TransformResult = Optional[Union[Transformed, Ignored]]
|
||||
@dataclass
|
||||
class Rule:
|
||||
left: str
|
||||
left_index: int
|
||||
name: str
|
||||
head: ArrowHead
|
||||
right: RightSide
|
||||
right_index: int
|
||||
|
||||
def right_result(self, path: PurePath) -> Union[str, Transformed, Ignored]:
|
||||
if isinstance(self.right, str):
|
||||
@ -345,6 +347,7 @@ def parse_eol(line: Line) -> None:
|
||||
|
||||
def parse_rule(line: Line) -> Rule:
|
||||
parse_zero_or_more_spaces(line)
|
||||
left_index = line.index
|
||||
left = parse_left(line)
|
||||
|
||||
parse_one_or_more_spaces(line)
|
||||
@ -354,19 +357,19 @@ def parse_rule(line: Line) -> Rule:
|
||||
line.expect("-")
|
||||
head = parse_arrow_head(line)
|
||||
|
||||
index = line.index
|
||||
right_index = line.index
|
||||
right: RightSide
|
||||
try:
|
||||
parse_zero_or_more_spaces(line)
|
||||
parse_eol(line)
|
||||
right = Empty()
|
||||
except RuleParseError:
|
||||
line.index = index
|
||||
line.index = right_index
|
||||
parse_one_or_more_spaces(line)
|
||||
right = parse_right(line)
|
||||
parse_eol(line)
|
||||
|
||||
return Rule(left, name, head, right)
|
||||
return Rule(left, left_index, name, head, right, right_index)
|
||||
|
||||
|
||||
def parse_transformation(line: Line) -> Transformation:
|
||||
@ -377,6 +380,9 @@ def parse_transformation(line: Line) -> Transformation:
|
||||
elif rule.name == "exact":
|
||||
return ExactTf(rule)
|
||||
elif rule.name == "name":
|
||||
if len(PurePath(rule.left).parts) > 1:
|
||||
line.index = rule.left_index
|
||||
raise RuleParseError(line, "Expected name, not multiple segments")
|
||||
return RenamingPartsTf(ExactTf(rule))
|
||||
elif rule.name == "re":
|
||||
return RenamingParentsTf(ExactReTf(rule))
|
||||
|
@ -1,2 +1,2 @@
|
||||
NAME = "PFERD"
|
||||
VERSION = "3.1.0"
|
||||
VERSION = "3.3.0"
|
||||
|
10
setup.cfg
10
setup.cfg
@ -6,11 +6,11 @@ version = attr: PFERD.version.VERSION
|
||||
packages = find:
|
||||
python_requires = >=3.8
|
||||
install_requires =
|
||||
aiohttp>=3.7.4.post0
|
||||
beautifulsoup4>=4.9.3
|
||||
rich>=10.1.0
|
||||
keyring>=23.0.1
|
||||
certifi>=2020.12.5
|
||||
aiohttp>=3.8.1
|
||||
beautifulsoup4>=4.10.0
|
||||
rich>=11.0.0
|
||||
keyring>=23.5.0
|
||||
certifi>=2021.10.8
|
||||
|
||||
[options.entry_points]
|
||||
console_scripts =
|
||||
|
Reference in New Issue
Block a user