Bump version to 3.8.3

Update changelog for 8caad00
Document usage of pilot.ilias.studium.kit.edu (#111 )
2025-07-12 14:12:30 +02:00 · 2025-07-01 14:28:30 +02:00 · 2025-07-01 14:26:10 +02:00 · 2025-05-30 17:13:45 +02:00 · 2025-05-28 17:04:57 +02:00 · 2025-05-27 16:25:59 +02:00
7 changed files with 206 additions and 81 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -22,8 +22,20 @@ ambiguous situations.

 ## Unreleased

+## 3.8.3 - 2025-07-01
+
+## Added
+- Support for link collections.  
+  In "fancy" mode, a single HTML file with multiple links is generated.
+  In all other modes, PFERD creates a folder for the collection and a new file
+  for every link inside.
+
 ## Fixed
 - Crawling of exercises with instructions
+- Don't download unavailable elements.  
+  Elements that are unavailable (for example, because their availability is
+  time restricted) will not download the HTML for the info page anymore.
+- `base_url` argument for `ilias-web` crawler causing crashes

 ## 3.8.2 - 2025-04-29

--- a/CONFIG.md
+++ b/CONFIG.md
@ -164,12 +164,13 @@ out of the box for the corresponding universities:
 [ilias-dl]: https://github.com/V3lop5/ilias-downloader/blob/main/configs "ilias-downloader configs"

 | University      | `base_url`                              | `login_type` | `client_id`   |
-|---------------|-----------------------------------------|--------------|---------------|
+|-----------------|-----------------------------------------|--------------|---------------|
 | FH Aachen       | https://www.ili.fh-aachen.de            | local        | elearning     |
 | Uni Köln        | https://www.ilias.uni-koeln.de/ilias    | local        | uk            |
 | Uni Konstanz    | https://ilias.uni-konstanz.de           | local        | ILIASKONSTANZ |
 | Uni Stuttgart   | https://ilias3.uni-stuttgart.de         | local        | Uni_Stuttgart |
 | Uni Tübingen    | https://ovidius.uni-tuebingen.de/ilias3 | shibboleth   |               |
+| KIT ILIAS Pilot | https://pilot.ilias.studium.kit.edu     | shibboleth   | pilot         |

 If your university isn't listed, try navigating to your instance's login page.
 Assuming no custom login service is used, the URL will look something like this:
--- a/PFERD/cli/command_ilias_web.py
+++ b/PFERD/cli/command_ilias_web.py
@ -45,8 +45,8 @@ def load(
    load_crawler(args, section)

    section["type"] = COMMAND_NAME
-    if args.ilias_url is not None:
-        section["base_url"] = args.ilias_url
+    if args.base_url is not None:
+        section["base_url"] = args.base_url
    if args.client_id is not None:
        section["client_id"] = args.client_id

--- a/PFERD/crawl/ilias/file_templates.py
+++ b/PFERD/crawl/ilias/file_templates.py
@ -1,3 +1,5 @@
+import dataclasses
+import re
 from enum import Enum
 from typing import Optional, cast

@ -12,7 +14,9 @@ _link_template_fancy = """
    <head>
        <meta charset="UTF-8">
        <title>ILIAS - Link: {{name}}</title>
+        <!-- REPEAT REMOVE START -->
        <meta http-equiv = "refresh" content = "{{redirect_delay}}; url = {{link}}" />
+        <!-- REPEAT REMOVE END -->
    </head>

    <style>
@ -23,6 +27,8 @@ _link_template_fancy = """
        display: flex;
        align-items: center;
        justify-content: center;
+        flex-direction: column;
+        gap: 4px;
    }
    body {
        padding: 0;
@ -31,11 +37,16 @@ _link_template_fancy = """
        font-family: "Open Sans", Verdana, Arial, Helvetica, sans-serif;
        height: 100vh;
    }
-    .row {
-        background-color: white;
+    .column {
        min-width: 500px;
        max-width: 90vw;
        display: flex;
+        flex-direction: column;
+        row-gap: 5px;
+    }
+    .row {
+        background-color: white;
+        display: flex;
        padding: 1em;
    }
    .logo {
@ -75,6 +86,8 @@ _link_template_fancy = """
    }
    </style>
    <body class="center-flex">
+        <div class="column">
+        <!-- REPEAT START -->
            <div class="row">
                <div class="logo center-flex">
                    <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24">
@ -89,6 +102,8 @@ _link_template_fancy = """
                </div>
                <div class="menu-button center-flex"> ⯆ </div>
            </div>
+        <!-- REPEAT END -->
+        </div>
    </body>
 </html>
 """.strip()  # noqa: E501 line too long
@ -255,6 +270,13 @@ def forum_thread_template(name: str, url: str, heading: bs4.Tag, content: bs4.Ta
        .replace("{{content}}", cast(str, content.prettify()))


+@dataclasses.dataclass
+class LinkData:
+    name: str
+    url: str
+    description: str
+
+
 class Links(Enum):
    IGNORE = "ignore"
    PLAINTEXT = "plaintext"
@ -272,6 +294,11 @@ class Links(Enum):
            return None
        raise ValueError("Missing switch case")

+    def collection_as_one(self) -> bool:
+        if self == Links.FANCY:
+            return True
+        return False
+
    def extension(self) -> Optional[str]:
        if self == Links.FANCY:
            return ".html"
@ -283,10 +310,48 @@ class Links(Enum):
            return None
        raise ValueError("Missing switch case")

+    def interpolate(self, redirect_delay: int, collection_name: str, links: list[LinkData]) -> str:
+        template = self.template()
+        if template is None:
+            raise ValueError("Cannot interpolate ignored links")
+
+        if len(links) == 1:
+            link = links[0]
+            content = template
+            content = content.replace("{{link}}", link.url)
+            content = content.replace("{{name}}", link.name)
+            content = content.replace("{{description}}", link.description)
+            content = content.replace("{{redirect_delay}}", str(redirect_delay))
+            return content
+        if self == Links.PLAINTEXT or self == Links.INTERNET_SHORTCUT:
+            return "\n".join(f"{link.url}" for link in links)
+
+        # All others get coerced to fancy
+        content = cast(str, Links.FANCY.template())
+        repeated_content = cast(
+            re.Match[str],
+            re.search(r"<!-- REPEAT START -->([\s\S]+)<!-- REPEAT END -->", content)
+        ).group(1)
+
+        parts = []
+        for link in links:
+            instance = repeated_content
+            instance = instance.replace("{{link}}", link.url)
+            instance = instance.replace("{{name}}", link.name)
+            instance = instance.replace("{{description}}", link.description)
+            instance = instance.replace("{{redirect_delay}}", str(redirect_delay))
+            parts.append(instance)
+
+        content = content.replace(repeated_content, "\n".join(parts))
+        content = content.replace("{{name}}", collection_name)
+        content = re.sub(r"<!-- REPEAT REMOVE START -->[\s\S]+<!-- REPEAT REMOVE END -->", "", content)
+
+        return content
+
    @staticmethod
    def from_string(string: str) -> "Links":
        try:
            return Links(string)
        except ValueError:
-            raise ValueError("must be one of 'ignore', 'plaintext',"
-                             " 'html', 'internet-shortcut'")
+            options = [f"'{option.value}'" for option in Links]
+            raise ValueError(f"must be one of {', '.join(options)}")
--- a/PFERD/crawl/ilias/ilias_web_crawler.py
+++ b/PFERD/crawl/ilias/ilias_web_crawler.py
@ -19,7 +19,7 @@ from ...utils import fmt_path, soupify, url_set_query_param
 from ..crawler import CrawlError, CrawlToken, CrawlWarning, DownloadToken, anoncritical
 from ..http_crawler import HttpCrawler, HttpCrawlerSection
 from .async_helper import _iorepeat
-from .file_templates import Links, forum_thread_template, learning_module_template
+from .file_templates import LinkData, Links, forum_thread_template, learning_module_template
 from .ilias_html_cleaner import clean, insert_base_markup
 from .kit_ilias_html import (IliasElementType, IliasForumThread, IliasLearningModulePage, IliasPage,
                             IliasPageElement, IliasSoup, _sanitize_path_name, parse_ilias_forum_export)
@ -329,6 +329,15 @@ instance's greatest bottleneck.
        # directory escape attacks.
        element_path = PurePath(parent_path, element.name)

+        # This is symptomatic of no access to the element, for example, because
+        # of time availability restrictions.
+        if "cmdClass=ilInfoScreenGUI" in element.url and "cmd=showSummary" in element.url:
+            log.explain(
+                "Skipping element as url points to info screen, "
+                "this should only happen with not-yet-released elements"
+            )
+            return None
+
        if element.type in _VIDEO_ELEMENTS:
            if not self._videos:
                log.status(
@ -437,6 +446,8 @@ instance's greatest bottleneck.
            return await self._handle_learning_module(element, element_path)
        elif element.type == IliasElementType.LINK:
            return await self._handle_link(element, element_path)
+        elif element.type == IliasElementType.LINK_COLLECTION:
+            return await self._handle_link(element, element_path)
        elif element.type == IliasElementType.BOOKING:
            return await self._handle_booking(element, element_path)
        elif element.type == IliasElementType.OPENCAST_VIDEO:
@ -462,45 +473,98 @@ instance's greatest bottleneck.
        log.explain_topic(f"Decision: Crawl Link {fmt_path(element_path)}")
        log.explain(f"Links type is {self._links}")

-        link_template_maybe = self._links.template()
-        link_extension = self._links.extension()
-        if not link_template_maybe or not link_extension:
+        export_url = url_set_query_param(element.url, "cmd", "exportHTML")
+        resolved = await self._resolve_link_target(export_url)
+        if resolved == "none":
+            links = [LinkData(element.name, "", element.description or "")]
+        else:
+            links = self._parse_link_content(element, cast(BeautifulSoup, resolved))
+
+        maybe_extension = self._links.extension()
+
+        if not maybe_extension:
            log.explain("Answer: No")
            return None
        else:
            log.explain("Answer: Yes")
-        element_path = element_path.with_name(element_path.name + link_extension)

+        if len(links) <= 1 or self._links.collection_as_one():
+            element_path = element_path.with_name(element_path.name + maybe_extension)
            maybe_dl = await self.download(element_path, mtime=element.mtime)
            if not maybe_dl:
                return None
+            return self._download_link(self._links, element.name, links, maybe_dl)

-        return self._download_link(element, link_template_maybe, maybe_dl)
+        maybe_cl = await self.crawl(element_path)
+        if not maybe_cl:
+            return None
+        # Required for download_all closure
+        cl = maybe_cl
+        extension = maybe_extension
+
+        async def download_all() -> None:
+            for link in links:
+                path = cl.path / (_sanitize_path_name(link.name) + extension)
+                if dl := await self.download(path, mtime=element.mtime):
+                    await self._download_link(self._links, element.name, [link], dl)
+
+        return download_all()

    @anoncritical
    @_iorepeat(3, "resolving link")
-    async def _download_link(self, element: IliasPageElement, link_template: str, dl: DownloadToken) -> None:
-        async with dl as (bar, sink):
-            export_url = element.url.replace("cmd=calldirectlink", "cmd=exportHTML")
-            real_url = await self._resolve_link_target(export_url)
-            self._write_link_content(link_template, real_url, element.name, element.description, sink)
-
-    def _write_link_content(
+    async def _download_link(
        self,
-        link_template: str,
-        url: str,
-        name: str,
-        description: Optional[str],
-        sink: FileSink,
+        link_renderer: Links,
+        collection_name: str,
+        links: list[LinkData],
+        dl: DownloadToken
    ) -> None:
-        content = link_template
-        content = content.replace("{{link}}", url)
-        content = content.replace("{{name}}", name)
-        content = content.replace("{{description}}", str(description))
-        content = content.replace("{{redirect_delay}}", str(self._link_file_redirect_delay))
-        sink.file.write(content.encode("utf-8"))
+        async with dl as (bar, sink):
+            rendered = link_renderer.interpolate(self._link_file_redirect_delay, collection_name, links)
+            sink.file.write(rendered.encode("utf-8"))
            sink.done()

+    async def _resolve_link_target(self, export_url: str) -> Union[BeautifulSoup, Literal['none']]:
+        async def impl() -> Optional[Union[BeautifulSoup, Literal['none']]]:
+            async with self.session.get(export_url, allow_redirects=False) as resp:
+                # No redirect means we were authenticated
+                if hdrs.LOCATION not in resp.headers:
+                    return soupify(await resp.read())  # .select_one("a").get("href").strip()  # type: ignore
+                # We are either unauthenticated or the link is not active
+                new_url = resp.headers[hdrs.LOCATION].lower()
+                if "baseclass=illinkresourcehandlergui" in new_url and "cmd=infoscreen" in new_url:
+                    return "none"
+                return None
+
+        auth_id = await self._current_auth_id()
+        target = await impl()
+        if target is not None:
+            return target
+
+        await self.authenticate(auth_id)
+
+        target = await impl()
+        if target is not None:
+            return target
+
+        raise CrawlError("resolve_link_target failed even after authenticating")
+
+    @staticmethod
+    def _parse_link_content(element: IliasPageElement, content: BeautifulSoup) -> list[LinkData]:
+        links = cast(list[Tag], list(content.select("a")))
+        if len(links) == 1:
+            url = str(links[0].get("href")).strip()
+            return [LinkData(name=element.name, description=element.description or "", url=url)]
+
+        results = []
+        for link in links:
+            url = str(link.get("href")).strip()
+            name = link.get_text(strip=True)
+            description = cast(Tag, link.find_next_sibling("dd")).get_text(strip=True)
+            results.append(LinkData(name=name, description=description, url=url.strip()))
+
+        return results
+
    async def _handle_booking(
        self,
        element: IliasPageElement,
@ -524,7 +588,7 @@ instance's greatest bottleneck.

        self._ensure_not_seen(element, element_path)

-        return self._download_booking(element, link_template_maybe, maybe_dl)
+        return self._download_booking(element, maybe_dl)

    @anoncritical
    @_iorepeat(1, "downloading description")
@ -545,36 +609,13 @@ instance's greatest bottleneck.
    async def _download_booking(
        self,
        element: IliasPageElement,
-        link_template: str,
        dl: DownloadToken,
    ) -> None:
        async with dl as (bar, sink):
-            self._write_link_content(link_template, element.url, element.name, element.description, sink)
-
-    async def _resolve_link_target(self, export_url: str) -> str:
-        async def impl() -> Optional[str]:
-            async with self.session.get(export_url, allow_redirects=False) as resp:
-                # No redirect means we were authenticated
-                if hdrs.LOCATION not in resp.headers:
-                    return soupify(await resp.read()).select_one("a").get("href").strip()  # type: ignore
-                # We are either unauthenticated or the link is not active
-                new_url = resp.headers[hdrs.LOCATION].lower()
-                if "baseclass=illinkresourcehandlergui" in new_url and "cmd=infoscreen" in new_url:
-                    return ""
-                return None
-
-        auth_id = await self._current_auth_id()
-        target = await impl()
-        if target is not None:
-            return target
-
-        await self.authenticate(auth_id)
-
-        target = await impl()
-        if target is not None:
-            return target
-
-        raise CrawlError("resolve_link_target failed even after authenticating")
+            links = [LinkData(name=element.name, description=element.description or "", url=element.url)]
+            rendered = self._links.interpolate(self._link_file_redirect_delay, element.name, links)
+            sink.file.write(rendered.encode("utf-8"))
+            sink.done()

    async def _handle_opencast_video(
        self,
--- a/PFERD/crawl/ilias/kit_ilias_html.py
+++ b/PFERD/crawl/ilias/kit_ilias_html.py
@ -109,6 +109,7 @@ class IliasElementType(Enum):
    LEARNING_MODULE_HTML = "learning_module_html"
    LITERATURE_LIST = "literature_list"
    LINK = "link"
+    LINK_COLLECTION = "link_collection"
    MEDIA_POOL = "media_pool"
    MEDIACAST_VIDEO = "mediacast_video"
    MEDIACAST_VIDEO_FOLDER = "mediacast_video_folder"
@ -202,7 +203,12 @@ class IliasElementType(Enum):
                        TypeMatcher.query("baseclass=illinkresourcehandlergui"),
                        TypeMatcher.query("calldirectlink"),
                    ),
-                    TypeMatcher.img_src("_webr.svg")
+                    TypeMatcher.img_src("_webr.svg")  # duplicated :(
+                )
+            case IliasElementType.LINK_COLLECTION:
+                return TypeMatcher.any(
+                    TypeMatcher.query("baseclass=illinkresourcehandlergui"),
+                    TypeMatcher.img_src("_webr.svg")  # duplicated :(
                )
            case IliasElementType.MEDIA_POOL:
                return TypeMatcher.any(
--- a/PFERD/version.py
+++ b/PFERD/version.py
@ -1,2 +1,2 @@
 NAME = "PFERD"
-VERSION = "3.8.2"
+VERSION = "3.8.3"
Author	SHA1	Message	Date
I-Al-Istannen	465f8b28c0	Bump version to 3.8.3	2025-07-01 14:28:30 +02:00
I-Al-Istannen	27e69af2f3	Update changelog for `8caad00`	2025-07-01 14:26:10 +02:00
Christian Schliz	56e3065950	Document usage of pilot.ilias.studium.kit.edu (#111 )	2025-05-30 17:13:45 +02:00
Mr. Pine	549ce6cce9	Ignore unavailable elements (#119 )	2025-05-28 17:04:57 +02:00
I-Al-Istannen	34564cedb4	Add support for link collections	2025-05-27 16:25:59 +02:00
I-Al-Istannen	2b0d20a1f6	Fix crawling of exercises with instructions We do not want a second path and the instruction field has an identical link...	2025-05-26 14:42:38 +02:00
PinieP	8caad0008d	Fix check for nonexistent `ilias_url` command attribute to `base_url` (#113 )	2025-05-05 22:05:54 +02:00