Compare commits

...

6 Commits

Author SHA1 Message Date
5de68a0400 Bump version 2020-12-30 17:20:30 +01:00
f0562049b6 Remove Python 3.9 method in crawler 2020-12-30 17:18:04 +01:00
0e1077bb50 Bump version 2020-12-30 14:50:49 +01:00
c978e9edf4 Resolve a few pylint warnings 2020-12-30 14:45:46 +01:00
2714ac6be6 Send CSRF token to Shibboleth 2020-12-30 14:34:11 +01:00
9b048a9cfc Canonize meeting names to a properly formatted date 2020-12-30 14:32:59 +01:00
7 changed files with 37 additions and 10 deletions

View File

@ -14,7 +14,7 @@ PRETTY = PrettyLogger(LOGGER)
try:
import keyring
except ImportError:
PRETTY.warning("Keyring module not found, KeyringAuthenticator won't work!")
pass
class TfaAuthenticator:

View File

@ -74,6 +74,8 @@ class KitShibbolethAuthenticator(IliasAuthenticator):
form = soup.find("form", {"class": "full content", "method": "post"})
action = form["action"]
csrf_token = form.find("input", {"name": "csrf_token"})["value"]
# Equivalent: Enter credentials in
# https://idp.scc.kit.edu/idp/profile/SAML2/Redirect/SSO
LOGGER.debug("Attempt to log in to Shibboleth using credentials")
@ -82,6 +84,7 @@ class KitShibbolethAuthenticator(IliasAuthenticator):
"_eventId_proceed": "",
"j_username": self._auth.username,
"j_password": self._auth.password,
"csrf_token": csrf_token
}
soup = soupify(sess.post(url, data=data))

View File

@ -40,6 +40,7 @@ class IliasElementType(Enum):
REGULAR_FILE = "REGULAR_FILE"
VIDEO_FILE = "VIDEO_FILE"
FORUM = "FORUM"
MEETING = "MEETING"
EXTERNAL_LINK = "EXTERNAL_LINK"
def is_folder(self) -> bool:
@ -241,6 +242,8 @@ class IliasCrawler:
entries_to_process += self._crawl_video_directory(entry.path, url)
continue
PRETTY.warning(f"Unknown type: {entry.entry_type}!")
return result
def _crawl_folder(self, folder_path: Path, url: str) -> List[IliasCrawlerEntry]:
@ -269,6 +272,25 @@ class IliasCrawler:
if element_type == IliasElementType.REGULAR_FILE:
result += self._crawl_file(folder_path, link, abs_url)
elif element_type == IliasElementType.MEETING:
meeting_name = str(element_path.name)
date_portion_str = meeting_name.split(" - ")[0]
date_portion = demangle_date(date_portion_str)
if not date_portion:
result += [IliasCrawlerEntry(element_path, abs_url, element_type, None)]
continue
rest_of_name = meeting_name
if rest_of_name.startswith(date_portion_str):
rest_of_name = rest_of_name[len(date_portion_str):]
new_name = datetime.datetime.strftime(date_portion, "%Y-%m-%d, %H:%M") \
+ rest_of_name
new_path = Path(folder_path, _sanitize_path_name(new_name))
result += [
IliasCrawlerEntry(new_path, abs_url, IliasElementType.REGULAR_FOLDER, None)
]
elif element_type is not None:
result += [IliasCrawlerEntry(element_path, abs_url, element_type, None)]
else:
@ -320,6 +342,8 @@ class IliasCrawler:
"""
# pylint: disable=too-many-return-statements
found_parent: Optional[bs4.Tag] = None
# We look for the outer div of our inner link, to find information around it
# (mostly the icon)
for parent in link_element.parents:
@ -350,6 +374,9 @@ class IliasCrawler:
if str(img_tag["src"]).endswith("frm.svg"):
return IliasElementType.FORUM
if str(img_tag["src"]).endswith("sess.svg"):
return IliasElementType.MEETING
return IliasElementType.REGULAR_FOLDER
@staticmethod

View File

@ -3,13 +3,10 @@ Contains a few logger utility functions and implementations.
"""
import logging
from pathlib import Path
from typing import List, Optional
from typing import Optional
from rich import print as rich_print
from rich._log_render import LogRender
from rich.console import Console
from rich.panel import Panel
from rich.style import Style
from rich.text import Text
from rich.theme import Theme

View File

@ -116,7 +116,7 @@ class Organizer(Location):
if self._is_marked(dst):
PRETTY.warning(f"File {str(dst_absolute)!r} was already written!")
conflict = ConflictType.MARKED_FILE_OVERWRITTEN
if self._resolve_conflict(f"Overwrite file?", dst_absolute, conflict, default=False):
if self._resolve_conflict("Overwrite file?", dst_absolute, conflict, default=False):
PRETTY.ignored_file(dst_absolute, "file was written previously")
return None

View File

@ -37,7 +37,7 @@ Ensure that you have at least Python 3.8 installed.
To install PFERD or update your installation to the latest version, run this
wherever you want to install or have already installed PFERD:
```
$ pip install git+https://github.com/Garmelon/PFERD@v2.5.0
$ pip install git+https://github.com/Garmelon/PFERD@v2.5.2
```
The use of [venv] is recommended.
@ -60,8 +60,8 @@ $ mkdir Vorlesungen
$ cd Vorlesungen
$ python3 -m venv .venv
$ .venv/bin/activate
$ pip install git+https://github.com/Garmelon/PFERD@v2.5.0
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.0/example_config.py
$ pip install git+https://github.com/Garmelon/PFERD@v2.5.2
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.2/example_config.py
$ python3 example_config.py
$ deactivate
```

View File

@ -2,7 +2,7 @@ from setuptools import find_packages, setup
setup(
name="PFERD",
version="2.5.0",
version="2.5.2",
packages=find_packages(),
install_requires=[
"requests>=2.21.0",