mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Compare commits
11 Commits
Author | SHA1 | Date | |
---|---|---|---|
52852d11a6 | |||
f94629a7fa | |||
c8ee456d33 | |||
2752e98621 | |||
1572e11da8 | |||
ea01dc7cb2 | |||
aba8d46d26 | |||
77056e6f8d | |||
064f12c14c | |||
2eb834afc3 | |||
d468a45662 |
1
.gitignore
vendored
1
.gitignore
vendored
@ -10,3 +10,4 @@ pyvenv.cfg
|
|||||||
|
|
||||||
.tmp/
|
.tmp/
|
||||||
pip-selfcheck.json
|
pip-selfcheck.json
|
||||||
|
.mypy_cache/
|
||||||
|
@ -23,16 +23,6 @@ logger = logging.getLogger(__name__)
|
|||||||
class ShibbolethAuthenticator:
|
class ShibbolethAuthenticator:
|
||||||
ILIAS_GOTO = "https://ilias.studium.kit.edu/goto.php"
|
ILIAS_GOTO = "https://ilias.studium.kit.edu/goto.php"
|
||||||
|
|
||||||
ALLOWED_CONTENT_TYPES = [
|
|
||||||
"application/pdf",
|
|
||||||
"application/zip",
|
|
||||||
"application/msword",
|
|
||||||
"text/xml",
|
|
||||||
"text/plain",
|
|
||||||
"image/jpeg",
|
|
||||||
"image/png",
|
|
||||||
]
|
|
||||||
|
|
||||||
def __init__(self, cookie_file) -> None:
|
def __init__(self, cookie_file) -> None:
|
||||||
# Because LWPCookieJar insists on the path being str-like instead of
|
# Because LWPCookieJar insists on the path being str-like instead of
|
||||||
# Path-like.
|
# Path-like.
|
||||||
@ -147,11 +137,7 @@ class ShibbolethAuthenticator:
|
|||||||
with self._session.get(url, params=params, stream=True) as r:
|
with self._session.get(url, params=params, stream=True) as r:
|
||||||
content_type = r.headers["content-type"]
|
content_type = r.headers["content-type"]
|
||||||
|
|
||||||
if content_type in self.ALLOWED_CONTENT_TYPES:
|
if content_type.startswith("text/html"):
|
||||||
# Yay, we got the file :)
|
|
||||||
stream_to_path(r, to_path)
|
|
||||||
return True
|
|
||||||
elif content_type == "text/html":
|
|
||||||
# Dangit, we're probably not logged in.
|
# Dangit, we're probably not logged in.
|
||||||
soup = bs4.BeautifulSoup(r.text, "html.parser")
|
soup = bs4.BeautifulSoup(r.text, "html.parser")
|
||||||
if self._is_logged_in(soup):
|
if self._is_logged_in(soup):
|
||||||
@ -159,9 +145,9 @@ class ShibbolethAuthenticator:
|
|||||||
"Attempting to download a web page, not a file")
|
"Attempting to download a web page, not a file")
|
||||||
return False
|
return False
|
||||||
else:
|
else:
|
||||||
# What *did* we get?
|
# Yay, we got the file :)
|
||||||
raise ContentTypeException(
|
stream_to_path(r, to_path)
|
||||||
f"Unknown file of type {content_type}")
|
return True
|
||||||
|
|
||||||
def download_file(self, file_id, to_path):
|
def download_file(self, file_id, to_path):
|
||||||
params = {"target": file_id}
|
params = {"target": file_id}
|
||||||
|
@ -4,6 +4,7 @@ import getpass
|
|||||||
import logging
|
import logging
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
import re
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
import bs4
|
import bs4
|
||||||
import requests
|
import requests
|
||||||
@ -43,7 +44,7 @@ class Ti:
|
|||||||
if filter(path):
|
if filter(path):
|
||||||
self._crawl(urlpart + address, path, orga, transform)
|
self._crawl(urlpart + address, path, orga, transform)
|
||||||
else:
|
else:
|
||||||
loggwe.info(f"Skipping {name}/")
|
logger.info(f"Skipping {name}/")
|
||||||
|
|
||||||
orga.clean_sync_dir()
|
orga.clean_sync_dir()
|
||||||
orga.clean_temp_dir()
|
orga.clean_temp_dir()
|
||||||
@ -76,7 +77,7 @@ class Ti:
|
|||||||
|
|
||||||
for filelink in soup.find_all("a", href=self.FILE_RE):
|
for filelink in soup.find_all("a", href=self.FILE_RE):
|
||||||
filepath = path / filelink["href"]
|
filepath = path / filelink["href"]
|
||||||
fileurl = url + "/" + filelink["href"]
|
fileurl = urljoin(url, filelink["href"])
|
||||||
|
|
||||||
new_path = transform(filepath)
|
new_path = transform(filepath)
|
||||||
if new_path is None:
|
if new_path is None:
|
||||||
|
@ -10,7 +10,7 @@ test it though).
|
|||||||
To install PFERD or update your installation to the latest version, run this
|
To install PFERD or update your installation to the latest version, run this
|
||||||
wherever you want to install/have installed PFERD:
|
wherever you want to install/have installed PFERD:
|
||||||
```
|
```
|
||||||
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.3
|
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.8
|
||||||
```
|
```
|
||||||
|
|
||||||
The use of [venv](https://docs.python.org/3/library/venv.html) is recommended.
|
The use of [venv](https://docs.python.org/3/library/venv.html) is recommended.
|
||||||
@ -25,7 +25,7 @@ $ mkdir Vorlesungen
|
|||||||
$ cd Vorlesungen
|
$ cd Vorlesungen
|
||||||
$ python3 -m venv .
|
$ python3 -m venv .
|
||||||
$ . bin/activate
|
$ . bin/activate
|
||||||
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.3
|
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.8
|
||||||
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/master/example_config.py
|
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/master/example_config.py
|
||||||
$ python3 example_config.py
|
$ python3 example_config.py
|
||||||
$ deactivate
|
$ deactivate
|
||||||
|
Reference in New Issue
Block a user