Compare commits

..

9 Commits

Author SHA1 Message Date
52852d11a6 Bump version to 1.1.8 2019-09-22 11:56:41 +00:00
f94629a7fa Fix exceptions with weird content types
(hopefully)
2019-09-22 11:55:47 +00:00
c8ee456d33 Bump version to 1.1.7 2019-07-26 08:14:55 +00:00
2752e98621 Fix relative url joining in ti downloader 2019-07-26 10:06:01 +02:00
1572e11da8 Bump version to 1.1.6 2019-07-05 08:49:26 +00:00
ea01dc7cb2 Allow even more types of files 2019-07-05 08:48:43 +00:00
aba8d46d26 Bump version to 1.1.5 2019-07-04 12:17:33 +00:00
77056e6f8d Allow more types of files 2019-07-04 12:16:42 +00:00
064f12c14c Ignore mypy files 2019-07-04 12:16:26 +00:00
5 changed files with 11 additions and 24 deletions

1
.gitignore vendored
View File

@ -10,3 +10,4 @@ pyvenv.cfg
.tmp/
pip-selfcheck.json
.mypy_cache/

View File

@ -23,17 +23,6 @@ logger = logging.getLogger(__name__)
class ShibbolethAuthenticator:
ILIAS_GOTO = "https://ilias.studium.kit.edu/goto.php"
ALLOWED_CONTENT_TYPES = [
"application/pdf",
"application/zip",
"application/msword",
"application/vnd.wolfram.nb",
"text/xml",
"text/plain",
"image/jpeg",
"image/png",
]
def __init__(self, cookie_file) -> None:
# Because LWPCookieJar insists on the path being str-like instead of
# Path-like.
@ -148,11 +137,7 @@ class ShibbolethAuthenticator:
with self._session.get(url, params=params, stream=True) as r:
content_type = r.headers["content-type"]
if content_type in self.ALLOWED_CONTENT_TYPES:
# Yay, we got the file :)
stream_to_path(r, to_path)
return True
elif content_type == "text/html":
if content_type.startswith("text/html"):
# Dangit, we're probably not logged in.
soup = bs4.BeautifulSoup(r.text, "html.parser")
if self._is_logged_in(soup):
@ -160,9 +145,9 @@ class ShibbolethAuthenticator:
"Attempting to download a web page, not a file")
return False
else:
# What *did* we get?
raise ContentTypeException(
f"Unknown file of type {content_type}")
# Yay, we got the file :)
stream_to_path(r, to_path)
return True
def download_file(self, file_id, to_path):
params = {"target": file_id}

View File

@ -4,6 +4,7 @@ import getpass
import logging
import pathlib
import re
from urllib.parse import urljoin
import bs4
import requests
@ -43,7 +44,7 @@ class Ti:
if filter(path):
self._crawl(urlpart + address, path, orga, transform)
else:
loggwe.info(f"Skipping {name}/")
logger.info(f"Skipping {name}/")
orga.clean_sync_dir()
orga.clean_temp_dir()
@ -76,7 +77,7 @@ class Ti:
for filelink in soup.find_all("a", href=self.FILE_RE):
filepath = path / filelink["href"]
fileurl = url + "/" + filelink["href"]
fileurl = urljoin(url, filelink["href"])
new_path = transform(filepath)
if new_path is None:

View File

@ -10,7 +10,7 @@ test it though).
To install PFERD or update your installation to the latest version, run this
wherever you want to install/have installed PFERD:
```
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.4
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.8
```
The use of [venv](https://docs.python.org/3/library/venv.html) is recommended.
@ -25,7 +25,7 @@ $ mkdir Vorlesungen
$ cd Vorlesungen
$ python3 -m venv .
$ . bin/activate
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.4
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.8
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/master/example_config.py
$ python3 example_config.py
$ deactivate

View File

@ -2,7 +2,7 @@ from setuptools import setup
setup(
name="PFERD",
version="1.1.4",
version="1.1.8",
packages=["PFERD"],
install_requires=[
"requests>=2.21.0",