Compare commits

...

11 Commits

Author SHA1 Message Date
52852d11a6 Bump version to 1.1.8 2019-09-22 11:56:41 +00:00
f94629a7fa Fix exceptions with weird content types
(hopefully)
2019-09-22 11:55:47 +00:00
c8ee456d33 Bump version to 1.1.7 2019-07-26 08:14:55 +00:00
2752e98621 Fix relative url joining in ti downloader 2019-07-26 10:06:01 +02:00
1572e11da8 Bump version to 1.1.6 2019-07-05 08:49:26 +00:00
ea01dc7cb2 Allow even more types of files 2019-07-05 08:48:43 +00:00
aba8d46d26 Bump version to 1.1.5 2019-07-04 12:17:33 +00:00
77056e6f8d Allow more types of files 2019-07-04 12:16:42 +00:00
064f12c14c Ignore mypy files 2019-07-04 12:16:26 +00:00
2eb834afc3 Bump version to 1.1.4 2019-06-11 12:46:40 +00:00
d468a45662 Allow wolfram files 2019-06-11 12:42:55 +00:00
5 changed files with 11 additions and 23 deletions

1
.gitignore vendored
View File

@ -10,3 +10,4 @@ pyvenv.cfg
.tmp/ .tmp/
pip-selfcheck.json pip-selfcheck.json
.mypy_cache/

View File

@ -23,16 +23,6 @@ logger = logging.getLogger(__name__)
class ShibbolethAuthenticator: class ShibbolethAuthenticator:
ILIAS_GOTO = "https://ilias.studium.kit.edu/goto.php" ILIAS_GOTO = "https://ilias.studium.kit.edu/goto.php"
ALLOWED_CONTENT_TYPES = [
"application/pdf",
"application/zip",
"application/msword",
"text/xml",
"text/plain",
"image/jpeg",
"image/png",
]
def __init__(self, cookie_file) -> None: def __init__(self, cookie_file) -> None:
# Because LWPCookieJar insists on the path being str-like instead of # Because LWPCookieJar insists on the path being str-like instead of
# Path-like. # Path-like.
@ -147,11 +137,7 @@ class ShibbolethAuthenticator:
with self._session.get(url, params=params, stream=True) as r: with self._session.get(url, params=params, stream=True) as r:
content_type = r.headers["content-type"] content_type = r.headers["content-type"]
if content_type in self.ALLOWED_CONTENT_TYPES: if content_type.startswith("text/html"):
# Yay, we got the file :)
stream_to_path(r, to_path)
return True
elif content_type == "text/html":
# Dangit, we're probably not logged in. # Dangit, we're probably not logged in.
soup = bs4.BeautifulSoup(r.text, "html.parser") soup = bs4.BeautifulSoup(r.text, "html.parser")
if self._is_logged_in(soup): if self._is_logged_in(soup):
@ -159,9 +145,9 @@ class ShibbolethAuthenticator:
"Attempting to download a web page, not a file") "Attempting to download a web page, not a file")
return False return False
else: else:
# What *did* we get? # Yay, we got the file :)
raise ContentTypeException( stream_to_path(r, to_path)
f"Unknown file of type {content_type}") return True
def download_file(self, file_id, to_path): def download_file(self, file_id, to_path):
params = {"target": file_id} params = {"target": file_id}

View File

@ -4,6 +4,7 @@ import getpass
import logging import logging
import pathlib import pathlib
import re import re
from urllib.parse import urljoin
import bs4 import bs4
import requests import requests
@ -43,7 +44,7 @@ class Ti:
if filter(path): if filter(path):
self._crawl(urlpart + address, path, orga, transform) self._crawl(urlpart + address, path, orga, transform)
else: else:
loggwe.info(f"Skipping {name}/") logger.info(f"Skipping {name}/")
orga.clean_sync_dir() orga.clean_sync_dir()
orga.clean_temp_dir() orga.clean_temp_dir()
@ -76,7 +77,7 @@ class Ti:
for filelink in soup.find_all("a", href=self.FILE_RE): for filelink in soup.find_all("a", href=self.FILE_RE):
filepath = path / filelink["href"] filepath = path / filelink["href"]
fileurl = url + "/" + filelink["href"] fileurl = urljoin(url, filelink["href"])
new_path = transform(filepath) new_path = transform(filepath)
if new_path is None: if new_path is None:

View File

@ -10,7 +10,7 @@ test it though).
To install PFERD or update your installation to the latest version, run this To install PFERD or update your installation to the latest version, run this
wherever you want to install/have installed PFERD: wherever you want to install/have installed PFERD:
``` ```
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.3 $ pip install git+https://github.com/Garmelon/PFERD@v1.1.8
``` ```
The use of [venv](https://docs.python.org/3/library/venv.html) is recommended. The use of [venv](https://docs.python.org/3/library/venv.html) is recommended.
@ -25,7 +25,7 @@ $ mkdir Vorlesungen
$ cd Vorlesungen $ cd Vorlesungen
$ python3 -m venv . $ python3 -m venv .
$ . bin/activate $ . bin/activate
$ pip install git+https://github.com/Garmelon/PFERD@v1.1.3 $ pip install git+https://github.com/Garmelon/PFERD@v1.1.8
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/master/example_config.py $ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/master/example_config.py
$ python3 example_config.py $ python3 example_config.py
$ deactivate $ deactivate

View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="PFERD", name="PFERD",
version="1.1.3", version="1.1.8",
packages=["PFERD"], packages=["PFERD"],
install_requires=[ install_requires=[
"requests>=2.21.0", "requests>=2.21.0",