mirrors
/
pferd
mirror of https://github.com/Garmelon/PFERD.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TODO

This commit is contained in:
Joscha 2022-11-24 15:47:01 +01:00
parent 5adfdfbd2b
commit bf27f4a686
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
PFERD/crawl/kit_ipd_crawler.py
View File

@ -155,11 +155,11 @@ class KitIpdCrawler(HttpCrawler):
sink.done()
async def _get_page(self) -> Tuple[BeautifulSoup, str]:
async with self.session.get(self._url) as request:
# The web page for Algorithmen für Routenplanung contains some
# weird comments that beautifulsoup doesn't parse correctly. This
# hack enables those pages to be crawled, and should hopefully not
# cause issues on other pages.
content = (await request.read()).decode("utf-8")
content = re.sub(r"<!--.*?-->", "", content)
return soupify(content.encode("utf-8")), str(request.url)
response = self.session.get(self._url)
# The web page for Algorithmen für Routenplanung contains some
# weird comments that beautifulsoup doesn't parse correctly. This
# hack enables those pages to be crawled, and should hopefully not
# cause issues on other pages.
content = re.sub(r"<!--.*?-->", "", response.text)
return soupify(content.encode("utf-8")), str(request.url)