mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix cookie loading and saving

Browse Source
This commit is contained in:
Joscha 2021-05-24 16:53:50 +02:00
parent 492ec6a932
commit 86ba47541b
1 changed files with 5 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
PFERD/crawl/http_crawler.py
View File

@ -38,7 +38,6 @@ class HttpCrawler(Crawler):
self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE) self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE)
self._shared_cookie_jar_paths: Optional[List[Path]] = None self._shared_cookie_jar_paths: Optional[List[Path]] = None
self._shared_auth = shared_auth self._shared_auth = shared_auth
self._current_cookie_jar: Optional[aiohttp.CookieJar] = None
self._output_dir.register_reserved(self.COOKIE_FILE) self._output_dir.register_reserved(self.COOKIE_FILE)
@ -106,6 +105,7 @@ class HttpCrawler(Crawler):
def _load_cookies(self) -> None: def _load_cookies(self) -> None:
log.explain_topic("Loading cookies") log.explain_topic("Loading cookies")
cookie_jar_path: Optional[Path] = None cookie_jar_path: Optional[Path] = None
if self._shared_cookie_jar_paths is None: if self._shared_cookie_jar_paths is None:
@ -132,32 +132,29 @@ class HttpCrawler(Crawler):
log.explain(f"Loading cookies from {fmt_real_path(cookie_jar_path)}") log.explain(f"Loading cookies from {fmt_real_path(cookie_jar_path)}")
try: try:
self._current_cookie_jar = aiohttp.CookieJar() self._cookie_jar.load(cookie_jar_path)
self._current_cookie_jar.load(cookie_jar_path)
except Exception as e: except Exception as e:
log.explain("Failed to load cookies") log.explain("Failed to load cookies")
log.explain(str(e)) log.explain(str(e))
def _save_cookies(self) -> None: def _save_cookies(self) -> None:
log.explain_topic("Saving cookies") log.explain_topic("Saving cookies")
if not self._current_cookie_jar:
log.explain("No cookie jar, save aborted")
return
try: try:
log.explain(f"Saving cookies to {fmt_real_path(self._cookie_jar_path)}") log.explain(f"Saving cookies to {fmt_real_path(self._cookie_jar_path)}")
self._current_cookie_jar.save(self._cookie_jar_path) self._cookie_jar.save(self._cookie_jar_path)
except Exception as e: except Exception as e:
log.warn(f"Failed to save cookies to {fmt_real_path(self._cookie_jar_path)}") log.warn(f"Failed to save cookies to {fmt_real_path(self._cookie_jar_path)}")
log.warn(str(e)) log.warn(str(e))
async def run(self) -> None: async def run(self) -> None:
self._request_count = 0 self._request_count = 0
self._cookie_jar = aiohttp.CookieJar()
self._load_cookies() self._load_cookies()
async with aiohttp.ClientSession( async with aiohttp.ClientSession(
headers={"User-Agent": f"{NAME}/{VERSION}"}, headers={"User-Agent": f"{NAME}/{VERSION}"},
cookie_jar=self._current_cookie_jar, cookie_jar=self._cookie_jar,
timeout=ClientTimeout(total=self._http_timeout) timeout=ClientTimeout(total=self._http_timeout)
) as session: ) as session:
self.session = session self.session = session