mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Log authentication attempts in HTTP crawler

Browse Source
This commit is contained in:
I-Al-Istannen 2021-05-24 16:22:11 +02:00
parent 5c76193045
commit d44f6966c2
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
PFERD/crawl/http_crawler.py
View File

@ -67,12 +67,18 @@ class HttpCrawler(Crawler):
the request was made. This ensures that authentication is not performed needlessly.
"""
async with self._authentication_lock:
log.explain_topic("Authenticating")
# Another thread successfully called authenticate in-between
# We do not want to perform auth again, so we return here. We can
# assume the other thread suceeded as authenticate will throw an error
# if it failed and aborts the crawl process.
if caller_auth_id != self._authentication_id:
log.explain(
"Authentication skipped due to auth id mismatch."
"A previous authentication beat us to the race."
)
return
log.explain("Calling crawler-specific authenticate")
await self._authenticate()
self._authentication_id += 1
# Saving the cookies after the first auth ensures we won't need to re-authenticate