Log authentication attempts in HTTP crawler

This commit is contained in:
I-Al-Istannen 2021-05-24 16:22:11 +02:00
parent 5c76193045
commit d44f6966c2

View File

@ -67,12 +67,18 @@ class HttpCrawler(Crawler):
the request was made. This ensures that authentication is not performed needlessly. the request was made. This ensures that authentication is not performed needlessly.
""" """
async with self._authentication_lock: async with self._authentication_lock:
log.explain_topic("Authenticating")
# Another thread successfully called authenticate in-between # Another thread successfully called authenticate in-between
# We do not want to perform auth again, so we return here. We can # We do not want to perform auth again, so we return here. We can
# assume the other thread suceeded as authenticate will throw an error # assume the other thread suceeded as authenticate will throw an error
# if it failed and aborts the crawl process. # if it failed and aborts the crawl process.
if caller_auth_id != self._authentication_id: if caller_auth_id != self._authentication_id:
log.explain(
"Authentication skipped due to auth id mismatch."
"A previous authentication beat us to the race."
)
return return
log.explain("Calling crawler-specific authenticate")
await self._authenticate() await self._authenticate()
self._authentication_id += 1 self._authentication_id += 1
# Saving the cookies after the first auth ensures we won't need to re-authenticate # Saving the cookies after the first auth ensures we won't need to re-authenticate