mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Do not set a timeout for whole HTTP request

Browse Source 
Downloads might take longer!
This commit is contained in:
I-Al-Istannen 2021-05-24 18:32:18 +02:00
parent 1739c54091
commit 9ce20216b5
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
PFERD/crawl/http_crawler.py
View File

@ -155,7 +155,15 @@ class HttpCrawler(Crawler):
async with aiohttp.ClientSession( async with aiohttp.ClientSession(
headers={"User-Agent": f"{NAME}/{VERSION}"}, headers={"User-Agent": f"{NAME}/{VERSION}"},
cookie_jar=self._cookie_jar, cookie_jar=self._cookie_jar,
timeout=ClientTimeout(total=self._http_timeout) timeout=ClientTimeout(
# 30 minutes. No download in the history of downloads was longer than 30 minutes.
# This is enough to transfer a 600 MB file over a 3 Mib/s connection.
# Allowing an arbitrary value could be annoying for overnight batch jobs
total=15 * 60,
connect=self._http_timeout,
sock_connect=self._http_timeout,
sock_read=self._http_timeout,
)
) as session: ) as session:
self.session = session self.session = session
try: try: