mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use file mtime in local crawler

Browse Source
This commit is contained in:
Joscha 2021-05-13 19:42:40 +02:00
parent 38bb66a776
commit 94d6a01cca
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
PFERD/crawlers/local.py
View File

@ -1,4 +1,5 @@
import asyncio import asyncio
import datetime
from pathlib import Path, PurePath from pathlib import Path, PurePath
from ..conductor import TerminalConductor from ..conductor import TerminalConductor
@ -48,12 +49,14 @@ class LocalCrawler(Crawler):
async def _crawl_file(self, path: Path, pure: PurePath) -> None: async def _crawl_file(self, path: Path, pure: PurePath) -> None:
async with self.download_bar(path) as bar: async with self.download_bar(path) as bar:
bar.set_total(path.stat().st_size) stat = path.stat()
mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
dl = await self.download(pure) dl = await self.download(pure, mtime=mtime)
if not dl: if not dl:
return return
bar.set_total(stat.st_size)
async with dl as sink: async with dl as sink:
with open(path, "rb") as f: with open(path, "rb") as f:
while True: while True: