Deduplicate path names in crawler

Also rename files so they follow the restrictions for windows file names if
we're on windows.
This commit is contained in:
Joscha
2021-05-25 11:58:01 +02:00
parent c21ddf225b
commit bce3dc384d
6 changed files with 114 additions and 2 deletions

View File

@@ -80,7 +80,7 @@ class LocalCrawler(Crawler):
))
for child in path.iterdir():
pure_child = pure / child.name
pure_child = cl.path / child.name
tasks.append(self._crawl_path(child, pure_child))
await self.gather(tasks)