Deduplicate path names in crawler

Also rename files so they follow the restrictions for windows file names if
we're on windows.
This commit is contained in:
Joscha
2021-05-25 11:58:01 +02:00
parent c21ddf225b
commit bce3dc384d
6 changed files with 114 additions and 2 deletions

View File

@ -114,6 +114,9 @@ class Report:
f.write("\n") # json.dump doesn't do this
def mark_reserved(self, path: PurePath) -> None:
if path in self.marked:
raise RuntimeError("Trying to reserve an already reserved file")
self.reserved_files.add(path)
def mark(self, path: PurePath) -> None: