|
|
|
@ -26,7 +26,7 @@ filenames.sort(key=os.path.getsize, reverse=True) |
|
|
|
|
print('Deduplicating by md5 hash...') |
|
|
|
|
|
|
|
|
|
for filename in filenames: |
|
|
|
|
# hash first 65536 bytes of each file first |
|
|
|
|
# hash first 65536 bytes of each file only |
|
|
|
|
CHUNK_SIZE = 65536 |
|
|
|
|
|
|
|
|
|
hasher = hashlib.md5() |
|
|
|
@ -59,8 +59,8 @@ count = 0 |
|
|
|
|
total = len(filenames) |
|
|
|
|
|
|
|
|
|
for filename in filenames: |
|
|
|
|
print('Hashing file', count, '/', total, end='\r') |
|
|
|
|
count += 1 |
|
|
|
|
print('Hashing file', count, '/', total, end='\r') |
|
|
|
|
|
|
|
|
|
if filename in delete: continue |
|
|
|
|
|
|
|
|
@ -85,4 +85,3 @@ print('Found', len(delete), 'total duplicate files.') |
|
|
|
|
print('Deleting...') |
|
|
|
|
for dupe in delete: |
|
|
|
|
os.remove(dupe) |
|
|
|
|
|
|
|
|
|