89 lines
2.1 KiB
Python
89 lines
2.1 KiB
Python
|
import glob
|
||
|
|
import os
|
||
|
|
import hashlib
|
||
|
|
from PIL import Image, UnidentifiedImageError
|
||
|
|
|
||
|
|
from imagehash import average_hash
|
||
|
|
from videohash import VideoHash
|
||
|
|
from videohash.exceptions import FFmpegFailedToExtractFrames
|
||
|
|
|
||
|
|
hashes = set()
|
||
|
|
delete = set()
|
||
|
|
|
||
|
|
filenames = [x for x in glob.glob('**', recursive=True) if os.path.isfile(x)]
|
||
|
|
|
||
|
|
print('Found', len(filenames), 'files in', os.getcwd() + '. Deduplicate them?')
|
||
|
|
print('ENTER to continue, ctrl+c to cancel.')
|
||
|
|
try:
|
||
|
|
input()
|
||
|
|
except KeyboardInterrupt:
|
||
|
|
print('\nCancelled.')
|
||
|
|
os._exit(0)
|
||
|
|
|
||
|
|
print('Sorting file list by size...')
|
||
|
|
filenames.sort(key=os.path.getsize, reverse=True)
|
||
|
|
|
||
|
|
print('Deduplicating by md5 hash...')
|
||
|
|
|
||
|
|
for filename in filenames:
|
||
|
|
# hash first 65536 bytes of each file first
|
||
|
|
CHUNK_SIZE = 65536
|
||
|
|
|
||
|
|
hasher = hashlib.md5()
|
||
|
|
with open(filename, 'rb') as file:
|
||
|
|
buf = file.read(CHUNK_SIZE)
|
||
|
|
hasher.update(buf)
|
||
|
|
digest = hasher.hexdigest()
|
||
|
|
if digest in hashes:
|
||
|
|
delete.add(filename)
|
||
|
|
else:
|
||
|
|
hashes.add(digest)
|
||
|
|
|
||
|
|
print('Found', len(delete), 'duplicates by md5 hash.')
|
||
|
|
print('Deduplicating by media fingerprint...')
|
||
|
|
|
||
|
|
def get_image_hash(filename):
|
||
|
|
try:
|
||
|
|
image = Image.open(filename)
|
||
|
|
return str(average_hash(image)) + '_image'
|
||
|
|
except UnidentifiedImageError:
|
||
|
|
return None
|
||
|
|
|
||
|
|
def get_video_hash(filename):
|
||
|
|
try:
|
||
|
|
return str(VideoHash(path=filename).hash_hex) + '_video'
|
||
|
|
except FFmpegFailedToExtractFrames:
|
||
|
|
return None
|
||
|
|
|
||
|
|
count = 0
|
||
|
|
total = len(filenames)
|
||
|
|
|
||
|
|
for filename in filenames:
|
||
|
|
print('Hashing file', count, '/', total, end='\r')
|
||
|
|
count += 1
|
||
|
|
|
||
|
|
if filename in delete: continue
|
||
|
|
|
||
|
|
try:
|
||
|
|
digest = get_image_hash(filename) or get_video_hash(filename)
|
||
|
|
except BaseException as e:
|
||
|
|
print()
|
||
|
|
print('Exception', e.__class__.__name__, str(e), 'while hashing:')
|
||
|
|
print(filename)
|
||
|
|
continue
|
||
|
|
|
||
|
|
if not digest: continue
|
||
|
|
|
||
|
|
if digest in hashes:
|
||
|
|
delete.add(filename)
|
||
|
|
else:
|
||
|
|
hashes.add(digest)
|
||
|
|
|
||
|
|
print()
|
||
|
|
print('Found', len(delete), 'total duplicate files.')
|
||
|
|
|
||
|
|
print('Deleting...')
|
||
|
|
for dupe in delete:
|
||
|
|
os.remove(dupe)
|
||
|
|
|