Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds file identification and verbose statements #3

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 129 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
.python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/
62 changes: 39 additions & 23 deletions DuplicateRemover.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,62 +4,78 @@
import numpy as np

class DuplicateRemover:
def __init__(self,dirname,hash_size = 8):

extensions = ['png', 'jpg', 'jpeg', 'gif']

def __init__(self, dirname, hash_size = 8):
self.dirname = dirname
self.hash_size = hash_size

def find_duplicates(self):
def find_duplicates(self, verbose=True):
"""
Find and Delete Duplicates
"""

fnames = os.listdir(self.dirname)
hashes = {}
duplicates = []
print("Finding Duplicates Now!\n")
if verbose:
print("Finding Duplicates Now!\n")
for image in fnames:
with Image.open(os.path.join(self.dirname,image)) as img:
temp_hash = imagehash.average_hash(img, self.hash_size)
if temp_hash in hashes:
print("Duplicate {} \nfound for Image {}!\n".format(image,hashes[temp_hash]))
duplicates.append(image)
else:
hashes[temp_hash] = image
if list(image.lower().split('.'))[-1] in self.extensions:
with Image.open(os.path.join(self.dirname,image)) as img:
temp_hash = imagehash.average_hash(img, self.hash_size)
if temp_hash in hashes:
if verbose:
print("Duplicate {} \nfound for Image {}!\n".format(image,hashes[temp_hash]))
duplicates.append(image)
else:
hashes[temp_hash] = image

if len(duplicates) != 0:
a = input("Do you want to delete these {} Images? Press Y or N: ".format(len(duplicates)))
if verbose:
a = input("Do you want to delete these {} Images? Press Y or N: ".format(len(duplicates)))
else:
a = 'y'
space_saved = 0
if(a.strip().lower() == "y"):
for duplicate in duplicates:
space_saved += os.path.getsize(os.path.join(self.dirname,duplicate))

os.remove(os.path.join(self.dirname,duplicate))
print("{} Deleted Succesfully!".format(duplicate))

print("\n\nYou saved {} mb of Space!".format(round(space_saved/1000000),2))
if verbose:
print("{} Deleted Succesfully!".format(duplicate))

if verbose:
print("\n\nYou saved {} mb of Space!".format(round(space_saved/1000000),2))
else:
print("Thank you for Using Duplicate Remover")
if verbose:
print("Thank you for Using Duplicate Remover")
else:
print("No Duplicates Found :(")
if verbose:
print("No Duplicates Found :(")




def find_similar(self,location,similarity=80):
def find_similar(self, location, similarity=80, verbose=True):
fnames = os.listdir(self.dirname)
threshold = 1 - similarity/100
diff_limit = int(threshold*(self.hash_size**2))

with Image.open(location) as img:
hash1 = imagehash.average_hash(img, self.hash_size).hash

print("Finding Similar Images to {} Now!\n".format(location))
if verbose:
print("Finding Similar Images to {} Now!\n".format(location))
for image in fnames:
with Image.open(os.path.join(self.dirname,image)) as img:
hash2 = imagehash.average_hash(img, self.hash_size).hash

if np.count_nonzero(hash1 != hash2) <= diff_limit:
print("{} image found {}% similar to {}".format(image,similarity,location))
if list(image.lower().split('.'))[-1] in self.extensions:
with Image.open(os.path.join(self.dirname,image)) as img:
hash2 = imagehash.average_hash(img, self.hash_size).hash

if np.count_nonzero(hash1 != hash2) <= diff_limit:
if verbose:
print("{} image found {}% similar to {}".format(image,similarity,location))



Expand Down
9 changes: 6 additions & 3 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
pillow
imagehash
numpy
ImageHash==4.2.0
numpy==1.20.2
Pillow==8.1.2
PyWavelets==1.1.1
scipy==1.6.2
six==1.15.0