hash_duplicate_delete.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hashlib\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as gridspec\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def file_hash(filepath):\n",
    "    with open(filepath, 'rb') as f:\n",
    "        return md5(f.read()).hexdigest()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_sfw = './dataset/train_set/sfw/'\n",
    "train_set_nsfw = './dataset/train_set/nsfw/'\n",
    "\n",
    "test_set_sfw = './dataset/test_set/sfw/'\n",
    "test_set_nsfw = './dataset/test_set/nsfw/'\n",
    "\n",
    "dir_list = [train_set_sfw, train_set_nsfw, test_set_sfw, test_set_nsfw]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of images in ./dataset/train_set/sfw/: 3207\n",
      "indexing...\n",
      "deleting...\n",
      "0 images has been removed from ./dataset/train_set/sfw/\n",
      "Number of images in ./dataset/train_set/nsfw/: 3059\n",
      "indexing...\n",
      "deleting...\n",
      "0 images has been removed from ./dataset/train_set/nsfw/\n",
      "Number of images in ./dataset/test_set/sfw/: 654\n",
      "indexing...\n",
      "deleting...\n",
      "0 images has been removed from ./dataset/test_set/sfw/\n",
      "Number of images in ./dataset/test_set/nsfw/: 648\n",
      "indexing...\n",
      "deleting...\n",
      ">file  27  has been deleted\n",
      ">file  296  has been deleted\n",
      ">file  303  has been deleted\n",
      "3 images has been removed from ./dataset/test_set/nsfw/\n"
     ]
    }
   ],
   "source": [
    "for target_dir in dir_list:\n",
    "    count_deleted = 0\n",
    "    os.chdir(target_dir)\n",
    "    os.getcwd()\n",
    "    file_list = os.listdir()\n",
    "    print('Number of images in ' + target_dir + ':', len(file_list))\n",
    "\n",
    "    duplicates = []\n",
    "    hash_keys = dict()\n",
    "    print('indexing...')\n",
    "    for index, filename in  enumerate(os.listdir('.')):\n",
    "        if os.path.isfile(filename):\n",
    "            with open(filename, 'rb') as f:\n",
    "                filehash = hashlib.md5(f.read()).hexdigest()\n",
    "            if filehash not in hash_keys: \n",
    "                hash_keys[filehash] = index\n",
    "            else:\n",
    "                duplicates.append((index,hash_keys[filehash]))\n",
    "\n",
    "#     print('showing images...')\n",
    "#     for file_indexes in duplicates:\n",
    "#         try:\n",
    "        \n",
    "#             plt.subplot(121),plt.imshow(imread(file_list[file_indexes[1]]))\n",
    "#             plt.title(file_indexes[1]), plt.xticks([]), plt.yticks([])\n",
    "\n",
    "#             plt.subplot(122),plt.imshow(imread(file_list[file_indexes[0]]))\n",
    "#             plt.title(str(file_indexes[0]) + ' duplicate'), plt.xticks([]), plt.yticks([])\n",
    "#             plt.show()\n",
    "        \n",
    "#         except OSError as e:\n",
    "#             continue\n",
    "\n",
    "    print('deleting...')\n",
    "    for index in duplicates:\n",
    "        os.remove(file_list[index[0]])\n",
    "        count_deleted += 1\n",
    "        print('>file ', index[0], ' has been deleted')\n",
    "\n",
    "    print(count_deleted, 'images has been removed from ' + target_dir)\n",
    "    os.chdir('../../../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}