scripts/retrieve_data_clean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rubencito/micromamba/envs/colorization/lib/python3.11/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: 'dlopen(/Users/rubencito/micromamba/envs/colorization/lib/python3.11/site-packages/torchvision/image.so, 0x0006): Symbol not found: __ZN3c1017RegisterOperatorsD1Ev\n",
      "  Referenced from: <103AD477-408D-3C2F-84B8-B7B8FFF88455> /Users/rubencito/micromamba/envs/colorization/lib/python3.11/site-packages/torchvision/image.so\n",
      "  Expected in:     <D1E1EC61-A5AA-3BBA-94B8-043C751C8A50> /Users/rubencito/micromamba/envs/colorization/lib/python3.11/site-packages/torch/lib/libtorch_cpu.dylib'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?\n",
      "  warn(\n"
     ]
    }
   ],
   "source": [
    "# %matplotlib inline\n",
    "# %config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from glob import glob\n",
    "import os\n",
    "from copy import deepcopy\n",
    "\n",
    "import numpy as np\n",
    "import cv2\n",
    "from PIL import Image\n",
    "from time import sleep\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "# import albumentations as A\n",
    "# from albumentations.pytorch import ToTensorV2\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision.transforms import v2\n",
    "from torchvision import transforms\n",
    "\n",
    "# Lorenz's libs\n",
    "import math\n",
    "import pandas as pd\n",
    "import requests\n",
    "from io import BytesIO\n",
    "from pyproj import Proj, Transformer\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "import folium\n",
    "from folium.plugins import MarkerCluster\n",
    "\n",
    "import concurrent.futures\n",
    "\n",
    "import cProfile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define helper functions/classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SwisstopoTileFetcher Class**\n",
    "\n",
    "This class facilitates fetching map tiles from the Swisstopo WMTS service. It converts geographic coordinates (latitude and longitude) into tile indices, constructs the appropriate URL for the tile image, and downloads the image. The class also provides a method to display the fetched tile image using matplotlib.\n",
    "\n",
    "Key Methods:\n",
    "\n",
    "*   **lat_lon_to_tile_indices():** Converts latitude and longitude to tile indices based on the zoom level.\n",
    "*   **fetch_tile():** Downloads the tile image from Swisstopo.\n",
    "*   **show_tile():** Displays the fetched tile image.\n",
    "\n",
    "Parameters:\n",
    "\n",
    "\n",
    "\n",
    "*   **longitude:** The longitude of the point for which the tile is to be fetched.\n",
    "*   **latitude:** The latitude of the point for which the tile is to be fetched.\n",
    "*   **zoom_level:** The zoom level for the map tile."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SwisstopoTileFetcher:\n",
    "    def __init__(self, longitude, latitude, zoom_level):\n",
    "        self.scheme = \"https\"\n",
    "        self.server_name = \"wmts0.geo.admin.ch\"  # Can be wmts0 to wmts9\n",
    "        self.version = \"1.0.0\"\n",
    "        self.layer_name = \"ch.swisstopo.swissimage\"\n",
    "        self.style_name = \"default\"\n",
    "        self.time = \"current\"\n",
    "        self.tile_matrix_set = \"3857\"\n",
    "        self.format_extension = \"jpeg\"\n",
    "        self.longitude = longitude\n",
    "        self.latitude = latitude\n",
    "        self.zoom_level = zoom_level\n",
    "\n",
    "    def lat_lon_to_tile_indices(self):\n",
    "        n = 2 ** self.zoom_level\n",
    "        lat_rad = math.radians(self.latitude)\n",
    "        x_tile = int((self.longitude + 180.0) / 360.0 * n)\n",
    "        y_tile = int((1.0 - math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) / math.pi) / 2.0 * n)\n",
    "        return x_tile, y_tile\n",
    "\n",
    "    def fetch_tile(self):\n",
    "        # Convert coordinates to tile indices\n",
    "        x, y = self.lat_lon_to_tile_indices()\n",
    "\n",
    "        # Construct the URL\n",
    "        url = f\"{self.scheme}://{self.server_name}/{self.version}/{self.layer_name}/{self.style_name}/{self.time}/{self.tile_matrix_set}/{self.zoom_level}/{x}/{y}.{self.format_extension}\"\n",
    "\n",
    "        # Download the tile\n",
    "\n",
    "        with requests.Session() as session:\n",
    "            with session.get(url) as response:\n",
    "                if response.status_code == 200:\n",
    "                    image = Image.open(BytesIO(response.content))\n",
    "                    return image, url\n",
    "                else:\n",
    "                    print(f\"Failed to download tile. Status code: {response.status_code}\")\n",
    "                    return None\n",
    "\n",
    "    def show_tile(self):\n",
    "        image = self.fetch_tile()\n",
    "        if image:\n",
    "            # Display the image\n",
    "            plt.imshow(image)\n",
    "            plt.axis('off')  # Hide the axis\n",
    "            plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**ArealstatistikSampler Class**\n",
    "\n",
    "This class is designed to sample geographic points from a dataset provided in LV95 coordinates and convert them to WGS84 coordinates. It reads the CSV file*, filters the data based on a specified column, and randomly selects a given number of points from each unique value in that column. The selected points are then transformed from the LV95 coordinate system to the WGS84 coordinate system.\n",
    "\n",
    "Key Methods:\n",
    "\n",
    "\n",
    "*   **lv95_to_wgs84(lon, lat):** Converts coordinates from LV95 to WGS84.\n",
    "*   **sample_points():** Samples the specified number of points for each unique value in the specified column, converts their coordinates, and returns a list of these points.\n",
    "\n",
    "\n",
    "Parameters:\n",
    "\n",
    "\n",
    "\n",
    "*   **file_path:** Path to the CSV file containing the data.\n",
    "*   **column_to_filter:** Column name used to filter and categorize the data.\n",
    "*   **num_samples:** Number of samples to select for each unique value in the column.\n",
    "*   **random_state:** Optional parameter to ensure reproducibility of random sampling.\n",
    "\n",
    "*available on https://www.bfs.admin.ch/bfs/en/home/services/geostat/swiss-federal-statistics-geodata/land-use-cover-suitability/swiss-land-use-statistics.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ArealstatistikSampler:\n",
    "    def __init__(self, file_path, column_to_filter, num_samples, random_state=None):\n",
    "        self.file_path = file_path\n",
    "        self.column_to_filter = column_to_filter\n",
    "        self.num_samples = num_samples\n",
    "        self.random_state = random_state\n",
    "\n",
    "    def lv95_to_wgs84(self, lon, lat):\n",
    "        in_proj = Proj(\"epsg:2056\")\n",
    "        out_proj = Proj(\"epsg:4326\")\n",
    "        transformer = Transformer.from_proj(in_proj, out_proj)\n",
    "        lon_wgs84, lat_wgs84 = transformer.transform(lon, lat)\n",
    "        return lon_wgs84, lat_wgs84\n",
    "\n",
    "    def sample_points(self):\n",
    "        # Read the CSV file into a DataFrame\n",
    "        df = pd.read_csv(self.file_path, delimiter=\";\")\n",
    "\n",
    "        # Filter out rows with missing values in the specified column\n",
    "        df_filtered = df.dropna(subset=[self.column_to_filter])\n",
    "\n",
    "        # Create an empty list to store the selected points\n",
    "        selected_points = []\n",
    "\n",
    "        # Set random state if provided\n",
    "        if self.random_state is not None:\n",
    "            random_state = self.random_state\n",
    "        else:\n",
    "            random_state = 42  # Default random state\n",
    "            \n",
    "        n_classes = df_filtered[self.column_to_filter].unique()\n",
    "\n",
    "        # Iterate over each unique value in the specified column\n",
    "        for class_value in n_classes:\n",
    "            # Filter rows for the current class value\n",
    "            class_df = df_filtered[df_filtered[self.column_to_filter] == class_value]\n",
    "\n",
    "            # Randomly select specified number of examples for the current class value\n",
    "            selected_samples = class_df.sample(n=self.num_samples, random_state=random_state)\n",
    "\n",
    "            # Convert LV95 coordinates to WGS84 and store them in the selected_points list\n",
    "            for _, row in selected_samples.iterrows():\n",
    "                lon_wgs84, lat_wgs84 = self.lv95_to_wgs84(row[\"E_COORD\"], row[\"N_COORD\"])\n",
    "                selected_points.append([lon_wgs84, lat_wgs84, class_value])\n",
    "\n",
    "        return selected_points\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file_path = \"/content/drive/MyDrive/CAS Avanced Machine Learning/Luftbild_Colarization/ag-b-00.03-37-area-csv.csv\"\n",
    "file_path = \"/Volumes/Ruben/datasets/land_use_data/ag-b-00.03-37-area-csv.csv\"\n",
    "# column_to_filter = \"AS18_72\" #column in the dataset with the classes\n",
    "column_to_filter = \"AS18_17\" #column in the dataset with the classes (less classe : 17)\n",
    "num_samples = 180 #number of samples per class, NOTE: I found out that the lower number of samples from class is 461, so larger number than this will give an error!\n",
    "random_state = 42\n",
    "zoom_levels = [16, 17, 18] #zoom levels to fetch images from randomly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Collect sample points and show the spatial distribution on a map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples collected: 12960\n"
     ]
    }
   ],
   "source": [
    "# Instantiate ArealstatistikSampler and sample points\n",
    "sampler = ArealstatistikSampler(file_path, column_to_filter, num_samples, random_state)\n",
    "coordinates = sampler.sample_points()\n",
    "\n",
    "# Print the number of samples collected\n",
    "print(\"Number of samples collected:\", len(coordinates))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try to make a faster fetcher class with concurrence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_images_with_random_zoom_levels_faster(point, indx, zoom_levels, save_to = None):\n",
    "    \"\"\"\n",
    "    Fetch images for sampled points using random zoom levels.\n",
    "\n",
    "    Args:\n",
    "        sampled_points (list): List of sampled points, where each point is represented as a list [lat, lon] or [lat, lon, class_value].\n",
    "        zoom_levels (list): List of zoom levels to choose from.\n",
    "        save_to (str): if provided, valid path where to save the  fetched image.\n",
    "\n",
    "    Returns:\n",
    "        list: List of dictionaries, each containing fetched image and its metadata (lat, lon, zoom_level, class).\n",
    "    \"\"\"\n",
    "\n",
    "    lat, lon = point[:2]  # Extract latitude and longitude\n",
    "    class_value = point[2] if len(point) > 2 else None\n",
    "    zoom_level = random.choice(zoom_levels)\n",
    "    tile_fetcher = SwisstopoTileFetcher(lon, lat, zoom_level)\n",
    "    image, url = tile_fetcher.fetch_tile()\n",
    "    sleep(0.15)\n",
    "    image_data = {\n",
    "        'img_id': indx,\n",
    "        'img_name': '_'.join(url.split(\"/\")[-4:]).split(\".\")[0],\n",
    "        'image': image,\n",
    "        'latitude': lat,\n",
    "        'longitude': lon,\n",
    "        'zoom_level': zoom_level,\n",
    "        'class': class_value,\n",
    "        'link':url, \n",
    "    }\n",
    "\n",
    "    if save_to:\n",
    "        assert isinstance(save_to, str), \"Path must be a valid string.\"\n",
    "        assert os.path.exists(save_to), f\"The path proveided '{save_to}' was not found. Make sure that there exists!\"\n",
    "\n",
    "        data_path = os.path.join(save_to, \"data\")\n",
    "        if not os.path.exists(data_path):\n",
    "            os.makedirs(data_path, exist_ok=True)\n",
    "        # print(f\"saving img_id: {indx}\")\n",
    "        image.save(os.path.join(data_path, f\"img_id_{image_data['img_id']}.jpg\"))\n",
    "           \n",
    "\n",
    "    return image_data\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we set multithreading to fetch images concurrently.\n",
    "This allows to speed significanlthly the download time of the images, when is a large number ~10.000 or so."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# import random\n",
    "# import os\n",
    "\n",
    "def fetch_images_concurrently(sampled_points, zoom_levels, save_to=None):\n",
    "    \"\"\"\n",
    "    Fetch images for sampled points concurrently using random zoom levels.\n",
    "\n",
    "    Args:\n",
    "        sampled_points (list): List of sampled points, where each point is represented as a list [lat, lon] or [lat, lon, class_value].\n",
    "        zoom_levels (list): List of zoom levels to choose from.\n",
    "        save_to (str): Optional path where to save the fetched images.\n",
    "\n",
    "    Returns:\n",
    "        list: List of dictionaries, each containing fetched image and its metadata (lat, lon, zoom_level, class).\n",
    "    \"\"\"\n",
    "    # def worker(point, indx, zoom_level):\n",
    "    #     return fetch_images_with_random_zoom_levels_faster(point, indx, zoom_level, save_to)\n",
    "    def worker(point, indx):\n",
    "        return fetch_images_with_random_zoom_levels_faster(point, indx, zoom_levels, save_to)\n",
    "\n",
    "    results = []\n",
    "    with concurrent.futures.ThreadPoolExecutor() as executor:\n",
    "        future_to_point = {executor.submit(worker, point, indx): (point, indx) for indx, point in enumerate(sampled_points)}\n",
    "        # future_to_point = {executor.submit(worker, point, indx, zoom_level): (point, indx, zoom_level) for indx, (point, zoom_level) in enumerate(zip(sampled_points, zoom_levels))}\n",
    "        for future in tqdm(concurrent.futures.as_completed(future_to_point), total=len(future_to_point), desc=\"Fetching images\"):\n",
    "            point, indx = future_to_point[future]\n",
    "            # point, indx, zoom_level = future_to_point[future]\n",
    "            try:\n",
    "                result = future.result()\n",
    "                results.append(result)\n",
    "            except Exception as e:\n",
    "                print(f\"Error fetching image for point {point} with index {indx}: {e}\")\n",
    "    \n",
    "    if save_to:\n",
    "        print(\"saving metadata\")\n",
    "        my_df = pd.DataFrame([ {key: d[key] for key in d if key != \"image\"} for d in results])\n",
    "        my_df.sort_values(by=['img_id'], ignore_index=True, inplace=True)\n",
    "        my_df.to_csv(os.path.join(save_to, \"metadata.csv\"))\n",
    "\n",
    "    print(\"#### Done ####\")\n",
    "    return results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fetch the images from the given coordinates in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cProfile.run('fetch_images_concurrently(coordinates, zoom_levels, r\"/Volumes/Ruben/datasets/fetched_raw_imgs_via_api_full\")')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetch the images\n",
    "# Fetch the images\n",
    "path_to_save_raw_images = r\"/Volumes/Ruben/datasets/fetched_raw_imgs_via_api_full\"\n",
    "# path_to_save_raw_images = r\"/Volumes/Ruben/datasets/fetched_raw_imgs_via_api\"\n",
    "    \n",
    "# fetched_images = fetch_images_with_random_zoom_levels(coordinates, zoom_levels, save_to=path_to_save_raw_images)\n",
    "fetched_images = fetch_images_concurrently(coordinates, zoom_levels, path_to_save_raw_images)\n",
    "len(fetched_images)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## debugging problematic images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code snippets are for debuggin purpose to investigate when retrived images were corrupted/missed.\n",
    "it is experimentl, so do not run the next code chunks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_to_data = os.path.join(path_to_save_raw_images, \"data\")\n",
    "os.path.exists(path_to_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12960"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(coordinates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[47.20178026276514, 8.264822308080678, 61],\n",
       " [47.03283524293044, 8.3608985971188, 61],\n",
       " [47.53628182279662, 9.500052084032554, 61],\n",
       " [46.860005678742134, 6.759326849716036, 61],\n",
       " [47.021068153553365, 8.95924400261676, 61]]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coordinates[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img_id</th>\n",
       "      <th>img_name</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>zoom_level</th>\n",
       "      <th>class</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3857_18_137090_91987</td>\n",
       "      <td>47.201780</td>\n",
       "      <td>8.264822</td>\n",
       "      <td>18</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3857_16_34290_23041</td>\n",
       "      <td>47.032835</td>\n",
       "      <td>8.360899</td>\n",
       "      <td>16</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3857_17_68994_45813</td>\n",
       "      <td>47.536282</td>\n",
       "      <td>9.500052</td>\n",
       "      <td>17</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3857_17_67996_46176</td>\n",
       "      <td>46.860006</td>\n",
       "      <td>6.759327</td>\n",
       "      <td>17</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3857_18_137595_92180</td>\n",
       "      <td>47.021068</td>\n",
       "      <td>8.959244</td>\n",
       "      <td>18</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>3857_18_136187_92237</td>\n",
       "      <td>46.967422</td>\n",
       "      <td>7.024716</td>\n",
       "      <td>18</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>3857_16_34116_23305</td>\n",
       "      <td>46.036235</td>\n",
       "      <td>7.408941</td>\n",
       "      <td>16</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>3857_17_67880_46388</td>\n",
       "      <td>46.459128</td>\n",
       "      <td>6.440222</td>\n",
       "      <td>17</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>3857_16_34388_23327</td>\n",
       "      <td>45.954755</td>\n",
       "      <td>8.902521</td>\n",
       "      <td>16</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>3857_16_34272_23000</td>\n",
       "      <td>47.188298</td>\n",
       "      <td>8.263295</td>\n",
       "      <td>16</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   img_id              img_name   latitude  longitude  zoom_level  class  \\\n",
       "0       0  3857_18_137090_91987  47.201780   8.264822          18     61   \n",
       "1       1   3857_16_34290_23041  47.032835   8.360899          16     61   \n",
       "2       2   3857_17_68994_45813  47.536282   9.500052          17     61   \n",
       "3       3   3857_17_67996_46176  46.860006   6.759327          17     61   \n",
       "4       4  3857_18_137595_92180  47.021068   8.959244          18     61   \n",
       "5       5  3857_18_136187_92237  46.967422   7.024716          18     61   \n",
       "6       6   3857_16_34116_23305  46.036235   7.408941          16     61   \n",
       "7       7   3857_17_67880_46388  46.459128   6.440222          17     61   \n",
       "8       8   3857_16_34388_23327  45.954755   8.902521          16     61   \n",
       "9       9   3857_16_34272_23000  47.188298   8.263295          16     61   \n",
       "\n",
       "                                                link  \n",
       "0  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "1  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "2  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "3  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "4  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "5  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "6  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "7  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "8  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "9  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(os.path.join(path_to_save_raw_images, \"metadata.csv\"), index_col=0)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        False\n",
       "1        False\n",
       "2        False\n",
       "3        False\n",
       "4        False\n",
       "         ...  \n",
       "12955    False\n",
       "12956    False\n",
       "12957    False\n",
       "12958    False\n",
       "12959    False\n",
       "Name: img_id, Length: 12960, dtype: bool"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['img_id'].duplicated()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>img_id</th>\n",
       "      <th>img_name</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>zoom_level</th>\n",
       "      <th>class</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed: 0, img_id, img_name, latitude, longitude, zoom_level, class, link]\n",
       "Index: []"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['img_id'].duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img_id</th>\n",
       "      <th>img_name</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>zoom_level</th>\n",
       "      <th>class</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2105</th>\n",
       "      <td>2105</td>\n",
       "      <td>3857_16_34346_23018</td>\n",
       "      <td>47.119002</td>\n",
       "      <td>8.669443</td>\n",
       "      <td>16</td>\n",
       "      <td>67</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2447</th>\n",
       "      <td>2447</td>\n",
       "      <td>3857_17_68486_45940</td>\n",
       "      <td>47.299968</td>\n",
       "      <td>8.103703</td>\n",
       "      <td>17</td>\n",
       "      <td>51</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2773</th>\n",
       "      <td>2773</td>\n",
       "      <td>3857_16_34291_23069</td>\n",
       "      <td>46.930224</td>\n",
       "      <td>8.368331</td>\n",
       "      <td>16</td>\n",
       "      <td>53</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3087</th>\n",
       "      <td>3087</td>\n",
       "      <td>3857_16_34262_23004</td>\n",
       "      <td>47.173383</td>\n",
       "      <td>8.208980</td>\n",
       "      <td>16</td>\n",
       "      <td>12</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3848</th>\n",
       "      <td>3848</td>\n",
       "      <td>3857_17_68232_45996</td>\n",
       "      <td>47.195740</td>\n",
       "      <td>7.405637</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12798</th>\n",
       "      <td>12798</td>\n",
       "      <td>3857_16_34340_23124</td>\n",
       "      <td>46.722584</td>\n",
       "      <td>8.639469</td>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12821</th>\n",
       "      <td>12821</td>\n",
       "      <td>3857_16_34092_23245</td>\n",
       "      <td>46.264607</td>\n",
       "      <td>7.275219</td>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12860</th>\n",
       "      <td>12860</td>\n",
       "      <td>3857_16_34470_23090</td>\n",
       "      <td>46.849588</td>\n",
       "      <td>9.354391</td>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12871</th>\n",
       "      <td>12871</td>\n",
       "      <td>3857_17_68561_46353</td>\n",
       "      <td>46.524969</td>\n",
       "      <td>8.309241</td>\n",
       "      <td>17</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12928</th>\n",
       "      <td>12928</td>\n",
       "      <td>3857_16_34333_23145</td>\n",
       "      <td>46.644725</td>\n",
       "      <td>8.599856</td>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>193 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       img_id             img_name   latitude  longitude  zoom_level  class  \\\n",
       "2105     2105  3857_16_34346_23018  47.119002   8.669443          16     67   \n",
       "2447     2447  3857_17_68486_45940  47.299968   8.103703          17     51   \n",
       "2773     2773  3857_16_34291_23069  46.930224   8.368331          16     53   \n",
       "3087     3087  3857_16_34262_23004  47.173383   8.208980          16     12   \n",
       "3848     3848  3857_17_68232_45996  47.195740   7.405637          17      3   \n",
       "...       ...                  ...        ...        ...         ...    ...   \n",
       "12798   12798  3857_16_34340_23124  46.722584   8.639469          16     66   \n",
       "12821   12821  3857_16_34092_23245  46.264607   7.275219          16     66   \n",
       "12860   12860  3857_16_34470_23090  46.849588   9.354391          16     66   \n",
       "12871   12871  3857_17_68561_46353  46.524969   8.309241          17     66   \n",
       "12928   12928  3857_16_34333_23145  46.644725   8.599856          16     66   \n",
       "\n",
       "                                                    link  \n",
       "2105   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "2447   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "2773   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "3087   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "3848   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "...                                                  ...  \n",
       "12798  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "12821  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "12860  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "12871  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "12928  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "\n",
       "[193 rows x 7 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['img_name'].duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img_id</th>\n",
       "      <th>img_name</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>zoom_level</th>\n",
       "      <th>class</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2105</th>\n",
       "      <td>2105</td>\n",
       "      <td>3857_16_34346_23018</td>\n",
       "      <td>47.119002</td>\n",
       "      <td>8.669443</td>\n",
       "      <td>16</td>\n",
       "      <td>67</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2447</th>\n",
       "      <td>2447</td>\n",
       "      <td>3857_17_68486_45940</td>\n",
       "      <td>47.299968</td>\n",
       "      <td>8.103703</td>\n",
       "      <td>17</td>\n",
       "      <td>51</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2773</th>\n",
       "      <td>2773</td>\n",
       "      <td>3857_16_34291_23069</td>\n",
       "      <td>46.930224</td>\n",
       "      <td>8.368331</td>\n",
       "      <td>16</td>\n",
       "      <td>53</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3087</th>\n",
       "      <td>3087</td>\n",
       "      <td>3857_16_34262_23004</td>\n",
       "      <td>47.173383</td>\n",
       "      <td>8.208980</td>\n",
       "      <td>16</td>\n",
       "      <td>12</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3848</th>\n",
       "      <td>3848</td>\n",
       "      <td>3857_17_68232_45996</td>\n",
       "      <td>47.195740</td>\n",
       "      <td>7.405637</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12798</th>\n",
       "      <td>12798</td>\n",
       "      <td>3857_16_34340_23124</td>\n",
       "      <td>46.722584</td>\n",
       "      <td>8.639469</td>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12821</th>\n",
       "      <td>12821</td>\n",
       "      <td>3857_16_34092_23245</td>\n",
       "      <td>46.264607</td>\n",
       "      <td>7.275219</td>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12860</th>\n",
       "      <td>12860</td>\n",
       "      <td>3857_16_34470_23090</td>\n",
       "      <td>46.849588</td>\n",
       "      <td>9.354391</td>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12871</th>\n",
       "      <td>12871</td>\n",
       "      <td>3857_17_68561_46353</td>\n",
       "      <td>46.524969</td>\n",
       "      <td>8.309241</td>\n",
       "      <td>17</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12928</th>\n",
       "      <td>12928</td>\n",
       "      <td>3857_16_34333_23145</td>\n",
       "      <td>46.644725</td>\n",
       "      <td>8.599856</td>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>193 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       img_id             img_name   latitude  longitude  zoom_level  class  \\\n",
       "2105     2105  3857_16_34346_23018  47.119002   8.669443          16     67   \n",
       "2447     2447  3857_17_68486_45940  47.299968   8.103703          17     51   \n",
       "2773     2773  3857_16_34291_23069  46.930224   8.368331          16     53   \n",
       "3087     3087  3857_16_34262_23004  47.173383   8.208980          16     12   \n",
       "3848     3848  3857_17_68232_45996  47.195740   7.405637          17      3   \n",
       "...       ...                  ...        ...        ...         ...    ...   \n",
       "12798   12798  3857_16_34340_23124  46.722584   8.639469          16     66   \n",
       "12821   12821  3857_16_34092_23245  46.264607   7.275219          16     66   \n",
       "12860   12860  3857_16_34470_23090  46.849588   9.354391          16     66   \n",
       "12871   12871  3857_17_68561_46353  46.524969   8.309241          17     66   \n",
       "12928   12928  3857_16_34333_23145  46.644725   8.599856          16     66   \n",
       "\n",
       "                                                    link  \n",
       "2105   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "2447   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "2773   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "3087   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "3848   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "...                                                  ...  \n",
       "12798  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "12821  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "12860  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "12871  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "12928  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "\n",
       "[193 rows x 7 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['link'].duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img_id</th>\n",
       "      <th>img_name</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>zoom_level</th>\n",
       "      <th>class</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>3857_18_136102_92270</td>\n",
       "      <td>46.936356</td>\n",
       "      <td>6.908075</td>\n",
       "      <td>18</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>188</td>\n",
       "      <td>3857_18_136222_92073</td>\n",
       "      <td>47.121405</td>\n",
       "      <td>7.073599</td>\n",
       "      <td>18</td>\n",
       "      <td>50</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     img_id              img_name   latitude  longitude  zoom_level  class  \\\n",
       "14       14  3857_18_136102_92270  46.936356   6.908075          18     61   \n",
       "188     188  3857_18_136222_92073  47.121405   7.073599          18     50   \n",
       "\n",
       "                                                  link  \n",
       "14   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "188  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_with_issues_id = [14, 188]\n",
    "df[df['img_id'].isin(img_with_issues_id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['3857_18_136102_92270', '3857_18_136222_92073'], dtype=object)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_with_issues_id = df[df['img_id'].isin(img_with_issues_id)][\"img_name\"].values\n",
    "img_with_issues_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img_id</th>\n",
       "      <th>img_name</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>zoom_level</th>\n",
       "      <th>class</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>3857_18_136102_92270</td>\n",
       "      <td>46.936356</td>\n",
       "      <td>6.908075</td>\n",
       "      <td>18</td>\n",
       "      <td>61</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>188</td>\n",
       "      <td>3857_18_136222_92073</td>\n",
       "      <td>47.121405</td>\n",
       "      <td>7.073599</td>\n",
       "      <td>18</td>\n",
       "      <td>50</td>\n",
       "      <td>https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     img_id              img_name   latitude  longitude  zoom_level  class  \\\n",
       "14       14  3857_18_136102_92270  46.936356   6.908075          18     61   \n",
       "188     188  3857_18_136222_92073  47.121405   7.073599          18     50   \n",
       "\n",
       "                                                  link  \n",
       "14   https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  \n",
       "188  https://wmts0.geo.admin.ch/1.0.0/ch.swisstopo....  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['img_name'].isin(img_with_issues_id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[46.93635599599829, 6.908074898533996, 61],\n",
       " [47.12140516279347, 7.073599476684011, 50]]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_coordinates = [[df.iloc[i][\"latitude\"], df.iloc[i][\"longitude\"], df.iloc[i][\"class\"]]  for i, rows in df[df['img_name'].isin(img_with_issues_id)].iterrows()]\n",
    "new_coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47.20178026276514"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[0][\"latitude\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 3)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(new_coordinates).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([17, 18, 16, 17, 18])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_zoom_levels = df[df['img_name'].isin(img_with_issues_id)][\"zoom_level\"].values\n",
    "new_zoom_levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching images: 100%|██████████| 2/2 [00:00<00:00,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving metadata\n",
      "#### Done ####\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fetched_images = fetch_images_concurrently(new_coordinates, \n",
    "#                                         #    new_zoom_levels,\n",
    "#                                        #  zoom_levels, \n",
    "#                                         [18], \n",
    "#                                            os.path.join(path_to_save_raw_images, \"test\"))\n",
    "fetched_images = fetch_images_with_random_zoom_levels(new_coordinates, \n",
    "                                        #    new_zoom_levels,\n",
    "                                       #  zoom_levels, \n",
    "                                        [18], \n",
    "                                           os.path.join(path_to_save_raw_images, \"test\"))\n",
    "len(fetched_images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[16, 17, 18]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zoom_levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Volumes/Ruben/datasets/fetched_raw_imgs_via_api_full'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_to_save_raw_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"HyperUnet_retrain_augmented_noise_corrected_Adam_history_dict\", 'rb') as my_file:\n",
    "    history = pickle.load(my_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'RootMeanSquaredError': [0.0422123484313488],\n",
       " 'loss': [0.030989430844783783],\n",
       " 'val_RootMeanSquaredError': [0.033434685319662094],\n",
       " 'val_loss': [0.025033673271536827]}"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}