From 9d5682284525bf8d78db6e2d18a7377edae4842c Mon Sep 17 00:00:00 2001 From: GKalliatakis Date: Thu, 21 Jun 2018 17:13:38 +0100 Subject: [PATCH] Initial Commit --- README.md | 19 +- applications/__init__.py | 0 applications/vgg16_places_365.py | 245 +++++++++++++++ dataset/__init__.py | 0 engine/__init__.py | 0 engine/bottleneck_features.py | 335 ++++++++++++++++++++ engine/elapsed_time.txt | 1 + evaluation/__init__.py | 0 evaluation/evaluate_applications.py | 127 ++++++++ evaluation/handcrafted_metrics.py | 453 ++++++++++++++++++++++++++++ examples/__init__.py | 0 train_routine.py | 15 + utils/__init__.py | 0 13 files changed, 1194 insertions(+), 1 deletion(-) create mode 100644 applications/__init__.py create mode 100644 applications/vgg16_places_365.py create mode 100644 dataset/__init__.py create mode 100644 engine/__init__.py create mode 100644 engine/bottleneck_features.py create mode 100644 engine/elapsed_time.txt create mode 100644 evaluation/__init__.py create mode 100644 evaluation/evaluate_applications.py create mode 100644 evaluation/handcrafted_metrics.py create mode 100644 examples/__init__.py create mode 100644 train_routine.py create mode 100644 utils/__init__.py diff --git a/README.md b/README.md index 7fa0978..1eed1b3 100644 --- a/README.md +++ b/README.md @@ -1 +1,18 @@ -# Traffic Analysis Image Dataset +# Traffic Analysis + + + +### Train simple CNN + +Use the train_routine.py. Main things to change are : + +```python +pre_trained_model='VGG16' + +data_augm_enabled = False +``` + +Also you can change the number of training epochs inside engine/bottleneck_features.py as well as the paths. + + + diff --git a/applications/__init__.py b/applications/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/applications/vgg16_places_365.py b/applications/vgg16_places_365.py new file mode 100644 index 0000000..97e3624 --- /dev/null +++ b/applications/vgg16_places_365.py @@ -0,0 +1,245 @@ +# -*- coding: utf-8 -*- +'''VGG16-places365 model for Keras + +# Reference: +- [Places: A 10 million Image Database for Scene Recognition](http://places2.csail.mit.edu/PAMI_places.pdf) +- [https://github.com/GKalliatakis/Keras-VGG16-places365] +''' + +from __future__ import division, print_function +import os + +import warnings + +from keras import backend as K +from keras.layers import Input +from keras.layers.core import Dense, Flatten +from keras.layers.pooling import MaxPooling2D +from keras.models import Model +from keras.layers import Conv2D +from keras.regularizers import l2 +from keras.layers.core import Dropout +from keras.layers import GlobalAveragePooling2D +from keras.layers import GlobalMaxPooling2D +from keras.applications.imagenet_utils import _obtain_input_shape +from keras.engine.topology import get_source_inputs +from keras.utils.data_utils import get_file +from keras.utils import layer_utils + +WEIGHTS_PATH = 'https://github.com/GKalliatakis/Keras-VGG16-places365/releases/download/v1.0/vgg16-places365_weights_tf_dim_ordering_tf_kernels.h5' +WEIGHTS_PATH_NO_TOP = 'https://github.com/GKalliatakis/Keras-VGG16-places365/releases/download/v1.0/vgg16-places365_weights_tf_dim_ordering_tf_kernels_notop.h5' + + +def VGG16_Places365(include_top=True, weights='places', + input_tensor=None, input_shape=None, + pooling=None, + classes=365): + """Instantiates the VGG16-places365 architecture. + + Optionally loads weights pre-trained + on Places. Note that when using TensorFlow, + for best performance you should set + `image_data_format="channels_last"` in your Keras config + at ~/.keras/keras.json. + + The model and the weights are compatible with both + TensorFlow and Theano. The data format + convention used by the model is the one + specified in your Keras config file. + + # Arguments + include_top: whether to include the 3 fully-connected + layers at the top of the network. + weights: one of `None` (random initialization), + 'places' (pre-training on Places), + or the path to the weights file to be loaded. + input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) + to use as image input for the model. + input_shape: optional shape tuple, only to be specified + if `include_top` is False (otherwise the input shape + has to be `(224, 224, 3)` (with `channels_last` data format) + or `(3, 224, 244)` (with `channels_first` data format). + It should have exactly 3 inputs channels, + and width and height should be no smaller than 48. + E.g. `(200, 200, 3)` would be one valid value. + pooling: Optional pooling mode for feature extraction + when `include_top` is `False`. + - `None` means that the output of the model will be + the 4D tensor output of the + last convolutional layer. + - `avg` means that global average pooling + will be applied to the output of the + last convolutional layer, and thus + the output of the model will be a 2D tensor. + - `max` means that global max pooling will + be applied. + classes: optional number of classes to classify images + into, only to be specified if `include_top` is True, and + if no `weights` argument is specified. + # Returns + A Keras model instance. + # Raises + ValueError: in case of invalid argument for `weights`, or invalid input shape + """ + if not (weights in {'places', None} or os.path.exists(weights)): + raise ValueError('The `weights` argument should be either ' + '`None` (random initialization), `places` ' + '(pre-training on Places), ' + 'or the path to the weights file to be loaded.') + + if weights == 'places' and include_top and classes != 365: + raise ValueError('If using `weights` as places with `include_top`' + ' as true, `classes` should be 365') + + + # Determine proper input shape + input_shape = _obtain_input_shape(input_shape, + default_size=224, + min_size=48, + data_format=K.image_data_format(), + require_flatten=include_top, + weights=weights) + + if input_tensor is None: + img_input = Input(shape=input_shape) + else: + if not K.is_keras_tensor(input_tensor): + img_input = Input(tensor=input_tensor, shape=input_shape) + else: + img_input = input_tensor + + # Block 1 + x = Conv2D(filters=64, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block1_conv1')(img_input) + + x = Conv2D(filters=64, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block1_conv2')(x) + + x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name="places_block1_pool", padding='valid')(x) + + # Block 2 + x = Conv2D(filters=128, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block2_conv1')(x) + + x = Conv2D(filters=128, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block2_conv2')(x) + + x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name="places_block2_pool", padding='valid')(x) + + # Block 3 + x = Conv2D(filters=256, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block3_conv1')(x) + + x = Conv2D(filters=256, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block3_conv2')(x) + + x = Conv2D(filters=256, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block3_conv3')(x) + + x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name="places_block3_pool", padding='valid')(x) + + # Block 4 + x = Conv2D(filters=512, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block4_conv1')(x) + + x = Conv2D(filters=512, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block4_conv2')(x) + + x = Conv2D(filters=512, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block4_conv3')(x) + + x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name="places_block4_pool", padding='valid')(x) + + # Block 5 + x = Conv2D(filters=512, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block5_conv1')(x) + + x = Conv2D(filters=512, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block5_conv2')(x) + + x = Conv2D(filters=512, kernel_size=3, strides=(1, 1), padding='same', + kernel_regularizer=l2(0.0002), + activation='relu', name='places_block5_conv3')(x) + + x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name="places_block5_pool", padding='valid')(x) + + if include_top: + # Classification block + x = Flatten(name='places_flatten')(x) + x = Dense(4096, activation='relu', name='places_fc1')(x) + if weights is None: + x = Dropout(0.5, name='places_drop_fc1')(x) + + x = Dense(4096, activation='relu', name='places_fc2')(x) + if weights is None: + x = Dropout(0.5, name='places_drop_fc2')(x) + + x = Dense(365, activation='softmax', name='places_predictions')(x) + + + else: + if pooling == 'avg': + x = GlobalAveragePooling2D()(x) + elif pooling == 'max': + x = GlobalMaxPooling2D()(x) + + # Ensure that the model takes into account + # any potential predecessors of `input_tensor`. + if input_tensor is not None: + inputs = get_source_inputs(input_tensor) + else: + inputs = img_input + + # Create model. + model = Model(inputs, x, name='vgg16-places365') + + # load weights + if weights == 'places': + if include_top: + weights_path = get_file('vgg16-places365_weights_tf_dim_ordering_tf_kernels.h5', + WEIGHTS_PATH, + cache_subdir='models') + else: + weights_path = get_file('vgg16-places365_weights_tf_dim_ordering_tf_kernels_notop.h5', + WEIGHTS_PATH_NO_TOP, + cache_subdir='models') + + model.load_weights(weights_path) + + if K.backend() == 'theano': + layer_utils.convert_all_kernels_in_model(model) + + if K.image_data_format() == 'channels_first': + if include_top: + maxpool = model.get_layer(name='block5_pool') + shape = maxpool.output_shape[1:] + dense = model.get_layer(name='fc1') + layer_utils.convert_dense_weights_data_format(dense, shape, 'channels_first') + + if K.backend() == 'tensorflow': + warnings.warn('You are using the TensorFlow backend, yet you ' + 'are using the Theano ' + 'image data format convention ' + '(`image_data_format="channels_first"`). ' + 'For best performance, set ' + '`image_data_format="channels_last"` in ' + 'your Keras config ' + 'at ~/.keras/keras.json.') + + elif weights is not None: + model.load_weights(weights) + + return model + diff --git a/dataset/__init__.py b/dataset/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/engine/__init__.py b/engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/engine/bottleneck_features.py b/engine/bottleneck_features.py new file mode 100644 index 0000000..d571680 --- /dev/null +++ b/engine/bottleneck_features.py @@ -0,0 +1,335 @@ +# -*- coding: utf-8 -*- +"""Leverage a pre-trained network (saved network previously trained on a large dataset) +in order to build an image recognition system and analyse traffic. + +Transfer image representations from popular deep learning models. + +[A] ConvNet as fixed feature extractor.`Feature extraction` will simply consist of taking the convolutional base +of a previously-trained network, running the new data through it, and training a new classifier on top of the output. +(i.e. train only the randomly initialized top layers while freezing all convolutional layers of the original model). + +# References +- [https://deeplearningsandbox.com/how-to-use-transfer-learning-and-fine-tuning-in-keras-and-tensorflow-to-build-an-image-recognition-94b0b02444f2] +- [https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html] + +""" + +import os +import sys + +from keras.preprocessing.image import ImageDataGenerator +from keras.applications.vgg19 import VGG19 +from keras.applications.resnet50 import ResNet50 +from keras.applications import VGG16 +from keras.optimizers import SGD +from keras.models import Model +from keras.layers import Input +from keras.layers import Dense, GlobalAveragePooling2D, GlobalMaxPooling2D, Flatten, Dropout + +from keras.callbacks import ModelCheckpoint, EarlyStopping, CSVLogger + +import matplotlib.pyplot as plt + +from applications.vgg16_places_365 import VGG16_Places365 + +import datetime + +# Preparation actions + +now = datetime.datetime.now + + +epochs = 1 + +# Base directory of raw jpg/png images +base_dir = '/home/sandbox/GKalliatakis-GitHub Account/Traffic-Analysis/dataset/MotorwayTraffic' + +# Base directory for saving the trained models +base_dir_trained_models = '/home/sandbox/GKalliatakis-GitHub Account/Traffic-Analysis/trained_models/' +bottleneck_features_dir = os.path.join(base_dir_trained_models, 'bottleneck_features/') +logs_dir = os.path.join(base_dir_trained_models, 'logs/') + + +train_dir = os.path.join(base_dir, 'train') +nb_train_samples = 360 + + +val_dir = os.path.join(base_dir, 'val') +nb_val_samples = 40 + + +classes = ['empty', 'fluid', 'heavy', 'jam'] + +# https://groups.google.com/forum/#!topic/keras-users/MUO6v3kRHUw +# To train unbalanced classes 'fairly', we want to increase the importance of the under-represented class(es). +# To do this, we need to chose a reference class. You can pick any class to serve as the reference, but conceptually, +# I like the majority class (the one with the most samples). +# Creating your class_weight dictionary: +# 1. determine the ratio of reference_class/other_class. If you choose class_0 as your reference, +# you'll have (1000/1000, 1000/500, 1000/100) = (1,2,10) +# 2. map the class label to the ratio: class_weight={0:1, 1:2, 2:10} +# class_weight = {0: 5.08, 1: 1, 2: 10.86, 3: 5.08, 4: 3.46, 5: 2.31, 6: 4.70, 7: 6.17, 8: 1.55} + +# Augmentation configuration with only rescaling. +# Rescale is a value by which we will multiply the data before any other processing. +# Our original images consist in RGB coefficients in the 0-255, but such values would +# be too high for our models to process (given a typical learning rate), +# so we target values between 0 and 1 instead by scaling with a 1/255. factor. +train_datagen = ImageDataGenerator(rescale=1. / 255) + +# This is the augmentation configuration we will use for training when data_augm_enabled argument is True +train_augmented_datagen = ImageDataGenerator( + rescale=1. / 255, + rotation_range=40, + width_shift_range=0.2, + height_shift_range=0.2, + shear_range=0.2, + zoom_range=0.2, + horizontal_flip=True, + fill_mode='nearest') + + +val_datagen = ImageDataGenerator(rescale=1. / 255) + +img_width, img_height = 224, 224 + +batch_size = 10 + + +train_generator = train_datagen.flow_from_directory(train_dir, target_size=(img_width, img_height), + classes=classes, class_mode='categorical', + batch_size=batch_size) + +augmented_train_generator = train_augmented_datagen.flow_from_directory(train_dir, target_size=(img_width, img_height), + classes=classes, class_mode='categorical', + batch_size=batch_size) + + +val_generator = val_datagen.flow_from_directory(val_dir, target_size=(img_width, img_height), + classes=classes, class_mode='categorical', + batch_size=batch_size) + + +steps_per_epoch = nb_train_samples // batch_size +validation_steps = nb_val_samples // batch_size + + + + +def retrain_classifier(pre_trained_model='VGG16', + pooling_mode='avg', + classes=4, + data_augm_enabled = False): + """ConvNet as fixed feature extractor, consist of taking the convolutional base of a previously-trained network, + running the new data through it, and training a new classifier on top of the output. + (i.e. train only the randomly initialized top layers while freezing all convolutional layers of the original model). + + # Arguments + pre_trained_model: one of `VGG16`, `VGG19`, `ResNet50`, `VGG16_Places365` + pooling_mode: Optional pooling_mode mode for feature extraction + - `None` means that the output of the model will be + the 4D tensor output of the + last convolutional layer. + - `avg` means that global average pooling_mode + will be applied to the output of the + last convolutional layer, and thus + the output of the model will be a 2D tensor. + - `max` means that global max pooling_mode will + be applied. + classes: optional number of classes to classify images into. + data_augm_enabled: whether to augment the samples during training + + # Returns + A Keras model instance. + + # Raises + ValueError: in case of invalid argument for `pre_trained_model`, `pooling_mode` or invalid input shape. + """ + + + if not (pre_trained_model in {'VGG16', 'VGG19', 'ResNet50', 'VGG16_Places365'}): + raise ValueError('The `pre_trained_model` argument should be either ' + '`VGG16`, `VGG19`, `ResNet50`, ' + 'or `VGG16_Places365`. Other models will be supported in future releases. ') + + if not (pooling_mode in {'avg', 'max', 'flatten'}): + raise ValueError('The `pooling_mode` argument should be either ' + '`avg` (GlobalAveragePooling2D), `max` ' + '(GlobalMaxPooling2D), ' + 'or `flatten` (Flatten).') + + + # Define the name of the model and its weights + if data_augm_enabled == True: + filepath = bottleneck_features_dir+'augm_bottleneck_features_' + pre_trained_model + '_' + pooling_mode + '_pool_weights_tf_dim_ordering_tf_kernels.h5' + log_filepath = logs_dir + 'augm_' + pre_trained_model + '_' + pooling_mode + '_log.csv' + else: + filepath = bottleneck_features_dir+'bottleneck_features_' + pre_trained_model + '_' + pooling_mode + '_pool_weights_tf_dim_ordering_tf_kernels.h5' + log_filepath = logs_dir + pre_trained_model + '_' + pooling_mode + '_log.csv' + + + # ModelCheckpoint + checkpointer = ModelCheckpoint(filepath=filepath, + monitor='val_loss', + verbose=1, + save_best_only=True, + mode='auto', + period=1, + save_weights_only=True) + + early_stop = EarlyStopping(monitor='val_loss', patience=5, mode='auto') + + csv_logger = CSVLogger(log_filepath, append=True, separator=',') + + callbacks_list = [checkpointer, early_stop, csv_logger] + + + input_tensor = Input(shape=(224, 224, 3)) + + # create the base pre-trained model for warm-up + if pre_trained_model == 'VGG16': + base_model = VGG16(weights='imagenet', include_top=False, input_tensor=input_tensor) + + elif pre_trained_model == 'VGG19': + base_model = VGG19(weights='imagenet', include_top=False, input_tensor=input_tensor) + + elif pre_trained_model == 'ResNet50': + base_model = ResNet50(weights='imagenet', include_top=False, input_tensor=input_tensor) + + elif pre_trained_model == 'VGG16_Places365': + base_model = VGG16_Places365(weights='places', include_top=False, input_tensor=input_tensor) + + print ('\n \n') + print('[INFO] Vanilla `' + pre_trained_model + '` pre-trained convnet was successfully initialised.\n') + + + x = base_model.output + + # Now we set-up transfer learning process - freeze all but the penultimate layer + # and re-train the last Dense layer with `classes` number of final outputs representing probabilities for the different classes. + # Build a randomly initialised classifier model to put on top of the convolutional model + + # both `avg`and `max`result in the same size of the Dense layer afterwards + # Both Flatten and GlobalAveragePooling2D are valid options. So is GlobalMaxPooling2D. + # Flatten will result in a larger Dense layer afterwards, which is more expensive + # and may result in worse overfitting. But if you have lots of data, it might also perform better. + # https://github.com/keras-team/keras/issues/8470 + if pooling_mode == 'avg': + x = GlobalAveragePooling2D(name='GAP')(x) + elif pooling_mode == 'max': + x = GlobalMaxPooling2D(name='GMP')(x) + elif pooling_mode == 'flatten': + x = Flatten(name='FLATTEN')(x) + + + x = Dense(256, activation='relu', name='FC1')(x) # let's add a fully-connected layer + + # When random init is enabled, we want to include Dropout, + # otherwise when loading a pre-trained HRA model we want to omit + # Dropout layer so the visualisations are done properly (there is an issue if it is included) + x = Dropout(0.5, name='DROPOUT')(x) + # and a logistic layer with the number of classes defined by the `classes` argument + predictions = Dense(classes, activation='softmax', name='PREDICTIONS')(x) # new softmax layer + + # this is the transfer learning model we will train + model = Model(inputs=base_model.input, outputs=predictions) + + print('[INFO] Randomly initialised classifier was successfully added on top of the original pre-trained conv. base. \n') + + print('[INFO] Number of trainable weights before freezing the conv. base of the original pre-trained convnet: ' + '' + str(len(model.trainable_weights))) + + # first: train only the top layers (which were randomly initialized) + # i.e. freeze all convolutional layers of the preliminary base model + for layer in base_model.layers: + layer.trainable = False + + print('[INFO] Number of trainable weights after freezing the conv. base of the pre-trained convnet: ' + '' + str(len(model.trainable_weights))) + + print ('\n') + + # compile the warm_up_model (should be done *after* setting layers to non-trainable) + + model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), + loss='categorical_crossentropy', + metrics=['accuracy']) + model.summary() + + + # # The attribute model.metrics_names will give you the display labels for the scalar outputs. + # print warm_up_model.metrics_names + + if data_augm_enabled: + print('[INFO] Using augmented samples for training. This may take a while ! \n') + + t = now() + + history = model.fit_generator(augmented_train_generator, + epochs= epochs, + steps_per_epoch=steps_per_epoch, + validation_data = val_generator, + validation_steps= validation_steps, + callbacks=callbacks_list) + + print('[INFO] Training time for re-training the last dense layer using augmented samples: %s' % (now() - t)) + + elapsed_time = now() - t + + + + else: + t = now() + history = model.fit_generator(train_generator, + epochs= epochs, + steps_per_epoch=steps_per_epoch, + validation_data = val_generator, + validation_steps= validation_steps, + callbacks=callbacks_list) + + print('[INFO] Training time for re-training the last dense layer: %s' % (now() - t)) + + elapsed_time = now() - t + + print ('\n') + + # summarize history for accuracy + plt.plot(history.history['acc']) + plt.plot(history.history['val_acc']) + plt.title('model accuracy') + plt.ylabel('accuracy') + plt.xlabel('epoch') + plt.legend(['train', 'test'], loc='upper left') + plt.show() + # summarize history for loss + plt.plot(history.history['loss']) + plt.plot(history.history['val_loss']) + plt.title('model loss') + plt.ylabel('loss') + plt.xlabel('epoch') + plt.legend(['train', 'test'], loc='upper left') + plt.show() + + elapsed_time_entry = pre_trained_model + '_' + pooling_mode + ': '+ str(elapsed_time) + + file = open('elapsed_time.txt', 'a+') + + file.write(elapsed_time_entry) + + file.close() + + return model, elapsed_time + + + + + +if __name__ == "__main__": + + transfer_learning_model = retrain_classifier(pre_trained_model='VGG16', + pooling_mode='avg', + data_augm_enabled=False) + + + + diff --git a/engine/elapsed_time.txt b/engine/elapsed_time.txt new file mode 100644 index 0000000..2a93c84 --- /dev/null +++ b/engine/elapsed_time.txt @@ -0,0 +1 @@ +VGG16_avg: 0:01:47.604987 \ No newline at end of file diff --git a/evaluation/__init__.py b/evaluation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evaluation/evaluate_applications.py b/evaluation/evaluate_applications.py new file mode 100644 index 0000000..90303aa --- /dev/null +++ b/evaluation/evaluate_applications.py @@ -0,0 +1,127 @@ +import matplotlib.pyplot as plt +import numpy as np + +from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, precision_score, average_precision_score,roc_curve + +from applications.hra_vgg16 import HRA_VGG16 +from applications.hra_vgg19 import HRA_VGG19 +from applications.hra_resnet50 import HRA_ResNet50 +from applications.hra_vgg16_places365 import HRA_VGG16_Places365 +from applications.compoundNet_vgg16 import CompoundNet_VGG16 +from applications.compoundNet_vgg19 import CompoundNet_VGG19 +from applications.compoundNet_resnet50 import CompoundNet_ResNet50 +from handcrafted_metrics import HRA_metrics +from handcrafted_metrics import plot_confusion_matrix +from applications.hra_baseline import baseline_model + +from handcrafted_metrics import top_k_accuracy_score + + +# from applications.latest.hra_vgg16_checkpoint import HRA_VGG16 +# from applications.latest.hra_vgg16_places365 import HRA_VGG16_Places365 +# from applications.latest.compoundNet_vgg16_checkpoint import CompoundNet_VGG16 + + +# ==== Baseline model =========================================================================================================================== +# model = baseline_model(classes=9, epochs=40, weights='HRA') +# model.summary() +# =============================================================================================================================================== + + +# ==== Feature extraction/Fine-tuing model ====================================================================================================== +# pooling_mode = 'avg' +# model = HRA_VGG16(weights='HRA', mode='fine_tuning', pooling_mode=pooling_mode, include_top=True, data_augm_enabled=False) +# model.summary() +# =============================================================================================================================================== + + +# ==== CompoundNet model / Early-fusion========================================================================================================== +# model= CompoundNet_VGG16(weights='HRA', mode= 'fine_tuning', fusion_strategy='average', pooling_mode='avg', data_augm_enabled=False) +# model.summary() +# =============================================================================================================================================== + + +# ==== Object-centric CompoundNet model ========================================================================================================= +# model= CompoundNet_VGG16_VGG19(weights='HRA', mode= 'fine_tuning', fusion_strategy='maximum', pooling_mode='max') +# model.summary() +# =============================================================================================================================================== + + +# ==== Late-fusion ========================================================================================================= +# pooling_mode = 'max' + +model_a = HRA_VGG16(weights='HRA', mode='fine_tuning', pooling_mode='max', include_top=True, data_augm_enabled=False) +model_a.summary() + +model_b = HRA_VGG16_Places365(weights='HRA', mode='fine_tuning', pooling_mode='flatten', include_top=True, data_augm_enabled=False) +model_b.summary() + +# =============================================================================================================================================== + + + +model_name='BEST_COVERAGE_late_fusion' + + + +metrics = HRA_metrics(main_test_dir ='/home/sandbox/Desktop/Human_Rights_Archive_DB/test_uniform') + +[y_true, y_pred, y_score] = metrics.predict_labels(model) + +[y_true, y_pred] = metrics.duo_ensemble_predict_labels(model_a=model_a, model_b= model_b) + + +# print y_true +top1_acc = accuracy_score(y_true, y_pred) + +# top5_acc = top_k_accuracy_score(y_true=y_true, y_pred=y_pred,k=3,normalize=True) +# coverage = metrics.coverage(model,prob_threshold=0.85) +coverage = metrics.coverage_duo_ensemble(model_a,model_b,prob_threshold=0.85) + + +# AP = average_precision_score (y_true = y_true, y_score=y_score) +# +# print AP + + + +print ('\n') +print ('=======================================================================================================') +print (model_name+' Top-1 acc. => '+str(top1_acc)) +print (model_name+' Coverage => '+str(coverage)+'%') + + + +target_names = ['arms', 'child_labour', 'child_marriage', 'detention_centres', 'disability_rights', 'displaced_populations', + 'environment', 'no_violation', 'out_of_school'] + +# result= model_name+' => '+ str(accuracy_score(y_true, y_pred))+ '\n' +# result= model_name+' => '+str(coverage)+'%'+ '\n' +# +# +# f=open("results/coverage_late_fusion.txt", "a+") +# f.write(result+'\n\n') +# # f.write(str(y_pred)+'\n\n') +# f.close() + +# print(classification_report(y_true, y_pred, target_names=target_names)) +# +# print (precision_score(y_true, y_pred, average=None)) +# +# cnf_matrix=confusion_matrix(y_true, y_pred) +# np.set_printoptions(precision=2) +# +# # Plot non-normalized confusion matrix +# plt.figure() +# plot_confusion_matrix(cnf_matrix, classes=target_names, +# title='Confusion matrix, without normalization') +# +# # Plot normalized confusion matrix +# plt.figure() +# plot_confusion_matrix(cnf_matrix, classes=target_names, normalize=True, +# title='Normalized confusion matrix') +# +# plt.show() +# +# +# print (cnf_matrix.diagonal()/cnf_matrix.sum(axis=1)) \ No newline at end of file diff --git a/evaluation/handcrafted_metrics.py b/evaluation/handcrafted_metrics.py new file mode 100644 index 0000000..23d1c59 --- /dev/null +++ b/evaluation/handcrafted_metrics.py @@ -0,0 +1,453 @@ +import os +from utils.predict import * +import itertools + + + +class HRA_metrics(): + """Perfofmance metrics base class. + """ + def __init__(self, + main_test_dir ='/home/sandbox/Desktop/Human_Rights_Archive_DB/test' + ): + + + self.main_test_dir = main_test_dir + self.total_nb_of_test_images = sum([len(files) for r, d, files in os.walk(main_test_dir)]) + self.sorted_categories_names = sorted(os.listdir(main_test_dir)) + self.support = [ 186, 945, 87, 186, 273, 408, 201, 153, 609] + + + + def predict_labels(self, + model): + """Computes the predicted and ground truth labels, as returned by a single classifier. + + # Arguments + model = keras model for which we want to predict the labels. + + # Returns + y_true : 1d array-like containing the ground truth (correct) labels. + y_pred : 1d array-like containing the predicted labels, as returned by a classifier. + """ + y_pred = [] + y_true= [] + y_score = [] + + for hra_class in self.sorted_categories_names: + + # variable that contains the main dir alongside the selected category + tmp = os.path.join(self.main_test_dir, hra_class) + img_names = sorted(os.listdir(tmp)) + + for raw_img in img_names: + + if hra_class == 'arms': + true_label = 0 + elif hra_class == 'child_labour': + true_label = 1 + elif hra_class == 'child_marriage': + true_label = 2 + elif hra_class == 'detention_centres': + true_label = 3 + elif hra_class == 'disability_rights': + true_label = 4 + elif hra_class == 'displaced_populations': + true_label = 5 + elif hra_class == 'environment': + true_label = 6 + elif hra_class == 'no_violation': + true_label = 7 + elif hra_class == 'out_of_school': + true_label = 8 + + y_true.append(true_label) + + + # variable that contains the final image to be loaded + print ('Processing [' + raw_img + ']') + final_img = os.path.join(tmp, raw_img) + img = image.load_img(final_img, target_size=(224, 224)) + + preds = predict(model, img, target_size) + + y_pred.append(int(preds[0][0])) + y_score.append(int(preds[0][2])) + + + print y_pred + + return y_true, y_pred, y_score + + + + def duo_ensemble_predict_labels(self, + model_a, + model_b): + """Computes the predicted and ground truth labels, as returned by an ansemble of 2 classifiers. + + # Arguments + model_a: 1st model + model_b: 2nd model + + # Returns + y_true : 1d array-like containing the ground truth (correct) labels. + y_pred : 1d array-like containing the predicted labels, as returned by a classifier. + """ + + y_pred = [] + y_true = [] + + for hra_class in self.sorted_categories_names: + # variable that contains the main dir alongside the selected category + tmp = os.path.join(self.main_test_dir, hra_class) + img_names = sorted(os.listdir(tmp)) + + for raw_img in img_names: + + if hra_class == 'arms': + true_label = 0 + elif hra_class == 'child_labour': + true_label = 1 + elif hra_class == 'child_marriage': + true_label = 2 + elif hra_class == 'detention_centres': + true_label = 3 + elif hra_class == 'disability_rights': + true_label = 4 + elif hra_class == 'displaced_populations': + true_label = 5 + elif hra_class == 'environment': + true_label = 6 + elif hra_class == 'no_violation': + true_label = 7 + elif hra_class == 'out_of_school': + true_label = 8 + + y_true.append(true_label) + + # variable that contains the final image to be loaded + print ('Processing [' + raw_img + ']') + final_img = os.path.join(tmp, raw_img) + img = image.load_img(final_img, target_size=(224, 224)) + + preds = duo_ensemble_predict(model_a, model_b, img, target_size) + + y_pred.append(int(preds[0][0])) + + print y_pred + + return y_true, y_pred + + + + def trio_ensemble_predict_labels(self, + model_a, + model_b, + model_c): + """Computes the predicted and ground truth labels, as returned by an ansemble of 3 classifiers. + + # Arguments + model_a: 1st model + model_b: 2nd model + model_c: 3rd model + + # Returns + y_true : 1d array-like containing the ground truth (correct) labels. + y_pred : 1d array-like containing the predicted labels, as returned by a classifier. + """ + + y_pred = [] + y_true = [] + + for hra_class in self.sorted_categories_names: + # variable that contains the main dir alongside the selected category + tmp = os.path.join(self.main_test_dir, hra_class) + img_names = sorted(os.listdir(tmp)) + + for raw_img in img_names: + + if hra_class == 'arms': + true_label = 0 + elif hra_class == 'child_labour': + true_label = 1 + elif hra_class == 'child_marriage': + true_label = 2 + elif hra_class == 'detention_centres': + true_label = 3 + elif hra_class == 'disability_rights': + true_label = 4 + elif hra_class == 'displaced_populations': + true_label = 5 + elif hra_class == 'environment': + true_label = 6 + elif hra_class == 'no_violation': + true_label = 7 + elif hra_class == 'out_of_school': + true_label = 8 + + y_true.append(true_label) + + # variable that contains the final image to be loaded + print ('Processing [' + raw_img + ']') + final_img = os.path.join(tmp, raw_img) + img = image.load_img(final_img, target_size=(224, 224)) + + preds = trio_ensemble_predict(model_a, model_b,model_c, img, target_size) + + y_pred.append(int(preds[0][0])) + + print y_pred + + return y_true, y_pred + + + + def quadruple_ensemble_predict_labels(self, + model_a, + model_b, + model_c, + model_d): + """Computes the predicted and ground truth labels, as returned by an ansemble of 4 classifiers. + + # Arguments + model_a: 1st model + model_b: 2nd model + model_c: 3rd model + model_d: 4th model + + # Returns + y_true : 1d array-like containing the ground truth (correct) labels. + y_pred : 1d array-like containing the predicted labels, as returned by a classifier. + """ + + y_pred = [] + y_true = [] + + for hra_class in self.sorted_categories_names: + # variable that contains the main dir alongside the selected category + tmp = os.path.join(self.main_test_dir, hra_class) + img_names = sorted(os.listdir(tmp)) + + for raw_img in img_names: + + if hra_class == 'arms': + true_label = 0 + elif hra_class == 'child_labour': + true_label = 1 + elif hra_class == 'child_marriage': + true_label = 2 + elif hra_class == 'detention_centres': + true_label = 3 + elif hra_class == 'disability_rights': + true_label = 4 + elif hra_class == 'displaced_populations': + true_label = 5 + elif hra_class == 'environment': + true_label = 6 + elif hra_class == 'no_violation': + true_label = 7 + elif hra_class == 'out_of_school': + true_label = 8 + + y_true.append(true_label) + + # variable that contains the final image to be loaded + print ('Processing [' + raw_img + ']') + final_img = os.path.join(tmp, raw_img) + img = image.load_img(final_img, target_size=(224, 224)) + + preds = quadruple_ensemble_predict(model_a, model_b,model_c,model_d, img, target_size) + + y_pred.append(int(preds[0][0])) + + print y_pred + + return y_true, y_pred + + + + def coverage(self, + model, + prob_threshold = 0.75): + """Coverage is the fraction of examples for which the ML system is able to produce a response. + """ + + + predicted_class_list = [] + actual_class_list = [] + coverage_count = 0 + + for category in self.sorted_categories_names: + # variable that contains the main dir alongside the selected category + tmp = os.path.join(self.main_test_dir, category) + img_names = sorted(os.listdir(tmp)) + + for raw_img in img_names: + # variable that contains the final image to be loaded + final_img = os.path.join(tmp, raw_img) + img = image.load_img(final_img, target_size=(224, 224)) + # preprocess image + x = image.img_to_array(img) + x = np.expand_dims(x, axis=0) + x = preprocess_input(x) + + preds = predict(model, img, target_size) + + top_1_predicted_probability = preds[0][2] + + # top_1_predicted = np.argmax(preds) + top_1_predicted_label = preds[0][1] + + if top_1_predicted_probability >= prob_threshold: + coverage_count += 1 + + print ('`' + category + '/' + raw_img + '` ===> `' + + top_1_predicted_label + '`' + ' with ' + str(top_1_predicted_probability) + ' P') + + predicted_class_list.append(top_1_predicted_label) + actual_class_list.append(category) + + total_coverage_per = (coverage_count * 100) / self.total_nb_of_test_images + + return total_coverage_per + + + def coverage_duo_ensemble(self, + model_a, + model_b, + prob_threshold = 0.75): + """Coverage is the fraction of examples for which the ML system is able to produce a response. + """ + + + predicted_class_list = [] + actual_class_list = [] + coverage_count = 0 + + for category in self.sorted_categories_names: + # variable that contains the main dir alongside the selected category + tmp = os.path.join(self.main_test_dir, category) + img_names = sorted(os.listdir(tmp)) + + for raw_img in img_names: + # variable that contains the final image to be loaded + final_img = os.path.join(tmp, raw_img) + img = image.load_img(final_img, target_size=(224, 224)) + # preprocess image + x = image.img_to_array(img) + x = np.expand_dims(x, axis=0) + x = preprocess_input(x) + + preds = duo_ensemble_predict(model_a, model_b, img, target_size) + # preds = predict(model, img, target_size) + + top_1_predicted_probability = preds[0][2] + + # top_1_predicted = np.argmax(preds) + top_1_predicted_label = preds[0][1] + + if top_1_predicted_probability >= prob_threshold: + coverage_count += 1 + + print ('`' + category + '/' + raw_img + '` ===> `' + + top_1_predicted_label + '`' + ' with ' + str(top_1_predicted_probability) + ' P') + + predicted_class_list.append(top_1_predicted_label) + actual_class_list.append(category) + + total_coverage_per = (coverage_count * 100) / self.total_nb_of_test_images + + return total_coverage_per + +def top_k_accuracy_score(y_true, y_pred, k=5, normalize=True): + """Top k Accuracy classification score. + For multiclass classification tasks, this metric returns the + number of times that the correct class was among the top k classes + predicted. + Parameters + ---------- + y_true : 1d array-like, or class indicator array / sparse matrix + shape num_samples or [num_samples, num_classes] + Ground truth (correct) classes. + y_pred : array-like, shape [num_samples, num_classes] + For each sample, each row represents the + likelihood of each possible class. + The number of columns must be at least as large as the set of possible + classes. + k : int, optional (default=5) predictions are counted as correct if + probability of correct class is in the top k classes. + normalize : bool, optional (default=True) + If ``False``, return the number of top k correctly classified samples. + Otherwise, return the fraction of top k correctly classified samples. + Returns + ------- + score : float + If ``normalize == True``, return the proportion of top k correctly + classified samples, (float), else it returns the number of top k + correctly classified samples (int.) + The best performance is 1 with ``normalize == True`` and the number + of samples with ``normalize == False``. + See also + -------- + accuracy_score + Notes + ----- + If k = 1, the result will be the same as the accuracy_score (though see + note below). If k is the same as the number of classes, this score will be + perfect and meaningless. + In cases where two or more classes are assigned equal likelihood, the + result may be incorrect if one of those classes falls at the threshold, as + one class must be chosen to be the nth class and the class chosen may not + be the correct one. + """ + if len(y_true.shape) == 2: + y_true = np.argmax(y_true, axis=1) + + num_obs, num_labels = y_pred.shape + idx = num_labels - k - 1 + counter = 0 + argsorted = np.argsort(y_pred, axis=1) + for i in range(num_obs): + if y_true[i] in argsorted[i, idx + 1:]: + counter += 1 + if normalize: + return counter / num_obs + else: + return counter + + + +def plot_confusion_matrix(cm, classes, + normalize=False, + title='Confusion matrix', + cmap=plt.cm.Blues): + """ + This function prints and plots the confusion matrix. + Normalization can be applied by setting `normalize=True`. + """ + if normalize: + cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + print("Normalized confusion matrix") + else: + print('Confusion matrix, without normalization') + + print(cm) + + plt.imshow(cm, interpolation='nearest') + plt.title(title) + plt.colorbar() + tick_marks = np.arange(len(classes)) + plt.xticks(tick_marks, classes, rotation=45) + plt.yticks(tick_marks, classes) + + fmt = '.2f' if normalize else 'd' + thresh = cm.max() / 2. + for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): + plt.text(j, i, format(cm[i, j], fmt), + horizontalalignment="center", + color="white" if cm[i, j] > thresh else "black") + + plt.tight_layout() + plt.ylabel('True label') + plt.xlabel('Predicted label') diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train_routine.py b/train_routine.py new file mode 100644 index 0000000..4e79e0a --- /dev/null +++ b/train_routine.py @@ -0,0 +1,15 @@ +from engine.bottleneck_features import retrain_classifier + + +# one of `VGG16`, `VGG19`, `ResNet50`, `VGG16_Places365` + + +model, elapsed_time = retrain_classifier(pre_trained_model='VGG16', + pooling_mode='avg', + classes=4, + data_augm_enabled = False) + + + + + diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29