From 93040b99b4d25880c9be7667989d71bb55edc0fb Mon Sep 17 00:00:00 2001 From: Martynas Janonis Date: Thu, 28 Feb 2019 03:19:47 +0000 Subject: [PATCH 1/2] Fixed transfer learning architecture --- T3D_keras.py | 89 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/T3D_keras.py b/T3D_keras.py index ae2fe0a..c8130a0 100644 --- a/T3D_keras.py +++ b/T3D_keras.py @@ -1,8 +1,9 @@ import keras from keras.models import Sequential, Model -from keras.layers import Input, BatchNormalization, Activation, Conv3D, Dropout, Concatenate, AveragePooling3D, MaxPooling3D, Dense, Flatten, GlobalAveragePooling2D +from keras.layers import Input, BatchNormalization, Activation, Conv3D, Dropout, Concatenate, AveragePooling3D, MaxPooling3D, Dense, Flatten, GlobalAveragePooling2D, GlobalAveragePooling3D from keras.activations import linear, softmax from keras.applications import densenet +from keras.layers import TimeDistributed __all__ = ['DenseNet', 'densenet121', 'densenet161'] # with DropOut @@ -82,9 +83,15 @@ def DenseNet3D(input_shape, growth_rate=32, block_config=(6, 12, 24, 16), """ #----------------------------------------------------------------- inp_2d = (Input(shape=(224,224,3), name='2d_input')) - pretrained_densenet = densenet.DenseNet169(include_top=False, input_shape=(224,224,3), input_tensor=inp_2d, weights='imagenet') - for layer in pretrained_densenet.layers: + batch_densenet = densenet.DenseNet169(include_top=False, input_shape=(224,224,3), input_tensor=inp_2d, weights='imagenet') + + for layer in batch_densenet.layers: layer.trainable = False + + # Configure the 2D CNN to take batches of pictures + inp_2d_batch = (Input(shape=(input_shape[0],224,224,3), name='2d_input_batch')) + batch_densenet = TimeDistributed(batch_densenet)(inp_2d_batch) + batch_densenet = Model(inputs=inp_2d_batch, outputs=batch_densenet) #----------------------------------------------------------------- # First convolution----------------------- @@ -127,8 +134,8 @@ def DenseNet3D(input_shape, growth_rate=32, block_config=(6, 12, 24, 16), x = AveragePooling3D(pool_size=(1, 7, 7))(x) x = Flatten(name='flatten_3d')(x) x = Dense(1024, activation='relu')(x) - #--------------fron 2d densenet model----------------- - y = GlobalAveragePooling2D(name='avg_pool_densnet2d')(pretrained_densenet.output) + #--------------from 2d densenet model----------------- + y = GlobalAveragePooling3D(name='avg_pool_densnet3d')(batch_densenet.output) y = Dense(1024, activation='relu')(y) #----------------------------------------------------- @@ -140,7 +147,77 @@ def DenseNet3D(input_shape, growth_rate=32, block_config=(6, 12, 24, 16), x = Dropout(0.35)(x) out = Dense(num_classes, activation='softmax')(x) - model = Model(inputs=[inp_2d, inp_3d], outputs=[out]) + model = Model(inputs=[inp_2d_batch, inp_3d], outputs=[out]) + model.summary() + + return model + + +# The T3D CNN standalone +def T3D(input_shape, growth_rate=32, block_config=(6, 12, 24, 16), + num_init_features=64, bn_size=4, drop_rate=0, num_classes=5): + r"""Densenet-BC model class, based on + `"Densely Connected Convolutional Networks" ` + Args: + growth_rate (int) - how many filters to add each layer (`k` in paper) + block_config (list of 4 ints) - how many layers in each pooling block + num_init_features (int) - the number of filters to learn in the first convolution layer + bn_size (int) - multiplicative factor for number of bottle neck layers + (i.e. bn_size * k features in the bottleneck layer) + drop_rate (float) - dropout rate after each dense layer + num_classes (int) - number of classification classes + """ + + # First convolution----------------------- + inp_3d = (Input(shape=input_shape, name='3d_input')) + + + # need to check padding + x = (Conv3D(num_init_features, kernel_size=(3, 7, 7), + strides=2, padding='same', use_bias=False))(inp_3d) + + x = BatchNormalization()(x) + x = Activation('relu')(x) + + # need to check padding + x = MaxPooling3D(pool_size=(3, 3, 3), strides=( + 2, 2, 2), padding='valid')(x) + + # Each denseblock + num_features = num_init_features + for i, num_layers in enumerate(block_config): + # print('Pass', i) + x = _DenseBlock(x, num_layers=num_layers, + bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + + num_features = num_features + num_layers * growth_rate + + if i != len(block_config) - 1: + # print('Not Last layer, so adding Temporal Transition Layer') + + x = _TTL(x) + # num_features = 128*3 + + x = _Transition(x, num_output_features=num_features) + num_features = num_features + + # Final batch norm + x = BatchNormalization()(x) + + x = Activation('relu')(x) + x = AveragePooling3D(pool_size=(1, 7, 7))(x) + x = Flatten(name='flatten_3d')(x) + x = Dense(1024, activation='relu')(x) + + #----------------------------------------------------- + x = Dropout(0.65)(x) + x = Dense(512, activation='relu')(x) + x = Dropout(0.5)(x) + x = Dense(128, activation='relu')(x) + x = Dropout(0.35)(x) + out = Dense(num_classes, activation='softmax')(x) + + model = Model(inputs=[inp_3d], outputs=[out]) model.summary() return model From 7574d1889d7076d70b00d7e329ad066b28464fa4 Mon Sep 17 00:00:00 2001 From: Martynas Janonis Date: Thu, 28 Feb 2019 03:28:08 +0000 Subject: [PATCH 2/2] Fixed the input shape for the batch densenet --- T3D_keras.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/T3D_keras.py b/T3D_keras.py index c8130a0..c51c15e 100644 --- a/T3D_keras.py +++ b/T3D_keras.py @@ -89,7 +89,7 @@ def DenseNet3D(input_shape, growth_rate=32, block_config=(6, 12, 24, 16), layer.trainable = False # Configure the 2D CNN to take batches of pictures - inp_2d_batch = (Input(shape=(input_shape[0],224,224,3), name='2d_input_batch')) + inp_2d_batch = (Input(shape=input_shape, name='2d_input_batch')) batch_densenet = TimeDistributed(batch_densenet)(inp_2d_batch) batch_densenet = Model(inputs=inp_2d_batch, outputs=batch_densenet) #-----------------------------------------------------------------