diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f3d74a9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +*~ diff --git a/deeplab_resnet/__init__.py b/deeplab_resnet/__init__.py index 366b5f3..51b4e63 100644 --- a/deeplab_resnet/__init__.py +++ b/deeplab_resnet/__init__.py @@ -1,3 +1,3 @@ from .model import DeepLabResNetModel from .image_reader import ImageReader -from .utils import decode_labels, inv_preprocess, prepare_label +from .utils import decode_labels, inv_preprocess, prepare_label, decode_labels_old diff --git a/deeplab_resnet/image_reader.py b/deeplab_resnet/image_reader.py index 2571eb4..87d2621 100644 --- a/deeplab_resnet/image_reader.py +++ b/deeplab_resnet/image_reader.py @@ -89,6 +89,7 @@ def read_labeled_image_list(data_dir, data_list): image = mask = line.strip("\n") images.append(data_dir + image) masks.append(data_dir + mask) + print(data_dir + image + " " + data_dir + mask) return images, masks def read_images_from_disk(input_queue, input_size, random_scale, random_mirror, ignore_label, img_mean): # optional pre-processing arguments diff --git a/deeplab_resnet/utils.py b/deeplab_resnet/utils.py index 5a12d51..9982ae9 100644 --- a/deeplab_resnet/utils.py +++ b/deeplab_resnet/utils.py @@ -38,6 +38,23 @@ def decode_labels(mask, num_images=1, num_classes=21): outputs[i] = np.array(img) return outputs +def decode_labels_old(mask, num_classes): + """Decode batch of segmentation masks. + + Args: + label_batch: result of inference after taking argmax. + + Returns: + An batch of RGB images of the same size + """ + img = Image.new('RGB', (len(mask[0]), len(mask))) + pixels = img.load() + for j_, j in enumerate(mask): + for k_, k in enumerate(j): + if k < num_classes: + pixels[k_,j_] = label_colours[k] + return np.array(img) + def prepare_label(input_batch, new_size, num_classes, one_hot=True): """Resize masks and perform one-hot encoding. diff --git a/evaluate_msc.py b/evaluate_msc.py index 0de0ddd..515b25a 100644 --- a/evaluate_msc.py +++ b/evaluate_msc.py @@ -11,11 +11,14 @@ import os import sys import time - +import pandas as pd import tensorflow as tf +import cv2 +from PIL import Image import numpy as np +import scipy.io as sio -from deeplab_resnet import DeepLabResNetModel, ImageReader, prepare_label +from deeplab_resnet import DeepLabResNetModel, ImageReader, prepare_label, decode_labels_old IMG_MEAN = np.array((104.00698793,116.66876762,122.67891434), dtype=np.float32) @@ -25,6 +28,8 @@ NUM_CLASSES = 21 NUM_STEPS = 1449 # Number of images in the validation set. RESTORE_FROM = './deeplab_resnet.ckpt' +OUTPUT_IMGS=True + def get_arguments(): """Parse all the arguments provided from the CLI. @@ -45,6 +50,10 @@ def get_arguments(): help="Number of images in the validation set.") parser.add_argument("--restore-from", type=str, default=RESTORE_FROM, help="Where restore model parameters from.") + parser.add_argument("--save-dir", type=str, default=None, + help="Save output directory") + parser.add_argument("--save-dir-ind", type=str, default=None, + help="Save output directory") return parser.parse_args() def load(saver, sess, ckpt_path): @@ -105,10 +114,10 @@ def main(): pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor. # mIoU - pred = tf.reshape(pred, [-1,]) + pred_lin = tf.reshape(pred, [-1, ]) gt = tf.reshape(label_batch, [-1,]) weights = tf.cast(tf.less_equal(gt, args.num_classes - 1), tf.int32) # Ignoring all labels greater than or equal to n_classes. - mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes, weights=weights) + mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred_lin, gt, num_classes=args.num_classes, weights=weights) # Set up tf session and initialize variables. config = tf.ConfigProto() @@ -124,14 +133,39 @@ def main(): if args.restore_from is not None: load(loader, sess, args.restore_from) + # Output dirs + if not os.path.exists(args.save_dir): + os.makedirs(args.save_dir) + if not os.path.exists(args.save_dir_ind): + os.makedirs(args.save_dir_ind) + # Iterate over test files + data_info = pd.read_csv(args.data_list, header=None) + num_test_files = data_info.shape[0] # For computing Iter + imgList = np.asarray(data_info.iloc[:, 0]) + NUM_STEPS = len(imgList) + print("Num test files = " + str(NUM_STEPS)) + # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. - for step in range(args.num_steps): - preds, _ = sess.run([pred, update_op]) + # for step in range(args.num_steps): + for step in range(NUM_STEPS): + preds, preds_lin, _ = sess.run([pred, pred_lin, update_op]) if step % 100 == 0: print('step {:d}'.format(step)) + if OUTPUT_IMGS: + # print(np.array(preds).shape) + msk = decode_labels_old(np.array(preds)[0, :, :, 0], args.num_classes) + im = Image.fromarray(msk) + im_name = imgList[step].split('/')[2] + im.save(args.save_dir + '/' + im_name) + + mask_ind = np.array(preds)[0, :, :, 0] + cv2.imwrite(args.save_dir_ind + '/' + im_name, mask_ind) + + + print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess))) coord.request_stop() coord.join(threads) diff --git a/scripts.sh b/scripts.sh new file mode 100644 index 0000000..2bead11 --- /dev/null +++ b/scripts.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# PATH=/home/garbade/anaconda2/bin:$PATH + +python train_msc.py --batch-size 2 \ + --data-dir /home/garbade/datasets/nyu_depth_v2/ \ + --data-list /home/garbade/datasets/nyu_depth_v2/filelists/train_drsleep.txt \ + --snapshot-dir /home/garbade/models_tf/08_nyu_depth_v2/01_02_firstTry_nyu11 \ + --grad-update-every 10 \ + --ignore-label 0 \ + --not-restore-last \ + --num-classes 11 \ + --num-steps 20001 \ + --random-mirror \ + --random-scale \ + --restore-from /home/garbade/models_tf/dr_sleep_models/tf_v0.12/deeplab_resnet.ckpt \ + --save-num-images 1 \ + --learning-rate 1.0e-4 + + + + +python evaluate_msc.py --data-dir /home/garbade/datasets/nyu_depth_v2/ \ + --data-list /home/garbade/datasets/nyu_depth_v2/filelists/test_drsleep.txt \ + --ignore-label 0 \ + --num-classes 11 \ + --num-steps 1 \ + --restore-from /home/garbade/models_tf/08_nyu_depth_v2/01_02_firstTry_nyu11/model.ckpt-10000 \ + --save-dir /home/garbade/models_tf/08_nyu_depth_v2/01_02_firstTry_nyu11/res \ + --save-dir-ind /home/garbade/models_tf/08_nyu_depth_v2/01_02_firstTry_nyu11/res_ind \ No newline at end of file diff --git a/train_msc.py b/train_msc.py index ffcd1c4..5640ba8 100644 --- a/train_msc.py +++ b/train_msc.py @@ -184,6 +184,11 @@ def main(): raw_prediction075 = tf.reshape(raw_output075, [-1, args.num_classes]) raw_prediction05 = tf.reshape(raw_output05, [-1, args.num_classes]) + # Cross Entropy Loss --> Be careful about 'gt' input format -->has to be 0-based: [0:n_classes -1] + # if args.ignore_label == 0: + # label_batch = tf.to_int32(label_batch) + # label_batch = tf.subtract(label_batch , 1) + label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w] label_proc075 = prepare_label(label_batch, tf.stack(raw_output075.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) label_proc05 = prepare_label(label_batch, tf.stack(raw_output05.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) @@ -192,9 +197,9 @@ def main(): raw_gt075 = tf.reshape(label_proc075, [-1,]) raw_gt05 = tf.reshape(label_proc05, [-1,]) - indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1) - indices075 = tf.squeeze(tf.where(tf.less_equal(raw_gt075, args.num_classes - 1)), 1) - indices05 = tf.squeeze(tf.where(tf.less_equal(raw_gt05, args.num_classes - 1)), 1) + indices = tf.squeeze(tf.where(tf.not_equal(raw_gt, args.ignore_label)), 1) + indices075 = tf.squeeze(tf.where(tf.not_equal(raw_gt075, args.ignore_label)), 1) + indices05 = tf.squeeze(tf.where(tf.not_equal(raw_gt05, args.ignore_label)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32) @@ -220,6 +225,9 @@ def main(): pred = tf.expand_dims(raw_output_up, dim=3) # Image summary. + # if args.ignore_label == 0: + # label_batch = tf.add(label_batch , 1) + images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8) labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8) preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8) @@ -234,7 +242,7 @@ def main(): base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) - + tf.summary.scalar('learning_rate', learning_rate) opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum) opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum) @@ -264,6 +272,12 @@ def main(): train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b) + # Log variables + + tf.summary.scalar("reduced_loss", reduced_loss) + for v in conv_trainable + fc_w_trainable + fc_b_trainable: + tf.summary.histogram(v.name.replace(":", "_"), v) + merged_summary_op = tf.summary.merge_all() # Set up tf session and initialize variables. config = tf.ConfigProto() @@ -274,7 +288,7 @@ def main(): sess.run(init) # Saver for storing checkpoints of the model. - saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10) + saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=4, keep_checkpoint_every_n_hours=4) # Load variables if the checkpoint is provided. if args.restore_from is not None: @@ -303,7 +317,8 @@ def main(): # Apply gradients. if step % args.save_pred_every == 0: - images, labels, summary, _ = sess.run([image_batch, label_batch, total_summary, train_op], feed_dict=feed_dict) + # if True: + images, labels, summary, _ = sess.run([image_batch, label_batch, merged_summary_op, train_op], feed_dict=feed_dict) summary_writer.add_summary(summary, step) save(saver, sess, args.snapshot_dir, step) else: