How to save the time for loading pre-trained model

[momo1986]

Hello guys,

Currently, my solution is restore the model every image or frame.
It will be very time-consuming.
I have tried to load it once, here is the code, in src/net.py:

def test_new_rule(self, checkpoint,graphpoint, cap):
    """Create session and restore weights
    """
    ops.reset_default_graph()
    g = tf.get_default_graph()
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        print(graphpoint)
        saver = tf.train.import_meta_graph(graphpoint)
        saver.restore(sess, checkpoint)
        ret, frame = cap.read()
        input_a = frame
        i = 0
        while ret:
            ret, frame = cap.read()
            if frame is None:
                continue
            input_b = frame
            input_a = cv2.resize(input_a, (384, 512))
            input_b = cv2.resize(input_b, (384, 512))
            # Convert from RGB -> BGR
            input_a = input_a[..., [2, 1, 0]]
            input_b = input_b[..., [2, 1, 0]]

            # Scale from [0, 255] -> [0.0, 1.0] if needed
            if input_a.max() > 1.0:
                input_a = input_a / 255.0
            if input_b.max() > 1.0:
                input_b = input_b / 255.0


            inputs = {
                'input_a': tf.expand_dims(tf.constant(input_a, dtype=tf.float32), 0),
                'input_b': tf.expand_dims(tf.constant(input_b, dtype=tf.float32), 0),
            }
            training_schedule = LONG_SCHEDULE

            predictions = self.model(inputs, training_schedule)
            #print(predictions)
            pred_flow = predictions['flow']
            pred_flow = sess.run(pred_flow)[0, :, :, :]

The api is called in src/flownet2/test.py:
net = FlowNet2(mode=Mode.TEST)
net.test_new_rule(
checkpoint='./checkpoints/FlowNet2/flownet-2.ckpt-0',
graphpoint='./checkpoints/FlowNet2/flownet-2.ckpt-0.meta',
cap=cap
)

**Here is error-log:

FailedPreconditionError (see above for traceback): Attempting to use uninitialized value FlowNet2/FlowNetCSS/FlowNetCS/FlowNetC/conv1/weights_1
[[node FlowNet2/FlowNetCSS/FlowNetCS/FlowNetC/conv1/weights_1/read (defined at /root/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/framework/python/ops/variables.py:277) = IdentityT=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:GPU:0"]]
[[{{node FlowNet2/ResizeBilinear/_459}} = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_1817_FlowNet2/ResizeBilinear", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]**

It will be very time-consuming for me to restore the model every time for image-processing.
Is there a work-around with the varaible definition of flownet-2 that I can transfer to test-file?
Thanks & regards!

[Iamanorange]

Building graph should be put out of camera capture loop:

predictions = self.model(inputs, training_schedule)
pred_flow = predictions['flow']
with tf.Session(config=config) as sess:
    saver.restore(blablabla)
    while ret:
        # Get frames and preprocessing
        flow = sess.run(pred_flow, feed_dict={
            input_a : frame_0,
            input_b : frame_1
        })[0, :, :, :]

[momo1986]

Building graph should be put out of camera capture loop:

predictions = self.model(inputs, training_schedule)
pred_flow = predictions['flow']
with tf.Session(config=config) as sess:
    saver.restore(blablabla)
    while ret:
        # Get frames and preprocessing
        flow = sess.run(pred_flow, feed_dict={
            input_a : frame_0,
            input_b : frame_1
        })[0, :, :, :]

Hello, thanks for your guidance,

I write my code like this:

def test_new_rule(self, checkpoint,graphpoint, cap):
    #Create session and restore weights
    tf.reset_default_graph()
    ret, frame = cap.read()

    input_a = frame
    input_b = frame
    input_a = cv2.resize(input_a, (384, 512))
    input_b = cv2.resize(input_b, (384, 512))

    # Convert from RGB -> BGR
    input_a = input_a[..., [2, 1, 0]]
    input_b = input_b[..., [2, 1, 0]]
    print(input_a.shape)
    # Scale from [0, 255] -> [0.0, 1.0] if needed
    if input_a.max() > 1.0:
        input_a = input_a / 255.0
    if input_b.max() > 1.0:
        input_b = input_b / 255.0

    # TODO: This is a hack, we should get rid of this

    training_schedule = LONG_SCHEDULE

    inputs = {
        'input_a': tf.expand_dims(tf.constant(input_a, dtype=tf.float32), 0),
        'input_b': tf.expand_dims(tf.constant(input_b, dtype=tf.float32), 0),
    }

    predictions = self.model(inputs, training_schedule)
    pred_flow = predictions['flow']

    saver = tf.train.Saver()

    with tf.Session() as sess:
        saver.restore(sess, checkpoint)
        ret, frame = cap.read()
        frame_0 = frame
        i = 0
        distance_u = 0
        distance_v = 0
        maxrad = 0
        previous_maxrad = 0
        max_maxrad = maxrad
        increase = False
        while ret:
            ret, frame = cap.read()
            if frame is None:
                continue
            frame_1 = frame
            frame_0 = cv2.resize(frame_0, (384, 512))
            frame_1 = cv2.resize(frame_1, (384, 512))
            # Convert from RGB -> BGR
            frame_0 = frame_0[..., [2, 1, 0]]
            frame_1 = frame_1[..., [2, 1, 0]]

            # Scale from [0, 255] -> [0.0, 1.0] if needed
            if frame_0.max() > 1.0:
                frame_0 = frame_0 / 255.0
            if frame_1.max() > 1.0:
                frame_1= frame_1 / 255.0
            frame_0 = tf.convert_to_tensor(frame_0, dtype=tf.float32)
            frame_1 = tf.convert_to_tensor(frame_1, dtype=tf.float32)

            flow = sess.run(pred_flow, feed_dict={
                input_a : frame_0,
                input_b : frame_1
            })[0, :, :, :]
     
            print(flow)
            print(distance_u)
            print(distance_v)
            print(maxrad)
            max_maxrad = max(max_maxrad, maxrad)
            if maxrad > previous_maxrad:
                increase = True
            else:
                if increase:
                    increase = False
                    print("Direction changed")
                    print(max_maxrad)
            previous_maxrad = maxrad
            frame_0 = frame_1
            i = i + 1
            if process_video == False:
                if cv2.waitKey(0) == ord('q'): break

However, it reports that the data-type is not correct:

Traceback (most recent call last):
File "/root/anaconda3/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/root/anaconda3/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/fast/flownet2-tf/src/flownet2/test_find_piano.py", line 54, in
main()
File "/fast/flownet2-tf/src/flownet2/test_find_piano.py", line 24, in main
cap=cap
File "/fast/flownet2-tf/src/net.py", line 154, in test_new_rule
input_b : frame_1
TypeError: unhashable type: 'numpy.ndarray'
(512, 384, 3)

[Iamanorange]

You should feed 2 placeholders to self.model(), instead of 2 constants.

[momo1986]

You should feed 2 placeholders to self.model(), instead of 2 constants.

How can I give the 2 placeholders with the input and give it to the self.model and then sess.run()?
Looks like pred_flow also depends on the input placeholder.
I have tried the different methods, always error?

[Iamanorange]

I haven't trained Flownet2. For inference:

# Build Graph
input_a = tf.placeholder(dtype=tf.float32, shape=[1, None, None, 3])
input_b = tf.placeholder(dtype=tf.float32, shape=[1, None, None, 3])
inputs = {
    'input_a': input_a,
    'input_b': input_b,
}
training_schedule = LONG_SCHEDULE
predictions = self.model(inputs, training_schedule)
pred_flow = predictions['flow']

with tf.Session(config=config) as sess:
    saver.restore(blablabla)
    while ret:
        # Get frames and preprocessing
        # Feed
        # There is no need to conver frame(ndarray) to tensor.
        flow = sess.run(pred_flow, feed_dict={
            input_a : frame_0,
            input_b : frame_1
        })[0, :, :, :]

[Banhalmi]

This worked for me to measure running time. Change it to handle frames.
The code in net.py was changed.
(added imports:
import cv2
import time
from numpy import array)

def test(self, checkpoint, input_a_path, input_b_path, out_path, save_image=True, save_flo=False):

		input_a_ = imread(input_a_path)
		input_b_ = imread(input_b_path)
		
		tf.reset_default_graph()
		
		input_a_ = cv2.resize(input_a_, dsize=(512, 384))#, interpolation=cv2.INTER_CUBIC)
		input_b_ = cv2.resize(input_b_, dsize=(512, 384))#, interpolation=cv2.INTER_CUBIC)

		# Convert from RGB -> BGR
		input_a_ = input_a_[..., [2, 1, 0]]
		input_b_ = input_b_[..., [2, 1, 0]]

		# Scale from [0, 255] -> [0.0, 1.0] if needed
		if input_a_.max() > 1.0:
			input_a_ = input_a_ / 255.0
		if input_b_.max() > 1.0:
			input_b_ = input_b_ / 255.0

		# TODO: This is a hack, we should get rid of this
		training_schedule = LONG_SCHEDULE

		input_a = tf.placeholder(dtype=tf.float32, shape=[1,384, 512, 3])
		input_b = tf.placeholder(dtype=tf.float32, shape=[1,384, 512, 3])
		inputs = {
			'input_a': input_a,
			'input_b': input_b,
		}

		image1 = array(input_a_).reshape(1, 384, 512, 3)
		image2 = array(input_b_).reshape(1, 384, 512, 3)

		predictions = self.model(inputs, training_schedule, False)
		pred_flow = predictions['flow']

		saver = tf.train.Saver()
		
		with tf.Session() as sess:
			saver.restore(sess, checkpoint)
			t0=time.time()
			for i in range(20):
				flow = sess.run(pred_flow, feed_dict={
		            input_a : image1,
				    input_b : image2
		        })[0, :, :, :]

				unique_name = 'flow-' + str(uuid.uuid4())
				if save_image:
					flow_img = flow_to_image(flow)
					full_out_path = os.path.join(out_path, unique_name + '.png')
					imsave(full_out_path, flow_img)

				if save_flo:
					full_out_path = os.path.join(out_path, unique_name + '.flo')
					write_flow(pred_flow, full_out_path)
		print('time for 20 iteration:{}'.format(time.time()-t0))