Tensorflow Tensorrt model not working properly in for loop

[123mw123]

I converted my model to tf-trt and it's working fine if run on single image. When I am trying to run on a folder, it's not detecting anything on some images and if i run on those undetected images again separately model is working fine. Below is the code I am using. Also if I read normal tf model instead of tf-trt model at line tf.gfile.GFile("east_trt_graph_32_batchnorm.pb", 'rb') as fid:, everything is working fine. Any idea why is this happening?

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.2
config.allow_soft_placement = True
outputs = ["feature_fusion/Conv_7/Sigmoid", "feature_fusion/concat_3"] 

detection_graph = tf.Graph()
with detection_graph.as_default():
    od_graph_def = tf.GraphDef()
    with tf.gfile.GFile("east_trt_graph_32_batchnorm.pb", 'rb') as fid:
        sess = tf.Session(config=config)
        serialized_graph = fid.read()
        od_graph_def.ParseFromString(serialized_graph)
        tf.import_graph_def(od_graph_def, name='')
        g = tf.get_default_graph()
        input_images = g.get_tensor_by_name("input_images:0")
        outputs = [n + ':0' for n in outputs]  # tensor names
        f_score = g.get_tensor_by_name(outputs[0])
        f_geometry = g.get_tensor_by_name(outputs[1])
        data_dir = "/home/saiteja/Documents/data/val_text_barcode"
        for im_name in os.listdir(data_dir):
            timer = {'net': 0, 'restore': 0, 'nms': 0}
            img = cv2.imread(os.path.join(data_dir, im_name))
            img_resized, (ratio_h, ratio_w) = resize_image(img, 512)
            score, geometry = sess.run([f_score, f_geometry], feed_dict={input_images: [img_resized]})
            boxes, timer = detect(score_map=score, geo_map=geometry, timer=timer)
            if boxes is not None:
                cv2.imwrite(os.path.join("./output_trt", im_name), draw_results(img, boxes, ratio_h, ratio_w))