Unable to convert TF OD API model for batch size > 1

[adroit91]

Hi,

I'm successfully able to convert model to FP32 TensorRT as well as FP16 TensorRT as long as my batch size is set to 1. But, as soon as I try increasing batch size (even to 2), I get errors.

Model config:
Model TF OD API configuration

Switches I use (with [Object Detection script] (https://github.com/tensorflow/tensorrt/blob/master/tftrt/examples/object_detection/object_detection.py) :
--data_dir /images_storage/BENCHMARK_IMGS --mode benchmark --batch_size 2 --input_saved_model_dir /data/models/saved_model --output_saved_model_dir /data/models/saved_model_TRT_FP32 --use_trt

Error I get:

---

TypeError Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _call_impl(self, args, kwargs, cancellation_manager)
1678 return self._call_with_structured_signature(args, kwargs,
-> 1679 cancellation_manager)
1680 except TypeError as structured_err:

/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _call_with_structured_signature(self, args, kwargs, cancellation_manager)
1755 self._function_spec.canonicalize_function_inputs(*args, **kwargs)
-> 1756 self._structured_signature_check_missing_args(args, kwargs)
1757 self._structured_signature_check_unexpected_args(args, kwargs)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _structured_signature_check_missing_args(self, args, kwargs)
1776 self._structured_signature_summary(),
-> 1777 ", ".join(sorted(missing_arguments))))
1778

TypeError: signature_wrapper(*, image_tensor) missing required arguments: image_tensor

During handling of the above exception, another exception occurred:

UnknownError Traceback (most recent call last)
in
9 display_every=args.display_every,
10 mode=args.mode,
---> 11 target_duration=args.target_duration)

in run_inference(graph_func, data_dir, annotation_path, batch_size, input_size, num_iterations, num_warmup_iterations, use_synthetic, display_every, mode, target_duration)
53 (i+1, num_iterations, iter_times[-1]*1000))
54 else:
---> 55 batch_preds = list(graph_func(batch_images).values())[0].numpy()
56 if i > 0 and target_duration is not None and
57 time.time() - initial_time > target_duration:

/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in call(self, *args, **kwargs)
1667 TypeError: If the arguments do not match the function's signature.
1668 """
-> 1669 return self._call_impl(args, kwargs)
1670
1671 def _call_impl(self, args, kwargs, cancellation_manager=None):

/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _call_impl(self, args, kwargs, cancellation_manager)
1681 try:
1682 return self._call_with_flat_signature(args, kwargs,
-> 1683 cancellation_manager)
1684 except TypeError:
1685 raise structured_err

/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _call_with_flat_signature(self, args, kwargs, cancellation_manager)
1734 "got {} ({})".format(self._flat_signature_summary(), i,
1735 type(arg).name, str(arg)))
-> 1736 return self._call_flat(args, self.captured_inputs, cancellation_manager)
1737
1738 def _call_with_structured_signature(self, args, kwargs, cancellation_manager):

/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/load.py in _call_flat(self, args, captured_inputs, cancellation_manager)
114 captured_inputs = list(map(get_unused_handle, captured_inputs))
115 return super(_WrapperFunction, self)._call_flat(args, captured_inputs,
--> 116 cancellation_manager)
117
118

/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
1917 # No tape is watching; skip to running the function.
1918 return self._build_call_outputs(self._inference_function.call(
-> 1919 ctx, args, cancellation_manager=cancellation_manager))
1920 forward_backward = self._select_forward_and_backward_functions(
1921 args,

/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in call(self, ctx, args, cancellation_manager)
558 inputs=args,
559 attrs=attrs,
--> 560 ctx=ctx)
561 else:
562 outputs = execute.execute_with_cancellation(

/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
58 ctx.ensure_initialized()
59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60 inputs, attrs, num_outputs)
61 except core._NotOkStatusException as e:
62 if name is not None:

UnknownError: 2 root error(s) found.
(0) Unknown: CUDNN_STATUS_INTERNAL_ERROR
in tensorflow/stream_executor/cuda/cuda_dnn.cc(3109): 'cudnnConvolutionForward( cudnn.handle(), alpha, input_nd.handle(), input_data.opaque(), filter_nd.handle(), filter_data.opaque(), conv.handle(), ToConvForwardAlgo(algorithm_desc), scratch_memory.opaque(), scratch_memory.size(), beta, output_nd.handle(), output_data.opaque())'
[[{{node FirstStageFeatureExtractor/InceptionResnetV2/InceptionResnetV2/Repeat_1/block17_19/Branch_1/Conv2d_0c_7x1/Conv2D}}]]
[[StatefulPartitionedCall/SecondStagePostprocessor/BatchMultiClassNonMaxSuppression/map/while/Identity/_231]]
(1) Unknown: CUDNN_STATUS_INTERNAL_ERROR
in tensorflow/stream_executor/cuda/cuda_dnn.cc(3109): 'cudnnConvolutionForward( cudnn.handle(), alpha, input_nd.handle(), input_data.opaque(), filter_nd.handle(), filter_data.opaque(), conv.handle(), ToConvForwardAlgo(algorithm_desc), scratch_memory.opaque(), scratch_memory.size(), beta, output_nd.handle(), output_data.opaque())'
[[{{node FirstStageFeatureExtractor/InceptionResnetV2/InceptionResnetV2/Repeat_1/block17_19/Branch_1/Conv2d_0c_7x1/Conv2D}}]]
[[StatefulPartitionedCall/SecondStagePostprocessor/BatchMultiClassNonMaxSuppression/map/while/TensorArrayWrite_4/TensorArrayWriteV3/_822]]
0 successful operations.
0 derived errors ignored. [Op:__inference_signature_wrapper_148011]

Function call stack:
signature_wrapper -> signature_wrapper