While training, got <UnknownError: AttributeError: 'NoneType' object has no attribute 'shape'>

Question

While training, got <UnknownError: AttributeError: 'NoneType' object has no attribute 'shape'>

PAIka2k opened this issue 4 years ago · 1 comments

PAIka2k commented 4 years ago

Hi, Thank you for your last response. After I solved data related issue, I got another issue while training

After first epoch, I got an error like below. I tried to change batch size but I think it didn't work.

Do you have any idea?

my tf version is 2.3.1 and my current state is as below.

training image = 90729 (train:val = 0.8:0.2)
initial_epoch = 0
epochs = 20
batch_size = 8

inputs, data = gt_util.sample_batch(1,0)
inputs.shape, data[0]
->((1, 300, 300, 3),
array([[ 0.43576389, 0.24691358, 0.59114583, 0.77777778, 24. ]]))

UnknownError Traceback (most recent call last)
~/.local/lib/python3.6/site-packages/tensorflow/python/eager/context.py in execution_mode(mode)
2101 ctx.executor = executor_new
-> 2102 yield
2103 finally:

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/iterator_ops.py in _next_internal(self)
757 output_types=self._flat_output_types,
--> 758 output_shapes=self._flat_output_shapes)
759

~/.local/lib/python3.6/site-packages/tensorflow/python/ops/gen_dataset_ops.py in iterator_get_next(iterator, output_types, output_shapes, name)
2609 except _core._NotOkStatusException as e:
-> 2610 _ops.raise_from_not_ok_status(e, name)
2611 except _core._FallbackException:

~/.local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in raise_from_not_ok_status(e, name)
6842 # pylint: disable=protected-access
-> 6843 six.raise_from(core._status_to_exception(e.code, message), None)
6844 # pylint: enable=protected-access

~/.local/lib/python3.6/site-packages/six.py in raise_from(value, from_value)

UnknownError: AttributeError: 'NoneType' object has no attribute 'shape'
Traceback (most recent call last):

File "/home/user/.local/lib/python3.6/site-packages/tensorflow/python/ops/script_ops.py", line 242, in call
return func(device, token, args)

File "/home/user/.local/lib/python3.6/site-packages/tensorflow/python/ops/script_ops.py", line 131, in call
ret = self._func(*args)

File "/home/user/.local/lib/python3.6/site-packages/tensorflow/python/autograph/impl/api.py", line 302, in wrapper
return func(*args, **kwargs)

File "/home/user/daejeon/ssd/ssd_data.py", line 519, in get_sample
img_tmp, y_tmp = self.random_sized_crop(img, y)

File "/home/user/daejeon/ssd/ssd_data.py", line 441, in random_sized_crop
img_h, img_w = img.shape[:2]

AttributeError: 'NoneType' object has no attribute 'shape'

 [[{{node EagerPyFunc}}]] [Op:IteratorGetNext]

During handling of the above exception, another exception occurred:

UnknownError Traceback (most recent call last)
in
53
54 for i in tqdm(range(gen_train.num_batches), 'training', leave=True):
---> 55 x, y_true = next(iterator_train)
56 metric_values = step(x, y_true, training=True)
57 metric_util.update(metric_values, training=True)

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/iterator_ops.py in next(self)
734
735 def next(self): # For Python 3 compatibility
--> 736 return self.next()
737
738 def _next_internal(self):

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/iterator_ops.py in next(self)
770 def next(self):
771 try:
--> 772 return self._next_internal()
773 except errors.OutOfRangeError:
774 raise StopIteration

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/iterator_ops.py in _next_internal(self)
762 return self._element_spec._from_compatible_tensor_list(ret) # pylint: disable=protected-access
763 except AttributeError:
--> 764 return structure.from_compatible_tensor_list(self._element_spec, ret)
765
766 @Property

/usr/lib/python3.6/contextlib.py in exit(self, type, value, traceback)
97 value = type()
98 try:
---> 99 self.gen.throw(type, value, traceback)
100 except StopIteration as exc:
101 # Suppress StopIteration unless it's the same exception that

~/.local/lib/python3.6/site-packages/tensorflow/python/eager/context.py in execution_mode(mode)
2103 finally:
2104 ctx.executor = executor_old
-> 2105 executor_new.wait()
2106
2107

~/.local/lib/python3.6/site-packages/tensorflow/python/eager/executor.py in wait(self)
65 def wait(self):
66 """Waits for ops dispatched in this executor to finish."""
---> 67 pywrap_tfe.TFE_ExecutorWaitForAllPendingNodes(self._handle)
68
69 def clear_error(self):