mila-iqia/blocks

self.dump_iteration_state(main_loop)

Closed this issue · 0 comments

During the training, the self.dump_iteration_state(main_loop) is not working, and the messages are as follows:

`Traceback (most recent call last):
File "main.py", line 37, in
main(configuration, get_tr_data(*_configuration))
File "/data/disk1/private/ayana/mini/init.py", line 188, in main
main_loop.run()
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/main_loop.py", line 197, in run
reraise_as(e)
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/utils/init.py", line 258, in reraise_as
six.reraise(type(new_exc), new_exc, orig_exc_traceback)
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/main_loop.py", line 183, in run
while self._run_epoch():
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/main_loop.py", line 232, in _run_epoch
while self._run_iteration():
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/main_loop.py", line 255, in _run_iteration
self._run_extensions('after_batch', batch)
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/main_loop.py", line 263, in run_extensions
extension.dispatch(CallbackName(method_name), *args)
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/extensions/init.py", line 338, in dispatch
self.do(callback_invoked, *(from_main_loop + tuple(arguments)))
File "/data/disk1/private/ayana/mini/checkpoint.py", line 123, in do
self.dump(self.main_loop)
File "/data/disk1/private/ayana/mini/checkpoint.py", line 90, in dump
self.dump_iteration_state(main_loop)
File "/data/disk1/private/ayana/mini/checkpoint.py", line 76, in dump_iteration_state
secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/serialization.py", line 224, in secure_dump
dump_function(object
, temp, *_kwargs)
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/serialization.py", line 200, in dump
_taradd(save_object, tar_file, '_pkl')
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/serialization.py", line 611, in taradd
func(temp_file)
File "/home/ayana/python-env/dl/local/lib/python2.7/site-packages/blocks/serialization.py", line 496, in call
p.dump(self.object
)
File "/usr/lib/python2.7/pickle.py", line 224, in dump
self.save(obj)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib/python2.7/pickle.py", line 548, in save_tuple
save(element)
File "/usr/lib/python2.7/pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib/python2.7/pickle.py", line 419, in save_reduce
save(state)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
save(v)
File "/usr/lib/python2.7/pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib/python2.7/pickle.py", line 419, in save_reduce
save(state)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib/python2.7/pickle.py", line 649, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/lib/python2.7/pickle.py", line 681, in _batch_setitems
save(v)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib/python2.7/pickle.py", line 600, in save_list
self._batch_appends(iter(obj))
File "/usr/lib/python2.7/pickle.py", line 633, in _batch_appends
save(x)
File "/usr/lib/python2.7/pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib/python2.7/pickle.py", line 419, in save_reduce
save(state)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib/python2.7/pickle.py", line 562, in save_tuple
save(element)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib/python2.7/pickle.py", line 486, in save_string
self.write(BINSTRING + pack("<i", n) + obj)
struct.error: 'i' format requires -2147483648 <= number <= 2147483647

Original exception:
error: 'i' format requires -2147483648 <= number <= 2147483647`

The parameters and log can be dumped correctly
What can I do to solve this ?