fix: pickle data truncated
Closed this issue · 0 comments
ocervell commented
When trying to get very long results during a scan in worker mode using a file broker / result backend, we get:
Traceback (most recent call last):
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 41, in _reraise_errors
yield
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 265, in loads
return decode(data)
^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 337, in unpickle
return pickle_loads(str_to_bytes(s))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 50, in pickle_loads
return load(BytesIO(s))
^^^^^^^^^^^^^^^^
_pickle.UnpicklingError: pickle data was truncated
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/osboxes/Workspace/secator/.venv/bin/secator", line 8, in <module>
sys.exit(cli())
^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/click/core.py", line 1157, in __call__
return self.main(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/rich_click/rich_group.py", line 21, in main
rv = super().main(*args, standalone_mode=False, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/click/core.py", line 1078, in main
rv = self.invoke(ctx)
^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/click/core.py", line 1688, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/click/core.py", line 1688, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/click/core.py", line 1434, in invoke
return ctx.invoke(self.callback, **ctx.params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/click/core.py", line 783, in invoke
return __callback(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/click/decorators.py", line 33, in new_func
return f(get_current_context(), *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/secator/decorators.py", line 311, in func
runner.run()
File "/home/osboxes/Workspace/secator/secator/runners/_base.py", line 217, in run
return list(self.__iter__())
^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/secator/runners/_base.py", line 226, in __iter__
for item in self.yielder():
File "/home/osboxes/Workspace/secator/secator/runners/scan.py", line 63, in yielder
yield from workflow
File "/home/osboxes/Workspace/secator/secator/runners/_base.py", line 226, in __iter__
for item in self.yielder():
File "/home/osboxes/Workspace/secator/secator/runners/workflow.py", line 61, in yielder
yield from results
File "/home/osboxes/Workspace/secator/secator/runners/_base.py", line 644, in process_live_tasks
for data in Runner.get_live_results(result):
File "/home/osboxes/Workspace/secator/secator/runners/_base.py", line 535, in get_live_results
yield from Runner.get_celery_results(result)
File "/home/osboxes/Workspace/secator/secator/runners/_base.py", line 559, in get_celery_results
data = get_task_data(task_id)
^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/secator/runners/_helpers.py", line 126, in get_task_data
if res.info and not isinstance(res.info, list):
^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/celery/result.py", line 466, in result
return self._get_task_meta()['result']
^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/celery/result.py", line 442, in _get_task_meta
return self._maybe_set_cache(self.backend.get_task_meta(self.id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/celery/backends/base.py", line 608, in get_task_meta
meta = self._get_task_meta_for(task_id)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/celery/backends/base.py", line 998, in _get_task_meta_for
return self.decode_result(meta)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/celery/backends/base.py", line 425, in decode_result
return self.meta_from_decoded(self.decode(payload))
^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/celery/backends/base.py", line 431, in decode
return loads(payload,
^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 264, in loads
with _reraise_errors(DecodeError):
File "/usr/lib/python3.11/contextlib.py", line 155, in __exit__
self.gen.throw(typ, value, traceback)
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 45, in _reraise_errors
reraise(wrapper, wrapper(exc), sys.exc_info()[2])
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/exceptions.py", line 34, in reraise
raise value.with_traceback(tb)
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 41, in _reraise_errors
yield
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 265, in loads
return decode(data)
^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 337, in unpickle
return pickle_loads(str_to_bytes(s))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/osboxes/Workspace/secator/.venv/lib/python3.11/site-packages/kombu/serialization.py", line 50, in pickle_loads
return load(BytesIO(s))
^^^^^^^^^^^^^^^^
kombu.exceptions.DecodeError: pickle data was truncated