error while running @concurrent.process repetitively
makesnosense opened this issue · 4 comments
makesnosense commented
Hey.
I use pebble to run a script every 1000 seconds, with timeout of 300. It works just fine for ~two days, then dies completely with the error below, which does not even seem to be handled or catched by the logger. After restarting, it works just fine (for some time more).
Maybe I do something wrong? Anyways, I would be very grateful for any suggestions on how to further investigate the issue.
Exception in thread Thread-81 (_worker_handler):
Traceback (most recent call last):
File "/usr/local/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
self.run()
File "/usr/local/lib/python3.12/threading.py", line 1010, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/site-packages/pebble/concurrent/process.py", line 137, in _worker_handler
if result.status == SUCCESS:
^^^^^^^^^^^^^
AttributeError: 'AttributeError' object has no attribute 'status'
The code I run is very-very similar to one of the examples that you provide:
logg = configure_logger(__name__)
@concurrent.process(timeout=300)
def main():
#some code
if __name__ == '__main__':
while True:
try:
future = main()
results = future.result()
except TimeoutError as error:
logg.info(f"Timeout {error.args[1]} seconds")
except Exception as error:
logg.error(error, exc_info=True)
logg.info(f'There's an error {error.__repr__()}')
time.sleep(1000)noxdafox commented
Hello,
will try to reproduce the issue. What OS are you using?
makesnosense commented
I use official Python docker image, python:3.12.2-bookworm, linux amd64, which is Debian 12.5
Host is Ubuntu 23.10, usual DO droplet with 2 GB Memory, if that matters.
makesnosense commented
I'm testing the new version after d32346d and it seems to handle arising exception as it should 👍 and overall program doesn't crash. (I have occasional error coming from google sheets API & gspread lib)
Looks good on my side, thank you!
Full error log just in case:
01.03.24 06:00:10 ERROR module: run __main__ func: <module> line: 57 msg: 'dict' object has no attribute 'text'
┆ File "/app/run.py", line 53, in <module>
┆ 49 if __name__ == '__main__':
┆ 50 while True:
┆ 51 try:
┆ 52 future = main()
┆ --> 53 results = future.result()
┆ 54 except TimeoutError as error:
┆ ..................................................
┆ __name__ = '__main__'
┆ future = <ProcessFuture at 0x7f464b201c70 state=finished raised Attri
┆ buteError>
┆ results = None
┆ future.result = <method 'Future.result' of <ProcessFuture at 0x7f464b201c70
┆ state=finished raised AttributeError> _base.py:428>
┆ error = AttributeError("'dict' object has no attribute 'text'")
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 456, in result
┆ 428 def result(self, timeout=None):
┆ (...)
┆ 452
┆ 453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
┆ 454 raise CancelledError()
┆ 455 elif self._state == FINISHED:
┆ --> 456 return self.__get_result()
┆ 457 else:
┆ ..................................................
┆ self = None
┆ timeout = None
┆ self._state = # AttributeError
┆ self = None
┆ CANCELLED = 'CANCELLED'
┆ CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
┆ FINISHED = 'FINISHED'
┆ self.__get_result = # AttributeError
┆ self = None
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
┆ 398 def __get_result(self):
┆ 399 if self._exception:
┆ 400 try:
┆ --> 401 raise self._exception
┆ 402 finally:
┆ ..................................................
┆ self = None
┆ self._exception = # AttributeError
┆ self = None
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/site-packages/pebble/concurrent/process.py", line 178, in _get_result
┆ 163 def _get_result(
┆ 164 future: ProcessFuture,
┆ 165 pipe: multiprocessing.Pipe,
┆ 166 timeout: float
┆ 167 ) -> Any:
┆ (...)
┆ 174 return Result(FAILURE, TimeoutError('Task Timeout', timeout))
┆ 175 if future.cancelled():
┆ 176 return Result(FAILURE, CancelledError())
┆ 177
┆ --> 178 return pipe.recv()
┆ 179 except (EOFError, OSError):
┆ ..................................................
┆ future = <ProcessFuture at 0x7f464b201c70 state=finished raised Attri
┆ buteError>
┆ pipe = <multiprocessing.connection.Connection object at 0x7f4649be4
┆ 770>
┆ multiprocessing.Pipe = <method 'BaseContext.Pipe' of <multiprocessing.context.Defau
┆ ltContext object at 0x7f4650658530> context.py:60>
┆ timeout = 500
┆ FAILURE = 1
┆ future.cancelled = <method 'Future.cancelled' of <ProcessFuture at 0x7f464b201c
┆ 70 state=finished raised AttributeError> _base.py:383>
┆ pipe.recv = <method '_ConnectionBase.recv' of <multiprocessing.connectio
┆ n.Connection object at 0x7f4649be4770> connection.py:246>
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/multiprocessing/connection.py", line 251, in recv
┆ 246 def recv(self):
┆ 247 """Receive a (picklable) object"""
┆ 248 self._check_closed()
┆ 249 self._check_readable()
┆ 250 buf = self._recv_bytes()
┆ --> 251 return _ForkingPickler.loads(buf.getbuffer())
┆ ..................................................
┆ self = <multiprocessing.connection.Connection object at 0x7f4649be4
┆ 770>
┆ self._check_closed = <method '_ConnectionBase._check_closed' of <multiprocessing.
┆ connection.Connection object at 0x7f4649be4770> connection.p
┆ y:135>
┆ self._check_readable = <method '_ConnectionBase._check_readable' of <multiprocessin
┆ g.connection.Connection object at 0x7f4649be4770> connection
┆ .py:139>
┆ buf = <_io.BytesIO object at 0x7f46491c19e0>
┆ self._recv_bytes = <method 'Connection._recv_bytes' of <multiprocessing.connect
┆ ion.Connection object at 0x7f4649be4770> connection.py:429>
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/site-packages/gspread/exceptions.py", line 43, in __init__
┆ 42 def __init__(self, response: Response):
┆ --> 43 super().__init__(self._extract_text(response))
┆ 44 self.response: Response = response
┆ ..................................................
┆ self = APIError({'code': 503, 'message': 'The service is currently
┆ unavailable.', 'status': 'UNAVAILABLE'})
┆ response = {'code': 503,
┆ 'message': 'The service is currently unavailable.',
┆ 'status': 'UNAVAILABLE'}
┆ self._extract_text = <method 'APIError._extract_text' of {'code': 503, 'message':
┆ 'The service is currently unavailable.', 'status': 'UNAVAIL
┆ ABLE'} exceptions.py:46>
┆ self.response = # AttributeError
┆ self = APIError({'code': 503, 'message': 'The service is cur
┆ rently unavailable.', 'status': 'UNAVAILABLE'})
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/site-packages/gspread/exceptions.py", line 49, in _extract_text
┆ 46 def _extract_text(
┆ 47 self, response: Response
┆ 48 ) -> Union[Dict[str, Union[int, str]], str]:
┆ --> 49 return self._text_from_detail(response) or response.text
┆ ..................................................
┆ self = APIError({'code': 503, 'message': 'The service is currently
┆ unavailable.', 'status': 'UNAVAILABLE'})
┆ response = {'code': 503,
┆ 'message': 'The service is currently unavailable.',
┆ 'status': 'UNAVAILABLE'}
┆ self._text_from_detail = <method 'APIError._text_from_detail' of {'code': 503, 'messa
┆ ge': 'The service is currently unavailable.', 'status': 'UNA
┆ VAILABLE'} exceptions.py:51>
┆ response.text = # AttributeError
┆ response = {'code': 503,
┆ 'message': 'The service is currently unavailable.',
┆ 'status': 'UNAVAILABLE'}
┆ ..................................................
┆
┆ ---- (full traceback above) ----
┆ File "/app/run.py", line 53, in <module>
┆ results = future.result()
┆ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 456, in result
┆ return self.__get_result()
┆ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
┆ raise self._exception
┆ File "/usr/local/lib/python3.12/site-packages/pebble/concurrent/process.py", line 178, in _get_result
┆ return pipe.recv()
┆ File "/usr/local/lib/python3.12/multiprocessing/connection.py", line 251, in recv
┆ return _ForkingPickler.loads(buf.getbuffer())
┆ File "/usr/local/lib/python3.12/site-packages/gspread/exceptions.py", line 43, in __init__
┆ super().__init__(self._extract_text(response))
┆ File "/usr/local/lib/python3.12/site-packages/gspread/exceptions.py", line 49, in _extract_text
┆ return self._text_from_detail(response) or response.text
┆
┆ AttributeError: 'dict' object has no attribute 'text'
```01.03.24 06:00:10 ERROR module: run __main__ func: <module> line: 57 msg: 'dict' object has no attribute 'text'
┆ File "/app/run.py", line 53, in <module>
┆ 49 if __name__ == '__main__':
┆ 50 while True:
┆ 51 try:
┆ 52 future = main()
┆ --> 53 results = future.result()
┆ 54 except TimeoutError as error:
┆ ..................................................
┆ __name__ = '__main__'
┆ future = <ProcessFuture at 0x7f464b201c70 state=finished raised Attri
┆ buteError>
┆ results = None
┆ future.result = <method 'Future.result' of <ProcessFuture at 0x7f464b201c70
┆ state=finished raised AttributeError> _base.py:428>
┆ error = AttributeError("'dict' object has no attribute 'text'")
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 456, in result
┆ 428 def result(self, timeout=None):
┆ (...)
┆ 452
┆ 453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
┆ 454 raise CancelledError()
┆ 455 elif self._state == FINISHED:
┆ --> 456 return self.__get_result()
┆ 457 else:
┆ ..................................................
┆ self = None
┆ timeout = None
┆ self._state = # AttributeError
┆ self = None
┆ CANCELLED = 'CANCELLED'
┆ CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
┆ FINISHED = 'FINISHED'
┆ self.__get_result = # AttributeError
┆ self = None
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
┆ 398 def __get_result(self):
┆ 399 if self._exception:
┆ 400 try:
┆ --> 401 raise self._exception
┆ 402 finally:
┆ ..................................................
┆ self = None
┆ self._exception = # AttributeError
┆ self = None
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/site-packages/pebble/concurrent/process.py", line 178, in _get_result
┆ 163 def _get_result(
┆ 164 future: ProcessFuture,
┆ 165 pipe: multiprocessing.Pipe,
┆ 166 timeout: float
┆ 167 ) -> Any:
┆ (...)
┆ 174 return Result(FAILURE, TimeoutError('Task Timeout', timeout))
┆ 175 if future.cancelled():
┆ 176 return Result(FAILURE, CancelledError())
┆ 177
┆ --> 178 return pipe.recv()
┆ 179 except (EOFError, OSError):
┆ ..................................................
┆ future = <ProcessFuture at 0x7f464b201c70 state=finished raised Attri
┆ buteError>
┆ pipe = <multiprocessing.connection.Connection object at 0x7f4649be4
┆ 770>
┆ multiprocessing.Pipe = <method 'BaseContext.Pipe' of <multiprocessing.context.Defau
┆ ltContext object at 0x7f4650658530> context.py:60>
┆ timeout = 500
┆ FAILURE = 1
┆ future.cancelled = <method 'Future.cancelled' of <ProcessFuture at 0x7f464b201c
┆ 70 state=finished raised AttributeError> _base.py:383>
┆ pipe.recv = <method '_ConnectionBase.recv' of <multiprocessing.connectio
┆ n.Connection object at 0x7f4649be4770> connection.py:246>
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/multiprocessing/connection.py", line 251, in recv
┆ 246 def recv(self):
┆ 247 """Receive a (picklable) object"""
┆ 248 self._check_closed()
┆ 249 self._check_readable()
┆ 250 buf = self._recv_bytes()
┆ --> 251 return _ForkingPickler.loads(buf.getbuffer())
┆ ..................................................
┆ self = <multiprocessing.connection.Connection object at 0x7f4649be4
┆ 770>
┆ self._check_closed = <method '_ConnectionBase._check_closed' of <multiprocessing.
┆ connection.Connection object at 0x7f4649be4770> connection.p
┆ y:135>
┆ self._check_readable = <method '_ConnectionBase._check_readable' of <multiprocessin
┆ g.connection.Connection object at 0x7f4649be4770> connection
┆ .py:139>
┆ buf = <_io.BytesIO object at 0x7f46491c19e0>
┆ self._recv_bytes = <method 'Connection._recv_bytes' of <multiprocessing.connect
┆ ion.Connection object at 0x7f4649be4770> connection.py:429>
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/site-packages/gspread/exceptions.py", line 43, in __init__
┆ 42 def __init__(self, response: Response):
┆ --> 43 super().__init__(self._extract_text(response))
┆ 44 self.response: Response = response
┆ ..................................................
┆ self = APIError({'code': 503, 'message': 'The service is currently
┆ unavailable.', 'status': 'UNAVAILABLE'})
┆ response = {'code': 503,
┆ 'message': 'The service is currently unavailable.',
┆ 'status': 'UNAVAILABLE'}
┆ self._extract_text = <method 'APIError._extract_text' of {'code': 503, 'message':
┆ 'The service is currently unavailable.', 'status': 'UNAVAIL
┆ ABLE'} exceptions.py:46>
┆ self.response = # AttributeError
┆ self = APIError({'code': 503, 'message': 'The service is cur
┆ rently unavailable.', 'status': 'UNAVAILABLE'})
┆ ..................................................
┆
┆ File "/usr/local/lib/python3.12/site-packages/gspread/exceptions.py", line 49, in _extract_text
┆ 46 def _extract_text(
┆ 47 self, response: Response
┆ 48 ) -> Union[Dict[str, Union[int, str]], str]:
┆ --> 49 return self._text_from_detail(response) or response.text
┆ ..................................................
┆ self = APIError({'code': 503, 'message': 'The service is currently
┆ unavailable.', 'status': 'UNAVAILABLE'})
┆ response = {'code': 503,
┆ 'message': 'The service is currently unavailable.',
┆ 'status': 'UNAVAILABLE'}
┆ self._text_from_detail = <method 'APIError._text_from_detail' of {'code': 503, 'messa
┆ ge': 'The service is currently unavailable.', 'status': 'UNA
┆ VAILABLE'} exceptions.py:51>
┆ response.text = # AttributeError
┆ response = {'code': 503,
┆ 'message': 'The service is currently unavailable.',
┆ 'status': 'UNAVAILABLE'}
┆ ..................................................
┆
┆ ---- (full traceback above) ----
┆ File "/app/run.py", line 53, in <module>
┆ results = future.result()
┆ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 456, in result
┆ return self.__get_result()
┆ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
┆ raise self._exception
┆ File "/usr/local/lib/python3.12/site-packages/pebble/concurrent/process.py", line 178, in _get_result
┆ return pipe.recv()
┆ File "/usr/local/lib/python3.12/multiprocessing/connection.py", line 251, in recv
┆ return _ForkingPickler.loads(buf.getbuffer())
┆ File "/usr/local/lib/python3.12/site-packages/gspread/exceptions.py", line 43, in __init__
┆ super().__init__(self._extract_text(response))
┆ File "/usr/local/lib/python3.12/site-packages/gspread/exceptions.py", line 49, in _extract_text
┆ return self._text_from_detail(response) or response.text
┆
┆ AttributeError: 'dict' object has no attribute 'text'
noxdafox commented
Issue fixed in release 5.0.7, please re-open this issue if the problem persists.