Stamina raises retriable exception before reaching max retries

[garciabruno]

I'm seeing some strange behavior from stamina, where a retriable exception is raised before the max attempts is reached.

Python: 3.10.4
stamina: 24.2.0

Sentry reports this log before the exception is raised:

stamina.retry_scheduled
{
  stamina.args: [],
  stamina.callable: <context block>,
  stamina.caused_by: ReadTimeout(''),
  stamina.kwargs: {},
  stamina.retry_num: 3,
  stamina.wait_for: 0.69,
  stamina.waited_so_far: 1.64
}

stamina.retry_num is 3, but attempts is set to 10 in retry_context

Code:

async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
        for attempt in stamina.retry_context(
            on=(
                httpx.RemoteProtocolError,
                httpx.ReadError,
                httpx.ReadTimeout,
            ),
            attempts=settings.HTTP_CLIENT_RETRIES,
        ):
            with attempt:
                return await self._wrapper.handle_async_request(request)

        # This code should be unreachable, since stamina will raise an exception
        # if the maximum number of retries is exceeded.
        raise Exception("Max retries exceeded.")  # pragma: no cover

Stack trace:

ReadTimeout: null
  File "httpx/_transports/default.py", line 69, in map_httpcore_exceptions
    yield
  File "httpx/_transports/default.py", line 373, in handle_async_request
    resp = await self._pool.handle_async_request(req)
  File "httpcore/_async/connection_pool.py", line 216, in handle_async_request
    raise exc from None
  File "httpcore/_async/connection_pool.py", line 196, in handle_async_request
    response = await connection.handle_async_request(
  File "httpcore/_async/connection.py", line 101, in handle_async_request
    return await self._connection.handle_async_request(request)
  File "httpcore/_async/http11.py", line 143, in handle_async_request
    raise exc
  File "httpcore/_async/http11.py", line 113, in handle_async_request
    ) = await self._receive_response_headers(**kwargs)
  File "httpcore/_async/http11.py", line 186, in _receive_response_headers
    event = await self._receive_event(timeout=timeout)
  File "httpcore/_async/http11.py", line 224, in _receive_event
    data = await self._network_stream.read(
  File "httpcore/_backends/anyio.py", line 31, in read
    with map_exceptions(exc_map):
  File "contextlib.py", line 153, in __exit__
    self.gen.throw(typ, value, traceback)
  File "httpcore/_exceptions.py", line 14, in map_exceptions
    raise to_exc(exc) from exc
ReadTimeout: null
  File "app/services/external/gamification.py", line 25, in get_entity_for_action
    entity_response = await get_blocks_pivot_data(field=EBlockField.ID, value=action.entity_id)
  File "app/http/courses.py", line 243, in get_blocks_pivot_data
    response = await pool.courses.get(
  File "httpx/_client.py", line 1801, in get
    return await self.request(
  File "httpx/_client.py", line 1574, in request
    return await self.send(request, auth=auth, follow_redirects=follow_redirects)
  File "httpx/_client.py", line 1661, in send
    response = await self._send_handling_auth(
  File "httpx/_client.py", line 1689, in _send_handling_auth
    response = await self._send_handling_redirects(
  File "httpx/_client.py", line 1726, in _send_handling_redirects
    response = await self._send_single_request(request)
  File "httpx/_client.py", line 1763, in _send_single_request
    response = await transport.handle_async_request(request)
  File "app/http/pool.py", line 16, in handle_async_request
    for attempt in stamina.retry_context(
  File "stamina/_core.py", line 439, in __iter__
    for r in _t.Retrying(
  File "__init__.py", line 347, in __iter__
    do = self.iter(retry_state=retry_state)
  File "__init__.py", line 325, in iter
    raise retry_exc.reraise()
  File "__init__.py", line 158, in reraise
    raise self.last_attempt.result()
  File "concurrent/futures/_base.py", line 451, in result
    return self.__get_result()
  File "concurrent/futures/_base.py", line 403, in __get_result
    raise self._exception
  File "app/http/pool.py", line 26, in handle_async_request
    return await self._wrapper.handle_async_request(request)
  File "httpx/_transports/default.py", line 372, in handle_async_request
    with map_httpcore_exceptions():
  File "contextlib.py", line 153, in __exit__
    self.gen.throw(typ, value, traceback)
  File "httpx/_transports/default.py", line 86, in map_httpcore_exceptions
    raise mapped_exc(message) from exc

Is there any direction you could point me to debug this issue, or is it a bug in the library?
Any help is appreciated.

[hynek]

The number of retries is also capped by the timeout argument so you probably have to raise it if you want to have all your retries to go through.

[garciabruno]

👍🏻 I was indeed hitting a timeout limit, thank you for the help.