google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling target function, last exception: 504 Deadline Exceeded
leomuseca opened this issue · 1 comments
leomuseca commented
Environment details
- OS type and version:
- Python version: 3.11
- google-cloud-datastore = "^2.13.2"
Steps to reproduce
- Service is unavailable 3-4 times a day
Stack trace
Traceback (most recent call last):
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/grpc_helpers.py", line 72, in error_remapped_callable
return callable_(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.DEADLINE_EXCEEDED
details = "Deadline Exceeded"
debug_error_string = "UNKNOWN:Deadline Exceeded {created_time:"2023-03-30T12:28:08.527586383+00:00", grpc_status:4}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/retry.py", line 191, in retry_target
return target()
^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/timeout.py", line 120, in func_with_timeout
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/grpc_helpers.py", line 74, in error_remapped_callable
raise exceptions.from_grpc_error(exc) from exc
google.api_core.exceptions.DeadlineExceeded: 504 Deadline Exceeded
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 419, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/fastapi/applications.py", line 270, in __call__
await super().__call__(scope, receive, send)
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/applications.py", line 124, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/middleware/errors.py", line 184, in __call__
raise exc
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/middleware/errors.py", line 162, in __call__
await self.app(scope, receive, _send)
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/middleware/base.py", line 106, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspaces/fast_api_muse/app/helpers/decorators.py", line 9, in wrapper
response = await func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspaces/fast_api_muse/app/main.py", line 71, in middelware
response = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/middleware/base.py", line 80, in call_next
raise app_exc
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/middleware/base.py", line 69, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
raise exc
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
await self.app(scope, receive, sender)
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
raise e
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/routing.py", line 706, in __call__
await route.handle(scope, receive, send)
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/routing.py", line 276, in handle
await self.app(scope, receive, send)
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/starlette/routing.py", line 66, in app
response = await func(request)
^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/fastapi/routing.py", line 237, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/fastapi/routing.py", line 163, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/page_iterator.py", line 208, in _items_iter
for page in self._page_iter(increment=False):
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/page_iterator.py", line 244, in _page_iter
page = self._next_page()
^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/cloud/datastore/query.py", line 627, in _next_page
response_pb = self.client._datastore_api.run_query(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/cloud/datastore_v1/services/datastore/client.py", line 635, in run_query
response = rpc(
^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/gapic_v1/method.py", line 113, in __call__
return wrapped_func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/retry.py", line 349, in retry_wrapped_func
return retry_target(
^^^^^^^^^^^^^
File "/home/vscode/.cache/pypoetry/virtualenvs/muse-fast-api-BwhkVub--py3.11/lib/python3.11/site-packages/google/api_core/retry.py", line 207, in retry_target
raise exceptions.RetryError(
google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling target function, last exception: 504 Deadline Exceeded
Thanks!
triplequark commented
Need more info to understand error and reproduce. Please re-open if this continues to be a concern.