Приложение `post` падает при отсутствии zipkin
Frodox opened this issue · 5 comments
Frodox commented
Hw#25
{
"_index": "fluentd-20180519",
"_type": "access_log",
"_id": "AWN46ikOv3Yq8u2uH3xC",
"_score": 1,
"_source": {
"log": "{\"event\": \"internal_error\", \"level\": \"error\", \"method\": \"GET\", \"path\": \"/posts?\", \"remote_addr\": \"172.18.0.3\", \"request_id\": \"d6f9abe5-afbd-4459-9964-4376481b4170\", \"service\": \"post\", \"timestamp\": \"2018-05-19 15:00:01\", \"traceback\": \"Traceback (most recent call last):\\n File \\\"/usr/local/lib/python3.6/site-packages/urllib3/connection.py\\\", line 141, in _new_conn\\n (self.host, self.port), self.timeout, **extra_kw)\\n File \\\"/usr/local/lib/python3.6/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n File \\\"/usr/local/lib/python3.6/socket.py\\\", line 743, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\nsocket.gaierror: [Errno -2] Name does not resolve\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py\\\", line 601, in urlopen\\n chunked=chunked)\\n File \\\"/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py\\\", line 357, in _make_request\\n conn.request(method, url, **httplib_request_kw)\\n File \\\"/usr/local/lib/python3.6/http/client.py\\\", line 1239, in request\\n self._send_request(method, url, body, headers, encode_chunked)\\n File \\\"/usr/local/lib/python3.6/http/client.py\\\", line 1285, in _send_request\\n self.endheaders(body, encode_chunked=encode_chunked)\\n File \\\"/usr/local/lib/python3.6/http/client.py\\\", line 1234, in endheaders\\n self._send_output(message_body, encode_chunked=encode_chunked)\\n File \\\"/usr/local/lib/python3.6/http/client.py\\\", line 1026, in _send_output\\n self.send(msg)\\n File \\\"/usr/local/lib/python3.6/http/client.py\\\", line 964, in send\\n self.connect()\\n File \\\"/usr/local/lib/python3.6/site-packages/urllib3/connection.py\\\", line 166, in connect\\n conn = self._new_conn()\\n File \\\"/usr/local/lib/python3.6/site-packages/urllib3/connection.py\\\", line 150, in _new_conn\\n self, \\\"Failed to establish a new connection: %s\\\" % e)\\nurllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f4fc4860358>: Failed to establish a new connection: [Errno -2] Name does not resolve\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/local/lib/python3.6/site-packages/requests/adapters.py\\\", line 440, in send\\n timeout=timeout\\n File \\\"/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py\\\", line 639, in urlopen\\n _stacktrace=sys.exc_info()[2])\\n File \\\"/usr/local/lib/python3.6/site-packages/urllib3/util/retry.py\\\", line 388, in increment\\n raise MaxRetryError(_pool, url, error or ResponseError(cause))\\nurllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='zipkin', port=9411): Max retries exceeded with url: /api/v1/spans (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f4fc4860358>: Failed to establish a new connection: [Errno -2] Name does not resolve',))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/local/lib/python3.6/site-packages/flask/app.py\\\", line 1612, in full_dispatch_request\\n rv = self.dispatch_request()\\n File \\\"/usr/local/lib/python3.6/site-packages/flask/app.py\\\", line 1598, in dispatch_request\\n return self.view_functions[rule.endpoint](**req.view_args)\\n File \\\"post_app.py\\\", line 96, in posts\\n posts = find_posts()\\n File \\\"/usr/local/lib/python3.6/site-packages/py_zipkin/zipkin.py\\\", line 296, in __exit__\\n self.stop(_exc_type, _exc_value, _exc_traceback)\\n File \\\"/usr/local/lib/python3.6/site-packages/py_zipkin/zipkin.py\\\", line 314, in stop\\n self.logging_context.stop()\\n File \\\"/usr/local/lib/python3.6/site-packages/py_zipkin/logging_helper.py\\\", line 76, in stop\\n self.log_spans()\\n File \\\"/usr/local/lib/python3.6/site-packages/py_zipkin/logging_helper.py\\\", line 185, in log_spans\\n transport_handler=self.transport_handler,\\n File \\\"/usr/local/lib/python3.6/site-packages/py_zipkin/logging_helper.py\\\", line 325, in log_span\\n transport_handler(message)\\n File \\\"post_app.py\\\", line 55, in http_transport\\n headers={'Content-Type': 'application/x-thrift'})\\n File \\\"/usr/local/lib/python3.6/site-packages/requests/api.py\\\", line 112, in post\\n return request('post', url, data=data, json=json, **kwargs)\\n File \\\"/usr/local/lib/python3.6/site-packages/requests/api.py\\\", line 58, in request\\n return session.request(method=method, url=url, **kwargs)\\n File \\\"/usr/local/lib/python3.6/site-packages/requests/sessions.py\\\", line 508, in request\\n resp = self.send(prep, **send_kwargs)\\n File \\\"/usr/local/lib/python3.6/site-packages/requests/sessions.py\\\", line 618, in send\\n r = adapter.send(request, **kwargs)\\n File \\\"/usr/local/lib/python3.6/site-packages/requests/adapters.py\\\", line 508, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPConnectionPool(host='zipkin', port=9411): Max retries exceeded with url: /api/v1/spans (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f4fc4860358>: Failed to establish a new connection: [Errno -2] Name does not resolve',))\\n\"}",
"container_id": "2ece463ebd7f7345becf89e01a70efb554248b499acfb440756d60bc8717ee33",
"container_name": "/docker_post_1",
"source": "stdout",
"@timestamp": "2018-05-19T15:00:01+00:00",
"@log_name": "service.post"
},
"fields": {
"@timestamp": [
1526742001000
]
}
}
serjs commented
Т.к. изначальный коммит содержит в себе изменения не только связанные с логами, быстро решить проблему не получается.
Я также заметил что пост при этом функционирует, но отображает посты только когда zipkin становится доступным.
Я рамках проблемы я бы переделал работу с zipkin во всех приложениях и добавил его подключение через ENV переменную
spinor72 commented
Нужно добавить, обработку ошибки в http_transport() можно еще там-же проверять значение ENV переменной перед отправкой запроса. Тогда не будет падать.
Например
try:
requests.post(ZIPKIN_URL,
data=body,
headers={'Content-Type': 'application/x-thrift'},
)
except requests.exceptions.RequestException:
tb = traceback.format_exc()
log.error('zipkin_error', service='post', traceback=tb)
serjs commented
#3 вмержили, задачу пока отставляю открытой, чтобы исправить сам эксепшн и обработку подключений к зипкин
virtualshuric commented
В ветке Logging необходимые изменения внесены