Too verbose timeout error from floyd logs
msoedov opened this issue · 3 comments
msoedov commented
floyd logs -t ....
After some time
.....
Traceback (most recent call last):
File "lib/python3.6/site-packages/urllib3/connectionpool.py", line 346, in _make_request
self._validate_conn(conn)
File "lib/python3.6/site-packages/urllib3/connectionpool.py", line 850, in _validate_conn
conn.connect()
File "lib/python3.6/site-packages/urllib3/connection.py", line 326, in connect
ssl_context=context)
File "lib/python3.6/site-packages/urllib3/util/ssl_.py", line 329, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py", line 401, in wrap_socket
_context=self, _session=session)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py", line 808, in __init__
self.do_handshake()
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py", line 1061, in do_handshake
self._sslobj.do_handshake()
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py", line 683, in do_handshake
self._sslobj.do_handshake()
socket.timeout: _ssl.c:732: The handshake operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "lib/python3.6/site-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "lib/python3.6/site-packages/urllib3/connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "lib/python3.6/site-packages/urllib3/util/retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "lib/python3.6/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "lib/python3.6/site-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "lib/python3.6/site-packages/urllib3/connectionpool.py", line 349, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
File "lib/python3.6/site-packages/urllib3/connectionpool.py", line 309, in _raise_timeout
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='www.floydhub.com', port=443): Read timed out. (read timeout=5)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "bin/floyd", line 11, in <module>
sys.exit(cli())
File "lib/python3.6/site-packages/click/core.py", line 722, in __call__
return self.main(*args, **kwargs)
File "lib/python3.6/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "lib/python3.6/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "lib/python3.6/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "lib/python3.6/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "lib/python3.6/site-packages/floyd/cli/experiment.py", line 174, in logs
log_file_contents = ResourceClient().get_content(instance_log_id)
File "lib/python3.6/site-packages/floyd/client/resource.py", line 31, in get_content
response = self.request('GET', self.URL_PREFIX + resource_id + "?content=true")
File "lib/python3.6/site-packages/floyd/client/base.py", line 62, in request
timeout=timeout)
File "lib/python3.6/site-packages/requests/api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "lib/python3.6/site-packages/requests/sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "lib/python3.6/site-packages/raven/breadcrumbs.py", line 303, in send
resp = real_send(self, request, *args, **kwargs)
File "lib/python3.6/site-packages/requests/sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "lib/python3.6/site-packages/requests/adapters.py", line 521, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='www.floydhub.com', port=443): Read timed out. (read timeout=5)
Sentry is attempting to send 1 pending error messages
Waiting up to 10 seconds
Press Ctrl-C to quit
Sentry responded with an error: <urlopen error The write operation timed out> (url: https://sentry.io/api/226940/store/)
Traceback (most recent call last):
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1065, in _send_output
self.send(chunk)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 986, in send
self.sock.sendall(data)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py", line 965, in sendall
v = self.send(data[count:])
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py", line 935, in send
return self._sslobj.write(data)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py", line 636, in write
return self._sslobj.write(data)
socket.timeout: The write operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "lib/python3.6/site-packages/raven/transport/threaded.py", line 165, in send_sync
super(ThreadedHTTPTransport, self).send(url, data, headers)
File "lib/python3.6/site-packages/raven/transport/http.py", line 43, in send
ca_certs=self.ca_certs,
File "lib/python3.6/site-packages/raven/utils/http.py", line 66, in urlopen
return opener.open(url, data, timeout)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 526, in open
response = self._open(req, data)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 544, in _open
'_open', req)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "lib/python3.6/site-packages/raven/utils/http.py", line 46, in https_open
return self.do_open(ValidHTTPSConnection, req)
File "/usr/local/Cellar/python3/3.6.2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error The write operation timed out>
["ReadTimeout: HTTPSConnectionPool(host='www.floydhub.com', port=443): Read timed out. (read timeout=5)", ' File "bin/floyd", line 11, in <module>', ' File "click/core.py", line 722, in __call__', ' File "click/core.py", line 697, in main', ' File "click/core.py", line 1066, in invoke', ' File "click/core.py", line 895, in invoke', ' File "click/core.py", line 535, in invoke', ' File "floyd/cli/experiment.py", line 174, in logs', ' File "floyd/client/resource.py", line 31, in get_content', ' File "floyd/client/base.py", line 62, in request', ' File "requests/api.py", line 58, in request', ' File "requests/sessions.py", line 508, in request', ' File "raven/breadcrumbs.py", line 303, in send', ' File "requests/sessions.py", line 618, in send', ' File "requests/adapters.py", line 521, in send']
houqp commented
Agreed, this exception needs to be caught in the code.
msoedov commented
I would suggest to do a retry on timeout error
houqp commented
Sorry that I missed your comment. The reason we didn't go with retry is for the following edge case:
- user fired the request to server
- server received the request, performed the change in the backend
- network issue happened while server streaming back the response
- user get a timeout error
For some of the actions like running jobs, it won't be safe to retry in this case otherwise users will get multiple jobs submitted.