Error while listing files
Closed this issue · 2 comments
mbaechtold commented
I'm using the latest release of this package (0.3.0) and when I try to list the file in my bucket, I get an exception.
My code
path = Path.home() / ".aws/exoscale"
exo = Exoscale(config_file=str(path))
bucket = exo.storage.get_bucket(bucket_name)
for item in bucket.list_files(prefix="stuff/"):
print(item)
The exception
Traceback (most recent call last):
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/connection.py", line 159, in _new_conn
conn = connection.create_connection(
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/util/connection.py", line 61, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/socket.py", line 918, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 8] nodename nor servname provided, or not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/httpsession.py", line 254, in send
urllib_response = conn.urlopen(
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 726, in urlopen
retries = retries.increment(
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/util/retry.py", line 379, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 381, in _make_request
self._validate_conn(conn)
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 978, in _validate_conn
conn.connect()
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/connection.py", line 309, in connect
conn = self._new_conn()
File "/path/to/virtualenv/lib/python3.8/site-packages/urllib3/connection.py", line 171, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <botocore.awsrequest.AWSHTTPSConnection object at 0x1104ac370>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/path/to/virtualenv/lib/python3.8/site-packages/exoscale/api/storage.py", line 570, in list_files
for page in pages:
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/paginate.py", line 255, in __iter__
response = self._make_request(current_kwargs)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/paginate.py", line 332, in _make_request
return self._method(**current_kwargs)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/client.py", line 337, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/client.py", line 642, in _make_api_call
http, parsed_response = self._make_request(
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/client.py", line 662, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/endpoint.py", line 102, in make_request
return self._send_request(request_dict, operation_model)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/endpoint.py", line 136, in _send_request
while self._needs_retry(attempts, operation_model, request_dict,
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/endpoint.py", line 253, in _needs_retry
responses = self._event_emitter.emit(
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/hooks.py", line 356, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/hooks.py", line 228, in emit
return self._emit(event_name, kwargs)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/hooks.py", line 211, in _emit
response = handler(**kwargs)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/retryhandler.py", line 183, in __call__
if self._checker(attempts, response, caught_exception):
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/retryhandler.py", line 250, in __call__
should_retry = self._should_retry(attempt_number, response,
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/retryhandler.py", line 277, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/retryhandler.py", line 316, in __call__
checker_response = checker(attempt_number, response,
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/retryhandler.py", line 222, in __call__
return self._check_caught_exception(
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
raise caught_exception
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/endpoint.py", line 200, in _do_get_response
http_response = self._send(request)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/endpoint.py", line 269, in _send
return self.http_session.send(request)
File "/path/to/virtualenv/lib/python3.8/site-packages/botocore/httpsession.py", line 283, in send
raise EndpointConnectionError(endpoint_url=request.url, error=e)
botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL: "https://s3.ch-dk-2.amazonaws.com/prometheus-bl?list-type=2&prefix=stuff%2F&max-keys=100&encoding-type=url"
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "script.py", line 21, in <module>
for item in bucket.list_files(prefix="stuff/"):
File "/path/to/virtualenv/lib/python3.8/site-packages/exoscale/api/storage.py", line 574, in list_files
raise APIException(e)
exoscale.api.APIException: Could not connect to the endpoint URL: "https://s3.ch-dk-2.amazonaws.com/prometheus-bl?list-type=2&prefix=stuff%2F&max-keys=100&encoding-type=url"
The observation
The url is strange.
It's a mixture of a Amazon URL but with the Exoscale zone: https://s3.ch-dk-2.amazonaws.com/prometheus-bl
What am I doing wrong here?
falzm commented
Hi @mbaechtold, I've tested with conditions similar to yours and I don't experience this issue:
$ cat exoscale.conf
[[profiles]]
name = "test"
api_key = "EXOxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
api_secret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
$ cat test.py
from exoscale import Exoscale
exo = Exoscale(config_file="exoscale.conf")
bucket = exo.storage.get_bucket("marc-test")
for item in bucket.list_files(prefix="x/"):
print(item)
$ python test.py
BucketFile(path='x/y')
BucketFile(path='x/z')Can you please share the (redacted) content of your .aws/exoscale config file?
falzm commented
Closing this issue due to inactivity, please re-open it if necessary.