python requests 请求下载webdriver失败
JokerYF opened this issue · 9 comments
本人写了一个Python小模块，用于更新selenium中webdriver：webdrivermanagercn
今天在做分布式UI自动化调研时，可能没有对GET请求做限制，以至于现在公司内部均无法正常通过该模块（底层使用requests GET请求获取）请求下载，并报如下错误：
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 793, in urlopen
response = self._make_request(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 491, in _make_request
raise new_e
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 467, in _make_request
self._validate_conn(conn)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1099, in _validate_conn
conn.connect()
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connection.py", line 653, in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connection.py", line 806, in ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/util/ssl.py", line 465, in ssl_wrap_socket
ssl_sock = ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/util/ssl.py", line 509, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/ssl.py", line 501, in wrap_socket
return self.sslsocket_class._create(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/ssl.py", line 1074, in _create
self.do_handshake()
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/ssl.py", line 1343, in do_handshake
self._sslobj.do_handshake()
ConnectionResetError: [Errno 54] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 847, in urlopen
retries = retries.increment(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment
raise reraise(type(error), error, _stacktrace)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/util/util.py", line 38, in reraise
raise value.with_traceback(tb)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 793, in urlopen
response = self._make_request(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 491, in _make_request
raise new_e
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 467, in _make_request
self._validate_conn(conn)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1099, in _validate_conn
conn.connect()
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connection.py", line 653, in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/connection.py", line 806, in ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/util/ssl.py", line 465, in ssl_wrap_socket
ssl_sock = ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/urllib3/util/ssl.py", line 509, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/ssl.py", line 501, in wrap_socket
return self.sslsocket_class._create(
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/ssl.py", line 1074, in _create
self.do_handshake()
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/ssl.py", line 1343, in do_handshake
self._sslobj.do_handshake()
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(54, 'Connection reset by peer'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/yf/PycharmProjects/webdrivermanagercn/test/demo2.py", line 4, in
res = requests.get(url)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/opt/miniconda3/envs/webdrivermanagercn/lib/python3.9/site-packages/requests/adapters.py", line 501, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(54, 'Connection reset by peer'))
请教一下还有可能继续下载吗？如果IP、MAC被封禁后，如何解封？或者多久会自动解封？很紧急，辛苦解决一下。
没有看到报错是什么？请问访问的 url 是那个？
没有看到报错是什么？请问访问的 url 是那个？
import requests
url = 'https://registry.npmmirror.com/-/binary/chrome-for-testing/124.0.6367.207/mac-arm64/chromedriver-mac-arm64.zip'
res = requests.get(url)
通过浏览器访问可以正常下载，使用requests请求则报以上错误，我有一台阿里云主机，主机上测试正常，公司内部网络下的设备均报以上错误
Windows端执行后报错如下：
Traceback (most recent call last):
File "urllib3\connectionpool.py", line 721, in urlopen
File "urllib3\connectionpool.py", line 403, in make_request
File "urllib3\connectionpool.py", line 1053, in validate_conn
File "urllib3\connection.py", line 429, in connect
File "urllib3\util\ssl.py", line 450, in ssl_wrap_socket
File "urllib3\util\ssl.py", line 493, in _ssl_wrap_socket_impl
File "ssl.py", line 423, in wrap_socket
File "ssl.py", line 870, in _create
File "ssl.py", line 1139, in do_handshake
ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "requests\adapters.py", line 497, in send
File "urllib3\connectionpool.py", line 799, in urlopen
File "urllib3\util\retry.py", line 550, in increment
File "urllib3\packages\six.py", line 769, in reraise
File "urllib3\connectionpool.py", line 721, in urlopen
File "urllib3\connectionpool.py", line 403, in make_request
File "urllib3\connectionpool.py", line 1053, in validate_conn
File "urllib3\connection.py", line 429, in connect
File "urllib3\util\ssl.py", line 450, in ssl_wrap_socket
File "urllib3\util\ssl.py", line 493, in _ssl_wrap_socket_impl
File "ssl.py", line 423, in wrap_socket
File "ssl.py", line 870, in _create
File "ssl.py", line 1139, in do_handshake
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, '远程主机强迫关闭了一个现有的连接 。', None, 10054, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "webdrivermanager_cn\core\driver.py", line 132, in install
File "webdrivermanager_cn\core\driver.py", line 115, in download
File "webdrivermanager_cn\core\download_manager.py", line 24, in download_file
File "requests\api.py", line 73, in get
File "requests\api.py", line 59, in request
File "requests\sessions.py", line 589, in request
File "requests\sessions.py", line 725, in send
File "requests\sessions.py", line 725, in
File "requests\sessions.py", line 274, in resolve_redirects
File "requests\sessions.py", line 703, in send
File "requests\adapters.py", line 501, in send
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, '远程主机强迫关闭了一个现有的连接。', None, 10054, None))
那这个就不是我能够解决的了。网络是通的
辛苦看一下是不是服务端有请求限制，我这边测试如果间隔较长时间不请求的话，请求一次是通的，如果马上请求第二次，就报错了，感觉像是有限流机制
没有任何限制，服务端限制会响应标准的 http 结果，你这个是网络异常
这个问题，我还是感觉像是服务端有限制，我分别测试了公司网络、移动热点、家庭网络，分别了用了不同的设备测试，结果都是一样的，都是上面的报错，辛苦再看看吧
你使用 curl 命令或者浏览器应该是可以打开的，如果 python 代码还是有问题，那就是代码问题了。
https://www.cnblogs.com/fengpingfan/p/14433251.html
辛苦再确认一下是否是频繁请求导致命中服务器防火墙相关安全措施，如果确定命中，能不能给一个请求频率的限制？
因为目前测试结果看，通过Linux云服务器测试均正常，可以排除Python代码层面问题，公司内网请求json数据正常，只有在GET请求下载zip包的时候，会直接报错，是否触发了某些反爬等安全机制？