Unable to run the proxy scraper

Question

Unable to run the proxy scraper

BukuBukuChagma opened this issue 2 years ago · 4 comments

When I run it after installing all the requirements, this is what I get

E:\scripts\proxy-scraper>python proxyScraper.py -p http
Traceback (most recent call last):
File "E:\scripts\proxy-scraper\proxyScraper.py", line 169, in
asyncio.run(scrape(args.proxy, args.output, args.verbose))
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 646, in run_until_complete
return future.result()
File "E:\scripts\proxy-scraper\proxyScraper.py", line 136, in scrape
await asyncio.gather(*tasks)
File "E:\scripts\proxy-scraper\proxyScraper.py", line 131, in scrape_scraper
proxies.extend(await scraper.scrape(client))
File "E:\scripts\proxy-scraper\proxyScraper.py", line 27, in scrape
response = await self.get_response(client)
File "E:\scripts\proxy-scraper\proxyScraper.py", line 21, in get_response
return await client.get(self.get_url())
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1757, in get
return await self.request(
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1533, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1620, in send
response = await self._send_handling_auth(
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1648, in _send_handling_auth
response = await self._send_handling_redirects(
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1685, in _send_handling_redirects
response = await self._send_single_request(request)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1722, in _send_single_request
response = await transport.handle_async_request(request)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_transports\default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection_pool.py", line 253, in handle_async_request
raise exc
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection_pool.py", line 237, in handle_async_request
response = await connection.handle_async_request(request)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 86, in handle_async_request
raise exc
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 63, in handle_async_request
stream = await self._connect(request)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 150, in _connect
stream = await stream.start_tls(**kwargs)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore\backends\asyncio.py", line 76, in start_tls
raise exc
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore\backends\asyncio.py", line 67, in start_tls
ssl_stream = await anyio.streams.tls.TLSStream.wrap(
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\anyio\streams\tls.py", line 122, in wrap
await wrapper._call_sslobject_method(ssl_object.do_handshake)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\anyio\streams\tls.py", line 130, in _call_sslobject_method
result = func(*args)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 975, in do_handshake
self._sslobj.do_handshake()
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:997)
Exception ignored in: <function _ProactorBasePipeTransport.del at 0x000001813AE9E3B0>
Traceback (most recent call last):
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 116, in del
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 108, in close
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 750, in call_soon
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 515, in _check_closed
RuntimeError: Event loop is closed
Exception ignored in: <function _ProactorBasePipeTransport.del at 0x000001813AE9E3B0>
Traceback (most recent call last):
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 116, in del
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 108, in close
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 750, in call_soon
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 515, in _check_closed
RuntimeError: Event loop is closed
Exception ignored in: <function _ProactorBasePipeTransport.del at 0x000001813AE9E3B0>
Traceback (most recent call last):
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 116, in del
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 108, in close
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 750, in call_soon
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 515, in _check_closed
RuntimeError: Event loop is closed
Exception ignored in: <function _ProactorBasePipeTransport.del at 0x000001813AE9E3B0>
Traceback (most recent call last):
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 116, in del
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 108, in close
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 750, in call_soon
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 515, in _check_closed
RuntimeError: Event loop is closed

E:\scripts\proxy-scraper>

Am I doing something wrong here?

Answer 1 · 2022-12-15T18:12:37.000Z

Hi,
I updated the source code.
Please get the new source code and test it again.
If you still have a problem, let me know; Because I don't have access to windows to test it.

Answer 2 · 2022-12-16T05:48:58.000Z

Cloned the updated repo but looks like the error is still there. Exact same error as the one above ssl handshake error.

Answer 3 · 2022-12-17T01:08:00.000Z

Sorry, my bad.
I updated it again. Please try again.
If it does not work again, try to remove lines from 169 to 179 and just keep:

        loop = asyncio.get_event_loop()
        loop.run_until_complete(scrape(args.proxy, args.output, args.verbose))
        loop.close()

You can read more about the problem here.

Answer 4 · 2022-12-17T05:48:35.000Z

The event loop problem is gone but this error still exists:

ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:997)

I removed all the lines from 169 onward and kept the ones you mentioned and it looks like this line is causing the error:

loop = asyncio.get_event_loop()
loop.run_until_complete(scrape(args.proxy, args.output, args.verbose))

Complete error:

E:\scripts\proxy-scraper>python proxyScraper.py -p http
E:\scripts\proxy-scraper\proxyScraper.py:170: DeprecationWarning: There is no current event loop
loop = asyncio.get_event_loop()
Traceback (most recent call last):
File "E:\scripts\proxy-scraper\proxyScraper.py", line 171, in
loop.run_until_complete(scrape(args.proxy, args.output, args.verbose))
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 646, in run_until_complete
return future.result()
File "E:\scripts\proxy-scraper\proxyScraper.py", line 137, in scrape
await asyncio.gather(*tasks)
File "E:\scripts\proxy-scraper\proxyScraper.py", line 132, in scrape_scraper
proxies.extend(await scraper.scrape(client))
File "E:\scripts\proxy-scraper\proxyScraper.py", line 28, in scrape
response = await self.get_response(client)
File "E:\scripts\proxy-scraper\proxyScraper.py", line 22, in get_response
return await client.get(self.get_url())
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1757, in get
return await self.request(
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1533, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1620, in send
response = await self._send_handling_auth(
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1648, in _send_handling_auth
response = await self._send_handling_redirects(
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1685, in _send_handling_redirects
response = await self._send_single_request(request)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1722, in _send_single_request
response = await transport.handle_async_request(request)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_transports\default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection_pool.py", line 253, in handle_async_request
raise exc
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection_pool.py", line 237, in handle_async_request
response = await connection.handle_async_request(request)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 86, in handle_async_request
raise exc
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 63, in handle_async_request
stream = await self._connect(request)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 150, in _connect
stream = await stream.start_tls(**kwargs)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore\backends\asyncio.py", line 76, in start_tls
raise exc
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore\backends\asyncio.py", line 67, in start_tls
ssl_stream = await anyio.streams.tls.TLSStream.wrap(
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\anyio\streams\tls.py", line 122, in wrap
await wrapper._call_sslobject_method(ssl_object.do_handshake)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\anyio\streams\tls.py", line 130, in _call_sslobject_method
result = func(*args)
File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 975, in do_handshake
self._sslobj.do_handshake()
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:997)