Poor performance running in VMware Linux
MMquant opened this issue · 1 comments
MMquant commented
When querying thousands of public resolvers asynchronously on dedicated Linux / OS X machine I'm getting about 760 errors in 10700 queries. When I run exactly same code in VMware Linux I'm getting about 7080 errors. Vast majority of errors are Timeouts.
My aiodns settings are:
AIODNS_TIMEOUT = 3
AIODNS_RETRY = 2
On Linux count of errors may be drastically lowered by setting
AIODNS_TIMEOUT = 7
AIODNS_RETRY = 5
which however has serious impact on performance.
See fully working debug code below
import asyncio
import aiodns
import requests
import re
AIODNS_TIMEOUT = 3
AIODNS_RETRY = 2
class Fetcher(object):
def __init__(self):
self.loop = asyncio.get_event_loop()
def get_records(self, names, query_type, resolvers):
coros = [self._query_sweep_resolvers(names, query_type, resolver) for resolver in resolvers]
tasks = asyncio.gather(*coros, return_exceptions=True)
records = self.loop.run_until_complete(tasks)
return records
async def _query_sweep_resolvers(self, name, query_type, nameserver):
resolver = aiodns.DNSResolver(
nameservers=[nameserver],
timeout=AIODNS_TIMEOUT,
tries=AIODNS_RETRY,
loop=self.loop
)
try:
result = await resolver.query(name, query_type)
except aiodns.error.DNSError as e:
result = e
return {'ns': nameserver,'name': name ,'type': query_type, 'result': result}
def errors_count(results):
count = 0
for result in results:
if type(result['result']) is aiodns.error.DNSError:
count += 1
return count
def get_resolvers():
data = requests.get('https://public-dns.info/nameservers.csv')
data_list = data.text.split('\n')
ips = []
for resolver in data_list[:-1]:
ip = resolver.split(',')[0]
reliability = resolver.split(',')[7]
if re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', ip) and reliability == '1.00':
ips.append(ip)
return ips
if __name__ == '__main__':
fetcher = Fetcher()
resolvers = get_resolvers()
results = fetcher.get_records('www.flickr.com', 'A', resolvers)
errors = errors_count(results)
pass
Output of ulimit
(.venv) root@kali:~/Programs/DNSweeper# ulimit -a
core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimited
scheduling priority (-e) 0
file size (blocks, -f) unlimited
pending signals (-i) 15552
max locked memory (kbytes, -l) 16384
max memory size (kbytes, -m) unlimited
open files (-n) 65535
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0
stack size (kbytes, -s) 8192
cpu time (seconds, -t) unlimited
max user processes (-u) 65535
virtual memory (kbytes, -v) unlimited
file locks (-x) unlimited
Output of uname
(.venv) root@kali:~/Programs/DNSweeper# uname -a
Linux kali 4.18.0-kali2-amd64 #1 SMP Debian 4.18.10-2kali1 (2018-10-09) x86_64 GNU/Linux
Running Linux in VMware Professional Version 10.1.5 (10950653)
Question is why I'm getting so many Timeout errors?