SQS too slow
Closed this issue · 1 comments
pvalsecc commented
It takes more time to push and pop from the SQS queue than to actually generate the tiles.
I did a small piece of code to compare SQS with Redis (on another machine) and I've got those results:
push_redis(): 1228.101962us/op
stats_redis(): 1159.500808us/op
pop_redis(): 1319.037752us/op
push_sqs(): 48042.700638us/op
stats_sqs(): 8742.319578us/op
pop_sqs(): 15600.274294us/op
The code:
#!/usr/bin/env python3
import boto.sqs
import boto.sqs.jsonmessage
import pickle
import redis
import timeit
NUMBER = 500
sqs_queue = None
redis_con = None
counter = 0
def connect_sqs():
global sqs_queue
connection = boto.sqs.connect_to_region('eu-west-1')
sqs_queue = connection.get_queue('XXXXXXXXXX')
sqs_queue.set_message_class(boto.sqs.jsonmessage.JSONMessage)
def push_sqs():
global sqs_queue, counter
counter += 1
sqs_message = sqs_queue.new_message()
sqs_message['z'] = 0
sqs_message['x'] = 1
sqs_message['y'] = counter
sqs_message['n'] = 3
sqs_message['metadata'] = {'layer': 'toto'}
sqs_queue.write(sqs_message)
def stats_sqs():
global sqs_queue
sqs_queue.get_attributes()
def pop_sqs():
global sqs_queue
sqs_message = sqs_queue.read()
assert sqs_message is not None
# print(sqs_message.get('y'))
sqs_message.delete()
def connect_redis():
global redis_con
redis_con = redis.StrictRedis(host="redis", port=6379)
def push_redis():
global redis_con, counter
counter += 1
message = dict(z=0, x=1, y=counter, n=3, metadata={'layer': 'toto'})
redis_con.lpush('test_queue', pickle.dumps(message))
def stats_redis():
global redis_con
redis_con.llen('test_queue')
def pop_redis():
global redis_con
_queue, message = redis_con.brpop('test_queue')
decoded = pickle.loads(message)
# print(repr(decoded))
def time_it(stmt):
measure = timeit.timeit(stmt=stmt, number=NUMBER, globals=globals())
print("%s: %fus/op" % (stmt, measure/NUMBER*1000000))
def main():
global redis_con, sqs_queue
connect_redis()
time_it('push_redis()')
time_it('stats_redis()')
time_it('pop_redis()')
print("Redis queue length: " + repr(redis_con.llen('test_queue')))
connect_sqs()
time_it('push_sqs()')
time_it('stats_sqs()')
time_it('pop_sqs()')
print("SQS queue attrs: " + repr(sqs_queue.get_attributes()))
main()