SQS too slow

[pvalsecc]

It takes more time to push and pop from the SQS queue than to actually generate the tiles.

I did a small piece of code to compare SQS with Redis (on another machine) and I've got those results:

push_redis(): 1228.101962us/op
stats_redis(): 1159.500808us/op
pop_redis(): 1319.037752us/op

push_sqs(): 48042.700638us/op
stats_sqs(): 8742.319578us/op
pop_sqs(): 15600.274294us/op

The code:

#!/usr/bin/env python3
import boto.sqs
import boto.sqs.jsonmessage
import pickle
import redis
import timeit

NUMBER = 500

sqs_queue = None
redis_con = None
counter = 0

def connect_sqs():
    global sqs_queue
    connection = boto.sqs.connect_to_region('eu-west-1')
    sqs_queue = connection.get_queue('XXXXXXXXXX')
    sqs_queue.set_message_class(boto.sqs.jsonmessage.JSONMessage)


def push_sqs():
    global sqs_queue, counter
    counter += 1
    sqs_message = sqs_queue.new_message()
    sqs_message['z'] = 0
    sqs_message['x'] = 1
    sqs_message['y'] = counter
    sqs_message['n'] = 3
    sqs_message['metadata'] = {'layer': 'toto'}

    sqs_queue.write(sqs_message)


def stats_sqs():
    global sqs_queue
    sqs_queue.get_attributes()


def pop_sqs():
    global sqs_queue
    sqs_message = sqs_queue.read()
    assert sqs_message is not None
    # print(sqs_message.get('y'))
    sqs_message.delete()


def connect_redis():
    global redis_con
    redis_con = redis.StrictRedis(host="redis", port=6379)


def push_redis():
    global redis_con, counter
    counter += 1
    message = dict(z=0, x=1, y=counter, n=3, metadata={'layer': 'toto'})
    redis_con.lpush('test_queue', pickle.dumps(message))


def stats_redis():
    global redis_con
    redis_con.llen('test_queue')


def pop_redis():
    global redis_con
    _queue, message = redis_con.brpop('test_queue')
    decoded = pickle.loads(message)
    # print(repr(decoded))


def time_it(stmt):
    measure = timeit.timeit(stmt=stmt, number=NUMBER, globals=globals())
    print("%s: %fus/op" % (stmt, measure/NUMBER*1000000))


def main():
    global redis_con, sqs_queue
    connect_redis()
    time_it('push_redis()')
    time_it('stats_redis()')
    time_it('pop_redis()')
    print("Redis queue length: " + repr(redis_con.llen('test_queue')))

    connect_sqs()
    time_it('push_sqs()')
    time_it('stats_sqs()')
    time_it('pop_sqs()')
    print("SQS queue attrs: " + repr(sqs_queue.get_attributes()))


main()

[pvalsecc]

Commit 4121880 introduced support for using redis as queue.