MD5 hash calculation of multiple simultaneous requests

tornado_multiasynchttpclient_example.py

#!/usr/bin/env python
# encoding: utf-8
import functools
import hashlib
import logging

import tornado.web
import tornado.gen
import tornado.httpclient
import tornado.ioloop


logger = logging.getLogger('asynchttpclient')
hashes = {}


@tornado.gen.coroutine
def calc_urls(urls):
    client = tornado.httpclient.AsyncHTTPClient()

    keys = []
    for i, url in enumerate(urls):
        key = '%sk' % i
        keys.append(key)
        hashes[key] = hashlib.md5()

        params = {
            'url': url,
            'streaming_callback': functools.partial(on_stream, key)
        }
        request = tornado.httpclient.HTTPRequest(**params)
        request.key = key
        client.fetch(request, callback=(yield tornado.gen.Callback(key)))

    responses = yield tornado.gen.WaitAll(keys)

    for rsp in responses:
        msg = rsp.error if rsp.error else hashes[rsp.request.key].hexdigest()

        print '{0}: {1} ({2})'.format(
            rsp.effective_url,
            msg,
            round(rsp.request_time, 4)
        )


def on_stream(key, data):
    hashes[key].update(data)


calc_urls(('http://ya.ru', 'http://nerabochaya-ssylka.ru', 'http://jopa.ru'))
tornado.ioloop.IOLoop.instance().start()