request_async.py

import argparse
import time
import requests
from concurrent.futures import ThreadPoolExecutor


def main(args):
    URLS = [
        'http://google.com.ar',
        'http://reddit.com',
        'http://imgur.com',
        'http://yahoo.com',
        'http://unca.edu.ar'
    ]

    URLS = URLS if args.count == 5 else URLS * args.count

    is_async = False
    if args.thread:
        is_async = True
        print('Ejecutando con multithreading...')
    else:
        print('Ejecutando sin multithreading...')
    print("     {} URLS - {} WORKERS".format(len(URLS), args.workers))
    t0 = time.time()
    if is_async:
        try:
            with ThreadPoolExecutor(max_workers=args.workers) as executor:
                for url, result in zip(URLS, executor.map(requests.get, URLS)):
                    print('     URL: {} - result: {}'.format(url, result))
        except Exception:
            print('Boom!')
    else:
        for url in URLS:
            print('     URL: {} - result: {}'.format(url, requests.get(url)))

    print('Terminado en {} secs'.format(time.time() - t0))


def _cliparse():
    parse = argparse.ArgumentParser()
    parse.add_argument('-t', '--thread', action='store_true',
                       help='Ejecutar requests con Threads')
    parse.add_argument('-c', '--count', default=5, help='Cantidad de URLS', type=int)
    parse.add_argument('-w', '--workers', default=5, help='Cantidad de Hilos', type=int)
    return parse.parse_args()


if __name__ == '__main__':
    args = _cliparse()
    main(args)

Uso

Sin arguments

python3 request_async.py

• Sin threads
• 5 URL's

Con arguments

python request_async.py -c 10 -t -w 10

• -t = Con threads
• -w = 10 Workers
• -c = 5 urls por defecto multiplicado por el valor de c = 10 URL's

Sin Threads

[gabo:~]$ python3 request_async.py      
Ejecutando sin multithreading...
     5 URLS - 5 WORKERS
     URL: http://google.com.ar - result: <Response [200]>
     URL: http://reddit.com - result: <Response [429]>
     URL: http://imgur.com - result: <Response [200]>
     URL: http://yahoo.com - result: <Response [200]>
     URL: http://unca.edu.ar - result: <Response [200]>
Terminado en 16.84122347831726 secs

Con Threads

[gabo:~]$ python3 request_async.py -t   
Ejecutando con multithreading...
     5 URLS - 5 WORKERS
     URL: http://google.com.ar - result: <Response [200]>
     URL: http://reddit.com - result: <Response [429]>
     URL: http://imgur.com - result: <Response [200]>
     URL: http://yahoo.com - result: <Response [200]>
     URL: http://unca.edu.ar - result: <Response [200]>
Terminado en 6.521490812301636 secs