proof of concept paginated api requests using trio and asks

async_get.py

import os
import logging
import asks
import trio
import time
import contextvars


logging.basicConfig(level='DEBUG')
logger = logging.getLogger(__name__)
num_conn = 30
num_pages = 5
max_conn = 5  # walmart api allows 5 requests/second
base_url = 'http://api.walmartlabs.com'
api_key = os.environ['WALMART_OPEN_API_KEY']
request_info = contextvars.ContextVar('request_info')


def log_debug(msg):
    request_tag = request_info.get()
    logger.debug("request %s: %s", request_tag, msg)


async def producer(send_channel, message):
        request_info.set('producer')
        await send_channel.send(message)


async def consumer(receive_channel):
        request_info.set('consumer')
        ep = await receive_channel.receive()
        return ep


async def _get_next_pages(s, rv, send_channel, receive_channel):
    request_info.set('GET')
   
    ep = await consumer(receive_channel) 
    url = base_url + ep
    log_debug(url)
    r = await s.get(url)

    log_debug(r)
    
    if r.status_code == 200:
        json = r.json()
        next_page = r.json()['nextPage']
        log_debug('next_page: ' + next_page)
        await producer(send_channel, next_page)
    else:
        log_debug(r.status_code)
        log_debug(r.text)

    return rv.append(next_page)


async def main(s, rv):
    request_info.set('MAIN')
    send_channel, receive_channel = trio.open_memory_channel(0) 
    async with send_channel, receive_channel:
        async with trio.open_nursery() as n:

            ep = '/v1/paginated/items?apiKey={}&format=json'.format(api_key)
            n.start_soon(producer, send_channel, ep)

            for i in range(num_pages):
                n.start_soon(_get_next_pages, s, rv, send_channel, receive_channel)
            
                log_debug(rv)	


if __name__ == '__main__':
    asks.init('trio')
    s = asks.Session(base_url, connections=num_pages)
    rv = []
    trio.run(main, s, rv)
    logger.info('len of rv: %s', len(rv))