Http Server Benchmarking tool based on Tornado

hbm.conf

#config
http_hosts={
    'localhost:8800': {'w':1,'cc':0,'sc':0},
    'localhost:8801': {'w':1,'cc':0,'sc':0},
}
http_method='POST'
post_file='post.data'
conn_timeout=5.0
request_timeout=5.0
clients=3
requests=100

httpd_benchmark.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2014 Zhang ZY<http://github.com/bufferx> 
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

'''fork from: https://gist.github.com/bufferx/5494125
'''

from bf4x_pyutil.common import CommonUtil

import functools
import logging

import os
import time

import tornado
from tornado import process

from tornado.gen import engine as gen_engine
from tornado.gen import Task as gen_Task

from tornado.httpclient import HTTPRequest
from tornado.httpclient import AsyncHTTPClient

from tornado.ioloop import IOLoop

from tornado.options import define
from tornado.options import options
from tornado.options import parse_command_line

FUNC_WDRR_SCHEDULE = CommonUtil.wdrr_schedule

define('config_file', default='hbm.conf', help='Configuration File Specifying \
Options')
define('http_hosts', default={}, help='HTTP HOSTS')
define('http_method', default='POST', help='HTTP METHOD')
define('post_file', default='post.data', help='HTTP POST DATA')
define('conn_timeout', default=1.0, help='Connect Timeout')
define('request_timeout', default=1.0, help='Request Timeout')
define('use_pycurl', default=False, help='Use Curl As Detector Client')
define('clients', default=20, help='Number of parallel connections')
define('requests', default=1000, help='Total number of requests')

HTTP_METHOD_SUPPORTED = ('GET', 'POST',)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - \
%(module)s.%(funcName)s:%(lineno)d - %(message)s', '')

logging.basicConfig(level=logging.DEBUG)

g_logger = logging.getLogger('root.main')
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
g_logger.addHandler(ch)
g_logger.propagate = False

log_time = lambda f, t: 'Function[%s] Consume %.3fms' % (f, (time.time() - t) * 1000)

stat_points = (50, 66, 75, 80, 90, 95, 98, 99, 100)
request_count = 0
request_times = []
post_data = ''
http_client = None


def time_it(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        start_time = time.time()
        r = func(*args, **kwargs)
        g_logger.info(log_time(func.__name__, start_time))
        return r
    return wrapper

def async_fetch(url):
    def callback(response):
        global request_count, request_times
        request_count += 1
        g_logger.info('%s\t%s\t%s\t%.3fs', response.effective_url, response.code, \
                      response.headers['Content-Length'] if 'Content-Length' in \
                      response.headers else 0, \
                      response.request_time)

        request_times.append(response.request_time)
        fire()

    request = HTTPRequest(url,
                          body=post_data,
                          method=options.http_method.upper(),
                          connect_timeout=options.conn_timeout,
                          request_timeout=options.request_timeout,
                          user_agent='PY_TORNADO_CLIENT',
                          )
    http_client.fetch(request, callback)


def stats():
    print '\nPercentage of the requests served within a certain time (ms)'
    _request_times = sorted(request_times)
    _idx = len(_request_times) - 1

    _ = (int(i/100.0*_idx) for i in stat_points) 
    for idx, p in enumerate(_):
        print '%d%%: %.3f' % (stat_points[idx], _request_times[p]*1000)

def fire():
    if request_count >= options.requests:
        IOLoop.instance().stop()
        g_logger.info('Over: %d\t%d', request_count, len(request_times))

        return

    host = FUNC_WDRR_SCHEDULE(options.http_hosts)
    url = 'http://%s/' % host

    async_fetch(url)

@time_it
def _main():
    global post_data, http_client

    parse_command_line()

    if os.path.exists(options.config_file):
        tornado.options.parse_config_file(options.config_file)

    for key, option in options.iteritems():
        g_logger.info('Options: (%s, %s)', key, option.value())

    if options.http_method.upper() not in HTTP_METHOD_SUPPORTED:
        print '%s Is Not Supported' % options.http_method
        return

    with open(options.post_file, 'r') as f:
        post_data = f.read()

    if options.use_pycurl:
        try:
            import pycurl
            AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
        except ImportError, e:
            g_logger.error('%s, Use Default AsyncHttpClient', e)

    process.fork_processes(options.clients)

    http_client = AsyncHTTPClient(max_clients=10)

    fire()

    IOLoop.instance().start()


def main():
    _main()
    stats()


if __name__ == '__main__':
    main()