Http Server Detector Base On Tornado, such as AsyncHttpClient, IOLoop.callback..

httpd_detector.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2013 Zhang ZY<http://idupx.blogspot.com/> 
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

'''fork from: https://gist.github.com/bufferx/5494125
'''

import functools
import logging

import time

from tornado.gen import engine as gen_engine
from tornado.gen import Task as gen_Task

from tornado.httpclient import HTTPRequest
from tornado.httpclient import AsyncHTTPClient

from tornado.ioloop import IOLoop
from tornado.ioloop import PeriodicCallback

from tornado.options import define
from tornado.options import options
from tornado.options import parse_command_line

define('http_url', default='http://www.google.com', help='HTTP METHOD')
define('http_method', default='HEAD', help='HTTP METHOD')
define('conn_timeout', default=1.0, help='Connect Timeout')
define('request_timeout', default=1.0, help='Request Timeout')
define('use_pycurl', default=False, help='Use Curl As Detector Client')
define('timer_interval', default=1.0, help='Timer Interval')

HTTP_METHOD_SUPPORTED = ('GET', 'HEAD',)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - \
%(module)s.%(funcName)s:%(lineno)d - %(message)s', '')

logging.basicConfig(level=logging.DEBUG)

g_logger = logging.getLogger('root.main')
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
g_logger.addHandler(ch)
g_logger.propagate = False

log_time = lambda f, t: 'Function[%s] Consume %.3fms' % (f, (time.time() - t) * 1000)


def time_it(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        start_time = time.time()
        r = func(*args, **kwargs)
        g_logger.info(log_time(func.__name__, start_time))
        return r
    return wrapper

@time_it
@gen_engine
def async_fetch(urls, callback):
    assert isinstance(urls, tuple)

    g_logger.debug('call async_fetch')

    http_client = AsyncHTTPClient(max_clients=100)

    task_list = []
    for url in urls:
        request = HTTPRequest(url,
                              method=options.http_method.upper(),
                              connect_timeout=options.conn_timeout,
                              request_timeout=options.request_timeout,
                              user_agent='PYTORNADO_CLIENT',
                              )
        task_list.append(gen_Task(http_client.fetch, request))

    g_logger.debug('gen task: %s', url)

    response_list = yield task_list

    callback(response_list)

def process_response(response_list):
    for response in response_list:
        g_logger.info('%s\t%s\t%s\t%.3fs', response.effective_url, response.code, \
                      response.headers['Content-Length'] if 'Content-Length' in \
                      response.headers else 0, \
                      response.request_time)

    if response.code == 599:
        IOLoop.instance().stop()

def _timer():
    URLS  = (options.http_url,)
    _callback = functools.partial(async_fetch, URLS, process_response)
    timer = PeriodicCallback(_callback, options.timer_interval * 1000)
    timer.start()

@time_it
def main():
    parse_command_line()
    for key, option in options.iteritems():
        g_logger.info('Options: (%s, %s)', key, option.value())

    if options.http_method.upper() not in HTTP_METHOD_SUPPORTED:
        print '%s Is Not Supported' % options.http_method
        return

    if options.use_pycurl:
        try:
            import pycurl
            AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
        except ImportError, e:
            g_logger.error('%s, Use Default AsyncHttpClient', e)

    _timer()

    IOLoop.instance().start()

if __name__ == '__main__':
    main()

Requirements

• Tornado2.3+

Usage

• python httpd_detector.py -http_url=http://google.com -http_method=HEAD -conn_timeout=1.0 -request_timeout=4.0 -timer_interval=1.0