chance-level-ap.md

import numpy as np
from sklearn.metrics import average_precision_score

def chance_level_ap(n_all, n_positive, trials=1000):
    return np.mean([
        average_precision_score([1] * n_positive + [0] * (n_all - n_positive), np.random.permutation(n_all))
        for _ in range(trials)])

import pytablewriter

def create_table(config_pairs):
    writer = pytablewriter.MarkdownTableWriter()
    writer.header_list = ['Num of examples', 'True positive rate', 'Chance level AP']
    writer.value_matrix = [
        [n_all, positive_rate, chance_level_ap(n_all, round(n_all * positive_rate))]
        for n_all, positive_rate in config_pairs
    ]
    return writer

import itertools

config_pairs = itertools.product([10, 100, 1000, 10000], [0.01, 0.1, 0.5, 0.9])

create_table(config_pairs)

/Users/akiba/.pyenv/versions/anaconda3-4.2.0/lib/python3.5/site-packages/sklearn/metrics/ranking.py:444: RuntimeWarning: invalid value encountered in true_divide
  recall = tps / tps[-1]

Num of examples	True positive rate	Chance level AP
10	0.01	NaN
10	0.10	0.29244
10	0.50	0.60708
10	0.90	0.91676
100	0.01	0.05494
100	0.10	0.13603
100	0.50	0.52077
100	0.90	0.90432
1000	0.01	0.01598
1000	0.10	0.10597
1000	0.50	0.50254
1000	0.90	0.90052
10000	0.01	0.01068
10000	0.10	0.10082
10000	0.50	0.50057
10000	0.90	0.90015