kurobako blog: basic-benchmark.sh

basic-benchmark.sh

# 1. Download kurobako binary.
$ curl -L https://github.com/sile/kurobako/releases/download/0.1.4/kurobako-0.1.4.linux-amd64 -o kurobako
$ chmod +x kurobako && sudo mv kurobako /usr/local/bin/
 
# 2. Download the data file for HPOBench (note that the file size is about 700MB).
$ curl -OL http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz
$ tar xf fcnet_tabular_benchmarks.tar.gz && cd fcnet_tabular_benchmarks/
 
# 3. Specify problems used in this benchmark.
#
# In this example, we use Protein Structure and Parkinsons Telemonitoring datasets.
$ kurobako problem hpobench fcnet_protein_structure_data.hdf5 > problems.json
$ kurobako problem hpobench fcnet_parkinsons_telemonitoring_data.hdf5 >> problems.json
$ cat problems.json
{"hpobench":{"dataset":"fcnet_protein_structure_data.hdf5"}}
{"hpobench":{"dataset":"fcnet_parkinsons_telemonitoring_data.hdf5"}}
 
# 4. Specify optimization algorithms ("solver" in Kurobako's terminology) used in this benchmark.
#
# In this example, we use the following four algorithms provided by Optuna:
# - RandomSampler without pruning
# - TPESampler without pruning
# - TPESampler with MedianPruner
# - TPESampler with SuccessiveHalvingPruner
$ kurobako solver --name 'RandomSampler' optuna --sampler random --pruner nop > solvers.json
$ kurobako solver --name 'TPESampler' optuna --sampler tpe --pruner nop >> solvers.json
$ kurobako solver --name 'TPESampler_with_MedianPruner' optuna --pruner median --median-warmup-steps 4 >> solvers.json
$ kurobako solver --name 'TPESampler_with_SuccessiveHalvingPruner' optuna --pruner asha --asha-min-resource 4 >> solvers.json
$ cat solvers.json
{"name":"RandomSampler","optuna":{"sampler":"random","pruner":"nop"}}
{"name":"TPESampler","optuna":{"pruner":"nop"}}
{"name":"TPESampler_with_MedianPruner","optuna":{"median_warmup_steps":4}}
{"name":"TPESampler_with_SuccessiveHalvingPruner","optuna":{"pruner":"asha","asha_min_resource":4}}
 
# 5. Run the benchmark.
$ kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) --repeats 30 --budget 80 > studies.json
$ cat studies.json | kurobako run --parallelism 10 > result.json
(ALL) [01:27:14] [STUDIES    240/240 100%] [ETA  0s] done
 
# 6. Generate the report of the benchmark result. 
$ cat result.json | kurobako report > report.md