Skip to content

Instantly share code, notes, and snippets.

@sile
Last active September 24, 2020 00:58

Revisions

  1. sile revised this gist Sep 24, 2020. 1 changed file with 9 additions and 9 deletions.
    18 changes: 9 additions & 9 deletions basic-benchmark.sh
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,5 @@
    # 1. Download kurobako binary.
    $ curl -L https://github.com/sile/kurobako/releases/download/0.1.4/kurobako-0.1.4.linux-amd64 -o kurobako
    $ curl -L https://github.com/sile/kurobako/releases/download/0.2.6/kurobako-0.2.6.linux-amd64 -o kurobako
    $ chmod +x kurobako && sudo mv kurobako /usr/local/bin/

    # 2. Download the data file for HPOBench (note that the file size is about 700MB).
    @@ -22,15 +22,15 @@ $ cat problems.json
    # - TPESampler without pruning
    # - TPESampler with MedianPruner
    # - TPESampler with SuccessiveHalvingPruner
    $ kurobako solver --name 'RandomSampler' optuna --sampler random --pruner nop > solvers.json
    $ kurobako solver --name 'TPESampler' optuna --sampler tpe --pruner nop >> solvers.json
    $ kurobako solver --name 'TPESampler_with_MedianPruner' optuna --pruner median --median-warmup-steps 4 >> solvers.json
    $ kurobako solver --name 'TPESampler_with_SuccessiveHalvingPruner' optuna --pruner asha --asha-min-resource 4 >> solvers.json
    $ kurobako solver --name 'RandomSampler' optuna --sampler RandomSampler --pruner NopPruner > solvers.json
    $ kurobako solver --name 'TPESampler' optuna --pruner NopPruner >> solvers.json
    $ kurobako solver --name 'TPESampler_with_MedianPruner' optuna --pruner MedianPruner --pruner-kwargs '{"warmup_steps": 4}' >> solvers.json
    $ kurobako solver --name 'TPESampler_with_SuccessiveHalvingPruner' optuna --pruner SuccessiveHalvingPruner --pruner-kwargs '{"min_resource": 4}' >> solvers.json
    $ cat solvers.json
    {"name":"RandomSampler","optuna":{"sampler":"random","pruner":"nop"}}
    {"name":"TPESampler","optuna":{"pruner":"nop"}}
    {"name":"TPESampler_with_MedianPruner","optuna":{"median_warmup_steps":4}}
    {"name":"TPESampler_with_SuccessiveHalvingPruner","optuna":{"pruner":"asha","asha_min_resource":4}}
    {"name":"RandomSampler","optuna":{"sampler":"RandomSampler","pruner":"NopPruner"}}
    {"name":"TPESampler","optuna":{"pruner":"NopPruner"}}
    {"name":"TPESampler_with_MedianPruner","optuna":{"pruner":"MedianPruner","pruner_kwargs":"{\"warmup_steps\": 4}"}}
    {"name":"TPESampler_with_SuccessiveHalvingPruner","optuna":{"pruner":"SuccessiveHalvingPruner","pruner_kwargs":"{\"min_resource\": 4}"}}

    # 5. Run the benchmark.
    $ kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) --repeats 30 --budget 80 > studies.json
  2. sile revised this gist Apr 15, 2020. 1 changed file with 4 additions and 4 deletions.
    8 changes: 4 additions & 4 deletions basic-benchmark.sh
    Original file line number Diff line number Diff line change
    @@ -24,13 +24,13 @@ $ cat problems.json
    # - TPESampler with SuccessiveHalvingPruner
    $ kurobako solver --name 'RandomSampler' optuna --sampler random --pruner nop > solvers.json
    $ kurobako solver --name 'TPESampler' optuna --sampler tpe --pruner nop >> solvers.json
    $ kurobako solver --name 'TPESampler with MedianPruner' optuna --pruner median --median-warmup-steps 4 >> solvers.json
    $ kurobako solver --name 'TPESampler with SuccessiveHalvingPruner' optuna --pruner asha --asha-min-resource 4 >> solvers.json
    $ kurobako solver --name 'TPESampler_with_MedianPruner' optuna --pruner median --median-warmup-steps 4 >> solvers.json
    $ kurobako solver --name 'TPESampler_with_SuccessiveHalvingPruner' optuna --pruner asha --asha-min-resource 4 >> solvers.json
    $ cat solvers.json
    {"name":"RandomSampler","optuna":{"sampler":"random","pruner":"nop"}}
    {"name":"TPESampler","optuna":{"pruner":"nop"}}
    {"name":"TPESampler with MedianPruner","optuna":{"median_warmup_steps":4}}
    {"name":"TPESampler with SuccessiveHalvingPruner","optuna":{"pruner":"asha","asha_min_resource":4}}
    {"name":"TPESampler_with_MedianPruner","optuna":{"median_warmup_steps":4}}
    {"name":"TPESampler_with_SuccessiveHalvingPruner","optuna":{"pruner":"asha","asha_min_resource":4}}

    # 5. Run the benchmark.
    $ kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) --repeats 30 --budget 80 > studies.json
  3. sile created this gist Jan 27, 2020.
    41 changes: 41 additions & 0 deletions basic-benchmark.sh
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,41 @@
    # 1. Download kurobako binary.
    $ curl -L https://github.com/sile/kurobako/releases/download/0.1.4/kurobako-0.1.4.linux-amd64 -o kurobako
    $ chmod +x kurobako && sudo mv kurobako /usr/local/bin/

    # 2. Download the data file for HPOBench (note that the file size is about 700MB).
    $ curl -OL http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz
    $ tar xf fcnet_tabular_benchmarks.tar.gz && cd fcnet_tabular_benchmarks/

    # 3. Specify problems used in this benchmark.
    #
    # In this example, we use Protein Structure and Parkinsons Telemonitoring datasets.
    $ kurobako problem hpobench fcnet_protein_structure_data.hdf5 > problems.json
    $ kurobako problem hpobench fcnet_parkinsons_telemonitoring_data.hdf5 >> problems.json
    $ cat problems.json
    {"hpobench":{"dataset":"fcnet_protein_structure_data.hdf5"}}
    {"hpobench":{"dataset":"fcnet_parkinsons_telemonitoring_data.hdf5"}}

    # 4. Specify optimization algorithms ("solver" in Kurobako's terminology) used in this benchmark.
    #
    # In this example, we use the following four algorithms provided by Optuna:
    # - RandomSampler without pruning
    # - TPESampler without pruning
    # - TPESampler with MedianPruner
    # - TPESampler with SuccessiveHalvingPruner
    $ kurobako solver --name 'RandomSampler' optuna --sampler random --pruner nop > solvers.json
    $ kurobako solver --name 'TPESampler' optuna --sampler tpe --pruner nop >> solvers.json
    $ kurobako solver --name 'TPESampler with MedianPruner' optuna --pruner median --median-warmup-steps 4 >> solvers.json
    $ kurobako solver --name 'TPESampler with SuccessiveHalvingPruner' optuna --pruner asha --asha-min-resource 4 >> solvers.json
    $ cat solvers.json
    {"name":"RandomSampler","optuna":{"sampler":"random","pruner":"nop"}}
    {"name":"TPESampler","optuna":{"pruner":"nop"}}
    {"name":"TPESampler with MedianPruner","optuna":{"median_warmup_steps":4}}
    {"name":"TPESampler with SuccessiveHalvingPruner","optuna":{"pruner":"asha","asha_min_resource":4}}

    # 5. Run the benchmark.
    $ kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) --repeats 30 --budget 80 > studies.json
    $ cat studies.json | kurobako run --parallelism 10 > result.json
    (ALL) [01:27:14] [STUDIES 240/240 100%] [ETA 0s] done

    # 6. Generate the report of the benchmark result.
    $ cat result.json | kurobako report > report.md