Last active
September 24, 2020 00:58
Revisions
-
sile revised this gist
Sep 24, 2020 . 1 changed file with 9 additions and 9 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,5 +1,5 @@ # 1. Download kurobako binary. $ curl -L https://github.com/sile/kurobako/releases/download/0.2.6/kurobako-0.2.6.linux-amd64 -o kurobako $ chmod +x kurobako && sudo mv kurobako /usr/local/bin/ # 2. Download the data file for HPOBench (note that the file size is about 700MB). @@ -22,15 +22,15 @@ $ cat problems.json # - TPESampler without pruning # - TPESampler with MedianPruner # - TPESampler with SuccessiveHalvingPruner $ kurobako solver --name 'RandomSampler' optuna --sampler RandomSampler --pruner NopPruner > solvers.json $ kurobako solver --name 'TPESampler' optuna --pruner NopPruner >> solvers.json $ kurobako solver --name 'TPESampler_with_MedianPruner' optuna --pruner MedianPruner --pruner-kwargs '{"warmup_steps": 4}' >> solvers.json $ kurobako solver --name 'TPESampler_with_SuccessiveHalvingPruner' optuna --pruner SuccessiveHalvingPruner --pruner-kwargs '{"min_resource": 4}' >> solvers.json $ cat solvers.json {"name":"RandomSampler","optuna":{"sampler":"RandomSampler","pruner":"NopPruner"}} {"name":"TPESampler","optuna":{"pruner":"NopPruner"}} {"name":"TPESampler_with_MedianPruner","optuna":{"pruner":"MedianPruner","pruner_kwargs":"{\"warmup_steps\": 4}"}} {"name":"TPESampler_with_SuccessiveHalvingPruner","optuna":{"pruner":"SuccessiveHalvingPruner","pruner_kwargs":"{\"min_resource\": 4}"}} # 5. Run the benchmark. $ kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) --repeats 30 --budget 80 > studies.json -
sile revised this gist
Apr 15, 2020 . 1 changed file with 4 additions and 4 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -24,13 +24,13 @@ $ cat problems.json # - TPESampler with SuccessiveHalvingPruner $ kurobako solver --name 'RandomSampler' optuna --sampler random --pruner nop > solvers.json $ kurobako solver --name 'TPESampler' optuna --sampler tpe --pruner nop >> solvers.json $ kurobako solver --name 'TPESampler_with_MedianPruner' optuna --pruner median --median-warmup-steps 4 >> solvers.json $ kurobako solver --name 'TPESampler_with_SuccessiveHalvingPruner' optuna --pruner asha --asha-min-resource 4 >> solvers.json $ cat solvers.json {"name":"RandomSampler","optuna":{"sampler":"random","pruner":"nop"}} {"name":"TPESampler","optuna":{"pruner":"nop"}} {"name":"TPESampler_with_MedianPruner","optuna":{"median_warmup_steps":4}} {"name":"TPESampler_with_SuccessiveHalvingPruner","optuna":{"pruner":"asha","asha_min_resource":4}} # 5. Run the benchmark. $ kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) --repeats 30 --budget 80 > studies.json -
sile created this gist
Jan 27, 2020 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,41 @@ # 1. Download kurobako binary. $ curl -L https://github.com/sile/kurobako/releases/download/0.1.4/kurobako-0.1.4.linux-amd64 -o kurobako $ chmod +x kurobako && sudo mv kurobako /usr/local/bin/ # 2. Download the data file for HPOBench (note that the file size is about 700MB). $ curl -OL http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz $ tar xf fcnet_tabular_benchmarks.tar.gz && cd fcnet_tabular_benchmarks/ # 3. Specify problems used in this benchmark. # # In this example, we use Protein Structure and Parkinsons Telemonitoring datasets. $ kurobako problem hpobench fcnet_protein_structure_data.hdf5 > problems.json $ kurobako problem hpobench fcnet_parkinsons_telemonitoring_data.hdf5 >> problems.json $ cat problems.json {"hpobench":{"dataset":"fcnet_protein_structure_data.hdf5"}} {"hpobench":{"dataset":"fcnet_parkinsons_telemonitoring_data.hdf5"}} # 4. Specify optimization algorithms ("solver" in Kurobako's terminology) used in this benchmark. # # In this example, we use the following four algorithms provided by Optuna: # - RandomSampler without pruning # - TPESampler without pruning # - TPESampler with MedianPruner # - TPESampler with SuccessiveHalvingPruner $ kurobako solver --name 'RandomSampler' optuna --sampler random --pruner nop > solvers.json $ kurobako solver --name 'TPESampler' optuna --sampler tpe --pruner nop >> solvers.json $ kurobako solver --name 'TPESampler with MedianPruner' optuna --pruner median --median-warmup-steps 4 >> solvers.json $ kurobako solver --name 'TPESampler with SuccessiveHalvingPruner' optuna --pruner asha --asha-min-resource 4 >> solvers.json $ cat solvers.json {"name":"RandomSampler","optuna":{"sampler":"random","pruner":"nop"}} {"name":"TPESampler","optuna":{"pruner":"nop"}} {"name":"TPESampler with MedianPruner","optuna":{"median_warmup_steps":4}} {"name":"TPESampler with SuccessiveHalvingPruner","optuna":{"pruner":"asha","asha_min_resource":4}} # 5. Run the benchmark. $ kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) --repeats 30 --budget 80 > studies.json $ cat studies.json | kurobako run --parallelism 10 > result.json (ALL) [01:27:14] [STUDIES 240/240 100%] [ETA 0s] done # 6. Generate the report of the benchmark result. $ cat result.json | kurobako report > report.md