diff --git a/.github/workflows/run_singularity_versions.yml b/.github/workflows/run_singularity_versions.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 96dd406d..7c77ce44 --- a/README.md +++ b/README.md @@ -45,6 +45,22 @@ fs = b.get_fidelity_space(seed=1) meta = b.get_meta_information() ``` +Multi-Objective benchmarks can be queried in a similar manner + +```python +from hpobench.container.benchmarks.mo.adult_benchmark import AdultBenchmark +b = AdultBenchmark(rng=1) +config = b.get_configuration_space(seed=1).sample_configuration() +result_dict = b.objective_function(configuration=config, fidelity={"budget": 66}, rng=1) +result_dict = b.objective_function(configuration=config, rng=1) + +>>> print(result_dict['function_values']) +{'misclassification_rate': 0.16572832429112494, + 'DSO': 0.197765453723867, + 'DEO': 0.1595593763542093, + 'DFP': 0.10465117283454546} +``` + ## Installation We recommend using a virtual environment. To install HPOBench, please run the following: diff --git a/changelog.md b/changelog.md old mode 100644 new mode 100755 diff --git a/ci_scripts/codestyle.sh b/ci_scripts/codestyle.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/container_examples.sh b/ci_scripts/container_examples.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/install.sh b/ci_scripts/install.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/install_singularity.sh b/ci_scripts/install_singularity.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/local_examples.sh b/ci_scripts/local_examples.sh old mode 100644 new mode 100755 diff --git a/ci_scripts/script.sh b/ci_scripts/script.sh old mode 100644 new mode 100755 diff --git a/codecov.yml b/codecov.yml old mode 100644 new mode 100755 diff --git a/examples/container/create_connection_to_benchmark.py b/examples/container/create_connection_to_benchmark.py old mode 100644 new mode 100755 diff --git a/examples/container/tabular_benchmark_example.py b/examples/container/tabular_benchmark_example.py old mode 100644 new mode 100755 diff --git a/examples/container/xgboost_with_container.py b/examples/container/xgboost_with_container.py old mode 100644 new mode 100755 diff --git a/examples/jahs_bench_201.json b/examples/jahs_bench_201.json old mode 100644 new mode 100755 diff --git a/examples/local/xgboost_local.py b/examples/local/xgboost_local.py old mode 100644 new mode 100755 diff --git a/examples/w_optimizer/SVMSurrogate_minicomparison.py b/examples/w_optimizer/SVMSurrogate_minicomparison.py old mode 100644 new mode 100755 diff --git a/examples/w_optimizer/cartpole_bohb.py b/examples/w_optimizer/cartpole_bohb.py old mode 100644 new mode 100755 diff --git a/examples/w_optimizer/cartpole_smachb.py b/examples/w_optimizer/cartpole_smachb.py old mode 100644 new mode 100755 diff --git a/extra_requirements/cartpole.json b/extra_requirements/cartpole.json old mode 100644 new mode 100755 diff --git a/extra_requirements/examples.json b/extra_requirements/examples.json old mode 100644 new mode 100755 diff --git a/extra_requirements/jahs_bench_201.json b/extra_requirements/jahs_bench_201.json new file mode 100644 index 00000000..fc220155 --- /dev/null +++ b/extra_requirements/jahs_bench_201.json @@ -0,0 +1,3 @@ +{ + "jahs_bench_201": ["jahs_bench_201@git+https://github.com/automl/jahs_bench_201.git"] +} diff --git a/extra_requirements/ml_mfbb.json b/extra_requirements/ml_mfbb.json old mode 100644 new mode 100755 diff --git a/extra_requirements/mo_cnn.json b/extra_requirements/mo_cnn.json old mode 100644 new mode 100755 diff --git a/extra_requirements/multi_objective.json b/extra_requirements/multi_objective.json old mode 100644 new mode 100755 diff --git a/extra_requirements/nasbench_101.json b/extra_requirements/nasbench_101.json old mode 100644 new mode 100755 diff --git a/extra_requirements/nasbench_1shot1.json b/extra_requirements/nasbench_1shot1.json old mode 100644 new mode 100755 diff --git a/extra_requirements/outlier_detection.json b/extra_requirements/outlier_detection.json old mode 100644 new mode 100755 diff --git a/extra_requirements/paramnet.json b/extra_requirements/paramnet.json old mode 100644 new mode 100755 diff --git a/extra_requirements/pybnn.json b/extra_requirements/pybnn.json old mode 100644 new mode 100755 diff --git a/extra_requirements/svm.json b/extra_requirements/svm.json old mode 100644 new mode 100755 diff --git a/extra_requirements/tabular_benchmarks.json b/extra_requirements/tabular_benchmarks.json old mode 100644 new mode 100755 diff --git a/extra_requirements/tests.json b/extra_requirements/tests.json old mode 100644 new mode 100755 diff --git a/extra_requirements/xgboost.json b/extra_requirements/xgboost.json old mode 100644 new mode 100755 diff --git a/extra_requirements/yahpo_gym.json b/extra_requirements/yahpo_gym.json old mode 100644 new mode 100755 diff --git a/hpobench/__init__.py b/hpobench/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/__version__.py b/hpobench/__version__.py old mode 100644 new mode 100755 diff --git a/hpobench/abstract_benchmark.py b/hpobench/abstract_benchmark.py old mode 100644 new mode 100755 index 6a2942af..bcc57542 --- a/hpobench/abstract_benchmark.py +++ b/hpobench/abstract_benchmark.py @@ -35,6 +35,7 @@ def __init__(self, rng: Union[int, np.random.RandomState, None] = None, **kwargs create a new random state. """ super(_BaseAbstractBenchmark, self).__init__(**kwargs) + print("base abstract benchamrk") self.rng = rng_helper.get_rng(rng=rng) self.configuration_space = self.get_configuration_space(self.rng.randint(0, 10000)) self.fidelity_space = self.get_fidelity_space(self.rng.randint(0, 10000)) diff --git a/hpobench/benchmarks/__init__.py b/hpobench/benchmarks/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/README.md b/hpobench/benchmarks/ml/README.md old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/__init__.py b/hpobench/benchmarks/ml/__init__.py old mode 100644 new mode 100755 index 64e399cd..145bf4b5 --- a/hpobench/benchmarks/ml/__init__.py +++ b/hpobench/benchmarks/ml/__init__.py @@ -1,22 +1,22 @@ -from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF -from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF -from hpobench.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF +from hpobench.benchmarks.ml.histgb_benchmark import HistGBBenchmark, HistGBBenchmarkBB, HistGBBenchmarkMF, HistGBBenchmarkMO +from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark, LRBenchmarkBB, LRBenchmarkMF, LRBenchmarkMO +from hpobench.benchmarks.ml.nn_benchmark import NNBenchmark, NNBenchmarkBB, NNBenchmarkMF, NNBenchmarkMO from hpobench.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ - RandomForestBenchmarkMF -from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF -from hpobench.benchmarks.ml.tabular_benchmark import TabularBenchmark + RandomForestBenchmarkMF, RandomForestBenchmarkMO +from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF, SVMBenchmarkMO +from hpobench.benchmarks.ml.tabular_benchmark import TabularBenchmark, TabularBenchmarkMO try: - from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF + from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF, XGBoostBenchmarkMO except ImportError: pass -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', - 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', - 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', - 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', - 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', - 'TabularBenchmark', - 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO', + 'LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO', + 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO', + 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'RandomForestBenchmarkMO', + 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', 'SVMBenchmarkMO', + 'TabularBenchmark', 'TabularBenchmarkMO', + 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmarkMO', ] diff --git a/hpobench/benchmarks/ml/histgb_benchmark.py b/hpobench/benchmarks/ml/histgb_benchmark.py old mode 100644 new mode 100755 index c4b0c468..097fefaa --- a/hpobench/benchmarks/ml/histgb_benchmark.py +++ b/hpobench/benchmarks/ml/histgb_benchmark.py @@ -15,18 +15,18 @@ from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark __version__ = '0.0.1' -class HistGBBenchmark(MLBenchmark): +class _HistGBBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(HistGBBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_HistGBBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -101,6 +101,13 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class HistGBBenchmark(_HistGBBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(HistGBBenchmark, self).__init__(task_id, rng, valid_size, data_path) class HistGBBenchmarkBB(HistGBBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -120,6 +127,14 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS HistGBBenchmark._get_fidelity_choices(ntrees_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class HistGBBenchmarkMO(_HistGBBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(HistGBBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF'] +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/lr_benchmark.py b/hpobench/benchmarks/ml/lr_benchmark.py old mode 100644 new mode 100755 index 8c317111..8357bd66 --- a/hpobench/benchmarks/ml/lr_benchmark.py +++ b/hpobench/benchmarks/ml/lr_benchmark.py @@ -14,19 +14,28 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.linear_model import SGDClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy -__version__ = '0.0.1' +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class LRBenchmark(MLBenchmark): + +class _LRBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_LRBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) self.cache_size = 500 @staticmethod @@ -107,6 +116,17 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class LRBenchmark(_LRBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + + super(LRBenchmark, self).__init__(task_id, rng, valid_size, data_path) + self.cache_size = 500 + + class LRBenchmarkBB(LRBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -126,6 +146,15 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS LRBenchmark._get_fidelity_choices(iter_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class LRBenchmarkMO(_LRBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(LRBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + self.cache_size = 500 -__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF'] +__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/nn_benchmark.py b/hpobench/benchmarks/ml/nn_benchmark.py old mode 100644 new mode 100755 index 06634661..cae60e12 --- a/hpobench/benchmarks/ml/nn_benchmark.py +++ b/hpobench/benchmarks/ml/nn_benchmark.py @@ -14,18 +14,28 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.neural_network import MLPClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark -__version__ = '0.0.1' +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy + +0.0.1: +* First implementation +''' + +__version__ = '0.0.2' -class NNBenchmark(MLBenchmark): +class _NNBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_NNBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -109,6 +119,13 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class NNBenchmark(_NNBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(NNBenchmark, self).__init__(task_id, rng, valid_size, data_path) class NNBenchmarkBB(NNBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -129,5 +146,12 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS ) return fidelity_space +class NNBenchmarkMO(_NNBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(NNBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) -__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF'] +__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/pybnn.py b/hpobench/benchmarks/ml/pybnn.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/rf_benchmark.py b/hpobench/benchmarks/ml/rf_benchmark.py old mode 100644 new mode 100755 index 596f03b6..55c0b0d5 --- a/hpobench/benchmarks/ml/rf_benchmark.py +++ b/hpobench/benchmarks/ml/rf_benchmark.py @@ -1,11 +1,3 @@ -""" -Changelog: -========== - -0.0.1: -* First implementation of the RF Benchmarks. -""" - from copy import deepcopy from typing import Union, Tuple, Dict @@ -14,18 +6,27 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.ensemble import RandomForestClassifier -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark + +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy -__version__ = '0.0.1' +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class RandomForestBenchmark(MLBenchmark): +class _RandomForestBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(RandomForestBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_RandomForestBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -103,6 +104,13 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class RandomForestBenchmark(_RandomForestBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(RandomForestBenchmark, self).__init__(task_id, rng, valid_size, data_path) class RandomForestBenchmarkBB(RandomForestBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -122,6 +130,17 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS RandomForestBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class RandomForestBenchmarkMO(_RandomForestBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(RandomForestBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + + -__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF'] +__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', + 'RandomForestBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/svm_benchmark.py b/hpobench/benchmarks/ml/svm_benchmark.py old mode 100644 new mode 100755 index 9462442f..d4bc5e8c --- a/hpobench/benchmarks/ml/svm_benchmark.py +++ b/hpobench/benchmarks/ml/svm_benchmark.py @@ -1,11 +1,3 @@ -""" -Changelog: -========== - -0.0.1: -* First implementation of the new SVM Benchmarks. -""" - from typing import Union, Dict import ConfigSpace as CS @@ -13,18 +5,27 @@ from ConfigSpace.hyperparameters import Hyperparameter from sklearn.svm import SVC -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy + +0.0.1: +* First implementation +''' -__version__ = '0.0.1' +__version__ = '0.0.2' -class SVMBenchmark(MLBenchmark): +class _SVMBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(SVMBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_SVMBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) self.cache_size = 200 @@ -81,6 +82,14 @@ def init_model(self, config: Union[CS.Configuration, Dict], ) return model +class SVMBenchmark(_SVMBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(SVMBenchmark, self).__init__(task_id, rng, valid_size, data_path) + class SVMBenchmarkBB(SVMBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -90,9 +99,19 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS SVMBenchmark._get_fidelity_choices(subsample_choice='fixed') ) return fidelity_space + +class SVMBenchmarkMO(_SVMBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(SVMBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + + # To keep the parity of the the overall design SVMBenchmarkMF = SVMBenchmark -__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB'] +__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB', 'SVMBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/svm_benchmark_old.py b/hpobench/benchmarks/ml/svm_benchmark_old.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/ml/tabular_benchmark.py b/hpobench/benchmarks/ml/tabular_benchmark.py old mode 100644 new mode 100755 index 72e5fb31..47850f0e --- a/hpobench/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/benchmarks/ml/tabular_benchmark.py @@ -13,14 +13,21 @@ import numpy as np from ConfigSpace.read_and_write import json as json_cs -from hpobench.abstract_benchmark import AbstractBenchmark +from hpobench.abstract_benchmark import AbstractSingleObjectiveBenchmark, AbstractMultiObjectiveBenchmark, AbstractBenchmark from hpobench.dependencies.ml.ml_benchmark_template import metrics from hpobench.util.data_manager import TabularDataManager -__version__ = '0.0.1' +__version__ = '0.0.2' +""" +Changelog: +========== +0.0.2: Added MO Tabular Benchmark + +0.0.1: First implementation of the Tabular Benchmark. +""" -class TabularBenchmark(AbstractBenchmark): +class _TabularBenchmarkBase: def __init__(self, model: str, task_id: int, @@ -35,6 +42,7 @@ def __init__(self, self.dm = TabularDataManager(model, task_id, data_dir) self.table, self.metadata = self.dm.load() + self.exp_args = self.metadata["exp_args"] self.config_spaces = self.metadata["config_spaces"] self.global_minimums = self.metadata["global_min"] @@ -42,33 +50,8 @@ def __init__(self, self.original_cs = json_cs.read(self.config_spaces['x']) self.original_fs = json_cs.read(self.config_spaces['z']) - super(TabularBenchmark, self).__init__(rng=rng, **kwargs) + super(_TabularBenchmarkBase, self).__init__(rng=rng) - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function(self, - configuration: Union[CS.Configuration, Dict], - fidelity: Union[Dict, CS.Configuration, None] = None, - rng: Union[np.random.RandomState, int, None] = None, - seed: Union[int, None] = None, - metric: Union[str, None] = 'acc', - **kwargs) -> Dict: - - result = self._objective(configuration, fidelity, seed, metric, evaluation="val") - return result - - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function_test(self, - configuration: Union[CS.Configuration, Dict], - fidelity: Union[Dict, CS.Configuration, None] = None, - rng: Union[np.random.RandomState, int, None] = None, - seed: Union[int, None] = None, - metric: Union[str, None] = 'acc', - **kwargs) -> Dict: - - result = self._objective(configuration, fidelity, seed, metric, evaluation="test") - return result # pylint: disable=arguments-differ def get_configuration_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -142,26 +125,54 @@ def get_fidelity_range(self) -> List: def _search_dataframe(self, row_dict, df): # https://stackoverflow.com/a/46165056/8363967 mask = np.array([True] * df.shape[0]) - for i, param in enumerate(df.drop("result", axis=1).columns): + #for some benchmarks result is comprehensive in the form of a dict + #for some the dictionary is flattened and column name is set by joining keys using delimiter '.' + + if 'result' in df.columns: + print("result found") + for i, param in enumerate(df.drop("result", axis=1).columns): + mask *= df[param].values == row_dict[param] + idx = np.where(mask) + + assert len(idx) == 1, 'The query has resulted into mulitple matches. This should not happen. ' \ + f'The Query was {row_dict}' + idx = idx[0][0] + print("idx",idx) + print("matched config",df.iloc[idx]) + result = df.iloc[idx]["result"] + return result + + + for i, param in enumerate(df.drop(df.filter(regex='result.').columns, axis=1)): mask *= df[param].values == row_dict[param] idx = np.where(mask) assert len(idx) == 1, 'The query has resulted into mulitple matches. This should not happen. ' \ f'The Query was {row_dict}' idx = idx[0][0] - result = df.iloc[idx]["result"] - return result - - def _objective( + result = df.iloc[idx] + result = result.filter(regex='result.') + #convert the result to dict + resultdict = result.to_dict() + result={} + for key, value in resultdict.items(): + keys = key.split('.') + d = result + for k in keys[:-1]: + if k not in d: + d[k] = {} + d = d[k] + d[keys[-1]] = value + + return result['result'] + + def _moobjective( self, config: Dict, fidelity: Dict, seed: Union[int, None] = None, - metric: Union[str, None] = "acc", evaluation: Union[str, None] = "" ) -> Dict: - metric_str = ', '.join(list(metrics.keys())) - assert metric in list(metrics.keys()), f"metric not found among: {metric_str}" score_key = f"{evaluation}_scores" cost_key = f"{evaluation}_scores" @@ -169,7 +180,7 @@ def _objective( for name in self.configuration_space.get_hyperparameter_names(): key_path[str(name)] = config[str(name)] for name in self.fidelity_space.get_hyperparameter_names(): - key_path[str(name)] = fidelity[str(name)] + key_path[str(name)] = float(fidelity[str(name)]) if seed is not None: assert seed in self._seeds_used() @@ -177,19 +188,135 @@ def _objective( else: seeds = self._seeds_used() - loss = [] - costs = 0.0 + + costs = dict(acc=0.0, f1=0.0, precision=0.0, bal_acc=0.0, model_cost=0.0) info = dict() + dict_metrics = dict(acc=[], f1=[], precision=[], bal_acc=[]) + + + for seed in seeds: key_path["seed"] = seed res = self._search_dataframe(key_path, self.table) - loss.append(1 - res["info"][score_key][metric]) - costs += res["info"]["model_cost"] + res["info"][cost_key][metric] + costs["model_cost"] = costs["model_cost"] + res["info"]["model_cost"] + for metric_key, metric_value in metrics.items(): + dict_metrics[metric_key].append(res["info"][score_key][metric_key]) + costs[metric_key] = costs[metric_key] + res["info"][cost_key][metric_key] info[seed] = res["info"] key_path.pop("seed") - loss = np.mean(loss) - result = dict(function_value=float(loss), cost=costs, info=info) + + result_dict = { + 'function_value': { + # The original benchmark returned the accuracy with range [0, 100]. + # We cast it to a minimization problem with range [0-1] to have a more standardized return value. + 'acc': float(np.mean(dict_metrics['acc'])), + 'precision': float(np.mean(dict_metrics['precision'])), + 'f1': float(np.mean(dict_metrics['f1'])), + 'bal_acc': float(np.mean(dict_metrics['bal_acc'])), + }, + 'cost': { + 'acc': float(costs['acc'] / len(seeds)), + 'precision': float(costs['precision'] / len(seeds)), + 'f1': float(costs['f1'] / len(seeds)), + 'bal_acc': float(costs['bal_acc'] / len(seeds)), + 'model_cost': float(costs['model_cost'] / len(seeds)), + }, + 'info': info + } + return result_dict + + +class TabularBenchmarkMO(_TabularBenchmarkBase, AbstractMultiObjectiveBenchmark): + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function(self, configuration: Union[CS.Configuration, Dict], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + seed: Union[int, None] = None, + **kwargs) -> Dict: + result_dict = self._moobjective(config=configuration, fidelity=fidelity, seed=seed, evaluation="val") + + + sensitivity = (result_dict['function_value']['f1'] * result_dict['function_value']['precision']) / (2 * result_dict['function_value']['precision'] - result_dict['function_value']['f1'] ) + + false_negative_rate = 1 - sensitivity + false_positive_rate = 1 - result_dict['function_value']['precision'] + #not considering cost for loss as it is not one of the objectives + cost = result_dict['cost']['model_cost'] + result_dict['cost']['precision'] + result_dict['cost']['f1'] + + result = { + 'function_value': { + 'fnr': float(false_negative_rate), + 'fpr': float(false_positive_rate), + + }, + 'cost': cost, + 'info': result_dict['info'] + } return result + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + seed: Union[int, None] = None, + **kwargs) -> Dict: + result_dict = self._moobjective(config=configuration, fidelity=fidelity, seed=seed, evaluation="test") + sensitivity = (result_dict['function_value']['f1'] * result_dict['function_value']['precision']) / (2 * result_dict['function_value']['precision'] - result_dict['function_value']['f1'] ) + false_negative_rate = 1 - sensitivity + false_positive_rate = 1 - result_dict['function_value']['precision'] + #not considering cost for loss as it is not one of the objectives + cost = result_dict['cost']['model_cost'] + result_dict['cost']['precision'] + result_dict['cost']['f1'] + + result = { + 'function_value': { + 'fnr': float(false_negative_rate), + 'fpr': float(false_positive_rate), + + }, + 'cost': cost, + 'info': result_dict['info'] + } + + return result + + @staticmethod + def get_objective_names() -> List[str]: + return ['fnr', 'fpr'] + + +class TabularBenchmark(_TabularBenchmarkBase, AbstractSingleObjectiveBenchmark): + # pylint: disable=arguments-differ + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + seed: Union[int, None] = None, + metric: Union[str, None] = 'acc', + **kwargs) -> Dict: -__all__ = ['TabularBenchmark'] + metric_str = ', '.join(list(metrics.keys())) + assert metric in list(metrics.keys()), f"metric not found among: {metric_str}" + result_dict = self._moobjective(configuration, fidelity, seed, evaluation="val") + result = dict(function_value=float(1-result_dict['function_value'][metric]), cost=result_dict['cost'][metric], info=result_dict['info']) + return result + + # pylint: disable=arguments-differ + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + seed: Union[int, None] = None, + metric: Union[str, None] = 'acc', + **kwargs) -> Dict: + + metric_str = ', '.join(list(metrics.keys())) + assert metric in list(metrics.keys()), f"metric not found among: {metric_str}" + result_dict = self._moobjective(configuration, fidelity, seed, evaluation="test") + result = dict(function_value=float(1-result_dict['function_value'][metric]), cost=result_dict['cost'][metric], info=result_dict['info']) + return result + +__all__ = ['TabularBenchmark', 'TabularBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/xgboost_benchmark.py b/hpobench/benchmarks/ml/xgboost_benchmark.py old mode 100644 new mode 100755 index ae554628..f2d65a99 --- a/hpobench/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/benchmarks/ml/xgboost_benchmark.py @@ -12,18 +12,27 @@ import xgboost as xgb from ConfigSpace.hyperparameters import Hyperparameter -from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark +from hpobench.dependencies.ml.ml_benchmark_template import MLBenchmark, MO_MLBenchmark -__version__ = '0.0.1' +''' +Changelog: +========== +0.0.2: +* Add the multiobjective version of this benchmark by returning val loss, precision, f1 and balanced accuracy +0.0.1: +* First implementation +''' +__version__ = '0.0.2' -class XGBoostBenchmark(MLBenchmark): + +class _XGBoostBenchmarkBase: def __init__(self, task_id: int, rng: Union[np.random.RandomState, int, None] = None, valid_size: float = 0.33, data_path: Union[str, None] = None): - super(XGBoostBenchmark, self).__init__(task_id, rng, valid_size, data_path) + super(_XGBoostBenchmarkBase, self).__init__(task_id, rng, valid_size, data_path) @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -108,6 +117,13 @@ def init_model(self, ) return model +class XGBoostBenchmark(_XGBoostBenchmarkBase, MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(XGBoostBenchmark, self).__init__(task_id, rng, valid_size, data_path) class XGBoostBenchmarkBB(XGBoostBenchmark): def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -127,6 +143,15 @@ def get_fidelity_space(self, seed: Union[int, None] = None) -> CS.ConfigurationS XGBoostBenchmark._get_fidelity_choices(n_estimators_choice='variable', subsample_choice='fixed') ) return fidelity_space + +class XGBoostBenchmarkMO(_XGBoostBenchmarkBase, MO_MLBenchmark): + def __init__(self, + task_id: int, + rng: Union[np.random.RandomState, int, None] = None, + valid_size: float = 0.33, + data_path: Union[str, None] = None): + super(XGBoostBenchmarkMO, self).__init__(task_id, rng, valid_size, data_path) + -__all__ = ['XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmark'] +__all__ = ['XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmark', 'XGBoostBenchmarkMO'] diff --git a/hpobench/benchmarks/ml/xgboost_benchmark_old.py b/hpobench/benchmarks/ml/xgboost_benchmark_old.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/mo/__init__.py b/hpobench/benchmarks/mo/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/mo/adult_benchmark.py b/hpobench/benchmarks/mo/adult_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/mo/cnn_benchmark.py b/hpobench/benchmarks/mo/cnn_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/__init__.py b/hpobench/benchmarks/nas/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/jahs_benchmarks.py b/hpobench/benchmarks/nas/jahs_benchmarks.py old mode 100644 new mode 100755 index e787fa7c..ba9096da --- a/hpobench/benchmarks/nas/jahs_benchmarks.py +++ b/hpobench/benchmarks/nas/jahs_benchmarks.py @@ -78,15 +78,6 @@ def download_and_extract_url(url, save_dir, filename): with tarfile.open(save_tar_file, 'r') as f: f.extractall(path=save_dir) - if save_tar_file.name == 'assembled_surrogates.tar': - from shutil import move - _dir = save_dir / 'assembled_surrogates' - _dir.mkdir(exist_ok=True, parents=True) - for dir_name in ['cifar10', 'colorectal_histology', 'fashion_mnist']: - _old_dir = save_dir / dir_name - _new_dir = _dir / dir_name - move(_old_dir, _new_dir) - logger.info("Done extracting") finish_flag.touch() diff --git a/hpobench/benchmarks/nas/nasbench_101.py b/hpobench/benchmarks/nas/nasbench_101.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/nasbench_1shot1.py b/hpobench/benchmarks/nas/nasbench_1shot1.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/nasbench_201.py b/hpobench/benchmarks/nas/nasbench_201.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/nas/tabular_benchmarks.py b/hpobench/benchmarks/nas/tabular_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/__init__.py b/hpobench/benchmarks/od/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/od_ae.py b/hpobench/benchmarks/od/od_ae.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/od_benchmarks.py b/hpobench/benchmarks/od/od_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/od_kde.py b/hpobench/benchmarks/od/od_kde.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/od/od_ocsvm.py b/hpobench/benchmarks/od/od_ocsvm.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/rl/__init__.py b/hpobench/benchmarks/rl/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/rl/cartpole.py b/hpobench/benchmarks/rl/cartpole.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/rl/learna_benchmark.py b/hpobench/benchmarks/rl/learna_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/surrogates/__init__.py b/hpobench/benchmarks/surrogates/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/surrogates/paramnet_benchmark.py b/hpobench/benchmarks/surrogates/paramnet_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/surrogates/svm_benchmark.py b/hpobench/benchmarks/surrogates/svm_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/benchmarks/surrogates/yahpo_gym.py b/hpobench/benchmarks/surrogates/yahpo_gym.py old mode 100644 new mode 100755 diff --git a/hpobench/config.py b/hpobench/config.py old mode 100644 new mode 100755 index 9d7964e0..81143257 --- a/hpobench/config.py +++ b/hpobench/config.py @@ -65,7 +65,7 @@ def __init__(self): # Options for the singularity container self.socket_dir = Path(self.socket_dir).expanduser().absolute() self.container_dir = self.cache_dir / f'hpobench-{os.getuid()}' - self.container_source = 'oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry' + self.container_source = 'oras://gitlab.tf.uni-freiburg.de:5050/robertsj/mo-hpobench' self.pyro_connect_max_wait = 400 # Read in the hpobenchrc file and set the default values if not specified diff --git a/hpobench/container/__init__.py b/hpobench/container/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/__init__.py b/hpobench/container/benchmarks/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/ml/__init__.py b/hpobench/container/benchmarks/ml/__init__.py old mode 100644 new mode 100755 index ed2ce40f..ddc66a34 --- a/hpobench/container/benchmarks/ml/__init__.py +++ b/hpobench/container/benchmarks/ml/__init__.py @@ -4,7 +4,7 @@ from hpobench.container.benchmarks.ml.rf_benchmark import RandomForestBenchmark, RandomForestBenchmarkBB, \ RandomForestBenchmarkMF from hpobench.container.benchmarks.ml.svm_benchmark import SVMBenchmark, SVMBenchmarkBB, SVMBenchmarkMF -from hpobench.container.benchmarks.ml.tabular_benchmark import TabularBenchmark +from hpobench.container.benchmarks.ml.tabular_benchmark import TabularBenchmark, TabularBenchmarkMO from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark, XGBoostBenchmarkBB, XGBoostBenchmarkMF @@ -13,5 +13,5 @@ 'NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'SVMBenchmark', 'SVMBenchmarkBB', 'SVMBenchmarkMF', - 'TabularBenchmark', + 'TabularBenchmark', 'TabularBenchmarkMO', 'XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF'] diff --git a/hpobench/container/benchmarks/ml/histgb_benchmark.py b/hpobench/container/benchmarks/ml/histgb_benchmark.py old mode 100644 new mode 100755 index dc7af088..14d1c9a2 --- a/hpobench/container/benchmarks/ml/histgb_benchmark.py +++ b/hpobench/container/benchmarks/ml/histgb_benchmark.py @@ -13,6 +13,12 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(HistGBBenchmark, self).__init__(**kwargs) +class HistGBBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'HistGBBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(HistGBBenchmark, self).__init__(**kwargs) class HistGBBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +36,8 @@ def __init__(self, **kwargs): super(HistGBBenchmarkMF, self).__init__(**kwargs) -__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF'] +<<<<<<< HEAD +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO'] +======= +__all__ = ['HistGBBenchmark', 'HistGBBenchmarkBB', 'HistGBBenchmarkMF', 'HistGBBenchmarkMO'] +>>>>>>> mo_tabular diff --git a/hpobench/container/benchmarks/ml/lr_benchmark.py b/hpobench/container/benchmarks/ml/lr_benchmark.py old mode 100644 new mode 100755 index 979cda3e..c6e61302 --- a/hpobench/container/benchmarks/ml/lr_benchmark.py +++ b/hpobench/container/benchmarks/ml/lr_benchmark.py @@ -13,6 +13,17 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(LRBenchmark, self).__init__(**kwargs) +class LRBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'LRBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' + # kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + # kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(LRBenchmarkMO, self).__init__(**kwargs) + class LRBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +41,8 @@ def __init__(self, **kwargs): super(LRBenchmarkMF, self).__init__(**kwargs) -__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF'] +<<<<<<< HEAD +__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO'] +======= +__all__ = ['LRBenchmark', 'LRBenchmarkBB', 'LRBenchmarkMF', 'LRBenchmarkMO'] +>>>>>>> mo_tabular diff --git a/hpobench/container/benchmarks/ml/nn_benchmark.py b/hpobench/container/benchmarks/ml/nn_benchmark.py old mode 100644 new mode 100755 index 04955e82..eb6090f3 --- a/hpobench/container/benchmarks/ml/nn_benchmark.py +++ b/hpobench/container/benchmarks/ml/nn_benchmark.py @@ -13,6 +13,18 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(NNBenchmark, self).__init__(**kwargs) +class NNBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'NNBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + + #Need to be changed to the production registry + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' + # kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + # kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(NNBenchmarkMO, self).__init__(**kwargs) class NNBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +42,8 @@ def __init__(self, **kwargs): super(NNBenchmarkMF, self).__init__(**kwargs) -__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF'] +<<<<<<< HEAD +__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] +======= +__all__ = ['NNBenchmark', 'NNBenchmarkBB', 'NNBenchmarkMF', 'NNBenchmarkMO'] +>>>>>>> mo_tabular diff --git a/hpobench/container/benchmarks/ml/pybnn.py b/hpobench/container/benchmarks/ml/pybnn.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/ml/rf_benchmark.py b/hpobench/container/benchmarks/ml/rf_benchmark.py old mode 100644 new mode 100755 index a414349d..bbbabda2 --- a/hpobench/container/benchmarks/ml/rf_benchmark.py +++ b/hpobench/container/benchmarks/ml/rf_benchmark.py @@ -14,6 +14,21 @@ def __init__(self, **kwargs): super(RandomForestBenchmark, self).__init__(**kwargs) + +class RandomForestBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + + #Need to be changed to the production registry + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' + # kwargs['container_name'] = kwargs.get('container_name', 'ml_mmfb') + # kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(RandomForestBenchmarkMO, self).__init__(**kwargs) + + class RandomForestBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'RandomForestBenchmarkBB') @@ -30,4 +45,8 @@ def __init__(self, **kwargs): super(RandomForestBenchmarkMF, self).__init__(**kwargs) -__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF'] +<<<<<<< HEAD +__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'RandomForestBenchmarkMO'] +======= +__all__ = ['RandomForestBenchmark', 'RandomForestBenchmarkBB', 'RandomForestBenchmarkMF', 'RandomForestBenchmarkMO'] +>>>>>>> mo_tabular diff --git a/hpobench/container/benchmarks/ml/svm_benchmark.py b/hpobench/container/benchmarks/ml/svm_benchmark.py old mode 100644 new mode 100755 index 7547a81a..564515b1 --- a/hpobench/container/benchmarks/ml/svm_benchmark.py +++ b/hpobench/container/benchmarks/ml/svm_benchmark.py @@ -13,6 +13,18 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(SVMBenchmark, self).__init__(**kwargs) +class SVMBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'SVMBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + + #Need to be changed to the production registry + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' + #kwargs['latest'] = kwargs.get('container_tag', '0.0.2') + super(SVMBenchmarkMO, self).__init__(**kwargs) + class SVMBenchmarkMF(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -30,4 +42,8 @@ def __init__(self, **kwargs): super(SVMBenchmarkBB, self).__init__(**kwargs) -__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB'] +<<<<<<< HEAD +__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB', 'SVMBenchmarkMO'] +======= +__all__ = ['SVMBenchmark', 'SVMBenchmarkMF', 'SVMBenchmarkBB', 'SVMBenchmarkMO'] +>>>>>>> mo_tabular diff --git a/hpobench/container/benchmarks/ml/svm_benchmark_old.py b/hpobench/container/benchmarks/ml/svm_benchmark_old.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/ml/tabular_benchmark.py b/hpobench/container/benchmarks/ml/tabular_benchmark.py old mode 100644 new mode 100755 index 6d19953b..1ea873be --- a/hpobench/container/benchmarks/ml/tabular_benchmark.py +++ b/hpobench/container/benchmarks/ml/tabular_benchmark.py @@ -13,5 +13,16 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(TabularBenchmark, self).__init__(**kwargs) +class TabularBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'TabularBenchmarkMO') + + + #Need to be changed to the production registry + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.8' + kwargs['latest'] = '0.0.8' + super(TabularBenchmarkMO, self).__init__(**kwargs) -__all__ = ['TabularBenchmark'] +__all__ = ['TabularBenchmark', 'TabularBenchmarkMO'] diff --git a/hpobench/container/benchmarks/ml/xgboost_benchmark.py b/hpobench/container/benchmarks/ml/xgboost_benchmark.py old mode 100644 new mode 100755 index c82ea606..644649df --- a/hpobench/container/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/container/benchmarks/ml/xgboost_benchmark.py @@ -13,6 +13,17 @@ def __init__(self, **kwargs): kwargs['latest'] = kwargs.get('container_tag', '0.0.1') super(XGBoostBenchmark, self).__init__(**kwargs) +class XGBoostBenchmarkMO(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'XGBoostBenchmarkMO') + kwargs['container_name'] = kwargs.get('container_name', 'mo_ml_mmfb') + + #Need to be changed to the production registry + kwargs['container_source'] = 'oras://gitlab.tf.uni-freiburg.de:5050/sharmaa/hpobench-registry' + kwargs['container_tag'] = '0.0.6' + kwargs['latest'] = '0.0.6' #kwargs.get('container_tag', '0.0.2') + super(XGBoostBenchmarkMO, self).__init__(**kwargs) + class XGBoostBenchmarkBB(AbstractBenchmarkClient): def __init__(self, **kwargs): @@ -38,4 +49,4 @@ def __init__(self, **kwargs): super(XGBoostSearchSpace3Benchmark, self).__init__(**kwargs) -__all__ = ['XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF'] +__all__ = ['XGBoostBenchmark', 'XGBoostBenchmarkBB', 'XGBoostBenchmarkMF', 'XGBoostBenchmarkMO'] \ No newline at end of file diff --git a/hpobench/container/benchmarks/ml/xgboost_benchmark_old.py b/hpobench/container/benchmarks/ml/xgboost_benchmark_old.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/mo/__init__.py b/hpobench/container/benchmarks/mo/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/mo/adult_benchmark.py b/hpobench/container/benchmarks/mo/adult_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/mo/cnn_benchmark.py b/hpobench/container/benchmarks/mo/cnn_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/__init__.py b/hpobench/container/benchmarks/nas/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/jahs_benchmarks.py b/hpobench/container/benchmarks/nas/jahs_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/nasbench_101.py b/hpobench/container/benchmarks/nas/nasbench_101.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/nasbench_1shot1.py b/hpobench/container/benchmarks/nas/nasbench_1shot1.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/nasbench_201.py b/hpobench/container/benchmarks/nas/nasbench_201.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/nas/tabular_benchmarks.py b/hpobench/container/benchmarks/nas/tabular_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/od/__init__.py b/hpobench/container/benchmarks/od/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/od/od_benchmarks.py b/hpobench/container/benchmarks/od/od_benchmarks.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/rl/__init__.py b/hpobench/container/benchmarks/rl/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/rl/cartpole.py b/hpobench/container/benchmarks/rl/cartpole.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/rl/learna_benchmark.py b/hpobench/container/benchmarks/rl/learna_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/surrogates/__init__.py b/hpobench/container/benchmarks/surrogates/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/surrogates/paramnet_benchmark.py b/hpobench/container/benchmarks/surrogates/paramnet_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/surrogates/svm_benchmark.py b/hpobench/container/benchmarks/surrogates/svm_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/container/benchmarks/surrogates/yahpo_gym.py b/hpobench/container/benchmarks/surrogates/yahpo_gym.py old mode 100644 new mode 100755 diff --git a/hpobench/container/client_abstract_benchmark.py b/hpobench/container/client_abstract_benchmark.py old mode 100644 new mode 100755 index d2963c00..722ffed1 --- a/hpobench/container/client_abstract_benchmark.py +++ b/hpobench/container/client_abstract_benchmark.py @@ -165,7 +165,10 @@ def _parse_fidelities(self, fidelity: Union[CS.Configuration, Dict, None] = None fidelity = fidelity else: raise ValueError(f'Type of fidelity not understood: {type(fidelity)}') + print("fidelity",type(fidelity)) + #print("fidelity type", type(fidelity['nepochs'])) f_str = json.dumps(fidelity, indent=None, cls=BenchmarkEncoder) + print("fstr",f_str) return f_str def load_benchmark(self, benchmark_name: str, container_name: str, container_source: Optional[str] = None, @@ -180,7 +183,7 @@ def load_benchmark(self, benchmark_name: str, container_name: str, container_sou self.container_source = container_source or self.config.container_source self.container_dir = Path(self.config.container_dir) - if (self.container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry') + if (self.container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/robertsj/mo-hpobench') and container_tag == 'latest'): assert 'latest' in kwargs, 'If the container is hosted on the gitlab registry, make sure that in the ' \ 'container init, the field \'latest\' is set.' @@ -222,7 +225,8 @@ def download_container(container_dir, container_name, container_source, containe # Currently, we can't see the available container tags on gitlab. Therefore, we create for each # "tag" a new entry in the registry. This might change in the future. But as long as we don't have # a fix for this, we need to map the container tag differently. - if container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry'): + if (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/muelleph/hpobench-registry')) or \ + (container_source.startswith('oras://gitlab.tf.uni-freiburg.de:5050/robertsj/mo-hpobench')): cmd += f'{container_source}/{container_name.lower()}/{container_tag}:latest' else: cmd += f'{container_source}/{container_name.lower()}:{container_tag}' @@ -389,9 +393,16 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], ------- Dict """ + logger.info(f"fidelity:{fidelity}") + + #logger.info(f"type fidelity:{type(fidelity['nepochs'])}") + logger.info(f"config:{configuration}") + #fidelity = fidelity['nepochs'] c_str = self._parse_configuration(configuration) f_str = self._parse_fidelities(fidelity) kwargs_str = self._parse_kwargs(rng, **kwargs) + print("c_str",c_str) + print("fidelity f_str",f_str) json_str = self.benchmark.objective_function(c_str, f_str, kwargs_str) return json.loads(json_str, cls=BenchmarkDecoder) diff --git a/hpobench/container/recipes/Singularity.template b/hpobench/container/recipes/Singularity.template old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.PyBNN b/hpobench/container/recipes/ml/Singularity.PyBNN old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.SupportVectorMachine b/hpobench/container/recipes/ml/Singularity.SupportVectorMachine old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.XGBoostBenchmark b/hpobench/container/recipes/ml/Singularity.XGBoostBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.ml_mmfb b/hpobench/container/recipes/ml/Singularity.ml_mmfb old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.ml_tabular_benchmarks b/hpobench/container/recipes/ml/Singularity.ml_tabular_benchmarks old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/ml/Singularity.mo_ml_mmfb b/hpobench/container/recipes/ml/Singularity.mo_ml_mmfb new file mode 100644 index 00000000..0249e727 --- /dev/null +++ b/hpobench/container/recipes/ml/Singularity.mo_ml_mmfb @@ -0,0 +1,25 @@ +Bootstrap: docker +From: python:3.8-slim + +%labels +MAINTAINER sharmaa@informatik.uni-freiburg.de +VERSION v0.0.2 + +%post + apt update -y \ + && apt install build-essential git -y \ + + cd /home \ + && git clone https://github.com/ayushi-3536/HPOBench.git \ + && cd HPOBench \ + && git checkout mo_tabular \ + && pip install ".[ml_mfbb]" \ + && cd / \ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ \ + && pip cache purge \ + && rm -rf /var/lib/apt/lists/* + + +%runscript + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml $@ diff --git a/hpobench/container/recipes/mo/Singularity.AdultBenchmark b/hpobench/container/recipes/mo/Singularity.AdultBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/mo/Singularity.CNNBenchmark b/hpobench/container/recipes/mo/Singularity.CNNBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/nas/Singularity.TabularBenchmarks b/hpobench/container/recipes/nas/Singularity.TabularBenchmarks old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/nas/Singularity.nasbench_101 b/hpobench/container/recipes/nas/Singularity.nasbench_101 old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/nas/Singularity.nasbench_1shot1 b/hpobench/container/recipes/nas/Singularity.nasbench_1shot1 old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/nas/Singularity.nasbench_201 b/hpobench/container/recipes/nas/Singularity.nasbench_201 old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/od/Singularity.ODBenchmarks b/hpobench/container/recipes/od/Singularity.ODBenchmarks old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/od/Singularity.ODKernelDensityEstimation b/hpobench/container/recipes/od/Singularity.ODKernelDensityEstimation old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/rl/Singularity.Cartpole b/hpobench/container/recipes/rl/Singularity.Cartpole old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/rl/Singularity.learnaBenchmark b/hpobench/container/recipes/rl/Singularity.learnaBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/surrogates/Singularity.ParamnetBenchmark b/hpobench/container/recipes/surrogates/Singularity.ParamnetBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/surrogates/Singularity.SupportVectorMachine b/hpobench/container/recipes/surrogates/Singularity.SupportVectorMachine old mode 100644 new mode 100755 diff --git a/hpobench/container/recipes/surrogates/Singularity.YAHPOGymBenchmark b/hpobench/container/recipes/surrogates/Singularity.YAHPOGymBenchmark old mode 100644 new mode 100755 diff --git a/hpobench/container/server_abstract_benchmark.py b/hpobench/container/server_abstract_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/__init__.py b/hpobench/dependencies/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/ml/__init__.py b/hpobench/dependencies/ml/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/ml/data_manager.py b/hpobench/dependencies/ml/data_manager.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/ml/ml_benchmark_template.py b/hpobench/dependencies/ml/ml_benchmark_template.py old mode 100644 new mode 100755 index 7cef515f..56e881da --- a/hpobench/dependencies/ml/ml_benchmark_template.py +++ b/hpobench/dependencies/ml/ml_benchmark_template.py @@ -1,16 +1,15 @@ import time from pathlib import Path from typing import Union, Dict, Iterable - import ConfigSpace as CS import numpy as np import pandas as pd from sklearn.metrics import make_scorer, accuracy_score, balanced_accuracy_score, \ precision_score, f1_score - -from hpobench.abstract_benchmark import AbstractBenchmark from hpobench.dependencies.ml.data_manager import OpenMLDataManager from hpobench.util.rng_helper import get_rng +from hpobench.abstract_benchmark import AbstractMultiObjectiveBenchmark, AbstractSingleObjectiveBenchmark +from typing import Union, Dict, List, Text, Tuple metrics = dict( acc=accuracy_score, @@ -27,7 +26,7 @@ ) -class MLBenchmark(AbstractBenchmark): +class _MLBenchmarkBase(): _issue_tasks = [3917, 3945] def __init__( @@ -38,7 +37,7 @@ def __init__( data_path: Union[str, Path, None] = None, global_seed: int = 1 ): - super(MLBenchmark, self).__init__(rng=rng) + super(_MLBenchmarkBase, self).__init__(rng=rng) if isinstance(rng, int): self.seed = rng @@ -197,17 +196,40 @@ def _train_objective(self, score_cost[k] = time.time() - _start train_loss = 1 - scores["acc"] return model, model_fit_time, train_loss, scores, score_cost - - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function(self, - configuration: Union[CS.Configuration, Dict], - fidelity: Union[CS.Configuration, Dict, None] = None, - shuffle: bool = False, - rng: Union[np.random.RandomState, int, None] = None, - **kwargs) -> Dict: + + @AbstractMultiObjectiveBenchmark.check_parameters + def _mo_objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: """Function that evaluates a 'config' on a 'fidelity' on the validation set + + Parameters + ---------- + configuration : Dict, CS.Configuration + fidelity: Dict, None + Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. + rng : np.random.RandomState, int, None + Random seed to use in the benchmark. + + To prevent overfitting on a single seed, it is possible to pass a + parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. + If this parameter is not given, the default random state is used. + kwargs + + Returns + ------- + Dict - + function_value : + ToDo: add return values + cost : runtime + info : Dict + fidelity : used fidelities in this evaluation + """ + model, model_fit_time, train_loss, train_scores, train_score_cost = self._train_objective( configuration, fidelity, shuffle, rng, evaluation="val" ) @@ -218,6 +240,9 @@ def objective_function(self, val_scores[k] = v(model, self.valid_X, self.valid_y) val_score_cost[k] = time.time() - _start val_loss = 1 - val_scores["acc"] + val_sensitivity = (val_scores['f1'] * val_scores['precision']) / (2 * val_scores['precision'] - val_scores['f1'] ) + val_scores['fnr'] = 1 - val_sensitivity + val_scores['fpr'] = 1 - val_scores['precision'] test_scores = dict() test_score_cost = dict() @@ -227,6 +252,10 @@ def objective_function(self, test_score_cost[k] = time.time() - _start test_loss = 1 - test_scores["acc"] + test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) + test_scores['fnr'] = 1 - test_sensitivity + test_scores['fpr'] = 1 - test_scores['precision'] + info = { 'train_loss': train_loss, 'val_loss': val_loss, @@ -243,15 +272,23 @@ def objective_function(self, 'config': configuration, } - return { - 'function_value': info['val_loss'], + result_dict = { + 'function_value': { + # The original benchmark returned the accuracy with range [0, 100]. + # We cast it to a minimization problem with range [0-1] to have a more standardized return value. + 'val_loss': float(val_loss), + 'fnr': float(val_scores['fnr']), + 'fpr': float(val_scores['fpr']), + + }, 'cost': model_fit_time + info['val_costs']['acc'], 'info': info } - # pylint: disable=arguments-differ - @AbstractBenchmark.check_parameters - def objective_function_test(self, + return result_dict + + @AbstractMultiObjectiveBenchmark.check_parameters + def _mo_objective_function_test(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[CS.Configuration, Dict, None] = None, shuffle: bool = False, @@ -270,6 +307,9 @@ def objective_function_test(self, test_score_cost[k] = time.time() - _start test_loss = 1 - test_scores["acc"] + test_sensitivity = (test_scores['f1'] * test_scores['precision']) / (2 * test_scores['precision'] - test_scores['f1'] ) + test_scores['fnr'] = 1 - test_sensitivity + test_scores['fpr'] = 1 - test_scores['precision'] info = { 'train_loss': train_loss, 'val_loss': None, @@ -287,7 +327,99 @@ def objective_function_test(self, } return { - 'function_value': float(info['test_loss']), + 'function_value': { + # The original benchmark returned the accuracy with range [0, 100]. + # We cast it to a minimization problem with range [0-1] to have a more standardized return value. + 'test_loss': float(info['test_loss']), + # 'f1': test_scores['f1'], + 'fpr': float(test_scores['fpr']), + # 'balanced_accuracy': test_scores['bal_acc'], + 'fnr': float(test_scores['fnr']), + }, 'cost': float(model_fit_time + info['test_costs']['acc']), 'info': info } + + +class MLBenchmark(_MLBenchmarkBase, AbstractSingleObjectiveBenchmark): + + + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + """Function that evaluates a 'config' on a 'fidelity' on the validation set + + Parameters + ---------- + configuration : Dict, CS.Configuration + fidelity: Dict, None + Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. + rng : np.random.RandomState, int, None + Random seed to use in the benchmark. + + To prevent overfitting on a single seed, it is possible to pass a + parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. + If this parameter is not given, the default random state is used. + kwargs + + Returns + ------- + Dict - + function_value : + ToDo: add return values + cost : runtime + info : Dict + fidelity : used fidelities in this evaluation + + """ + results = self._mo_objective_function( + configuration=configuration, fidelity=fidelity, rng=rng, **kwargs + ) + results['function_value'] = results['function_value']['val_loss'] + return results + + @AbstractSingleObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + """Function that evaluates a 'config' on a 'fidelity' on the test set + """ + + results = self._mo_objective_function_test( + configuration=configuration, fidelity=fidelity, rng=rng, **kwargs + ) + results['function_value'] = results['function_value']['test_loss'] + return results + +class MO_MLBenchmark(_MLBenchmarkBase, AbstractMultiObjectiveBenchmark): + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + return self._mo_objective_function(configuration=configuration, fidelity=fidelity, shuffle=shuffle, rng=rng, + **kwargs) + + @AbstractMultiObjectiveBenchmark.check_parameters + def objective_function_test(self, + configuration: Union[CS.Configuration, Dict], + fidelity: Union[CS.Configuration, Dict, None] = None, + shuffle: bool = False, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + return self._mo_objective_function_test(configuration=configuration, fidelity=fidelity, shuffle=shuffle, rng=rng, + **kwargs) + @staticmethod + def get_objective_names() -> List[str]: + return ['val_loss', 'fnr', 'fpr'] + diff --git a/hpobench/dependencies/mo/__init__.py b/hpobench/dependencies/mo/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/mo/fairness_metrics.py b/hpobench/dependencies/mo/fairness_metrics.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/mo/scalar.py b/hpobench/dependencies/mo/scalar.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/__init__.py b/hpobench/dependencies/od/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/backbones/__init__.py b/hpobench/dependencies/od/backbones/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/backbones/mlp.py b/hpobench/dependencies/od/backbones/mlp.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/callbacks/__init__.py b/hpobench/dependencies/od/callbacks/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/callbacks/checkpoint_saver.py b/hpobench/dependencies/od/callbacks/checkpoint_saver.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/callbacks/earlystopping.py b/hpobench/dependencies/od/callbacks/earlystopping.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/data_manager.py b/hpobench/dependencies/od/data_manager.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/models/__init__.py b/hpobench/dependencies/od/models/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/models/autoencoder.py b/hpobench/dependencies/od/models/autoencoder.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/traditional_benchmark.py b/hpobench/dependencies/od/traditional_benchmark.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/utils/__init__.py b/hpobench/dependencies/od/utils/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/utils/activations.py b/hpobench/dependencies/od/utils/activations.py old mode 100644 new mode 100755 diff --git a/hpobench/dependencies/od/utils/scaler.py b/hpobench/dependencies/od/utils/scaler.py old mode 100644 new mode 100755 diff --git a/hpobench/util/__init__.py b/hpobench/util/__init__.py old mode 100644 new mode 100755 diff --git a/hpobench/util/clean_up_script.py b/hpobench/util/clean_up_script.py old mode 100644 new mode 100755 diff --git a/hpobench/util/container_utils.py b/hpobench/util/container_utils.py old mode 100644 new mode 100755 diff --git a/hpobench/util/data_manager.py b/hpobench/util/data_manager.py old mode 100644 new mode 100755 index c72305e1..f2b21ee8 --- a/hpobench/util/data_manager.py +++ b/hpobench/util/data_manager.py @@ -1172,7 +1172,7 @@ def get_workclass(x): class TabularDataManager(DataManager): - def __init__(self, model: str, task_id: [int, str], data_dir: [str, Path, None] = None): + def __init__(self, model: str, task_id: Union[int, str], data_dir: Union[str, Path, None] = None): super(TabularDataManager, self).__init__() url_dict = dict( @@ -1182,7 +1182,7 @@ def __init__(self, model: str, task_id: [int, str], data_dir: [str, Path, None] rf="https://ndownloader.figshare.com/files/30469089", nn="https://ndownloader.figshare.com/files/30379005" ) - + print("url dict", url_dict) assert model in url_dict.keys(), \ f'Model has to be one of {list(url_dict.keys())} but was {model}' @@ -1190,6 +1190,7 @@ def __init__(self, model: str, task_id: [int, str], data_dir: [str, Path, None] self.task_id = str(task_id) self.url_to_use = url_dict.get(model) + print("url_to_use",self.url_to_use) if data_dir is None: data_dir = hpobench.config_file.data_dir / "TabularData" @@ -1209,9 +1210,11 @@ def load(self): return table, metadata # We have to download the entire zip file and etract then extract the parquet file. + print("save dir", self.save_dir) self._download_file_with_progressbar(self.url_to_use, self._save_dir / f'{self.model}.zip') self._unzip_data(self._save_dir / f'{self.model}.zip', self._save_dir) table = self._load_parquet(self.parquet_file) + print("table", table) metadata = self._load_json(self.metadata_file) return table, metadata diff --git a/hpobench/util/dependencies.py b/hpobench/util/dependencies.py old mode 100644 new mode 100755 diff --git a/hpobench/util/example_utils.py b/hpobench/util/example_utils.py old mode 100644 new mode 100755 diff --git a/hpobench/util/openml_data_manager.py b/hpobench/util/openml_data_manager.py old mode 100644 new mode 100755 index 6af5709f..093f417f --- a/hpobench/util/openml_data_manager.py +++ b/hpobench/util/openml_data_manager.py @@ -69,6 +69,7 @@ def get_openmlcc18_taskids(): def _load_data(task_id: int): """ Helper-function to load the data from the OpenML website. """ + print("openml task id", task_id) task = openml.tasks.get_task(task_id) try: diff --git a/hpobench/util/rng_helper.py b/hpobench/util/rng_helper.py old mode 100644 new mode 100755 diff --git a/hpobench/util/test_utils.py b/hpobench/util/test_utils.py old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 diff --git a/tests/__init__.py b/tests/__init__.py old mode 100644 new mode 100755 diff --git a/tests/test_abstract_benchmark.py b/tests/test_abstract_benchmark.py old mode 100644 new mode 100755 diff --git a/tests/test_adult.py b/tests/test_adult.py old mode 100644 new mode 100755 index b52c37ed..0b2a7a0b --- a/tests/test_adult.py +++ b/tests/test_adult.py @@ -6,9 +6,17 @@ def test_adult_benchmark(): from hpobench.container.benchmarks.mo.adult_benchmark import AdultBenchmark + from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark + from hpobench.benchmarks.mo.adult_benchmark import AdultBenchmark # Check Seeding benchmark = AdultBenchmark(rng=0) + print("got adult bench") + print("getting ml bench") + b = XGBoostBenchmark(task_id=167149) + print("b:",b) + + print("got ml bench") cs = benchmark.get_configuration_space(seed=0) cfg_1 = cs.sample_configuration() @@ -35,3 +43,4 @@ def test_adult_benchmark(): result_1 = benchmark.objective_function_test(test_config, rng=1, fidelity={'budget': 3}) assert 1 - result_1['function_value']['misclassification_rate'] == pytest.approx(0.76377, rel=0.001) assert 1 - result_1['function_value']['misclassification_rate'] == result_1['info']['test_accuracy'] +test_adult_benchmark() \ No newline at end of file diff --git a/tests/test_check_configuration.py b/tests/test_check_configuration.py old mode 100644 new mode 100755 diff --git a/tests/test_container_availbable.py b/tests/test_container_availbable.py old mode 100644 new mode 100755 diff --git a/tests/test_data_manager.py b/tests/test_data_manager.py old mode 100644 new mode 100755 diff --git a/tests/test_hpobenchconfig.py b/tests/test_hpobenchconfig.py old mode 100644 new mode 100755 diff --git a/tests/test_mo_cnn.py b/tests/test_mo_cnn.py old mode 100644 new mode 100755 diff --git a/tests/test_nasbench_101.py b/tests/test_nasbench_101.py old mode 100644 new mode 100755 diff --git a/tests/test_nasbench_201.py b/tests/test_nasbench_201.py old mode 100644 new mode 100755 diff --git a/tests/test_od.py b/tests/test_od.py old mode 100644 new mode 100755 diff --git a/tests/test_openml_datamanager.py b/tests/test_openml_datamanager.py old mode 100644 new mode 100755 diff --git a/tests/test_paramnet.py b/tests/test_paramnet.py old mode 100644 new mode 100755 diff --git a/tests/test_pybnn.py b/tests/test_pybnn.py old mode 100644 new mode 100755 diff --git a/tests/test_server.py b/tests/test_server.py old mode 100644 new mode 100755 diff --git a/tests/test_svm.py b/tests/test_svm.py old mode 100644 new mode 100755 index c3acf007..444f6dd5 --- a/tests/test_svm.py +++ b/tests/test_svm.py @@ -1,6 +1,6 @@ import pytest -from hpobench.container.benchmarks.ml.svm_benchmark_old import SupportVectorMachine +from hpobench.benchmarks.ml.svm_benchmark import SVMBenchmark from hpobench.util.openml_data_manager import get_openmlcc18_taskids task_ids = get_openmlcc18_taskids() @@ -8,9 +8,8 @@ import logging logging.basicConfig(level=logging.DEBUG) - def test_svm_init(): - benchmark = SupportVectorMachine(task_id=task_ids[0]) + benchmark = SVMBenchmark(task_id=task_ids[0]) fs = benchmark.get_fidelity_space(seed=0) fidelity = fs.sample_configuration().get_dictionary() @@ -34,3 +33,29 @@ def test_svm_init(): result = benchmark.objective_function_test(configuration=config) assert result['function_value'] == pytest.approx(0.4648, abs=0.1) assert result['cost'] is not None + +# def test_svm_init(): +# benchmark = SupportVectorMachine(task_id=task_ids[0]) + +# fs = benchmark.get_fidelity_space(seed=0) +# fidelity = fs.sample_configuration().get_dictionary() +# assert fidelity['dataset_fraction'] == pytest.approx(0.54881, abs=0.001) + +# meta = benchmark.get_meta_information() +# assert meta is not None + +# cs = benchmark.get_configuration_space(seed=0) +# config = cs.sample_configuration().get_dictionary() +# assert config['C'] == pytest.approx(0.9762, abs=0.001) +# assert config['gamma'] == pytest.approx(4.3037, abs=0.001) + +# result = benchmark.objective_function(configuration=config, fidelity=fidelity) +# assert result['function_value'] == pytest.approx(0.4837, abs=0.1) +# assert result['cost'] is not None + +# with pytest.raises(AssertionError): +# result = benchmark.objective_function_test(configuration=config, fidelity=fidelity) + +# result = benchmark.objective_function_test(configuration=config) +# assert result['function_value'] == pytest.approx(0.4648, abs=0.1) +# assert result['cost'] is not None diff --git a/tests/test_tabular_benchmarks.py b/tests/test_tabular_benchmarks.py old mode 100644 new mode 100755 diff --git a/tests/test_utils.py b/tests/test_utils.py old mode 100644 new mode 100755 diff --git a/tests/test_whitebox.py b/tests/test_whitebox.py old mode 100644 new mode 100755 diff --git a/tests/test_yahpo.py b/tests/test_yahpo.py old mode 100644 new mode 100755