From c41b20b029ca28f127b18a175484ab6232e98561 Mon Sep 17 00:00:00 2001 From: Archit Bansal Date: Mon, 18 Jan 2021 14:34:08 +0100 Subject: [PATCH] Added a local dummy benchmark class for testing interfaces. --- hpobench/benchmarks/dummy_local_benchmarks.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 hpobench/benchmarks/dummy_local_benchmarks.py diff --git a/hpobench/benchmarks/dummy_local_benchmarks.py b/hpobench/benchmarks/dummy_local_benchmarks.py new file mode 100644 index 00000000..741607c8 --- /dev/null +++ b/hpobench/benchmarks/dummy_local_benchmarks.py @@ -0,0 +1,106 @@ +import logging +import time +from typing import Sequence, Callable, Optional, Dict, Union +import numpy as np +from hpobench.abstract_benchmark import AbstractBenchmark +import ConfigSpace + +_log = logging.getLogger(__name__) + + +def sleep_objective_fn(obj, configuration: Dict, + fidelity: Union[Dict, None] = None, + rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: + """ A dummy objective function that does not really perform any computation but simply introduces a delay defined + by the attribute '_sleep_duration' of the calling object. It returns a random value in [0, 1) for the key + 'function_value' and the value of 'obj_sleep_duration' for the cost. It also attaches a dictionary to the key + 'info' containing the rng, configuration and fidelity values the function was called with in the keys 'rng', + 'config' and 'fidelity' respectively. """ + + _log.debug("Dummy objective function 'sleep_objective_fn' called.") + for kwarg in kwargs.items(): + _log.debug("Ignoring additional keyword argument %s to objective function call." % str(kwarg)) + + time.sleep(obj._sleep_duration) + return { + 'function_value': np.random.random(), + 'cost': obj._sleep_duration, + 'info': { + 'rng': rng, + 'config': configuration, + 'fidelity': fidelity + } + } + + +class DummyBenchmark(AbstractBenchmark): + def __init__(self, + configspace_params: Sequence[ConfigSpace.hyperparameters.Hyperparameter], + fidelity_params: Sequence[ConfigSpace.hyperparameters.Hyperparameter], + sleep_duration: Optional[float] = 1.0, + obj_fn: Optional[Callable] = sleep_objective_fn, rng: Union[int, np.random.RandomState, None] = None, + **kwargs): + """ + Initializes a DummyBenchmark object with the specified characteristics. DummyBenchmark objects are not + expected to perform any actual computations but instead serve as aids in debugging interfaces for HPOBench. + They should be configured to expose characteristics expected of an actual benchmark. Note that DummyBenchmark + objects are only useful after the object has been initialized and therefore are not suitable for tests designed + to work on pre-initialization class methods only, such as those for calls to 'get_configuration_space()' on a + Benchmark class itself. To this end, 'get_configuration_space()', 'get_fidelity_space()' and + 'get_meta_information()' have been re-defined as instance functions of the initialized object instead. + + Parameters + ---------- + configspace_params: Sequence of ConfigSpace.hyperparameter.Hyperparameter objects + This list of Hyperparameter objects is directly used to define the DummyBenchmark object's configuration + space. + fidelity_params: Sequence of ConfigSpace.hyperparameter.Hyperparameter objects + This list of Hyperparameter objects is directly used to define the DummyBenchmark object's fidelity space. + sleep_duration: float (optional) + By default, the objective function of DummyBenchmark does not perform any computation and simply + introduces a delay of 'sleep_duration' seconds. The default value is 1.0 (seconds). + obj_fn: a callable object (optional) + This enables the objective function of the dummy benchmark to be changed to a user specified function + instead. Default: sleep_objective_fn. + rng: int, np.random.RandomState, None + The default random state for the benchmark. If type is int, a np.random.RandomState with seed `rng` is + created. If type is None, create a new random state. Ignored by DummyBenchmark, directly passed to + AbstractBenchmark. + """ + + _log.debug("Initializing DummyBenchmark object with %s configuration space parameters and %d fidelity space " + "parameters." % (len(configspace_params), len(fidelity_params))) + self._sleep_duration = sleep_duration + self._config_params = configspace_params + self._fidelity_params = fidelity_params + self._obj_fn = obj_fn + self._meta = {'sleep_duration': sleep_duration, 'obj_fn': obj_fn.__name__} + super(DummyBenchmark, self).__init__(rng=rng) + _log.info("DummyBenchmark initializer received additional keyword arguments:\n%s" % str(kwargs)) + + def get_configuration_space(self, seed: Union[int, None] = None) -> ConfigSpace.ConfigurationSpace: + csp = ConfigSpace.ConfigurationSpace(name="Dummy Configuration Space", seed=seed) + csp.add_hyperparameters(self._config_params) + return csp + + def get_fidelity_space(self, seed: Union[int, None] = None) -> ConfigSpace.ConfigurationSpace: + fsp = ConfigSpace.ConfigurationSpace(name="Dummy Fidelity Space", seed=seed) + fsp.add_hyperparameters(self._fidelity_params) + return fsp + + @AbstractBenchmark.check_parameters + def objective_function(self, configuration: Union[ConfigSpace.Configuration, Dict], + fidelity: Union[Dict, ConfigSpace.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + return self._obj_fn(self, configuration, fidelity, rng, **kwargs) + + @AbstractBenchmark.check_parameters + def objective_function_test(self, configuration: Union[ConfigSpace.Configuration, Dict], + fidelity: Union[Dict, ConfigSpace.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, + **kwargs) -> Dict: + return self._obj_fn(self, configuration, fidelity, rng, **kwargs) + + def get_meta_information(self) -> Dict: + return self._meta