From a367cd0f08c4fc3c399e0f0fa30a2ac806b6c953 Mon Sep 17 00:00:00 2001 From: Anda Zhou <83614683+azhou-determined@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:01:18 -0700 Subject: [PATCH] chore: deprecate Custom Searcher [MD-504] (#9829) Add deprecation warnings for Custom Searcher --- .../apis-howto/deepspeed/autotuning.rst | 5 +++++ .../search-methods/hp-custom.rst | 6 +++++ docs/reference/custom-searcher-reference.rst | 5 +++++ .../reference/experiment-config-reference.rst | 4 +--- .../deprecate-custom-searchers.rst | 8 +++++++ .../fixtures/custom_searcher/searchers.py | 22 +++++++++++++++++++ .../searcher/_remote_search_runner.py | 8 +++++++ harness/determined/searcher/_search_method.py | 10 +++++++++ harness/determined/searcher/_search_runner.py | 8 +++++++ 9 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 docs/release-notes/deprecate-custom-searchers.rst diff --git a/docs/model-dev-guide/api-guides/apis-howto/deepspeed/autotuning.rst b/docs/model-dev-guide/api-guides/apis-howto/deepspeed/autotuning.rst index e3454f577b9..143895b5d36 100644 --- a/docs/model-dev-guide/api-guides/apis-howto/deepspeed/autotuning.rst +++ b/docs/model-dev-guide/api-guides/apis-howto/deepspeed/autotuning.rst @@ -4,6 +4,11 @@ DeepSpeed Autotune: User Guide ################################ +.. important:: + + **Deprecation Notice**: DeepSpeed Autotune is deprecated and will be removed in an upcoming + release. For information on supported search methods, please visit :ref:`search-methods`. + .. meta:: :description: This user guide demonstrates how to optimize DeepSpeed parameters in order to take full advantage of the user's hardware and model. diff --git a/docs/model-dev-guide/hyperparameter/search-methods/hp-custom.rst b/docs/model-dev-guide/hyperparameter/search-methods/hp-custom.rst index ba3f8dd3a91..bf8e047cd1d 100644 --- a/docs/model-dev-guide/hyperparameter/search-methods/hp-custom.rst +++ b/docs/model-dev-guide/hyperparameter/search-methods/hp-custom.rst @@ -4,6 +4,12 @@ Custom Search Methods ####################### +.. important:: + + **Deprecation Notice**: Support for all custom search methods have been deprecated and will be + removed in a future release. Please see :ref:`search-methods` for details on supported preset + searchers. + +----------------------------------------------------------------+ | API reference | +================================================================+ diff --git a/docs/reference/custom-searcher-reference.rst b/docs/reference/custom-searcher-reference.rst index c669c6010be..02c111abb19 100644 --- a/docs/reference/custom-searcher-reference.rst +++ b/docs/reference/custom-searcher-reference.rst @@ -4,6 +4,11 @@ Custom Searcher Reference ########################### +.. important:: + + **Deprecation Notice**: Custom Searcher is deprecated and will be removed in a future release. + Please see :ref:`search-methods` for details on supported preset searchers. + ******************************************* ``determined.searcher.LocalSearchRunner`` ******************************************* diff --git a/docs/reference/experiment-config-reference.rst b/docs/reference/experiment-config-reference.rst index 2304587d742..7b492ba196e 100644 --- a/docs/reference/experiment-config-reference.rst +++ b/docs/reference/experiment-config-reference.rst @@ -868,9 +868,7 @@ points in the grid for this hyperparameter. Grid points are evenly spaced betwee The ``searcher`` section defines how the experiment's hyperparameter space will be explored. To run an experiment that trains a single trial with fixed hyperparameters, specify the ``single`` searcher and specify constant values for the model's hyperparameters. Otherwise, Determined supports three -different hyperparameter search algorithms: ``adaptive_asha``, ``random``, and ``grid``. To define -your own hyperparameter search algorithm, specify the ``custom`` searcher. For more information -about custom search algorithms, see :ref:`topic-guides_hp-tuning-det_custom`. +different hyperparameter search algorithms: ``adaptive_asha``, ``random``, and ``grid``. The name of the hyperparameter search algorithm to use is configured via the ``name`` field; the remaining fields configure the behavior of the searcher and depend on the searcher being used. For diff --git a/docs/release-notes/deprecate-custom-searchers.rst b/docs/release-notes/deprecate-custom-searchers.rst new file mode 100644 index 00000000000..d0fc16d2969 --- /dev/null +++ b/docs/release-notes/deprecate-custom-searchers.rst @@ -0,0 +1,8 @@ +:orphan: + +**Deprecation** + +- Custom Searchers: All custom searchers (including DeepSpeed Autotune) have been deprecated. This + feature will be removed in a future release. We will maintain first-class support for a variety + of preset searchers, which can be easily configured for any experiment. Please see + :ref:`search-methods` for details. diff --git a/e2e_tests/tests/fixtures/custom_searcher/searchers.py b/e2e_tests/tests/fixtures/custom_searcher/searchers.py index ed87b4ad862..7ad1a562c59 100644 --- a/e2e_tests/tests/fixtures/custom_searcher/searchers.py +++ b/e2e_tests/tests/fixtures/custom_searcher/searchers.py @@ -6,6 +6,7 @@ import random import sys import uuid +import warnings from typing import Any, Dict, List, Optional, Set from urllib3 import connectionpool @@ -15,6 +16,13 @@ class SingleSearchMethod(searcher.SearchMethod): def __init__(self, experiment_config: dict, max_length: int) -> None: + warnings.warn( + "`SingleSearchMethod` and all custom searchers have been deprecated. " + "This feature will be removed in a future release. Consider configuring a preset " + "searcher instead (see Determined docs for details).", + FutureWarning, + stacklevel=2, + ) # since this is a single trial the hyperparameter space comprises a single point self.hyperparameters = experiment_config["hyperparameters"] self.max_length = max_length @@ -74,6 +82,13 @@ def __init__( exception_points: Optional[List[str]] = None, metric_as_dict: bool = False, ) -> None: + warnings.warn( + "`RandomSearchMethod` and all custom searchers have been deprecated. " + "This feature will be removed in a future release. Consider configuring a preset " + "searcher instead (see Determined docs for details).", + FutureWarning, + stacklevel=2, + ) self.max_trials = max_trials self.max_concurrent_trials = max_concurrent_trials self.max_length = max_length @@ -363,6 +378,13 @@ def __init__( max_concurrent_trials: int = 16, exception_points: Optional[List[str]] = None, ) -> None: + warnings.warn( + "`ASHASearchMethod` and all custom searchers have been deprecated. " + "This feature will be removed in a future release. Consider configuring a preset " + "searcher instead (see Determined docs for details).", + FutureWarning, + stacklevel=2, + ) self.asha_search_state = ASHASearchMethodState( max_length, max_trials, num_rungs, divisor, max_concurrent_trials ) diff --git a/harness/determined/searcher/_remote_search_runner.py b/harness/determined/searcher/_remote_search_runner.py index 60eec141707..c4d300089c5 100644 --- a/harness/determined/searcher/_remote_search_runner.py +++ b/harness/determined/searcher/_remote_search_runner.py @@ -2,6 +2,7 @@ import os import pathlib import pickle +import warnings from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import determined as det @@ -21,6 +22,13 @@ class RemoteSearchRunner(searcher.SearchRunner): """ def __init__(self, search_method: searcher.SearchMethod, context: det.core.Context) -> None: + warnings.warn( + "`RemoteSearchRunner` and all custom searchers have been deprecated. " + "This feature will be removed in a future release. Consider configuring a preset " + "searcher instead (see Determined docs for details).", + FutureWarning, + stacklevel=2, + ) super().__init__(search_method) self.context = context info = det.get_cluster_info() diff --git a/harness/determined/searcher/_search_method.py b/harness/determined/searcher/_search_method.py index 7ede061c9b2..8190f461f35 100644 --- a/harness/determined/searcher/_search_method.py +++ b/harness/determined/searcher/_search_method.py @@ -4,6 +4,7 @@ import json import pathlib import uuid +import warnings from typing import Any, Dict, List, Optional, Set, Tuple from determined import experimental @@ -224,6 +225,15 @@ class SearchMethod: Do not modify ``searcher_state`` passed into event handlers. """ + def __init__(self) -> None: + warnings.warn( + "`SearchMethod` and all custom searchers have been deprecated. " + "This feature will be removed in a future release. Consider configuring a preset " + "searcher instead (see Determined docs for details).", + FutureWarning, + stacklevel=2, + ) + @abc.abstractmethod def initial_operations(self, searcher_state: SearcherState) -> List[Operation]: """ diff --git a/harness/determined/searcher/_search_runner.py b/harness/determined/searcher/_search_runner.py index 060e7401fad..2081f7145e8 100644 --- a/harness/determined/searcher/_search_runner.py +++ b/harness/determined/searcher/_search_runner.py @@ -5,6 +5,7 @@ import pickle import time import uuid +import warnings from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union from determined import searcher @@ -256,6 +257,13 @@ def __init__( searcher_dir: Optional[pathlib.Path] = None, session: Optional[api.Session] = None, ): + warnings.warn( + "`LocalSearchRunner` and all custom searchers have been deprecated. " + "This feature will be removed in a future release. Consider configuring a preset " + "searcher instead (see Determined docs for details).", + FutureWarning, + stacklevel=2, + ) super().__init__(search_method) self.state_path = None self.session = session