diff --git a/docs/reference/training/api-keras-reference.rst b/docs/reference/training/api-keras-reference.rst index 32689141f21..150722d88be 100644 --- a/docs/reference/training/api-keras-reference.rst +++ b/docs/reference/training/api-keras-reference.rst @@ -29,7 +29,7 @@ Deprecated APIs ################# -The following APIs have been deprecated as of Determined XXYYZZ and will be removed in a future +The following APIs have been deprecated as of Determined 0.38.0 and will be removed in a future version. Please migrate your ``TFKerasTrial``-based training to use the new :class:`~determined.keras.DeterminedCallback` instead. diff --git a/harness/determined/cli/experiment.py b/harness/determined/cli/experiment.py index 7fd091f0d84..8ec1b208cf2 100644 --- a/harness/determined/cli/experiment.py +++ b/harness/determined/cli/experiment.py @@ -190,7 +190,7 @@ def submit_experiment(args: argparse.Namespace) -> None: if args.test_mode: warnings.warn( - "The --test flag to det experiment create has been deprecated in Determined XXYYZZ and " + "The --test flag to det experiment create has been deprecated in Determined 0.38.0 and " "will be removed in a future version. The searcher.max_length setting of the " "experiment config has also been deprecated, and since --test mode relies on that " "setting, --test mode will cease to work as soon as you remove max_length.", @@ -277,7 +277,7 @@ def local_experiment(args: argparse.Namespace) -> None: warnings.warn( "The --local and --test flags to det experiment create have both been deprecated in " - "Determined XXYYZZ and will be removed in a future version. The searcher.max_length " + "Determined 0.38.0 and will be removed in a future version. The searcher.max_length " "setting of the experiment config has also been deprecated, and since --test mode relies " "on that setting, --test mode will cease to work as soon as you remove max_length. " "Additionally, --local mode should no longer be necessary, as you should be able to just " diff --git a/harness/determined/core/_searcher.py b/harness/determined/core/_searcher.py index 806be3fb66c..5b8efe12d11 100644 --- a/harness/determined/core/_searcher.py +++ b/harness/determined/core/_searcher.py @@ -56,7 +56,7 @@ def convert_key(key: Any) -> Optional[Unit]: class SearcherOperation: """ .. warning:: - SearcherOperation is deprecated in XXYYZZ, and will be removed in a future version. + SearcherOperation is deprecated in 0.38.0, and will be removed in a future version. A ``SearcherOperation`` is a request from the hyperparameter-search logic for the training script to execute one train-validate-report cycle. @@ -89,7 +89,7 @@ def __init__( def length(self) -> int: """ .. warning:: - SearcherOperation.length is deprecated in XXYYZZ, and will be removed in a future + SearcherOperation.length is deprecated in 0.38.0, and will be removed in a future version. Instead, you should directly specify your training length in your training code. @@ -101,7 +101,7 @@ def length(self) -> int: def report_progress(self, length: float) -> None: """ .. warning:: - SearcherOperation.report_progress is deprecated in XXYYZZ, and will be removed in a + SearcherOperation.report_progress is deprecated in 0.38.0, and will be removed in a future version. Instead, report progess with :meth:`~determined.core.TrainContext.report_progress`. @@ -128,7 +128,7 @@ def report_progress(self, length: float) -> None: def report_completed(self, searcher_metric: Any) -> None: """ .. warning:: - SearcherOperation.report_completed is deprecated in XXYYZZ, and will be removed in a + SearcherOperation.report_completed is deprecated in 0.38.0, and will be removed in a future version. Instead, just exit 0 when your training is complete. ``report_completed()`` is the final step of a train-validate-report cycle. @@ -148,7 +148,7 @@ def report_completed(self, searcher_metric: Any) -> None: class SearcherMode(enum.Enum): """ .. warning:: - SearcherMode is deprecated in XXYYZZ, and will be removed in a future version. + SearcherMode is deprecated in 0.38.0, and will be removed in a future version. ``SearcherMode`` defines the calling behavior of the ``SearcherContext.operations()`` call. @@ -168,7 +168,7 @@ class SearcherMode(enum.Enum): class SearcherContext: """ .. warning:: - SearcherContext is deprecated in XXYYZZ, and will be removed in a future version. Instead + SearcherContext is deprecated in 0.38.0, and will be removed in a future version. Instead of using ``SearcherContext.operations()`` to decide how long to train for, you should set your training length directly in your training code. @@ -241,7 +241,7 @@ def operations( ) -> Iterator[SearcherOperation]: """ .. warning:: - SearcherContext.operations is deprecated in XXYYZZ, and will be removed in a future + SearcherContext.operations is deprecated in 0.38.0, and will be removed in a future version. Instead, you should set your training length directly in your training code. This method no longer talks to the Determined master; it just yields a single @@ -250,7 +250,7 @@ def operations( """ warnings.warn( - "SearcherContext.operations() was deprecated in Determined XXYYZZ and will be removed " + "SearcherContext.operations() was deprecated in Determined 0.38.0 and will be removed " "in a future version. Instead, you should set your training length directly in your " "training code.", FutureWarning, @@ -300,7 +300,7 @@ def _operations( def acknowledge_out_of_ops(self) -> None: """ .. warning:: - SearcherContext.acknowledge_out_of_ops() is deprecated in XXYYZZ, and will be removed in + SearcherContext.acknowledge_out_of_ops() is deprecated in 0.38.0, and will be removed in a future version. Current calls to this function are ignored, and there should not need to be a replacement. """ @@ -309,7 +309,7 @@ def acknowledge_out_of_ops(self) -> None: def get_configured_units(self) -> Optional[Unit]: """ .. warning:: - SearcherContext.get_configured_units() is deprecated in XXYYZZ, and will be removed in + SearcherContext.get_configured_units() is deprecated in 0.38.0, and will be removed in a future version. Note that the ``searcher.max_length`` filed of the experiment config is also deprecated and will be removed as well. Instead, you should directly specify your training length in your training code. @@ -374,7 +374,7 @@ def operations( auto_ack: bool = True, ) -> Iterator[SearcherOperation]: warnings.warn( - "SearcherContext.operations() was deprecated in Determined XXYYZZ and will be removed " + "SearcherContext.operations() was deprecated in Determined 0.38.0 and will be removed " "in a future version. Instead, you should set your training length directly in your " "training code.", FutureWarning, diff --git a/harness/determined/exec/harness.py b/harness/determined/exec/harness.py index 188855e7399..65b9932c746 100644 --- a/harness/determined/exec/harness.py +++ b/harness/determined/exec/harness.py @@ -3,6 +3,7 @@ import faulthandler import logging import sys +import warnings from typing import Iterator, Optional, Type import determined as det @@ -42,9 +43,9 @@ def main(train_entrypoint: str) -> int: if hasattr(det.pytorch, "deepspeed") and issubclass( trial_class, det.pytorch.deepspeed.DeepSpeedTrial ): - return _run_deepspeed_trial(trial_class, info) + return _run_deepspeed_trial(trial_class, info, train_entrypoint) elif issubclass(trial_class, det.pytorch.PyTorchTrial): - return _run_pytorch_trial(trial_class, info) + return _run_pytorch_trial(trial_class, info, train_entrypoint) # TODO: Don't include EnvContext object in the future high-level APIs for PyTorch or Keras. # It was natural to create this big-blob-of-config object, but it was a mistake to pass it into @@ -138,11 +139,25 @@ def main(train_entrypoint: str) -> int: def _run_pytorch_trial( trial_class: "Type[det.pytorch.PyTorchTrial]", info: det.ClusterInfo, + train_entrypoint: str, ) -> int: from determined import pytorch det.common.set_logger(info.trial._debug) + # Only warn here if the user set a legacy entrypoint, not if we arrived here after user passed + # a --trial argument to a launcher. + if train_entrypoint == info.trial._config["entrypoint"]: + warnings.warn( + f"Support for legacy entrypoint format ({train_entrypoint}) has been deprecated in " + "Determined 0.38.0 and will be removed in a future version. You can keep your " + "PyTorchTrial, but please replace your model_def:TrialClass-style entrypoint " + "with a script-style entrypoint, and use the det.pytorch.Trainer() to train your " + "PyTorchTrial.", + FutureWarning, + stacklevel=2, + ) + logger.debug("Starting harness.") with maybe_periodic_stacktraces(info.trial._debug): @@ -202,12 +217,26 @@ def _run_pytorch_trial( def _run_deepspeed_trial( trial_class: "Type[det.pytorch.deepspeed.DeepSpeedTrial]", info: det.ClusterInfo, + train_entrypoint: str, ) -> int: from determined import pytorch from determined.pytorch import deepspeed as det_ds det.common.set_logger(info.trial._debug) + # Only warn here if the user set a legacy entrypoint, not if we arrived here after user passed + # a --trial argument to a launcher. + if train_entrypoint == info.trial._config["entrypoint"]: + warnings.warn( + f"Support for legacy entrypoint format ({train_entrypoint}) has been deprecated in " + "Determined 0.38.0 and will be removed in a future version. You can keep your " + "DeepSpeedTrial, but please replace your model_def:TrialClass-style entrypoint " + "with a script-style entrypoint, and use the new det.pytorch.deepspeed.Trainer() " + "to train your DeepSpeedTrial.", + FutureWarning, + stacklevel=2, + ) + logger.debug("Starting harness.") with det_ds.init( diff --git a/harness/determined/keras/_load.py b/harness/determined/keras/_load.py index 7c8359df1b0..8aa68d8b212 100644 --- a/harness/determined/keras/_load.py +++ b/harness/determined/keras/_load.py @@ -38,7 +38,7 @@ def load_model_from_checkpoint_path( .. warning:: - load_model_from_checkpoint_path has been deprecated in Determined XXYYZZ and will be removed + load_model_from_checkpoint_path has been deprecated in Determined 0.38.0 and will be removed in a future version. This function is designed to work with TFKerasTrial, which is also deprecated. Please use the new :class:`~determined.keras.DeterminedCallback` for training instead, which allows you to use ``model.load_weights()`` to restore from @@ -46,7 +46,7 @@ def load_model_from_checkpoint_path( """ warnings.warn( - "load_model_from_checkpoint_path has been deprecated in Determined XXYYZZ and will be " + "load_model_from_checkpoint_path has been deprecated in Determined 0.38.0 and will be " "removedin a future version. This function is designed to work with TFKerasTrial, which " "is alsodeprecated. Please use the new det.keras.DeterminedCallback fortraining instead, " "which allows you to use ``model.load_weights()`` to restore fromcheckpoints.", diff --git a/harness/determined/keras/_tf_keras_trial.py b/harness/determined/keras/_tf_keras_trial.py index b4f77f81233..5e832c02fa3 100644 --- a/harness/determined/keras/_tf_keras_trial.py +++ b/harness/determined/keras/_tf_keras_trial.py @@ -183,7 +183,7 @@ def pre_execute_hook( # TFKerasTrial's __init__ method may not be called by user-defined subclasses, so we fire # the warning here. Also, this will show up before some of the worst tf log vomit, I hope. warnings.warn( - "TFKerasTrial has been deprecated in Determined XXYYZZ and will be removed in a future " + "TFKerasTrial has been deprecated in Determined 0.38.0 and will be removed in a future " "version. Please use the new det.keras.DeterminedCallback for training.", FutureWarning, stacklevel=2, @@ -1016,7 +1016,7 @@ class TFKerasTrial(det.LegacyTrial): .. warning:: - TFKerasTrial has been deprecated in Determined XXYYZZ and will be removed in a future + TFKerasTrial has been deprecated in Determined 0.38.0 and will be removed in a future version. Please use the new :class:`~determined.keras.DeterminedCallback` for training. """ diff --git a/harness/determined/keras/callbacks.py b/harness/determined/keras/callbacks.py index 3e3afe85db2..f0f03823adc 100644 --- a/harness/determined/keras/callbacks.py +++ b/harness/determined/keras/callbacks.py @@ -45,7 +45,7 @@ class Callback(tf.keras.callbacks.Callback): # type: ignore .. warning:: - det.keras.callbacks.Callback has been deprecated in Determined XXYYZZ and will be removed + det.keras.callbacks.Callback has been deprecated in Determined 0.38.0 and will be removed in a future version. This Callback class is designed to work with TFKerasTrial, which is also deprecated. Please use the new :class:`~determined.keras.DeterminedCallback` for training, and use normal keras Callabacks with it. @@ -550,7 +550,7 @@ class EarlyStopping(tf.keras.callbacks.EarlyStopping, Callback): # type: ignore .. warning:: - EarlyStopping has been deprecated in Determined XXYYZZ and will be removed in a future + EarlyStopping has been deprecated in Determined 0.38.0 and will be removed in a future version. Determined's EarlyStopping is a customization of keras' EarlyStopping callback that is specific to TFKerasTrial, which is also deprecated. Please use the new :class:`~determined.keras.DeterminedCallback` for training, and use keras' EarlyStopping @@ -593,7 +593,7 @@ def __init__(self, *arg: Any, **kwarg: Any) -> None: self.test_end_count = 0 warnings.warn( - "EarlyStopping has been deprecated in Determined XXYYZZ and will be removed in a " + "EarlyStopping has been deprecated in Determined 0.38.0 and will be removed in a " "future version. Determined's EarlyStopping is a customization of keras' " "EarlyStopping callback that is specific to TFKerasTrial, which is also deprecated. " "Please use the new det.keras.DeterminedCallback for training, and use keras' " @@ -634,7 +634,7 @@ class ReduceLROnPlateau(tf.keras.callbacks.ReduceLROnPlateau, Callback): # type .. warning:: - ReduceLROnPlateau has been deprecated in Determined XXYYZZ and will be removed in a future + ReduceLROnPlateau has been deprecated in Determined 0.38.0 and will be removed in a future version. Determined's ReduceLROnPlateau is a customization of keras' ReduceLROnPlateau callback that is specific to TFKerasTrial, which is also deprecated. Please use the new :class:`~determined.keras.DeterminedCallback` for training, and use keras' @@ -678,7 +678,7 @@ def __init__(self, *arg: Any, **kwarg: Any) -> None: self.test_end_count = 0 warnings.warn( - "ReduceLROnPlateau has been deprecated in Determined XXYYZZ and will be removed in a " + "ReduceLROnPlateau has been deprecated in Determined 0.38.0 and will be removed in a " "future version. Determined's ReduceLROnPlateau is a customization of keras' " "ReduceLROnPlateau callback that is specific to TFKerasTrial, which is also " "deprecated. Please use the new det.keras.DeterminedCallback for training, and use " @@ -714,7 +714,7 @@ class TensorBoard(tf.keras.callbacks.TensorBoard, Callback): # type: ignore .. warning:: - det.keras.callbacks.TensorBoard has been deprecated in Determined XXYYZZ and will be removed + det.keras.callbacks.TensorBoard has been deprecated in Determined 0.38.0 and will be removed in a future version. This version of keras' TensorBoard callback is designed to work with TFKerasTrial, which is also deprecated. Please use the new :class:`~determined.keras.DeterminedCallback` for training, and use the new @@ -735,7 +735,7 @@ def __init__(self, *args: Any, **kwargs: Any): tf.keras.callbacks.TensorBoard.__init__(self, log_dir=log_dir, *args, **kwargs) warnings.warn( - "det.keras.callbacks.TensorBoard has been deprecated in Determined XXYYZZ and will be " + "det.keras.callbacks.TensorBoard has been deprecated in Determined 0.38.0 and will be " "removed in a future version. This version of keras' TensorBoard callback is designed " "to work with TFKerasTrial, which is also deprecated. Please use the new " "det.keras.DeterminedCallback for training, and use the new det.keras.TensorBoard with " diff --git a/harness/determined/launch/deepspeed.py b/harness/determined/launch/deepspeed.py index 6cff0c91d91..648c0244c00 100644 --- a/harness/determined/launch/deepspeed.py +++ b/harness/determined/launch/deepspeed.py @@ -12,6 +12,7 @@ import subprocess import sys import time +import warnings from typing import Dict, List, Mapping, Optional import deepspeed @@ -375,8 +376,8 @@ def parse_args(args: List[str]) -> List[str]: parser.add_argument( "--trial", help=( - "use a Trial class as the entrypoint to training. When --trial is used, the SCRIPT " - "positional argument must be omitted." + "(deprecated) use a Trial class as the entrypoint to training. When --trial is used, " + "the SCRIPT positional argument must be omitted." ), ) # For training scripts. @@ -396,6 +397,14 @@ def parse_args(args: List[str]) -> List[str]: parser.print_usage() print("error: extra arguments to --trial:", script, file=sys.stderr) sys.exit(1) + warnings.warn( + "Support for --trial argument to determined.launch.deepspeed has been deprecated " + "in Determined 0.38.0 and will be removed in a future version. You can keep your " + "DeepSpeedTrial, but please replace your --trial argument with a script that uses the " + "new det.pytorch.deepspeed.Trainer() to train your DeepSpeedTrial.", + FutureWarning, + stacklevel=2, + ) script = det.util.legacy_trial_entrypoint_to_script(parsed.trial) elif not script: # There needs to be at least one script argument. diff --git a/harness/determined/launch/horovod.py b/harness/determined/launch/horovod.py index 899a290c364..a5da95fb73a 100644 --- a/harness/determined/launch/horovod.py +++ b/harness/determined/launch/horovod.py @@ -11,6 +11,7 @@ import subprocess import sys import time +import warnings from typing import List, Tuple import determined as det @@ -223,8 +224,8 @@ def parse_args(args: List[str]) -> Tuple[List[str], List[str], bool]: parser.add_argument( "--trial", help=( - "use a Trial class as the entrypoint to training. When --trial is used, the SCRIPT " - "positional argument must be omitted." + "(deprecated) use a Trial class as the entrypoint to training. When --trial is used, " + "the SCRIPT positional argument must be omitted." ), ) # For training scripts. @@ -270,5 +271,14 @@ def parse_args(args: List[str]) -> Tuple[List[str], List[str], bool]: if __name__ == "__main__": + warnings.warn( + "determined.launch.horovod has been deprecated in Determined 0.38.0 and will be " + "removed in a future version. For PyTorchTrial users, please switch to " + "determined.launch.torch_distributed instead. For TFKerasTrial users, please migrate " + "to the new det.keras.DeterminedCallback for training and use the new " + "determined.launch.tensorflow launcher with it.", + FutureWarning, + stacklevel=2, + ) hvd_args, script, autohorovod = parse_args(sys.argv[1:]) sys.exit(main(hvd_args, script, autohorovod)) diff --git a/harness/determined/launch/torch_distributed.py b/harness/determined/launch/torch_distributed.py index 5795c401f15..b22946b6e9d 100644 --- a/harness/determined/launch/torch_distributed.py +++ b/harness/determined/launch/torch_distributed.py @@ -3,6 +3,7 @@ import os import subprocess import sys +import warnings from typing import List, Tuple import determined as det @@ -127,8 +128,8 @@ def parse_args(args: List[str]) -> Tuple[List[str], List[str]]: parser.add_argument( "--trial", help=( - "use a Trial class as the entrypoint to training. When --trial is used, the SCRIPT " - "positional argument must be omitted." + "(deprecated) use a Trial class as the entrypoint to training. When --trial is used, " + "the SCRIPT positional argument must be omitted." ), ) # For training scripts. @@ -148,6 +149,19 @@ def parse_args(args: List[str]) -> Tuple[List[str], List[str]]: parser.print_usage() print("error: extra arguments to --trial:", script, file=sys.stderr) sys.exit(1) + # We may have arrived here indirectly if the user specified a legacy entrypoint, in which + # case we have a warning for them in exec/harness.py. Only issue the warning if they + # explicitly configured the --trial argument. + info = det.get_cluster_info() + if info and "--trial" in info.trial._config["entrypoint"]: + warnings.warn( + "Support for --trial argument to determined.launch.torch_distributed has been " + "deprecated in Determined 0.38.0 and will be removed in a future version. You can " + "keep your PyTorchTrial, but please replace your --trial argument with a script " + "that uses the det.pytorch.Trainer() to train your PyTorchTrial.", + FutureWarning, + stacklevel=2, + ) script = det.util.legacy_trial_entrypoint_to_script(parsed.trial) elif not script: # There needs to be at least one script argument. diff --git a/harness/determined/pytorch/_trainer.py b/harness/determined/pytorch/_trainer.py index bba0f21bd00..bdb51208e70 100644 --- a/harness/determined/pytorch/_trainer.py +++ b/harness/determined/pytorch/_trainer.py @@ -194,7 +194,7 @@ def fit( if max_length_val: warnings.warn( "Configuring `max_length` from the `searcher.max_length` experiment " - "config, which was deprecated in XXYYZZ and will be removed in a future " + "config, which was deprecated in 0.38.0 and will be removed in a future " "release. Please set `fit(max_length=X)` with your desired training length " "directly.", FutureWarning, diff --git a/harness/determined/pytorch/deepspeed/_trainer.py b/harness/determined/pytorch/deepspeed/_trainer.py index de2514dcf0f..8e36f345235 100644 --- a/harness/determined/pytorch/deepspeed/_trainer.py +++ b/harness/determined/pytorch/deepspeed/_trainer.py @@ -158,7 +158,7 @@ def fit( if max_length_val: warnings.warn( "Configuring `max_length` from the `searcher.max_length` experiment " - "config, which was deprecated in XXYYZZ and will be removed in a future " + "config, which was deprecated in 0.38.0 and will be removed in a future " "release. Please set `fit(max_length=X)` with your desired training length " "directly.", FutureWarning,