diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index 9be6827c5c..8faf447c3c 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -1142,10 +1142,10 @@ executed in the background while you continue to use the App. There are a variety of options available for configuring whether a given operation should be delegated or executed immediately. -.. _operator-delegation-configuration: +.. _operator-execution-options: -Delegation configuration -~~~~~~~~~~~~~~~~~~~~~~~~ +Execution options +~~~~~~~~~~~~~~~~~ You can provide the optional properties described below in the :ref:`operator's config ` to specify the available execution @@ -1183,12 +1183,12 @@ user to choose between the supported options if there are multiple: .. image:: /images/plugins/operators/operator-execute-button.png :align: center -.. _operator-execution-options: +.. _dynamic-execution-options: -Execution options -~~~~~~~~~~~~~~~~~ +Dynamic execution options +~~~~~~~~~~~~~~~~~~~~~~~~~ -Operators can implement +Operators may also implement :meth:`resolve_execution_options() ` to dynamically configure the available execution options based on the current execution context: @@ -1238,14 +1238,9 @@ of the current view: # Force delegation for large views and immediate execution for small views return len(ctx.view) > 1000 -.. note:: - - If :meth:`resolve_delegation() ` - is not implemented or returns `None`, then the choice of execution mode is - deferred to - :meth:`resolve_execution_options() ` - to specify the available execution options as described in the previous - section. +If :meth:`resolve_delegation() ` +is not implemented or returns `None`, then the choice of execution mode is +deferred to the prior mechanisms described above. .. _operator-reporting-progress: diff --git a/docs/source/plugins/using_plugins.rst b/docs/source/plugins/using_plugins.rst index 0a03ef9ebc..fc4adba6c8 100644 --- a/docs/source/plugins/using_plugins.rst +++ b/docs/source/plugins/using_plugins.rst @@ -594,8 +594,8 @@ operator can be invoked like so: dataset = foz.load_zoo_dataset("quickstart") compute_metadata = foo.get_operator("@voxel51/utils/compute_metadata") - # Schedule a delegated operation to (re)compute metadata - compute_metadata(dataset, overwrite=True, delegate=True) + # (Re)compute the dataset's metadata + compute_metadata(dataset, overwrite=True) .. note:: @@ -614,13 +614,11 @@ follows: sample_collection, overwrite=False, num_workers=None, - delegate=False, ): ctx = dict(view=sample_collection.view()) params = dict( overwrite=overwrite, num_workers=num_workers, - delegate=delegate, ) return foo.execute_operator(self.uri, ctx, params=params) @@ -655,71 +653,63 @@ data, you can access it via the ``result`` property of the returned result = await op(...) print(result.result) # {...} -.. _delegating-function-calls: - -Delegating function calls -------------------------- +Requesting delegation +~~~~~~~~~~~~~~~~~~~~~ -The -`@voxel51/utils/delegate `_ -operator provides a general purpose utility for -:ref:`delegating execution ` of an arbitrary function -call that can be expressed in any of the following forms: +Operators that support :ref:`delegated execution ` can +support this via the `__call__()` syntax by passing the +`request_delegation=True` flag to +:func:`execute_operator() `. -- Execute an arbitrary function: `fcn(*args, **kwargs)` -- Apply a function to a dataset or view: - `fcn(dataset_or_view, *args, **kwargs)` -- Call an instance method of a dataset or view: - `dataset_or_view.fcn(*args, **kwargs)` - -Here's some examples of delegating common tasks that can be expressed in the -above forms: +In fact, the +`@voxel51/utils/compute_metadata `_ +operator does just that: .. code-block:: python :linenos: - import fiftyone as fo - import fiftyone.operators as foo - import fiftyone.zoo as foz + class ComputeMetadata(foo.Operator): + return foo.OperatorConfig( + ... + allow_immediate_execution=True, + allow_delegated_execution=True, + ) - dataset = foz.load_zoo_dataset("quickstart") - delegate = foo.get_operator("@voxel51/utils/delegate") + def __call__( + self, + sample_collection, + overwrite=False, + num_workers=None, + delegate=False, + ): + ctx = dict(view=sample_collection.view()) + params = dict( + overwrite=overwrite, + num_workers=num_workers, + ) + return foo.execute_operator( + self.uri, + ctx, + params=params, + request_delegation=delegate, + ) - # Compute metadata - delegate("compute_metadata", dataset=dataset) +which means that it can be invoked like so: - # Compute visualization - delegate( - "fiftyone.brain.compute_visualization", - dataset=dataset, - brain_key="img_viz", - ) +.. code-block:: python + :linenos: - # Export a view - delegate( - "export", - view=dataset.to_patches("ground_truth"), - export_dir="/tmp/patches", - dataset_type="fiftyone.types.ImageClassificationDirectoryTree", - label_field="ground_truth", - ) + compute_metadata = foo.get_operator("@voxel51/utils/compute_metadata") - # Load the exported patches into a new dataset - delegate( - "fiftyone.Dataset.from_dir", - dataset_dir="/tmp/patches", - dataset_type="fiftyone.types.ImageClassificationDirectoryTree", - label_field="ground_truth", - name="patches", - persistent=True, - ) + # Schedule a delegated operation to (re)compute metadata + compute_metadata(dataset, overwrite=True, delegate=True) .. _direct-operator-execution: Direct execution ---------------- -You can also programmatically execute any operator by directly calling +You can programmatically execute any operator by directly calling :func:`execute_operator() `: .. code-block:: python @@ -738,25 +728,11 @@ You can also programmatically execute any operator by directly calling dataset_type="COCO", labels_path=dict(absolute_path="/tmp/coco/labels.json"), label_field="ground_truth", - delegate=False, # False: execute immediately, True: delegate ) } foo.execute_operator("@voxel51/io/export_samples", ctx) -In the above example, the `delegate=True/False` parameter controls whether -execution happens immediately or is -:ref:`delegated ` because the operator implements -its -:meth:`resolve_delegation() ` -as follows: - -.. code-block:: python - :linenos: - - def resolve_delegation(self, ctx): - return ctx.params.get("delegate", False) - .. note:: In general, to use @@ -787,13 +763,11 @@ data, you can access it via the ``result`` property of the returned result = await foo.execute_operator("@an-operator/with-results", ctx) print(result.result) # {...} -.. _requesting-operator-delegation: - Requesting delegation ---------------------- +~~~~~~~~~~~~~~~~~~~~~ -If an operation supports both immediate and delegated execution as specified -either by its :ref:`configuration ` or +If an operation supports both immediate and +:ref:`delegated execution ` as specified by its :ref:`execution options `, you can request delegated execution by passing the `request_delegation=True` flag to :func:`execute_operator() `: @@ -823,6 +797,65 @@ operator's input modal when executing it from within the App: delegation_target="overnight", ) +.. _delegating-function-calls: + +Delegating function calls +------------------------- + +The +`@voxel51/utils/delegate `_ +operator provides a general purpose utility for +:ref:`delegating execution ` of an arbitrary function +call that can be expressed in any of the following forms: + +- Execute an arbitrary function: `fcn(*args, **kwargs)` +- Apply a function to a dataset or view: + `fcn(dataset_or_view, *args, **kwargs)` +- Call an instance method of a dataset or view: + `dataset_or_view.fcn(*args, **kwargs)` + +Here's some examples of delegating common tasks that can be expressed in the +above forms: + +.. code-block:: python + :linenos: + + import fiftyone as fo + import fiftyone.operators as foo + import fiftyone.zoo as foz + + dataset = foz.load_zoo_dataset("quickstart") + delegate = foo.get_operator("@voxel51/utils/delegate") + + # Compute metadata + delegate("compute_metadata", dataset=dataset) + + # Compute visualization + delegate( + "fiftyone.brain.compute_visualization", + dataset=dataset, + brain_key="img_viz", + ) + + # Export a view + delegate( + "export", + view=dataset.to_patches("ground_truth"), + export_dir="/tmp/patches", + dataset_type="fiftyone.types.ImageClassificationDirectoryTree", + label_field="ground_truth", + ) + + # Load the exported patches into a new dataset + delegate( + "fiftyone.Dataset.from_dir", + dataset_dir="/tmp/patches", + dataset_type="fiftyone.types.ImageClassificationDirectoryTree", + label_field="ground_truth", + name="patches", + persistent=True, + ) + .. _delegated-operations: Delegated operations @@ -893,7 +926,7 @@ delegated operations and execute them serially in its process. You must also ensure that the :ref:`allow_legacy_orchestrators ` config flag is set -in the environment where you run the App, e.g. by setting: +in the environment where you run the App/SDK, e.g. by setting: .. code-block:: shell diff --git a/fiftyone/operators/executor.py b/fiftyone/operators/executor.py index a2b50e7af0..1a7fe76a4c 100644 --- a/fiftyone/operators/executor.py +++ b/fiftyone/operators/executor.py @@ -261,12 +261,13 @@ async def execute_or_delegate_operator( ctx.request_params["delegated"] = True metadata = {"inputs_schema": None, "outputs_schema": None} - try: - metadata["inputs_schema"] = inputs.to_json() - except Exception as e: - logger.warning( - f"Failed to resolve inputs schema for the operation: {str(e)}" - ) + if inputs is not None: + try: + metadata["inputs_schema"] = inputs.to_json() + except Exception as e: + logger.warning( + f"Failed to resolve inputs schema for the operation: {str(e)}" + ) op = DelegatedOperationService().queue_operation( operator=operator.uri,