Skip to content

Commit

Permalink
Add error check for filtering out all of the metrics in a workflow
Browse files Browse the repository at this point in the history
This isn't a valid use case so just raise for it.
  • Loading branch information
naddeoa committed Mar 26, 2024
1 parent 59f9abe commit 74177d1
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 0 deletions.
5 changes: 5 additions & 0 deletions langkit/core/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,11 @@ def run(self, data: Union[pd.DataFrame, Row, Dict[str, str]], options: Optional[
if options and options.metric_filter and options.metric_filter.by_required_inputs:
by_required_inputs_set = frozenset([frozenset(x) for x in options.metric_filter.by_required_inputs])
metrics_to_run = [metric for metric in self.metrics_config.metrics if frozenset(metric.input_names) in by_required_inputs_set]
if not metrics_to_run:
raise ValueError(
f"No metrics to run. Filters {options.metric_filter.by_required_inputs} did "
f"not match any metrics {self.get_metric_names()}"
)
else:
metrics_to_run = self.metrics_config.metrics

Expand Down
10 changes: 10 additions & 0 deletions tests/langkit/metrics/test_workflow.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from typing import List

import pytest

from langkit.core.workflow import MetricFilterOptions, RunOptions, Workflow
from langkit.metrics.library import lib
from langkit.validators.library import lib as validator_lib
Expand Down Expand Up @@ -129,6 +131,14 @@ def test_metric_filter_both_prompt_and_response():
]


def test_metric_filter_no_metrics_left():
wf = Workflow(metrics=[lib.presets.recommended(), lib.response.similarity.prompt()])
options = RunOptions(metric_filter=MetricFilterOptions(by_required_inputs=[["doesnt exist"]]))

with pytest.raises(ValueError):
wf.run({"prompt": "hi", "response": "hello"}, options)


def test_metric_filter_include_everything():
wf = Workflow(metrics=[lib.presets.recommended(), lib.response.similarity.prompt()])
options = RunOptions(metric_filter=MetricFilterOptions(by_required_inputs=[["prompt", "response"], ["prompt"], ["response"]]))
Expand Down

0 comments on commit 74177d1

Please sign in to comment.