Skip to content

Commit

Permalink
Enable lf.eval.inputs_from to read csv files.
Browse files Browse the repository at this point in the history
Usage:

```
dataset = lf.eval.inputs_from('<csv_path>')()
```

or providing `pd.read_csv` arguments:

```
dataset = lf.eval.inputs_from('<csv_path>', index_col=0, header=0)()
```
PiperOrigin-RevId: 688580326
  • Loading branch information
chengrunyang authored and langfun authors committed Oct 22, 2024
1 parent c2c60d3 commit 232fd57
Showing 1 changed file with 15 additions and 2 deletions.
17 changes: 15 additions & 2 deletions langfun/core/eval/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import langfun.core.coding as lf_coding
from langfun.core.llms.cache import in_memory
import langfun.core.structured as lf_structured
import pandas as pd
import pyglove as pg


Expand Down Expand Up @@ -1684,10 +1685,22 @@ def visualize(cls, evaluations: list['Evaluation']) -> str | None:


@pg.functor()
def inputs_from(path: str | list[str]) -> list[Any]:
def inputs_from(path: str | list[str], **kwargs) -> list[Any]:
"""A functor that returns a list of user-defined objects as eval inputs."""
if isinstance(path, str):
return pg.load(path)
if path.endswith('.json'):
return pg.load(path)
elif path.endswith('.csv'):
dataset_df = pd.read_csv(path, **kwargs)
dataset = []
for i in range(dataset_df.shape[0]):
row = {}
for col in dataset_df.columns:
row[col] = dataset_df.iloc[i][col]
dataset.append(row)
return dataset
else:
raise ValueError(f'Unsupported file format: {path}')
examples = []
for p in path:
examples.extend(pg.load(p))
Expand Down

0 comments on commit 232fd57

Please sign in to comment.