Skip to content

Commit

Permalink
Add "Logs" panel for evaluation v2.
Browse files Browse the repository at this point in the history
Evaluation classes could use `self.debug`/`self.info`/`self.warning`/`self.error`/`self.fatal` to write logs to this panel.

PiperOrigin-RevId: 707683046
  • Loading branch information
daiyip authored and langfun authors committed Dec 18, 2024
1 parent 96d3c24 commit b5601f0
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 6 deletions.
78 changes: 72 additions & 6 deletions langfun/core/eval/v2/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"""Base class for Langfun evaluation tasks."""

import abc
import datetime
import functools
import time

Expand Down Expand Up @@ -63,6 +64,7 @@ def _on_bound(self):
self.__dict__.pop('is_leaf', None)
self.__dict__.pop('children', None)
super()._on_bound()
self._log_entries = []

#
# Handling evaluation hierarchy (materialized vs. hyper evaluations).
Expand Down Expand Up @@ -277,6 +279,40 @@ def _reset(self) -> None:
for metric in self.metrics:
metric.reset()

#
# Evaluation-level logging.
#

def _log(self, level: lf.logging.LogLevel, message: str, **kwargs):
self._log_entries.append(
lf.logging.LogEntry(
level=level,
time=datetime.datetime.now(),
message=message,
metadata=kwargs,
)
)

def debug(self, message: str, **kwargs):
"""Logs a debug message to the session."""
self._log('debug', message, **kwargs)

def info(self, message: str, **kwargs):
"""Logs an info message to the session."""
self._log('info', message, **kwargs)

def warning(self, message: str, **kwargs):
"""Logs a warning message to the session."""
self._log('warning', message, **kwargs)

def error(self, message: str, **kwargs):
"""Logs an error message to the session."""
self._log('error', message, **kwargs)

def fatal(self, message: str, **kwargs):
"""Logs a fatal message to the session."""
self._log('fatal', message, **kwargs)

#
# HTML views.
#
Expand Down Expand Up @@ -465,6 +501,27 @@ def _metric_value_tab(
)
)

def _logs_tab() -> pg.views.html.controls.Tab:
"""Renders a tab for the logs of the evaluation."""
return pg.views.html.controls.Tab(
label='Logs',
content=pg.Html.element(
'div',
[
pg.Html.element(
'textarea',
[
pg.Html.escape(
'\n'.join(str(l) for l in self._log_entries)
)
],
readonly=True,
css_classes=['logs-textarea'],
)
]
)
)

def _main_tabs() -> pg.Html:
return pg.Html.element(
'div',
Expand All @@ -474,6 +531,8 @@ def _main_tabs() -> pg.Html:
_definition_tab(),
] + [
_metric_tab(m) for m in self.metrics
] + [
_logs_tab()
],
selected=1,
)
Expand Down Expand Up @@ -593,6 +652,14 @@ def _html_tree_view_css_styles(self) -> list[str]:
width:100%;
height:100%;
}
.logs-textarea {
width: 100%;
height: 500px;
padding: 5px;
border: 1px solid #DDD;
background-color: #EEE;
resize: vertical;
}
"""
]

Expand All @@ -615,16 +682,15 @@ def load(
assert isinstance(example, example_lib.Example), example
self._evaluated_examples[example.id] = example

@property
def evaluated_examples(self) -> dict[int, example_lib.Example]:
"""Returns the examples in the state."""
return self._evaluated_examples

def get(self, example_id: int) -> example_lib.Example | None:
"""Returns the example with the given ID."""
return self._evaluated_examples.get(example_id)

def update(self, example: example_lib.Example) -> None:
"""Updates the state with the given example."""
self._evaluated_examples[example.id] = example

@property
def evaluated_examples(self) -> dict[int, example_lib.Example]:
"""Returns the examples in the state."""
return self._evaluated_examples

21 changes: 21 additions & 0 deletions langfun/core/logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,27 @@ class LogEntry(pg.Object, pg.views.HtmlTreeView.Extension):
def should_output(self, min_log_level: LogLevel) -> bool:
return _LOG_LEVELS.index(self.level) >= _LOG_LEVELS.index(min_log_level)

def format(self,
compact: bool = False,
verbose: bool = True,
root_indent: int = 0,
*,
text_format: bool = True,
**kwargs):
if text_format:
escaped_message = self.message.replace('\n', '\\n')
s = f"""{self.time.strftime('%H:%M:%S')} {self.level.upper()} - {escaped_message}"""
if self.metadata:
metadata_str = repr(self.metadata).replace('\n', '\\n')
s += f' (metadata: {metadata_str})'
return s
return super().format(
compact=compact,
verbose=verbose,
root_indent=root_indent,
**kwargs
)

def _html_tree_view_summary(
self,
view: pg.views.HtmlTreeView,
Expand Down
19 changes: 19 additions & 0 deletions langfun/core/logging_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,25 @@ def assert_html_content(self, html, expected):
print(actual)
self.assertEqual(actual, expected)

def test_format(self):
time = datetime.datetime(2024, 10, 10, 12, 30, 45)
self.assertEqual(
str(
logging.LogEntry(
level='info', message='hello\nworld',
time=time, metadata=dict(x=1),
)
),
'12:30:45 INFO - hello\\nworld (metadata: {x=1})',
)
self.assertIn(
'LogEntry(',
logging.LogEntry(
level='info', message='hello\nworld',
time=time, metadata=dict(x=1),
).format(text_format=False),
)

def test_html(self):
time = datetime.datetime(2024, 10, 10, 12, 30, 45)
self.assert_html_content(
Expand Down

0 comments on commit b5601f0

Please sign in to comment.