Skip to content

Commit

Permalink
Ensure checkpoint files are written immediately after evaluation comp…
Browse files Browse the repository at this point in the history
…letes.

Previously, checkpoint files were written when the writer object was destroyed, relying on the `del` statement to trigger garbage collection. However, `del` only decreases the reference count of the object and does not guarantee its immediate destruction. This change explicitly closes the writer to ensure checkpoint files are saved promptly after evaluation.

PiperOrigin-RevId: 707287518
  • Loading branch information
daiyip authored and langfun authors committed Dec 18, 2024
1 parent 2ab5e48 commit 16809a0
Showing 1 changed file with 12 additions and 3 deletions.
15 changes: 12 additions & 3 deletions langfun/core/eval/v2/checkpointing.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def save_state(example: Example):
)
)
writer.add(example)
del writer
writer.close()
runner.background_run(save_state, example)

def _file_prefix_and_ext(self, filename: str) -> tuple[str, str]:
Expand Down Expand Up @@ -128,6 +128,8 @@ def on_run_abort(
) -> None:
with self._lock:
if self._sequence_writer is not None:
for writer in self._sequence_writer.values():
writer.close()
self._sequence_writer.clear()

def on_run_complete(
Expand Down Expand Up @@ -174,6 +176,9 @@ def on_experiment_complete(
assert experiment.id in self._sequence_writer
with self._lock:
if self._sequence_writer is not None:
# Make sure the writer is closed without delay so the file will be
# available immediately.
self._sequence_writer[experiment.id].close()
del self._sequence_writer[experiment.id]

def on_example_complete(
Expand Down Expand Up @@ -207,9 +212,13 @@ def add(self, example: Example):
return
self._sequence_writer.add(example_blob)

def __del__(self):
def close(self):
# Make sure there is no write in progress.
with self._lock:
assert self._sequence_writer is not None
if self._sequence_writer is None:
return
self._sequence_writer.close()
self._sequence_writer = None

def __del__(self):
self.close()

0 comments on commit 16809a0

Please sign in to comment.