Skip to content

Commit

Permalink
extra performance metric PU vs PUlabels
Browse files Browse the repository at this point in the history
  • Loading branch information
Orieus committed Sep 13, 2023
1 parent 1e2c1e2 commit 0d42908
Showing 1 changed file with 16 additions and 2 deletions.
18 changes: 16 additions & 2 deletions src/task_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,7 @@ def get_labels_from_scores(self, n_max: int = 50_000, s_min: float = 1.0,
# Save parameters in metadata file
self.metadata[tag] = {
'doc_selection': {
'method': 'Imported from data files',
'n_max': n_max,
's_min': s_min}}

Expand Down Expand Up @@ -1065,6 +1066,12 @@ def performance_metrics_PU(self):
"""
Compute all performance metrics for the PU model, based on the data
available at the current dataset
This methods compares three types of labels/predictions:
PUlabels: Labels produced by the document selection process
PU: Predictions from the model trained with the PUlabels
Annotations: Ground-truth labels, typically annotated by the user.
"""

# Check if a classifier object exists
Expand All @@ -1084,6 +1091,10 @@ def performance_metrics_PU(self):
self._performance_metrics("PU", ANNOTATIONS, "unused")
self._performance_metrics("PU", ANNOTATIONS, "all")

# Test PU predictions against annotations
self._performance_metrics("PUlabels", ANNOTATIONS, "test")
self._performance_metrics("PUlabels", ANNOTATIONS, "unused")
self._performance_metrics("PUlabels", ANNOTATIONS, "all")
return

def performance_metrics_PN(self):
Expand Down Expand Up @@ -1607,7 +1618,7 @@ def get_labels_by_keywords(self):
elif method == 'c':
method = 'count'

# Get keywords and labels
# Get keywords and a label name
self.keywords = self._ask_keywords()
tag = self._ask_label_tag()

Expand Down Expand Up @@ -1713,9 +1724,12 @@ def get_labels_from_scores(self):
convert_to=float,
default=self.global_parameters['score_based_selection']['s_min'])

# As a name for the new labels
tag = self._ask_label_tag()

# ##########
# Get labels
super().get_labels_from_scores(n_max=n_max, s_min=s_min)
super().get_labels_from_scores(n_max=n_max, s_min=s_min, tag=tag)

return

Expand Down

0 comments on commit 0d42908

Please sign in to comment.