Added SetRelP (as alias of new SetP(relative=True))

#1

ir_measures/measures/__init__.py

@@ -25,7 +25,7 @@ def register(measure, aliases=[], name=None):
 from .rbp import RBP, _RBP
 from .rprec import Rprec, RPrec, _Rprec
 from .rr import RR, MRR, _RR
-from .set_measures import SetP, _SetP, SetR, _SetR, SetF, _SetF
+from .set_measures import SetP, SetRelP, _SetP, SetR, _SetR, SetF, _SetF
 from .success import Success, _Success
 
 # enable from "ir_measures.measures import *" --- on purpuse, do not include _-prefixed versions,

ir_measures/measures/set_measures.py

@@ -9,11 +9,14 @@ class _SetP(measures.Measure):
     __name__ = 'SetP'
     NAME = __name__
     SUPPORTED_PARAMS = {
-        'rel': measures.ParamInfo(dtype=int, default=1, desc='minimum relevance score to be considered relevant (inclusive)')
+        'rel': measures.ParamInfo(dtype=int, default=1, desc='minimum relevance score to be considered relevant (inclusive)'),
+        'relative': measures.ParamInfo(dtype=bool, default=False, desc='calculate the measure using the maximum possible SetP for the provided result size'),
     }
 
 SetP = _SetP()
+SetRelP = _SetP(relative=True)
 measures.register(SetP)
+measures.register(SetRelP, name='SetRelP')
 
 
 class _SetR(measures.Measure):

ir_measures/providers/pytrec_eval_provider.py

@@ -33,7 +33,7 @@ class PytrecEvalProvider(providers.Provider):
         measures._NumRet(rel=Any()),
         measures._NumQ(),
         measures._NumRel(rel=Choices(1)), # for some reason, relevance_level doesn't flow through to num_rel, so can only support rel=1
-        measures._SetP(rel=Any()),
+        measures._SetP(rel=Any(), relative=Any()),
         measures._SetR(rel=Any()),
         measures._Success(rel=Any(), cutoff=Any()),
         measures._IPrec(recall=Any()),
@@ -129,8 +129,12 @@ def _build_invokers(self, measures, qrels):
                 else:
                     measure_str = f'set_F_{measure["beta"]}'
             elif measure.NAME == 'SetP':
-                invocation_key = (measure['rel'], 0)
-                measure_str = f'set_P'
+                if measure['relative']:
+                    invocation_key = (measure['rel'], 0)
+                    measure_str = f'set_relative_P'
+                else:
+                    invocation_key = (measure['rel'], 0)
+                    measure_str = f'set_P'
             elif measure.NAME == 'SetR':
                 invocation_key = (measure['rel'], 0)
                 measure_str = f'set_recall'

ir_measures/util.py

@@ -312,7 +312,7 @@ def parse_trec_measure(measure: str) -> List['Measure']:
         'recall': (ir_measures.R, 'cutoff', [5, 10, 15, 20, 30, 100, 200, 500, 1000]),
         'set_recall': (ir_measures.SetR, None, None),
         'utility': (None, None, None),
-        'set_relative_P': (None, None, None),
+        'set_relative_P': (ir_measures.SetRelP, None, None),
         'num_ret': (ir_measures.NumRet, None, None),
         'num_rel': (ir_measures.NumRel, None, None),
         'ndcg_cut': (ir_measures.nDCG, 'cutoff', [5, 10, 15, 20, 30, 100, 200, 500, 1000]),

test/test_pytrec_eval.py

@@ -201,6 +201,14 @@ def test_SetP(self):
         self.assertEqual(result[1].value, .25)
         self.assertEqual(provider.calc_aggregate([measure], qrels, run)[measure], 0.425)
 
+        measure = ir_measures.SetRelP(rel=1)
+        result = list(provider.iter_calc([measure], qrels, run))
+        self.assertEqual(result[0].query_id, "0")
+        self.assertEqual(result[0].value, 1.0)
+        self.assertEqual(result[1].query_id, "1")
+        self.assertEqual(result[1].value, .5)
+        self.assertEqual(provider.calc_aggregate([measure], qrels, run)[measure], 0.75)
+
         measure = ir_measures.SetP(rel=2)
         result = list(provider.iter_calc([measure], qrels, run))
         self.assertEqual(result[0].query_id, "0")
@@ -209,6 +217,14 @@ def test_SetP(self):
         self.assertEqual(result[1].value, .25)
         self.assertEqual(provider.calc_aggregate([measure], qrels, run)[measure], 0.325)
 
+        measure = ir_measures.SetRelP(rel=2)
+        result = list(provider.iter_calc([measure], qrels, run))
+        self.assertEqual(result[0].query_id, "0")
+        self.assertEqual(result[0].value, 1.0)
+        self.assertEqual(result[1].query_id, "1")
+        self.assertEqual(result[1].value, 1.0)
+        self.assertEqual(provider.calc_aggregate([measure], qrels, run)[measure], 1.0)
+
         measure = ir_measures.SetP(rel=3)
         result = list(provider.iter_calc([measure], qrels, run))
         self.assertEqual(result[0].query_id, "0")

test/test_util.py

@@ -22,6 +22,7 @@ def test_parse_trec_measure(self):
             'set_P': [SetP],
             'set_recall': [SetR],
             'set_F': [SetF],
+            'set_relative_P': [SetRelP],
             'set_F.1.0,0.5,2.4': [SetF, SetF(beta=0.5), SetF(beta=2.4)],
             'official': [P@5, P@10, P@15, P@20, P@30, P@100, P@200, P@500, P@1000, Rprec, Bpref, IPrec@0.0, IPrec@0.1, IPrec@0.2, IPrec@0.3, IPrec@0.4, IPrec@0.5, IPrec@0.6, IPrec@0.7, IPrec@0.8, IPrec@0.9, IPrec@1.0, AP, NumQ, NumRel, NumRelRet, NumRet, RR],
         }