Merge pull request #7 from annahedstroem/new-feature-batch-size

include batch size as arg meta_evaluation.py

metaquantus/benchmarking.py

@@ -29,6 +29,7 @@ def __init__(
         keep_results: bool = False,
         channel_first: Optional[bool] = True,
         softmax: Optional[bool] = False,
+        batch_size: Optional[int] = 64,
         device: Optional[str] = None,
     ):
         """
@@ -56,6 +57,8 @@ def __init__(
             Indicates if channels is first.
         softmax: bool
             Indicates if the softmax (or logits) are used.
+        batch_size: int 
+            The batch size to run Quantus evaluation with.
         device: torch.device
             The device used, to enable GPUs.
 
@@ -73,6 +76,7 @@ def __init__(
         self.write_to_file = write_to_file
         self.channel_first = channel_first
         self.softmax = softmax
+        self.batch_size = batch_size
         self.device = device
         self.name = self.master.fname
 
@@ -127,6 +131,7 @@ def __call__(self, *args, **kwargs) -> dict:
                             s_batch=settings_data["s_batch"],
                             channel_first=self.channel_first,
                             softmax=self.softmax,
+                            batch_size=self.batch_size,
                             device=self.device,
                             score_direction=estimator["score_direction"],
                         )

metaquantus/meta_evaluation.py

@@ -130,6 +130,7 @@ def __call__(
         channel_first: Optional[bool] = True,
         softmax: Optional[bool] = False,
         device: Optional[str] = None,
+        batch_size: Optional[int] = 64,
         model_predict_kwargs: Optional[Dict[str, Any]] = {},
         score_direction: Optional[str] = None,
     ):
@@ -154,6 +155,8 @@ def __call__(
             Indicates if channels is first.
         softmax: bool
             Indicates if the softmax (or logits) are used.
+        batch_size: int 
+            The batch size to run Quantus evaluation with.
         device: torch.device
             The device used, to enable GPUs.
         model_predict_kwargs: dict
@@ -179,6 +182,7 @@ def __call__(
             channel_first=channel_first,
             softmax=softmax,
             device=device,
+            batch_size=batch_size,
         )
 
         # Run inference.
@@ -255,6 +259,7 @@ def run_perturbation_analysis(
         s_batch: Union[np.array, None] = None,
         channel_first: Optional[bool] = True,
         softmax: Optional[bool] = False,
+        batch_size: Optional[int] = 64,
         device: Optional[str] = None,
         model_predict_kwargs: Optional[Dict[str, Any]] = {},
     ):
@@ -352,6 +357,7 @@ def run_perturbation_analysis(
                         },
                         model_predict_kwargs=model_predict_kwargs,
                         softmax=softmax,
+                        batch_size=batch_size,
                         device=device,
                     )
 
@@ -397,6 +403,7 @@ def run_perturbation_analysis(
                         model_predict_kwargs=model_predict_kwargs,
                         softmax=softmax,
                         device=device,
+                        batch_size=batch_size,
                     )
 
                 self.results_eval_scores_perturbed[test_name][i] = scores_perturbed

metaquantus/perturbation_tests/base.py

@@ -34,6 +34,7 @@ def __call__(
         explain_func_kwargs: Optional[Dict[str, Any]],
         model_predict_kwargs: Optional[Dict],
         softmax: Optional[bool],
+        batch_size: Optional[int],
         device: Optional[str],
     ) -> Union[int, float, list, dict, None]:
         raise NotImplementedError

metaquantus/perturbation_tests/ipt.py

@@ -52,6 +52,7 @@ def __call__(
         explain_func: Optional[Callable],
         model_predict_kwargs: Optional[Dict],
         softmax: Optional[bool],
+        batch_size: Optional[int],
         device: Optional[str],
     ) -> Tuple[dict, np.ndarray, dict]:
         """
@@ -83,6 +84,8 @@ def __call__(
             Extra kwargs when running model.predict.
         softmax: bool
             Indicates if the softmax (or logits) are used.
+        batch_size: int 
+            The batch size to run Quantus evaluation with.
         device: torch.device
             The device used, to enable GPUs.
 
@@ -175,6 +178,7 @@ def __call__(
                     },
                     model_predict_kwargs=model_predict_kwargs,
                     channel_first=channel_first,
+                    batch_size=batch_size,
                     softmax=softmax,
                     device=device,
                 )

metaquantus/perturbation_tests/mpt.py

@@ -59,6 +59,7 @@ def __call__(
         explain_func: Optional[Callable],
         model_predict_kwargs: Optional[Dict],
         softmax: Optional[bool],
+        batch_size: Optional[int],
         device: Optional[str],
     ) -> Tuple[dict, np.ndarray, dict]:
         """
@@ -90,6 +91,8 @@ def __call__(
             Extra kwargs when running model.predict.
         softmax: bool
             Indicates if the softmax (or logits) are used.
+        batch_size: int 
+            The batch size to run Quantus evaluation with.
         device: torch.device
             The device used, to enable GPUs.
 
@@ -184,6 +187,7 @@ def __call__(
                     model_predict_kwargs=model_predict_kwargs,
                     channel_first=channel_first,
                     softmax=softmax,
+                    batch_size=batch_size,
                     device=device,
                 )
 

requirements_test.txt

@@ -2,6 +2,6 @@
 coverage>=7.2.3
 black==22.10.0
 numpy>=1.20.3
-pytest>=6.2.5
+pytest<=7.4.4 # pytest-lazyfixture issue https://github.com/annahedstroem/MetaQuantus/actions/runs/8449892962/job/23144947207
 pytest-cov>=3.0.0
 pytest-lazy-fixture>=0.6.3

tests/test_benchmarking.py

@@ -86,6 +86,7 @@ def test_benchmarking_mnist(
         keep_results=True,
         channel_first=True,
         softmax=False,
+        batch_size=64,
         device=device,
         path=os.getcwd()+"tests/assets/results/",
         save=False,

tests/test_meta_evaluation.py

@@ -100,6 +100,7 @@ def test_meta_evaluation_mnist(
         s_batch=s_batch,
         channel_first=True,
         softmax=False,
+        batch_size=50,
         device=device,
         score_direction=score_direction,
     )