General Cleanup and Following the Better Tracing Merger

tests/_regtest_outputs/test_processing.test_process_scan_below.out

@@ -1,5 +1,6 @@
                   image_size_x_m  image_size_y_m  image_area_m2  image_size_x_px  image_size_y_px  image_area_px2  grains_number_above  grains_per_m2_above  grains_number_below  grains_per_m2_below  rms_roughness
 image                                                                                                                                                                                                               
 minicircle_small      1.2646e-07      1.2646e-07     1.5993e-14               64               64            4096                    0           0.0000e+00                    1           6.2526e+13     6.8208e-10
-    centre_x   centre_y  grain_number  radius_min  radius_max  radius_mean  radius_median  height_min  height_max  height_median  height_mean      volume       area  area_cartesian_bbox  smallest_bounding_width  smallest_bounding_length  smallest_bounding_area  aspect_ratio threshold  max_feret  min_feret             image  grain_endpoints  grain_junctions  total_branch_lengths  num_crossings avg_crossing_confidence min_crossing_confidence  num_mols  total_contour_length  average_end_to_end_distance
-0 3.2366e-08 1.4036e-08             0  7.7690e-10  1.2272e-08   6.4301e-09     6.4170e-09 -3.7937e-10 -2.1207e-10    -2.4477e-10  -2.6816e-10 -3.0364e-26 1.1323e-16           3.0066e-16               7.0841e-09                2.1505e-08              1.5234e-16    3.2941e-01     below 2.2092e-08 7.0841e-09  minicircle_small                2                0            1.3493e-08              0                    None                    None         1            1.0799e-08                   1.0076e-08
+               centre_x   centre_y  radius_min  radius_max  radius_mean  radius_median  height_min  height_max  height_median  height_mean      volume       area  area_cartesian_bbox  smallest_bounding_width  smallest_bounding_length  smallest_bounding_area  aspect_ratio threshold  max_feret  min_feret             image
+grain_number                                                                                                                                                                                                                                                                                                                     
+0            3.2366e-08 1.4036e-08  7.7690e-10  1.2272e-08   6.4301e-09     6.4170e-09 -3.7937e-10 -2.1207e-10    -2.4477e-10  -2.6816e-10 -3.0364e-26 1.1323e-16           3.0066e-16               7.0841e-09                2.1505e-08              1.5234e-16    3.2941e-01     below 2.2092e-08 7.0841e-09  minicircle_small

tests/_regtest_outputs/test_processing.test_process_scan_both.out

@@ -1,8 +1,7 @@
                   image_size_x_m  image_size_y_m  image_area_m2  image_size_x_px  image_size_y_px  image_area_px2  grains_number_above  grains_per_m2_above  grains_number_below  grains_per_m2_below  rms_roughness
 image                                                                                                                                                                                                               
-minicircle_small      1.2646e-07      1.2646e-07     1.5993e-14               64               64            4096                    3           1.8758e+14                    1           6.2526e+13     6.8208e-10
-    centre_x   centre_y  grain_number  radius_min  radius_max  radius_mean  radius_median  height_min  height_max  height_median  height_mean      volume       area  area_cartesian_bbox  smallest_bounding_width  smallest_bounding_length  smallest_bounding_area  aspect_ratio threshold  max_feret  min_feret             image  grain_endpoints  grain_junctions  total_branch_lengths  num_crossings avg_crossing_confidence min_crossing_confidence  num_mols writhe_string  total_contour_length  average_end_to_end_distance
-0 3.2366e-08 1.4036e-08             0  7.7690e-10  1.2272e-08   6.4301e-09     6.4170e-09 -3.7937e-10 -2.1207e-10    -2.4477e-10  -2.6816e-10 -3.0364e-26 1.1323e-16           3.0066e-16               7.0841e-09                2.1505e-08              1.5234e-16    3.2941e-01     below 2.2092e-08 7.0841e-09  minicircle_small                2                0            1.3493e-08              0                    None                    None         1           NaN            1.0799e-08                   1.0076e-08
-1 7.5100e-08 4.7559e-08             0  3.9431e-09  2.5631e-08   1.6016e-08     1.6680e-08  9.1991e-10  2.6422e-09     1.5338e-09   1.5341e-09  1.0543e-24 6.8721e-16           1.3198e-15               2.0539e-08                5.0379e-08              1.0347e-15    4.0769e-01     above 5.0379e-08 2.0539e-08  minicircle_small                1                1            8.4571e-08              1                     NaN                     NaN         2                          6.5881e-08                   8.8370e-09
-2 8.0241e-08 7.8677e-08             1  6.8951e-09  2.7188e-08   1.6272e-08     1.6263e-08  9.0630e-10  2.4586e-09     1.6144e-09   1.6264e-09  1.0352e-24 6.3645e-16           1.5931e-15               2.0174e-08                5.1212e-08              1.0332e-15    3.9394e-01     above 5.1262e-08 2.0174e-08  minicircle_small                0                0            7.3054e-08              0                     NaN                     NaN         1           NaN            5.8272e-08                   0.0000e+00
-3 4.0012e-08 7.5644e-08             2  9.9461e-09  2.3654e-08   1.7561e-08     1.8364e-08  9.0641e-10  2.1066e-09     1.5939e-09   1.5493e-09  1.1192e-24 7.2236e-16           1.5462e-15               3.3592e-08                4.1496e-08              1.3940e-15    8.0952e-01     above 4.4405e-08 3.2528e-08  minicircle_small                0                0            1.0447e-07              0                     NaN                     NaN         1           NaN            8.7183e-08                   0.0000e+00
+minicircle_small      1.2646e-07      1.2646e-07     1.5993e-14               64               64            4096                    3           1.8758e+14                    0           0.0000e+00     6.8208e-10
+    centre_x   centre_y  grain_number  radius_min  radius_max  radius_mean  radius_median  height_min  height_max  height_median  height_mean     volume       area  area_cartesian_bbox  smallest_bounding_width  smallest_bounding_length  smallest_bounding_area  aspect_ratio threshold  max_feret  min_feret             image  grain_endpoints  grain_junctions  total_branch_lengths  total_contour_length  average_end_to_end_distance
+0 7.5100e-08 4.7559e-08             0  3.9431e-09  2.5631e-08   1.6016e-08     1.6680e-08  9.1991e-10  2.6422e-09     1.5338e-09   1.5341e-09 1.0543e-24 6.8721e-16           1.3198e-15               2.0539e-08                5.0379e-08              1.0347e-15    4.0769e-01     above 5.0379e-08 2.0539e-08  minicircle_small       1.0000e+00       1.0000e+00            8.4571e-08            6.5881e-08                   8.8370e-09
+1 8.0241e-08 7.8677e-08             1  6.8951e-09  2.7188e-08   1.6272e-08     1.6263e-08  9.0630e-10  2.4586e-09     1.6144e-09   1.6264e-09 1.0352e-24 6.3645e-16           1.5931e-15               2.0174e-08                5.1212e-08              1.0332e-15    3.9394e-01     above 5.1262e-08 2.0174e-08  minicircle_small       0.0000e+00       0.0000e+00            7.3054e-08            5.8272e-08                   0.0000e+00
+2 4.0012e-08 7.5644e-08             2  9.9461e-09  2.3654e-08   1.7561e-08     1.8364e-08  9.0641e-10  2.1066e-09     1.5939e-09   1.5493e-09 1.1192e-24 7.2236e-16           1.5462e-15               3.3592e-08                4.1496e-08              1.3940e-15    8.0952e-01     above 4.4405e-08 3.2528e-08  minicircle_small       0.0000e+00       0.0000e+00            1.0447e-07            8.7183e-08                   0.0000e+00

tests/test_io.py

@@ -710,6 +710,7 @@ def test_load_scan_get_data_check_image_size_and_add_to_dict(
     load_scan_spm: LoadScans, x: int, y: int, log_msg: str, caplog, tmp_path
 ) -> None:
     """Test errors are raised when images that are too small are passed."""
+    caplog.set_level(logging.DEBUG, LOGGER_NAME)
     load_scan_spm.filename = "minicircle"
     load_scan_spm.img_path = tmp_path
     load_scan_spm.image = np.ndarray((x, y))

tests/test_plotting.py

@@ -105,8 +105,8 @@ def test_var_to_label_config(tmp_path: Path) -> None:
 @pytest.mark.parametrize(
     ("var", "expected_label"),
     [
-        ("contour_length", "Contour Length"),
-        ("end_to_end_distance", "End to End Distance"),
+        ("total_contour_length", "Total Contour Length"),
+        ("average_end_to_end_distance", "Average End to End Distance"),
         ("grain_bound_len", "Circumference"),
         ("grain_curvature1", "Smaller Curvature"),
     ],

tests/test_processing.py

@@ -1,5 +1,6 @@
 """Test end-to-end running of topostats."""
 
+import logging
 import pickle
 from pathlib import Path
 
@@ -12,6 +13,7 @@
 
 from topostats.io import LoadScans, hdf5_to_dict
 from topostats.processing import (
+    LOGGER_NAME,
     check_run_steps,
     process_scan,
     run_filters,
@@ -23,8 +25,6 @@
 BASE_DIR = Path.cwd()
 RESOURCES = BASE_DIR / "tests/resources"
 
-# pylint: disable=too-many-positional-arguments
-
 
 # Can't see a way of parameterising with pytest-regtest as it writes to a file based on the file/function
 # so instead we run three regression tests.
@@ -637,6 +637,7 @@ def test_process_stages(
     Grainstats are disabled. Whislt possible it is expected that users understand the need to run earlier stages before
     later stages can run and do not disable earlier stages.
     """
+    caplog.set_level(logging.DEBUG, LOGGER_NAME)
     img_dic = load_scan_data.img_dict
     process_scan_config["filter"]["run"] = filter_run
     process_scan_config["grains"]["run"] = grains_run

tests/test_utils.py

@@ -1,12 +1,14 @@
 """Test the utils module."""
 
+import logging
 from pathlib import Path
 
 import numpy as np
 import numpy.typing as npt
 import pytest
 
 from topostats.utils import (
+    LOGGER_NAME,
     bound_padded_coordinates_to_image,
     convert_path,
     convolve_skeleton,
@@ -34,6 +36,7 @@ def test_convert_path(tmp_path: Path) -> None:
 
 def test_update_config(caplog) -> None:
     """Test updating configuration."""
+    caplog.set_level(logging.DEBUG, LOGGER_NAME)
     sample_config = {"a": 1, "b": 2, "c": "something", "base_dir": "here", "output_dir": "there"}
     new_values = {"c": "something new"}
     updated_config = update_config(sample_config, new_values)
@@ -138,6 +141,7 @@ def test_update_plotting_config(
 )
 def test_udpate_plotting_config_adding_required_options(plotting_config: dict, target_config: dict, caplog) -> None:
     """Only updates plotting_dict parameters from parent plotting config if value is not None."""
+    caplog.set_level(logging.DEBUG, LOGGER_NAME)
     update_plotting_config(plotting_config)
     assert plotting_config == target_config
     if plotting_config["savefig_dpi"] == 600:

topostats/filters.py

@@ -172,10 +172,10 @@ def median_flatten(
         image = image.copy()
         if mask is not None:
             read_matrix = np.ma.masked_array(image, mask=mask, fill_value=np.nan).filled()
-            LOGGER.info(f"[{self.filename}] : Median flattening with mask")
+            LOGGER.debug(f"[{self.filename}] : Median flattening with mask")
         else:
             read_matrix = image
-            LOGGER.info(f"[{self.filename}] : Median flattening without mask")
+            LOGGER.debug(f"[{self.filename}] : Median flattening without mask")
 
         for row in range(image.shape[0]):
             # Get the median of the row
@@ -185,7 +185,7 @@ def median_flatten(
             else:
                 LOGGER.warning(
                     """f[{self.filename}] Large grain detected image can not be
-processed, please refer to <url to page where we document common problems> for more information."""
+processed, please refer to https://github.com/AFM-SPM/TopoStats/discussions for more information."""
                 )
 
         return image
@@ -212,10 +212,10 @@ def remove_tilt(self, image: npt.NDArray, mask: npt.NDArray = None) -> npt.NDArr
         image = image.copy()
         if mask is not None:
             read_matrix = np.ma.masked_array(image, mask=mask, fill_value=np.nan).filled()
-            LOGGER.info(f"[{self.filename}] : Plane tilt removal with mask")
+            LOGGER.debug(f"[{self.filename}] : Plane tilt removal with mask")
         else:
             read_matrix = image
-            LOGGER.info(f"[{self.filename}] : Plane tilt removal without mask")
+            LOGGER.debug(f"[{self.filename}] : Plane tilt removal without mask")
 
         # Line of best fit
         # Calculate medians
@@ -226,31 +226,31 @@ def remove_tilt(self, image: npt.NDArray, mask: npt.NDArray = None) -> npt.NDArr
 
         # Fit linear x
         px = np.polyfit(range(0, len(medians_x)), medians_x, 1)
-        LOGGER.info(f"[{self.filename}] : x-polyfit 1st order: {px}")
+        LOGGER.debug(f"[{self.filename}] : x-polyfit 1st order: {px}")
         py = np.polyfit(range(0, len(medians_y)), medians_y, 1)
-        LOGGER.info(f"[{self.filename}] : y-polyfit 1st order: {py}")
+        LOGGER.debug(f"[{self.filename}] : y-polyfit 1st order: {py}")
 
         if px[0] != 0:
             if not np.isnan(px[0]):
-                LOGGER.info(f"[{self.filename}] : Removing x plane tilt")
+                LOGGER.debug(f"[{self.filename}] : Removing x plane tilt")
                 for row in range(0, image.shape[0]):
                     for col in range(0, image.shape[1]):
                         image[row, col] -= px[0] * (col)
             else:
-                LOGGER.info(f"[{self.filename}] : x gradient is nan, skipping plane tilt x removal")
+                LOGGER.debug(f"[{self.filename}] : x gradient is nan, skipping plane tilt x removal")
         else:
-            LOGGER.info("[{self.filename}] : x gradient is zero, skipping plane tilt x removal")
+            LOGGER.debug("[{self.filename}] : x gradient is zero, skipping plane tilt x removal")
 
         if py[0] != 0:
             if not np.isnan(py[0]):
-                LOGGER.info(f"[{self.filename}] : removing y plane tilt")
+                LOGGER.debug(f"[{self.filename}] : removing y plane tilt")
                 for row in range(0, image.shape[0]):
                     for col in range(0, image.shape[1]):
                         image[row, col] -= py[0] * (row)
             else:
-                LOGGER.info("[{self.filename}] : y gradient is nan, skipping plane tilt y removal")
+                LOGGER.debug("[{self.filename}] : y gradient is nan, skipping plane tilt y removal")
         else:
-            LOGGER.info("[{self.filename}] : y gradient is zero, skipping plane tilt y removal")
+            LOGGER.debug("[{self.filename}] : y gradient is zero, skipping plane tilt y removal")
 
         return image
 
@@ -344,7 +344,7 @@ def model_func(x: float, y: float, a: float, b: float, c: float, d: float) -> fl
 
         # Unpack the optimised parameters
         a, b, c, d = popt
-        LOGGER.info(
+        LOGGER.debug(
             f"[{self.filename}] : Nonlinear polynomial removal optimal params: const: {a} xy: {b} x: {c} y: {d}"
         )
 
@@ -377,17 +377,17 @@ def remove_quadratic(self, image: npt.NDArray, mask: npt.NDArray = None) -> npt.
         image = image.copy()
         if mask is not None:
             read_matrix = np.ma.masked_array(image, mask=mask, fill_value=np.nan).filled()
-            LOGGER.info(f"[{self.filename}] : Remove quadratic bow with mask")
+            LOGGER.debug(f"[{self.filename}] : Remove quadratic bow with mask")
         else:
             read_matrix = image
-            LOGGER.info(f"[{self.filename}] : Remove quadratic bow without mask")
+            LOGGER.debug(f"[{self.filename}] : Remove quadratic bow without mask")
 
         # Calculate medians
         medians_x = [np.nanmedian(read_matrix[:, i]) for i in range(read_matrix.shape[1])]
 
         # Fit quadratic x
         px = np.polyfit(range(0, len(medians_x)), medians_x, 2)
-        LOGGER.info(f"[{self.filename}] : x polyfit 2nd order: {px}")
+        LOGGER.debug(f"[{self.filename}] : x polyfit 2nd order: {px}")
 
         # Handle divide by zero
         if px[0] != 0:
@@ -398,9 +398,9 @@ def remove_quadratic(self, image: npt.NDArray, mask: npt.NDArray = None) -> npt.
                     for col in range(0, image.shape[1]):
                         image[row, col] -= px[0] * (col - cx) ** 2
             else:
-                LOGGER.info(f"[{self.filename}] : Quadratic polyfit returns nan, skipping quadratic removal")
+                LOGGER.debug(f"[{self.filename}] : Quadratic polyfit returns nan, skipping quadratic removal")
         else:
-            LOGGER.info(f"[{self.filename}] : Quadratic polyfit returns zero, skipping quadratic removal")
+            LOGGER.debug(f"[{self.filename}] : Quadratic polyfit returns zero, skipping quadratic removal")
 
         return image
 
@@ -458,7 +458,7 @@ def average_background(self, image: npt.NDArray, mask: npt.NDArray = None) -> np
         if mask is None:
             mask = np.zeros_like(image)
         mean = np.mean(image[mask == 0])
-        LOGGER.info(f"[{self.filename}] : Zero averaging background : {mean} nm")
+        LOGGER.debug(f"[{self.filename}] : Zero averaging background : {mean} nm")
         return image - mean
 
     def gaussian_filter(self, image: npt.NDArray, **kwargs) -> npt.NDArray:
@@ -477,7 +477,7 @@ def gaussian_filter(self, image: npt.NDArray, **kwargs) -> npt.NDArray:
         npt.NDArray
             Numpy array that represent the image after Gaussian filtering.
         """
-        LOGGER.info(
+        LOGGER.debug(
             f"[{self.filename}] : Applying Gaussian filter (mode : {self.gaussian_mode};"
             f" Gaussian blur (px) : {self.gaussian_size})."
         )
@@ -520,14 +520,14 @@ def filter_image(self) -> None:  # numpydoc: ignore=GL07
         # Remove scars
         run_scar_removal = self.remove_scars_config.pop("run")
         if run_scar_removal:
-            LOGGER.info(f"[{self.filename}] : Initial scar removal")
+            LOGGER.debug(f"[{self.filename}] : Initial scar removal")
             self.images["initial_scar_removal"], _ = scars.remove_scars(
                 self.images["initial_nonlinear_polynomial_removal"],
                 filename=self.filename,
                 **self.remove_scars_config,
             )
         else:
-            LOGGER.info(f"[{self.filename}] : Skipping scar removal as requested from config")
+            LOGGER.debug(f"[{self.filename}] : Skipping scar removal as requested from config")
             self.images["initial_scar_removal"] = self.images["initial_nonlinear_polynomial_removal"]
 
         # Zero the data before thresholding, helps with absolute thresholding
@@ -565,15 +565,15 @@ def filter_image(self) -> None:  # numpydoc: ignore=GL07
         )
         # Remove scars
         if run_scar_removal:
-            LOGGER.info(f"[{self.filename}] : Secondary scar removal")
+            LOGGER.debug(f"[{self.filename}] : Secondary scar removal")
             self.images["secondary_scar_removal"], scar_mask = scars.remove_scars(
                 self.images["masked_nonlinear_polynomial_removal"],
                 filename=self.filename,
                 **self.remove_scars_config,
             )
             self.images["scar_mask"] = scar_mask
         else:
-            LOGGER.info(f"[{self.filename}] : Skipping scar removal as requested from config")
+            LOGGER.debug(f"[{self.filename}] : Skipping scar removal as requested from config")
             self.images["secondary_scar_removal"] = self.images["masked_nonlinear_polynomial_removal"]
         self.images["final_zero_average_background"] = self.average_background(
             self.images["secondary_scar_removal"], self.images["mask"]