Merge pull request #111 from HealthyPear/maintenance-update_TRAINING_…

…test

Update TRAINING integration tests

protopipe/aux/example_config_files/analysis.yaml

@@ -2,7 +2,7 @@
 # NOTE: only Prod3b simulations are currently supported.
 General:
  config_name: 'v0.4.0_dev1'
- site: 'south'  # 'north' or 'south'
+ site: 'north'  # 'north' or 'south'
  # array can be either
  # - 'subarray_LSTs', 'subarray_MSTs', 'subarray_SSTs' or 'full_array'
  # - a custom list of telescope IDs
@@ -18,8 +18,8 @@ ImageCleaning:
  biggest:
   tail:  #
    thresholds:  # picture, boundary
-    - LSTCam: [6.61, 3.30]  # TBC
-    - NectarCam: [5.75, 2.88]  # TBC
+    - LSTCam: [6.61, 3.30]
+    - NectarCam: [5.75, 2.88]
     - FlashCam: [4,2] # dummy values for reliable unit-testing
     - ASTRICam: [4,2] # dummy values for reliable unit-testing
     - DigiCam: [0,0] # values left unset for future studies
@@ -52,8 +52,8 @@ ImageCleaning:
  extended:
   tail:  #
    thresholds:  # picture, boundary
-    - LSTCam: [6.61, 3.30]  # TBC
-    - NectarCam: [5.75, 2.88]  # TBC
+    - LSTCam: [6.61, 3.30]
+    - NectarCam: [5.75, 2.88]
     - FlashCam: [4,2] # dummy values for reliable unit-testing
     - ASTRICam: [4,2] # dummy values for reliable unit-testing
     - DigiCam: [0,0] # values left unset for future studies

protopipe/pipeline/temp.py

@@ -10,6 +10,10 @@
 
 """
 
+import os
+import logging
+from pathlib import Path
+
 import numpy as np
 from numpy import nan
 from scipy.sparse import lil_matrix, csr_matrix
@@ -22,6 +26,7 @@
 from ctapipe.reco import HillasReconstructor
 from ctapipe.reco.HillasReconstructor import HillasPlane
 
+logger = logging.getLogger(__name__)
 
 class HillasParametersTelescopeFrameContainer(Container):
     container_prefix = "hillas"
@@ -199,3 +204,58 @@ def initialize_hillas_planes(
                 weight=moments.intensity * (moments.length / moments.width),
             )
             self.hillas_planes[tel_id] = circle
+
+
+try:
+    import ctapipe_resources
+
+    has_resources = True
+except ImportError:
+    has_resources = False
+
+DEFAULT_URL = "http://cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.3/"
+def get_dataset_path(filename, url=DEFAULT_URL):
+    """
+    Returns the full file path to an auxiliary dataset needed by
+    ctapipe, given the dataset's full name (filename with no directory).
+    This will first search for the file in directories listed in
+    tne environment variable CTAPIPE_SVC_PATH (if set), and if not found,
+    will look in the ctapipe_resources module
+    (if installed with the ctapipe-extra package), which contains the defaults.
+    Parameters
+    ----------
+    filename: str
+        name of dataset to fetch
+    Returns
+    -------
+    string with full path to the given dataset
+    """
+
+    searchpath = os.getenv("CTAPIPE_SVC_PATH")
+
+    if searchpath:
+        filepath = find_in_path(filename=filename, searchpath=searchpath, url=url)
+
+        if filepath:
+            return filepath
+
+    if has_resources:
+        logger.debug(
+            "Resource '{}' not found in CTAPIPE_SVC_PATH, looking in "
+            "ctapipe_resources...".format(filename)
+        )
+
+        return Path(ctapipe_resources.get(filename))
+
+    # last, try downloading the data
+    try:
+        return download_file_cached(filename, default_url=url, progress=True)
+    except HTTPError as e:
+        # let 404 raise the FileNotFoundError instead of HTTPError
+        if e.response.status_code != 404:
+            raise
+
+    raise FileNotFoundError(
+        f"Couldn't find resource: '{filename}',"
+        " You might want to install ctapipe_resources"
+    )

protopipe/scripts/tests/test_config_analysis_north.yaml

@@ -0,0 +1,106 @@
+# General informations
+# NOTE: only Prod3b simulations are currently supported.
+General:
+ config_name: 'v0.4.0_dev1'
+ site: 'north'  # 'north' or 'south'
+ # array can be either
+ # - 'subarray_LSTs', 'subarray_MSTs', 'subarray_SSTs' or 'full_array'
+ # - a custom list of telescope IDs
+ # WARNING: for simulations containing multiple copies of the telescopes,
+ # only 'full_array' or custom list are supported options!
+ array: full_array
+ cam_id_list : ['LSTCam', 'NectarCam'] # Selected cameras (disabled option)
+
+# Cleaning for reconstruction
+ImageCleaning:
+
+ # Cleaning for reconstruction
+ biggest:
+  tail:  #
+   thresholds:  # picture, boundary
+    - LSTCam: [6.61, 3.30]
+    - NectarCam: [5.75, 2.88]
+    - FlashCam: [4,2] # dummy values for reliable unit-testing
+    - ASTRICam: [4,2] # dummy values for reliable unit-testing
+    - DigiCam: [0,0] # values left unset for future studies
+    - CHEC: [0,0] # values left unset for future studies
+    - SCTCam: [0,0] # values left unset for future studies
+   keep_isolated_pixels: False
+   min_number_picture_neighbors: 1
+
+  wave:
+   # Directory to write temporary files
+   #tmp_files_directory: '/dev/shm/'
+   tmp_files_directory: './'
+   options:
+    LSTCam:
+     type_of_filtering: 'hard_filtering'
+     filter_thresholds: [3, 0.2]
+     last_scale_treatment: 'drop'
+     kill_isolated_pixels: True
+     detect_only_positive_structures: False
+     clusters_threshold: 0
+    NectarCam:  # TBC
+     type_of_filtering: 'hard_filtering'
+     filter_thresholds: [3, 0.2]
+     last_scale_treatment: 'drop'
+     kill_isolated_pixels: True
+     detect_only_positive_structures: False
+     clusters_threshold: 0
+
+ # Cleaning for energy/score estimation
+ extended:
+  tail:  #
+   thresholds:  # picture, boundary
+    - LSTCam: [6.61, 3.30]
+    - NectarCam: [5.75, 2.88]
+    - FlashCam: [4,2] # dummy values for reliable unit-testing
+    - ASTRICam: [4,2] # dummy values for reliable unit-testing
+    - DigiCam: [0,0] # values left unset for future studies
+    - CHEC: [0,0] # values left unset for future studies
+    - SCTCam: [0,0] # values left unset for future studies
+   keep_isolated_pixels: False
+   min_number_picture_neighbors: 1
+
+  wave:
+   # Directory to write temporary files
+   #tmp_files_directory: '/dev/shm/'
+   tmp_files_directory: './'
+   options:
+    LSTCam:
+     type_of_filtering: 'hard_filtering'
+     filter_thresholds: [3, 0.2]
+     last_scale_treatment: 'posmask'
+     kill_isolated_pixels: True
+     detect_only_positive_structures: False
+     clusters_threshold: 0
+    NectarCam:  # TBC
+     type_of_filtering: 'hard_filtering'
+     filter_thresholds: [3, 0.2]
+     last_scale_treatment: 'posmask'
+     kill_isolated_pixels: True
+     detect_only_positive_structures: False
+     clusters_threshold: 0
+
+# Cut for image selection
+ImageSelection:
+ charge: [50., 1e10]
+ pixel: [3, 1e10]
+ ellipticity: [0.1, 0.6]
+ nominal_distance: [0., 0.8]  # in camera radius
+
+# Minimal number of telescopes to consider events
+Reconstruction:
+ min_tel: 2
+
+# Parameters for energy estimation
+EnergyRegressor:
+ # Name of the regression method (e.g. AdaBoostRegressor, etc.)
+ method_name: 'AdaBoostRegressor'
+
+# Parameters for g/h separation
+GammaHadronClassifier:
+ # Name of the classification method (e.g. AdaBoostRegressor, etc.)
+ method_name: 'RandomForestClassifier'
+ # Use probability output or score
+ use_proba: True

protopipe/scripts/tests/test_config_analysis_south.yaml

@@ -0,0 +1,106 @@
+# General informations
+# NOTE: only Prod3b simulations are currently supported.
+General:
+ config_name: 'v0.4.0_dev1'
+ site: 'south'  # 'north' or 'south'
+ # array can be either
+ # - 'subarray_LSTs', 'subarray_MSTs', 'subarray_SSTs' or 'full_array'
+ # - a custom list of telescope IDs
+ # WARNING: for simulations containing multiple copies of the telescopes,
+ # only 'full_array' or custom list are supported options!
+ array: full_array
+ cam_id_list : ['LSTCam', 'NectarCam'] # Selected cameras (disabled option)
+
+# Cleaning for reconstruction
+ImageCleaning:
+
+ # Cleaning for reconstruction
+ biggest:
+  tail:  #
+   thresholds:  # picture, boundary
+    - LSTCam: [6.61, 3.30]
+    - NectarCam: [5.75, 2.88]
+    - FlashCam: [4,2] # dummy values for reliable unit-testing
+    - ASTRICam: [4,2] # dummy values for reliable unit-testing
+    - DigiCam: [0,0] # values left unset for future studies
+    - CHEC: [0,0] # values left unset for future studies
+    - SCTCam: [0,0] # values left unset for future studies
+   keep_isolated_pixels: False
+   min_number_picture_neighbors: 1
+
+  wave:
+   # Directory to write temporary files
+   #tmp_files_directory: '/dev/shm/'
+   tmp_files_directory: './'
+   options:
+    LSTCam:
+     type_of_filtering: 'hard_filtering'
+     filter_thresholds: [3, 0.2]
+     last_scale_treatment: 'drop'
+     kill_isolated_pixels: True
+     detect_only_positive_structures: False
+     clusters_threshold: 0
+    NectarCam:  # TBC
+     type_of_filtering: 'hard_filtering'
+     filter_thresholds: [3, 0.2]
+     last_scale_treatment: 'drop'
+     kill_isolated_pixels: True
+     detect_only_positive_structures: False
+     clusters_threshold: 0
+
+ # Cleaning for energy/score estimation
+ extended:
+  tail:  #
+   thresholds:  # picture, boundary
+    - LSTCam: [6.61, 3.30]
+    - NectarCam: [5.75, 2.88]
+    - FlashCam: [4,2] # dummy values for reliable unit-testing
+    - ASTRICam: [4,2] # dummy values for reliable unit-testing
+    - DigiCam: [0,0] # values left unset for future studies
+    - CHEC: [0,0] # values left unset for future studies
+    - SCTCam: [0,0] # values left unset for future studies
+   keep_isolated_pixels: False
+   min_number_picture_neighbors: 1
+
+  wave:
+   # Directory to write temporary files
+   #tmp_files_directory: '/dev/shm/'
+   tmp_files_directory: './'
+   options:
+    LSTCam:
+     type_of_filtering: 'hard_filtering'
+     filter_thresholds: [3, 0.2]
+     last_scale_treatment: 'posmask'
+     kill_isolated_pixels: True
+     detect_only_positive_structures: False
+     clusters_threshold: 0
+    NectarCam:  # TBC
+     type_of_filtering: 'hard_filtering'
+     filter_thresholds: [3, 0.2]
+     last_scale_treatment: 'posmask'
+     kill_isolated_pixels: True
+     detect_only_positive_structures: False
+     clusters_threshold: 0
+
+# Cut for image selection
+ImageSelection:
+ charge: [50., 1e10]
+ pixel: [3, 1e10]
+ ellipticity: [0.1, 0.6]
+ nominal_distance: [0., 0.8]  # in camera radius
+
+# Minimal number of telescopes to consider events
+Reconstruction:
+ min_tel: 2
+
+# Parameters for energy estimation
+EnergyRegressor:
+ # Name of the regression method (e.g. AdaBoostRegressor, etc.)
+ method_name: 'AdaBoostRegressor'
+
+# Parameters for g/h separation
+GammaHadronClassifier:
+ # Name of the classification method (e.g. AdaBoostRegressor, etc.)
+ method_name: 'RandomForestClassifier'
+ # Use probability output or score
+ use_proba: True