From bab78f75c6e7df7ab5ef2a18c5e43db5fa42b323 Mon Sep 17 00:00:00 2001 From: s4294967296 Date: Tue, 14 Feb 2023 23:49:29 +0100 Subject: [PATCH 1/9] RingCounting.py: added model loading and documentation --- UserTools/RingCounting/RingCounting.py | 168 ++++++++++++++++++++++--- 1 file changed, 154 insertions(+), 14 deletions(-) diff --git a/UserTools/RingCounting/RingCounting.py b/UserTools/RingCounting/RingCounting.py index 442316778..2079cbeba 100644 --- a/UserTools/RingCounting/RingCounting.py +++ b/UserTools/RingCounting/RingCounting.py @@ -1,39 +1,179 @@ -# python Tool script -# ------------------ +######################################################################################################################## +# +# 8888888b. d8b .d8888b. 888 d8b +# 888 Y88b Y8P d88P Y88b 888 Y8P +# 888 888 888 888 888 +# 888 d88P 888 88888b. .d88b. 888 .d88b. 888 888 88888b. 888888 888 88888b. .d88b. +# 8888888P" 888 888 "88b d88P"88b 888 d88""88b 888 888 888 "88b 888 888 888 "88b d88P"88b +# 888 T88b 888 888 888 888 888 888 888 888 888 888 888 888 888 888 888 888 888 888 888 +# 888 T88b 888 888 888 Y88b 888 Y88b d88P Y88..88P Y88b 888 888 888 Y88b. 888 888 888 Y88b 888 +# 888 T88b 888 888 888 "Y88888 "Y8888P" "Y88P" "Y88888 888 888 "Y888 888 888 888 "Y88888 +# 888 888 +# Y8b d88P Y8b d88P +# "Y88P" "Y88P" +# -------------------------------------------------------------------------------------------------------------------- +# Daniel Tobias Schmid, Feb. 2023, dschmid@fnal.gov / d.schmid@students.uni-mainz.de +# -------------------------------------------------------------------------------------------------------------------- +# +# The Cherenkov-ring-counting tool is used to classify events as single- or multi-ring by analyzing PMT hit maps loaded +# from a CSV file of CNNImage displays. It uses a keras based neural network. To use this tool in a ToolChain, users +# must populate the the corresponding "/configfiles/YourToolChain/RingCountingConfig" file with the following +# information: +# (Square brackets imply a variable in the config file) +# 1. The path to a file containing a list of files of the PMT data to be used.Must be in CNNImage format. +# -> defined by setting [[files_to_load]] +# 2. The model version +# -> defined by setting [[version]] +# Check the documentation! +# 3. The model directory path (not including the model's filename, but including last slash) +# -> defined by setting [[model_path]] +# 4. Which PMT mask to use (some PMTs have been turned off in the training); check documentation for which model +# requires what mask. +# -> defined by setting [[pmt_mask]] +# An example config file can also be found in in the RingCountingStore/documentation/ folders mentioned below. +# +# Documentation on the tool, model versions and performance can be found in (anniegpvm-machine): +# /pnfs/annie/persistent/users/dschmid/RingCountingStore/documentation/ +# All models are located in (anniegpvm-machine): +# /pnfs/annie/persistent/users/dschmid/RingCountingStore/models/ +# +######################################################################################################################## from Tool import * -class TestPython(Tool): - - # declare member variables here - demovariable = 'TestString' - +import numpy as np +import tensorflow as tf + + +class RingCountingGlobals: + PMT_MASKS = { + "none": [], + # From per-PMT p.e. distribution histograms -> which PMTs have different MC / data curves + "pe_curve_divergent_november_22": [56, 58, 84, 85, 87, 88, 89, 100, 118, 120, 121, 137, 138, 139], + # Difference in MC and data on specific datasets used for training + "active_divergent_november_22": [22, 124], + # Both november 22 lists + "november_22": [22, 56, 58, 84, 85, 87, 88, 89, 100, 118, 120, 121, 124, 137, 138, 139], + # Load from config TODO + "custom": [] + } + + +class RingCounting(Tool, RingCountingGlobals): + # ---------------------------------------------------------------------------------------------------- + # Data stuff + cnn_image_pmt: np.array = None # To be loaded with PMT data + + # ---------------------------------------------------------------------------------------------------- + # Config stuff + files_to_load = std.string() # List of files to be loaded (must be in CNNImage format) + version = std.string() # Model version + model_path = std.string() # Path to model directory + pmt_mask = std.string() + + # ---------------------------------------------------------------------------------------------------- + # Model stuff + model = None + predicted = None + def Initialise(self): - self.m_log.Log(__file__+" Initialising", self.v_debug, self.m_verbosity) - self.m_variables.Print() + # ---------------------------------------------------------------------------------------------------- + # Debug area + self.m_log.Log(__file__ + " Initialising", self.v_debug, self.m_verbosity) + + # ---------------------------------------------------------------------------------------------------- + # Config area + self.m_variables.Get("files_to_load", self.files_to_load) + self.m_variables.Get("version", self.version) + self.m_variables.Get("model_path", self.model_path) + self.m_variables.Get("pmt_mask", self.pmt_mask) + self.pmt_mask = PMT_MASKS[self.pmt_mask] + self._initialise_from_config() + + # ---------------------------------------------------------------------------------------------------- + # Loading data + self.load_data() + self.mask_pmts() + + # ---------------------------------------------------------------------------------------------------- + # Loading model + self.load_model() + return 1 - + + def load_data(self): + """ + Todo: Docstr. + """ + self.cnn_image_pmt = np.array([]) + with open(self.files_to_load) as file: + for data_path in file: + arr = np.loadtxt(data_path, delimiter=",") + if arr.shape == (160,): + # Explicitly adding the extra dimension using np.array([]) is done to ensure the data is + # reshaped into the shape (-1, 160) in case of having only a single entry. + arr = np.array([arr]) + arr = np.reshape(arr, (-1, 160)) + + if len(self.cnn_image_pmt) > 0: + self.cnn_image_pmt = np.concatenate([arr, self.cnn_image_pmt], axis=0) + elif len(arr) > 0: + self.cnn_image_pmt = arr + else: + self.m_log.Log(__file__ + f" Attempted to load an empty PMT-datafile (CNNImage). ({data_path})", + self.v_debug, self.m_verbosity) + + def mask_pmts(self): + """ + TODO: Docstr. + """ + for i in range(len(self.cnn_image_pmt)): + self.cnn_image_pmt[i] = np.put(self.cnn_image_pmt, self.PMT_MASKS[self.pmt_mask], 0, mode='raise') + + pass + + def load_model(self): + self.model = tf.keras.models.load_model(self.model_path + f"RC_model_v{self.version}.model") + def Execute(self): - self.m_log.Log(__file__+" Executing", self.v_debug, self.m_verbosity) + self.m_log.Log(__file__ + " Executing", self.v_debug, self.m_verbosity) + self.predict() return 1 - + def Finalise(self): - self.m_log.Log(__file__+" Finalising", self.v_debug, self.m_verbosity) + self.m_log.Log(__file__ + " Finalising", self.v_debug, self.m_verbosity) return 1 + def predict(self): + """ + Classify events in single- and multi-ring events using a keras model. PMT data is supplied as an ordered list + of arrays, saved in the path specified in the config file. Return a list of 2-dimensional predictions + (same order as input). When used for RC, predictions are given as [MR prediction, SR prediction]. + ---------------------------------------------------------------------------------------------------------------- + The following variables - to be set in the config file - will modify the behaviour of this method: + - NONE - + """ + self.predicted = self.model.predict(self.cnn_image_pmt) + + ################### # ↓ Boilerplate ↓ # ################### -thistool = TestPython() + +thistool = RingCounting() + def SetToolChainVars(m_data_in, m_variables_in, m_log_in): return thistool.SetToolChainVars(m_data_in, m_variables_in, m_log_in) + def Initialise(): return thistool.Initialise() + def Execute(): return thistool.Execute() + def Finalise(): return thistool.Finalise() From 374eb6f188c1077650541aa9e92c4573a8e4664b Mon Sep 17 00:00:00 2001 From: s4294967296 Date: Wed, 15 Feb 2023 00:29:32 +0100 Subject: [PATCH 2/9] RingCounting.py: fix bug --- UserTools/RingCounting/RingCounting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/UserTools/RingCounting/RingCounting.py b/UserTools/RingCounting/RingCounting.py index 2079cbeba..1d1d24312 100644 --- a/UserTools/RingCounting/RingCounting.py +++ b/UserTools/RingCounting/RingCounting.py @@ -86,7 +86,7 @@ def Initialise(self): self.m_variables.Get("version", self.version) self.m_variables.Get("model_path", self.model_path) self.m_variables.Get("pmt_mask", self.pmt_mask) - self.pmt_mask = PMT_MASKS[self.pmt_mask] + self.pmt_mask = self.PMT_MASKS[self.pmt_mask] self._initialise_from_config() # ---------------------------------------------------------------------------------------------------- @@ -127,7 +127,7 @@ def mask_pmts(self): TODO: Docstr. """ for i in range(len(self.cnn_image_pmt)): - self.cnn_image_pmt[i] = np.put(self.cnn_image_pmt, self.PMT_MASKS[self.pmt_mask], 0, mode='raise') + self.cnn_image_pmt[i] = np.put(self.cnn_image_pmt, self.pmt_mask, 0, mode='raise') pass From 63ea78c8a548808b20ae9a3377a6f8b9426f1374 Mon Sep 17 00:00:00 2001 From: s4294967296 Date: Thu, 16 Feb 2023 20:07:39 +0100 Subject: [PATCH 3/9] RingCounting.py: rm unimplemented method call --- UserTools/RingCounting/RingCounting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/UserTools/RingCounting/RingCounting.py b/UserTools/RingCounting/RingCounting.py index 1d1d24312..d1d18e2c6 100644 --- a/UserTools/RingCounting/RingCounting.py +++ b/UserTools/RingCounting/RingCounting.py @@ -87,7 +87,6 @@ def Initialise(self): self.m_variables.Get("model_path", self.model_path) self.m_variables.Get("pmt_mask", self.pmt_mask) self.pmt_mask = self.PMT_MASKS[self.pmt_mask] - self._initialise_from_config() # ---------------------------------------------------------------------------------------------------- # Loading data From e987e42d7ee5caba0deb4c25f27ce42e0132fd5a Mon Sep 17 00:00:00 2001 From: "Daniel T. Schmid" Date: Thu, 16 Feb 2023 15:19:03 -0600 Subject: [PATCH 4/9] RingCounting.py: fixed pmt masking, fixed file loading, added files_to_load path line commenting support --- UserTools/RingCounting/RingCounting.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/UserTools/RingCounting/RingCounting.py b/UserTools/RingCounting/RingCounting.py index d1d18e2c6..0399aae75 100644 --- a/UserTools/RingCounting/RingCounting.py +++ b/UserTools/RingCounting/RingCounting.py @@ -40,6 +40,8 @@ ######################################################################################################################## from Tool import * +import os + import numpy as np import tensorflow as tf @@ -61,7 +63,7 @@ class RingCountingGlobals: class RingCounting(Tool, RingCountingGlobals): # ---------------------------------------------------------------------------------------------------- # Data stuff - cnn_image_pmt: np.array = None # To be loaded with PMT data + cnn_image_pmt = None # To be loaded with PMT data # ---------------------------------------------------------------------------------------------------- # Config stuff @@ -104,8 +106,12 @@ def load_data(self): Todo: Docstr. """ self.cnn_image_pmt = np.array([]) - with open(self.files_to_load) as file: + with open(str(self.files_to_load), "r") as file: for data_path in file: + data_path = data_path.strip(" \n") + if data_path[0] == "#": + continue + arr = np.loadtxt(data_path, delimiter=",") if arr.shape == (160,): # Explicitly adding the extra dimension using np.array([]) is done to ensure the data is @@ -125,8 +131,8 @@ def mask_pmts(self): """ TODO: Docstr. """ - for i in range(len(self.cnn_image_pmt)): - self.cnn_image_pmt[i] = np.put(self.cnn_image_pmt, self.pmt_mask, 0, mode='raise') + for event in self.cnn_image_pmt: + np.put(event, self.pmt_mask, 0, mode='raise') pass @@ -136,6 +142,9 @@ def load_model(self): def Execute(self): self.m_log.Log(__file__ + " Executing", self.v_debug, self.m_verbosity) self.predict() + print(self.predicted) + print("predicting zero arr") + print(self.model.predict(np.zeros((1, 10, 16, 1)))) return 1 def Finalise(self): @@ -151,7 +160,9 @@ def predict(self): The following variables - to be set in the config file - will modify the behaviour of this method: - NONE - """ - self.predicted = self.model.predict(self.cnn_image_pmt) + print("Predicting...") + print(self.cnn_image_pmt) + self.predicted = self.model.predict(np.reshape(self.cnn_image_pmt, newshape=(-1, 10, 16, 1))) ################### From 8c60e45bdeaa15cd70fc2250ad5a034134e5d72f Mon Sep 17 00:00:00 2001 From: s4294967296 Date: Fri, 17 Feb 2023 18:16:23 +0100 Subject: [PATCH 5/9] RingCounting.py: + docStrings + saving controlled by [[save_to]] --- UserTools/RingCounting/RingCounting.py | 81 +++++++++++++++++--------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/UserTools/RingCounting/RingCounting.py b/UserTools/RingCounting/RingCounting.py index 0399aae75..b78ddcb7d 100644 --- a/UserTools/RingCounting/RingCounting.py +++ b/UserTools/RingCounting/RingCounting.py @@ -30,23 +30,28 @@ # 4. Which PMT mask to use (some PMTs have been turned off in the training); check documentation for which model # requires what mask. # -> defined by setting [[pmt_mask]] +# 5. Where to save the predictions. +# -> defined by setting [[save_to]] # An example config file can also be found in in the RingCountingStore/documentation/ folders mentioned below. # # Documentation on the tool, model versions and performance can be found in (anniegpvm-machine): -# /pnfs/annie/persistent/users/dschmid/RingCountingStore/documentation/ +# /pnfs/annie/persistent/users/dschmid/RingCountingStore/documentation/ **TODO** # All models are located in (anniegpvm-machine): # /pnfs/annie/persistent/users/dschmid/RingCountingStore/models/ # ######################################################################################################################## from Tool import * -import os - import numpy as np import tensorflow as tf class RingCountingGlobals: + # The reason to mask some PMTs to 0 is to disable PMTs that differ in the MC and experimental datasets. + # If a model has been trained with a specific mask toggled on, it is crucial that when evaluating experimental + # data with this model, that the same specific mask is also toggled on. This is due to the model not having + # encountered these PMTs being enabled during training, and thus probably producing worse predictions with + # the un-encountered PMTs enabled when predicting. PMT_MASKS = { "none": [], # From per-PMT p.e. distribution histograms -> which PMTs have different MC / data curves @@ -70,7 +75,8 @@ class RingCounting(Tool, RingCountingGlobals): files_to_load = std.string() # List of files to be loaded (must be in CNNImage format) version = std.string() # Model version model_path = std.string() # Path to model directory - pmt_mask = std.string() + pmt_mask = std.string() # See RingCountingGlobals + save_to = std.string() # Where to save the predictions to # ---------------------------------------------------------------------------------------------------- # Model stuff @@ -78,6 +84,14 @@ class RingCounting(Tool, RingCountingGlobals): predicted = None def Initialise(self): + """ Initialise RingCounting tool object in following these steps: + 1. Load necessary config + 2. Load data + 2.1 Modify data + 3. Load model + + For specific details, see compartmentalised function docstrings. + """ # ---------------------------------------------------------------------------------------------------- # Debug area self.m_log.Log(__file__ + " Initialising", self.v_debug, self.m_verbosity) @@ -88,6 +102,7 @@ def Initialise(self): self.m_variables.Get("version", self.version) self.m_variables.Get("model_path", self.model_path) self.m_variables.Get("pmt_mask", self.pmt_mask) + self.m_variables.Get("save_to", self.save_to) self.pmt_mask = self.PMT_MASKS[self.pmt_mask] # ---------------------------------------------------------------------------------------------------- @@ -102,13 +117,22 @@ def Initialise(self): return 1 def load_data(self): - """ - Todo: Docstr. + """ Load data in the CNNImage format. + + The files to load are specified in a txt file which filepath is defined in the RingCountingConfig file under + [[files_to_load]]. The CNNImage format is defined in the CNNImage tool. In short, a 160-entry list of numbers + (cast-able to numpy float-types) in comma separated format, where events are separated by newlines, is expected. + + Uncommenting a filepath in the [[files_to_load]] file is supported by placing "#" as the first character. + + Handles empty files by skipping and logging a warning. """ self.cnn_image_pmt = np.array([]) - with open(str(self.files_to_load), "r") as file: + with open(self.files_to_load, "r") as file: for data_path in file: + # Removes trailing and leading whitespace and "\n" data_path = data_path.strip(" \n") + # Supports commenting out a file if data_path[0] == "#": continue @@ -124,45 +148,46 @@ def load_data(self): elif len(arr) > 0: self.cnn_image_pmt = arr else: - self.m_log.Log(__file__ + f" Attempted to load an empty PMT-datafile (CNNImage). ({data_path})", + self.m_log.Log("WARNING: " + __file__ + f" Attempted to load an empty PMT-datafile (CNNImage). ({data_path})", self.v_debug, self.m_verbosity) + def save_data(self): + """ Save the data to the specified [[save_to]]-file. """ + np.savetxt(self.save_to, self.predicted, delimiter=",") + def mask_pmts(self): - """ - TODO: Docstr. + """ Mask PMTs to 0. The PMTs to be masked is given as a list of indices, defined by setting [[pmt_mask]]. + For further details check the RingCountingGlobals class. """ for event in self.cnn_image_pmt: np.put(event, self.pmt_mask, 0, mode='raise') - pass - def load_model(self): + """ Load the specified model [[version]].""" self.model = tf.keras.models.load_model(self.model_path + f"RC_model_v{self.version}.model") + def predict(self): + """ + Classify events in single- and multi-ring events using a keras model. Save a list of 2-dimensional predictions + (same order as input) to self.predicted. Predictions are given as [MR prediction, SR prediction]. + """ + print("Predicting...") + print(self.cnn_image_pmt) + self.predicted = self.model.predict(np.reshape(self.cnn_image_pmt, newshape=(-1, 10, 16, 1))) + def Execute(self): + """ Execute the tool by generating model predictions on the supplied data. """ self.m_log.Log(__file__ + " Executing", self.v_debug, self.m_verbosity) self.predict() - print(self.predicted) - print("predicting zero arr") - print(self.model.predict(np.zeros((1, 10, 16, 1)))) + return 1 def Finalise(self): + """ Finalise the tool by saving the predictions. """ self.m_log.Log(__file__ + " Finalising", self.v_debug, self.m_verbosity) - return 1 + self.save_data() - def predict(self): - """ - Classify events in single- and multi-ring events using a keras model. PMT data is supplied as an ordered list - of arrays, saved in the path specified in the config file. Return a list of 2-dimensional predictions - (same order as input). When used for RC, predictions are given as [MR prediction, SR prediction]. - ---------------------------------------------------------------------------------------------------------------- - The following variables - to be set in the config file - will modify the behaviour of this method: - - NONE - - """ - print("Predicting...") - print(self.cnn_image_pmt) - self.predicted = self.model.predict(np.reshape(self.cnn_image_pmt, newshape=(-1, 10, 16, 1))) + return 1 ################### From 28625c453620c5fffa3fcbe242050f71ab5ebe81 Mon Sep 17 00:00:00 2001 From: "Daniel T. Schmid" Date: Fri, 17 Feb 2023 11:47:32 -0600 Subject: [PATCH 6/9] added RingCountingConfig, fixed loading error in RingCountingTool --- UserTools/RingCounting/RingCounting.py | 4 ++++ configfiles/RingCounting/README.md | 25 ++++++++++++++++++++ configfiles/RingCounting/RingCountingConfig | 20 ++++++++++++++++ configfiles/RingCounting/ToolChainConfig | 26 +++++++++++++++++++++ configfiles/RingCounting/ToolsConfig | 1 + configfiles/RingCounting/files_to_load.txt | 7 ++++++ 6 files changed, 83 insertions(+) create mode 100644 configfiles/RingCounting/README.md create mode 100644 configfiles/RingCounting/RingCountingConfig create mode 100644 configfiles/RingCounting/ToolChainConfig create mode 100644 configfiles/RingCounting/ToolsConfig create mode 100644 configfiles/RingCounting/files_to_load.txt diff --git a/UserTools/RingCounting/RingCounting.py b/UserTools/RingCounting/RingCounting.py index b78ddcb7d..1537359b9 100644 --- a/UserTools/RingCounting/RingCounting.py +++ b/UserTools/RingCounting/RingCounting.py @@ -99,10 +99,12 @@ def Initialise(self): # ---------------------------------------------------------------------------------------------------- # Config area self.m_variables.Get("files_to_load", self.files_to_load) + self.files_to_load = str(self.files_to_load) self.m_variables.Get("version", self.version) self.m_variables.Get("model_path", self.model_path) self.m_variables.Get("pmt_mask", self.pmt_mask) self.m_variables.Get("save_to", self.save_to) + self.save_to = str(self.save_to) self.pmt_mask = self.PMT_MASKS[self.pmt_mask] # ---------------------------------------------------------------------------------------------------- @@ -128,6 +130,8 @@ def load_data(self): Handles empty files by skipping and logging a warning. """ self.cnn_image_pmt = np.array([]) + # Removes trailing and leading whitespace and "\n" + self.files_to_load = self.files_to_load.strip(" \n") with open(self.files_to_load, "r") as file: for data_path in file: # Removes trailing and leading whitespace and "\n" diff --git a/configfiles/RingCounting/README.md b/configfiles/RingCounting/README.md new file mode 100644 index 000000000..78d3e2015 --- /dev/null +++ b/configfiles/RingCounting/README.md @@ -0,0 +1,25 @@ +# Configure files + +*********************** +#Description +********************** + +Configure files are simple text files for passing variables to the Tools. + +Text files are read by the Store class (src/Store) and automatically assigned to an internal map for the relevant Tool to use. + + +************************ +#Usage +************************ + +Any line starting with a "#" will be ignored by the Store, as will blank lines. + +Variables should be stored one per line as follows: + + +Name Value #Comments + + +Note: Only one value is permitted per name and they are stored in a string stream and template cast back to the type given. + diff --git a/configfiles/RingCounting/RingCountingConfig b/configfiles/RingCounting/RingCountingConfig new file mode 100644 index 000000000..033f3eaf2 --- /dev/null +++ b/configfiles/RingCounting/RingCountingConfig @@ -0,0 +1,20 @@ +PythonScript RingCounting + +InitialiseFunction Initialise +ExecuteFunction Execute +FinaliseFunction Finalise + +verbose 1 + +# analysis specific settings +# +# Define file containing (multiple) filepath(s) to be loaded +files_to_load configfiles/RingCounting/files_to_load.txt +# Model version -> check documentation +version 1_0_0 +# Folder containing models +model_path /annie/app/users/dschmid/RingCountingStore/models/ +# Masked PMTs (to 0) config -> check documentation +pmt_mask november_22 +# Output file +save_to RC_output.csv diff --git a/configfiles/RingCounting/ToolChainConfig b/configfiles/RingCounting/ToolChainConfig new file mode 100644 index 000000000..d24302d64 --- /dev/null +++ b/configfiles/RingCounting/ToolChainConfig @@ -0,0 +1,26 @@ +#ToolChain dynamic setup file + +##### Runtime Parameters ##### +verbose 1 ## Verbosity level of ToolChain +error_level 0 # 0= do not exit, 1= exit on unhandled errors only, 2= exit on unhandled errors and handled errors +attempt_recover 1 ## 1= will attempt to finalise if an execute fails +remote_port 24002 +IO_Threads 1 ## Number of threads for network traffic (~ 1/Gbps) + +###### Logging ##### +log_mode Interactive # Interactive=cout , Remote= remote logging system "serservice_name Remote_Logging" , Local = local file log; +log_local_path ./log +log_service LogStore + + +###### Service discovery ##### Ignore these settings for local analysis +service_publish_sec -1 +service_kick_sec -1 + +##### Tools To Add ##### +Tools_File configfiles/RingCounting/ToolsConfig ## list of tools to run and their config files + +##### Run Type ##### +Inline 1 ## number of Execute steps in program, -1 infinite loop that is ended by user +Interactive 0 ## set to 1 if you want to run the code interactively + diff --git a/configfiles/RingCounting/ToolsConfig b/configfiles/RingCounting/ToolsConfig new file mode 100644 index 000000000..2e1af9c88 --- /dev/null +++ b/configfiles/RingCounting/ToolsConfig @@ -0,0 +1 @@ +myRingCounting PythonScript configfiles/RingCounting/RingCountingConfig diff --git a/configfiles/RingCounting/files_to_load.txt b/configfiles/RingCounting/files_to_load.txt new file mode 100644 index 000000000..ba0ee1c19 --- /dev/null +++ b/configfiles/RingCounting/files_to_load.txt @@ -0,0 +1,7 @@ +singleEntry.csv +singleEntry.csv +singleEntry.csv +singleEntry.csv +/annie/app/users/dschmid/ToolAnalysisDev/R1613p1_CNNImage_charge.csv +singleEntry.csv +R1613p1_CNNImage_charge.csv From eb4ae298ab82d259ee6efb58d71f683f2f07e17b Mon Sep 17 00:00:00 2001 From: s4294967296 Date: Fri, 17 Feb 2023 18:52:15 +0100 Subject: [PATCH 7/9] RingCounting.py: + logging of successful file loading + pep8 formatting --- UserTools/RingCounting/RingCounting.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/UserTools/RingCounting/RingCounting.py b/UserTools/RingCounting/RingCounting.py index 1537359b9..f47ca9497 100644 --- a/UserTools/RingCounting/RingCounting.py +++ b/UserTools/RingCounting/RingCounting.py @@ -75,8 +75,8 @@ class RingCounting(Tool, RingCountingGlobals): files_to_load = std.string() # List of files to be loaded (must be in CNNImage format) version = std.string() # Model version model_path = std.string() # Path to model directory - pmt_mask = std.string() # See RingCountingGlobals - save_to = std.string() # Where to save the predictions to + pmt_mask = std.string() # See RingCountingGlobals + save_to = std.string() # Where to save the predictions to # ---------------------------------------------------------------------------------------------------- # Model stuff @@ -99,12 +99,12 @@ def Initialise(self): # ---------------------------------------------------------------------------------------------------- # Config area self.m_variables.Get("files_to_load", self.files_to_load) - self.files_to_load = str(self.files_to_load) + self.files_to_load = str(self.files_to_load) # cast to str since std.string =/= str self.m_variables.Get("version", self.version) self.m_variables.Get("model_path", self.model_path) self.m_variables.Get("pmt_mask", self.pmt_mask) self.m_variables.Get("save_to", self.save_to) - self.save_to = str(self.save_to) + self.save_to = str(self.save_to) # cast to str since std.string =/= str self.pmt_mask = self.PMT_MASKS[self.pmt_mask] # ---------------------------------------------------------------------------------------------------- @@ -149,10 +149,15 @@ def load_data(self): if len(self.cnn_image_pmt) > 0: self.cnn_image_pmt = np.concatenate([arr, self.cnn_image_pmt], axis=0) + self.m_log.Log(__file__ + f" Successfully loaded CNNImage file {data_path} with {len(arr)} events.", + self.v_debug, self.m_verbosity) elif len(arr) > 0: + self.m_log.Log(__file__ + f" Successfully loaded CNNImage file {data_path} with {len(arr)} events.", + self.v_debug, self.m_verbosity) self.cnn_image_pmt = arr else: - self.m_log.Log("WARNING: " + __file__ + f" Attempted to load an empty PMT-datafile (CNNImage). ({data_path})", + self.m_log.Log("WARNING: " + __file__ + f" Attempted to load an empty PMT-datafile (CNNImage). " + f"({data_path})", self.v_debug, self.m_verbosity) def save_data(self): From 2fa64c017f44d2de3ca5c78ccc27da9027160b76 Mon Sep 17 00:00:00 2001 From: s4294967296 Date: Fri, 17 Feb 2023 19:06:39 +0100 Subject: [PATCH 8/9] Filled RingCountingTool README.md --- UserTools/RingCounting/README.md | 38 +++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/UserTools/RingCounting/README.md b/UserTools/RingCounting/README.md index a94e890ed..df32f38d8 100644 --- a/UserTools/RingCounting/README.md +++ b/UserTools/RingCounting/README.md @@ -1,20 +1,38 @@ # RingCounting -RingCounting +The RingCounting Tool is used to classify events into single- and multi-Cherenkov-ring-events. +For this a machine learning approach is used. +--- +This Tool uses PMT data in the CNNImage format (see UserTools/CNNImage). For further details on the tool and +the models used (including performance etc.) see the documentation found on the anniegpvm-machines at (**Todo**) +``` +/pnfs/annie/persistent/users/dschmid/RingCountingStore/documentation/ +``` +--- +All models can be found at +``` +/pnfs/annie/persistent/users/dschmid/RingCountingStore/models/ +``` +and (some) at +``` +/annie/app/users/dschmid/RingCountingStore/models/ +``` ## Data -Describe any data formats RingCounting creates, destroys, changes, or analyzes. E.G. - -**RawLAPPDData** `map>>` -* Takes this data from the `ANNIEEvent` store and finds the number of peaks - +- Currently does not add predictions to any BoostStore. This is planned in a future update (**Todo**) +--- ## Configuration -Describe any configuration variables for RingCounting. - +The following configuration variables must be set for the tool to function properly (further details) +are found at the top of the RingCounting.py file. +--- +Exemplary configuration: ``` -param1 value1 -param2 value2 +files_to_load configfiles/RingCounting/files_to_load.txt +version 1_0_0 +model_path /annie/app/users/dschmid/RingCountingStore/models/ +pmt_mask november_22 +save_to RC_output.csv ``` From 5fc781dc544f920d4e6501c10ef521f0d71d2c2a Mon Sep 17 00:00:00 2001 From: "Daniel T. Schmid" Date: Sun, 15 Oct 2023 08:20:25 -0500 Subject: [PATCH 9/9] Fixed mrdtotalentries being smaller than tanktotalentries throwing a fatal error in LoadRawData.cpp when build type is set to "MRD", since in this case only MRD data is expected to be loaded. --- UserTools/LoadRawData/LoadRawData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserTools/LoadRawData/LoadRawData.cpp b/UserTools/LoadRawData/LoadRawData.cpp index 0a266dff8..bbddfcde9 100644 --- a/UserTools/LoadRawData/LoadRawData.cpp +++ b/UserTools/LoadRawData/LoadRawData.cpp @@ -163,7 +163,7 @@ bool LoadRawData::Execute(){ } //If more MRD events than VME events abort - if (mrdtotalentries > tanktotalentries){ + if (mrdtotalentries > tanktotalentries && BuildType != "MRD"){ std::cout << "LoadRawData tool ERROR: More MRD entries than VME entries! Stopping toolchain" << std::endl; m_data->vars.Set("StopLoop",1); }