diff --git a/doc/user_manual/existing_interfaces.tex b/doc/user_manual/existing_interfaces.tex index a277e47ba6..32b6d5179a 100755 --- a/doc/user_manual/existing_interfaces.tex +++ b/doc/user_manual/existing_interfaces.tex @@ -83,7 +83,7 @@ \subsection{Generic Interface} the GenericCode interface can be invoked using the \xmlNode{outputFile} node in which the output file name (CSV only) must be specified. For example, in the previous example, say instead of \texttt{-a gen.two} and \texttt{-o myOut} -in the command line, the code always produce a CSV file named ``fixed@output.csv''; +in the command line, the code always produce a CSV file named ``fixed@output.csv''; Then, our example XML for the code would be @@ -134,9 +134,9 @@ \subsection{RAVEN Interface} \label{subsec:RAVENInterface} The RAVEN interface is meant to provide the possibility to execute a RAVEN input file driving a set of SLAVE RAVEN calculations. For example, if the user wants to optimize the parameters -of a surrogate model (e.g. minimizing the distance between the surrogate predictions and the real data), he +of a surrogate model (e.g. minimizing the distance between the surrogate predictions and the real data), he can achieve this task by setting up a RAVEN input file (master) that performs an optimization on the feature -space characterized by the surrogate model parameters, whose training and validation assessment is performed in the SLAVE +space characterized by the surrogate model parameters, whose training and validation assessment is performed in the SLAVE RAVEN runs. \\ There are some limitations for this interface: \begin{itemize} @@ -150,7 +150,7 @@ \subsection{RAVEN Interface} \\ Similarly to any other code interface, the user provides paths to executables and aliases for sampled variables within the \xmlNode{Models} block. The \xmlNode{Code} block will contain attributes \xmlAttr{name} and \xmlAttr{subType}. \xmlAttr{name} identifies that particular \xmlNode{Code} model within RAVEN, and -\xmlAttr{subType} specifies which code interface the model will use (In this case \xmlAttr{subType}=``RAVEN''). +\xmlAttr{subType} specifies which code interface the model will use (In this case \xmlAttr{subType}=``RAVEN''). The \xmlNode{executable} block should contain the absolute or relative (with respect to the current working directory) path to the RAVEN framework script (\textbf{raven\_framework}). @@ -167,37 +167,37 @@ \subsection{RAVEN Interface} \begin{lstlisting}[language=python] def manipulateScalarSampledVariables(sampledVariables): """ - This method is aimed to manipulate scalar variables. - The user can create new variables based on the + This method is aimed to manipulate scalar variables. + The user can create new variables based on the variables sampled by RAVEN - @ In, sampledVariables, dict, dictionary of + @ In, sampledVariables, dict, dictionary of sampled variables ({"var1":value1,"var2":value2}) - @ Out, None, the new variables should be + @ Out, None, the new variables should be added in the "sampledVariables" dictionary """ - newVariableValue = - sampledVariables['Distributions|Uniform@name:a_dist|lowerBound'] + newVariableValue = + sampledVariables['Distributions|Uniform@name:a_dist|lowerBound'] + 1.0 - sampledVariables['Distributions|Uniform@name:a_dist|upperBound'] = + sampledVariables['Distributions|Uniform@name:a_dist|upperBound'] = newVariableValue return \end{lstlisting} - \item \textbf{\textit{convertNotScalarSampledVariables}}, a method that is aimed to convert not scalar variables (e.g. 1D arrays) into multiple scalar variables + \item \textbf{\textit{convertNotScalarSampledVariables}}, a method that is aimed to convert not scalar variables (e.g. 1D arrays) into multiple scalar variables (e.g. \xmlNode{constant}(s) in a sampling strategy). - This method is going to be required in case not scalar variables are detected by the interface. + This method is going to be required in case not scalar variables are detected by the interface. Example: \begin{lstlisting}[language=python] def convertNotScalarSampledVariables(noScalarVariables): """ - This method is aimed to convert not scalar + This method is aimed to convert not scalar variables into multiple scalar variables. The user MUST create new variables based on the not Scalar Variables sampled (and passed in) by RAVEN - @ In, noScalarVariables, dict, dictionary of sampled + @ In, noScalarVariables, dict, dictionary of sampled variables that are not scalar ({"var1":1Darray1,"var2":1Darray2}) - @ Out, newVars, dict, the new variables that have - been created based on the not scalar variables + @ Out, newVars, dict, the new variables that have + been created based on the not scalar variables contained in "noScalarVariables" dictionary """ oneDimensionalArray = @@ -205,7 +205,7 @@ \subsection{RAVEN Interface} newVars = {} for cnt, value in enumerate(oneDimensionalArray): newVars['Samplers|MonteCarlo@name:myMC|constant'+ - '@name=temperatureHistory'+str(cnt)] = + '@name=temperatureHistory'+str(cnt)] = oneDimensionalArray[cnt] return newVars \end{lstlisting} @@ -225,7 +225,7 @@ \subsection{RAVEN Interface} \end{lstlisting} -Like for every other interface, the syntax of the variable names is important to make the parser understand how to perturb an input file. +Like for every other interface, the syntax of the variable names is important to make the parser understand how to perturb an input file. \\ For the RAVEN interface, a syntax inspired by the XPath nomenclature is used. \begin{lstlisting}[style=XML] @@ -251,7 +251,7 @@ \subsection{RAVEN Interface} \begin{lstlisting}[style=XML] - ... + ... 10.0 ... @@ -261,7 +261,7 @@ \subsection{RAVEN Interface} \begin{lstlisting}[style=XML] - ... + ... 0.0001 ... @@ -275,9 +275,9 @@ \subsection{RAVEN Interface} ... 0 1 - ... + ... - + ... @@ -295,6 +295,12 @@ \subsection{RAVEN Interface} \end{lstlisting} +\subsubsection{ExternalXML and RAVEN interface} +Care must be taken if the SLAVE RAVEN uses \xmlNode{ExternalXML} nodes. In this case, each file containing +external XML nodes must be added in the \xmlNode{Step} as an \xmlNode{Input} class \xmlAttr{Files} to make sure it gets copied to +the individual run directory. The type for these files can be anything, with the exception of type +\xmlString{raven}. + %%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%% RELAP5 INTERFACE %%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1303,20 +1309,20 @@ \subsubsection{Models} ... \end{lstlisting} -RAVEN works best with Comma-Separated Value (CSV) files. Therefore, the default +RAVEN works best with Comma-Separated Value (CSV) files. Therefore, the default .mat output type needs to be converted to .csv output. -The Dymola interface will automatically convert the .mat output to human-readable +The Dymola interface will automatically convert the .mat output to human-readable forms, i.e., .csv output, through its implementation of the finalizeCodeOutput function. \\In order to speed up the reading and conversion of the .mat file, the user can specify -the list of variables (in addition to the Time variable) that need to be imported and -converted into a csv file minimizing -the IO memory usage as much as possible. Within the \xmlNode{Code} the following -XML +the list of variables (in addition to the Time variable) that need to be imported and +converted into a csv file minimizing +the IO memory usage as much as possible. Within the \xmlNode{Code} the following +XML node (in addition ot the \xmlNode{executable} one) can be inputted: \begin{itemize} - \item \xmlNode{outputVariablesToLoad}, \xmlDesc{space separated list, optional - parameter}, a space separated list of variables that need be exported from the .mat + \item \xmlNode{outputVariablesToLoad}, \xmlDesc{space separated list, optional + parameter}, a space separated list of variables that need be exported from the .mat file (in addition to the Time variable). \default{all the variables in the .mat file}. \end{itemize} For example: diff --git a/framework/CodeInterfaces/RAVEN/RAVENparser.py b/framework/CodeInterfaces/RAVEN/RAVENparser.py index 534372b528..518aaba73f 100644 --- a/framework/CodeInterfaces/RAVEN/RAVENparser.py +++ b/framework/CodeInterfaces/RAVEN/RAVENparser.py @@ -30,6 +30,8 @@ import numpy as np from collections import OrderedDict +from utils import xmlUtils + class RAVENparser(): """ Import the RAVEN input as xml tree, provide methods to add/change entries and print it back @@ -51,6 +53,11 @@ def __init__(self, inputFile): except IOError as e: raise IOError(self.printTag+' ERROR: Input Parsing error!\n' +str(e)+'\n') self.tree = tree.getroot() + + # expand the ExteranlXML nodes + cwd = os.path.dirname(inputFile) + xmlUtils.expandExternalXML(self.tree,cwd) + # get the variable groups variableGroup = self.tree.find('VariableGroups') if variableGroup is not None: diff --git a/framework/Driver.py b/framework/Driver.py index f117ddf7e4..c2e809f6d3 100755 --- a/framework/Driver.py +++ b/framework/Driver.py @@ -217,7 +217,8 @@ def checkVersions(): sys.exit(1) # call the function to load the external xml files into the input tree - simulation.XMLpreprocess(root,inputFileName=inputFile) + cwd = os.path.dirname(os.path.abspath(inputFile)) + simulation.XMLpreprocess(root,cwd) #generate all the components of the simulation #Call the function to read and construct each single module of the simulation simulation.XMLread(root,runInfoSkip=set(["DefaultInputFile"]),xmlFilename=inputFile) diff --git a/framework/Simulation.py b/framework/Simulation.py index d83a32e7f4..197de024a5 100644 --- a/framework/Simulation.py +++ b/framework/Simulation.py @@ -47,8 +47,7 @@ from JobHandler import JobHandler import MessageHandler import VariableGroups -from utils import utils -from utils import TreeStructure +from utils import utils,TreeStructure,xmlUtils from Application import __QtAvailable from Interaction import Interaction if __QtAvailable: @@ -370,49 +369,14 @@ def __createAbsPath(self,fileIn): path = os.path.normpath(self.runInfoDict['WorkingDir']) curfile.prependPath(path) #this respects existing path from the user input, if any - def ExternalXMLread(self,externalXMLFile,externalXMLNode,xmlFileName=None): - """ - parses the external xml input file - @ In, externalXMLFile, string, the filename for the external xml file that will be loaded - @ In, externalXMLNode, string, decribes which node will be loaded to raven input file - @ In, xmlFileName, string, optional, the raven input file name - @ Out, externalElemment, xml.etree.ElementTree.Element, object that will be added to the current tree of raven input - """ - #TODO make one for getpot too - if '~' in externalXMLFile: - externalXMLFile = os.path.expanduser(externalXMLFile) - if not os.path.isabs(externalXMLFile): - if xmlFileName == None: - self.raiseAnError(IOError,'Relative working directory requested but input xmlFileName is None.') - xmlDirectory = os.path.dirname(os.path.abspath(xmlFileName)) - externalXMLFile = os.path.join(xmlDirectory,externalXMLFile) - if os.path.exists(externalXMLFile): - externalTree = TreeStructure.parse(externalXMLFile) - externalElement = externalTree.getroot() - if externalElement.tag != externalXMLNode: - self.raiseAnError(IOError,'The required node is: ' + externalXMLNode + 'is different from the provided external xml type: ' + externalElement.tag) - else: - self.raiseAnError(IOError,'The external xml input file ' + externalXMLFile + ' does not exist!') - return externalElement - - def XMLpreprocess(self,node,inputFileName=None): + def XMLpreprocess(self,node,cwd): """ Preprocess the input file, load external xml files into the main ET @ In, node, TreeStructure.InputNode, element of RAVEN input file - @ In, inputFileName, string, optional, the raven input file name + @ In, cwd, string, current working directory (for relative path searches) @ Out, None """ - self.verbosity = node.attrib.get('verbosity','all').lower() - for element in node.iter(): - for subElement in element: - if subElement.tag == 'ExternalXML': - self.raiseADebug('-'*2+' Loading external xml within block '+ element.tag+ ' for: {0:15}'.format(str(subElement.attrib['node']))+2*'-') - nodeName = subElement.attrib['node'] - xmlToLoad = subElement.attrib['xmlToLoad'].strip() - newElement = self.ExternalXMLread(xmlToLoad,nodeName,inputFileName) - element.append(newElement) - element.remove(subElement) - self.XMLpreprocess(node,inputFileName) + xmlUtils.expandExternalXML(node,cwd) def XMLread(self,xmlNode,runInfoSkip = set(),xmlFilename=None): """ diff --git a/framework/utils/TreeStructure.py b/framework/utils/TreeStructure.py index daf8865706..e43a53d955 100644 --- a/framework/utils/TreeStructure.py +++ b/framework/utils/TreeStructure.py @@ -435,6 +435,16 @@ def __getitem__(self, index): """ return self.children[index] + def __setitem__(self,index,value): + """ + Sets a specific child node. + @ In, index, int, the index for the child + @ In, value, Node, the child itself + @ Out, None + """ + value = self.assureIsNode(value) + self.children[index] = value + def __repr__(self): """ String representation. @@ -468,7 +478,7 @@ def append(self,node): @ In, node, Node, node to append to children @ Out, None """ - assert isinstance(node,InputNode) + node = self.assureIsNode(node) self.children.append(node) def find(self,nodeName): @@ -524,6 +534,21 @@ def iter(self, name=None): for e in e.iter(name): yield e + def assureIsNode(self,node): + """ + Takes care of translating XML to Node on demand. + @ In, node, Node or ET.Element, node to fix up + @ Out, node, fixed node + """ + if not isinstance(node,InputNode): + # if XML, convert to InputNode + if isinstance(node,ET.Element): + tree = ET.ElementTree(node) + node = xmlToInputTree(tree).getroot() + else: + raise TypeError('TREE-STRUCTURE ERROR: When trying to use node "{}", unrecognized type "{}"!'.format(node,type(node))) + return node + def printXML(self): """ Returns string representation of tree (in XML format). diff --git a/framework/utils/xmlUtils.py b/framework/utils/xmlUtils.py index 6afbcdd49c..9db553f179 100644 --- a/framework/utils/xmlUtils.py +++ b/framework/utils/xmlUtils.py @@ -308,3 +308,43 @@ def fixXmlTag(msg): print('XML UTILS: Prepending "_" to illegal tag "'+msg+'"') msg = '_' + msg return msg + +def expandExternalXML(root,workingDir): + """ + Expands "ExternalXML" nodes with the associated nodes and returns the full tree. + @ In, root, xml.etree.ElementTree.Element, main node whose children might be ExternalXML nodes + @ In, workingDir, string, base location from which to find additional xml files + @ Out, None + """ + # find instances of ExteranlXML nodes to replace + for i,subElement in enumerate(root): + if subElement.tag == 'ExternalXML': + nodeName = subElement.attrib['node'] + xmlToLoad = subElement.attrib['xmlToLoad'].strip() + newElement = readExternalXML(xmlToLoad,nodeName,workingDir) + root[i] = newElement + subElement = newElement + # whether expanded or not, search each subnodes for more external xml + expandExternalXML(subElement,workingDir) + +def readExternalXML(extFile,extNode,cwd): + """ + Loads external XML into nodes. + @ In, extFile, string, filename for the external xml file + @ In, extNode, string, tag of node to load + @ In, cwd, string, current working directory (for relative paths) + @ Out, externalElement, xml.etree.ElementTree.Element, object from file + """ + # expand user tilde + if '~' in extFile: + extFile = os.path.expanduser(extFile) + # check if absolute or relative found + if not os.path.isabs(extFile): + extFile = os.path.join(cwd,extFile) + if not os.path.exists(extFile): + raise IOError('XML UTILS ERROR: External XML file not found: "{}"'.format(os.path.abspath(extFile))) + # find the element to read + root = ET.parse(open(extFile,'r')).getroot() + if root.tag != extNode.strip(): + raise IOError('XML UTILS ERROR: Node "{}" is not the root node of "{}"!'.format(extNode,extFile)) + return root diff --git a/tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/ext_dataobjects.xml b/tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/ext_dataobjects.xml new file mode 100644 index 0000000000..d303cfcbd1 --- /dev/null +++ b/tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/ext_dataobjects.xml @@ -0,0 +1,30 @@ + + + DeltaTimeScramToAux,DG1recoveryTime + OutputPlaceHolder + + + DeltaTimeScramToAux,DG1recoveryTime + CladTempThreshold + + + DeltaTimeScramToAux,DG1recoveryTime + OutputPlaceHolder + + + DeltaTimeScramToAux,DG1recoveryTime + OutputPlaceHolder + + + DeltaTimeScramToAux,DG1recoveryTime + CladTempThreshold + + + DeltaTimeScramToAux,DG1recoveryTime + CladTempThreshold + + + DeltaTimeScramToAux,DG1recoveryTime + CladTempThreshold + + diff --git a/tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/test_rom_trainer.xml b/tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/test_rom_trainer.xml index 3747bac653..65139c16ed 100644 --- a/tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/test_rom_trainer.xml +++ b/tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/test_rom_trainer.xml @@ -141,35 +141,6 @@ - - - DeltaTimeScramToAux,DG1recoveryTime - OutputPlaceHolder - - - DeltaTimeScramToAux,DG1recoveryTime - CladTempThreshold - - - DeltaTimeScramToAux,DG1recoveryTime - OutputPlaceHolder - - - DeltaTimeScramToAux,DG1recoveryTime - OutputPlaceHolder - - - DeltaTimeScramToAux,DG1recoveryTime - CladTempThreshold - - - DeltaTimeScramToAux,DG1recoveryTime - CladTempThreshold - - - DeltaTimeScramToAux,DG1recoveryTime - CladTempThreshold - - + diff --git a/tests/framework/CodeInterfaceTests/test_raven_running_raven_int_models.xml b/tests/framework/CodeInterfaceTests/test_raven_running_raven_int_models.xml index bbeb424e06..5b1466ceb9 100644 --- a/tests/framework/CodeInterfaceTests/test_raven_running_raven_int_models.xml +++ b/tests/framework/CodeInterfaceTests/test_raven_running_raven_int_models.xml @@ -23,12 +23,14 @@ test_rom_trainer.xml + ext_dataobjects.xml test_rom_trainer.xml + ext_dataobjects.xml raven_running_rom MC_external testPrintHistorySet @@ -48,7 +50,7 @@ Models|ROM@subType:SciKitLearn@name:ROM1|C Models|ROM@subType:SciKitLearn@name:ROM1|tol Samplers|Grid@name:gridRom|constant@name:DG1recoveryTime - + diff --git a/tests/framework/utils/testXmlUtils.py b/tests/framework/utils/testXmlUtils.py index 80d6fa1ad7..4c3fcd3672 100644 --- a/tests/framework/utils/testXmlUtils.py +++ b/tests/framework/utils/testXmlUtils.py @@ -253,6 +253,7 @@ def attemptFileClear(fName,later): found = xmlUtils.findPathEllipsesParents(xmlTree.getroot(),'child/cchild') print ('ellipses') print(xmlUtils.prettify(found,doc=True)) +# TODO is there supposed to be a test here? #test bad XML tags # rule 1: only start with letter or underscore, can't start with xml @@ -284,6 +285,68 @@ def attemptFileClear(fName,later): print('ERROR: Fixing legal XML tag "'+ok+'" FAILED:',fixed,'should be',ok) results['fail']+=1 + +# test readExternalXML, use relative path +extFile = 'GoodExternalXMLFile.xml' +extNode = 'testMainNode' +cwd = os.path.join(os.path.dirname(__file__),'xml') +node = xmlUtils.readExternalXML(extFile,extNode,cwd) +strNode = """ + + firstFirstSubText + + + secondFirstSubText + +""" +if strNode != ET.tostring(node): + print('ERROR: loaded XML node:') + print(ET.tostring(node)) + print(' ----- does not match expected:') + print(strNode) + results['fail']+=1 +else: + results['pass']+=1 + + +# test expandExternalXML, two substitutions +strNode = """ + + + + + +""" +root = ET.fromstring(strNode) +cwd = os.path.join(os.path.dirname(__file__),'xml') +xmlUtils.expandExternalXML(root,cwd) +correct = """ + + + firstFirstSubText + + + secondFirstSubText + + + + + firstFirstSubText + + + secondFirstSubText + + +""" +if correct != ET.tostring(root): + print('ERROR: expanded XML node:') + print(ET.tostring(root)) + print(' ----- does not match expected:') + print(correct) + results['fail']+=1 +else: + results['pass']+=1 + print(results) sys.exit(results["fail"]) diff --git a/tests/framework/utils/xml/GoodExternalXMLFile.xml b/tests/framework/utils/xml/GoodExternalXMLFile.xml new file mode 100644 index 0000000000..48e428fb10 --- /dev/null +++ b/tests/framework/utils/xml/GoodExternalXMLFile.xml @@ -0,0 +1,8 @@ + + + firstFirstSubText + + + secondFirstSubText + +