[rework] ExternalXML in RAVEN Code Interface

doc/user_manual/existing_interfaces.tex

@@ -295,6 +295,12 @@ \subsection{RAVEN Interface}
 </Files>
 \end{lstlisting}
 
+\subsubsection{ExternalXML and RAVEN interface}
+Care must be taken if the SLAVE RAVEN uses \xmlNode{ExternalXML} nodes.  In this case, each file containing
+external XML nodes must be added in the \xmlNode{Step} as an \xmlNode{Input} class \xmlAttr{Files} to make sure it gets copied to
+the individual run directory.  The type for these files can be anything, with the exception of type
+\xmlString{raven}.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%% RELAP5  INTERFACE  %%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%

framework/CodeInterfaces/RAVEN/RAVENparser.py

@@ -30,6 +30,8 @@
 import numpy as np
 from collections import OrderedDict
 
+from utils import xmlUtils
+
 class RAVENparser():
   """
     Import the RAVEN input as xml tree, provide methods to add/change entries and print it back
@@ -51,6 +53,11 @@ def __init__(self, inputFile):
     except IOError as e:
       raise IOError(self.printTag+' ERROR: Input Parsing error!\n' +str(e)+'\n')
     self.tree = tree.getroot()
+
+    # expand the ExteranlXML nodes
+    cwd = os.path.dirname(inputFile)
+    xmlUtils.expandExternalXML(self.tree,cwd)
+
     # get the variable groups
     variableGroup = self.tree.find('VariableGroups')
     if variableGroup is not None:

framework/Driver.py

@@ -217,7 +217,8 @@ def checkVersions():
       sys.exit(1)
 
     # call the function to load the external xml files into the input tree
-    simulation.XMLpreprocess(root,inputFileName=inputFile)
+    cwd = os.path.dirname(os.path.abspath(inputFile))
+    simulation.XMLpreprocess(root,cwd)
     #generate all the components of the simulation
     #Call the function to read and construct each single module of the simulation
     simulation.XMLread(root,runInfoSkip=set(["DefaultInputFile"]),xmlFilename=inputFile)

framework/Simulation.py

@@ -47,8 +47,7 @@
 from JobHandler import JobHandler
 import MessageHandler
 import VariableGroups
-from utils import utils
-from utils import TreeStructure
+from utils import utils,TreeStructure,xmlUtils
 from Application import __QtAvailable
 from Interaction import Interaction
 if __QtAvailable:
@@ -370,49 +369,14 @@ def __createAbsPath(self,fileIn):
     path = os.path.normpath(self.runInfoDict['WorkingDir'])
     curfile.prependPath(path) #this respects existing path from the user input, if any
 
-  def ExternalXMLread(self,externalXMLFile,externalXMLNode,xmlFileName=None):
-    """
-      parses the external xml input file
-      @ In, externalXMLFile, string, the filename for the external xml file that will be loaded
-      @ In, externalXMLNode, string, decribes which node will be loaded to raven input file
-      @ In, xmlFileName, string, optional, the raven input file name
-      @ Out, externalElemment, xml.etree.ElementTree.Element, object that will be added to the current tree of raven input
-    """
-    #TODO make one for getpot too
-    if '~' in externalXMLFile:
-      externalXMLFile = os.path.expanduser(externalXMLFile)
-    if not os.path.isabs(externalXMLFile):
-      if xmlFileName == None:
-        self.raiseAnError(IOError,'Relative working directory requested but input xmlFileName is None.')
-      xmlDirectory = os.path.dirname(os.path.abspath(xmlFileName))
-      externalXMLFile = os.path.join(xmlDirectory,externalXMLFile)
-    if os.path.exists(externalXMLFile):
-      externalTree = TreeStructure.parse(externalXMLFile)
-      externalElement = externalTree.getroot()
-      if externalElement.tag != externalXMLNode:
-        self.raiseAnError(IOError,'The required node is: ' + externalXMLNode + 'is different from the provided external xml type: ' + externalElement.tag)
-    else:
-      self.raiseAnError(IOError,'The external xml input file ' + externalXMLFile + ' does not exist!')
-    return externalElement
-
-  def XMLpreprocess(self,node,inputFileName=None):
+  def XMLpreprocess(self,node,cwd):
     """
       Preprocess the input file, load external xml files into the main ET
       @ In, node, TreeStructure.InputNode, element of RAVEN input file
-      @ In, inputFileName, string, optional, the raven input file name
+      @ In, cwd, string, current working directory (for relative path searches)
       @ Out, None
     """
-    self.verbosity = node.attrib.get('verbosity','all').lower()
-    for element in node.iter():
-      for subElement in element:
-        if subElement.tag == 'ExternalXML':
-          self.raiseADebug('-'*2+' Loading external xml within block '+ element.tag+ ' for: {0:15}'.format(str(subElement.attrib['node']))+2*'-')
-          nodeName = subElement.attrib['node']
-          xmlToLoad = subElement.attrib['xmlToLoad'].strip()
-          newElement = self.ExternalXMLread(xmlToLoad,nodeName,inputFileName)
-          element.append(newElement)
-          element.remove(subElement)
-          self.XMLpreprocess(node,inputFileName)
+    xmlUtils.expandExternalXML(node,cwd)
 
   def XMLread(self,xmlNode,runInfoSkip = set(),xmlFilename=None):
     """

framework/utils/TreeStructure.py

@@ -435,6 +435,16 @@ def __getitem__(self, index):
     """
     return self.children[index]
 
+  def __setitem__(self,index,value):
+    """
+      Sets a specific child node.
+      @ In, index, int, the index for the child
+      @ In, value, Node, the child itself
+      @ Out, None
+    """
+    value = self.assureIsNode(value)
+    self.children[index] = value
+
   def __repr__(self):
     """
       String representation.
@@ -468,7 +478,7 @@ def append(self,node):
       @ In, node, Node, node to append to children
       @ Out, None
     """
-    assert isinstance(node,InputNode)
+    node = self.assureIsNode(node)
     self.children.append(node)
 
   def find(self,nodeName):
@@ -524,6 +534,21 @@ def iter(self, name=None):
       for e in e.iter(name):
         yield e
 
+  def assureIsNode(self,node):
+    """
+      Takes care of translating XML to Node on demand.
+      @ In, node, Node or ET.Element, node to fix up
+      @ Out, node, fixed node
+    """
+    if not isinstance(node,InputNode):
+      # if XML, convert to InputNode
+      if isinstance(node,ET.Element):
+        tree = ET.ElementTree(node)
+        node = xmlToInputTree(tree).getroot()
+      else:
+        raise TypeError('TREE-STRUCTURE ERROR: When trying to use node "{}", unrecognized type "{}"!'.format(node,type(node)))
+    return node
+
   def printXML(self):
     """
       Returns string representation of tree (in XML format).

framework/utils/xmlUtils.py

@@ -308,3 +308,43 @@ def fixXmlTag(msg):
     print('XML UTILS: Prepending "_" to illegal tag "'+msg+'"')
     msg = '_' + msg
   return msg
+
+def expandExternalXML(root,workingDir):
+  """
+    Expands "ExternalXML" nodes with the associated nodes and returns the full tree.
+    @ In, root, xml.etree.ElementTree.Element, main node whose children might be ExternalXML nodes
+    @ In, workingDir, string, base location from which to find additional xml files
+    @ Out, None
+  """
+  # find instances of ExteranlXML nodes to replace
+  for i,subElement in enumerate(root):
+    if subElement.tag == 'ExternalXML':
+      nodeName = subElement.attrib['node']
+      xmlToLoad = subElement.attrib['xmlToLoad'].strip()
+      newElement = readExternalXML(xmlToLoad,nodeName,workingDir)
+      root[i] = newElement
+      subElement = newElement
+    # whether expanded or not, search each subnodes for more external xml
+    expandExternalXML(subElement,workingDir)
+
+def readExternalXML(extFile,extNode,cwd):
+  """
+    Loads external XML into nodes.
+    @ In, extFile, string, filename for the external xml file
+    @ In, extNode, string, tag of node to load
+    @ In, cwd, string, current working directory (for relative paths)
+    @ Out, externalElement, xml.etree.ElementTree.Element, object from file
+  """
+  # expand user tilde
+  if '~' in extFile:
+    extFile = os.path.expanduser(extFile)
+  # check if absolute or relative found
+  if not os.path.isabs(extFile):
+    extFile = os.path.join(cwd,extFile)
+  if not os.path.exists(extFile):
+    raise IOError('XML UTILS ERROR: External XML file not found: "{}"'.format(os.path.abspath(extFile)))
+  # find the element to read
+  root = ET.parse(open(extFile,'r')).getroot()
+  if root.tag != extNode.strip():
+    raise IOError('XML UTILS ERROR: Node "{}" is not the root node of "{}"!'.format(extNode,extFile))
+  return root

tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/ext_dataobjects.xml

@@ -0,0 +1,34 @@
+<DataObjects>
+	<PointSet name="inputHolder">
+		<Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
+		<Output>OutputPlaceHolder</Output>
+	</PointSet>
+	<PointSet name="Pointset_from_database_for_rom_trainer">
+		<Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
+		<Output>CladTempThreshold,time</Output>
+	</PointSet>
+	<HistorySet name="Historyset_from_database_for_rom_trainer">
+		<Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
+		<Output>CladTempThreshold</Output>
+	</HistorySet>
+	<PointSet name="data_for_sampling_empty_at_begin">
+		<Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
+		<Output>OutputPlaceHolder</Output>
+	</PointSet>
+	<PointSet name="data_for_sampling_empty_at_begin_nd">
+		<Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
+		<Output>OutputPlaceHolder</Output>
+	</PointSet>
+	<PointSet name="outputMontecarloRom">
+		<Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
+		<!--Output>CladTempThreshold</Output-->
+	</PointSet>
+	<HistorySet name="outputMontecarloRomHS">
+			<Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
+			<Output>CladTempThreshold,time</Output>
+	</HistorySet>
+	<PointSet name="outputMontecarloRomND">
+		<Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
+		<Output>CladTempThreshold</Output>
+	</PointSet>
+</DataObjects>

tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/test_rom_trainer.xml

@@ -169,39 +169,6 @@
     <HDF5 name="MC_TEST_EXTRACT_STEP_FOR_ROM_TRAINER" readMode="overwrite"/>
   </Databases>
 
-  <DataObjects>
-    <PointSet name="inputHolder">
-      <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <Output>OutputPlaceHolder</Output>
-    </PointSet>
-    <PointSet name="Pointset_from_database_for_rom_trainer">
-      <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <Output>CladTempThreshold,time</Output>
-    </PointSet>
-    <HistorySet name="Historyset_from_database_for_rom_trainer">
-      <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <Output>CladTempThreshold</Output>
-    </HistorySet>
-    <PointSet name="data_for_sampling_empty_at_begin">
-      <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <Output>OutputPlaceHolder</Output>
-    </PointSet>
-    <PointSet name="data_for_sampling_empty_at_begin_nd">
-      <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <Output>OutputPlaceHolder</Output>
-    </PointSet>
-    <PointSet name="outputMontecarloRom">
-      <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <!--Output>CladTempThreshold</Output-->
-    </PointSet>
-    <HistorySet name="outputMontecarloRomHS">
-        <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-        <Output>CladTempThreshold,time</Output>
-    </HistorySet>
-    <PointSet name="outputMontecarloRomND">
-      <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <Output>CladTempThreshold</Output>
-    </PointSet>
-  </DataObjects>
+  <ExternalXML node="DataObjects" xmlToLoad="../ext_dataobjects.xml"/>
 
 </Simulation>

tests/framework/CodeInterfaceTests/test_raven_running_raven_int_models.xml

@@ -23,12 +23,14 @@
 
 <Files>
     <Input name="test_rom_trainer.xml" type="raven" >test_rom_trainer.xml</Input>
+    <Input name="ext_dataobjects.xml" type="ExternalXML" >ext_dataobjects.xml</Input>
 </Files>
 
   <!-- STEPS -->
   <Steps>
     <MultiRun name="FirstMRun" re-seeding="1">
       <Input class="Files" type="raven">test_rom_trainer.xml</Input>
+      <Input class="Files" type="raven">ext_dataobjects.xml</Input>
       <Model class="Models" type="Code">raven_running_rom</Model>
       <Sampler class="Samplers" type="MonteCarlo">MC_external</Sampler>
       <Output class="DataObjects" type="HistorySet">testPrintHistorySet</Output>

tests/framework/utils/testXmlUtils.py

@@ -253,6 +253,7 @@ def attemptFileClear(fName,later):
 found = xmlUtils.findPathEllipsesParents(xmlTree.getroot(),'child/cchild')
 print ('ellipses')
 print(xmlUtils.prettify(found,doc=True))
+# TODO is there supposed to be a test here?
 
 #test bad XML tags
 # rule 1: only start with letter or underscore, can't start with xml
@@ -284,6 +285,68 @@ def attemptFileClear(fName,later):
     print('ERROR: Fixing legal XML tag "'+ok+'" FAILED:',fixed,'should be',ok)
     results['fail']+=1
 
+
+# test readExternalXML, use relative path
+extFile = 'GoodExternalXMLFile.xml'
+extNode = 'testMainNode'
+cwd = os.path.join(os.path.dirname(__file__),'xml')
+node = xmlUtils.readExternalXML(extFile,extNode,cwd)
+strNode = """<testMainNode att=\"attrib1\">
+  <firstSubNode>
+    <firstFirstSubNode att=\"attrib1.1.1\">firstFirstSubText</firstFirstSubNode>
+  </firstSubNode>
+  <secondSubNode att=\"attrib1.2\">
+    <secondFirstSubNode>secondFirstSubText</secondFirstSubNode>
+  </secondSubNode>
+</testMainNode>"""
+if strNode != ET.tostring(node):
+  print('ERROR: loaded XML node:')
+  print(ET.tostring(node))
+  print(' ----- does not match expected:')
+  print(strNode)
+  results['fail']+=1
+else:
+  results['pass']+=1
+
+
+# test expandExternalXML, two substitutions
+strNode = """<root>
+  <ExternalXML node="testMainNode" xmlToLoad="GoodExternalXMLFile.xml"/>
+  <rootsub>
+    <ExternalXML node="testMainNode" xmlToLoad="GoodExternalXMLFile.xml"/>
+  </rootsub>
+</root>
+"""
+root = ET.fromstring(strNode)
+cwd = os.path.join(os.path.dirname(__file__),'xml')
+xmlUtils.expandExternalXML(root,cwd)
+correct = """<root>
+  <testMainNode att="attrib1">
+  <firstSubNode>
+    <firstFirstSubNode att="attrib1.1.1">firstFirstSubText</firstFirstSubNode>
+  </firstSubNode>
+  <secondSubNode att="attrib1.2">
+    <secondFirstSubNode>secondFirstSubText</secondFirstSubNode>
+  </secondSubNode>
+</testMainNode><rootsub>
+    <testMainNode att="attrib1">
+  <firstSubNode>
+    <firstFirstSubNode att="attrib1.1.1">firstFirstSubText</firstFirstSubNode>
+  </firstSubNode>
+  <secondSubNode att="attrib1.2">
+    <secondFirstSubNode>secondFirstSubText</secondFirstSubNode>
+  </secondSubNode>
+</testMainNode></rootsub>
+</root>"""
+if correct != ET.tostring(root):
+  print('ERROR: expanded XML node:')
+  print(ET.tostring(root))
+  print(' ----- does not match expected:')
+  print(correct)
+  results['fail']+=1
+else:
+  results['pass']+=1
+
 print(results)
 
 sys.exit(results["fail"])

tests/framework/utils/xml/GoodExternalXMLFile.xml

@@ -0,0 +1,8 @@
+<testMainNode att="attrib1">
+  <firstSubNode>
+    <firstFirstSubNode att="attrib1.1.1">firstFirstSubText</firstFirstSubNode>
+  </firstSubNode>
+  <secondSubNode att="attrib1.2">
+    <secondFirstSubNode>secondFirstSubText</secondFirstSubNode>
+  </secondSubNode>
+</testMainNode>