fix: solve smooks#204 & smooks#224 by parameterising the encoding in …

…the DFDL schema and setting it from the execution context with org.smooks.api.ExecutionContext#setContentEncoding
cjmamo · Sep 3, 2023 · a58682f · a58682f
1 parent 9aa25ac
commit a58682f
Show file tree

Hide file tree

Showing 21 changed files with 637 additions and 153 deletions.
diff --git a/common-schemas/src/main/resources/EDIFACT-Common/EDIFACT-Interchange.dfdl.xsd.mustache b/common-schemas/src/main/resources/EDIFACT-Common/EDIFACT-Interchange.dfdl.xsd.mustache
@@ -37,8 +37,8 @@
     <xsd:element name="Interchange" dfdl:lengthKind="implicit">
         <xsd:complexType>
             <xsd:sequence>
-                <xsd:element dfdl:terminator="%NL;%WSP*; %WSP*;" dfdl:initiator="UNA" dfdl:length="6" dfdl:lengthKind="explicit" name="UNA" type="srv:UNA" minOccurs="0"/>
-                <xsd:element dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNB" name="UNB" type="srv:UNB-InterchangeHeader">
+                <xsd:element name="UNA" type="srv:UNA" minOccurs="0" dfdl:terminator="%NL;%WSP*; %WSP*;" dfdl:initiator="UNA" dfdl:length="6" dfdl:lengthKind="explicit" dfdl:encoding="iso-8859-1"/>
+                <xsd:element name="UNB" type="srv:UNB-InterchangeHeader" dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNB">
                     <xsd:annotation>
                         <xsd:appinfo source="http://www.ogf.org/dfdl/">
                             <!-- Supports syntax versions 3 and 4 -->
@@ -51,23 +51,23 @@
                     <xsd:element name="FunctionGroup" maxOccurs="unbounded">
                         <xsd:complexType>
                             <xsd:sequence>
-                                <xsd:element dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNG" name="UNG" type="srv:UNG-GroupHeader"/>
+                                <xsd:element name="UNG" type="srv:UNG-GroupHeader" dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNG"/>
                                 <xsd:element ref="{{version}}:Message" maxOccurs="unbounded"/>
-                                <xsd:element dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNE" name="UNE" type="srv:UNE-GroupTrailer"/>
+                                <xsd:element name="UNE" type="srv:UNE-GroupTrailer" dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNE"/>
                             </xsd:sequence>
                         </xsd:complexType>
                     </xsd:element>
                     <xsd:element ref="{{version}}:Message" maxOccurs="unbounded"/>
                 </xsd:choice>
-                <xsd:element dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNZ" name="UNZ" type="srv:UNZ-InterchangeTrailer"/>
+                <xsd:element name="UNZ" type="srv:UNZ-InterchangeTrailer" dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNZ"/>
             </xsd:sequence>
         </xsd:complexType>
     </xsd:element>
 
     <xsd:element name="Message">
         <xsd:complexType>
             <xsd:sequence>
-                <xsd:element dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNH" name="UNH" type="srv:UNH-MessageHeader"/>
+                <xsd:element name="UNH" type="srv:UNH-MessageHeader" dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNH"/>
                 <xsd:choice>
                     <xsd:sequence>
                         <xsd:choice dfdl:choiceDispatchKey="{ ./UNH/S009/E0065 }">
@@ -94,11 +94,11 @@
                              </xsd:element>
                             {{/messageTypes}}
                         </xsd:choice>
-                        <xsd:element dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNT" name="UNT" type="srv:UNT-MessageTrailer"/>
+                        <xsd:element name="UNT" type="srv:UNT-MessageTrailer" dfdl:ref="ibmEdiFmt:EDISegmentFormat" dfdl:initiator="UNT"/>
                     </xsd:sequence>
                     <xsd:sequence>
                         <xsd:element ref="{{version}}:BadMessage"/>
-                        <xsd:element dfdl:initiator="UNT" dfdl:ref="ibmEdiFmt:EDISegmentFormat" name="UNT" type="srv:UNT-MessageTrailer"/>
+                        <xsd:element name="UNT" type="srv:UNT-MessageTrailer" dfdl:initiator="UNT" dfdl:ref="ibmEdiFmt:EDISegmentFormat"/>
                     </xsd:sequence>
                 </xsd:choice>
             </xsd:sequence>

diff --git a/common-schemas/src/main/resources/EDIFACT-Common/IBM_EDI_Format.dfdl.xsd b/common-schemas/src/main/resources/EDIFACT-Common/IBM_EDI_Format.dfdl.xsd
@@ -53,9 +53,10 @@
             <dfdl:defineVariable defaultValue="*" name="RepeatSep" type="xsd:string" external="true"/>
             <dfdl:defineVariable defaultValue="." name="DecimalSep" type="xsd:string" external="true"/>
             <dfdl:defineVariable defaultValue="," name="GroupingSep" type="xsd:string" external="true"/>
+            <dfdl:defineVariable defaultValue="utf-8" name="Encoding" type="xsd:string" external="true"/>
 
             <dfdl:defineFormat name="EDIFormat">
-                <dfdl:format encoding="iso-8859-1"
+                <dfdl:format encoding="{$ibmEdiFmt:Encoding}"
                              ignoreCase="yes"
                              byteOrder="bigEndian"
                              representation="text"

diff --git a/edg/src/main/resources/EDIFACT-Templates/EDIFACT-Messages.dfdl.xsd.mustache b/edg/src/main/resources/EDIFACT-Templates/EDIFACT-Messages.dfdl.xsd.mustache
@@ -90,7 +90,7 @@
                                     </xsd:appinfo>
                                 </xsd:annotation>
                             </xsd:element>
-                            <xsd:element name="Data" type="xsd:hexBinary" dfdl:lengthUnits="bytes"/>
+                            <xsd:element name="Data" type="xsd:hexBinary" dfdl:lengthUnits="bytes" dfdl:encoding="iso-8859-1"/>
                         </xsd:sequence>
                     </xsd:complexType>
                 </xsd:element>

diff --git a/edg/src/test/java/org/smooks/edi/edg/EdifactDfdlSchemaGeneratorTestCase.java b/edg/src/test/java/org/smooks/edi/edg/EdifactDfdlSchemaGeneratorTestCase.java
@@ -53,7 +53,9 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -80,21 +82,27 @@ public void testMain() throws Throwable {
 
     @Test
     public void testDfdlSchema() throws Throwable {
-        EdifactDfdlSchemaGenerator.main(new String[]{"/d03b.zip,org.smooks.edi.ect.formats.unedifact.parser.UnEdifactDirectoryParser", "target/generated-test-resources"});
+        generateSchema("D01B", Collections.singletonList("RECADV"));
+        generateSchema("D03B", Arrays.asList("INVOIC", "PAXLST"));
+
+        Runner runner = Runner.apply("parse.tdml");
+        runner.runOneTest("PAXLST", true);
+        runner.runOneTest("INVOIC", true);
+        runner.runOneTest("BAD-PAXLST", true);
+        runner.runOneTest("RECADV", true);
+    }
 
-        File generatedSchema = new File("target/generated-test-resources/d03b/EDIFACT-Interchange.dfdl.xsd");
+    private void generateSchema(String version, List<String> messageTypes) throws IOException {
+        EdifactDfdlSchemaGenerator.main(new String[]{String.format("/%s.zip,org.smooks.edi.ect.formats.unedifact.parser.UnEdifactDirectoryParser", version.toLowerCase()), "target/generated-test-resources"});
+
+        File generatedSchema = new File(String.format("target/generated-test-resources/%s/EDIFACT-Interchange.dfdl.xsd", version.toLowerCase()));
         Mustache mustache = new DefaultMustacheFactory().compile("EDIFACT-Common/EDIFACT-Interchange.dfdl.xsd.mustache");
         try (FileWriter fileWriter = new FileWriter(generatedSchema)) {
             mustache.execute(fileWriter, new HashMap<String, Object>() {{
                 this.put("schemaLocation", "EDIFACT-Messages.dfdl.xsd");
-                this.put("messageTypes", Arrays.asList("INVOIC", "PAXLST"));
-                this.put("version", "D03B");
+                this.put("messageTypes", messageTypes);
+                this.put("version", version);
             }});
         }
-
-        Runner runner = Runner.apply("parse.tdml");
-        runner.runOneTest("PAXLST", true);
-        runner.runOneTest("INVOIC", true);
-        runner.runOneTest("BAD-PAXLST", true);
     }
 }
diff --git a/edg/src/test/resources/d01b.zip b/edg/src/test/resources/d01b.zip
diff --git a/edg/src/test/resources/log4j2-test.xml b/edg/src/test/resources/log4j2-test.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ========================LICENSE_START=================================
+  smooks-edg
+  %%
+  Copyright (C) 2020 - 2023 Smooks
+  %%
+  Licensed under the terms of the Apache License Version 2.0, or
+  the GNU Lesser General Public License version 3.0 or later.
+  
+  SPDX-License-Identifier: Apache-2.0 OR LGPL-3.0-or-later
+  
+  ======================================================================
+  
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+  
+      http://www.apache.org/licenses/LICENSE-2.0
+  
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  
+  ======================================================================
+  
+  This program is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 3 of the License, or (at your option) any later version.
+  
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+  
+  You should have received a copy of the GNU Lesser General Public License
+  along with this program; if not, write to the Free Software Foundation,
+  Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+  =========================LICENSE_END==================================
+  -->
+
+<Configuration>
+    <Appenders>
+        <Console name="Console">
+            <PatternLayout pattern="%-5p %d [%t] %c: %m%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="info">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
+</Configuration>
diff --git a/edg/src/test/resources/parse.tdml b/edg/src/test/resources/parse.tdml
@@ -39,4 +39,14 @@
             <tdml:error>RFF occurred '2' times when it was expected to be a minimum of '0' and a maximum of '1' times.</tdml:error>
         </tdml:validationErrors>
     </tdml:parserTestCase>
+
+    <tdml:parserTestCase name="RECADV" validation="off" root="Interchange" model="target/generated-test-resources/d01b/EDIFACT-Interchange.dfdl.xsd">
+        <tdml:document>
+            <tdml:documentPart encoding="UTF-8" type="file">recadv/recadv.txt</tdml:documentPart>
+        </tdml:document>
+
+        <tdml:infoset>
+            <tdml:dfdlInfoset type="file">recadv/expected.xml</tdml:dfdlInfoset>
+        </tdml:infoset>
+    </tdml:parserTestCase>
 </tdml:testSuite>