Skip to content

Commit

Permalink
feat: set DFDL variables during execution instead of start-up so that…
Browse files Browse the repository at this point in the history 
… the content-encoding can be set in the EDIFACT cartridge and fix smooks/smooks-edi-cartridge#224 and smooks/smooks-edi-cartridge#204 (smooks#150)
  • Loading branch information
@cjmamo
cjmamo authored Sep 3, 2023
1 parent b844241 commit 841b529
Show file tree
Hide file tree
Showing 9 changed files with 118 additions and 58 deletions.
14 changes: 1 addition & 13 deletions src/main/java/org/smooks/cartridges/dfdl/DataProcessorFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,11 @@
import org.slf4j.LoggerFactory;
import org.smooks.api.ApplicationContext;
import org.smooks.api.SmooksConfigException;
import org.smooks.api.resource.config.Parameter;
import org.smooks.api.resource.config.ResourceConfig;

import javax.inject.Inject;
import javax.inject.Named;
import java.net.URI;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

Expand All @@ -75,16 +72,7 @@ public class DataProcessorFactory {

public DataProcessor createDataProcessor() {
try {
final Map<String, String> variables = new HashMap<>();
final List<Parameter<?>> variablesParameters = resourceConfig.getParameters("variables");
if (variablesParameters != null) {
for (Parameter<?> variablesParameter : variablesParameters) {
final Map.Entry<String, String> variable = (Map.Entry<String, String>) variablesParameter.getValue();
variables.put(variable.getKey(), variable.getValue());
}
}

final DfdlSchema dfdlSchema = new DfdlSchema(new URI(schemaUri), variables,
final DfdlSchema dfdlSchema = new DfdlSchema(new URI(schemaUri),
ValidationMode.valueOf(resourceConfig.getParameterValue("validationMode", String.class, "Off")),
Boolean.parseBoolean(resourceConfig.getParameterValue("cacheOnDisk", String.class, "false")),
Boolean.parseBoolean(resourceConfig.getParameterValue("debugging", String.class, "false")),
Expand Down
14 changes: 3 additions & 11 deletions src/main/java/org/smooks/cartridges/dfdl/DfdlSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,29 +60,25 @@
import java.net.URI;
import java.nio.channels.Channels;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class DfdlSchema {

public static final String WORKING_DIRECTORY = ".smooks/dfdl-cartridge/";
protected static final Logger LOGGER = LoggerFactory.getLogger(DfdlSchema.class);

protected final URI uri;
protected final Map<String, String> variables;
protected final ValidationMode validationMode;
protected final boolean cacheOnDisk;
protected final boolean debugging;
protected final String distinguishedRootNode;
private final String schematronUrl;
private final boolean schematronValidation;

public DfdlSchema(final URI uri, final Map<String, String> variables, final ValidationMode validationMode, final boolean cacheOnDisk,
public DfdlSchema(final URI uri, final ValidationMode validationMode, final boolean cacheOnDisk,
final boolean debugging, final String distinguishedRootNode, final String schematronUrl,
final boolean schematronValidation) {
this.uri = uri;
this.variables = variables;
this.validationMode = validationMode;
this.cacheOnDisk = cacheOnDisk;
this.debugging = debugging;
Expand All @@ -95,10 +91,6 @@ public URI getUri() {
return uri;
}

public Map<String, String> getVariables() {
return variables;
}

public ValidationMode getValidationMode() {
return validationMode;
}
Expand All @@ -112,7 +104,7 @@ public boolean isDebugging() {
}

public String getName() {
return uri + ":" + validationMode + ":" + cacheOnDisk + ":" + debugging + ":" + variables.toString();
return uri + ":" + validationMode + ":" + cacheOnDisk + ":" + debugging;
}

public DataProcessor compile() throws Throwable {
Expand All @@ -137,7 +129,7 @@ public DataProcessor compile() throws Throwable {
}


dataProcessor = dataProcessor.withValidationMode(validationMode).withExternalVariables(new HashMap<>(variables));
dataProcessor = dataProcessor.withValidationMode(validationMode);

if (schematronValidation) {
final SchematronValidator schematronValidator;
Expand Down
44 changes: 34 additions & 10 deletions src/main/java/org/smooks/cartridges/dfdl/parser/DfdlParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@

import org.apache.daffodil.japi.DataProcessor;
import org.apache.daffodil.japi.Diagnostic;
import org.apache.daffodil.japi.ExternalVariableException;
import org.apache.daffodil.japi.ParseResult;
import org.apache.daffodil.japi.ValidationMode;
import org.apache.daffodil.japi.io.InputSourceDataInputStream;
Expand All @@ -53,6 +54,7 @@
import org.smooks.api.ExecutionContext;
import org.smooks.api.SmooksException;
import org.smooks.api.TypedKey;
import org.smooks.api.resource.config.Parameter;
import org.smooks.api.resource.config.ResourceConfig;
import org.smooks.api.resource.reader.SmooksXMLReader;
import org.smooks.cartridges.dfdl.DataProcessorFactory;
Expand All @@ -70,7 +72,10 @@
import jakarta.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
import java.util.AbstractMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class DfdlParser implements SmooksXMLReader {

Expand All @@ -84,27 +89,27 @@ public class DfdlParser implements SmooksXMLReader {
protected ApplicationContext applicationContext;

@Inject
private ResourceConfig resourceConfig;
protected ResourceConfig resourceConfig;

@Inject
@Named("dataProcessorFactory")
private Class<? extends DataProcessorFactory> dataProcessorFactoryClass;
protected Class<? extends DataProcessorFactory> dataProcessorFactoryClass;

@Inject
@Named("schemaUri")
private String schemaUri;
protected String schemaUri;

@Inject
@Named("validationMode")
private ValidationMode validationMode = ValidationMode.Off;
protected ValidationMode validationMode = ValidationMode.Off;

@Inject
private Boolean indent = false;
protected Boolean indent = false;

private ContentHandler contentHandler;
private ErrorHandler errorHandler;
private DTDHandler dtdHandler;
private ExecutionContext executionContext;
protected ContentHandler contentHandler;
protected ErrorHandler errorHandler;
protected DTDHandler dtdHandler;
protected ExecutionContext executionContext;

@Override
public void setExecutionContext(final ExecutionContext executionContext) {
Expand Down Expand Up @@ -177,12 +182,31 @@ public void postConstruct() throws IllegalAccessException, InstantiationExceptio
dataProcessor = dataProcessorFactory.createDataProcessor();
}

protected AbstractMap<String, String> getVariables() {
final List<Parameter<?>> variablesParameters = resourceConfig.getParameters("variables");
final AbstractMap<String, String> variables = new HashMap<>();
if (variablesParameters != null) {
for (Parameter<?> variablesParameter : variablesParameters) {
final Map.Entry<String, String> variable = (Map.Entry<String, String>) variablesParameter.getValue();
variables.put(variable.getKey(), variable.getValue());
}
}

return variables;
}

@Override
public void parse(final InputSource input) {
final InputSourceDataInputStream inputSourceDataInputStream = new InputSourceDataInputStream(input.getByteStream());
final DataProcessor copyDataProcessor;
try {
copyDataProcessor = dataProcessor.withExternalVariables(getVariables());
} catch (ExternalVariableException e) {
throw new SmooksException(e);
}
ParseResult parseResult = null;
while (parseResult == null || inputSourceDataInputStream.hasData()) {
parseResult = dataProcessor.parse(inputSourceDataInputStream, new ContentHandlerInfosetOutputter(contentHandler, indent));
parseResult = copyDataProcessor.parse(inputSourceDataInputStream, new ContentHandlerInfosetOutputter(contentHandler, indent));
if (parseResult.isError()) {
executionContext.put(DIAGNOSTICS_TYPED_KEY, parseResult.getDiagnostics());
for (Diagnostic diagnostic : parseResult.getDiagnostics()) {
Expand Down
43 changes: 40 additions & 3 deletions src/main/java/org/smooks/cartridges/dfdl/unparser/DfdlUnparser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,16 @@
import org.apache.daffodil.japi.DaffodilUnparseContentHandler;
import org.apache.daffodil.japi.DataProcessor;
import org.apache.daffodil.japi.Diagnostic;
import org.apache.daffodil.japi.ExternalVariableException;
import org.apache.daffodil.japi.UnparseResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.smooks.api.ExecutionContext;
import org.smooks.api.SmooksException;
import org.smooks.api.delivery.sax.StreamResultWriter;
import org.smooks.api.memento.MementoCaretaker;
import org.smooks.api.resource.config.Parameter;
import org.smooks.api.resource.config.ResourceConfig;
import org.smooks.api.resource.visitor.sax.ng.AfterVisitor;
import org.smooks.api.resource.visitor.sax.ng.BeforeVisitor;
import org.smooks.api.resource.visitor.sax.ng.ChildrenVisitor;
Expand All @@ -64,17 +67,25 @@
import org.w3c.dom.Node;
import org.xml.sax.helpers.AttributesImpl;

import javax.inject.Inject;
import javax.xml.XMLConstants;
import java.io.IOException;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
import java.util.AbstractMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@StreamResultWriter
public class DfdlUnparser implements BeforeVisitor, AfterVisitor, ChildrenVisitor {

protected static final Logger LOGGER = LoggerFactory.getLogger(DfdlUnparser.class);
protected final DataProcessor dataProcessor;

@Inject
protected ResourceConfig resourceConfig;

public DfdlUnparser(final DataProcessor dataProcessor) {
this.dataProcessor = dataProcessor;
}
Expand Down Expand Up @@ -112,12 +123,13 @@ protected DaffodilUnparseContentHandlerMemento getOrCreateDaffodilUnparseContent
}

final WritableByteChannel writableByteChannel;
final DaffodilUnparseContentHandler daffodilUnparseContentHandler;
try {
writableByteChannel = Channels.newChannel(WriterOutputStream.builder().setCharset(executionContext.getContentEncoding()).setBufferSize(1024).setWriteImmediately(true).setWriter(Stream.out(executionContext)).get());
} catch (IOException e) {
daffodilUnparseContentHandler = dataProcessor.withExternalVariables(getVariables(executionContext)).newContentHandlerInstance(writableByteChannel);
} catch (ExternalVariableException | IOException e) {
throw new SmooksException(e);
}
final DaffodilUnparseContentHandler daffodilUnparseContentHandler = dataProcessor.newContentHandlerInstance(writableByteChannel);
daffodilUnparseContentHandler.startDocument();

final DaffodilUnparseContentHandlerMemento daffodilUnparseContentHandlerMemento = new DaffodilUnparseContentHandlerMemento(new NodeFragment(node), this);
Expand All @@ -126,7 +138,20 @@ protected DaffodilUnparseContentHandlerMemento getOrCreateDaffodilUnparseContent

return daffodilUnparseContentHandlerMemento;
}


protected AbstractMap<String, String> getVariables(ExecutionContext executionContext) {
final List<Parameter<?>> variablesParameters = resourceConfig.getParameters("variables");
final AbstractMap<String, String> variables = new HashMap<>();
if (variablesParameters != null) {
for (Parameter<?> variablesParameter : variablesParameters) {
final Map.Entry<String, String> variable = (Map.Entry<String, String>) variablesParameter.getValue();
variables.put(variable.getKey(), variable.getValue());
}
}

return variables;
}

@Override
public void visitBefore(Element element, ExecutionContext executionContext) {
final DaffodilUnparseContentHandler daffodilUnparseContentHandler = getOrCreateDaffodilUnparseContentHandlerMemento(element, executionContext).getDaffodilUnparseContentHandler();
Expand Down Expand Up @@ -176,4 +201,16 @@ protected void throwIfError(final UnparseResult unparseResult) {
}
}
}

public DataProcessor getDataProcessor() {
return dataProcessor;
}

public ResourceConfig getResourceConfig() {
return resourceConfig;
}

public void setResourceConfig(ResourceConfig resourceConfig) {
this.resourceConfig = resourceConfig;
}
}
16 changes: 10 additions & 6 deletions src/main/java/org/smooks/cartridges/dfdl/unparser/DfdlUnparserContentHandlerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,21 +53,21 @@

import javax.inject.Inject;

public class DfdlUnparserContentHandlerFactory implements ContentHandlerFactory<DfdlUnparser> {
public class DfdlUnparserContentHandlerFactory<T extends DfdlUnparser> implements ContentHandlerFactory<DfdlUnparser> {

@Inject
protected ApplicationContext applicationContext;

@Override
public DfdlUnparser create(final ResourceConfig smooksResourceConfiguration) throws SmooksConfigException {
public T create(final ResourceConfig resourceConfig) throws SmooksConfigException {
try {
final String dataProcessorFactoryClassName = smooksResourceConfiguration.getParameterValue("dataProcessorFactory", String.class);
final String dataProcessorFactoryClassName = resourceConfig.getParameterValue("dataProcessorFactory", String.class);
final Class<? extends DataProcessorFactory> dataProcessorFactoryClass = (Class<? extends DataProcessorFactory>) Class.forName(dataProcessorFactoryClassName);
final DataProcessorFactory dataProcessorFactory = dataProcessorFactoryClass.newInstance();

applicationContext.getRegistry().lookup(new LifecycleManagerLookup()).applyPhase(dataProcessorFactory, new PostConstructLifecyclePhase(new Scope(applicationContext.getRegistry(), smooksResourceConfiguration, dataProcessorFactory)));
final DfdlUnparser dfdlUnparser = new DfdlUnparser(dataProcessorFactory.createDataProcessor());
applicationContext.getRegistry().lookup(new LifecycleManagerLookup()).applyPhase(dfdlUnparser, new PostConstructLifecyclePhase(new Scope(applicationContext.getRegistry(), smooksResourceConfiguration, dfdlUnparser)));
applicationContext.getRegistry().lookup(new LifecycleManagerLookup()).applyPhase(dataProcessorFactory, new PostConstructLifecyclePhase(new Scope(applicationContext.getRegistry(), resourceConfig, dataProcessorFactory)));
final T dfdlUnparser = newDfdlUnparser(dataProcessorFactory);
applicationContext.getRegistry().lookup(new LifecycleManagerLookup()).applyPhase(dfdlUnparser, new PostConstructLifecyclePhase(new Scope(applicationContext.getRegistry(), resourceConfig, dfdlUnparser)));

return dfdlUnparser;
} catch (Throwable t) {
Expand All @@ -87,4 +87,8 @@ public void setApplicationContext(ApplicationContext applicationContext) {
public ApplicationContext getApplicationContext() {
return applicationContext;
}

protected T newDfdlUnparser(DataProcessorFactory dataProcessorFactory) {
return (T) new DfdlUnparser(dataProcessorFactory.createDataProcessor());
}
}
7 changes: 3 additions & 4 deletions src/test/java/org/smooks/cartridges/dfdl/DfdlSchemaTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@

import java.io.File;
import java.net.URI;
import java.util.HashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;

Expand All @@ -59,7 +58,7 @@ public class DfdlSchemaTestCase extends AbstractTestCase {
@Test
public void testCompileHitsCacheGivenCacheOnDiskIsSetToTrue() throws Throwable {
CountDownLatch countDownLatch = new CountDownLatch(2);
DfdlSchema dfdlSchema = new DfdlSchema(new URI("/csv.dfdl.xsd"), new HashMap<>(), TestKit.getRandomItem(TestKit.getCacheOnDiskSupportedValidationModes()), true, ThreadLocalRandom.current().nextBoolean(), null, null, false) {
DfdlSchema dfdlSchema = new DfdlSchema(new URI("/csv.dfdl.xsd"), TestKit.getRandomItem(TestKit.getCacheOnDiskSupportedValidationModes()), true, ThreadLocalRandom.current().nextBoolean(), null, null, false) {
@Override
protected DataProcessor compileSource() throws Throwable {
countDownLatch.countDown();
Expand All @@ -74,14 +73,14 @@ protected DataProcessor compileSource() throws Throwable {

@Test
public void testCompileGivenCacheOnDiskIsSetToTrue() throws Throwable {
DfdlSchema dfdlSchema = new DfdlSchema(new URI("/csv.dfdl.xsd"), new HashMap<>(), TestKit.getRandomItem(TestKit.getCacheOnDiskSupportedValidationModes()), true, ThreadLocalRandom.current().nextBoolean(), null, null, false);
DfdlSchema dfdlSchema = new DfdlSchema(new URI("/csv.dfdl.xsd"), TestKit.getRandomItem(TestKit.getCacheOnDiskSupportedValidationModes()), true, ThreadLocalRandom.current().nextBoolean(), null, null, false);
dfdlSchema.compile();
assertTrue(new File(DfdlSchema.WORKING_DIRECTORY + "/csv.dfdl.xsd.dat").exists());
}

@Test
public void testCompileGivenCacheOnDiskIsSetToFalse() throws Throwable {
DfdlSchema dfdlSchema = new DfdlSchema(new URI("/csv.dfdl.xsd"), new HashMap<>(), ValidationMode.values()[ThreadLocalRandom.current().nextInt(ValidationMode.values().length)], false, ThreadLocalRandom.current().nextBoolean(), null, null, false);
DfdlSchema dfdlSchema = new DfdlSchema(new URI("/csv.dfdl.xsd"), ValidationMode.values()[ThreadLocalRandom.current().nextInt(ValidationMode.values().length)], false, ThreadLocalRandom.current().nextBoolean(), null, null, false);
dfdlSchema.compile();
assertFalse(new File(DfdlSchema.WORKING_DIRECTORY + "/csv.dfdl.xsd.dat").exists());
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/smooks/cartridges/dfdl/FunctionalTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public void testSmooksConfigGivenMissingUnparseOnNodeAttributeOnDfdlUnparser() t

@Test
public void testSmooksGivenDfdlUnparserVisitor() throws Throwable {
DfdlSchema dfdlSchema = new DfdlSchema(new URI("/csv.dfdl.xsd"), new HashMap<>(), ValidationMode.Full, false, false, null, null, false);
DfdlSchema dfdlSchema = new DfdlSchema(new URI("/csv.dfdl.xsd"), ValidationMode.Full, false, false, null, null, false);
DfdlUnparser dfdlUnparser = new DfdlUnparser(dfdlSchema.compile());

smooks.setFilterSettings(FilterSettings.newSaxNgSettings().setDefaultSerializationOn(false));
Expand Down
Loading

0 comments on commit 841b529

Please sign in to comment.