Multimodal

pom.xml

@@ -607,6 +607,21 @@
             <type>pom</type>
         </dependency>
 
+        <dependency>
+            <groupId>org.bytedeco</groupId>
+            <artifactId>javacv</artifactId>
+            <version>1.5.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.bytedeco</groupId>
+            <artifactId>javacv-platform</artifactId>
+            <version>1.5.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.bytedeco</groupId>
+            <artifactId>ffmpeg-platform</artifactId>
+            <version>6.1.1-1.5.10</version>
+        </dependency>
         <dependency>
             <groupId>org.jsoup</groupId>
             <artifactId>jsoup</artifactId>

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/DUUIComposer.java

@@ -450,6 +450,7 @@ public void run() {
                     // TODO thread safety needed for here?
                     DUUISegmentationStrategy segmentationStrategy = i.getSegmentationStrategy();
                     if (segmentationStrategy instanceof DUUISegmentationStrategyNone) {
+
                         i.getDriver().run(i.getUUID(), _jc, perf, composer);
                     } else {
                         segmentationStrategy.initialize(_jc);

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/DUUIFallbackCommunicationLayer.java

@@ -1,6 +1,7 @@
 package org.texttechnologylab.DockerUnifiedUIMAInterface;
 
 import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.uima.cas.CASException;
 import org.apache.uima.cas.impl.XmiCasDeserializer;
 import org.apache.uima.cas.impl.XmiCasSerializer;
 import org.apache.uima.jcas.JCas;
@@ -15,10 +16,10 @@
 import java.util.Map;
 
 public class DUUIFallbackCommunicationLayer implements IDUUICommunicationLayer {
-    public void serialize(JCas jc, ByteArrayOutputStream out, Map<String,String> parameters) throws CompressorException, IOException, SAXException {
+    public void serialize(JCas jc, ByteArrayOutputStream out, Map<String,String> parameters, String sourceView) throws CompressorException, IOException, SAXException, CASException {
         JSONObject obj = new JSONObject();
         ByteArrayOutputStream arr = new ByteArrayOutputStream();
-        XmiCasSerializer.serialize(jc.getCas(), null, arr);
+        XmiCasSerializer.serialize(jc.getView(sourceView).getCas(), null, arr);
 
         StringWriter writer = new StringWriter();
         TypeSystemUtil.typeSystem2TypeSystemDescription(jc.getTypeSystem()).toXML(writer);
@@ -34,7 +35,7 @@ public void serialize(JCas jc, ByteArrayOutputStream out, Map<String,String> par
         out.write(obj.toString().getBytes(StandardCharsets.UTF_8));
     }
 
-    public void deserialize(JCas jc, ByteArrayInputStream input) throws IOException, SAXException {
+    public void deserialize(JCas jc, ByteArrayInputStream input, String targetView) throws IOException, SAXException {
         String body = new String(input.readAllBytes(), Charset.defaultCharset());
         JSONObject response = new JSONObject(body);
         if (response.has("cas") || response.has("error")) {

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/IDUUICommunicationLayer.java

@@ -1,6 +1,7 @@
 package org.texttechnologylab.DockerUnifiedUIMAInterface;
 
 import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.uima.cas.CASException;
 import org.apache.uima.jcas.JCas;
 import org.xml.sax.SAXException;
 
@@ -12,6 +13,11 @@
  * Interface for communication between the DUUI composer {@link DUUIComposer} and the components {@link org.texttechnologylab.DockerUnifiedUIMAInterface.driver.IDUUIDriverInterface}.
  */
 public interface IDUUICommunicationLayer {
+
+  public void serialize(JCas jc, ByteArrayOutputStream out, Map<String,String> parameters, String sourceView) throws CompressorException, IOException, SAXException, CASException;
+
+  public void deserialize(JCas jc, ByteArrayInputStream input, String targetView) throws IOException, SAXException, CASException;
+
     /**
      * Serializes a JCas to a byte array output stream by using the LUA script provided by the component.
      * @param jc Input JCas.

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/composer/DUUISegmentedWorker.java

@@ -73,9 +73,11 @@ public void run() {
             try {
                 DUUIPipelineDocumentPerformance perf = new DUUIPipelineDocumentPerformance(name, waitTimeEnd - waitTimeStart, jCas, trackErrorDocs);
 
+                pipelinePart.getDriver().run(pipelinePart.getUUID(), jCas, perf, null);
                 // TODO!!!! @Daniel
                 //pipelinePart.getDriver().run(pipelinePart.getUUID(), jCas, perf);
 
+
                 if (backend != null) {
                     backend.addMetricsForDocument(perf);
                 }

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIDockerDriver.java

@@ -198,7 +198,7 @@ public static IDUUICommunicationLayer responsiveAfterTime(String url, JCas jc, i
         ByteArrayOutputStream stream = new ByteArrayOutputStream();
         try {
             //TODO: Make this accept options to better check the instantiation!
-            layer.serialize(jc, stream, null);
+            layer.serialize(jc, stream, null, "_InitialView");
         } catch (Exception e) {
             e.printStackTrace();
             throw new Exception(format("The serialization step of the communication layer fails for implementing class %s", layer.getClass().getCanonicalName()));
@@ -213,7 +213,7 @@ public static IDUUICommunicationLayer responsiveAfterTime(String url, JCas jc, i
         if (resp.statusCode() == 200) {
             ByteArrayInputStream inputStream = new ByteArrayInputStream(resp.body());
             try {
-                layer.deserialize(jc, inputStream);
+                layer.deserialize(jc, inputStream, "_InitialView");
             } catch (Exception e) {
                 System.err.printf("Caught exception printing response %s\n", new String(resp.body(), StandardCharsets.UTF_8));
                 throw e;
@@ -493,6 +493,8 @@ static class InstantiatedComponent implements IDUUIInstantiatedPipelineComponent
         private String _reg_username;
         private String _uniqueComponentKey;
         private Map<String, String> _parameters;
+        private String _sourceView;
+        private String _targetView;
         private DUUIPipelineComponent _component;
 
 
@@ -514,6 +516,8 @@ public void addComponent(IDUUIUrlAccessible access) {
             _component = comp;
             _image_name = comp.getDockerImageName();
             _parameters = comp.getParameters();
+            _targetView = comp.getTargetView();
+            _sourceView = comp.getSourceView();
             if (_image_name == null) {
                 throw new InvalidParameterException("The image name was not set! This is mandatory for the DockerLocalDriver Class.");
             }
@@ -585,6 +589,10 @@ public Map<String, String> getParameters() {
             return _parameters;
         }
 
+        public String getSourceView() {return _sourceView; }
+
+        public String getTargetView() {return _targetView; }
+
         public boolean isWebsocket() {
             return _websocket;
         }
@@ -602,6 +610,21 @@ public Component withParameter(String key, String value) {
             return this;
         }
 
+        public Component withView(String viewName) {
+            _component.withView(viewName);
+            return this;
+        }
+
+        public Component withSourceView(String viewName) {
+            _component.withSourceView(viewName);
+            return this;
+        }
+
+        public Component withTargetView(String viewName) {
+            _component.withTargetView(viewName);
+            return this;
+        }
+
         public Component(String target) throws URISyntaxException, IOException {
             _component = new DUUIPipelineComponent();
             _component.withDockerImageName(target);

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIKubernetesDriver.java

@@ -439,6 +439,8 @@ static class InstantiatedComponent implements IDUUIInstantiatedPipelineComponent
         private int _scale;
         private boolean _withImageFetching;
         private Map<String, String> _parameters;
+        private String _sourceView;
+        private String _targetView;
         private DUUIPipelineComponent _component;
 
         private final boolean _websocket;
@@ -451,6 +453,8 @@ static class InstantiatedComponent implements IDUUIInstantiatedPipelineComponent
             _component = comp;
             _image_name = comp.getDockerImageName();
             _parameters = comp.getParameters();
+            _targetView = comp.getTargetView();
+            _sourceView = comp.getSourceView();
             if (_image_name == null) {
                 throw new InvalidParameterException("The image name was not set! This is mandatory for the DockerLocalDriver Class.");
             }
@@ -523,6 +527,10 @@ public Map<String, String> getParameters() {
             return _parameters;
         }
 
+        public String getSourceView() {return _sourceView; }
+
+        public String getTargetView() {return _targetView; }
+
         @Override
         public String getUniqueComponentKey() {
             return _uniqueComponentKey;
@@ -633,6 +641,21 @@ public Component withParameter(String key, String value) {
             return this;
         }
 
+        public Component withView(String viewName) {
+            _component.withView(viewName);
+            return this;
+        }
+
+        public Component withSourceView(String viewName) {
+            _component.withSourceView(viewName);
+            return this;
+        }
+
+        public Component withTargetView(String viewName) {
+            _component.withTargetView(viewName);
+            return this;
+        }
+
         /**
          * Builds the component.
          *

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java

@@ -65,6 +65,8 @@ public class DUUIPipelineComponent {
     private static String driverName = "driver";
     private static String descriptionName = "description";
 
+    private static String sourceView = "sourceView";
+    private static String targetView = "targetView";
 
     private String getVersion() throws URISyntaxException, IOException {
         ClassLoader classLoader = DUUIPipelineComponent.class.getClassLoader();
@@ -80,6 +82,7 @@ public DUUIPipelineComponent() throws URISyntaxException, IOException {
         _options = new HashMap<>();
         _finalizedEncoded = null;
         _parameters = new HashMap<>();
+
         String version = getVersion();
         if(version == null) {
             _options.put(versionInformation,"Unknown");
@@ -487,6 +490,22 @@ public DUUIPipelineComponent withParameter(String key, String value) {
         return this;
     }
 
+    public DUUIPipelineComponent withView(String viewName){
+        withSourceView(viewName);
+        withTargetView(viewName);
+        return this;
+    }
+
+    public DUUIPipelineComponent withSourceView(String viewName) {
+        _options.put(sourceView, viewName);
+        return this;
+    }
+
+    public DUUIPipelineComponent withTargetView(String viewName) {
+        _options.put(targetView, viewName);
+        return this;
+    }
+
     public static DUUIPipelineComponent fromJson(String json) throws URISyntaxException, IOException {
         JSONObject jobj = new JSONObject(json);
 
@@ -560,6 +579,22 @@ public final Map<String,String> getParameters() {
         return _parameters;
     }
 
+    public String getSourceView() {
+        String result = _options.get(sourceView);
+        if(result == null) {
+            return "_InitialView";
+        }
+        return result;
+    }
+
+    public String getTargetView() {
+        String result = _options.get(targetView);
+        if(result == null) {
+            return "_InitialView";
+        }
+        return result;
+    }
+
     public DUUIPipelineComponent clearParameters() {
         _parameters.clear();
         return this;

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIRemoteDriver.java

@@ -81,6 +81,21 @@ public Component withParameter(String key, String value) {
             return this;
         }
 
+        public Component withView(String viewName) {
+            component.withView(viewName);
+            return this;
+        }
+
+        public Component withSourceView(String viewName) {
+            component.withSourceView(viewName);
+            return this;
+        }
+
+        public Component withTargetView(String viewName) {
+            component.withTargetView(viewName);
+            return this;
+        }
+
         public Component withWebsocket(boolean b) {
             component.withWebsocket(b);
             return this;
@@ -152,6 +167,8 @@ private static class InstantiatedComponent implements IDUUIInstantiatedPipelineC
         private ConcurrentLinkedQueue<ComponentInstance> _components;
         private String _uniqueComponentKey;
         private Map<String, String> _parameters;
+        private String _sourceView;
+        private String _targetView;
         private DUUIPipelineComponent _component;
         private boolean _websocket;
         private int _ws_elements;
@@ -179,6 +196,8 @@ public InstantiatedComponent(DUUIPipelineComponent comp) {
             }
 
             _parameters = comp.getParameters();
+            _targetView = comp.getTargetView();
+            _sourceView = comp.getSourceView();
 
             _uniqueComponentKey = "";
 
@@ -208,6 +227,10 @@ public Map<String, String> getParameters() {
             return _parameters;
         }
 
+        public String getSourceView() {return _sourceView; }
+
+        public String getTargetView() {return _targetView; }
+
         public boolean isWebsocket() {
             return _websocket;
         }

src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUISwarmDriver.java

@@ -264,6 +264,8 @@ private static class InstantiatedComponent implements IDUUIInstantiatedPipelineC
         private final String _reg_password;
         private final String _reg_username;
         private final Map<String, String> _parameters;
+        private String _sourceView;
+        private String _targetView;
         private DUUIPipelineComponent _component;
 
 
@@ -275,6 +277,8 @@ private static class InstantiatedComponent implements IDUUIInstantiatedPipelineC
             }
 
             _parameters = comp.getParameters();
+            _targetView = comp.getTargetView();
+            _sourceView = comp.getSourceView();
             _scale = comp.getScale(1);
             _constraints.addAll(comp.getConstraints());
             _components = new ConcurrentLinkedQueue<>();
@@ -374,6 +378,9 @@ public Map<String, String> getParameters() {
             return _parameters;
         }
 
+        public String getSourceView() {return _sourceView; }
+
+        public String getTargetView() {return _targetView; }
 
         public Triplet<IDUUIUrlAccessible, Long, Long> getComponent() {
             long mutexStart = System.nanoTime();
@@ -412,6 +419,21 @@ public Component withParameter(String key, String value) {
             return this;
         }
 
+        public Component withView(String viewName) {
+            component.withView(viewName);
+            return this;
+        }
+
+        public Component withSourceView(String viewName) {
+            component.withSourceView(viewName);
+            return this;
+        }
+
+        public Component withTargetView(String viewName) {
+            component.withTargetView(viewName);
+            return this;
+        }
+
         public Component withScale(int scale) {
             component.withScale(scale);
             return this;