Reworking fcrepo indexer to be file system aware

islandora-connector-houdini/src/main/java/ca/islandora/alpaca/connector/houdini/HoudiniConnector.java

@@ -61,13 +61,12 @@ public void configure() {
             .setHeader("Accept", simple("${exchangeProperty.event.attachment.content.mimetype}"))
             .setHeader("X-Islandora-Args", simple("${exchangeProperty.event.attachment.content.args}"))
             .setHeader("Apix-Ldp-Resource", simple("${exchangeProperty.event.attachment.content.sourceUri}"))
-            .transform(simple("${null}"))
+            .setBody(simple("${null}"))
             .to("{{houdini.convert.url}}")
 
             // PUT the media.
             .removeHeaders("*", "Authorization", "Content-Type")
-            .setHeader("Content-Disposition",
-                    simple("attachment; filename=\"${exchangeProperty.event.attachment.content.filename}\""))
+            .setHeader("Content-Location", simple("${exchangeProperty.event.attachment.content.fileUploadUri}"))
             .setHeader(Exchange.HTTP_METHOD, constant("PUT"))
             .toD("${exchangeProperty.event.attachment.content.destinationUri}");
     }

islandora-connector-houdini/src/main/java/ca/islandora/alpaca/connector/houdini/event/AS2AttachmentContent.java

@@ -86,23 +86,24 @@ public void setArgs(final String args) {
     }
 
     /**
-     * @return  Filename
+     * @return  File upload uri
      */
-    public String getFilename() {
-        return filename;
+    @JsonProperty(value = "file_upload_uri")
+    public String getFileUploadUri() {
+        return fileUploadUri;
     }
 
     /**
-     * @param   filename    Filename
+     * @param   fileUploadUri    File upload uri
      */
-    public void setFilename(final String filename) {
-        this.filename = filename;
+    public void setFileUploadUri(final String fileUploadUri) {
+        this.fileUploadUri = fileUploadUri;
     }
 
     private String sourceUri;
     private String destinationUri;
     private String mimetype;
     private String args;
-    private String filename;
+    private String fileUploadUri;
 
 }

islandora-connector-houdini/src/test/java/ca/islandora/alpaca/connector/houdini/HoudiniConnectorTest.java

@@ -74,17 +74,17 @@ public void configure() throws Exception {
                                 exchange.getIn().setBody("SOME DERIVATIVE", String.class);
                             });
 
-                    mockEndpointsAndSkip("http://localhost:8000/node/1/media/image/3");
+                    mockEndpointsAndSkip("http://localhost:8000/node/2/media/image/3");
                 }
         });
         context.start();
 
-        final MockEndpoint endpoint = getMockEndpoint("mock:http:localhost:8000/node/1/media/image/3");
+        final MockEndpoint endpoint = getMockEndpoint("mock:http:localhost:8000/node/2/media/image/3");
 
         endpoint.expectedMessageCount(1);
         endpoint.expectedHeaderReceived(Exchange.HTTP_METHOD, "PUT");
         endpoint.expectedHeaderReceived(CONTENT_TYPE, "image/jpeg");
-        endpoint.expectedHeaderReceived("Content-Disposition", "attachment; filename=\"1 - ServiceFile.jpeg\"");
+        endpoint.expectedHeaderReceived("Content-Location", "public://2018-08/2-Service File.jpg");
         endpoint.expectedHeaderReceived("Authorization", "Bearer islandora");
 
         template.send(exchange -> {

islandora-connector-houdini/src/test/resources/AS2Event.jsonld

@@ -2,7 +2,7 @@
    "@context":"https:\/\/www.w3.org\/ns\/activitystreams",
    "actor":{
       "type":"Person",
-      "id":"urn:uuid:01e93c43-bb4a-4c7e-9402-f15242472853",
+      "id":"urn:uuid:9029a0c0-d845-4ddd-864c-2198d45839da",
       "url":[
          {
             "name":"Canonical",
@@ -14,26 +14,28 @@
       ]
    },
    "object":{
-      "id":"urn:uuid:3d3550ff-1d4c-4484-a406-8f97c1673e5c",
+      "id":"urn:uuid:72358916-51e9-4712-b756-4b0404c91b1d",
       "url":[
          {
-            "name":"Canoncial",
+            "name":"Canonical",
             "type":"Link",
-            "href":"http:\/\/localhost:8000\/node\/1",
+            "href":"http:\/\/localhost:8000\/node\/2",
             "mediaType":"text\/html",
             "rel":"canonical"
          },
          {
-            "name":"JSONLD",
+            "name":"JSON",
             "type":"Link",
-            "href":"http:\/\/localhost:8000\/node\/1?_format=jsonld",
-            "mediaType":"application\/ld+json"
+            "href":"http:\/\/localhost:8000\/node\/2?_format=json",
+            "mediaType":"application\/json",
+            "rel":"alternate"
          },
          {
-            "name":"JSON",
+            "name":"JSONLD",
             "type":"Link",
-            "href":"http:\/\/localhost:8000\/node\/1?_format=json",
-            "mediaType":"application\/json"
+            "href":"http:\/\/localhost:8000\/node\/2?_format=jsonld",
+            "mediaType":"application\/ld+json",
+            "rel":"alternate"
          }
       ]
    },
@@ -44,9 +46,9 @@
       "content":{
          "mimetype":"image\/jpeg",
          "args":"",
-         "source_uri":"http:\/\/localhost:8000\/sites\/default\/files\/2018-05\/Test.jpg",
-         "destination_uri":"http:\/\/localhost:8000\/node\/1\/media\/image\/3",
-         "filename":"1 - ServiceFile.jpeg"
+         "source_uri":"http:\/\/localhost:8000\/_flysystem\/fedora\/2018-07\/Egyptology.jpg",
+         "destination_uri":"http:\/\/localhost:8000\/node\/2\/media\/image\/3",
+         "file_upload_uri":"public:\/\/2018-08\/2-Service File.jpg"
       },
       "mediaType":"application\/json"
    }

islandora-indexing-fcrepo/build.gradle

@@ -6,6 +6,7 @@ dependencies {
     compile group: 'org.apache.camel', name: 'camel-core', version: camelVersion
     compile group: 'org.apache.camel', name: 'camel-blueprint', version: camelVersion
     compile group: 'org.apache.camel', name: 'camel-http4', version: camelVersion
+    compile group: 'org.apache.camel', name: 'camel-jackson', version: camelVersion
     compile group: 'org.apache.camel', name: 'camel-jsonpath', version: camelVersion
     compile group: 'org.slf4j', name: 'slf4j-api', version: slf4jVersion
     compile group: 'commons-io', name: 'commons-io', version: commonsIoVersion 

islandora-indexing-fcrepo/src/main/cfg/ca.islandora.alpaca.indexing.fcrepo.cfg

@@ -2,10 +2,12 @@
 error.maxRedeliveries=5
 
 # Input queues
-content.stream=activemq:queue:islandora-indexing-fcrepo-content
-file.stream=activemq:queue:islandora-indexing-fcrepo-file
+node.stream=activemq:queue:islandora-indexing-fcrepo-content
+node.delete.stream=activemq:queue:islandora-indexing-fcrepo-delete
 media.stream=activemq:queue:islandora-indexing-fcrepo-media
-delete.stream=activemq:queue:islandora-indexing-fcrepo-delete
+file.stream=activemq:queue:islandora-indexing-fcrepo-file
+file.delete.stream=activemq:queue:islandora-indexing-fcrepo-file-delete
 
-# Base url for Milliner microservice
+# Base url for microservices
 milliner.baseUrl=http://localhost:8000/milliner/
+gemini.baseUrl=http://localhost:8000/gemini/

islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexer.java

@@ -22,12 +22,14 @@
 import static org.apache.camel.LoggingLevel.INFO;
 import static org.slf4j.LoggerFactory.getLogger;
 
+import ca.islandora.alpaca.indexing.fcrepo.event.AS2Event;
 import org.apache.camel.Exchange;
 import org.apache.camel.Predicate;
 import org.apache.camel.PropertyInject;
 import org.apache.camel.builder.RouteBuilder;
 import org.apache.camel.builder.PredicateBuilder;
 import org.apache.camel.http.common.HttpOperationFailedException;
+import org.apache.camel.model.dataformat.JsonLibrary;
 import org.slf4j.Logger;
 
 /**
@@ -53,9 +55,7 @@ public void setMaxRedeliveries(final int maxRedeliveries) {
      * @return  Milliner base url
      */
     public String getMillinerBaseUrl() {
-        // Enforce trailing slash.
-        final String trimmed = millinerBaseUrl.trim();
-        return trimmed.endsWith("/") ? trimmed : trimmed + "/";
+        return enforceTrailingSlash(millinerBaseUrl);
     }
 
     /**
@@ -65,12 +65,34 @@ public void setMillinerBaseUrl(final String millinerBaseUrl) {
         this.millinerBaseUrl = millinerBaseUrl;
     }
 
+    /**
+     * @return  Gemini base url
+     */
+    public String getGeminiBaseUrl() {
+        return enforceTrailingSlash(geminiBaseUrl);
+    }
+
+    /**
+     * @param   geminiBaseUrl Gemini base url
+     */
+    public void setGeminiBaseUrl(final String geminiBaseUrl) {
+        this.geminiBaseUrl = geminiBaseUrl;
+    }
+
+    private String enforceTrailingSlash(final String baseUrl) {
+        final String trimmed = baseUrl.trim();
+        return trimmed.endsWith("/") ? trimmed : trimmed + "/";
+    }
+
     @PropertyInject("error.maxRedeliveries")
     private int maxRedeliveries;
 
     @PropertyInject("milliner.baseUrl")
     private String millinerBaseUrl;
 
+    @PropertyInject("gemini.baseUrl")
+    private String geminiBaseUrl;
+
     private static final Logger LOGGER = getLogger(FcrepoIndexer.class);
 
     @Override
@@ -97,29 +119,28 @@ public void configure() {
                         "Error indexing resource in fcrepo: ${exception.message}\n\n${exception.stacktrace}"
                 );
 
-        from("{{content.stream}}")
-                .routeId("FcrepoIndexerContent")
-                .removeHeaders("*", "Authorization")
-                .setHeader(Exchange.CONTENT_TYPE, constant("application/ld+json"))
-                .setHeader(Exchange.HTTP_METHOD, constant("POST"))
-                .toD(getMillinerBaseUrl() + "content");
+        from("{{node.stream}}")
+                .routeId("FcrepoIndexerNode")
 
-        from("{{file.stream}}")
-                .routeId("FcrepoIndexerFile")
-                .removeHeaders("*", "Authorization")
-                .setHeader(Exchange.CONTENT_TYPE, constant("application/ld+json"))
-                .setHeader(Exchange.HTTP_METHOD, constant("POST"))
-                .to(getMillinerBaseUrl() + "file");
+                // Parse the event into a POJO.
+                .unmarshal().json(JsonLibrary.Jackson, AS2Event.class)
 
-        from("{{media.stream}}")
-                .routeId("FcrepoIndexerMedia")
+                // Extract relevant data from the event.
+                .setProperty("event").simple("${body}")
+                .setProperty("uuid").simple("${exchangeProperty.event.object.id.replaceAll(\"urn:uuid:\",\"\")}")
+                .setProperty("jsonldUrl").simple("${exchangeProperty.event.object.url[2].href}")
+
+                // Prepare the message.
                 .removeHeaders("*", "Authorization")
-                .setHeader(Exchange.CONTENT_TYPE, constant("application/ld+json"))
                 .setHeader(Exchange.HTTP_METHOD, constant("POST"))
-                .to(getMillinerBaseUrl() + "media");
+                .setHeader("Content-Location", simple("${exchangeProperty.jsonldUrl}"))
+                .setBody(simple("${null}"))
+
+                // Pass it to milliner.
+                .toD(getMillinerBaseUrl() + "node/${exchangeProperty.uuid}");
 
-        from("{{delete.stream}}")
-                .routeId("FcrepoIndexerDelete")
+        from("{{node.delete.stream}}")
+                .routeId("FcrepoIndexerDeleteNode")
                 .onException(HttpOperationFailedException.class)
                         .onWhen(is404)
                         .useOriginalMessage()
@@ -130,10 +151,81 @@ public void configure() {
                                 "Received 404 from Milliner, skipping de-indexing."
                         )
                         .end()
-                .setProperty("urn").jsonpath("$.object.id")
-                .setProperty("uuid").simple("${exchangeProperty.urn.replaceAll(\"urn:uuid:\",\"\")}")
+                // Parse the event into a POJO.
+                .unmarshal().json(JsonLibrary.Jackson, AS2Event.class)
+
+                // Extract relevant data from the event.
+                .setProperty("event").simple("${body}")
+                .setProperty("uuid").simple("${exchangeProperty.event.object.id.replaceAll(\"urn:uuid:\",\"\")}")
+
+                // Prepare the message.
                 .removeHeaders("*", "Authorization")
                 .setHeader(Exchange.HTTP_METHOD, constant("DELETE"))
-                .toD(getMillinerBaseUrl() + "resource/${exchangeProperty.uuid}");
+                .setBody(simple("${null}"))
+
+                // Remove the file from Gemini.
+                .toD(getMillinerBaseUrl() + "node/${exchangeProperty.uuid}");
+
+        from("{{media.stream}}")
+                .routeId("FcrepoIndexerMedia")
+
+                // Parse the event into a POJO.
+                .unmarshal().json(JsonLibrary.Jackson, AS2Event.class)
+
+                // Extract relevant data from the event.
+                .setProperty("event").simple("${body}")
+                .setProperty("sourceField").simple("${exchangeProperty.event.attachment.content.sourceField}")
+                .setProperty("jsonUrl").simple("${exchangeProperty.event.object.url[1].href}")
+
+                // Prepare the message.
+                .removeHeaders("*", "Authorization")
+                .setHeader(Exchange.HTTP_METHOD, constant("POST"))
+                .setHeader("Content-Location", simple("${exchangeProperty.jsonUrl}"))
+                .setBody(simple("${null}"))
+
+                // Pass it to milliner.
+                .toD(getMillinerBaseUrl() + "media/${exchangeProperty.sourceField}");
+
+        from("{{file.stream}}")
+                .routeId("FcrepoIndexerFile")
+
+                // Parse the event into a POJO.
+                .unmarshal().json(JsonLibrary.Jackson, AS2Event.class)
+
+                // Extract relevant data from the event.
+                .setProperty("event").simple("${body}")
+                .setProperty("uuid").simple("${exchangeProperty.event.object.id.replaceAll(\"urn:uuid:\",\"\")}")
+                .setProperty("drupal").simple("${exchangeProperty.event.object.url[0].href}")
+                .setProperty("fedora").simple("${exchangeProperty.event.attachment.content.fedoraUri}")
+
+                // Prepare the message.
+                .removeHeaders("*", "Authorization")
+                .setHeader(Exchange.CONTENT_TYPE, constant("application/json"))
+                .setHeader(Exchange.HTTP_METHOD, constant("PUT"))
+                .setBody(simple(
+                    "{\"drupal\": \"${exchangeProperty.drupal}\", \"fedora\": \"${exchangeProperty.fedora}\"}")
+                )
+
+                // Index the file in Gemini.
+                .toD(getGeminiBaseUrl() + "${exchangeProperty.uuid}");
+
+        from("{{file.delete.stream}}")
+                .routeId("FcrepoIndexerFileDelete")
+
+                // Parse the event into a POJO.
+                .unmarshal().json(JsonLibrary.Jackson, AS2Event.class)
+
+                // Extract relevant data from the event.
+                .setProperty("event").simple("${body}")
+                .setProperty("uuid").simple("${exchangeProperty.event.object.id.replaceAll(\"urn:uuid:\",\"\")}")
+
+                // Prepare the message.
+                .removeHeaders("*", "Authorization")
+                .setHeader(Exchange.HTTP_METHOD, constant("DELETE"))
+                .setBody(simple("${null}"))
+
+                // Remove the file from Gemini.
+                .toD(getGeminiBaseUrl() + "${exchangeProperty.uuid}");
+
     }
 }