From 442f447605b5170eb62a27870b1446fffa39b7c4 Mon Sep 17 00:00:00 2001
From: Fabian Steeg <steeg@hbz-nrw.de>
Date: Wed, 11 Nov 2020 09:46:14 +0100
Subject: [PATCH 1/2] Add extract-element flux command based on CSS-style
 selector

See:

https://github.com/metafacture/metafacture-core/issues/312
https://gitlab.com/oersi/oersi-etl/-/issues/27
https://jsoup.org/cookbook/extracting-data/selector-syntax
---
 .../metafacture/html/ElementExtractor.java    | 61 +++++++++++++++++
 .../main/resources/flux-commands.properties   |  1 +
 .../html/ElementExtractorTest.java            | 67 +++++++++++++++++++
 3 files changed, 129 insertions(+)
 create mode 100644 metafacture-html/src/main/java/org/metafacture/html/ElementExtractor.java
 create mode 100644 metafacture-html/src/test/java/org/metafacture/html/ElementExtractorTest.java

diff --git a/metafacture-html/src/main/java/org/metafacture/html/ElementExtractor.java b/metafacture-html/src/main/java/org/metafacture/html/ElementExtractor.java
new file mode 100644
index 000000000..19acdc868
--- /dev/null
+++ b/metafacture-html/src/main/java/org/metafacture/html/ElementExtractor.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Fabian Steeg, hbz
+ *
+ * Licensed under the Apache License, Version 2.0 the "License";
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.metafacture.html;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.commons.io.IOUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.metafacture.framework.FluxCommand;
+import org.metafacture.framework.ObjectReceiver;
+import org.metafacture.framework.annotations.Description;
+import org.metafacture.framework.annotations.In;
+import org.metafacture.framework.annotations.Out;
+import org.metafacture.framework.helpers.DefaultObjectPipe;
+
+/**
+ * Extracts the the specified element from an HTML document
+ *
+ * @author Fabian Steeg
+ */
+@Description("Extracts the specified element from an HTML document")
+@In(Reader.class)
+@Out(String.class)
+@FluxCommand("extract-element")
+public class ElementExtractor extends DefaultObjectPipe<Reader, ObjectReceiver<String>> {
+    private String selector;
+
+    /**
+     * @param selector The CSS-style jsoup selector, see https://jsoup.org/cookbook/extracting-data/selector-syntax
+     */
+    public ElementExtractor(final String selector) {
+        this.selector = selector;
+    }
+
+    @Override
+    public void process(final Reader reader) {
+        try {
+            Document document = Jsoup.parse(IOUtils.toString(reader));
+            Element firstElement = document.select(selector).first();
+            getReceiver().process(firstElement.data());
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/metafacture-html/src/main/resources/flux-commands.properties b/metafacture-html/src/main/resources/flux-commands.properties
index 95f4e031c..3d1443da4 100644
--- a/metafacture-html/src/main/resources/flux-commands.properties
+++ b/metafacture-html/src/main/resources/flux-commands.properties
@@ -15,3 +15,4 @@
 #
 decode-html org.metafacture.html.HtmlDecoder
 extract-script org.metafacture.html.ScriptExtractor
+extract-element org.metafacture.html.ElementExtractor
diff --git a/metafacture-html/src/test/java/org/metafacture/html/ElementExtractorTest.java b/metafacture-html/src/test/java/org/metafacture/html/ElementExtractorTest.java
new file mode 100644
index 000000000..860af9f34
--- /dev/null
+++ b/metafacture-html/src/test/java/org/metafacture/html/ElementExtractorTest.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2020 Fabian Steeg, hbz
+ *
+ * Licensed under the Apache License, Version 2.0 the "License";
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.metafacture.html;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import java.io.StringReader;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.metafacture.framework.ObjectReceiver;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/**
+ * Tests for {@link ElementExtractor}.
+ *
+ * @author Fabian Steeg
+ *
+ */
+public final class ElementExtractorTest {
+
+    private static final StringReader IN = new StringReader("<html>"
+            + "<script data-test='site-head-data'>{\"code\":\"hey\"}</script>"
+            + "<script data-test='model-linked-data'>{\"code\":\"yo\"}");
+    
+    private static final String OUT = "{\"code\":\"yo\"}";
+
+    private ElementExtractor elementExtractor;
+
+    @Mock
+    private ObjectReceiver<String> receiver;
+
+    @Before
+    public void setup() {
+        MockitoAnnotations.initMocks(this);
+        elementExtractor = new ElementExtractor("script[data-test=model-linked-data]");
+        elementExtractor.setReceiver(receiver);
+    }
+
+    @Test
+    public void testShouldProcessRecordsFollowedbySeparator() {
+        elementExtractor.process(IN);
+        verify(receiver).process(OUT);
+        verifyNoMoreInteractions(receiver);
+    }
+
+    @After
+    public void cleanup() {
+        elementExtractor.closeStream();
+    }
+}

From 99bc94186df12467e6c7b5abb8443f793c0d02d9 Mon Sep 17 00:00:00 2001
From: Fabian Steeg <steeg@hbz-nrw.de>
Date: Tue, 8 Dec 2020 16:32:25 +0100
Subject: [PATCH 2/2] Remove unreleased ScriptExtractor

Replaced by generic ElementExtractor, see 442f447
---
 .../org/metafacture/html/ScriptExtractor.java | 52 ---------------
 .../main/resources/flux-commands.properties   |  1 -
 .../metafacture/html/ScriptExtractorTest.java | 64 -------------------
 3 files changed, 117 deletions(-)
 delete mode 100644 metafacture-html/src/main/java/org/metafacture/html/ScriptExtractor.java
 delete mode 100644 metafacture-html/src/test/java/org/metafacture/html/ScriptExtractorTest.java

diff --git a/metafacture-html/src/main/java/org/metafacture/html/ScriptExtractor.java b/metafacture-html/src/main/java/org/metafacture/html/ScriptExtractor.java
deleted file mode 100644
index fdcecbfcb..000000000
--- a/metafacture-html/src/main/java/org/metafacture/html/ScriptExtractor.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2020 Fabian Steeg, hbz
- *
- * Licensed under the Apache License, Version 2.0 the "License";
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.metafacture.html;
-
-import java.io.IOException;
-import java.io.Reader;
-
-import org.apache.commons.io.IOUtils;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.metafacture.framework.FluxCommand;
-import org.metafacture.framework.ObjectReceiver;
-import org.metafacture.framework.annotations.Description;
-import org.metafacture.framework.annotations.In;
-import org.metafacture.framework.annotations.Out;
-import org.metafacture.framework.helpers.DefaultObjectPipe;
-
-/**
- * Extracts the first script from an HTML document
- *
- * @author Fabian Steeg
- */
-@Description("Extracts the first script from an HTML document")
-@In(Reader.class)
-@Out(String.class)
-@FluxCommand("extract-script")
-public class ScriptExtractor extends DefaultObjectPipe<Reader, ObjectReceiver<String>> {
-    @Override
-    public void process(final Reader reader) {
-        try {
-            Document document = Jsoup.parse(IOUtils.toString(reader));
-            Element firstScript = document.select("script").first();
-            getReceiver().process(firstScript.data());
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-}
diff --git a/metafacture-html/src/main/resources/flux-commands.properties b/metafacture-html/src/main/resources/flux-commands.properties
index 3d1443da4..e6046cd3f 100644
--- a/metafacture-html/src/main/resources/flux-commands.properties
+++ b/metafacture-html/src/main/resources/flux-commands.properties
@@ -14,5 +14,4 @@
 # limitations under the License.
 #
 decode-html org.metafacture.html.HtmlDecoder
-extract-script org.metafacture.html.ScriptExtractor
 extract-element org.metafacture.html.ElementExtractor
diff --git a/metafacture-html/src/test/java/org/metafacture/html/ScriptExtractorTest.java b/metafacture-html/src/test/java/org/metafacture/html/ScriptExtractorTest.java
deleted file mode 100644
index 6045fad7a..000000000
--- a/metafacture-html/src/test/java/org/metafacture/html/ScriptExtractorTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2020 Fabian Steeg, hbz
- *
- * Licensed under the Apache License, Version 2.0 the "License";
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.metafacture.html;
-
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-
-import java.io.StringReader;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.metafacture.framework.ObjectReceiver;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-
-/**
- * Tests for {@link ScriptExtractor}.
- *
- * @author Fabian Steeg
- *
- */
-public final class ScriptExtractorTest {
-
-    private static final StringReader IN = new StringReader("<html><script>{\"code\":\"yo\"}");
-    private static final String OUT = "{\"code\":\"yo\"}";
-
-    private ScriptExtractor scriptExtractor;
-
-    @Mock
-    private ObjectReceiver<String> receiver;
-
-    @Before
-    public void setup() {
-        MockitoAnnotations.initMocks(this);
-        scriptExtractor = new ScriptExtractor();
-        scriptExtractor.setReceiver(receiver);
-    }
-
-    @Test
-    public void testShouldProcessRecordsFollowedbySeparator() {
-        scriptExtractor.process(IN);
-        verify(receiver).process(OUT);
-        verifyNoMoreInteractions(receiver);
-    }
-
-    @After
-    public void cleanup() {
-        scriptExtractor.closeStream();
-    }
-}