Merge branch 'master' into 1828-relatedIssn

.github/workflows/build.yml

@@ -6,27 +6,28 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 11
     - name: Cache Maven packages
       uses: actions/cache@v2
       with:
         path: ~/.m2
         key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
         restore-keys: ${{ runner.os }}-m2
-        # - name: Install metafacture-core snapshots
-        #run: |
-        #git clone https://github.com/metafacture/metafacture-core.git
-        #cd metafacture-core
-        #./gradlew publishToMavenLocal
-        #cd ..
+    #- name: Install metafacture-core-snapshots
+    #  run: |
+    #    git clone https://github.com/metafacture/metafacture-core.git
+    #    cd metafacture-core
+    #    git checkout 5.7.0-rc1
+    #    ./gradlew publishToMavenLocal
+    #    cd ..
     - name: Install metafacture-fix
       run: |
         git clone https://github.com/metafacture/metafacture-fix.git
         cd metafacture-fix
-        git reset --hard 0.6.1
+        git checkout 1.0.0
         ./gradlew publishToMavenLocal
         cd ..
     - name: Build with Maven

.gitmodules

@@ -4,6 +4,3 @@
 [submodule "vocabs"]
 	path = vocabs
 	url = https://github.com/lobid/vocabs
-[submodule "link-templates"]
-	path = link-templates
-	url = https://github.com/hbz/link-templates

README.textile

@@ -12,7 +12,7 @@ h1. Build
 
 "!https://github.com/hbz/lobid-resources/workflows/Build/badge.svg?branch=master!":https://github.com/hbz/lobid-resources/actions?query=branch%3Amaster
 
-Prerequisites: Java 8, Maven 3; verify with @mvn -version@
+Prerequisites: Java 11, Maven 3; verify with @mvn -version@
 
 Create and change into a folder where you want to store the projects:
 
@@ -49,11 +49,11 @@ In the online test the data is indexed into a living elasticsearch instance.
 This instance is only reachable within our internal network, thus this test
 must be executed manually. Then elasticsearch can be looked up like this:
 
-https://alma.lobid.org/resources/990054215550206441
+https://lobid.org/resources/990054215550206441
 
 For querying it you can use the elasticsearch query DSL, like:
 
-https://alma.lobid.org/resources/search?q=title:%22Moby%20dick%22
+https://lobid.org/resources/search?q=title:%22Moby%20dick%22
 
 h1. Developer instructions
 
@@ -100,11 +100,23 @@ If you've added and commited everything, check again if all is ok:
 
 @mvn clean install@
 
-This should result in *BUILD SUCCESS*. Push your changes. You're done :)
+This should result in *BUILD SUCCESS*. Push your changes.
+
+Check if the play tests work, e.g.:
+
+@sbt "test:testOnly *IntegrationTest"@
+
+If that fails, check the tests. Most of the time the "fix" is to update the test
+as new data introduce more/less hits.
+Then, at last:
+
+You're done :)
 
 h2. Tables as gitsubmodules
+
 Some lookup tables are provided through gitsubmodules. To make a @git pull@ also
-update these tables you can e.g. do @git config --local submodule.recurse true@ once.
+update these tables you can e.g. do @git config --local submodule.recurse true@ once and @git submodule update --recursive --remote@ after every @git pull@ ! This is necessary
+to be on the HEAD of the master of the submodules.
 
 h2. Propagate the context.json to lobid-resources-web
 

pom.xml

@@ -3,114 +3,115 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.lobid</groupId>
 	<artifactId>lobid-resources</artifactId>
-	<version>0.6.0-SNAPSHOT</version>
+	<version>1.0.1-SNAPSHOT</version>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<target.jdk>1.8</target.jdk>
+		<target.jdk>11</target.jdk>
 		<junit.version>4.8.2</junit.version>
-		<logback.version>0.9.30</logback.version>
+		<logback.version>1.3.14</logback.version>
 	</properties>
 	<dependencies>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-io</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-files</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-json</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
+			<exclusions>
+				<exclusion>
+					<groupId>commons-logging</groupId>
+					<artifactId>commons-logging</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-api</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-biblio</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-formeta</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-monitoring</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-strings</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-formatting</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-triples</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-flowcontrol</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metamorph</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-plumbing</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metamorph-test</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-xml</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-mangling</artifactId>
-			<version>5.6.0</version>
+			<version>6.0.0</version>
 		</dependency>
 		<dependency>
-			<groupId>org.apache.jena</groupId>
-			<artifactId>jena-arq</artifactId>
-			<version>3.17.0</version>
+			<groupId>org.metafacture</groupId>
+			<artifactId>metafix</artifactId>
+			<version>1.0.0</version>
 			<exclusions>
-			<exclusion>
-				<groupId>log4j</groupId>
-				<artifactId>log4j</artifactId>
-			</exclusion>
-			<exclusion>
-				<groupId>org.apache.httpcomponents</groupId>
-				<artifactId>httpclient</artifactId>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-simple</artifactId>
 			</exclusion>
 		</exclusions>
 		</dependency>
-		<dependency>
-			<groupId>org.metafacture</groupId>
-			<artifactId>metafix</artifactId>
-			<version>0.6.1</version>
-		</dependency>
 		<dependency>
 			<groupId>com.github.jsonld-java</groupId>
 			<artifactId>jsonld-java</artifactId>
-			<version>0.13.2</version>
+			<version>0.13.6</version>
 			<exclusions>
 				<exclusion>
 					<groupId>commons-codec</groupId>
@@ -129,7 +130,11 @@
 			<version>5.6.3</version>
 			<exclusions>
 				<exclusion>
-					<groupId>log4j</groupId>
+					<groupId>commons-logging</groupId>
+					<artifactId>commons-logging</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
 					<artifactId>log4j-api</artifactId>
 				</exclusion>
 			</exclusions>
@@ -138,6 +143,12 @@
 			<groupId>org.elasticsearch.client</groupId>
 			<artifactId>transport</artifactId>
 			<version>5.6.3</version>
+			<exclusions>
+				<exclusion>
+					<groupId>commons-logging</groupId>
+					<artifactId>commons-logging</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>junit</groupId>
@@ -164,6 +175,21 @@
 			<artifactId>jaxb-api</artifactId>
 			<version>2.3.0</version>
 		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-classic</artifactId>
+			<version>${logback.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-core</artifactId>
+			<version>${logback.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>log4j-over-slf4j</artifactId>
+			<version>1.7.36</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<resources>

src/main/java/de/hbz/lobid/helper/Etikett.java

@@ -233,4 +233,4 @@ public Map<String, String> getMultilangLabel() {
 		return multilangLabel;
 	}
 
-}
+}

src/main/java/de/hbz/lobid/helper/EtikettMaker.java

@@ -169,13 +169,15 @@ private void initContext() {
     }
 
     /**
-     * Generates context.json based on labels.json. Stores into filesystem.
+     * Generates context.json based on labels.json.
+     * Stores into filesystem, alphabetically sorted.
      */
     public void writeContext() {
         logger.info("Writing context file ...");
         try {
             ObjectMapper objectMapper = new ObjectMapper();
-            objectMapper.enable(SerializationFeature.INDENT_OUTPUT)
+            objectMapper.enable(SerializationFeature.INDENT_OUTPUT) //
+                .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) //
                 .writeValue(new File(getContextLocation()), context);
             logger.info(
                 "... done writing context file to " + getContextLocation() + ".");

src/main/java/org/lobid/resources/run/AlmaMarcXmlFix2lobidJsonEs.java

@@ -237,7 +237,7 @@ private static ElasticsearchIndexer getElasticsearchIndexer() {
 
     private static StringReader receiverThread() {
         StreamBatchLogger batchLogger = new StreamBatchLogger();
-        batchLogger.setBatchSize(100000);
+        batchLogger.setBatchSize(10000);
         MarcXmlHandler marcXmlHandler = new MarcXmlHandler();
         marcXmlHandler.setNamespace(null);
         EtikettJson etikettJson = new EtikettJson();

src/main/resources/alma/alma.fix

@@ -16,7 +16,7 @@ do list(path:"880??","var":"$i")
   copy_field("$i.6","$i.linkageTest")
   replace_all("$i.linkageTest","\\d{3}(-\\d{2}).*","880$1")
   copy_field("$i.6","$i.@script.id")
-  copy_field("$i.6","$i.@script.label") 
+  copy_field("$i.6","$i.@script.label")
   replace_all("$i.@script.id","(\\d{3}-\\d{2})/(.*)","https://unicode.org/iso15924/iso15924.txt#$2")
   replace_all("$i.@script.label","(\\d{3}-\\d{2})/(.*)","$2")
   lookup("$i.@script.label","ISO15924-to-script")

src/main/resources/alma/fix/contribution.fix

@@ -55,7 +55,7 @@ do list(path:"100[01] ", "var":"$i")
         end
       end
       # name
-      call_macro("gndPersonCombinedLabel",field:"$i") 
+      call_macro("gndPersonCombinedLabel",field:"$i")
       copy_field("$i.@combinedLabel","contribution[].$last.agent.label")
       # type
       set_array("contribution[].$last.agent.type[]","Person")
@@ -105,7 +105,7 @@ do list(path:"700[01] ", "var":"$i")
         end
       end
       # name
-      call_macro("gndPersonCombinedLabel",field:"$i") 
+      call_macro("gndPersonCombinedLabel",field:"$i")
       copy_field("$i.@combinedLabel","contribution[].$last.agent.label")
       # type
       set_array("contribution[].$last.agent.type[]","Person")
@@ -263,7 +263,7 @@ do list(path:"111[012] |711[012] ", "var":"$i")
       end
       # name
       call_macro("gndEventCombinedLabel",field:"$i")
-      copy_field("$i.@combinedLabel", "contribution[].$last.agent.label")        
+      copy_field("$i.@combinedLabel", "contribution[].$last.agent.label")
       # type
       set_array("contribution[].$last.agent.type[]","ConferenceOrEvent")
       # role
@@ -339,7 +339,7 @@ do list (path: "contribution[]", "var": "$i")
   set_array("$i.agent.altLabel[]")
   do list(path:"GPN??", "var": "$z")
     if in ("$i.agent.@gndIdn", "$z.B") # Person labels have no character between $a (Name) and $b (Number).
-      call_macro("gndPersonCombinedLabel",field:"$z") 
+      call_macro("gndPersonCombinedLabel",field:"$z")
       copy_field("$z.@combinedLabel", "$i.agent.altLabel[].$append")
     end
   end

src/main/resources/alma/fix/identifiers.fix

@@ -76,7 +76,7 @@ do list(path:"022? ", "var":"$i")
   copy_field("$i.a", "issn[].$append")
 end
 replace_all("issn[].*", "-","")
-
+uniq("issn[]")
 
 # 024 - Other Standard Identifier (R) - Subfield a (NR) 1. Indicator 2 = ISMN
 set_array("ismn[]")
@@ -98,9 +98,9 @@ end
 # 856 - Electronic Location and Access (R) - Subfield: $u (R) $3 (NR)
 # 1. Indicator: 4 = HTTP
 do list(path:"8564?", "var":"$i")
-  if all_match("$i.u", ".*(10\\.(\\d)+/(\\S)+).*") # Volltext
+  if all_match("$i.u", ".*doi.org.*(10\\.(\\d)+/(\\S)+).*") # Volltext
     copy_field("$i.u", "doi[].$append")
-    replace_all("doi[].$last", ".*(10\\.(\\d)+/(\\S)+).*", "$1")
+    replace_all("doi[].$last", ".*doi.org.*(10\\.(\\d)+/(\\S)+).*", "$1")
   end
 end
 uniq("doi[]")