From 86d90818cb691260edcebda7f5631826d849d56f Mon Sep 17 00:00:00 2001 From: Pascal Christoph Date: Fri, 6 Oct 2023 08:59:49 +0200 Subject: [PATCH 1/2] Separate logging destinations (#1847) - switch to logback - keep log4j2 because of elasticsearch local node, see https://discuss.elastic.co/t/classcastexception-in-estestcase-slf4j/69040 - introduce logging of ETL to a separate file - introduce logging of elasticsearch to a separate file - change log pattern to be more concise - exclude some logging libraries from dependencies - add missing log messages --- pom.xml | 60 +++++++++++----- .../java/de/hbz/lobid/helper/Etikett.java | 2 +- src/main/resources/log4j.xml | 21 ------ src/main/resources/log4j2.xml | 2 +- src/main/resources/logback.xml | 69 +++++++++++++++++++ .../controllers/resources/WebhookAlmaFix.java | 20 ++++-- web/build.sbt | 24 +++---- web/conf/logback.xml | 46 ------------- 8 files changed, 139 insertions(+), 105 deletions(-) delete mode 100644 src/main/resources/log4j.xml create mode 100644 src/main/resources/logback.xml delete mode 100644 web/conf/logback.xml diff --git a/pom.xml b/pom.xml index a3897d2e9b..9756ec0352 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ UTF-8 1.8 4.8.2 - 0.9.30 + 1.2.12 @@ -26,6 +26,16 @@ org.metafacture metafacture-json 5.6.0 + + + commons-logging + commons-logging + + + org.slf4j + slf4j-api + + org.metafacture @@ -87,25 +97,16 @@ metafacture-mangling 5.6.0 - - org.apache.jena - jena-arq - 3.17.0 - - - log4j - log4j - - - org.apache.httpcomponents - httpclient - - - org.metafacture metafix 0.6.1 + + + org.slf4j + slf4j-simple + + com.github.jsonld-java @@ -129,7 +130,11 @@ 5.6.3 - log4j + commons-logging + commons-logging + + + org.apache.logging.log4j log4j-api @@ -138,6 +143,12 @@ org.elasticsearch.client transport 5.6.3 + + + commons-logging + commons-logging + + junit @@ -164,6 +175,21 @@ jaxb-api 2.3.0 + + ch.qos.logback + logback-classic + ${logback.version} + + + ch.qos.logback + logback-core + ${logback.version} + + + org.slf4j + log4j-over-slf4j + 1.7.36 + diff --git a/src/main/java/de/hbz/lobid/helper/Etikett.java b/src/main/java/de/hbz/lobid/helper/Etikett.java index 23d282e61d..7641d49426 100644 --- a/src/main/java/de/hbz/lobid/helper/Etikett.java +++ b/src/main/java/de/hbz/lobid/helper/Etikett.java @@ -233,4 +233,4 @@ public Map getMultilangLabel() { return multilangLabel; } -} \ No newline at end of file +} diff --git a/src/main/resources/log4j.xml b/src/main/resources/log4j.xml deleted file mode 100644 index 412b596f74..0000000000 --- a/src/main/resources/log4j.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 09de0a31e8..6754b1838b 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -1,7 +1,7 @@ - + diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000000..4fd580ef13 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,69 @@ + + + ./logs/application.log + + + application-log-%d{yyyy-MM-dd}.%i.gz + + 21 + + 100MB + + + %date{yyyy-MM-dd HH:mm:ss ZZZZ} [%level] from %logger{36} - %message%n%xException + + + + + ./logs/etl.log + + + etl-log-%d{yyyy-MM-dd}.%i.gz + + 21 + + 100MB + + + %date{yyyy-MM-dd HH:mm:ss} [%level] %logger{52} in %thread - %message%n%xException + + + + + + %logger{15} - %message%n%xException{10} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/app/controllers/resources/WebhookAlmaFix.java b/web/app/controllers/resources/WebhookAlmaFix.java index fe742b1230..f779aaaa4c 100644 --- a/web/app/controllers/resources/WebhookAlmaFix.java +++ b/web/app/controllers/resources/WebhookAlmaFix.java @@ -5,6 +5,7 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import org.lobid.resources.run.AlmaMarcXmlFix2lobidJsonEs; + import play.Logger; import play.mvc.Controller; import play.mvc.Http; @@ -87,19 +88,22 @@ public static Result updateAlma(final String GIVEN_TOKEN) { Logger.error(msg); AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false, msg); + Logger.warn(msg); return status(500, msg); } } catch (IOException e) { - msg = composeMessage(e.getMessage()); + msg = composeMessage("Problems with data file\n" + e.getMessage()); AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false, msg); - return status(500, "Problems with data file\n" + msg); + Logger.warn(msg); + return status(500, msg); } if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) { msg = composeMessage(String.format(MSG_UPDATE_ALREADY_RUNNING, indexNameOfUpdate)); AlmaMarcXmlFix2lobidJsonEs.sendMail(ETL_OF + KIND, false, msg); + Logger.warn(msg); return status(423, msg); } msg = composeMessage(String.format(msgStartEtl, KIND)); @@ -156,17 +160,21 @@ public static Result basedumpAlma(final String GIVEN_TOKEN) { Logger.error(msg); AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false, msg); + Logger.warn(msg); return status(500, msg); } } catch (IOException e) { - return status(500, "Problems with data file\n" + e); + msg = composeMessage("IO Problems with data file\n" + e); + Logger.warn(msg); + return status(500, msg); } createIndexNameOfBasedump = indexNameOfBasedump + "-" + LocalDateTime.now() .format(DateTimeFormatter.ofPattern("yyyyMMdd-kkmm")); if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) { msg = composeMessage(String.format(MSG_CREATE_INDEX_ALREADY_RUNNING, createIndexNameOfBasedump)); AlmaMarcXmlFix2lobidJsonEs.sendMail(ETL_OF + KIND, false, msg); + Logger.warn(msg); return status(423, msg); } msg = composeMessage(String.format(msgStartEtl, KIND)); @@ -200,29 +208,31 @@ public static Result switchEsAlias(final String GIVEN_TOKEN) { String msg = composeMessage(subject); Logger.info(msg); if (!GIVEN_TOKEN.equals(token)) { - Logger.info("Wrong token: " + GIVEN_TOKEN); return wrongToken(subject, GIVEN_TOKEN); } boolean success; if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) { msg = composeMessage(String.format(MSG_CREATE_INDEX_ALREADY_RUNNING, createIndexNameOfBasedump)); AlmaMarcXmlFix2lobidJsonEs.sendMail("Failed: " + subject, false, msg); + Logger.warn(msg); return status(423, msg); } AlmaMarcXmlFix2lobidJsonEs.setSwitchVariables(alias1, alias2, clusterHost, basedumpSwitchMindocs, basedumpSwitchMinsize); success = AlmaMarcXmlFix2lobidJsonEs.switchAlias(); if (success) { msg = composeMessage(AlmaMarcXmlFix2lobidJsonEs.MSG_SUCCESS + subject); + Logger.info(msg); return ok(msg); } msg = composeMessage(AlmaMarcXmlFix2lobidJsonEs.MSG_FAIL + subject); + Logger.warn(msg); return internalServerError(msg); } private static Result wrongToken(final String KIND, final String GIVEN_TOKEN) { String msg = composeMessage(String.format(msgWrongToken, GIVEN_TOKEN, KIND)); - Logger.error(msg); + Logger.warn(msg); AlmaMarcXmlFix2lobidJsonEs.sendMail(KIND, false, msg); return forbidden(msg); } diff --git a/web/build.sbt b/web/build.sbt index 18d4c8a3f9..bf24350e44 100644 --- a/web/build.sbt +++ b/web/build.sbt @@ -1,6 +1,6 @@ name := "lobid-resources-web" -version := "0.3.0-SNAPSHOT" +version := "0.3.1-SNAPSHOT" scalaVersion := "2.11.11" @@ -11,23 +11,18 @@ libraryDependencies ++= Seq( cache, javaWs, "com.typesafe.play" % "play-test_2.11" % "2.4.11", - "ch.qos.logback" % "logback-classic" % "1.3.11", - "ch.qos.logback" % "logback-core" % "1.3.11", - "org.elasticsearch.plugin" % "parent-join-client" % "5.6.3" withSources(), - "org.elasticsearch" % "elasticsearch" % "5.6.3" withSources(), - "org.elasticsearch.client" % "transport" % "5.6.3" withSources() - // otherwise javaWs won't work - exclude ("io.netty", "netty"), - "org.mockito" % "mockito-core" % "1.9.5", + "org.apache.logging.log4j" % "log4j-core" % "2.9.1", + "org.elasticsearch.plugin" % "parent-join-client" % "5.6.3", + "org.mockito" % "mockito-core" % "1.9.5" %Test, "com.google.gdata" % "core" % "1.47.1" exclude ("com.google.guava", "guava"), - "org.easytesting" % "fest-assert" % "1.4" % "test", - "org.apache.logging.log4j" % "log4j-core" % "2.20.0", - "org.xbib.elasticsearch.plugin" % "elasticsearch-plugin-bundle" % "5.4.1.0" + "org.easytesting" % "fest-assert" % "1.4" %Test, + "org.xbib.elasticsearch.plugin" % "elasticsearch-plugin-bundle" % "5.4.1.0", + "org.lobid" % "lobid-resources" % "0.6.0-SNAPSHOT" changing() ) -resolvers += "Local Maven Repository" at Path.userHome.asFile.toURI.toURL + ".m2/repository" +excludeDependencies += "org.jboss.logging" % "jboss-logging" -libraryDependencies += "org.lobid" % "lobid-resources" % "0.6.0-SNAPSHOT" changing() +resolvers += "Local Maven Repository" at Path.userHome.asFile.toURI.toURL + ".m2/repository" lazy val root = (project in file(".")).enablePlugins(PlayJava) @@ -38,3 +33,4 @@ import com.typesafe.sbteclipse.core.EclipsePlugin.EclipseKeys EclipseKeys.projectFlavor := EclipseProjectFlavor.Java // Java project. Don't expect Scala IDE EclipseKeys.createSrc := EclipseCreateSrc.ValueSet(EclipseCreateSrc.ManagedClasses, EclipseCreateSrc.ManagedResources) // Use .class files instead of generated .scala files for views and routes EclipseKeys.preTasks := Seq(compile in Compile) // Compile the project before generating Eclipse files, so that .class files for views and routes are present + diff --git a/web/conf/logback.xml b/web/conf/logback.xml deleted file mode 100644 index 9a4127b714..0000000000 --- a/web/conf/logback.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - ./logs/application.log - - - application-log-%d{yyyy-MM-dd}.%i.gz - - 21 - - 100MB - - - %date{yyyy-MM-dd HH:mm:ss ZZZZ} [%level] from %logger in %thread - %message%n%xException - - - - - - %coloredLevel %logger{15} - %message%n%xException{10} - - - - - - - - - - - - - - - - - - - - - - - - - - From f06f7193e4a9113f24d920979f983bd4bc66c65d Mon Sep 17 00:00:00 2001 From: Pascal Christoph Date: Thu, 26 Oct 2023 13:52:28 +0200 Subject: [PATCH 2/2] Reintroduce jboss logging (#1847) Play won't start on the server, complains with: "java.lang.NoClassDefFoundError: org/jboss/logging/BasicLogger" --- web/build.sbt | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/build.sbt b/web/build.sbt index bf24350e44..6e846cb254 100644 --- a/web/build.sbt +++ b/web/build.sbt @@ -20,8 +20,6 @@ libraryDependencies ++= Seq( "org.lobid" % "lobid-resources" % "0.6.0-SNAPSHOT" changing() ) -excludeDependencies += "org.jboss.logging" % "jboss-logging" - resolvers += "Local Maven Repository" at Path.userHome.asFile.toURI.toURL + ".m2/repository" lazy val root = (project in file(".")).enablePlugins(PlayJava)