Skip to content

Commit

Permalink
Merge #1930 from remote-tracking branch 'origin/1847-separateLogFiles'
Browse files Browse the repository at this point in the history
  • Loading branch information
@dr0i
dr0i committed Oct 26, 2023
2 parents 33c69bd + f06f719 commit 93c4d09
Show file tree
Hide file tree
Showing 8 changed files with 137 additions and 105 deletions.
60 changes: 43 additions & 17 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<target.jdk>1.8</target.jdk>
<junit.version>4.8.2</junit.version>
<logback.version>0.9.30</logback.version>
<logback.version>1.2.12</logback.version>
</properties>
<dependencies>
<dependency>
Expand All @@ -26,6 +26,16 @@
<groupId>org.metafacture</groupId>
<artifactId>metafacture-json</artifactId>
<version>5.6.0</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.metafacture</groupId>
Expand Down Expand Up @@ -87,25 +97,16 @@
<artifactId>metafacture-mangling</artifactId>
<version>5.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.jena</groupId>
<artifactId>jena-arq</artifactId>
<version>3.17.0</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.metafacture</groupId>
<artifactId>metafix</artifactId>
<version>0.6.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.github.jsonld-java</groupId>
Expand All @@ -129,7 +130,11 @@
<version>5.6.3</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
</exclusions>
Expand All @@ -138,6 +143,12 @@
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>5.6.3</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
Expand All @@ -164,6 +175,21 @@
<artifactId>jaxb-api</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback.version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>${logback.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
<version>1.7.36</version>
</dependency>
</dependencies>
<build>
<resources>
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/de/hbz/lobid/helper/Etikett.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,4 +233,4 @@ public Map<String, String> getMultilangLabel() {
return multilangLabel;
}

}
}
21 changes: 0 additions & 21 deletions src/main/resources/log4j.xml
View file

This file was deleted.

2 changes: 1 addition & 1 deletion src/main/resources/log4j2.xml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration>
<Appenders>
<RollingFile name="RollingFile" fileName="logs/ETL.log" filePattern="ETL-log-%d{yyyy-ww}.gz" immediateFlush="true" append="true">
<RollingFile name="RollingFile" fileName="./logs/elasticsearch.log" filePattern="elasticsearch-log-%d{yyyy-ww}.gz" immediateFlush="true" append="true">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %level [%l] - %m%n"/>
<!-- Weekly rollover with compression -->
<Policies>
Expand Down
69 changes: 69 additions & 0 deletions src/main/resources/logback.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
<configuration>
<appender name="FILE_APPLICATION" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>./logs/application.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- Daily rollover with compression -->
<fileNamePattern>application-log-%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
<!-- keep 3 weeks worth of history -->
<maxHistory>21</maxHistory>
<!-- roll log file > 100MB -->
<maxFileSize>100MB</maxFileSize>
</rollingPolicy>
<encoder>
<pattern>%date{yyyy-MM-dd HH:mm:ss ZZZZ} [%level] from %logger{36} - %message%n%xException</pattern>
</encoder>
</appender>

<appender name="FILE_ETL" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>./logs/etl.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- Daily rollover with compression -->
<fileNamePattern>etl-log-%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
<!-- keep 3 weeks worth of history -->
<maxHistory>21</maxHistory>
<!-- roll log file > 100MB -->
<maxFileSize>100MB</maxFileSize>
</rollingPolicy>
<encoder>
<pattern>%date{yyyy-MM-dd HH:mm:ss} [%level] %logger{52} in %thread - %message%n%xException</pattern>
</encoder>
</appender>

<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%logger{15} - %message%n%xException{10}</pattern>
</encoder>
</appender>

<appender name="ASYNCSTDOUT" class="ch.qos.logback.classic.AsyncAppender">
<appender-ref ref="STDOUT" />
</appender>

<!-- Off these ones as they are annoying, and anyway we manage configuration ourself -->
<logger name="com.avaje.ebean.config.PropertyMapLoader" level="OFF" />
<logger name="com.avaje.ebeaninternal.server.core.XmlConfigLoader" level="OFF" />
<logger name="com.avaje.ebeaninternal.server.lib.BackgroundThread" level="OFF" />
<logger name="com.gargoylesoftware.htmlunit.javascript" level="OFF" />

<logger name="play" level="INFO" >
<appender-ref ref="FILE_APPLICATION" />
</logger>
<logger name="application" level="INFO" >
<appender-ref ref="FILE_APPLICATION" />
</logger>

<logger name="org.metafacture" level="INFO" >
<appender-ref ref="FILE_ETL" />
</logger>
<logger name="org.lobid" level="INFO" >
<appender-ref ref="FILE_ETL" />
</logger>
<logger name="de.hbz.lobid" level="INFO" >
<appender-ref ref="FILE_ETL" />
</logger>

<root level="INFO">
<appender-ref ref="ASYNCSTDOUT" />
</root>

</configuration>
20 changes: 15 additions & 5 deletions web/app/controllers/resources/WebhookAlmaFix.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import org.lobid.resources.run.AlmaMarcXmlFix2lobidJsonEs;

import play.Logger;
import play.mvc.Controller;
import play.mvc.Http;
Expand Down Expand Up @@ -87,19 +88,22 @@ public static Result updateAlma(final String GIVEN_TOKEN) {
Logger.error(msg);
AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false,
msg);
Logger.warn(msg);
return status(500, msg);
}
}
catch (IOException e) {
msg = composeMessage(e.getMessage());
msg = composeMessage("Problems with data file\n" + e.getMessage());
AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false,
msg);
return status(500, "Problems with data file\n" + msg);
Logger.warn(msg);
return status(500, msg);
}
if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) {
msg = composeMessage(String.format(MSG_UPDATE_ALREADY_RUNNING, indexNameOfUpdate));
AlmaMarcXmlFix2lobidJsonEs.sendMail(ETL_OF + KIND, false,
msg);
Logger.warn(msg);
return status(423, msg);
}
msg = composeMessage(String.format(msgStartEtl, KIND));
Expand Down Expand Up @@ -156,17 +160,21 @@ public static Result basedumpAlma(final String GIVEN_TOKEN) {
Logger.error(msg);
AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false,
msg);
Logger.warn(msg);
return status(500, msg);
}
}
catch (IOException e) {
return status(500, "Problems with data file\n" + e);
msg = composeMessage("IO Problems with data file\n" + e);
Logger.warn(msg);
return status(500, msg);
}
createIndexNameOfBasedump = indexNameOfBasedump + "-" + LocalDateTime.now()
.format(DateTimeFormatter.ofPattern("yyyyMMdd-kkmm"));
if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) {
msg = composeMessage(String.format(MSG_CREATE_INDEX_ALREADY_RUNNING, createIndexNameOfBasedump));
AlmaMarcXmlFix2lobidJsonEs.sendMail(ETL_OF + KIND, false, msg);
Logger.warn(msg);
return status(423, msg);
}
msg = composeMessage(String.format(msgStartEtl, KIND));
Expand Down Expand Up @@ -200,29 +208,31 @@ public static Result switchEsAlias(final String GIVEN_TOKEN) {
String msg = composeMessage(subject);
Logger.info(msg);
if (!GIVEN_TOKEN.equals(token)) {
Logger.info("Wrong token: " + GIVEN_TOKEN);
return wrongToken(subject, GIVEN_TOKEN);
}
boolean success;
if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) {
msg = composeMessage(String.format(MSG_CREATE_INDEX_ALREADY_RUNNING, createIndexNameOfBasedump));
AlmaMarcXmlFix2lobidJsonEs.sendMail("Failed: " + subject, false, msg);
Logger.warn(msg);
return status(423, msg);
}
AlmaMarcXmlFix2lobidJsonEs.setSwitchVariables(alias1, alias2, clusterHost, basedumpSwitchMindocs, basedumpSwitchMinsize);
success = AlmaMarcXmlFix2lobidJsonEs.switchAlias();
if (success) {
msg = composeMessage(AlmaMarcXmlFix2lobidJsonEs.MSG_SUCCESS + subject);
Logger.info(msg);
return ok(msg);
}
msg = composeMessage(AlmaMarcXmlFix2lobidJsonEs.MSG_FAIL + subject);
Logger.warn(msg);
return internalServerError(msg);
}

private static Result wrongToken(final String KIND,
final String GIVEN_TOKEN) {
String msg = composeMessage(String.format(msgWrongToken, GIVEN_TOKEN, KIND));
Logger.error(msg);
Logger.warn(msg);
AlmaMarcXmlFix2lobidJsonEs.sendMail(KIND, false, msg);
return forbidden(msg);
}
Expand Down
22 changes: 8 additions & 14 deletions web/build.sbt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name := "lobid-resources-web"

version := "0.3.0-SNAPSHOT"
version := "0.3.1-SNAPSHOT"

scalaVersion := "2.11.11"

Expand All @@ -11,24 +11,17 @@ libraryDependencies ++= Seq(
cache,
javaWs,
"com.typesafe.play" % "play-test_2.11" % "2.4.11",
"ch.qos.logback" % "logback-classic" % "1.3.11",
"ch.qos.logback" % "logback-core" % "1.3.11",
"org.elasticsearch.plugin" % "parent-join-client" % "5.6.3" withSources(),
"org.elasticsearch" % "elasticsearch" % "5.6.3" withSources(),
"org.elasticsearch.client" % "transport" % "5.6.3" withSources()
// otherwise javaWs won't work
exclude ("io.netty", "netty"),
"org.mockito" % "mockito-core" % "1.9.5",
"org.apache.logging.log4j" % "log4j-core" % "2.9.1",
"org.elasticsearch.plugin" % "parent-join-client" % "5.6.3",
"org.mockito" % "mockito-core" % "1.9.5" %Test,
"com.google.gdata" % "core" % "1.47.1" exclude ("com.google.guava", "guava"),
"org.easytesting" % "fest-assert" % "1.4" % "test",
"org.apache.logging.log4j" % "log4j-core" % "2.20.0",
"org.xbib.elasticsearch.plugin" % "elasticsearch-plugin-bundle" % "5.4.1.0"
"org.easytesting" % "fest-assert" % "1.4" %Test,
"org.xbib.elasticsearch.plugin" % "elasticsearch-plugin-bundle" % "5.4.1.0",
"org.lobid" % "lobid-resources" % "0.6.0-SNAPSHOT" changing()
)

resolvers += "Local Maven Repository" at Path.userHome.asFile.toURI.toURL + ".m2/repository"

libraryDependencies += "org.lobid" % "lobid-resources" % "0.6.0-SNAPSHOT" changing()

lazy val root = (project in file(".")).enablePlugins(PlayJava)

javacOptions ++= Seq("-source", "1.8", "-target", "1.8")
Expand All @@ -38,3 +31,4 @@ import com.typesafe.sbteclipse.core.EclipsePlugin.EclipseKeys
EclipseKeys.projectFlavor := EclipseProjectFlavor.Java // Java project. Don't expect Scala IDE
EclipseKeys.createSrc := EclipseCreateSrc.ValueSet(EclipseCreateSrc.ManagedClasses, EclipseCreateSrc.ManagedResources) // Use .class files instead of generated .scala files for views and routes
EclipseKeys.preTasks := Seq(compile in Compile) // Compile the project before generating Eclipse files, so that .class files for views and routes are present

46 changes: 0 additions & 46 deletions web/conf/logback.xml
View file

This file was deleted.

0 comments on commit 93c4d09

Please sign in to comment.