Skip to content
This repository has been archived by the owner on Oct 16, 2023. It is now read-only.

Gh-986: Migrate log4j to reload4j #992

Merged
merged 3 commits into from
Mar 25, 2022
Merged

Gh-986: Migrate log4j to reload4j #992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4f4f0f6
gh-984: Change usage of log4j to slf4j
t92549 Mar 24, 2022
f35aa54
gh-984: Checkstyle
t92549 Mar 24, 2022
b310cc2
Exclude Log4j dependencies
GCHQDeveloper314 Mar 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions mini-accumulo-cluster/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,49 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-minicluster</artifactId>
<version>${accumulo.version}</version>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
<version>${accumulo.version}</version>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

Expand Down
5 changes: 3 additions & 2 deletions ...r/src/main/java/uk/gov/gchq/gaffer/miniaccumulocluster/MiniAccumuloClusterController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -55,7 +56,7 @@ public class MiniAccumuloClusterController {
public static final String SHUTDOWN_FILENAME = "shutdown";
public static final boolean DEFAULT_IS_TEMP_DIR = false;

private static final Logger LOGGER = Logger.getLogger(MiniAccumuloClusterController.class);
private static final Logger LOGGER = LoggerFactory.getLogger(MiniAccumuloClusterController.class);
protected static boolean shutdownHookAdded = false;

protected String dirName;
Expand Down
6 changes: 4 additions & 2 deletions ...ter/src/main/java/uk/gov/gchq/gaffer/miniaccumulocluster/MiniAccumuloShellController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@

import org.apache.accumulo.shell.Shell;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


import java.io.File;
import java.io.FileInputStream;
Expand All @@ -29,7 +31,7 @@


public final class MiniAccumuloShellController {
private static final Logger LOGGER = Logger.getLogger(MiniAccumuloShellController.class);
private static final Logger LOGGER = LoggerFactory.getLogger(MiniAccumuloShellController.class);

private static final String ACCUMULO_INSTANCE = "accumulo.instance";
private static final String ACCUMULO_USER = "accumulo.user";
Expand Down
22 changes: 22 additions & 0 deletions performance-testing/performance-testing-accumulo-store/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,34 @@
<artifactId>accumulo-minicluster</artifactId>
<version>${accumulo.version}</version>
<scope>test</scope>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>uk.gov.gchq.gaffer</groupId>
Expand Down
11 changes: 11 additions & 0 deletions performance-testing/performance-testing-aws/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,17 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-server-base</artifactId>
<version>${accumulo.version}</version>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>

<!--
Expand Down
22 changes: 22 additions & 0 deletions performance-testing/performance-testing-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,34 @@
<artifactId>accumulo-minicluster</artifactId>
<version>${accumulo.version}</version>
<scope>test</scope>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>uk.gov.gchq.gaffer</groupId>
Expand Down
5 changes: 3 additions & 2 deletions ...in/java/uk/gov/gchq/gaffer/randomelementgeneration/cache/PreferentialAttachmentCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
*/
package uk.gov.gchq.gaffer.randomelementgeneration.cache;

import org.apache.log4j.Logger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.Collections;
Expand All @@ -41,7 +42,7 @@
* when the same element is added multiple times causing many aggregations.
*/
public class PreferentialAttachmentCache<T> implements Cache<T> {
private static final Logger LOGGER = Logger.getLogger(PreferentialAttachmentCache.class);
private static final Logger LOGGER = LoggerFactory.getLogger(PreferentialAttachmentCache.class);

private final Random random = new Random();
private TreeMap<Long, Set<T>> mapFreqToItems;
Expand Down
40 changes: 40 additions & 0 deletions slider/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,17 @@
<groupId>org.apache.slider</groupId>
<artifactId>slider-core</artifactId>
<version>${slider.version}</version>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand All @@ -65,6 +76,15 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand All @@ -83,6 +103,15 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
</exclusion>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand Down Expand Up @@ -122,6 +151,17 @@
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<!-- Conflicts with Reload4j -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

Expand Down
24 changes: 10 additions & 14 deletions slider/src/main/java/uk/gov/gchq/gaffer/slider/util/AppConfigGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,11 @@
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.log4j.Logger;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
Expand All @@ -49,7 +50,7 @@
*/
public class AppConfigGenerator implements Runnable {

private static final Logger LOGGER = Logger.getLogger(AppConfigGenerator.class);
private static final Logger LOGGER = LoggerFactory.getLogger(AppConfigGenerator.class);

static class SliderAppConfig {

Expand Down Expand Up @@ -263,7 +264,7 @@ private SliderAppConfig generateSliderAppConfigForMultiNode(final SliderAppConfi

totalCoresAvailable = Math.round((float) totalCoresAvailable * ((float) this.clusterUsagePercent / 100f));
totalMemoryAvailable = Math.round((float) totalMemoryAvailable * ((float) this.clusterUsagePercent / 100f));
LOGGER.info(String.format("Trying to use %s%% of available resources across cluster = cores: %s mem: %s", this.clusterUsagePercent, totalCoresAvailable, totalMemoryAvailable));
LOGGER.info("Trying to use {}% of available resources across cluster = cores: {} mem: {}", this.clusterUsagePercent, totalCoresAvailable, totalMemoryAvailable);

// Slider Application Master
totalCoresAvailable -= ResourceKeys.DEF_YARN_CORES;
Expand Down Expand Up @@ -320,7 +321,7 @@ private SliderAppConfig generateSliderAppConfigForSingleNode(final SliderAppConf

int coresRemainingPerNode = Math.round((float) availableResources.getMaxCores() * ((float) this.clusterUsagePercent / 100f));
int memoryRemainingPerNode = Math.round((float) availableResources.getMaxMemory() * ((float) this.clusterUsagePercent / 100f));
LOGGER.info(String.format("Trying to use %s%% of available resources per node = cores: %s mem: %s", this.clusterUsagePercent, coresRemainingPerNode, memoryRemainingPerNode));
LOGGER.info("Trying to use {}% of available resources per node = cores: {} mem: {}", this.clusterUsagePercent, coresRemainingPerNode, memoryRemainingPerNode);

// Slider Application Master
coresRemainingPerNode -= ResourceKeys.DEF_YARN_CORES;
Expand Down Expand Up @@ -406,27 +407,22 @@ public void run() {
final ConfTreeSerDeser parser = new ConfTreeSerDeser();

final ConfTree initialAppConfig = parser.fromFile(new File(this.initialAppConfigPath));
LOGGER.info("Initial appConfig.json:");
LOGGER.info(initialAppConfig);
LOGGER.info("Initial appConfig.json: {}", initialAppConfig);

AvailableResources availableClusterResources = null;

availableClusterResources = this.getYarnResources();
LOGGER.info("Available Cluster Resources:");
LOGGER.info(availableClusterResources);
LOGGER.info("Available Cluster Resources: {}", availableClusterResources);

// We query twice because for some reason YARN on EMR lies about the max resources
// available per node the first time round :S
// TODO: Work out why this is the case!
availableClusterResources = this.getYarnResources();
LOGGER.info("Available Cluster Resources:");
LOGGER.info(availableClusterResources);
LOGGER.info("Available Cluster Resources: {}", availableClusterResources);

final SliderAppConfig config = this.generateSliderAppConfig(initialAppConfig, availableClusterResources);
LOGGER.info("Generated appConfig.json:");
LOGGER.info(config.getAppConfig());
LOGGER.info("Generated resources.json:");
LOGGER.info(config.getResources());
LOGGER.info("Generated appConfig.json: {}", config.getAppConfig());
LOGGER.info("Generated resources.json: {}", config.getResources());

parser.save(config.getAppConfig(), new File(this.appConfigOutputPath));
parser.save(config.getResources(), new File(this.resourcesOutputPath));
Expand Down
10 changes: 5 additions & 5 deletions slider/src/test/java/uk/gov/gchq/gaffer/slider/GafferSliderDeployer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
*/
public class GafferSliderDeployer extends AgentCommandTestBase {

private final Logger log = LoggerFactory.getLogger(GafferSliderDeployer.class);
private static final Logger LOGGER = LoggerFactory.getLogger(GafferSliderDeployer.class);

private static final String CLUSTER_NAME = "gaffer-slider-integration-tests";

Expand Down Expand Up @@ -103,7 +103,7 @@ private String createGafferAccumuloUser(final Connector accumulo, final String u
"privateVisibility"
));

log.info("Created Accumulo user called {} with password {} and authorizations {}",
LOGGER.info("Created Accumulo user called {} with password {} and authorizations {}",
user, password, accumulo.securityOperations().getUserAuthorizations(user).toString());

return password;
Expand Down Expand Up @@ -158,9 +158,9 @@ public void setup() throws Exception {
tracePassword = rootPassword;
}

log.info("Accumulo Root Password: {}", rootPassword);
log.info("Accumulo Instance Secret: {}", instanceSecret);
log.info("Accumulo Tracer Password: {}", tracePassword);
LOGGER.info("Accumulo Root Password: {}", rootPassword);
LOGGER.info("Accumulo Instance Secret: {}", instanceSecret);
LOGGER.info("Accumulo Tracer Password: {}", tracePassword);

File passwordFile = AccumuloSliderUtils.generatePasswordFile(this.folder, rootPassword, instanceSecret, tracePassword);

Expand Down