From 668761763aba59c4370dc467ec00693017763ac5 Mon Sep 17 00:00:00 2001 From: GCHQDeveloper314 <94527357+GCHQDeveloper314@users.noreply.github.com> Date: Tue, 3 May 2022 13:07:57 +0100 Subject: [PATCH] Gh-987: Upgrade dependencies (#995) * Update selenium to the latest version * Add missing plugin version This was defaulting to a very old beta version * Exclude Log4j dependencies * Adjust plugin property changed in Gaffer * Remove plugin versions which are now managed centrally * Replace usage of CloseableIterable and WrappedCloseableIterable with Iterable * Replace findbugs with spotbugs * Updated Gaffer version to 2.0.0-alpha-0.2 (#996) * Updated Gaffer version to 2.0.0-alpha-0.2 * Added new koryphe function to gafferpy Co-authored-by: Gaffer * Replace tomcat7-maven-plugin with cargo-maven3-plugin, upgrade to Tomcat 9 * Fix Selenium tests to be able to run again Co-authored-by: t92549 <80890692+t92549@users.noreply.github.com> Co-authored-by: Gaffer --- NOTICES | 2 +- cd/setup_ui_system_tests.sh | 4 +- mini-accumulo-cluster/pom.xml | 36 +- .../pom.xml | 22 + .../ingest/BytesWritableElementGenerator.java | 3 +- .../performance-testing-aws/pom.xml | 11 + .../performance-testing-core/pom.xml | 22 + .../performancetesting/query/QueryTest.java | 3 +- pom.xml | 11 +- python-shell/src/__init__.py | 2 +- python-shell/src/gafferpy/__init__.py | 2 +- python-shell/src/gafferpy/gaffer_functions.py | 17 +- python-shell/src/test/test_connector.py | 2 +- .../src/test/test_connector_requests.py | 2 +- ui/pom.xml | 385 ++++++++++-------- .../uk/gov/gchq/gaffer/ui/QueryBuilderST.java | 5 +- 16 files changed, 344 insertions(+), 185 deletions(-) diff --git a/NOTICES b/NOTICES index 6e55cb6f7..891b105db 100755 --- a/NOTICES +++ b/NOTICES @@ -20,7 +20,7 @@ Gaffer tools is built using maven. This process will automatically pull in depen projects below. -Gaffer (uk.gov.gchq.gaffer:gaffer2:2.0.0-alpha-0.1): +Gaffer (uk.gov.gchq.gaffer:gaffer2:2.0.0-alpha-0.2): - Apache License, Version 2.0 diff --git a/cd/setup_ui_system_tests.sh b/cd/setup_ui_system_tests.sh index 1b8cf945d..82d3d0448 100755 --- a/cd/setup_ui_system_tests.sh +++ b/cd/setup_ui_system_tests.sh @@ -3,7 +3,7 @@ set -e mvn install -P quick,road-traffic-demo -B -pl ui & -curl -OL https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz -tar -xf geckodriver-v0.23.0-linux64.tar.gz +curl -OL https://github.com/mozilla/geckodriver/releases/download/v0.31.0/geckodriver-v0.31.0-linux64.tar.gz +tar -xf geckodriver-v0.31.0-linux64.tar.gz mv geckodriver ui/ sleep 1m diff --git a/mini-accumulo-cluster/pom.xml b/mini-accumulo-cluster/pom.xml index 6549760d4..31329505f 100755 --- a/mini-accumulo-cluster/pom.xml +++ b/mini-accumulo-cluster/pom.xml @@ -1,6 +1,6 @@ + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + org.apache.accumulo accumulo-core ${accumulo.version} + + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + org.apache.hadoop hadoop-hdfs ${hadoop.version} + + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + @@ -58,6 +91,7 @@ org.apache.maven.plugins maven-assembly-plugin + 3.3.0 ${skip.jar-with-dependencies} diff --git a/performance-testing/performance-testing-accumulo-store/pom.xml b/performance-testing/performance-testing-accumulo-store/pom.xml index 7ff55da92..a6839a3b6 100644 --- a/performance-testing/performance-testing-accumulo-store/pom.xml +++ b/performance-testing/performance-testing-accumulo-store/pom.xml @@ -32,12 +32,34 @@ accumulo-minicluster ${accumulo.version} test + + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + org.apache.hadoop hadoop-hdfs ${hadoop.version} test + + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + uk.gov.gchq.gaffer diff --git a/performance-testing/performance-testing-accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/performancetesting/ingest/BytesWritableElementGenerator.java b/performance-testing/performance-testing-accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/performancetesting/ingest/BytesWritableElementGenerator.java index e7da53806..4d27b7d80 100644 --- a/performance-testing/performance-testing-accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/performancetesting/ingest/BytesWritableElementGenerator.java +++ b/performance-testing/performance-testing-accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/performancetesting/ingest/BytesWritableElementGenerator.java @@ -17,7 +17,6 @@ import org.apache.hadoop.io.BytesWritable; -import uk.gov.gchq.gaffer.commonutil.iterable.WrappedCloseableIterable; import uk.gov.gchq.gaffer.data.element.Element; import uk.gov.gchq.gaffer.data.generator.ElementGenerator; import uk.gov.gchq.gaffer.exception.SerialisationException; @@ -46,6 +45,6 @@ public Iterable apply(final Iterable } }); final Iterable elementIterable = elementStream::iterator; - return new WrappedCloseableIterable<>(elementIterable::iterator); + return elementIterable::iterator; } } diff --git a/performance-testing/performance-testing-aws/pom.xml b/performance-testing/performance-testing-aws/pom.xml index 2c3924d08..93399dfbd 100644 --- a/performance-testing/performance-testing-aws/pom.xml +++ b/performance-testing/performance-testing-aws/pom.xml @@ -32,6 +32,17 @@ org.apache.accumulo accumulo-server-base ${accumulo.version} + + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + org.apache.hadoop hadoop-hdfs ${hadoop.version} test + + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + uk.gov.gchq.gaffer diff --git a/performance-testing/performance-testing-core/src/main/java/uk/gov/gchq/gaffer/performancetesting/query/QueryTest.java b/performance-testing/performance-testing-core/src/main/java/uk/gov/gchq/gaffer/performancetesting/query/QueryTest.java index 1c6cf4ebb..6ede82c09 100644 --- a/performance-testing/performance-testing-core/src/main/java/uk/gov/gchq/gaffer/performancetesting/query/QueryTest.java +++ b/performance-testing/performance-testing-core/src/main/java/uk/gov/gchq/gaffer/performancetesting/query/QueryTest.java @@ -18,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable; import uk.gov.gchq.gaffer.data.element.Element; import uk.gov.gchq.gaffer.graph.Graph; import uk.gov.gchq.gaffer.graph.GraphConfig; @@ -114,7 +113,7 @@ private void queryBatch(final Supplier elementSeedSupplie final long startTime = System.currentTimeMillis(); final long numResults; try { - final CloseableIterable results = graph.execute(getElements, new User()); + final Iterable results = graph.execute(getElements, new User()); numResults = StreamSupport.stream(results.spliterator(), false).count(); } catch (final OperationException e) { LOGGER.error("OperationException thrown after " + (System.currentTimeMillis() - startTime) / 1000.0 diff --git a/pom.xml b/pom.xml index 177237314..d4507eb9e 100755 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ uk.gov.gchq.gaffer gaffer2 - 2.0.0-alpha-0.1 + 2.0.0-alpha-0.2 gaffer-tools @@ -36,7 +36,7 @@ - 2.0.0-alpha-0.1 + 2.0.0-alpha-0.2 https://github.com/gchq/gaffer-tools @@ -153,7 +153,8 @@ org.apache.maven.plugins maven-failsafe-plugin - ${failsafe.version} + + 2.22.2 system-test @@ -188,8 +189,8 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin analyze-compile diff --git a/python-shell/src/__init__.py b/python-shell/src/__init__.py index 7d5be1e2a..121d3a323 100644 --- a/python-shell/src/__init__.py +++ b/python-shell/src/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.0-alpha-0.1" +__version__ = "2.0.0-alpha-0.2" __title__ = "gafferpy" __description__ = "Gaffer Python Shell" diff --git a/python-shell/src/gafferpy/__init__.py b/python-shell/src/gafferpy/__init__.py index 7d5be1e2a..121d3a323 100644 --- a/python-shell/src/gafferpy/__init__.py +++ b/python-shell/src/gafferpy/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.0-alpha-0.1" +__version__ = "2.0.0-alpha-0.2" __title__ = "gafferpy" __description__ = "Gaffer Python Shell" diff --git a/python-shell/src/gafferpy/gaffer_functions.py b/python-shell/src/gafferpy/gaffer_functions.py index 1f5130386..e8ccc41c5 100755 --- a/python-shell/src/gafferpy/gaffer_functions.py +++ b/python-shell/src/gafferpy/gaffer_functions.py @@ -1090,11 +1090,16 @@ def to_json(self): class ToSet(AbstractFunction): CLASS = 'uk.gov.gchq.koryphe.impl.function.ToSet' - def __init__(self): + def __init__(self, implementation=None): super().__init__(_class_name=self.CLASS) + self.implementation = implementation def to_json(self): - return super().to_json() + function_json = super().to_json() + if (self.implementation is not None): + function_json["implementation"] = self.implementation + + return function_json class ToArray(AbstractFunction): @@ -1338,6 +1343,14 @@ def to_json(self): return function_json +class ToDouble(AbstractFunction): + CLASS = "uk.gov.gchq.koryphe.impl.function.ToDouble" + + def __init__(self): + super().__init__(_class_name=self.CLASS) + + def to_json(self): + return super().to_json() class FirstValid(AbstractFunction): CLASS = "uk.gov.gchq.koryphe.impl.function.FirstValid" diff --git a/python-shell/src/test/test_connector.py b/python-shell/src/test/test_connector.py index 95ca43b57..f4eea7858 100755 --- a/python-shell/src/test/test_connector.py +++ b/python-shell/src/test/test_connector.py @@ -129,7 +129,7 @@ def test_is_operation_supported(self): "exampleJson": { "class": "uk.gov.gchq.gaffer.operation.impl.get.GetAllElements" }, - "outputClassName": "uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable" + "outputClassName": "java.lang.Iterable" } ''' diff --git a/python-shell/src/test/test_connector_requests.py b/python-shell/src/test/test_connector_requests.py index 9f48b4033..e7f9c2776 100755 --- a/python-shell/src/test/test_connector_requests.py +++ b/python-shell/src/test/test_connector_requests.py @@ -129,7 +129,7 @@ def test_is_operation_supported(self): "exampleJson": { "class": "uk.gov.gchq.gaffer.operation.impl.get.GetAllElements" }, - "outputClassName": "uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable" + "outputClassName": "java.lang.Iterable" } ''' diff --git a/ui/pom.xml b/ui/pom.xml index a6e0b814d..7362f56f6 100644 --- a/ui/pom.xml +++ b/ui/pom.xml @@ -28,7 +28,7 @@ ui rest 8080 - 3.4.0 + 3.141.59 2.2 **/*-spec.js @@ -43,7 +43,8 @@ com.google.guava guava - 21.0 + + 25.0-jre test @@ -58,13 +59,33 @@ ${gaffer.version} test + + uk.gov.gchq.gaffer + federated-demo + ${gaffer.version} + war + test + + + uk.gov.gchq.gaffer + road-traffic-rest + ${gaffer.version} + war + test + + + uk.gov.gchq.gaffer + map-rest + ${gaffer.version} + war + test + org.apache.maven.plugins maven-war-plugin - ${maven-war-plugin.version} @@ -133,19 +154,30 @@ - org.apache.tomcat.maven - tomcat7-maven-plugin - 2.0 + org.codehaus.cargo + cargo-maven3-plugin + ${cargo.plugin.version} - /${standalone-ui-path} - ${standalone-port} + + + + /${standalone-ui-path} + + + + + ${standalone-port} + + + tomcat9x + deploy integration-test - run-war + run @@ -164,7 +196,6 @@ org.apache.maven.plugins maven-war-plugin - ${maven-war-plugin.version} @@ -177,81 +208,89 @@ - org.apache.tomcat.maven - tomcat7-maven-plugin - 2.0 + org.codehaus.cargo + cargo-maven3-plugin + ${cargo.plugin.version} - / - ${standalone-port} - - - /${standalone-ui-path} - + + uk.gov.gchq.gaffer ui - ${project.parent.version} war - true - - - /${standalone-rest-path} - + + /${standalone-ui-path} + + + uk.gov.gchq.gaffer road-traffic-rest - ${gaffer.version} war - true - - - - - Road Traffic Example - - - Example using road traffic data. - The sample data used is taken from the - Department for Transport - <a - href='http://data.dft.gov.uk/gb-traffic-matrix/Raw_count_data_major_roads.zip' - >GB Road Traffic Counts</a>, - which is licensed under the - <a - href='http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3' - >Open Government Licence</a>. - - - ${standalone-rest-path} - - - ${project.basedir}/example/road-traffic/graphConfig.json - - - ${project.basedir}/example/road-traffic/schema - - - ${project.basedir}/example/road-traffic/map/store.properties - - - ${project.basedir}/example/road-traffic/roadTrafficSampleData.csv - - - ${project.basedir}/example/road-traffic/addOperationsToChain.json - - - false - - - images/logo.png - - - + + /${standalone-rest-path} + + + + + ${standalone-port} + + + tomcat9x + + + uk.gov.gchq.gaffer + road-traffic-rest + war + + + + + Road Traffic Example + + + Example using road traffic data. + The sample data used is taken from the + Department for Transport + <a + href='http://data.dft.gov.uk/gb-traffic-matrix/Raw_count_data_major_roads.zip' + >GB Road Traffic Counts</a>, + which is licensed under the + <a + href='http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3' + >Open Government Licence</a>. + + + ${standalone-rest-path} + + + ${project.basedir}/example/road-traffic/graphConfig.json + + + ${project.basedir}/example/road-traffic/schema + + + ${project.basedir}/example/road-traffic/map/store.properties + + + ${project.basedir}/example/road-traffic/roadTrafficSampleData.csv + + + ${project.basedir}/example/road-traffic/addOperationsToChain.json + + + false + + + images/logo.png + + + deploy integration-test - run-war + run @@ -270,7 +309,6 @@ org.apache.maven.plugins maven-war-plugin - ${maven-war-plugin.version} @@ -283,62 +321,71 @@ - org.apache.tomcat.maven - tomcat7-maven-plugin - 2.0 + org.codehaus.cargo + cargo-maven3-plugin + ${cargo.plugin.version} - / - ${standalone-port} - - - /${standalone-ui-path} - + + uk.gov.gchq.gaffer ui - ${project.parent.version} war - true - - - /${standalone-rest-path} - + + /${standalone-ui-path} + + + uk.gov.gchq.gaffer federated-demo - ${gaffer.version} war - true - - - - - Federated Gaffer Example - - - Example of a Gaffer Federated Store - - - ${standalone-rest-path} - - - ${project.basedir}/example/federated/graphConfig.json - - - ${project.basedir}/example/federated/federatedStore.properties - - - false - - - images/logo.png - - + + /${standalone-rest-path} + + + + + ${standalone-port} + + + tomcat9x + + + uk.gov.gchq.gaffer + federated-demo + war + + + + + Federated Gaffer Example + + + Example of a Gaffer Federated Store + + + ${standalone-rest-path} + + + ${project.basedir}/example/federated/graphConfig.json + + + ${project.basedir}/example/federated/federatedStore.properties + + + false + + + images/logo.png + + + deploy integration-test - run-war + run @@ -357,7 +404,6 @@ org.apache.maven.plugins maven-war-plugin - ${maven-war-plugin.version} @@ -370,65 +416,74 @@ - org.apache.tomcat.maven - tomcat7-maven-plugin - 2.0 + org.codehaus.cargo + cargo-maven3-plugin + ${cargo.plugin.version} - / - ${standalone-port} - - - /${standalone-ui-path} - + + uk.gov.gchq.gaffer ui - ${project.parent.version} war - true - - - /${standalone-rest-path} - + + /${standalone-ui-path} + + + uk.gov.gchq.gaffer map-rest - ${gaffer.version} war - true - - - - - TypeSubTypeValue Gaffer Example - - - Example of a Gaffer Store with TypeSubTypeValue vertices - - - ${standalone-rest-path} - - - ${project.basedir}/example/type-subtype-value/graphConfig.json - - - ${project.basedir}/example/type-subtype-value/schema - - - ${project.basedir}/example/type-subtype-value/map/store.properties - - - true - - - images/logo.png - - + + /${standalone-rest-path} + + + + + ${standalone-port} + + + tomcat9x + + + uk.gov.gchq.gaffer + map-rest + war + + + + + TypeSubTypeValue Gaffer Example + + + Example of a Gaffer Store with TypeSubTypeValue vertices + + + ${standalone-rest-path} + + + ${project.basedir}/example/type-subtype-value/graphConfig.json + + + ${project.basedir}/example/type-subtype-value/schema + + + ${project.basedir}/example/type-subtype-value/map/store.properties + + + true + + + images/logo.png + + + deploy integration-test - run-war + run @@ -457,8 +512,8 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin analyze-compile diff --git a/ui/src/test/java/uk/gov/gchq/gaffer/ui/QueryBuilderST.java b/ui/src/test/java/uk/gov/gchq/gaffer/ui/QueryBuilderST.java index 416d7c61b..d748c3ada 100644 --- a/ui/src/test/java/uk/gov/gchq/gaffer/ui/QueryBuilderST.java +++ b/ui/src/test/java/uk/gov/gchq/gaffer/ui/QueryBuilderST.java @@ -28,6 +28,7 @@ import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.firefox.FirefoxDriver; +import org.openqa.selenium.firefox.FirefoxOptions; import org.openqa.selenium.firefox.FirefoxProfile; import uk.gov.gchq.gaffer.exception.SerialisationException; @@ -129,7 +130,9 @@ public static void beforeClass() throws OperationException { FirefoxProfile profile = new FirefoxProfile(); profile.setPreference("intl.accept_languages", "en-GB"); // for dates - driver = new FirefoxDriver(profile); + FirefoxOptions options = new FirefoxOptions(); + options.setProfile(profile); + driver = new FirefoxDriver(options); // Create a large window to ensure we don't need to scroll